pax_global_header00006660000000000000000000000064152705237120014516gustar00rootroot0000000000000052 comment=12841f706ea15d17a5fe48b3e9e246dd565efdd8 kytensor-2.49.0.7/000077500000000000000000000000001527052371200136355ustar00rootroot00000000000000kytensor-2.49.0.7/CMakeLists.txt000066400000000000000000000647321527052371200164110ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.16) project(tritonserver LANGUAGES C CXX) include(CMakeDependentOption) # Use C++17 standard as Triton's minimum required. set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.") set(TRITON_VERSION "2.49.0" CACHE STRING "The version of the Triton shared library" ) option(TRITON_ENABLE_SERVER "Enable build triton server" ON) option(TRITON_ENABLE_CLIENT "Enable build triton client" OFF) option(TRITON_ENABLE_ONNX_BACKEND "Enable build triton onnx backend" ON) option(TRITON_ENABLE_LLAMACPP_BACKEND "Enable build triton llamacpp backend" OFF) option(TRITON_ENABLE_PYTHON_BACKEND "Enable build triton llamacpp backend" OFF) option(TRITON_ENABLE_LOGGING "Include logging support in server" ON) option(TRITON_ENABLE_STATS "Include statistics collections in server" ON) option(TRITON_ENABLE_TRACING "Include tracing support in server" OFF) option(TRITON_ENABLE_NVTX "Include NVTX support in server" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in server" ON) option(TRITON_ENABLE_MALI_GPU "Enable Arm Mali GPU support in server" OFF) option(TRITON_IGPU_BUILD "Enable options for iGPU compilation in sever" OFF) set(TRITON_MIN_COMPUTE_CAPABILITY "6.0" CACHE STRING "The minimum CUDA compute capability supported by Triton" ) set(TRITON_EXTRA_LIB_PATHS "" CACHE PATH "Extra library paths for Triton Server build") # Ensemble option(TRITON_ENABLE_ENSEMBLE "Include ensemble support in server" OFF) # Endpoints option(TRITON_ENABLE_HTTP "Include HTTP API in server" ON) option(TRITON_ENABLE_GRPC "Include GRPC API in server" ON) option(TRITON_ENABLE_SAGEMAKER "Include AWS SageMaker API in server" OFF) option(TRITON_ENABLE_VERTEX_AI "Include Vertex AI API in server" OFF) # Metrics option(TRITON_ENABLE_METRICS "Include metrics support in server" ON) option(TRITON_ENABLE_METRICS_GPU "Include GPU metrics support in server" ON) option(TRITON_ENABLE_METRICS_CPU "Include CPU metrics support in server" ON) # Cloud storage option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF) option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF) option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF) # Need to know if TensorRT is available when building unit tests option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF) # ASAN option(TRITON_ENABLE_ASAN "Build with address sanitizer" OFF) # Repo tags set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_THIRD_PARTY_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/third_party repo") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") # Triton module source addr set(TRITON_MODULE_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/.." CACHE STRING "Addr for triton source") message(STATUS "TRITON_MODULE_SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}") # Third-party location set(TRITON_THIRD_PARTY_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/third-party" CACHE STRING "Location of third-party build") set(TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/third-party-src" CACHE STRING "Location of third-party source") if(TRITON_ENABLE_METRICS AND NOT TRITON_ENABLE_STATS) message(FATAL_ERROR "TRITON_ENABLE_METRICS=ON requires TRITON_ENABLE_STATS=ON") endif() if(TRITON_ENABLE_TRACING AND NOT TRITON_ENABLE_STATS) message(FATAL_ERROR "TRITON_ENABLE_TRACING=ON requires TRITON_ENABLE_STATS=ON") endif() if (TRITON_ENABLE_METRICS_CPU AND NOT TRITON_ENABLE_METRICS) message(FATAL_ERROR "TRITON_ENABLE_METRICS_CPU=ON requires TRITON_ENABLE_METRICS=ON") endif() if (TRITON_ENABLE_METRICS_GPU AND NOT TRITON_ENABLE_METRICS) message(FATAL_ERROR "TRITON_ENABLE_METRICS_GPU=ON requires TRITON_ENABLE_METRICS=ON") endif() if (TRITON_ENABLE_METRICS_GPU AND NOT TRITON_ENABLE_GPU) message(FATAL_ERROR "TRITON_ENABLE_METRICS_GPU=ON requires TRITON_ENABLE_GPU=ON") endif() if(TRITON_ENABLE_ASAN AND TRITON_ENABLE_GPU) message(FATAL_ERROR "TRITON_ENABLE_ASAN=ON requires TRITON_ENABLE_GPU=OFF") endif() # onnx backend set(TRITON_ONNXRUNTIME_INCLUDE_PATHS "" CACHE PATH "Paths to ONNXRuntime includes") set(TRITON_ONNXRUNTIME_LIB_PATHS "" CACHE PATH "Paths to ONNXRuntime libraries") # llamacpp backend set(TRITON_LLAMACPP_INCLUDE_PATHS "" CACHE PATH "Paths to llamacpp includes") set(TRITON_LLAMACPP_LIB_PATHS "" CACHE PATH "Paths to llamacpp libraries") # # Dependencies # include(FetchContent) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-third-party SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/third_party ) # Some libs are installed to ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib64 instead # of ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib on Centos set(LIB_DIR "lib") if(LINUX) file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE") if(${DISTRO_ID_LIKE} MATCHES "rhel|centos") set (LIB_DIR "lib64") endif(${DISTRO_ID_LIKE} MATCHES "rhel|centos") endif(LINUX) set(TRITON_CORE_HEADERS_ONLY OFF) FetchContent_MakeAvailable(repo-third-party repo-core) ### #add_subdirectory(${TRITON_MODULE_SOURCE_DIR}/common) add_subdirectory(${TRITON_MODULE_SOURCE_DIR}/backend) # # Triton server executable and examples # # Need to use ExternalProject for our builds so that we can get the # correct dependencies between Triton executable and the # ExternalProject dependencies (found in the third_party repo) include(ExternalProject) # If CMAKE_TOOLCHAIN_FILE is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "") if (CMAKE_TOOLCHAIN_FILE) set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "-DCMAKE_TOOLCHAIN_FILE:PATH=${CMAKE_TOOLCHAIN_FILE}") endif() # If VCPKG_TARGET_TRIPLET is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "") if (VCPKG_TARGET_TRIPLET) set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "-DVCPKG_TARGET_TRIPLET:STRING=${VCPKG_TARGET_TRIPLET}") endif() # If OPENSSL_ROOT_DIR is set, propagate that hint path to the external # projects with OpenSSL dependency. set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "") if (OPENSSL_ROOT_DIR) set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") endif() # Location where protobuf-config.cmake will be installed varies by # platform if (WIN32) set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/cmake") else() set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf") endif() # Triton with Opentelemetry is not supported on Windows # FIXME: add location for Windows, when support is added # JIRA DLIS-4786 if (WIN32) set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "") else() set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/opentelemetry-cpp/${LIB_DIR}/cmake/opentelemetry-cpp") endif() if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(TRITON_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/install) else() set(TRITON_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) endif() #set(PYTHONBACKEND_DEPENDS Boost) #set(TRITON_DEPENDS triton-core protobuf googletest re2) if(${TRITON_ENABLE_GCS}) set(TRITON_DEPENDS ${TRITON_DEPENDS} google-cloud-cpp) endif() # TRITON_ENABLE_GCS if(${TRITON_ENABLE_S3}) set(TRITON_DEPENDS ${TRITON_DEPENDS} aws-sdk-cpp) endif() # TRITON_ENABLE_S3 if(${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) set(TRITON_DEPENDS ${TRITON_DEPENDS} libevent libevhtp) endif() # TRITON_ENABLE_HTTP || TRITON_ENABLE_METRICS || TRITON_ENABLE_SAGEMAKER || TRITON_ENABLE_VERTEX_AI if(${TRITON_ENABLE_GRPC}) # set(TRITON_DEPENDS ${TRITON_DEPENDS} grpc) endif() # TRITON_ENABLE_GRPC if(NOT WIN32 AND ${TRITON_ENABLE_TRACING}) set(TRITON_DEPENDS ${TRITON_DEPENDS} opentelemetry-cpp) endif() # TRITON_ENABLE_TRACING # server if(${TRITON_ENABLE_SERVER}) ExternalProject_Add(triton-server PREFIX triton-server SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/server/triton-server" CMAKE_CACHE_ARGS -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DBoost_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/boost -DGTEST_ROOT:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/googletest -DgRPC_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc/lib/cmake/grpc -Dc-ares_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares/${LIB_DIR}/cmake/c-ares -Dre2_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/re2/${LIB_DIR}/cmake/re2 -Dabsl_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl/${LIB_DIR}/cmake/absl -DCURL_DIR:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/curl/${LIB_DIR}/cmake/CURL -Dnlohmann_json_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/nlohmann_json/${LIB_DIR}/cmake/nlohmann_json -DLibevent_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevent/lib/cmake/libevent -Dlibevhtp_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevhtp/lib/cmake/libevhtp -Dstorage_client_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/google-cloud-cpp/${LIB_DIR}/cmake/storage_client -Dgoogle_cloud_cpp_common_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/google-cloud-cpp/${LIB_DIR}/cmake/google_cloud_cpp_common -DCrc32c_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/crc32c/${LIB_DIR}/cmake/Crc32c -DAWSSDK_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/AWSSDK -Daws-cpp-sdk-core_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/aws-cpp-sdk-core -Daws-cpp-sdk-s3_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/aws-cpp-sdk-s3 -Daws-c-event-stream_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-c-event-stream/cmake -Daws-c-common_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-c-common/cmake -Daws-checksums_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-checksums/cmake -Dopentelemetry-cpp_DIR:PATH=${_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR} -DTRITON_CORE_HEADERS_ONLY:BOOL=${TRITON_CORE_HEADERS_ONLY} -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_MODULE_SOURCE_DIR:STRING=${TRITON_MODULE_SOURCE_DIR} -DTRITON_IGPU_BUILD:BOOL=${TRITON_IGPU_BUILD} -DTRITON_THIRD_PARTY_REPO_TAG:STRING=${TRITON_THIRD_PARTY_REPO_TAG} -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} -DTRITON_EXTRA_LIB_PATHS:PATH=${TRITON_EXTRA_LIB_PATHS} -DTRITON_ENABLE_ASAN:BOOL=${TRITON_ENABLE_ASAN} -DTRITON_ENABLE_NVTX:BOOL=${TRITON_ENABLE_NVTX} -DTRITON_ENABLE_TRACING:BOOL=${TRITON_ENABLE_TRACING} -DTRITON_ENABLE_LOGGING:BOOL=${TRITON_ENABLE_LOGGING} -DTRITON_ENABLE_STATS:BOOL=${TRITON_ENABLE_STATS} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DTRITON_ENABLE_MALI_GPU:BOOL=${TRITON_ENABLE_MALI_GPU} -DTRITON_ENABLE_HTTP:BOOL=${TRITON_ENABLE_HTTP} -DTRITON_ENABLE_SAGEMAKER:BOOL=${TRITON_ENABLE_SAGEMAKER} -DTRITON_ENABLE_VERTEX_AI:BOOL=${TRITON_ENABLE_VERTEX_AI} -DTRITON_ENABLE_GRPC:BOOL=${TRITON_ENABLE_GRPC} -DTRITON_MIN_COMPUTE_CAPABILITY:STRING=${TRITON_MIN_COMPUTE_CAPABILITY} -DTRITON_ENABLE_METRICS:BOOL=${TRITON_ENABLE_METRICS} -DTRITON_ENABLE_METRICS_GPU:BOOL=${TRITON_ENABLE_METRICS_GPU} -DTRITON_ENABLE_METRICS_CPU:BOOL=${TRITON_ENABLE_METRICS_CPU} -DTRITON_ENABLE_GCS:BOOL=${TRITON_ENABLE_GCS} -DTRITON_ENABLE_AZURE_STORAGE:BOOL=${TRITON_ENABLE_AZURE_STORAGE} -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3} -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT} -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}/server -DTRITON_VERSION:STRING=${TRITON_VERSION} INSTALL_COMMAND "" DEPENDS ${TRITON_DEPENDS} ) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/server/triton-server/kytensor DESTINATION ${CMAKE_INSTALL_BINDIR}) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/server/triton-server/_deps/repo-core-build/triton-core/libtritonserver.so DESTINATION ${CMAKE_INSTALL_LIBDIR}) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/server/kytensor.service DESTINATION /usr/lib/systemd/system/) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/_deps/repo-core-build/triton-core/_deps/repo-common-build/protobuf/libproto-library.so DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() # TRITON_ENABLE_SERVER # clent if(${TRITON_ENABLE_CLIENT}) # # Options # option(TRITON_ENABLE_CC_HTTP "Build C++ HTTP client libraries" OFF) option(TRITON_ENABLE_CC_GRPC "Build C++ GRPC client libraries" OFF) option(TRITON_ENABLE_PYTHON_HTTP "Enable Python HTTP client libraries" OFF) option(TRITON_ENABLE_PYTHON_GRPC "Enable Python GRPC client libraries" OFF) option(TRITON_ENABLE_JAVA_HTTP "Enable JAVA HTTP client libraries" OFF) option(TRITON_ENABLE_EXAMPLES "Include examples in build" OFF) option(TRITON_ENABLE_TESTS "Include tests in build" OFF) option(TRITON_ENABLE_ZLIB "Include ZLIB library in build" ON) # Package Perf Analyzer with the Python Client -- Intended to be used by PA repo option(TRITON_PACKAGE_PERF_ANALYZER "Include Perf Analyzer in pip wheel" OFF) if(NOT TRITON_ENABLE_PYTHON_GRPC) set(TRITON_COMMON_ENABLE_PROTOBUF_PYTHON OFF) endif() if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(TRITON_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/cc-clients/install) else() set(TRITON_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) endif() if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC) set(_cc_client_depends re2) if(${TRITON_ENABLE_CC_HTTP}) set(_cc_client_depends ${_cc_client_depends} curl) endif() # TRITON_ENABLE_CC_HTTP if(${TRITON_ENABLE_CC_GRPC}) set(_cc_client_depends ${_cc_client_depends} grpc protobuf) endif() # TRITON_ENABLE_CC_GRPC ExternalProject_Add(cc-clients PREFIX cc-clients SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/client/src/c++" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients" CMAKE_CACHE_ARGS ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DCURL_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/curl/${LIB_DIR}/cmake/CURL -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} -DgRPC_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc/lib/cmake/grpc -Dabsl_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl/${LIB_DIR}/cmake/absl -Dre2_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/re2/${LIB_DIR}/cmake/re2 -Dc-ares_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares/${LIB_DIR}/cmake/c-ares -DGTEST_ROOT:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/googletest -DTRITON_MODULE_SOURCE_DIR:STRING=${TRITON_MODULE_SOURCE_DIR} -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} -DTRITON_ENABLE_CC_HTTP:BOOL=${TRITON_ENABLE_CC_HTTP} -DTRITON_ENABLE_CC_GRPC:BOOL=${TRITON_ENABLE_CC_GRPC} -DTRITON_ENABLE_EXAMPLES:BOOL=${TRITON_ENABLE_EXAMPLES} -DTRITON_ENABLE_TESTS:BOOL=${TRITON_ENABLE_TESTS} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DTRITON_ENABLE_ZLIB:BOOL=${TRITON_ENABLE_ZLIB} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}/cc-clients INSTALL_COMMAND "" #DEPENDS ${_cc_client_depends} ) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/library/libhttpclient.so DESTINATION ${CMAKE_INSTALL_LIBDIR}) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/library/libgrpcclient.so DESTINATION ${CMAKE_INSTALL_LIBDIR}) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/library/libhttpclient_static.a DESTINATION ${CMAKE_INSTALL_LIBDIR}) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/library/libgrpcclient_static.a DESTINATION ${CMAKE_INSTALL_LIBDIR}) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/library/TritonClientConfig.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/TritonClient) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/client/TritonClientTargets.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/TritonClient) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/client/src/c++/library/grpc_client.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/client/) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/client/src/c++/library/http_client.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/client/) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/client/src/c++/library/shm_utils.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/client/) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/client/src/c++/library/common.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/client/) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/client/src/c++/library/ipc.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/client/) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/_deps/repo-common-build/protobuf/grpc_service.grpc.pb.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/client/) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/_deps/repo-common-build/protobuf/grpc_service.pb.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/client/) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/_deps/repo-common-build/protobuf/model_config.pb.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/client/) #INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/tests/cc_client_test DESTINATION /opt/appdata/kylin-ai/libtriton-client-unittest/) #INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/cc-clients/tests/client_timeout_test DESTINATION /opt/appdata/kylin-ai/libtriton-client-unittest/) #install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/client/unittest_models/ DESTINATION /opt/appdata/kylin-ai/libtriton-client-unittest//unittest_models/) #install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/client/src/c++/tests/README.md DESTINATION /opt/appdata/kylin-ai/libtriton-client-unittest/) endif() # TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC if(TRITON_ENABLE_PYTHON_HTTP OR TRITON_ENABLE_PYTHON_GRPC) ExternalProject_Add(python-clients PREFIX python-clients SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/client/src/python" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/client/python-clients" CMAKE_CACHE_ARGS ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} -DgRPC_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc/lib/cmake/grpc -Dabsl_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl/${LIB_DIR}/cmake/absl -Dc-ares_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares/${LIB_DIR}/cmake/c-ares -Dre2_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/re2/${LIB_DIR}/cmake/re2 -DTRITON_MODULE_SOURCE_DIR:STRING=${TRITON_MODULE_SOURCE_DIR} -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} -DTRITON_VERSION:STRING=${TRITON_VERSION} -DTRITON_ENABLE_PYTHON_HTTP:BOOL=${TRITON_ENABLE_PYTHON_HTTP} -DTRITON_ENABLE_PYTHON_GRPC:BOOL=${TRITON_ENABLE_PYTHON_GRPC} -DTRITON_ENABLE_EXAMPLES:BOOL=${TRITON_ENABLE_EXAMPLES} -DTRITON_ENABLE_TESTS:BOOL=${TRITON_ENABLE_TESTS} -DTRITON_PACKAGE_PERF_ANALYZER:BOOL=${TRITON_PACKAGE_PERF_ANALYZER} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}/python-clients ) endif() # TRITON_ENABLE_PYTHON_HTTP OR TRITON_ENABLE_PYTHON_GRPC if(TRITON_ENABLE_JAVA_HTTP) ExternalProject_Add(java-clients PREFIX java-clients SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/client/src/java" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/client/java-clients" CMAKE_CACHE_ARGS ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_VERSION:STRING=${TRITON_VERSION} -DTRITON_ENABLE_JAVA_HTTP:BOOL=${TRITON_ENABLE_JAVA_HTTP} -DTRITON_ENABLE_EXAMPLES:BOOL=${TRITON_ENABLE_EXAMPLES} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}/java-clients INSTALL_COMMAND "" ) endif() # TRITON_ENABLE_JAVA_HTTP endif() # TRITON_ENABLE_CLIENT # onnx backend if(${TRITON_ENABLE_ONNX_BACKEND}) ExternalProject_Add(onnxruntimebackend PREFIX onnxruntimebackend SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/framework_backends/onnxruntime_backend" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_backend" CMAKE_CACHE_ARGS -DTritonCommon_DIR:PATH=${CMAKE_CURRENT_BINARY_DIR}/_deps/repo-common-build -DTritonCore_DIR:PATH=${CMAKE_CURRENT_BINARY_DIR}/_deps/repo-core-build -DTritonBackend_DIR:PATH=${CMAKE_CURRENT_BINARY_DIR}/backend -DTRITON_ONNXRUNTIME_INCLUDE_PATHS:PATH=${TRITON_ONNXRUNTIME_INCLUDE_PATHS} -DTRITON_ONNXRUNTIME_LIB_PATHS:PATH=${TRITON_ONNXRUNTIME_LIB_PATHS} -DTRITON_MODULE_SOURCE_DIR:STRING=${TRITON_MODULE_SOURCE_DIR} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DTRITON_ENABLE_NVTX:BOOL=${TRITON_ENABLE_NVTX} -DTRITON_ENABLE_STATS:BOOL=${TRITON_ENABLE_STATS} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}/backends -DTRITON_VERSION:STRING=${TRITON_VERSION} INSTALL_COMMAND "" DEPENDS triton-core-serverstub triton-backend-utils triton-common-async-work-queue ) if(CMAKE_INSTALL_PREFIX STREQUAL "/usr") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_backend/libtriton_onnxruntime.so DESTINATION /opt/tritonserver/backends/onnxruntime) else() INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_backend/libtriton_onnxruntime.so DESTINATION ${CMAKE_INSTALL_LIBDIR}/backends/onnxruntime) endif() endif() # TRITON_ENABLE_ONNX_BACKEND # llamacpp backend if(${TRITON_ENABLE_LLAMACPP_BACKEND}) ExternalProject_Add(llamacppbackend PREFIX llamacppbackend SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/framework_backends/llamacpp_backend" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/llamacpp_backend" CMAKE_CACHE_ARGS -DTRITON_LLAMACPP_INCLUDE_PATHS:PATH=${TRITON_LLAMACPP_INCLUDE_PATHS} -DTRITON_LLAMACPP_LIB_PATHS:PATH=${TRITON_LLAMACPP_LIB_PATHS} -DTRITON_ENABLE_METRICS:BOOL=${TRITON_ENABLE_METRICS} -DTRITON_MODULE_SOURCE_DIR:STRING=${TRITON_MODULE_SOURCE_DIR} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}/backends -DTRITON_VERSION:STRING=${TRITON_VERSION} INSTALL_COMMAND "" # DEPENDS libllama ) if(CMAKE_INSTALL_PREFIX STREQUAL "/usr") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/llamacpp_backend/libtriton_llamacpp.so DESTINATION /opt/tritonserver/backends/llamacpp) else() INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/llamacpp_backend/libtriton_llamacpp.so DESTINATION ${CMAKE_INSTALL_LIBDIR}/backends/llamacpp) endif() endif() # TRITON_ENABLE_LLAMACPP_BACKEND # python backend if(${TRITON_ENABLE_PYTHON_BACKEND}) ExternalProject_Add(pythonbackend PREFIX pythonbackend SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/framework_backends/python_backend" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/python_backend" CMAKE_CACHE_ARGS -DBoost_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/boost -DTRITON_MODULE_SOURCE_DIR:STRING=${TRITON_MODULE_SOURCE_DIR} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DTRITON_ENABLE_NVTX:BOOL=${TRITON_ENABLE_NVTX} -DTRITON_ENABLE_STATS:BOOL=${TRITON_ENABLE_STATS} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}/backends -DTRITON_VERSION:STRING=${TRITON_VERSION} INSTALL_COMMAND "" DEPENDS ${PYTHONBACKEND_DEPENDS} ) ExternalProject_Add(identitybackend PREFIX identitybackend SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/framework_backends/identity_backend" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/identity_backend" CMAKE_CACHE_ARGS -DTRITON_MODULE_SOURCE_DIR:STRING=${TRITON_MODULE_SOURCE_DIR} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DTRITON_ENABLE_NVTX:BOOL=${TRITON_ENABLE_NVTX} -DTRITON_ENABLE_STATS:BOOL=${TRITON_ENABLE_STATS} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}/backends -DTRITON_VERSION:STRING=${TRITON_VERSION} INSTALL_COMMAND "" DEPENDS ${PYTHONBACKEND_DEPENDS} ) if(CMAKE_INSTALL_PREFIX STREQUAL "/usr") INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/python_backend/libtriton_python.so DESTINATION /opt/tritonserver/backends/python) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/python_backend/triton_python_backend_stub DESTINATION /opt/tritonserver/backends/python) INSTALL(FILES ${TRITON_MODULE_SOURCE_DIR}/framework_backends/python_backend/src/resources/triton_python_backend_utils.py DESTINATION /opt/tritonserver/backends/python) else() INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/python_backend/libtriton_python.so DESTINATION ${CMAKE_INSTALL_LIBDIR}/backends/python) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/python_backend/triton_python_backend_stub DESTINATION ${CMAKE_INSTALL_LIBDIR}/backends/python) INSTALL(FILES ${TRITON_MODULE_SOURCE_DIR}/framework_backends/python_backend/src/resources/triton_python_backend_utils.py DESTINATION ${CMAKE_INSTALL_LIBDIR}/backends/python) endif() endif() # TRITON_ENABLE_PYTHON_BACKEND kytensor-2.49.0.7/README.md000066400000000000000000000022051527052371200151130ustar00rootroot00000000000000# Kytensor 推理服务 kytensor推理服务是基于[NVIDIA Triton Inference Server](https://github.com/triton-inference-server)构建，将其各个模块重新组织在一个工程中便于管理和开发。 - server - client - core - backend - common - third_party - framework_backends ## 编译 ### server ``` cmake -DCMAKE_INSTALL_PREFIX=`pwd`/install -DTRITON_ENABLE_SERVER=ON -DTRITON_ENABLE_CLIENT=OFF -DTRITON_ENABLE_ONNX_BACKEND=OFF .. ``` ### client ``` cmake -DCMAKE_INSTALL_PREFIX=`pwd`/install -DTRITON_ENABLE_SERVER=OFF -DTRITON_ENABLE_CLIENT=ON -DTRITON_ENABLE_ONNX_BACKEND=OFF -DTRITON_ENABLE_CC_HTTP=ON -DTRITON_ENABLE_CC_GRPC=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_ENABLE_TESTS=ON -DTRITON_ENABLE_PYTHON_HTTP=ON -DTRITON_ENABLE_PYTHON_GRPC=ON .. ``` ## 测试 ``` cmake -DCMAKE_INSTALL_PREFIX=`pwd`/install -DTRITON_ENABLE_SERVER=OFF -DTRITON_ENABLE_CLIENT=ON -DTRITON_ENABLE_ONNX_BACKEND=OFF -DTRITON_ENABLE_CC_HTTP=ON -DTRITON_ENABLE_CC_GRPC=ON -DTRITON_ENABLE_EXAMPLES=OFF -DTRITON_ENABLE_TESTS=ON -DTRITON_ENABLE_PYTHON_HTTP=ON -DTRITON_ENABLE_PYTHON_GRPC=ON -DTRITON_ENABLE_GPU=OFF -DTRITON_ENABLE_METRICS_GPU=OFF .. ``` kytensor-2.49.0.7/backend/000077500000000000000000000000001527052371200152245ustar00rootroot00000000000000kytensor-2.49.0.7/backend/.clang-format000066400000000000000000000014261527052371200176020ustar00rootroot00000000000000--- BasedOnStyle: Google IndentWidth: 2 ColumnLimit: 80 ContinuationIndentWidth: 4 UseTab: Never MaxEmptyLinesToKeep: 2 SortIncludes: true CompactNamespaces: true ReflowComments: true DerivePointerAlignment: false PointerAlignment: Left AllowShortIfStatementsOnASingleLine: false AllowShortBlocksOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AlwaysBreakAfterReturnType: TopLevelDefinitions AlignAfterOpenBracket: AlwaysBreak BreakBeforeBraces: Custom BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: true AfterNamespace: false AfterStruct: false AfterUnion: false BeforeCatch: true BinPackArguments: true BinPackParameters: true ConstructorInitializerAllOnOneLineOrOnePerLine: false IndentCaseLabels: true kytensor-2.49.0.7/backend/.github/000077500000000000000000000000001527052371200165645ustar00rootroot00000000000000kytensor-2.49.0.7/backend/.github/workflows/000077500000000000000000000000001527052371200206215ustar00rootroot00000000000000kytensor-2.49.0.7/backend/.github/workflows/pre-commit.yml000066400000000000000000000033201527052371200234160ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: pre-commit on: pull_request: jobs: pre-commit: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - uses: pre-commit/action@v3.0.0 kytensor-2.49.0.7/backend/.pre-commit-config.yaml000066400000000000000000000056111527052371200215100ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black rev: 23.1.0 hooks: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] types_or: [python, cython] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.5 hooks: - id: clang-format types_or: [c, c++, cuda, proto, textproto, java] args: ["-fallback-style=none", "-style=file", "-i"] - repo: https://github.com/codespell-project/codespell rev: v2.2.4 hooks: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$) # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-json - id: check-toml - id: check-yaml - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace kytensor-2.49.0.7/backend/CMakeLists.txt000066400000000000000000000205271527052371200177720ustar00rootroot00000000000000# Copyright 2020-2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonbackend LANGUAGES C CXX) # # Options # option(TRITON_ENABLE_GPU "Enable GPU support in backend utilities" ON) option(TRITON_ENABLE_MALI_GPU "Enable Arm MALI GPU support in backend utilities" OFF) option(TRITON_ENABLE_STATS "Include statistics collections in backend utilities" ON) # Default OFF unless backend explicitly request to use provided implementation option(TRITON_ENABLE_MEMORY_TRACKER "Include device memory tracker in backend utilities" OFF) set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") # # Setting C++ standard # set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard whose features are requested to build this target.") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() if(TRITON_ENABLE_MEMORY_TRACKER AND NOT TRITON_ENABLE_GPU) message(WARNING "TRITON_ENABLE_MEMORY_TRACKER=ON requires TRITON_ENABLE_GPU=ON, TRITON_ENABLE_MEMORY_TRACKER will be disable") set(TRITON_ENABLE_MEMORY_TRACKER OFF CACHE BOOL "Device memory tracker disabled" FORCE) endif() # # Dependencies # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_MakeAvailable(repo-common repo-core) # # CUDA # if(${TRITON_ENABLE_GPU}) find_package(CUDAToolkit REQUIRED) set(CMAKE_CUDA_RUNTIME_LIBRARY Shared) message(STATUS "Using CUDA ${CUDAToolkit_VERSION}") if(CUDAToolkit_VERSION VERSION_GREATER "10.1" OR CUDAToolkit_VERSION VERSION_EQUAL "10.1") add_definitions(-DTRITON_ENABLE_CUDA_GRAPH=1) else() message(WARNING "CUDA ${CUDA_VERSION} does not support CUDA graphs.") endif() endif() # TRITON_ENABLE_GPU # # Backend library containing useful source and utilities # set(SRC_FILES "src/backend_common.cc" "src/backend_input_collector.cc" "src/backend_memory.cc" "src/backend_model_instance.cc" "src/backend_model.cc" "src/backend_output_responder.cc" ) if(${TRITON_ENABLE_GPU}) set(SRC_FILES ${SRC_FILES} "src/kernel.h") if(${TRITON_ENABLE_MEMORY_TRACKER}) set(SRC_FILES ${SRC_FILES} "src/device_memory_tracker.cc") endif() # TRITON_ENABLE_MEMORY_TRACKER endif() # TRITON_ENABLE_GPU add_library( triton-backend-utils ${SRC_FILES} ) if(${TRITON_ENABLE_GPU}) add_library( kernel_library_new src/kernel.cu src/kernel.h ) enable_language(CUDA) set_target_properties(kernel_library_new PROPERTIES LANGUAGE CUDA) set_target_properties(kernel_library_new PROPERTIES CUDA_ARCHITECTURES "53;60-real;62-real;70-real;72-real;75-real;80-real;86-real;89-real;90-real") set_target_properties(kernel_library_new PROPERTIES POSITION_INDEPENDENT_CODE ON) set_target_properties(kernel_library_new PROPERTIES LINKER_LANGUAGE CUDA) target_compile_features(kernel_library_new PUBLIC cxx_std_${TRITON_MIN_CXX_STANDARD}) set_target_properties(kernel_library_new PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) endif() # TRITON_ENABLE_GPU add_library( TritonBackend::triton-backend-utils ALIAS triton-backend-utils ) target_include_directories( triton-backend-utils PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message("Using MSVC as compiler, default target on Windows 10. " "If the target system is not Windows 10, please update _WIN32_WINNT " "to corresponding value.") endif() target_compile_features(triton-backend-utils PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-backend-utils PRIVATE $<$,$,$>: -Wall -Wextra -Wno-unused-parameter -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) # TRITON_ENABLE_GPU exposed in header so set PUBLIC if(${TRITON_ENABLE_GPU}) target_compile_definitions( triton-backend-utils PUBLIC TRITON_ENABLE_GPU=1 ) if(${TRITON_ENABLE_MEMORY_TRACKER}) target_compile_definitions( triton-backend-utils PUBLIC TRITON_ENABLE_MEMORY_TRACKER=1 ) endif() # TRITON_ENABLE_MEMORY_TRACKER endif() # TRITON_ENABLE_GPU # TRITON_ENABLE_MALI_GPU exposed in header so set PUBLIC if(${TRITON_ENABLE_MALI_GPU}) target_compile_definitions( triton-backend-utils PUBLIC TRITON_ENABLE_MALI_GPU=1 ) endif() # TRITON_ENABLE_MALI_GPU # TRITON_ENABLE_STATS exposed in header so set PUBLIC if(${TRITON_ENABLE_STATS}) target_compile_definitions( triton-backend-utils PUBLIC TRITON_ENABLE_STATS=1 ) endif() # TRITON_ENABLE_STATS set_target_properties( triton-backend-utils PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE POSITION_INDEPENDENT_CODE ON OUTPUT_NAME tritonbackendutils ) target_link_libraries( triton-backend-utils PUBLIC triton-core-backendapi # from repo-core triton-core-serverapi # from repo-core triton-common-async-work-queue # from repo-common triton-common-json # from repo-common ) if(${TRITON_ENABLE_GPU}) target_link_libraries( triton-backend-utils PUBLIC CUDA::cudart PRIVATE kernel_library_new ) if(${TRITON_ENABLE_MEMORY_TRACKER}) target_link_libraries( triton-backend-utils PUBLIC CUDA::cupti ) endif() # TRITON_ENABLE_MEMORY_TRACKER endif() # TRITON_ENABLE_GPU # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonBackend) install( TARGETS triton-backend-utils EXPORT triton-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) if(${TRITON_ENABLE_GPU}) install( TARGETS kernel_library_new EXPORT triton-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) endif() # TRITON_ENABLE_GPU install( DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) install( EXPORT triton-backend-targets FILE TritonBackendTargets.cmake NAMESPACE TritonBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonBackendTargets.cmake NAMESPACE TritonBackend:: ) export(PACKAGE TritonBackend) kytensor-2.49.0.7/backend/LICENSE000066400000000000000000000027221527052371200162340ustar00rootroot00000000000000Copyright (c) 2018-2020, NVIDIA CORPORATION. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/backend/README.md000066400000000000000000000665511527052371200165200ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Triton Inference Server Backend A Triton *backend* is the implementation that executes a model. A backend can be a wrapper around a deep-learning framework, like PyTorch, TensorFlow, TensorRT or ONNX Runtime. Or a backend can be custom C/C++ logic performing any operation (for example, image pre-processing). This repo contains documentation on Triton backends and also source, scripts and utilities for creating Triton backends. You do not need to use anything provided in this repo to create a Triton backend but you will likely find its contents useful. ## Frequently Asked Questions Full documentation is included below but these shortcuts can help you get started in the right direction. ### Where can I ask general questions about Triton and Triton backends? Be sure to read all the information below as well as the [general Triton documentation](https://github.com/triton-inference-server/server#triton-inference-server) available in the main [server](https://github.com/triton-inference-server/server) repo. If you don't find your answer there you can ask questions on the main Triton [issues page](https://github.com/triton-inference-server/server/issues). ### Where can I find all the backends that are available for Triton? Anyone can develop a Triton backend, so it isn't possible for us to know about all available backends. But the Triton project does provide a set of supported backends that are tested and updated with each Triton release. **TensorRT**: The TensorRT backend is used to execute TensorRT models. The [tensorrt_backend](https://github.com/triton-inference-server/tensorrt_backend) repo contains the source for the backend. **ONNX Runtime**: The ONNX Runtime backend is used to execute ONNX models. The [onnxruntime_backend](https://github.com/triton-inference-server/onnxruntime_backend) repo contains the documentation and source for the backend. **TensorFlow**: The TensorFlow backend is used to execute TensorFlow models in both GraphDef and SavedModel formats. The same backend is used to execute both TensorFlow 1 and TensorFlow 2 models. The [tensorflow_backend](https://github.com/triton-inference-server/tensorflow_backend) repo contains the documentation and source for the backend. **PyTorch**: The PyTorch backend is used to execute PyTorch models in both TorchScript and PyTorch 2.0 formats. The [pytorch_backend](https://github.com/triton-inference-server/pytorch_backend) repo contains the documentation and source for the backend. **OpenVINO**: The OpenVINO backend is used to execute [OpenVINO](https://docs.openvinotoolkit.org/latest/index.html) models. The [openvino_backend](https://github.com/triton-inference-server/openvino_backend) repo contains the documentation and source for the backend. **Python**: The Python backend allows you to write your model logic in Python. For example, you can use this backend to execute pre/post processing code written in Python, or to execute a PyTorch Python script directly (instead of first converting it to TorchScript and then using the PyTorch backend). The [python_backend](https://github.com/triton-inference-server/python_backend) repo contains the documentation and source for the backend. **DALI**: [DALI](https://github.com/NVIDIA/DALI) is a collection of highly optimized building blocks and an execution engine that accelerates the pre-processing of the input data for deep learning applications. The DALI backend allows you to execute your DALI pipeline within Triton. The [dali_backend](https://github.com/triton-inference-server/dali_backend) repo contains the documentation and source for the backend. **FIL**: The FIL ([Forest Inference Library](https://github.com/rapidsai/cuml/tree/branch-21.10/python/cuml/fil)) backend is used to execute a variety of tree-based ML models, including XGBoost models, LightGBM models, Scikit-Learn random forest models, and cuML random forest models. The [fil_backend](https://github.com/triton-inference-server/fil_backend) repo contains the documentation and source for the backend. **vLLM**: The vLLM backend is designed to run [supported models](https://vllm.readthedocs.io/en/latest/models/supported_models.html) on a [vLLM engine](https://github.com/vllm-project/vllm/blob/main/vllm/engine/async_llm_engine.py). This backend depends on [python_backend](https://github.com/triton-inference-server/python_backend) to load and serve models. The [vllm_backend](https://github.com/triton-inference-server/vllm_backend) repo contains the documentation and source for the backend. **Important Note!** Not all the above backends are supported on every platform supported by Triton. Look at the [Backend-Platform Support Matrix](docs/backend_platform_support_matrix.md) to learn about the same. ### How can I develop my own Triton backend? First you probably want to ask on the main Triton [issues page](https://github.com/triton-inference-server/server/issues) to make sure you are not duplicating a backend that already exists. Then follow the [tutorial](examples/README.md) to learn how to create your first simple Triton backend and incrementally improve it to add more features. You should also read the complete documentation on [Triton backends](#backends). ### Can I add (or remove) a backend to an existing Triton installation? Yes. See [Backend Shared Library](#backend-shared-library) for general information about how the shared library implementing a backend is managed by Triton, and [Triton with Unsupported and Custom Backends](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/compose.md#triton-with-unsupported-and-custom-backends) for documentation on how to add your backend to the released Triton Docker image. For a standard install the globally available backends are in /opt/tritonserver/backends. ### What about backends developed using the "legacy custom backend" API. The legacy custom API is removed from Triton. If you have custom backends that you developed using this older API you must port them to the new [Triton Backend API](#triton-backend-api). ## Backends A Triton *backend* is the implementation that executes a model. A backend can be a wrapper around a deep-learning framework, like PyTorch, TensorFlow, TensorRT, ONNX Runtime or OpenVINO. A backend can also implement any functionality you want as long as it adheres to the [backend API](#triton-backend-api). Triton uses this API to send requests to the backend for execution and the backend uses the API to communicate with Triton. Every model must be associated with a backend. A model's backend is specified in the model's configuration using the `backend` setting. For using TensorRT backend, the value of this setting should be `tensorrt`. Similarly, for using PyTorch, ONNX and TensorFlow backends, the `backend` field should be set to `pytorch`, `onnxruntime` or `tensorflow` respectively. For all other backends, `backend` must be set to the name of the backend. Some backends may also check the `platform` setting for categorizing the model, for example, in TensorFlow backend, `platform` should be set to `tensorflow_savedmodel` or `tensorflow_graphdef` according to the model format. Please refer to the specific backend repository on whether `platform` is used. ### Backend Shared Library Each backend must be implemented as a shared library and the name of the shared library must be *libtriton_\.so*. For example, if the name of the backend is "mybackend", a model indicates that it uses the backend by setting the model configuration 'backend' setting to "mybackend", and Triton looks for *libtriton_mybackend.so* as the shared library that implements the backend. The [tutorial](examples/README.md) shows examples of how to build your backend logic into the appropriate shared library. For a model, *M* that specifies backend *B*, Triton searches for the backend shared library in the following places, in this order: * \/M/\/libtriton_B.so * \/M/libtriton_B.so * \/B/libtriton_B.so Where \ is by default /opt/tritonserver/backends. The --backend-directory flag can be used to override the default. Typically you will install your backend into the global backend directory. For example, if using Triton Docker images you can follow the instructions in [Triton with Unsupported and Custom Backends](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/compose.md#triton-with-unsupported-and-custom-backends). Continuing the example of a backend names "mybackend", you would install into the Triton image as: ``` /opt/ tritonserver/ backends/ mybackend/ libtriton_mybackend.so ... # other files needed by mybackend ``` Starting from 24.01, the default backend shared library name can be changed by providing the `runtime` setting in the model configuration. For example, ``` runtime: "my_backend_shared_library_name.so" ``` A model may choose a specific runtime implementation provided by the backend. ### Triton Backend API A Triton backend must implement the C interface defined in [tritonbackend.h](https://github.com/triton-inference-server/core/tree/main/include/triton/core/tritonbackend.h). The following abstractions are used by the API. #### TRITONBACKEND_Backend A TRITONBACKEND_Backend object represents the backend itself. The same backend object is shared across all models that use the backend. The associated API, like TRITONBACKEND_BackendName, is used to get information about the backend and to associate a user-defined state with the backend. A backend can optionally implement TRITONBACKEND_Initialize and TRITONBACKEND_Finalize to get notification of when the backend object is created and destroyed (for more information see [backend lifecycles](#backend-lifecycles)). #### TRITONBACKEND_Model A TRITONBACKEND_Model object represents a model. Each model loaded by Triton is associated with a TRITONBACKEND_Model. Each model can use the TRITONBACKEND_ModelBackend API to get the backend object representing the backend that is used by the model. The same model object is shared across all instances of that model. The associated API, like TRITONBACKEND_ModelName, is used to get information about the model and to associate a user-defined state with the model. Most backends will implement TRITONBACKEND_ModelInitialize and TRITONBACKEND_ModelFinalize to initialize the backend for a given model and to manage the user-defined state associated with the model (for more information see [backend lifecycles](#backend-lifecycles)). The backend must take into account threading concerns when implementing TRITONBACKEND_ModelInitialize and TRITONBACKEND_ModelFinalize. Triton will not perform multiple simultaneous calls to these functions for a given model; however, if a backend is used by multiple models Triton may simultaneously call the functions with a different thread for each model. As a result, the backend must be able to handle multiple simultaneous calls to the functions. Best practice for backend implementations is to use only function-local and model-specific user-defined state in these functions, as is shown in the [tutorial](examples/README.md). #### TRITONBACKEND_ModelInstance A TRITONBACKEND_ModelInstance object represents a model *instance*. Triton creates one or more instances of the model based on the *instance_group* settings specified in the model configuration. Each of these instances is associated with a TRITONBACKEND_ModelInstance object. The only function that the backend must implement is TRITONBACKEND_ModelInstanceExecute. The TRITONBACKEND_ModelInstanceExecute function is called by Triton to perform inference/computation on a batch of inference requests. Most backends will also implement TRITONBACKEND_ModelInstanceInitialize and TRITONBACKEND_ModelInstanceFinalize to initialize the backend for a given model instance and to manage the user-defined state associated with the model (for more information see [backend lifecycles](#backend-lifecycles)). The backend must take into account threading concerns when implementing TRITONBACKEND_ModelInstanceInitialize, TRITONBACKEND_ModelInstanceFinalize and TRITONBACKEND_ModelInstanceExecute. Triton will not perform multiple simultaneous calls to these functions for a given model instance; however, if a backend is used by a model with multiple instances or by multiple models Triton may simultaneously call the functions with a different thread for each model instance. As a result, the backend must be able to handle multiple simultaneous calls to the functions. Best practice for backend implementations is to use only function-local and model-specific user-defined state in these functions, as is shown in the [tutorial](examples/README.md). #### TRITONBACKEND_Request A TRITONBACKEND_Request object represents an inference request made to the model. The backend takes ownership of the request object(s) in TRITONBACKEND_ModelInstanceExecute and must release each request by calling TRITONBACKEND_RequestRelease. However, the ownership of request object is returned back to Triton in case TRITONBACKEND_ModelInstanceExecute returns an error. See [Inference Requests and Responses](#inference-requests-and-responses) for more information about request lifecycle. The Triton Backend API allows the backend to get information about the request as well as the input and request output tensors of the request. Each request input is represented by a TRITONBACKEND_Input object. #### TRITONBACKEND_Response A TRITONBACKEND_Response object represents a response sent by the backend for a specific request. The backend uses the response API to set the name, shape, datatype and tensor values for each output tensor included in the response. The response can indicate either a failed or a successful request. See [Inference Requests and Responses](#inference-requests-and-responses) for more information about request-response lifecycle. #### TRITONBACKEND_BackendAttribute A `TRITONBACKEND_BackendAttribute` allows a backend to set certain attributes which are queried by Triton to inform certain feature support, preferred configurations, and other types of backend-specific behavior. When initializing a backend, Triton will query the `TRITONBACKEND_GetBackendAttribute` function if implemented by the backend. This function is optional to implement, but is generally used to call the related `TRITONBACKEND_BackendAttribute` APIs for setting backend-specific attributes. Some of the relevant BackendAttribute setter APIs are listed below: - `TRITONBACKEND_BackendSetExecutionPolicy` - `TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup` - Defines a priority list of instance groups to prefer for this backend if a model config doesn't explicitly define any instance groups. - `TRITONBACKEND_BackendAttributeSetParallelModelInstanceLoading` - Defines whether the backend can safely handle concurrent calls to `TRITONBACKEND_ModelInstanceInitialize` or not. - Loading model instances in parallel can improve server startup times for large instance counts. - By default, this attribute is set to false, meaning that parallel instance loading is disabled for all backends unless explicitly enabled. - The following official backends currently support loading model instances in parallel: - Python - ONNXRuntime The full list of `TRITONBACKEND_BackendAttribute` related APIs are defined in [tritonbackend.h](https://github.com/triton-inference-server/core/blob/main/include/triton/core/tritonbackend.h). ### Backend Lifecycles A backend must carefully manage the lifecycle of the backend itself, the models and model instances that use the backend and the inference requests that execute on the model instances using the backend. #### Backend and Model Backend, model and model instance initialization is triggered when Triton loads a model. * If the model requires a backend that is not already in use by an already loaded model, then: * Triton [loads the shared library](#backend-shared-library) that implements the backend required by the model. * Triton creates the TRITONBACKEND_Backend object that represents the backend. * Triton calls TRITONBACKEND_Initialize if it is implemented in the backend shared library. TRITONBACKEND_Initialize should not return until the backend is completely initialized. If TRITONBACKEND_Initialize returns an error, Triton will report that the model failed to load. * Triton creates the TRITONBACKEND_Model object that represents the model. Triton calls TRITONBACKEND_ModelInitialize if it is implemented in the backend shared library. TRITONBACKEND_ModelInitialize should not return until the backend is completely initialized for the model. If TRITONBACKEND_ModelInitialize returns an error, Triton will show that the model failed to load. * For each model instance specified for the model in the model configuration: * Triton creates the TRITONBACKEND_ModelInstance object that represents the model instance. * Triton calls TRITONBACKEND_ModelInstanceInitialize if it is implemented in the backend shared library. TRITONBACKEND_ModelInstanceInitialize should not return until the backend is completely initialized for the instance. If TRITONBACKEND_ModelInstanceInitialize returns an error, Triton will show that the model failed to load. Backend, model and model instance finalization is triggered when Triton unloads a model. * For each model instance: * Triton calls TRITONBACKEND_ModelInstanceFinalize if it is implemented in the backend shared library. TRITONBACKEND_ModelInstanceFinalize should not return until the backend is completely finalized, including stopping any threads create for the model instance and freeing any user-defined state created for the model instance. * Triton destroys the TRITONBACKEND_ModelInstance object that represents the model instance. * Triton calls TRITONBACKEND_ModelFinalize if it is implemented in the backend shared library. TRITONBACKEND_ModelFinalize should not return until the backend is completely finalized, including stopping any threads create for the model and freeing any user-defined state created for the model. * Triton destroys the TRITONBACKEND_Model object that represents the model. * Even if no other loaded model requires the backend, Triton does not finalize and unload the backend until the tritonserver process is exiting. When the tritonserver process exits: * Triton calls TRITONBACKEND_Finalize if it is implemented in the backend shared library. TRITONBACKEND_ModelFinalize should not return until the backend is completely finalized, including stopping any threads create for the backend and freeing any user-defined state created for the backend. * Triton destroys the TRITONBACKEND_Backend object that represents the backend. #### Inference Requests and Responses Triton calls TRITONBACKEND_ModelInstanceExecute to execute inference requests on a model instance. Each call to TRITONBACKEND_ModelInstanceExecute communicates a batch of requests to execute and the instance of the model that should be used to execute those requests. The backend should not allow the caller thread to return from TRITONBACKEND_ModelInstanceExecute until that instance is ready to handle another set of requests. Typically this means that the TRITONBACKEND_ModelInstanceExecute function will create responses and release the requests before returning. However, in case TRITONBACKEND_ModelInstanceExecute returns an error, the ownership of requests is transferred back to Triton which will then be responsible for releasing them. Therefore, in the case where TRITONBACKEND_ModelInstanceExecute returns an error, the backend must not retain references to the requests or access them in any way. For more detailed description of request/response lifetimes, study the documentation of TRITONBACKEND_ModelInstanceExecute in [tritonbackend.h](https://github.com/triton-inference-server/core/blob/main/include/triton/core/tritonbackend.h). ##### Single Response Most backends will create a single response for each request. For that kind of backend, executing a single inference request requires the following steps: * Create a response for the request using TRITONBACKEND_ResponseNew. * For each request input tensor use TRITONBACKEND_InputProperties to get shape and datatype of the input as well as the buffer(s) containing the tensor contents. * For each output tensor which the request expects to be returned, use TRITONBACKEND_ResponseOutput to create the output tensor of the required datatype and shape. Use TRITONBACKEND_OutputBuffer to get a pointer to the buffer where the tensor's contents should be written. * Use the inputs to perform the inference computation that produces the requested output tensor contents into the appropriate output buffers. * Optionally set parameters in the response. * Send the response using TRITONBACKEND_ResponseSend. * Release the request using TRITONBACKEND_RequestRelease. For a batch of requests the backend should attempt to combine the execution of the individual requests as much as possible to increase performance. ##### Decoupled Responses It is also possible for a backend to send multiple responses for a request. A backend may also send responses out-of-order relative to the order that the request batches are executed. Such backends are called *decoupled* backends. The decoupled backends use one `ResponseFactory` object per request to create and send any number of responses for the request. They must send at least one final response per request (even if it is a flags-only response). You can send a flags-only response with TRITONBACKEND_ResponseFactorySendFlags. For this kind of backend, executing a single inference request typically requires the following steps: 1. For each request input tensor, use TRITONBACKEND_InputProperties to get shape and datatype of the input as well as the buffer(s) containing the tensor contents. 2. Create a `ResponseFactory` object for the request using TRITONBACKEND_ResponseFactoryNew. 3. Create a response from the `ResponseFactory` object using TRITONBACKEND_ResponseNewFromFactory. As long as you have the `ResponseFactory` object, you can continue creating responses. 4. For each output tensor which the request expects to be returned, use TRITONBACKEND_ResponseOutput to create the output tensor of the required datatype and shape. Use TRITONBACKEND_OutputBuffer to get a pointer to the buffer where the tensor's contents should be written. 5. Use the inputs to perform the inference computation that produces the requested output tensor contents into the appropriate output buffers. 6. Optionally set parameters in the response. 7. Send the response using TRITONBACKEND_ResponseSend. 8. Repeat steps 3-7 until there are no more responses. 9. Send the last response for a request using either TRIONBACKEND_ResponseSend with a TRITONSERVER_ResponseCompleteFlag or after all responses have been sent for a request using TRITONBACKEND_ResponseFactorySendFlags. This is required for every request. 10. Release the request using TRITONBACKEND_RequestRelease. ###### Special Cases The decoupled API is powerful and supports various special cases: * The model can also send responses out-of-order in which it received requests. * The backend can copy out the contents of the input buffer(s) if request is to be released before the contents are completely consumed to generate responses. After copy, the request can be released anytime before exiting TRITONBACKEND_ModelInstanceExecute. The copies and `ResponseFactory` object can be passed to a separate thread in backend. This means main caller thread can exit from TRITONBACKEND_ModelInstanceExecute and the backend can still continue generating responses as long as it holds `ResponseFactory` object. The [repeat example](examples/README.md) demonstrates full power of what can be achieved from decoupled API. Study documentation of these TRITONBACKEND_* functions in [tritonbackend.h](https://github.com/triton-inference-server/core/blob/main/include/triton/core/tritonbackend.h) for more details on these APIs. Read [Decoupled Backends and Models](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/decoupled_models.md) for more details on how to host a decoupled model. ## Build the Backend Utilities The source in this repo builds into a single "backend utilities" library that is useful when building backends. You don't need to use these utilities but they will be helpful for most backends. Typically you don't need to build this repo directly but instead you can include it in the build of your backend as is shown in the CMakeLists.txt files of the [tutorial examples](examples/README.md). To build and install in a local directory use the following commands. ``` $ mkdir build $ cd build $ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install .. $ make install ``` The following required Triton repositories will be pulled and used in the build. By default the "main" branch/tag will be used for each repo but the listed CMake argument can be used to override. * triton-inference-server/common: -DTRITON_COMMON_REPO_TAG=[tag] * triton-inference-server/core: -DTRITON_CORE_REPO_TAG=[tag] See the [CMakeLists.txt](CMakeLists.txt) file for other build options. ## Python-based Backends Triton also provides an option to create [Python-based backends](docs/python_based_backends.md). These backends should implement the [`TritonPythonModel` interface](https://github.com/triton-inference-server/python_backend#usage), which could be re-used as a backend by multiple models. While the only required function is `execute`, you may find it helpful to enhance your implementation by adding `initialize`, `finalize`, and any other helper functions. For examples, please refer to the [vLLM backend](https://github.com/triton-inference-server/vllm_backend), which provides a common python script to serve models supported by vLLM. kytensor-2.49.0.7/backend/cmake/000077500000000000000000000000001527052371200163045ustar00rootroot00000000000000kytensor-2.49.0.7/backend/cmake/TritonBackendConfig.cmake.in000066400000000000000000000035631527052371200235770ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONBACKEND_CMAKE_DIR}) if(NOT TARGET TritonBackend::triton-backend-utils) include("${TRITONBACKEND_CMAKE_DIR}/TritonBackendTargets.cmake") endif() set(TRITONBACKEND_LIBRARIES TritonBackend::triton-backend-utils) kytensor-2.49.0.7/backend/docs/000077500000000000000000000000001527052371200161545ustar00rootroot00000000000000kytensor-2.49.0.7/backend/docs/backend_platform_support_matrix.md000066400000000000000000000122651527052371200251570ustar00rootroot00000000000000 # Backend-Platform Support Matrix Even though Triton supports inference across various platforms such as cloud, data center, edge and embedded devices on NVIDIA GPUs, x86 and ARM CPU, or AWS Inferentia, it does so by relying on the backends. Note that not all Triton backends support every platform. The purpose of this document is to go over what all compute platforms are supported by each of these Triton backends. GPU in this document refers to Nvidia GPU. See [GPU, Driver, and CUDA Support Matrix](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html) to learn more about supported GPUs. ## Ubuntu 22.04 The table below describes target device(s) supported for inference by each backend on different platforms. | Backend | x86 | ARM-SBSA | | ------------ | --------- | ------------- | | TensorRT | :heavy_check_mark: GPU
:x: CPU | :heavy_check_mark: GPU
:x: CPU | | ONNX Runtime | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU
:heavy_check_mark: CPU | | TensorFlow | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU
:heavy_check_mark: CPU | | PyTorch | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU
:heavy_check_mark: CPU | | OpenVINO | :x: GPU
:heavy_check_mark: CPU | :x: GPU
:x: CPU | | Python[^1] | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU
:heavy_check_mark: CPU | | DALI | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU[^2]
:heavy_check_mark: CPU[^2] | | FIL | :heavy_check_mark: GPU
:heavy_check_mark: CPU | Unsupported | | vLLM | :heavy_check_mark: GPU
:heavy_check_mark: CPU | Unsupported | ## Windows 10 Only TensorRT and ONNX Runtime backends are supported on Windows. | Backend | x86 | ARM-SBSA | | ------------ | --------- | ------------- | | TensorRT | :heavy_check_mark: GPU
:x: CPU | :heavy_check_mark: GPU
:x: CPU | | ONNX Runtime | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU
:heavy_check_mark: CPU | ## Jetson JetPack Following backends are currently supported on Jetson Jetpack: | Backend | Jetson | | ------------ | --------- | | TensorRT | :heavy_check_mark: GPU
:x: CPU | | ONNX Runtime | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU
:heavy_check_mark: CPU | | TensorFlow | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU
:heavy_check_mark: CPU | | PyTorch | :heavy_check_mark: GPU
:heavy_check_mark: CPU | :heavy_check_mark: GPU
:heavy_check_mark: CPU | | Python[^1] | :x: GPU
:heavy_check_mark: CPU | Look at the [Triton Inference Server Support for Jetson and JetPack](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/jetson.md). ## AWS Inferentia Currently, inference on AWS Inferentia is only supported via [python backend](https://github.com/triton-inference-server/python_backend#running-with-inferentia) where the deployed python script invokes AWS Neuron SDK. [^1]: The supported devices for Python Backend are mentioned with respect to Triton. The python script running in Python Backend can be used to execute inference on any hardware if there are available python APIs to do so. AWS inferentia is one such example. Triton core is largely unaware of the fact that inference will run on Inferentia. [^2]: In case of ARM-SBSA, some operations are not fully supported. kytensor-2.49.0.7/backend/docs/python_based_backends.md000066400000000000000000000165251527052371200230200ustar00rootroot00000000000000 # Python-based Backends Python-based backend is a special type of Triton's backends, which does not require any C++ code. However, this type of backends depends on [Python backend](https://github.com/triton-inference-server/python_backend) and requires the following artifacts being present: `libtriton_python.so`, `triton_python_backend_stub`, and `triton_python_backend_utils.py`. ## Usage To implement and use a Python-based backend, make sure to follow these steps. * Implement the [`TritonPythonModel` interface](https://github.com/triton-inference-server/python_backend#usage), which could be re-used as a backend by multiple models. This script should be named `model.py`. * Create a folder for your custom backend under the backends directory (ex: /opt/tritonserver/backends) with the corresponding backend name, containing the `model.py`. For example, for a backend named `my_python_based_backend`, Triton would expect to find the full path `/opt/tritonserver/backends/my_python_based_backend/model.py`. * Make sure that `libtriton_python.so`, `triton_python_backend_stub`, and `triton_python_backend_utils.py` are present either under `/opt/tritonserver/backends/my_python_based_backend/` or `/opt/tritonserver/backends/python/`. When both locations contain mentioned artifacts, custom backend's artifacts will take priority over Python backend's artifacts. This way, if custom backends needs to use a different Python version than what is shipped by default, it can easily be done. Please, refer to [customization](#customization) section for more details. * Specify `my_python_based_backend` as a backend in `config.pbtxt` for any model, that should use this backend. ``` ... backend: "my_python_based_backend" ... ``` Since Triton uses Python backend under the hood, it is expected, to see `python` backend entry in server logs, even when Python backend is not explicitly used. ``` I1013 21:52:45.756456 18668 server.cc:619] +-------------------------+-------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | Backend | Path | Config | +-------------------------+-------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | python | /opt/tritonserver/backends/python/libtriton_python.so | {"cmdline":{"auto-complete-config":"true","backend-directory":"/opt/tritonserver/backends","min-compute-capability" | | | | :"6.000000","default-max-batch-size":"4"}} | | my_python_based_backend | /opt/tritonserver/backends/my_python_based_backend/model.py | {"cmdline":{"auto-complete-config":"true","backend-directory":"/opt/tritonserver/backends","min-compute-capability" | | | | :"6.000000","default-max-batch-size":"4"}} | +-------------------------+-------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------+ ``` ## Customization Python backend shipped in the NVIDIA GPU Cloud containers uses Python 3.10. Python backend is able to use the libraries that exist in the current Python environment. These libraries can be installed in a virtualenv, conda environment, or the global system Python, and will only be used if the Python version matches the Python version of the Python backend's stub executable (`triton_python_backend_stub`). For example, if you install a set of libraries in a Python 3.9 environment and your Python backend stub is compiled with Python 3.10 these libraries will *NOT* be available. You would need to [compile](https://github.com/triton-inference-server/python_backend#building-custom-python-backend-stub) the stub executable with Python 3.9. If you want to create a tar file that contains all your Python dependencies or you want to use different Python environments for each Python model you need to create a [Custom Execution Environment](https://github.com/triton-inference-server/python_backend#creating-custom-execution-environments) in Python backend. ## Background In some use cases, it is sufficient to implement [`TritonPythonModel` interface](https://github.com/triton-inference-server/python_backend#usage) only once and re-use it across multiple models. As an example, please refer to the [vLLM backend](https://github.com/triton-inference-server/vllm_backend), which provides a common python script to serve models supported by vLLM. Triton Inference Server can handle this special case and treats common `model.py` script as a Python-based backend. In the scenario, when model relies on a custom Python-based backend, Triton loads `libtriton_python.so` first, this ensures that Triton knows how to send requests to the backend for execution and the backend knows how to communicate with Triton. Then, Triton makes sure to use common `model.py` from the backend's repository, and not look for it in the model repository. While the only required function is `execute`, it is typically helpful to enhance your implementation by adding `initialize`, `finalize`, and any other helper functions. Users are also encouraged to make use of the [`auto_complete_config`](https://github.com/triton-inference-server/python_backend#auto_complete_config) function to define standardized input and output properties upfront. kytensor-2.49.0.7/backend/examples/000077500000000000000000000000001527052371200170425ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/README.md000066400000000000000000000533441527052371200203320ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Triton Example Backends To learn how to create a Triton backend, and to see a best-practices baseline onto which you can add your own backend log, follow the [Tutorial](#tutorial). Triton also provides a couple of example backends that demonstrate specific aspects of the backend API not covered by the [Tutorial](#tutorial). * The [*repeat*](https://github.com/triton-inference-server/repeat_backend) backend shows a more advanced example of how a backend can produce multiple responses per request. * The [*stateful*](https://github.com/triton-inference-server/stateful_backend) backend shows an example of how a backend can manage model state tensors on the server-side for the [sequence batcher](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#sequence-batcher) to avoid transferring state tensors between client and server. Triton also implements [Implicit State Management](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/architecture.md#implicit-state-management) which allows backends to behave in a stateless manner and leave the state management to Triton. ## Tutorial The [Triton Backend API](../README.md#triton-backend-api) exposes a large number of features. The backend utilities and classes provide many functions commonly used when creating a backend. But to create a functional backend it is not necessary to use most of the backend API or utilities. The tutorial starts with an implementation that shows a *minimal* backend and then adds on recommended and optional enhancements. The tutorial implementations follow best practices for Triton backends and so can be used as templates for your own backend. ### *Minimal* Triton Backend The source code for the *minimal* backend is contained in [minimal.cc](backends/minimal/src/minimal.cc). The source code contains extensive documentation describing the operation of the backend and the use of the [Triton Backend API](../README.md#triton-backend-api) and the backend utilities. Before reading the source code, make sure you understand the concepts associated with Triton backend abstractions [TRITONBACKEND_Backend](../README.md#tritonbackend_backend), [TRITONBACKEND_Model](../README.md#tritonbackend_model), and [TRITONBACKEND_ModelInstance](../README.md#tritonbackend_modelinstance). The *minimal* backend does not do any interesting operation, it simply copies a single input tensor to a single output tensor, but it does demonstrate the basic organization required for a Triton backend. The *minimal* backend is complete but for clarity leaves out some important aspects of writing a full-featured backend that are described in [*Recommended* Triton Backend](#recommended-triton-backend). When creating your own backend use the [*Recommended* Triton Backend](#recommended-triton-backend) as a starting point. #### Building the *Minimal* Backend [backends/minimal/CMakeLists.txt](backends/minimal/CMakeLists.txt) shows the recommended build and install script for a Triton backend. To build the *minimal* backend and install in a local directory use the following commands. ``` $ cd backends/minimal $ mkdir build $ cd build $ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install .. $ make install ``` The following required Triton repositories will be pulled and used in the build. By default the "main" branch/tag will be used for each repo but the listed CMake argument can be used to override. * triton-inference-server/backend: -DTRITON_BACKEND_REPO_TAG=[tag] * triton-inference-server/core: -DTRITON_CORE_REPO_TAG=[tag] * triton-inference-server/common: -DTRITON_COMMON_REPO_TAG=[tag] If you are building on a release branch (or on a development branch that is based off of a release branch), then you must set these cmake arguments to point to that release branch as well. For example, if you are building the r21.10 identity_backend branch then you need to use the following additional cmake flags: ``` -DTRITON_BACKEND_REPO_TAG=r21.10 -DTRITON_CORE_REPO_TAG=r21.10 -DTRITON_COMMON_REPO_TAG=r21.10 ``` After building the install directory will contain a backends/minimal directory that contains the *minimal* backend. Instructions for adding this backend to the Triton server are described in [Backend Shared Library](../README.md#backend-shared-library). #### Running Triton with the *Minimal* Backend After adding the *minimal* backend to the Triton server as described in [Backend Shared Library](../README.md#backend-shared-library), you can run Triton and have it load the models in [model_repos/minimal_models](model_repos/minimal_models). Assuming you have created a *tritonserver* Docker image by adding the *minimal* backend to Triton, the following command will run Triton: ``` $ docker run --rm -it --net=host -v/path/to/model_repos/minimal_models:/models tritonserver --model-repository=/models ``` The console output will show similar to the following indicating that the *batching* and *nonbatching* models from the minimal_models repository have loaded correctly. Note that the model repository has two models that both use the *minimal* backend. A backend can support any number of different models. ``` I1215 23:46:00.250284 68 server.cc:589] +-------------+---------+--------+ | Model | Version | Status | +-------------+---------+--------+ | batching | 1 | READY | | nonbatching | 1 | READY | +-------------+---------+--------+ ``` The models are identical except that the *batching* model enabled the [dynamic batcher](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher) and supports batch sizes up to 8. Note that the *batching* model sets the [batch delay](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#delayed-batching) to 5 seconds so that the example client described below can demonstrate how the *minimal* backend receives a batch of requests. #### Testing the *Minimal* Backend The [clients](clients) directory holds example clients. The [minimal_client](clients/minimal_client) Python script demonstrates sending a couple of inference requests to the *minimal* backend. With Triton running as described in [Running Triton with the *Minimal* Backend](#running-triton-with-the-minimal-backend), execute the client: ``` $ clients/minimal_client ``` The minimal_client first sends a single request to nonbatching model. From the output you can see that the input value is returned in the output. ``` ========= Sending request to nonbatching model: IN0 = [1 2 3 4] Response: {'model_name': 'nonbatching', 'model_version': '1', 'outputs': [{'name': 'OUT0', 'datatype': 'INT32', 'shape': [4], 'parameters': {'binary_data_size': 16}}]} OUT0 = [1 2 3 4] ``` In the Triton console output you can see the log message printed by the *minimal* backend that indicates that it received a batch containing the single request. ``` I1221 18:14:12.964836 86 minimal.cc:348] model nonbatching: requests in batch 1 I1221 18:14:12.964857 86 minimal.cc:356] batched IN0 value: [ 1, 2, 3, 4 ] ``` The minimal_client next sends 2 requests at the same time to the batching model. Triton will dynamically batch those requests into a single batch and send that single batch to the *minimal* backend. ``` ========= Sending request to batching model: IN0 = [[10 11 12 13]] Sending request to batching model: IN0 = [[20 21 22 23]] Response: {'model_name': 'batching', 'model_version': '1', 'outputs': [{'name': 'OUT0', 'datatype': 'INT32', 'shape': [1, 4], 'parameters': {'binary_data_size': 16}}]} OUT0 = [[10 11 12 13]] Response: {'model_name': 'batching', 'model_version': '1', 'outputs': [{'name': 'OUT0', 'datatype': 'INT32', 'shape': [1, 4], 'parameters': {'binary_data_size': 16}}]} OUT0 = [[20 21 22 23]] ``` In the Triton console output you can see the log message indicating that the *minimal* backend received a batch containing both requests. ``` I1221 18:14:17.965982 86 minimal.cc:348] model batching: requests in batch 2 I1221 18:14:17.966035 86 minimal.cc:356] batched IN0 value: [ 10, 11, 12, 13, 20, 21, 22, 23 ] ``` ### *Recommended* Triton Backend The source code for the *recommended* backend is contained in [recommended.cc](backends/recommended/src/recommended.cc). The source code contains extensive documentation describing the operation of the backend and the use of the [Triton Backend API](../README.md#triton-backend-api) and the backend utilities. Before reading the source code, make sure you understand the concepts associated with Triton backend abstractions [TRITONBACKEND_Backend](../README.md#tritonbackend_backend), [TRITONBACKEND_Model](../README.md#tritonbackend_model), and [TRITONBACKEND_ModelInstance](../README.md#tritonbackend_modelinstance). The *recommended* backend improves the [*minimal* backend](#minimal-triton-backend) to include the following features which should be present in any robust backend implementation: * Enhances the backend to support models with input/output tensors that have datatypes other than INT32. * Enhances the backend to support models with input/output tensors that have any shape. * Uses the Triton backend metric APIs to record statistics about requests executing in the backend. These metrics can then we queried using the Triton [metrics](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md) and [statistics](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_statistics.md) APIs. * Additional error checking to ensure that the backend's version is compatible with Triton and that each model's configuration is compatible with the backend. As with the *minimal* backend, the *recommended* backend just returns the input tensor value in the output tensor. Because of the additions described above, the *recommended* backend can serve as a starting point for your backend. #### Building the *Recommended* Backend [backends/recommended/CMakeLists.txt](backends/recommended/CMakeLists.txt) shows the recommended build and install script for a Triton backend. Building and installing is the same as described in [Building the *Minimal* Backend](#building-the-minimal-backend). #### Running Triton with the *Recommended* Backend After adding the *recommended* backend to the Triton server as described in [Backend Shared Library](../README.md#backend-shared-library), you can run Triton and have it load the models in [model_repos/recommended_models](model_repos/recommended_models). Assuming you have created a *tritonserver* Docker image by adding the *recommended* backend to Triton, the following command will run Triton: ``` $ docker run --rm -it --net=host -v/path/to/model_repos/recommended_models:/models tritonserver --model-repository=/models ``` The console output will show similar to the following indicating that the *batching* model from the recommended_models repository have loaded correctly. ``` I1215 23:46:00.250284 68 server.cc:589] +-------------+---------+--------+ | Model | Version | Status | +-------------+---------+--------+ | batching | 1 | READY | +-------------+---------+--------+ ``` #### Testing the *Recommended* Backend The [clients](clients) directory holds example clients. The [recommended_client](clients/recommended_client) Python script demonstrates sending a couple of inference requests to the *recommended* backend. With Triton running as described in [Running Triton with the *Recommended* Backend](#running-triton-with-the-recommended-backend), execute the client: ``` $ clients/recommended_client ``` The recommended_client next sends 2 requests at the same time to the batching model, similar to what was done above with the *minimal* backend. Triton will dynamically batch those requests into a single batch and send that single batch to the *recommended* backend. In this model, batching is supported, the datatype is FP32 and the tensor shape is [ -1, 4, 4 ]. ``` ========= Sending request to batching model: input = [[[1. 1.1 1.2 1.3] [2. 2.1 2.2 2.3] [3. 3.1 3.2 3.3] [4. 4.1 4.2 4.3]]] Sending request to batching model: input = [[[10. 10.1 10.2 10.3] [20. 20.1 20.2 20.3] [30. 30.1 30.2 30.3] [40. 40.1 40.2 40.3]]] Response: {'model_name': 'batching', 'model_version': '1', 'outputs': [{'name': 'OUTPUT', 'datatype': 'FP32', 'shape': [1, 4, 4], 'parameters': {'binary_data_size': 64}}]} OUTPUT = [[[1. 1.1 1.2 1.3] [2. 2.1 2.2 2.3] [3. 3.1 3.2 3.3] [4. 4.1 4.2 4.3]]] Response: {'model_name': 'batching', 'model_version': '1', 'outputs': [{'name': 'OUTPUT', 'datatype': 'FP32', 'shape': [1, 4, 4], 'parameters': {'binary_data_size': 64}}]} OUTPUT = [[[10. 10.1 10.2 10.3] [20. 20.1 20.2 20.3] [30. 30.1 30.2 30.3] [40. 40.1 40.2 40.3]]] ``` In the Triton console output you can see the log message indicating that the *recommended* backend received a batch containing both requests. ``` I1221 18:30:52.223226 127 recommended.cc:604] model batching: requests in batch 2 I1221 18:30:52.223313 127 recommended.cc:613] batched INPUT value: [ 1.000000, 1.100000, 1.200000, 1.300000, 2.000000, 2.100000, 2.200000, 2.300000, 3.000000, 3.100000, 3.200000, 3.300000, 4.000000, 4.100000, 4.200000, 4.300000, 10.000000, 10.100000, 10.200000, 10.300000, 20.000000, 20.100000, 20.200001, 20.299999, 30.000000, 30.100000, 30.200001, 30.299999, 40.000000, 40.099998, 40.200001, 40.299999 ] ``` Because the *recommended* backend can support models that have input/output tensors with any datatype and shape, you can edit the model configuration and the client to experiment with these options. To see the metrics collected for these two inference requests, use the following command to access Triton's metrics endpoint. ``` $ curl localhost:8002/metrics ``` The output will be metric values in Prometheus data format. The [metrics documentation](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md) gives a description of these metric values. ``` # HELP nv_inference_request_success Number of successful inference requests, all batch sizes # TYPE nv_inference_request_success counter nv_inference_request_success{model="batching",version="1"} 2.000000 # HELP nv_inference_request_failure Number of failed inference requests, all batch sizes # TYPE nv_inference_request_failure counter nv_inference_request_failure{model="batching",version="1"} 0.000000 # HELP nv_inference_count Number of inferences performed # TYPE nv_inference_count counter nv_inference_count{model="batching",version="1"} 2.000000 # HELP nv_inference_exec_count Number of model executions performed # TYPE nv_inference_exec_count counter nv_inference_exec_count{model="batching",version="1"} 1.000000 ... ``` You can also see the collected statistics using the [statistics endpoint](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_statistics.md). ``` $ curl localhost:8000/v2/models/batching/stats {"model_stats":[{"name":"batching","version":"1","last_inference":1640111452223,"inference_count":2,"execution_count":1,"inference_stats":{"success":{"count":2,"ns":9997025869},"fail":{"count":0,"ns":0},"queue":{"count":2,"ns":9996491319},"compute_input":{"count":2,"ns":95288},"compute_infer":{"count":2,"ns":232202},"compute_output":{"count":2,"ns":195850}},"batch_stats":[{"batch_size":2,"compute_input":{"count":1,"ns":47644},"compute_infer":{"count":1,"ns":116101},"compute_output":{"count":1,"ns":97925}}]}]} ``` ### *BLS* Triton Backend Please see the [documentation](backends/bls/README.md) of *BLS* Backend. ### Custom Batching When using the dynamic batcher, Triton allows you to set custom batching rules. These rules are added on top of the specified dynamic batcher behavior. To set them, you pass in a library that implements the custom batching API. Two example batching libraries are located in the [batching_strategies directory](batching_strategies). For this tutorial, you can use the [volume_batching](batching_strategies/volume_batching) example to set up a maximum byte volume per request. To build the library and install in a local directory, use the following commands: ``` $ cd batch_strategies/volume_batching $ mkdir build $ cd build $ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install .. $ make install ``` Next, move the library to the desired location. You can pass the file location via the model configuration. If not specified, Triton will look for a library called `batchstrategy.so` in the model version, model, and backend directories, in that order. For ease, we'll pass it via the model configuration. Select a model to use this strategy with. Then, update the model configuration to have these fields: ``` dynamic_batching { } parameters: { key: "TRITON_BATCH_STRATEGY_PATH", value: {string_value: "/path/to/libtriton_volumebatching.so"}} parameters { key: "MAX_BATCH_VOLUME_BYTES" value: {string_value: "96"}} ``` You can update the path to the filepath of your library. You can also update the value of `MAX_BATCH_VOLUME_BYTES` to the maximum volume per batch for your use case. After starting Triton, you should see the scheduler apply a volume constraint per batch on top of default batching behavior for your model. This can be made more visible by setting a [max queue delay](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#delayed-batching) to give the scheduler more time for each batch to be completed. For example, you could set the delay to 100,000 microseconds. ### Enhancements This section describes several optional features that you can add to enhance the capabilities of your backend. #### Automatically Model Configuration Generation [Automatic model configuration generation](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#auto-generated-model-configuration) is enabled by the backend implementing the appropriate logic (for example, in a function called AutoCompleteConfig) during TRITONBACKEND_ModelInitialize. For the *recommended* backend you would add a call to AutoCompleteConfig in the ModelState constructor just before the call to ValidateModelConfig. The AutoCompleteConfig function can update the model configuration with input tensor, output tensor, and max-batch-size configuration; and then update the configuration using TRITONBACKEND_ModelSetConfig. Examples can be found in [ONNXRuntime backend](https://github.com/triton-inference-server/onnxruntime_backend), [TensorFlow backend](https://github.com/triton-inference-server/tensorflow_backend) and other backends. #### Add Key-Value Parameters to a Response A backend can add a key-value pair to a response any time after the response is created and before it is sent. The parameter key must be a string and the parameter value can be a string, integer or boolean. The following example shows the TRITONBACKEND API used to set response parameters. Error checking code is not shown to improve clarity. ``` TRITONBACKEND_ResponseSetStringParameter(response, "param0", "an example string parameter"); TRITONBACKEND_ResponseSetIntParameter(responses[r], "param1", 42); TRITONBACKEND_ResponseSetBoolParameter(responses[r], "param2", false); ``` #### Access Model Artifacts in the Model Repository A backend can access any of the files in a model's area of the model registry. These files are typically needed during TRITONBACKEND_ModelInitialize but can be accessed at other times as well. The TRITONBACKEND_ModelRepository API gives the location of the model's repository. For example, the following code can be run during TRITONBACKEND_ModelInitialize to write the location to the log. ``` // Can get location of the model artifacts. Normally we would need // to check the artifact type to make sure it was something we can // handle... but we are just going to log the location so we don't // need the check. We would use the location if we wanted to load // something from the model's repo. TRITONBACKEND_ArtifactType artifact_type; const char* clocation; RETURN_IF_ERROR( TRITONBACKEND_ModelRepository(model, &artifact_type, &clocation)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Repository location: ") + clocation).c_str()); ``` The framework backends (for example, TensorRT, ONNXRuntime, TensorFlow, PyTorch) read the actual model file from the model repository using this API. See those backends for examples of how it can be used. kytensor-2.49.0.7/backend/examples/backends/000077500000000000000000000000001527052371200206145ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/bls/000077500000000000000000000000001527052371200213745ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/bls/CMakeLists.txt000066400000000000000000000121601527052371200241340ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonblsbackend LANGUAGES C CXX) # # Options # # Must include options required for this project as well as any # projects included in this one by FetchContent. # # GPU support is disabled by default because BLS backend doesn't # support GPUs. # option(TRITON_ENABLE_GPU "Enable GPU support in backend" OFF) option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON) set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") # # Setting C++ min standard # set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard whose features are requested to build this target.") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # # FetchContent's composability isn't very good. We must include the # transitive closure of all repos so that we can override the tag. # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-backend SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/backend ) FetchContent_MakeAvailable(repo-common repo-core repo-backend) # # Shared library implementing the Triton Backend API # configure_file(src/libtriton_bls.ldscript libtriton_bls.ldscript COPYONLY) add_library( triton-bls-backend SHARED src/backend.cc src/bls.h src/bls.cc src/bls_utils.h src/bls_utils.cc ) add_library( TritonBLSBackend::triton-bls-backend ALIAS triton-bls-backend ) target_include_directories( triton-bls-backend PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_compile_features(triton-bls-backend PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-bls-backend PRIVATE $<$,$,$>: -Wall -Wextra -Wno-unused-parameter -Werror> ) target_link_libraries( triton-bls-backend PRIVATE triton-core-serverstub # from repo-core triton-backend-utils # from repo-backend ) set_target_properties( triton-bls-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_bls LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_bls.ldscript LINK_FLAGS "-Wl,--version-script libtriton_bls.ldscript" ) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonBLSBackend) install( TARGETS triton-bls-backend EXPORT triton-bls-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/bls ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/bls ) install( EXPORT triton-bls-backend-targets FILE TritonBLSBackendTargets.cmake NAMESPACE TritonBLSBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonBLSBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonBLSBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonBLSBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-bls-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonBLSBackendTargets.cmake NAMESPACE TritonBLSBackend:: ) export(PACKAGE TritonBLSBackend) kytensor-2.49.0.7/backend/examples/backends/bls/README.md000066400000000000000000000144341527052371200226610ustar00rootroot00000000000000 # *BLS* Triton Backend The [*BLS*](../bls) backend demonstrates using in-process C-API to execute inferences within the backend. This backend serves as an example to backend developers for implementing their own custom pipeline in C++. For Python use cases, please refer to [Business Logic Scripting](https://github.com/triton-inference-server/python_backend/blob/main/README.md#business-logic-scripting) section in Python backend. The source code for the *bls* backend is contained in [src](./src). * [backend.cc](./src/backend.cc) contains the main backend implementation. The content of this file is not BLS specific. It only includes the required Triton backend functions that is standard for any backend implementation. The BLS logic is set off in the `TRITONBACKEND_ModelInstanceExecute` with lines `bls_executor.Execute(requests[r], &responses[r]);`. * [bls.h](./src/bls.h) is where the BLS (class `BLSExecutor`) of this example is located. You can refer to this file to see how to interact with Triton in-process C-API to build the custom execution pipeline. * [bls_utils.h](./src/bls_utils.h) is where all the utilities that are not BLS dependent are located. The source code contains extensive documentation describing the operation of the backend and the use of the [Triton Backend API](../../../README.md#triton-backend-api) and the [Triton Server API](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/inference_protocols.md#in-process-triton-server-api). Before reading the source code, make sure you understand the concepts associated with Triton backend abstractions [TRITONBACKEND_Backend](../../../README.md#tritonbackend_backend), [TRITONBACKEND_Model](../../../README.md#tritonbackend_model), and [TRITONBACKEND_ModelInstance](../../../README.md#tritonbackend_modelinstance). The *bls* backend will send two requests on the 'addsub_python' and 'addsub_tf' models. After the inference requests are completed, this backend will extract OUTPUT0 from the 'addsub_python' and OUTPUT1 from the 'addsub_tf' model to construct the final inference response object using these tensors. There are some self-imposed limitations that were made for the simplicity of this example: 1. This backend does not support batching. 2. This backend does not support decoupled models. 3. This backend does not support GPU tensors. 4. The model configuration should be strictly set as the comments described in [backend.cc](./src/backend.cc). You can implement your custom backend that is not limited to the limitations mentioned above. ## Building the *BLS* Backend [backends/bls/CMakeLists.txt](CMakeLists.txt) shows the recommended build and install script for a Triton backend. Building and installing is the same as described in [Building the *Minimal* Backend](../../README.md#building-the-minimal-backend). ## Running Triton with the *BLS* Backend After adding the *bls* backend to the Triton server as described in [Backend Shared Library](../../../README.md#backend-shared-library), you can run Triton and have it load the models in [model_repos/bls_models](../../model_repos/bls_models). Assuming you have created a *tritonserver* Docker image by adding the *bls* backend to Triton, the following command will run Triton: ``` $ docker run --rm -it --net=host -v/path/to/model_repos/bls_models:/models tritonserver --model-repository=/models ``` The console output will show similar to the following indicating that the *bls_fp32*, *addsub_python* and *addsub_tf* models from the bls_models repository have loaded correctly. ``` I0616 09:34:47.767433 19214 server.cc:629] +---------------+---------+--------+ | Model | Version | Status | +---------------+---------+--------+ | addsub_python | 1 | READY | | addsub_tf | 1 | READY | | bls_fp32 | 1 | READY | +---------------+---------+--------+ ``` ## Testing the *BLS* Backend The [clients](../../clients) directory holds example clients. The [bls_client](../../clients/bls_client) Python script demonstrates sending an inference requests to the *bls* backend. With Triton running as described in [Running Triton with the *BLS* Backend](#running-triton-with-the-bls-backend), execute the client: ``` $ clients/bls_client ``` You should see an output similar to the output below: ``` INPUT0 ([0.42935285 0.51512766 0.43625894 ... 0.6670954 0.17747518 0.7976901 ]) + INPUT1 ([6.7752063e-01 2.4223252e-01 6.7743927e-01 ... 4.1531715e-01 2.5451833e-01 7.9097062e-01]) = OUTPUT0 ([1.1068735 0.75736016 1.1136982 ... 1.0824126 0.4319935 1.5886607 ]) INPUT0 ([0.42935285 0.51512766 0.43625894 ... 0.6670954 0.17747518 0.7976901 ]) - INPUT1 ([6.7752063e-01 2.4223252e-01 6.7743927e-01 ... 4.1531715e-01 2.5451833e-01 7.9097062e-01]) = OUTPUT1 ([-0.24816778 0.27289516 -0.24118033 ... 0.25177827 -0.07704315 0.00671947]) PASS ``` kytensor-2.49.0.7/backend/examples/backends/bls/cmake/000077500000000000000000000000001527052371200224545ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/bls/cmake/TritonBLSBackendConfig.cmake.in000066400000000000000000000036151527052371200303060ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONBLSBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONBLSBACKEND_CMAKE_DIR}) if(NOT TARGET TritonBLSBackend::triton-bls-backend) include("${TRITONBLSBACKEND_CMAKE_DIR}/TritonBLSBackendTargets.cmake") endif() set(TRITONBLSBACKEND_LIBRARIES TritonBLSBackend::triton-bls-backend) kytensor-2.49.0.7/backend/examples/backends/bls/src/000077500000000000000000000000001527052371200221635ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/bls/src/backend.cc000066400000000000000000000467301527052371200240730ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "bls.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" // // Backend that demonstrates using in-process C-API to execute inferences // within the backend. // // Two particular models, 'addsub_python' and 'addsub_tf', must be loaded on // the server for a successful inference execution on this backend. // // The model configuration should be set as follows in order to be in line with // the 'addsub_python' and 'addsub_tf' models. This backend does not support // batching. These limitations are only for this specific backend. You can // implement your custom BLS backend with less limitations. // // Model Configuration: // - Input 'INPUT0' must have shape [16] and datatype must be TYPE_FP32. // // - Input 'INPUT1' must have shape [16] and datatype must be TYPE_FP32. // // - For each response, output 'OUTPUT0' must have shape [16] and // datatype TYPE_FP32. // // - For each response, output 'OUTPUT1' must have shape [16] and // datatype TYPE_FP32. // // This backend will send two requests on the 'addsub_python' and 'addsub_tf' // models. After the inference requests are completed, this backend // will extract OUTPUT0 from the 'addsub_python' and OUTPUT1 from the // 'addsub_tf' model to construct the final inference response object using // these tensors. namespace triton { namespace backend { namespace bls { // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; // Validate that model configuration is supported by this backend. TRITONSERVER_Error* ValidateModelConfig(); private: ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model) {} }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } TRITONSERVER_Error* ModelState::ValidateModelConfig() { // We have the json DOM for the model configuration... common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(model_config_.PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model configuration:\n") + buffer.Contents()).c_str()); // max_batch_size must be 0 because this backend does not support // batching int64_t max_batch_size; RETURN_IF_ERROR(model_config_.MemberAsInt("max_batch_size", &max_batch_size)); RETURN_ERROR_IF_FALSE( max_batch_size == 0, TRITONSERVER_ERROR_INVALID_ARG, std::string("bls backend only supports models with max_batch_size == 0")); common::TritonJson::Value inputs, outputs; RETURN_IF_ERROR(model_config_.MemberAsArray("input", &inputs)); RETURN_IF_ERROR(model_config_.MemberAsArray("output", &outputs)); // There must be 2 inputs and 2 outputs. RETURN_ERROR_IF_FALSE( inputs.ArraySize() == 2, TRITONSERVER_ERROR_INVALID_ARG, std::string("expected 2 inputs, got ") + std::to_string(inputs.ArraySize())); RETURN_ERROR_IF_FALSE( outputs.ArraySize() == 2, TRITONSERVER_ERROR_INVALID_ARG, std::string("expected 2 outputs, got ") + std::to_string(outputs.ArraySize())); // Here we rely on the model configuration listing the inputs and // outputs in a specific order, which we shouldn't really require... common::TritonJson::Value input0, input1, output0, output1; RETURN_IF_ERROR(inputs.IndexAsObject(0, &input0)); RETURN_IF_ERROR(inputs.IndexAsObject(1, &input1)); RETURN_IF_ERROR(outputs.IndexAsObject(0, &output0)); RETURN_IF_ERROR(outputs.IndexAsObject(1, &output1)); // Check tensor names std::string in0_name, in1_name, out0_name, out1_name; RETURN_IF_ERROR(input0.MemberAsString("name", &in0_name)); RETURN_IF_ERROR(input1.MemberAsString("name", &in1_name)); RETURN_IF_ERROR(output0.MemberAsString("name", &out0_name)); RETURN_IF_ERROR(output1.MemberAsString("name", &out1_name)); RETURN_ERROR_IF_FALSE( in0_name == "INPUT0", TRITONSERVER_ERROR_INVALID_ARG, std::string("expected first input tensor name to be INPUT0, got ") + in0_name); RETURN_ERROR_IF_FALSE( in1_name == "INPUT1", TRITONSERVER_ERROR_INVALID_ARG, std::string("expected second input tensor name to be INPUT1, got ") + in1_name); RETURN_ERROR_IF_FALSE( out0_name == "OUTPUT0", TRITONSERVER_ERROR_INVALID_ARG, std::string("expected first output tensor name to be OUTPUT0, got ") + out0_name); RETURN_ERROR_IF_FALSE( out1_name == "OUTPUT1", TRITONSERVER_ERROR_INVALID_ARG, std::string("expected second output tensor name to be OUTPUT1, got ") + out1_name); // Check shapes std::vector in0_shape, in1_shape, out0_shape, out1_shape; RETURN_IF_ERROR(backend::ParseShape(input0, "dims", &in0_shape)); RETURN_IF_ERROR(backend::ParseShape(input1, "dims", &in1_shape)); RETURN_IF_ERROR(backend::ParseShape(output0, "dims", &out0_shape)); RETURN_IF_ERROR(backend::ParseShape(output1, "dims", &out1_shape)); RETURN_ERROR_IF_FALSE( in0_shape.size() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string("expected INPUT0 shape to have one dimension, got ") + backend::ShapeToString(in0_shape)); RETURN_ERROR_IF_FALSE( in1_shape.size() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string("expected INPUT1 shape to have one dimension, got ") + backend::ShapeToString(in1_shape)); RETURN_ERROR_IF_FALSE( out0_shape.size() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string("expected OUTPUT0 shape to have one dimension, got ") + backend::ShapeToString(out0_shape)); RETURN_ERROR_IF_FALSE( out1_shape.size() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string("expected OUTPUT1 shape to have one dimension, got ") + backend::ShapeToString(out1_shape)); // Check datatypes std::string in0_dtype, in1_dtype, out0_dtype, out1_dtype; RETURN_IF_ERROR(input0.MemberAsString("data_type", &in0_dtype)); RETURN_IF_ERROR(input1.MemberAsString("data_type", &in1_dtype)); RETURN_IF_ERROR(output0.MemberAsString("data_type", &out0_dtype)); RETURN_IF_ERROR(output1.MemberAsString("data_type", &out1_dtype)); RETURN_ERROR_IF_FALSE( in0_dtype == "TYPE_FP32", TRITONSERVER_ERROR_INVALID_ARG, std::string("expected INPUT0 datatype to be TYPE_FP32, got ") + in0_dtype); RETURN_ERROR_IF_FALSE( in1_dtype == "TYPE_FP32", TRITONSERVER_ERROR_INVALID_ARG, std::string("expected INPUT1 datatype to be TYPE_FP32, got ") + in1_dtype); RETURN_ERROR_IF_FALSE( out0_dtype == "TYPE_FP32", TRITONSERVER_ERROR_INVALID_ARG, std::string("expected OUTPUT0 datatype to be TYPE_FP32, got ") + out0_dtype); RETURN_ERROR_IF_FALSE( out1_dtype == "TYPE_FP32", TRITONSERVER_ERROR_INVALID_ARG, std::string("expected OUTPUT1 datatype to be TYPE_FP32, got ") + out1_dtype); return nullptr; // success } // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each TRITONBACKEND_ModelInstance. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState() = default; void ProcessRequests( TRITONBACKEND_Request** requests, const uint32_t request_count); private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance) { } }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } void ModelInstanceState::ProcessRequests( TRITONBACKEND_Request** requests, const uint32_t request_count) { uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); for (size_t i = 0; i < request_count; i++) { // If we get a nullptr request then something is badly wrong. Fail // and release all requests. if (requests[i] == nullptr) { RequestsRespondWithError( requests, request_count, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "null request given to BLS backend for '" + Name() + "'") .c_str())); return; } } // At this point we accept ownership of 'requests', which means that // even if something goes wrong we must still return success from // this function. If something does go wrong in processing a // particular request then we send an error response just for the // specific request. std::vector responses; responses.reserve(request_count); for (size_t i = 0; i < request_count; i++) { TRITONBACKEND_Response* response; auto err = TRITONBACKEND_ResponseNew(&response, requests[i]); if (err == nullptr) { responses.emplace_back(response); } else { responses.emplace_back(nullptr); LOG_MESSAGE(TRITONSERVER_LOG_ERROR, "Fail to create response"); TRITONSERVER_ErrorDelete(err); } } ModelState* model_state = reinterpret_cast(Model()); // The way we collect these batch timestamps is not entirely // accurate. Normally, in a performant backend you would execute all // the requests at the same time, and so there would be a single // compute-start / compute-end time-range. But here we execute each // request separately so there is no single range. As a result we // just show the entire execute time as being the compute time as // well. uint64_t compute_start_ns = 0; SET_TIMESTAMP(compute_start_ns); // Create a BLSExecutor object. To separate from standard backend // implementation, the BLS logic is placed inside class BLSExecutor. BLSExecutor bls_executor(model_state->TritonServer()); for (size_t r = 0; r < request_count; r++) { bls_executor.Execute(requests[r], &responses[r]); } uint64_t compute_end_ns = 0; SET_TIMESTAMP(compute_end_ns); uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); // Send all the responses that haven't already been sent because of // an earlier error. Note that the responses are not set to nullptr // here as we need that indication below to determine if the request // we successful or not. for (auto& response : responses) { if (response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr), "failed to send BLS backend response"); } } // Report statistics for each request. for (uint32_t r = 0; r < request_count; ++r) { auto& request = requests[r]; LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting request statistics"); LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } // Report the entire batch statistics. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( TritonModelInstance(), 1 /*total_batch_size*/, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting batch request statistics"); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("TRITONBACKEND_ModelExecute: model ") + Name() + " released " + std::to_string(request_count) + " requests") .c_str()); } ///////////// extern "C" { // Implementing TRITONBACKEND_ModelInitialize is optional. The backend // should initialize any state that is intended to be shared across // all instances of the model. TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); // With each model we create a ModelState object and associate it // with the TRITONBACKEND_Model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast(model_state))); // One of the primary things to do in ModelInitialize is to examine // the model configuration to ensure that it is something that this // backend can support. If not, returning an error from this // function will prevent the model from loading. RETURN_IF_ERROR(model_state->ValidateModelConfig()); return nullptr; // success } // Implementing TRITONBACKEND_ModelFinalize is optional unless state // is set using TRITONBACKEND_ModelSetState. The backend must free // this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceInitialize is optional. The // backend should initialize any state that is required for a model // instance. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); // The instance can access the corresponding model as well... here // we get the model and from that get the model's state. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast(vmodelstate); // With each instance we create a ModelInstanceState object and // associate it with the TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast(instance_state))); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("TRITONBACKEND_ModelInstanceInitialize: instance " "initialization successful ") + name + " (device " + std::to_string(device_id) + ")") .c_str()); return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceFinalize is optional unless // state is set using TRITONBACKEND_ModelInstanceSetState. The backend // must free this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceExecute is required. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Suggested practice for this is to use only // function-local and model-instance-specific state (obtained from // 'instance'), which is what we do here. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast(&instance_state))); ModelState* model_state = reinterpret_cast(instance_state->Model()); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("model ") + model_state->Name() + ", instance " + instance_state->Name() + ", executing " + std::to_string(request_count) + " requests") .c_str()); instance_state->ProcessRequests(requests, request_count); return nullptr; // success } } // extern "C" }}} // namespace triton::backend::bls kytensor-2.49.0.7/backend/examples/backends/bls/src/bls.cc000066400000000000000000000276041527052371200232630ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "bls.h" namespace triton { namespace backend { namespace bls { BLSExecutor::BLSExecutor(TRITONSERVER_Server* server) : server_(server), model_executor_(server) { } TRITONSERVER_Error* BLSExecutor::PrepareInferenceRequest( TRITONBACKEND_Request* bls_request, TRITONSERVER_InferenceRequest** irequest, const std::string model_name) { // Get request_id, correlation_id, and flags from the current request // for preparing a new inference request that we will send to 'addsub_python' // or 'addsub_tf' model later. const char* request_id; uint64_t correlation_id; uint32_t flags; RETURN_IF_ERROR(TRITONBACKEND_RequestId(bls_request, &request_id)); RETURN_IF_ERROR( TRITONBACKEND_RequestCorrelationId(bls_request, &correlation_id)); RETURN_IF_ERROR(TRITONBACKEND_RequestFlags(bls_request, &flags)); // Create an inference request object. The inference request object // is where we set the name of the model we want to use for // inference and the input tensors. RETURN_IF_ERROR(TRITONSERVER_InferenceRequestNew( irequest, server_, model_name.c_str(), -1 /* model_version */)); // Set request_id, correlation_id, and flags for the new request. RETURN_IF_ERROR(TRITONSERVER_InferenceRequestSetId(*irequest, request_id)); RETURN_IF_ERROR( TRITONSERVER_InferenceRequestSetCorrelationId(*irequest, correlation_id)); RETURN_IF_ERROR(TRITONSERVER_InferenceRequestSetFlags(*irequest, flags)); RETURN_IF_ERROR(TRITONSERVER_InferenceRequestSetReleaseCallback( *irequest, InferRequestComplete, nullptr /* request_release_userp */)); return nullptr; // success } TRITONSERVER_Error* BLSExecutor::PrepareInferenceInput( TRITONBACKEND_Request* bls_request, TRITONSERVER_InferenceRequest* irequest) { // Get the properties of the two inputs from the current request. // Then, add the two input tensors and append the input data to the new // request. uint32_t input_count; RETURN_IF_ERROR(TRITONBACKEND_RequestInputCount(bls_request, &input_count)); TRITONBACKEND_Input* input; const char* name; TRITONSERVER_DataType datatype; const int64_t* shape; uint32_t dims_count; size_t data_byte_size; TRITONSERVER_MemoryType data_memory_type; int64_t data_memory_id; const char* data_buffer; for (size_t count = 0; count < input_count; count++) { RETURN_IF_ERROR(TRITONBACKEND_RequestInputByIndex( bls_request, count /* index */, &input)); RETURN_IF_ERROR(TRITONBACKEND_InputProperties( input, &name, &datatype, &shape, &dims_count, nullptr, nullptr)); RETURN_IF_ERROR(TRITONBACKEND_InputBuffer( input, 0 /* idx */, reinterpret_cast(&data_buffer), &data_byte_size, &data_memory_type, &data_memory_id)); RETURN_IF_ERROR(TRITONSERVER_InferenceRequestAddInput( irequest, name, datatype, shape, dims_count)); RETURN_IF_ERROR(TRITONSERVER_InferenceRequestAppendInputData( irequest, name, &data_buffer[0], data_byte_size, data_memory_type, data_memory_id)); } return nullptr; // success } TRITONSERVER_Error* BLSExecutor::PrepareInferenceOutput( TRITONBACKEND_Request* bls_request, TRITONSERVER_InferenceRequest* irequest) { // Indicate the output tensors to be calculated and returned // for the inference request. uint32_t output_count; RETURN_IF_ERROR(TRITONBACKEND_RequestOutputCount(bls_request, &output_count)); const char* output_name; for (size_t count = 0; count < output_count; count++) { RETURN_IF_ERROR(TRITONBACKEND_RequestOutputName( bls_request, count /* index */, &output_name)); RETURN_IF_ERROR( TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output_name)); } return nullptr; // success } void BLSExecutor::Execute( TRITONBACKEND_Request* bls_request, TRITONBACKEND_Response** response) { // The names of the models that we will send internal requests on. std::vector model_names = {"addsub_python", "addsub_tf"}; // Check if both models are valid before executing request. try { for (size_t i = 0; i < 2; i++) { // Check if the model is ready. bool is_ready = false; THROW_IF_TRITON_ERROR(TRITONSERVER_ServerModelIsReady( server_, model_names[i].c_str(), -1 /* model_version */, &is_ready)); if (!is_ready) { throw BLSBackendException( (std::string("Failed to execute the inference request. Model '") + model_names[i].c_str() + "' is not ready.") .c_str()); } // For simplicity, decoupled API is not supported in this BLS backend. You // can implement your own backend that supports decoupled models. uint32_t txn_flags; THROW_IF_TRITON_ERROR(TRITONSERVER_ServerModelTransactionProperties( server_, model_names[i].c_str(), -1 /* model_version */, &txn_flags, nullptr /* voidp */)); if ((txn_flags & TRITONSERVER_TXN_DECOUPLED) != 0) { throw BLSBackendException( std::string("Model '") + model_names[i].c_str() + "' is using the decoupled. This BLS Backend doesn't support models " "using the decoupled transaction policy."); } } } catch (const BLSBackendException& bls_exception) { LOG_MESSAGE(TRITONSERVER_LOG_ERROR, bls_exception.what()); RESPOND_AND_SET_NULL_IF_ERROR( response, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to send inference requests")); return; } // Prepare std::future for each model. Since this BLS backend // can handle requests in parallel, we will send all the inference // requests first and then retrieve them later. std::vector> futures(2); // The inference request object for sending internal requests. TRITONSERVER_InferenceRequest* irequest = nullptr; // For each inference request, the backend sends two requests on the // 'addsub_python' and 'addsub_tf' models. try { for (size_t icount = 0; icount < 2; icount++) { // Initialize the inference request with required information. THROW_IF_TRITON_ERROR( PrepareInferenceRequest(bls_request, &irequest, model_names[icount])); THROW_IF_TRITON_ERROR(PrepareInferenceInput(bls_request, irequest)); THROW_IF_TRITON_ERROR(PrepareInferenceOutput(bls_request, irequest)); // Execute inference request. THROW_IF_TRITON_ERROR( model_executor_.AsyncExecute(irequest, &futures[icount])); } } catch (const BLSBackendException& bls_exception) { LOG_MESSAGE(TRITONSERVER_LOG_ERROR, bls_exception.what()); LOG_IF_ERROR( TRITONSERVER_InferenceRequestDelete(irequest), "Failed to delete inference request."); RESPOND_AND_SET_NULL_IF_ERROR( response, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to send inference requests")); return; } // If both internal requests are sent successfully, retrieve the output from // each request and construct the final response. ConstructFinalResponse(response, std::move(futures)); } void BLSExecutor::ConstructFinalResponse( TRITONBACKEND_Response** response, std::vector> futures) { // Prepare two TRITONSERVER_InferenceResponse* objects for 'addsub_python' and // 'addsub_tf' respectively. std::vector completed_responses = { nullptr, nullptr}; const char* output_name; TRITONSERVER_DataType output_datatype; const int64_t* output_shape; uint64_t dims_count; size_t output_byte_size; TRITONSERVER_MemoryType output_memory_type; int64_t output_memory_id; const void* output_base; void* userp; for (size_t icount = 0; icount < 2; icount++) { // Retrieve the corresponding TRITONSERVER_InferenceResponse object from // 'futures'. The InferResponseComplete function sets the std::promise // so that this thread will block until the response is returned. completed_responses[icount] = futures[icount].get(); try { THROW_IF_TRITON_ERROR( TRITONSERVER_InferenceResponseError(completed_responses[icount])); } catch (const BLSBackendException& bls_exception) { LOG_MESSAGE(TRITONSERVER_LOG_ERROR, bls_exception.what()); if (completed_responses[icount] != nullptr) { LOG_IF_ERROR( TRITONSERVER_InferenceResponseDelete(completed_responses[icount]), "Failed to delete inference response."); } return; } // Retrieve outputs from 'completed_responses'. // Extract OUTPUT0 from the 'addsub_python' and OUTPUT1 from the // 'addsub_tf' model to form the final inference response object. // Get all the information about the output tensor. RESPOND_AND_SET_NULL_IF_ERROR( response, TRITONSERVER_InferenceResponseOutput( completed_responses[icount], icount, &output_name, &output_datatype, &output_shape, &dims_count, &output_base, &output_byte_size, &output_memory_type, &output_memory_id, &userp)); // Create an output tensor in the final response with // the information retrieved above. TRITONBACKEND_Output* output; RESPOND_AND_SET_NULL_IF_ERROR( response, TRITONBACKEND_ResponseOutput( *response, &output, output_name, output_datatype, output_shape, dims_count)); // Get a buffer that holds the tensor data for the output. // We request a buffer in CPU memory but we have to handle any returned // type. If we get back a buffer in GPU memory we just fail the request. void* output_buffer; output_memory_type = TRITONSERVER_MEMORY_CPU; RESPOND_AND_SET_NULL_IF_ERROR( response, TRITONBACKEND_OutputBuffer( output, &output_buffer, output_byte_size, &output_memory_type, &output_memory_id)); if (output_memory_type == TRITONSERVER_MEMORY_GPU) { RESPOND_AND_SET_NULL_IF_ERROR( response, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "failed to create output buffer in CPU memory")); } // Fill the BLS output buffer with output data returned by internal // requests. memcpy(output_buffer, output_base, output_byte_size); LOG_IF_ERROR( TRITONSERVER_InferenceResponseDelete(completed_responses[icount]), "Failed to delete inference response."); } } }}} // namespace triton::backend::bls kytensor-2.49.0.7/backend/examples/backends/bls/src/bls.h000066400000000000000000000063461527052371200231250ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include "bls_utils.h" #include "triton/backend/backend_common.h" #include "triton/core/tritonbackend.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { namespace bls { // // BLSExecutor // // Includes the custom BLS logic for this backend. // This class shows how to utilize Triton in-process C-API to build the // execution pipeline. // class BLSExecutor { public: BLSExecutor(TRITONSERVER_Server* server); // Prepares the inference request that will be used internally. TRITONSERVER_Error* PrepareInferenceRequest( TRITONBACKEND_Request* bls_request, TRITONSERVER_InferenceRequest** irequest, const std::string model_name); // Prepares the input for the internal inference request. TRITONSERVER_Error* PrepareInferenceInput( TRITONBACKEND_Request* bls_request, TRITONSERVER_InferenceRequest* irequest); // Prepares the output for the internal inference request. TRITONSERVER_Error* PrepareInferenceOutput( TRITONBACKEND_Request* bls_request, TRITONSERVER_InferenceRequest* irequest); // Performs the whole BLS pipeline. void Execute( TRITONBACKEND_Request* bls_request, TRITONBACKEND_Response** response); // Constructs the final response. void ConstructFinalResponse( TRITONBACKEND_Response** response, std::vector> futures); private: // The server object that encapsulates all the functionality of the Triton // server and allows access to the Triton server API. TRITONSERVER_Server* server_; // The ModelExecutor object for executing inference request on a model. ModelExecutor model_executor_; }; }}} // namespace triton::backend::bls kytensor-2.49.0.7/backend/examples/backends/bls/src/bls_utils.cc000066400000000000000000000157251527052371200245040ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "bls_utils.h" namespace triton { namespace backend { namespace bls { TRITONSERVER_Error* CPUAllocator( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { // For simplicity, this backend example always uses CPU memory regardless of // the preferred memory type. You can make the actual memory type and id that // we allocate be the same as preferred memory type. You can also provide a // customized allocator to support different preferred_memory_type, and reuse // memory buffer when possible. *actual_memory_type = TRITONSERVER_MEMORY_CPU; *actual_memory_type_id = preferred_memory_type_id; // If 'byte_size' is zero just return 'buffer' == nullptr, we don't // need to do any other book-keeping. if (byte_size == 0) { *buffer = nullptr; *buffer_userp = nullptr; LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, ("allocated " + std::to_string(byte_size) + " bytes for result tensor " + tensor_name) .c_str()); } else { void* allocated_ptr = nullptr; *actual_memory_type = TRITONSERVER_MEMORY_CPU; allocated_ptr = malloc(byte_size); // Pass the tensor name with buffer_userp so we can show it when // releasing the buffer. if (allocated_ptr != nullptr) { *buffer = allocated_ptr; *buffer_userp = new std::string(tensor_name); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, ("allocated " + std::to_string(byte_size) + " bytes in " + TRITONSERVER_MemoryTypeString(*actual_memory_type) + " for result tensor " + tensor_name) .c_str()); } } return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { std::string* name = nullptr; if (buffer_userp != nullptr) { name = reinterpret_cast(buffer_userp); } else { name = new std::string(""); } std::stringstream ss; ss << buffer; std::string buffer_str = ss.str(); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, ("Releasing buffer " + buffer_str + " of size " + std::to_string(byte_size) + " in " + TRITONSERVER_MemoryTypeString(memory_type) + " for result '" + *name) .c_str()); switch (memory_type) { case TRITONSERVER_MEMORY_CPU: free(buffer); break; default: LOG_MESSAGE( TRITONSERVER_LOG_ERROR, std::string( "error: unexpected buffer allocated in CUDA managed memory") .c_str()); break; } delete name; return nullptr; // Success } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { if (request != nullptr) { LOG_IF_ERROR( TRITONSERVER_InferenceRequestDelete(request), "Failed to delete inference request."); } } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { // The following logic only works for non-decoupled models as for decoupled // models it may send multiple responses for a request or not send any // responses for a request. Need to modify this function if the model is using // decoupled API. if (response != nullptr) { // Send 'response' to the future. std::promise* p = reinterpret_cast*>(userp); p->set_value(response); delete p; } } ModelExecutor::ModelExecutor(TRITONSERVER_Server* server) : server_(server) { // When triton needs a buffer to hold an output tensor, it will ask // us to provide the buffer. In this way we can have any buffer // management and sharing strategy that we want. To communicate to // triton the functions that we want it to call to perform the // allocations, we create a "response allocator" object. We pass // this response allocate object to triton when requesting // inference. We can reuse this response allocator object for any // number of inference requests. allocator_ = nullptr; THROW_IF_TRITON_ERROR(TRITONSERVER_ResponseAllocatorNew( &allocator_, CPUAllocator, ResponseRelease, nullptr /* start_fn */)); } TRITONSERVER_Error* ModelExecutor::AsyncExecute( TRITONSERVER_InferenceRequest* irequest, std::future* future) { // Perform inference by calling TRITONSERVER_ServerInferAsync. This // call is asynchronous and therefore returns immediately. The // completion of the inference and delivery of the response is done // by triton by calling the "response complete" callback functions // (InferResponseComplete in this case). auto p = new std::promise(); *future = p->get_future(); RETURN_IF_ERROR(TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(p))); RETURN_IF_ERROR( TRITONSERVER_ServerInferAsync(server_, irequest, nullptr /* trace */)); return nullptr; // success } }}} // namespace triton::backend::bls kytensor-2.49.0.7/backend/examples/backends/bls/src/bls_utils.h000066400000000000000000000077511527052371200243460ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "triton/backend/backend_common.h" #include "triton/core/tritonbackend.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { namespace bls { #define THROW_IF_TRITON_ERROR(X) \ do { \ TRITONSERVER_Error* tie_err__ = (X); \ if (tie_err__ != nullptr) { \ throw BLSBackendException(TRITONSERVER_ErrorMessage(tie_err__)); \ } \ } while (false) // // BLSBackendException // // Exception thrown if error occurs in BLSBackend. // struct BLSBackendException : std::exception { BLSBackendException(const std::string& message) : message_(message) {} const char* what() const throw() { return message_.c_str(); } std::string message_; }; // Performs the allocations of output tensors. TRITONSERVER_Error* CPUAllocator( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id); // Callback functions for server inference. TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp); void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); // // ModelExecutor // // Execute inference request on a model. // class ModelExecutor { public: ModelExecutor(TRITONSERVER_Server* server); // Performs async inference request. TRITONSERVER_Error* AsyncExecute( TRITONSERVER_InferenceRequest* irequest, std::future* future); private: // The server object that encapsulates all the functionality of the Triton // server and allows access to the Triton server API. TRITONSERVER_Server* server_; // The allocator object that will be used for allocating output tensors. TRITONSERVER_ResponseAllocator* allocator_; }; }}} // namespace triton::backend::bls kytensor-2.49.0.7/backend/examples/backends/bls/src/libtriton_bls.ldscript000066400000000000000000000030661527052371200266040ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; kytensor-2.49.0.7/backend/examples/backends/minimal/000077500000000000000000000000001527052371200222425ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/minimal/CMakeLists.txt000066400000000000000000000131441527052371200250050ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tutorialminimalbackend LANGUAGES C CXX) # # Options # # Must include options required for this project as well as any # projects included in this one by FetchContent. # # GPU support is disabled by default because minimal backend doesn't # use GPUs. # option(TRITON_ENABLE_GPU "Enable GPU support in backend" OFF) option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON) set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") # # Setting C++ min standard # set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard whose features are requested to build this target.") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # # FetchContent requires us to include the transitive closure of all # repos that we depend on so that we can override the tags. # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-backend SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/backend ) FetchContent_MakeAvailable(repo-common repo-core repo-backend) # # The backend must be built into a shared library. Use an ldscript to # hide all symbols except for the TRITONBACKEND API. # configure_file(src/libtriton_minimal.ldscript libtriton_minimal.ldscript COPYONLY) add_library( triton-minimal-backend SHARED src/minimal.cc ) add_library( TutorialMinimalBackend::triton-minimal-backend ALIAS triton-minimal-backend ) target_include_directories( triton-minimal-backend PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_compile_features(triton-minimal-backend PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-minimal-backend PRIVATE $<$,$,$>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) target_link_libraries( triton-minimal-backend PRIVATE triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core triton-backend-utils # from repo-backend ) if(WIN32) set_target_properties( triton-minimal-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_minimal ) else() set_target_properties( triton-minimal-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_minimal LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_minimal.ldscript LINK_FLAGS "-Wl,--version-script libtriton_minimal.ldscript" ) endif() # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TutorialMinimalBackend) install( TARGETS triton-minimal-backend EXPORT triton-minimal-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/minimal RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/minimal ) install( EXPORT triton-minimal-backend-targets FILE TutorialMinimalBackendTargets.cmake NAMESPACE TutorialMinimalBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TutorialMinimalBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TutorialMinimalBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TutorialMinimalBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-minimal-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TutorialMinimalBackendTargets.cmake NAMESPACE TutorialMinimalBackend:: ) export(PACKAGE TutorialMinimalBackend) kytensor-2.49.0.7/backend/examples/backends/minimal/cmake/000077500000000000000000000000001527052371200233225ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/minimal/cmake/TutorialMinimalBackendConfig.cmake.in000066400000000000000000000036771527052371200324560ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TUTORIALMINIMALBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TUTORIALMINIMALBACKEND_CMAKE_DIR}) if(NOT TARGET TutorialMinimalBackend::triton-minimal-backend) include("${TUTORIALMINIMALBACKEND_CMAKE_DIR}/TutorialMinimalBackendTargets.cmake") endif() set(TUTORIALMINIMALBACKEND_LIBRARIES TutorialMinimalBackend::triton-minimal-backend) kytensor-2.49.0.7/backend/examples/backends/minimal/src/000077500000000000000000000000001527052371200230315ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/minimal/src/libtriton_minimal.ldscript000066400000000000000000000030661527052371200303200ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; kytensor-2.49.0.7/backend/examples/backends/minimal/src/minimal.cc000066400000000000000000000413021527052371200247660ustar00rootroot00000000000000// Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/backend_common.h" #include "triton/backend/backend_input_collector.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" #include "triton/backend/backend_output_responder.h" #include "triton/core/tritonbackend.h" namespace triton { namespace backend { namespace minimal { // // Minimal backend that demonstrates the TRITONBACKEND API. This // backend works for any model that has 1 input called "IN0" with // INT32 datatype and shape [ 4 ] and 1 output called "OUT0" with // INT32 datatype and shape [ 4 ]. The backend supports both batching // and non-batching models. // // For each batch of requests, the backend returns the input tensor // value in the output tensor. // ///////////// // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. ModelState is derived from BackendModel class // provided in the backend utilities that provides many common // functions. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; private: ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model) {} }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } extern "C" { // Triton calls TRITONBACKEND_ModelInitialize when a model is loaded // to allow the backend to create any state associated with the model, // and to also examine the model configuration to determine if the // configuration is suitable for the backend. Any errors reported by // this function will prevent the model from loading. // TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { // Create a ModelState object and associate it with the // TRITONBACKEND_Model. If anything goes wrong with initialization // of the model state then an error is returned and Triton will fail // to load the model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast(model_state))); return nullptr; // success } // Triton calls TRITONBACKEND_ModelFinalize when a model is no longer // needed. The backend should cleanup any state associated with the // model. This function will not be called until all model instances // of the model have been finalized. // TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast(vstate); delete model_state; return nullptr; // success } } // extern "C" ///////////// // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each // TRITONBACKEND_ModelInstance. ModelInstanceState is derived from // BackendModelInstance class provided in the backend utilities that // provides many common functions. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState() = default; // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state) { } ModelState* model_state_; }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } extern "C" { // Triton calls TRITONBACKEND_ModelInstanceInitialize when a model // instance is created to allow the backend to initialize any state // associated with the instance. // TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { // Get the model state associated with this instance's model. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast(vmodelstate); // Create a ModelInstanceState object and associate it with the // TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast(instance_state))); return nullptr; // success } // Triton calls TRITONBACKEND_ModelInstanceFinalize when a model // instance is no longer needed. The backend should cleanup any state // associated with the model instance. // TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast(vstate); delete instance_state; return nullptr; // success } } // extern "C" ///////////// extern "C" { // When Triton calls TRITONBACKEND_ModelInstanceExecute it is required // that a backend create a response for each request in the batch. A // response may be the output tensors required for that request or may // be an error that is returned in the response. // TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Best practice for a high-performance // implementation is to avoid introducing mutex/lock and instead use // only function-local and model-instance-specific state. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast(&instance_state))); ModelState* model_state = instance_state->StateForModel(); // 'responses' is initialized as a parallel array to 'requests', // with one TRITONBACKEND_Response object for each // TRITONBACKEND_Request object. If something goes wrong while // creating these response objects, the backend simply returns an // error from TRITONBACKEND_ModelInstanceExecute, indicating to // Triton that this backend did not create or send any responses and // so it is up to Triton to create and send an appropriate error // response for each request. RETURN_IF_ERROR is one of several // useful macros for error handling that can be found in // backend_common.h. std::vector responses; responses.reserve(request_count); for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); responses.push_back(response); } // At this point, the backend takes ownership of 'requests', which // means that it is responsible for sending a response for every // request. From here, even if something goes wrong in processing, // the backend must return 'nullptr' from this function to indicate // success. Any errors and failures must be communicated via the // response objects. // // To simplify error handling, the backend utilities manage // 'responses' in a specific way and it is recommended that backends // follow this same pattern. When an error is detected in the // processing of a request, an appropriate error response is sent // and the corresponding TRITONBACKEND_Response object within // 'responses' is set to nullptr to indicate that the // request/response has already been handled and no further processing // should be performed for that request. Even if all responses fail, // the backend still allows execution to flow to the end of the // function. RESPOND_AND_SET_NULL_IF_ERROR, and // RESPOND_ALL_AND_SET_NULL_IF_ERROR are macros from // backend_common.h that assist in this management of response // objects. // The backend could iterate over the 'requests' and process each // one separately. But for performance reasons it is usually // preferred to create batched input tensors that are processed // simultaneously. This is especially true for devices like GPUs // that are capable of exploiting the large amount parallelism // exposed by larger data sets. // // The backend utilities provide a "collector" to facilitate this // batching process. The 'collector's ProcessTensor function will // combine a tensor's value from each request in the batch into a // single contiguous buffer. The buffer can be provided by the // backend or 'collector' can create and manage it. In this backend, // there is not a specific buffer into which the batch should be // created, so use ProcessTensor arguments that cause collector to // manage it. BackendInputCollector collector( requests, request_count, &responses, model_state->TritonMemoryManager(), false /* pinned_enabled */, nullptr /* stream*/); // To instruct ProcessTensor to "gather" the entire batch of IN0 // input tensors into a single contiguous buffer in CPU memory, set // the "allowed input types" to be the CPU ones (see tritonserver.h // in the triton-inference-server/core repo for allowed memory // types). std::vector> allowed_input_types = {{TRITONSERVER_MEMORY_CPU_PINNED, 0}, {TRITONSERVER_MEMORY_CPU, 0}}; const char* input_buffer; size_t input_buffer_byte_size; TRITONSERVER_MemoryType input_buffer_memory_type; int64_t input_buffer_memory_type_id; RESPOND_ALL_AND_SET_NULL_IF_ERROR( responses, request_count, collector.ProcessTensor( "IN0", nullptr /* existing_buffer */, 0 /* existing_buffer_byte_size */, allowed_input_types, &input_buffer, &input_buffer_byte_size, &input_buffer_memory_type, &input_buffer_memory_type_id)); // Finalize the collector. If 'true' is returned, 'input_buffer' // will not be valid until the backend synchronizes the CUDA // stream or event that was used when creating the collector. For // this backend, GPU is not supported and so no CUDA sync should // be needed; so if 'true' is returned simply log an error. const bool need_cuda_input_sync = collector.Finalize(); if (need_cuda_input_sync) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, "'minimal' backend: unexpected CUDA sync required by collector"); } // 'input_buffer' contains the batched "IN0" tensor. The backend can // implement whatever logic is necessary to produce "OUT0". This // backend simply returns the IN0 value in OUT0 so no actual // computation is needed. LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model ") + model_state->Name() + ": requests in batch " + std::to_string(request_count)) .c_str()); std::string tstr; IGNORE_ERROR(BufferAsTypedString( tstr, input_buffer, input_buffer_byte_size, TRITONSERVER_TYPE_INT32)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("batched IN0 value: ") + tstr).c_str()); const char* output_buffer = input_buffer; TRITONSERVER_MemoryType output_buffer_memory_type = input_buffer_memory_type; int64_t output_buffer_memory_type_id = input_buffer_memory_type_id; // This backend supports models that batch along the first dimension // and those that don't batch. For non-batch models the output shape // will be [ 4 ]. For batch models the output shape will be [ -1, 4 // ] and the backend "responder" utility below will set the // appropriate batch dimension value for each response. std::vector output_batch_shape; bool supports_first_dim_batching; RESPOND_ALL_AND_SET_NULL_IF_ERROR( responses, request_count, model_state->SupportsFirstDimBatching(&supports_first_dim_batching)); if (supports_first_dim_batching) { output_batch_shape.push_back(-1); } output_batch_shape.push_back(4); // Because the OUT0 values are concatenated into a single contiguous // 'output_buffer', the backend must "scatter" them out to the // individual response OUT0 tensors. The backend utilities provide // a "responder" to facilitate this scattering process. // The 'responders's ProcessTensor function will copy the portion of // 'output_buffer' corresponding to each request's output into the // response for that request. BackendOutputResponder responder( requests, request_count, &responses, model_state->TritonMemoryManager(), supports_first_dim_batching, false /* pinned_enabled */, nullptr /* stream*/); responder.ProcessTensor( "OUT0", TRITONSERVER_TYPE_INT32, output_batch_shape, output_buffer, output_buffer_memory_type, output_buffer_memory_type_id); // Finalize the responder. If 'true' is returned, the OUT0 // tensors' data will not be valid until the backend synchronizes // the CUDA stream or event that was used when creating the // responder. For this backend, GPU is not supported and so no // CUDA sync should be needed; so if 'true' is returned simply log // an error. const bool need_cuda_output_sync = responder.Finalize(); if (need_cuda_output_sync) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, "'minimal' backend: unexpected CUDA sync required by responder"); } // Send all the responses that haven't already been sent because of // an earlier error. for (auto& response : responses) { if (response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr), "failed to send response"); } } // Done with the request objects so release them. for (uint32_t r = 0; r < request_count; ++r) { auto& request = requests[r]; LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } return nullptr; // success } } // extern "C" }}} // namespace triton::backend::minimal kytensor-2.49.0.7/backend/examples/backends/recommended/000077500000000000000000000000001527052371200230765ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/recommended/CMakeLists.txt000066400000000000000000000133541527052371200256440ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tutorialrecommendedbackend LANGUAGES C CXX) # # Options # # Must include options required for this project as well as any # projects included in this one by FetchContent. # # GPU support is disabled by default because recommended backend # doesn't use GPUs. # option(TRITON_ENABLE_GPU "Enable GPU support in backend" OFF) option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON) set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") # # Setting C++ min standard # set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard whose features are requested to build this target.") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # # FetchContent requires us to include the transitive closure of all # repos that we depend on so that we can override the tags. # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-backend SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/backend ) FetchContent_MakeAvailable(repo-common repo-core repo-backend) # # The backend must be built into a shared library. Use an ldscript to # hide all symbols except for the TRITONBACKEND API. # configure_file(src/libtriton_recommended.ldscript libtriton_recommended.ldscript COPYONLY) add_library( triton-recommended-backend SHARED src/recommended.cc ) add_library( TutorialRecommendedBackend::triton-recommended-backend ALIAS triton-recommended-backend ) target_include_directories( triton-recommended-backend PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_compile_features(triton-recommended-backend PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-recommended-backend PRIVATE $<$,$,$>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) target_link_libraries( triton-recommended-backend PRIVATE triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core triton-backend-utils # from repo-backend ) if(WIN32) set_target_properties( triton-recommended-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_recommended ) else() set_target_properties( triton-recommended-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_recommended LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_recommended.ldscript LINK_FLAGS "-Wl,--version-script libtriton_recommended.ldscript" ) endif() # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TutorialRecommendedBackend) install( TARGETS triton-recommended-backend EXPORT triton-recommended-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/recommended RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/recommended ) install( EXPORT triton-recommended-backend-targets FILE TutorialRecommendedBackendTargets.cmake NAMESPACE TutorialRecommendedBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TutorialRecommendedBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TutorialRecommendedBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TutorialRecommendedBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-recommended-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TutorialRecommendedBackendTargets.cmake NAMESPACE TutorialRecommendedBackend:: ) export(PACKAGE TutorialRecommendedBackend) kytensor-2.49.0.7/backend/examples/backends/recommended/cmake/000077500000000000000000000000001527052371200241565ustar00rootroot00000000000000TutorialRecommendedBackendConfig.cmake.in000066400000000000000000000037431527052371200340610ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/recommended/cmake# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TUTORIALRECOMMENDEDBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TUTORIALRECOMMENDEDBACKEND_CMAKE_DIR}) if(NOT TARGET TutorialRecommendedBackend::triton-recommended-backend) include("${TUTORIALRECOMMENDEDBACKEND_CMAKE_DIR}/TutorialRecommendedBackendTargets.cmake") endif() set(TUTORIALRECOMMENDEDBACKEND_LIBRARIES TutorialRecommendedBackend::triton-recommended-backend) kytensor-2.49.0.7/backend/examples/backends/recommended/src/000077500000000000000000000000001527052371200236655ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/backends/recommended/src/libtriton_recommended.ldscript000066400000000000000000000030661527052371200320100ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; kytensor-2.49.0.7/backend/examples/backends/recommended/src/recommended.cc000066400000000000000000000677731527052371200265020ustar00rootroot00000000000000// Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/backend_common.h" #include "triton/backend/backend_input_collector.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" #include "triton/backend/backend_output_responder.h" #include "triton/core/tritonbackend.h" namespace triton { namespace backend { namespace recommended { // // Backend that demonstrates the TRITONBACKEND API. This backend works // for any model that has 1 input with any datatype and any shape and // 1 output with the same shape and datatype as the input. The backend // supports both batching and non-batching models. // // For each batch of requests, the backend returns the input tensor // value in the output tensor. // ///////////// extern "C" { // Triton calls TRITONBACKEND_Initialize when a backend is loaded into // Triton to allow the backend to create and initialize any state that // is intended to be shared across all models and model instances that // use the backend. The backend should also verify version // compatibility with Triton in this function. // TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); // Check the backend API version that Triton supports vs. what this // backend was compiled against. Make sure that the Triton major // version is the same and the minor version is >= what this backend // uses. uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Triton TRITONBACKEND API version: ") + std::to_string(api_version_major) + "." + std::to_string(api_version_minor)) .c_str()); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("'") + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "triton backend API version does not support this backend"); } // The backend configuration may contain information needed by the // backend, such as tritonserver command-line arguments. This // backend doesn't use any such configuration but for this example // print whatever is available. TRITONSERVER_Message* backend_config_message; RETURN_IF_ERROR( TRITONBACKEND_BackendConfig(backend, &backend_config_message)); const char* buffer; size_t byte_size; RETURN_IF_ERROR(TRITONSERVER_MessageSerializeToJson( backend_config_message, &buffer, &byte_size)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("backend configuration:\n") + buffer).c_str()); // This backend does not require any "global" state but as an // example create a string to demonstrate. std::string* state = new std::string("backend state"); RETURN_IF_ERROR( TRITONBACKEND_BackendSetState(backend, reinterpret_cast(state))); return nullptr; // success } // Triton calls TRITONBACKEND_Finalize when a backend is no longer // needed. // TRITONSERVER_Error* TRITONBACKEND_Finalize(TRITONBACKEND_Backend* backend) { // Delete the "global" state associated with the backend. void* vstate; RETURN_IF_ERROR(TRITONBACKEND_BackendState(backend, &vstate)); std::string* state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Finalize: state is '") + *state + "'") .c_str()); delete state; return nullptr; // success } } // extern "C" ///////////// // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. ModelState is derived from BackendModel class // provided in the backend utilities that provides many common // functions. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; // Name of the input and output tensor const std::string& InputTensorName() const { return input_name_; } const std::string& OutputTensorName() const { return output_name_; } // Datatype of the input and output tensor TRITONSERVER_DataType TensorDataType() const { return datatype_; } // Shape of the input and output tensor as given in the model // configuration file. This shape will not include the batch // dimension (if the model has one). const std::vector& TensorNonBatchShape() const { return nb_shape_; } // Shape of the input and output tensor, including the batch // dimension (if the model has one). This method cannot be called // until the model is completely loaded and initialized, including // all instances of the model. In practice, this means that backend // should only call it in TRITONBACKEND_ModelInstanceExecute. TRITONSERVER_Error* TensorShape(std::vector& shape); // Validate that this model is supported by this backend. TRITONSERVER_Error* ValidateModelConfig(); private: ModelState(TRITONBACKEND_Model* triton_model); std::string input_name_; std::string output_name_; TRITONSERVER_DataType datatype_; bool shape_initialized_; std::vector nb_shape_; std::vector shape_; }; ModelState::ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model), shape_initialized_(false) { // Validate that the model's configuration matches what is supported // by this backend. THROW_IF_BACKEND_MODEL_ERROR(ValidateModelConfig()); } TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } TRITONSERVER_Error* ModelState::TensorShape(std::vector& shape) { // This backend supports models that batch along the first dimension // and those that don't batch. For non-batch models the output shape // will be the shape from the model configuration. For batch models // the output shape will be the shape from the model configuration // prepended with [ -1 ] to represent the batch dimension. The // backend "responder" utility used below will set the appropriate // batch dimension value for each response. The shape needs to be // initialized lazily because the SupportsFirstDimBatching function // cannot be used until the model is completely loaded. if (!shape_initialized_) { bool supports_first_dim_batching; RETURN_IF_ERROR(SupportsFirstDimBatching(&supports_first_dim_batching)); if (supports_first_dim_batching) { shape_.push_back(-1); } shape_.insert(shape_.end(), nb_shape_.begin(), nb_shape_.end()); shape_initialized_ = true; } shape = shape_; return nullptr; // success } TRITONSERVER_Error* ModelState::ValidateModelConfig() { // If verbose logging is enabled, dump the model's configuration as // JSON into the console output. if (TRITONSERVER_LogIsEnabled(TRITONSERVER_LOG_VERBOSE)) { common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(ModelConfig().PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("model configuration:\n") + buffer.Contents()).c_str()); } // ModelConfig is the model configuration as a TritonJson // object. Use the TritonJson utilities to parse the JSON and // determine if the configuration is supported by this backend. common::TritonJson::Value inputs, outputs; RETURN_IF_ERROR(ModelConfig().MemberAsArray("input", &inputs)); RETURN_IF_ERROR(ModelConfig().MemberAsArray("output", &outputs)); // The model must have exactly 1 input and 1 output. RETURN_ERROR_IF_FALSE( inputs.ArraySize() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string("model configuration must have 1 input")); RETURN_ERROR_IF_FALSE( outputs.ArraySize() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string("model configuration must have 1 output")); common::TritonJson::Value input, output; RETURN_IF_ERROR(inputs.IndexAsObject(0, &input)); RETURN_IF_ERROR(outputs.IndexAsObject(0, &output)); // Record the input and output name in the model state. const char* input_name; size_t input_name_len; RETURN_IF_ERROR(input.MemberAsString("name", &input_name, &input_name_len)); input_name_ = std::string(input_name); const char* output_name; size_t output_name_len; RETURN_IF_ERROR( output.MemberAsString("name", &output_name, &output_name_len)); output_name_ = std::string(output_name); // Input and output must have same datatype std::string input_dtype, output_dtype; RETURN_IF_ERROR(input.MemberAsString("data_type", &input_dtype)); RETURN_IF_ERROR(output.MemberAsString("data_type", &output_dtype)); RETURN_ERROR_IF_FALSE( input_dtype == output_dtype, TRITONSERVER_ERROR_INVALID_ARG, std::string("expected input and output datatype to match, got ") + input_dtype + " and " + output_dtype); datatype_ = ModelConfigDataTypeToTritonServerDataType(input_dtype); // Input and output must have same shape. Reshape is not supported // on either input or output so flag an error is the model // configuration uses it. triton::common::TritonJson::Value reshape; RETURN_ERROR_IF_TRUE( input.Find("reshape", &reshape), TRITONSERVER_ERROR_UNSUPPORTED, std::string("reshape not supported for input tensor")); RETURN_ERROR_IF_TRUE( output.Find("reshape", &reshape), TRITONSERVER_ERROR_UNSUPPORTED, std::string("reshape not supported for output tensor")); std::vector input_shape, output_shape; RETURN_IF_ERROR(backend::ParseShape(input, "dims", &input_shape)); RETURN_IF_ERROR(backend::ParseShape(output, "dims", &output_shape)); RETURN_ERROR_IF_FALSE( input_shape == output_shape, TRITONSERVER_ERROR_INVALID_ARG, std::string("expected input and output shape to match, got ") + backend::ShapeToString(input_shape) + " and " + backend::ShapeToString(output_shape)); nb_shape_ = input_shape; return nullptr; // success } extern "C" { // Triton calls TRITONBACKEND_ModelInitialize when a model is loaded // to allow the backend to create any state associated with the model, // and to also examine the model configuration to determine if the // configuration is suitable for the backend. Any errors reported by // this function will prevent the model from loading. // TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { // Create a ModelState object and associate it with the // TRITONBACKEND_Model. If anything goes wrong with initialization // of the model state then an error is returned and Triton will fail // to load the model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast(model_state))); return nullptr; // success } // Triton calls TRITONBACKEND_ModelFinalize when a model is no longer // needed. The backend should cleanup any state associated with the // model. This function will not be called until all model instances // of the model have been finalized. // TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast(vstate); delete model_state; return nullptr; // success } } // extern "C" ///////////// // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each // TRITONBACKEND_ModelInstance. ModelInstanceState is derived from // BackendModelInstance class provided in the backend utilities that // provides many common functions. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState() = default; // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state) { } ModelState* model_state_; }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } extern "C" { // Triton calls TRITONBACKEND_ModelInstanceInitialize when a model // instance is created to allow the backend to initialize any state // associated with the instance. // TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { // Get the model state associated with this instance's model. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast(vmodelstate); // Create a ModelInstanceState object and associate it with the // TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast(instance_state))); return nullptr; // success } // Triton calls TRITONBACKEND_ModelInstanceFinalize when a model // instance is no longer needed. The backend should cleanup any state // associated with the model instance. // TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast(vstate); delete instance_state; return nullptr; // success } } // extern "C" ///////////// extern "C" { // When Triton calls TRITONBACKEND_ModelInstanceExecute it is required // that a backend create a response for each request in the batch. A // response may be the output tensors required for that request or may // be an error that is returned in the response. // TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Collect various timestamps during the execution of this batch or // requests. These values are reported below before returning from // the function. uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Best practice for a high-performance // implementation is to avoid introducing mutex/lock and instead use // only function-local and model-instance-specific state. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast(&instance_state))); ModelState* model_state = instance_state->StateForModel(); // 'responses' is initialized as a parallel array to 'requests', // with one TRITONBACKEND_Response object for each // TRITONBACKEND_Request object. If something goes wrong while // creating these response objects, the backend simply returns an // error from TRITONBACKEND_ModelInstanceExecute, indicating to // Triton that this backend did not create or send any responses and // so it is up to Triton to create and send an appropriate error // response for each request. RETURN_IF_ERROR is one of several // useful macros for error handling that can be found in // backend_common.h. std::vector responses; responses.reserve(request_count); for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); responses.push_back(response); } // At this point, the backend takes ownership of 'requests', which // means that it is responsible for sending a response for every // request. From here, even if something goes wrong in processing, // the backend must return 'nullptr' from this function to indicate // success. Any errors and failures must be communicated via the // response objects. // // To simplify error handling, the backend utilities manage // 'responses' in a specific way and it is recommended that backends // follow this same pattern. When an error is detected in the // processing of a request, an appropriate error response is sent // and the corresponding TRITONBACKEND_Response object within // 'responses' is set to nullptr to indicate that the // request/response has already been handled and no further processing // should be performed for that request. Even if all responses fail, // the backend still allows execution to flow to the end of the // function so that statistics are correctly reported by the calls // to TRITONBACKEND_ModelInstanceReportStatistics and // TRITONBACKEND_ModelInstanceReportBatchStatistics. // RESPOND_AND_SET_NULL_IF_ERROR, and // RESPOND_ALL_AND_SET_NULL_IF_ERROR are macros from // backend_common.h that assist in this management of response // objects. // The backend could iterate over the 'requests' and process each // one separately. But for performance reasons it is usually // preferred to create batched input tensors that are processed // simultaneously. This is especially true for devices like GPUs // that are capable of exploiting the large amount parallelism // exposed by larger data sets. // // The backend utilities provide a "collector" to facilitate this // batching process. The 'collector's ProcessTensor function will // combine a tensor's value from each request in the batch into a // single contiguous buffer. The buffer can be provided by the // backend or 'collector' can create and manage it. In this backend, // there is not a specific buffer into which the batch should be // created, so use ProcessTensor arguments that cause collector to // manage it. ProcessTensor does NOT support TRITONSERVER_TYPE_BYTES // data type. BackendInputCollector collector( requests, request_count, &responses, model_state->TritonMemoryManager(), false /* pinned_enabled */, nullptr /* stream*/); // To instruct ProcessTensor to "gather" the entire batch of input // tensors into a single contiguous buffer in CPU memory, set the // "allowed input types" to be the CPU ones (see tritonserver.h in // the triton-inference-server/core repo for allowed memory types). std::vector> allowed_input_types = {{TRITONSERVER_MEMORY_CPU_PINNED, 0}, {TRITONSERVER_MEMORY_CPU, 0}}; const char* input_buffer; size_t input_buffer_byte_size; TRITONSERVER_MemoryType input_buffer_memory_type; int64_t input_buffer_memory_type_id; RESPOND_ALL_AND_SET_NULL_IF_ERROR( responses, request_count, collector.ProcessTensor( model_state->InputTensorName().c_str(), nullptr /* existing_buffer */, 0 /* existing_buffer_byte_size */, allowed_input_types, &input_buffer, &input_buffer_byte_size, &input_buffer_memory_type, &input_buffer_memory_type_id)); // Finalize the collector. If 'true' is returned, 'input_buffer' // will not be valid until the backend synchronizes the CUDA // stream or event that was used when creating the collector. For // this backend, GPU is not supported and so no CUDA sync should // be needed; so if 'true' is returned simply log an error. const bool need_cuda_input_sync = collector.Finalize(); if (need_cuda_input_sync) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, "'recommended' backend: unexpected CUDA sync required by collector"); } // 'input_buffer' contains the batched input tensor. The backend can // implement whatever logic is necessary to produce the output // tensor. This backend simply logs the input tensor value and then // returns the input tensor value in the output tensor so no actual // computation is needed. uint64_t compute_start_ns = 0; SET_TIMESTAMP(compute_start_ns); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model ") + model_state->Name() + ": requests in batch " + std::to_string(request_count)) .c_str()); std::string tstr; IGNORE_ERROR(BufferAsTypedString( tstr, input_buffer, input_buffer_byte_size, model_state->TensorDataType())); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("batched " + model_state->InputTensorName() + " value: ") + tstr) .c_str()); const char* output_buffer = input_buffer; TRITONSERVER_MemoryType output_buffer_memory_type = input_buffer_memory_type; int64_t output_buffer_memory_type_id = input_buffer_memory_type_id; uint64_t compute_end_ns = 0; SET_TIMESTAMP(compute_end_ns); bool supports_first_dim_batching; RESPOND_ALL_AND_SET_NULL_IF_ERROR( responses, request_count, model_state->SupportsFirstDimBatching(&supports_first_dim_batching)); std::vector tensor_shape; RESPOND_ALL_AND_SET_NULL_IF_ERROR( responses, request_count, model_state->TensorShape(tensor_shape)); // Because the output tensor values are concatenated into a single // contiguous 'output_buffer', the backend must "scatter" them out // to the individual response output tensors. The backend utilities // provide a "responder" to facilitate this scattering process. // BackendOutputResponder does NOT support TRITONSERVER_TYPE_BYTES // data type. // The 'responders's ProcessTensor function will copy the portion of // 'output_buffer' corresponding to each request's output into the // response for that request. BackendOutputResponder responder( requests, request_count, &responses, model_state->TritonMemoryManager(), supports_first_dim_batching, false /* pinned_enabled */, nullptr /* stream*/); responder.ProcessTensor( model_state->OutputTensorName().c_str(), model_state->TensorDataType(), tensor_shape, output_buffer, output_buffer_memory_type, output_buffer_memory_type_id); // Finalize the responder. If 'true' is returned, the output // tensors' data will not be valid until the backend synchronizes // the CUDA stream or event that was used when creating the // responder. For this backend, GPU is not supported and so no CUDA // sync should be needed; so if 'true' is returned simply log an // error. const bool need_cuda_output_sync = responder.Finalize(); if (need_cuda_output_sync) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, "'recommended' backend: unexpected CUDA sync required by responder"); } // Send all the responses that haven't already been sent because of // an earlier error. for (auto& response : responses) { if (response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr), "failed to send response"); } } uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); #ifdef TRITON_ENABLE_STATS // For batch statistics need to know the total batch size of the // requests. This is not necessarily just the number of requests, // because if the model supports batching then any request can be a // batched request itself. size_t total_batch_size = 0; if (!supports_first_dim_batching) { total_batch_size = request_count; } else { for (uint32_t r = 0; r < request_count; ++r) { auto& request = requests[r]; TRITONBACKEND_Input* input = nullptr; LOG_IF_ERROR( TRITONBACKEND_RequestInputByIndex(request, 0 /* index */, &input), "failed getting request input"); if (input != nullptr) { const int64_t* shape = nullptr; LOG_IF_ERROR( TRITONBACKEND_InputProperties( input, nullptr, nullptr, &shape, nullptr, nullptr, nullptr), "failed getting input properties"); if (shape != nullptr) { total_batch_size += shape[0]; } } } } #else (void)exec_start_ns; (void)exec_end_ns; (void)compute_start_ns; (void)compute_end_ns; #endif // TRITON_ENABLE_STATS // Report statistics for each request, and then release the request. for (uint32_t r = 0; r < request_count; ++r) { auto& request = requests[r]; #ifdef TRITON_ENABLE_STATS LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_state->TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting request statistics"); #endif // TRITON_ENABLE_STATS LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } #ifdef TRITON_ENABLE_STATS // Report batch statistics. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( instance_state->TritonModelInstance(), total_batch_size, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting batch request statistics"); #endif // TRITON_ENABLE_STATS return nullptr; // success } } // extern "C" }}} // namespace triton::backend::recommended kytensor-2.49.0.7/backend/examples/batching_strategies/000077500000000000000000000000001527052371200230535ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/batching_strategies/single_batching/000077500000000000000000000000001527052371200261735ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/batching_strategies/single_batching/CMakeLists.txt000066400000000000000000000123001527052371200307270ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(singlebatching LANGUAGES C CXX) set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") # # Setting C++ min standard # set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard whose features are requested to build this target.") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # # FetchContent requires us to include the transitive closure of all # repos that we depend on so that we can override the tags. # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-backend SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/backend ) FetchContent_MakeAvailable(repo-common repo-core repo-backend) # # The backend must be built into a shared library. Use an ldscript to # hide all symbols except for the TRITONBACKEND_ModelBatch API. # configure_file(src/libtriton_singlebatching.ldscript libtriton_singlebatching.ldscript COPYONLY) add_library( triton-single-batching SHARED src/single_batching.cc ) target_include_directories( triton-single-batching PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_compile_features(triton-single-batching PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-single-batching PRIVATE $<$,$,$>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) target_link_libraries( triton-single-batching PRIVATE triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core triton-backend-utils # from repo-backend ) if(WIN32) set_target_properties( triton-single-batching PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_singlebatching ) else() set_target_properties( triton-single-batching PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_singlebatching LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_singlebatching.ldscript LINK_FLAGS "-Wl,--version-script libtriton_singlebatching.ldscript" ) endif() # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/SingleBatching) install( TARGETS triton-single-batching EXPORT triton-single-batching-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/batching/single_batching RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/batching/single_batching ) install( EXPORT triton-single-batching-targets FILE SingleBatchingTargets.cmake NAMESPACE triton-single-batching DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/triton-single-batching.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/triton-single-batching.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/triton-single-batching.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-single-batching-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/triton-single-batching.cmake NAMESPACE triton-single-batching ) export(PACKAGE triton-single-batching) kytensor-2.49.0.7/backend/examples/batching_strategies/single_batching/cmake/000077500000000000000000000000001527052371200272535ustar00rootroot00000000000000triton-single-batching.cmake.in000066400000000000000000000035471527052371200351670ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/batching_strategies/single_batching/cmake# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( SINGLEBATCHING_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${SINGLEBATCHING_CMAKE_DIR}) if(NOT TARGET triton-single-batching) include("${SINGLEBATCHING_CMAKE_DIR}/SingleBatchingTargets.cmake") endif() set(SINGLEBATCHING_LIBRARIES triton-single-batching) kytensor-2.49.0.7/backend/examples/batching_strategies/single_batching/src/000077500000000000000000000000001527052371200267625ustar00rootroot00000000000000libtriton_singlebatching.ldscript000066400000000000000000000031001527052371200355120ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/batching_strategies/single_batching/src# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_ModelBatch*; local: *; }; kytensor-2.49.0.7/backend/examples/batching_strategies/single_batching/src/single_batching.cc000066400000000000000000000116041527052371200324130ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include "triton/core/tritonbackend.h" namespace triton { namespace core { namespace single_batching { // // Minimal custom batching strategy that demonstrates the // TRITONBACKEND_ModelBatch API. This custom batching strategy dynamically // creates batches up to 1 request. // ///////////// extern "C" { /// Check whether a request should be added to the pending model batch. /// /// \param request The request to be added to the pending batch. /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. When the callback returns, this should reflect /// the latest batch information. /// \param should_include The pointer to be updated on whether the request /// should be included in the batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatchIncludeRequest( TRITONBACKEND_Request* request, void* userp, bool* should_include) { // Check if the batch is empty. // If so, include this request. Otherwise, do not. bool* empty = static_cast(userp); if (*empty) { *should_include = true; *empty = false; } else { *should_include = false; } return nullptr; // success } /// Callback to be invoked when Triton has begun forming a batch. /// /// \param batcher The read-only placeholder for backend to retrieve // information about the batching strategy for this model. /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatchInitialize( const TRITONBACKEND_Batcher* batcher, void** userp) { // Userp will point to a boolean indicating whether the batch is empty. *userp = new bool(true); return nullptr; // success } /// Callback to be invoked when Triton has finishing forming a batch. /// /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatchFinalize(void* userp) { delete static_cast(userp); return nullptr; // success } /// Create a new batcher for use with custom batching. This is called during /// model loading. The batcher will point to a user-defined data structure that /// holds read-only data used for custom batching. /// /// \param batcher User-defined placeholder for backend to store and /// retrieve information about the batching strategy for this model. /// return a TRITONSERVER_Error indicating success or failure. /// \param model The backend model for which Triton is forming a batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatcherInitialize( TRITONBACKEND_Batcher** batcher, TRITONBACKEND_Model* model) { return nullptr; // success } /// Free memory associated with batcher. This is called during model unloading. /// /// \param batcher User-defined placeholder for backend to store and /// retrieve information about the batching strategy for this model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatcherFinalize(TRITONBACKEND_Batcher* batcher) { return nullptr; // success } } // extern "C" }}} // namespace triton::core::single_batching kytensor-2.49.0.7/backend/examples/batching_strategies/volume_batching/000077500000000000000000000000001527052371200262215ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/batching_strategies/volume_batching/CMakeLists.txt000066400000000000000000000123001527052371200307550ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(volumebatching LANGUAGES C CXX) set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") # # Setting C++ min standard # set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard whose features are requested to build this target.") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # # FetchContent requires us to include the transitive closure of all # repos that we depend on so that we can override the tags. # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-backend SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/backend ) FetchContent_MakeAvailable(repo-common repo-core repo-backend) # # The backend must be built into a shared library. Use an ldscript to # hide all symbols except for the TRITONBACKEND_ModelBatch API. # configure_file(src/libtriton_volumebatching.ldscript libtriton_volumebatching.ldscript COPYONLY) add_library( triton-volume-batching SHARED src/volume_batching.cc ) target_include_directories( triton-volume-batching PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_compile_features(triton-volume-batching PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-volume-batching PRIVATE $<$,$,$>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) target_link_libraries( triton-volume-batching PRIVATE triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core triton-backend-utils # from repo-backend ) if(WIN32) set_target_properties( triton-volume-batching PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_volumebatching ) else() set_target_properties( triton-volume-batching PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_volumebatching LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_volumebatching.ldscript LINK_FLAGS "-Wl,--version-script libtriton_volumebatching.ldscript" ) endif() # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/volumeBatching) install( TARGETS triton-volume-batching EXPORT triton-volume-batching-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/batching/volume_batching RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/batching/volume_batching ) install( EXPORT triton-volume-batching-targets FILE VolumeBatchingTargets.cmake NAMESPACE triton-volume-batching DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/triton-volume-batching.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/triton-volume-batching.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/triton-volume-batching.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-volume-batching-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/triton-volume-batching.cmake NAMESPACE triton-volume-batching ) export(PACKAGE triton-volume-batching) kytensor-2.49.0.7/backend/examples/batching_strategies/volume_batching/cmake/000077500000000000000000000000001527052371200273015ustar00rootroot00000000000000triton-volume-batching.cmake.in000066400000000000000000000035471527052371200352430ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/batching_strategies/volume_batching/cmake# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( VOLUMEBATCHING_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${VOLUMEBATCHING_CMAKE_DIR}) if(NOT TARGET triton-volume-batching) include("${VOLUMEBATCHING_CMAKE_DIR}/VolumeBatchingTargets.cmake") endif() set(VOLUMEBATCHING_LIBRARIES triton-volume-batching) kytensor-2.49.0.7/backend/examples/batching_strategies/volume_batching/src/000077500000000000000000000000001527052371200270105ustar00rootroot00000000000000libtriton_volumebatching.ldscript000066400000000000000000000031001527052371200355660ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/batching_strategies/volume_batching/src# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_ModelBatch*; local: *; }; kytensor-2.49.0.7/backend/examples/batching_strategies/volume_batching/src/volume_batching.cc000066400000000000000000000177331527052371200325000ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include "triton/core/tritonbackend.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr #include "triton/common/triton_json.h" namespace triton { namespace core { namespace volume_batching { // // Minimal custom batching strategy that demonstrates the // TRITONBACKEND_ModelBatch API. This custom batching strategy dynamically // creates batches up to 1 request. // ///////////// extern "C" { /// Check whether a request should be added to the pending model batch. /// /// \param request The request to be added to the pending batch. /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. When the callback returns, this should reflect /// the latest batch information. /// \param should_include The pointer to be updated on whether the request /// should be included in the batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatchIncludeRequest( TRITONBACKEND_Request* request, void* userp, bool* should_include) { // Default should_include to false in case function returns error. *should_include = false; // Get current remaining batch volume. unsigned int* remaining_volume = static_cast(userp); // Get request's volume in bytes. unsigned int pending_volume = 0; uint32_t input_count; auto err = TRITONBACKEND_RequestInputCount(request, &input_count); if (err) return err; TRITONBACKEND_Input* input; size_t data_byte_size; for (size_t count = 0; count < input_count; count++) { auto err = TRITONBACKEND_RequestInputByIndex(request, count /* index */, &input); if (err) return err; err = TRITONBACKEND_InputProperties( input, nullptr, nullptr, nullptr, nullptr, &data_byte_size, nullptr); if (err) return err; pending_volume += static_cast(data_byte_size); } // Print remaining volume for debugging purposes. std::cout << "Pending volume : " << pending_volume << std::endl; std::cout << "Remaining volume : " << *remaining_volume << std::endl; // Check if there is enough remaining volume for this request. // If so, include this request. Otherwise, do not. if (pending_volume <= *remaining_volume) { *should_include = true; *remaining_volume = *remaining_volume - pending_volume; } else { *should_include = false; } return nullptr; // success } /// Callback to be invoked when Triton has begun forming a batch. /// /// \param batcher The read-only placeholder for backend to retrieve // information about the batching strategy for this model. /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatchInitialize( const TRITONBACKEND_Batcher* batcher, void** userp) { // Userp will point to an unsigned integer representing the remaining volume // in bytes for this batch. *userp = new unsigned int(*reinterpret_cast(batcher)); return nullptr; // success } /// Callback to be invoked when Triton has finishing forming a batch. /// /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatchFinalize(void* userp) { delete static_cast(userp); return nullptr; // success } /// Create a new batcher for use with custom batching. This is called during /// model loading. The batcher will point to a user-defined data structure that /// holds read-only data used for custom batching. /// /// \param batcher User-defined placeholder for backend to store and /// retrieve information about the batching strategy for this model. /// return a TRITONSERVER_Error indicating success or failure. /// \param model The backend model for which Triton is forming a batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatcherInitialize( TRITONBACKEND_Batcher** batcher, TRITONBACKEND_Model* model) { // Batcher will point to an unsigned integer representing the maximum // volume in bytes for each batch. // Read the user-specified bytes from the model config. TRITONSERVER_Message* config_message; TRITONBACKEND_ModelConfig(model, 1 /* config_version */, &config_message); const char* buffer; size_t byte_size; uint64_t max_volume_bytes = 0; std::string max_volume_bytes_str; auto err = TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size); if (err) return err; triton::common::TritonJson::Value model_config, params, volume_param; err = model_config.Parse(buffer, byte_size); TRITONSERVER_MessageDelete(config_message); if (!model_config.Find("parameters", ¶ms)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, "Unable to find parameters in model config"); } std::vector param_keys; if (!params.Find("MAX_BATCH_VOLUME_BYTES", &volume_param)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, "Unable to find MAX_BATCH_VOLUME_BYTES parameter in model config"); } err = volume_param.MemberAsString("string_value", &max_volume_bytes_str); if (err) return err; try { max_volume_bytes = static_cast(std::stoul(max_volume_bytes_str)); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("failed to convert '") + max_volume_bytes_str + "' to unsigned int64") .c_str()); } *batcher = reinterpret_cast( new unsigned int(max_volume_bytes)); return nullptr; // success } /// Free memory associated with batcher. This is called during model unloading. /// /// \param batcher User-defined placeholder for backend to store and /// retrieve information about the batching strategy for this model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TRITONBACKEND_ModelBatcherFinalize(TRITONBACKEND_Batcher* batcher) { delete reinterpret_cast(batcher); return nullptr; // success } } // extern "C" }}} // namespace triton::core::volume_batching kytensor-2.49.0.7/backend/examples/clients/000077500000000000000000000000001527052371200205035ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/clients/bls_client000077500000000000000000000071771527052371200225630ustar00rootroot00000000000000#!/usr/bin/python # Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonhttpclient as httpclient from tritonclientutils import np_to_triton_dtype if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() model_name = "bls_fp32" shape = [16] with httpclient.InferenceServerClient(url=FLAGS.url) as client: input0_data = np.random.rand(*shape).astype(np.float32) input1_data = np.random.rand(*shape).astype(np.float32) inputs = [ httpclient.InferInput( "INPUT0", input0_data.shape, np_to_triton_dtype(input0_data.dtype), ), httpclient.InferInput( "INPUT1", input1_data.shape, np_to_triton_dtype(input1_data.dtype), ), ] inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs = [ httpclient.InferRequestedOutput("OUTPUT0"), httpclient.InferRequestedOutput("OUTPUT1"), ] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() output0_data = response.as_numpy("OUTPUT0") output1_data = response.as_numpy("OUTPUT1") print( "INPUT0 ({}) + INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output0_data ) ) print( "INPUT0 ({}) - INPUT1 ({}) = OUTPUT1 ({})".format( input0_data, input1_data, output1_data ) ) if not np.allclose(input0_data + input1_data, output0_data): print("error: incorrect sum") sys.exit(1) if not np.allclose(input0_data - input1_data, output1_data): print("error: incorrect difference") sys.exit(1) print("\nPASS") sys.exit(0) kytensor-2.49.0.7/backend/examples/clients/minimal_client000077500000000000000000000103051527052371200234140ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonclient.http as httpclient if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() # For the HTTP client, need to specify large enough concurrency to # issue all the inference requests to the server in parallel. For # this example we want to be able to send 2 requests concurrently. try: concurrent_request_count = 2 triton_client = httpclient.InferenceServerClient( url=FLAGS.url, concurrency=concurrent_request_count ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) # First send a single request to the nonbatching model. print("=========") input0_data = np.array([1, 2, 3, 4], dtype=np.int32) print("Sending request to nonbatching model: IN0 = {}".format(input0_data)) inputs = [httpclient.InferInput("IN0", [4], "INT32")] inputs[0].set_data_from_numpy(input0_data) result = triton_client.infer("nonbatching", inputs) print("Response: {}".format(result.get_response())) print("OUT0 = {}".format(result.as_numpy("OUT0"))) # Send 2 requests to the batching model. Because these are sent # asynchronously and Triton's dynamic batcher is configured to # delay up to 5 seconds when forming a batch for this model, we # expect these 2 requests to be batched within Triton and sent to # the minimal backend as a single batch. print("\n=========") async_requests = [] input0_data = np.array([[10, 11, 12, 13]], dtype=np.int32) print("Sending request to batching model: IN0 = {}".format(input0_data)) inputs = [httpclient.InferInput("IN0", [1, 4], "INT32")] inputs[0].set_data_from_numpy(input0_data) async_requests.append(triton_client.async_infer("batching", inputs)) input0_data = np.array([[20, 21, 22, 23]], dtype=np.int32) print("Sending request to batching model: IN0 = {}".format(input0_data)) inputs = [httpclient.InferInput("IN0", [1, 4], "INT32")] inputs[0].set_data_from_numpy(input0_data) async_requests.append(triton_client.async_infer("batching", inputs)) for async_request in async_requests: # Get the result from the initiated asynchronous inference # request. This call will block till the server responds. result = async_request.get_result() print("Response: {}".format(result.get_response())) print("OUT0 = {}".format(result.as_numpy("OUT0"))) kytensor-2.49.0.7/backend/examples/clients/recommended_client000077500000000000000000000105251527052371200242540ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonclient.http as httpclient if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() # For the HTTP client, need to specify large enough concurrency to # issue all the inference requests to the server in parallel. For # this example we want to be able to send 2 requests concurrently. try: concurrent_request_count = 2 triton_client = httpclient.InferenceServerClient( url=FLAGS.url, concurrency=concurrent_request_count ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) # Send 2 requests to the batching model. Because these are sent # asynchronously and Triton's dynamic batcher is configured to # delay up to 5 seconds when forming a batch for this model, we # expect these 2 requests to be batched within Triton and sent to # the backend as a single batch. # # The recommended backend can handle any model with 1 input and 1 # output as long as the input and output datatype and shape are # the same. The batching model uses datatype FP32 and shape # [ 4, 4 ]. print("\n=========") async_requests = [] input0_data = np.array( [ [ [1.0, 1.1, 1.2, 1.3], [2.0, 2.1, 2.2, 2.3], [3.0, 3.1, 3.2, 3.3], [4.0, 4.1, 4.2, 4.3], ] ], dtype=np.float32, ) print("Sending request to batching model: input = {}".format(input0_data)) inputs = [httpclient.InferInput("INPUT", [1, 4, 4], "FP32")] inputs[0].set_data_from_numpy(input0_data) async_requests.append(triton_client.async_infer("batching", inputs)) input0_data = np.array( [ [ [10.0, 10.1, 10.2, 10.3], [20.0, 20.1, 20.2, 20.3], [30.0, 30.1, 30.2, 30.3], [40.0, 40.1, 40.2, 40.3], ] ], dtype=np.float32, ) print("Sending request to batching model: input = {}".format(input0_data)) inputs = [httpclient.InferInput("INPUT", [1, 4, 4], "FP32")] inputs[0].set_data_from_numpy(input0_data) async_requests.append(triton_client.async_infer("batching", inputs)) for async_request in async_requests: # Get the result from the initiated asynchronous inference # request. This call will block till the server responds. result = async_request.get_result() print("Response: {}".format(result.get_response())) print("OUTPUT = {}".format(result.as_numpy("OUTPUT"))) kytensor-2.49.0.7/backend/examples/model_repos/000077500000000000000000000000001527052371200213525ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/bls_models/000077500000000000000000000000001527052371200234755ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_python/000077500000000000000000000000001527052371200263405ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_python/1/000077500000000000000000000000001527052371200265005ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_python/1/model.py000066400000000000000000000063011527052371200301520ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import triton_python_backend_utils as pb_utils # This model calculates the sum and difference of the INPUT0 and INPUT1 and put # the results in OUTPUT0 and OUTPUT1 respectively. For more information # regarding how this model.py was written, please refer to Python Backend. class TritonPythonModel: def initialize(self, args): self.model_config = model_config = json.loads(args["model_config"]) output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") self.output0_dtype = pb_utils.triton_string_to_numpy( output0_config["data_type"] ) self.output1_dtype = pb_utils.triton_string_to_numpy( output1_config["data_type"] ) def execute(self, requests): output0_dtype = self.output0_dtype output1_dtype = self.output1_dtype responses = [] for request in requests: in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") out_0, out_1 = ( in_0.as_numpy() + in_1.as_numpy(), in_0.as_numpy() - in_1.as_numpy(), ) out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(output0_dtype)) out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(output1_dtype)) inference_response = pb_utils.InferenceResponse( output_tensors=[out_tensor_0, out_tensor_1] ) responses.append(inference_response) return responses def finalize(self): print("Cleaning up...") kytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_python/config.pbtxt000066400000000000000000000036021527052371200306710ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "addsub_python" backend: "python" max_batch_size: 0 input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 16 ] } ] input [ { name: "INPUT1" data_type: TYPE_FP32 dims: [ 16 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 16 ] } ] output [ { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 16 ] } ] kytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_tf/000077500000000000000000000000001527052371200254305ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_tf/1/000077500000000000000000000000001527052371200255705ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_tf/1/model.savedmodel/000077500000000000000000000000001527052371200310125ustar00rootroot00000000000000saved_model.pb000077500000000000000000000016771527052371200335560ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_tf/1/model.savedmodel  : Add x"T y"T z"T" Ttype: 2   . Identity input"T output"T" Ttype C Placeholder output"dtype" dtypetype" shapeshape: : Sub x"T y"T z"T" Ttype: 2  "serve*1.15.52unknown V TENSOR_INPUT0 Placeholder* dtype0* shape:* _output_shapes : V TENSOR_INPUT1 Placeholder* _output_shapes :* shape:* dtype0 M ADDAdd TENSOR_INPUT0 TENSOR_INPUT1* T0* _output_shapes : M SUBSub TENSOR_INPUT0 TENSOR_INPUT1* _output_shapes :* T0 D TENSOR_OUTPUT0IdentityADD* T0* _output_shapes : D TENSOR_OUTPUT1IdentitySUB* T0* _output_shapes :"* serving_default # INPUT0 TENSOR_INPUT0:0 # INPUT1 TENSOR_INPUT1:0% OUTPUT1 TENSOR_OUTPUT1:0% OUTPUT0 TENSOR_OUTPUT0:0tensorflow/serving/predictkytensor-2.49.0.7/backend/examples/model_repos/bls_models/addsub_tf/config.pbtxt000066400000000000000000000005641527052371200277650ustar00rootroot00000000000000name: "addsub_tf" platform: "tensorflow_savedmodel" max_batch_size: 0 input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 16 ] }, { name: "INPUT1" data_type: TYPE_FP32 dims: [ 16 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 16 ] }, { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 16 ] } ] kytensor-2.49.0.7/backend/examples/model_repos/bls_models/bls_fp32/000077500000000000000000000000001527052371200251075ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/bls_models/bls_fp32/config.pbtxt000066400000000000000000000036501527052371200274430ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "bls_fp32" backend: "bls" max_batch_size: 0 input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 16 ] } ] input [ { name: "INPUT1" data_type: TYPE_FP32 dims: [ 16 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 16 ] } ] output [ { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 16 ] } ] instance_group [ { kind: KIND_CPU } ] kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/000077500000000000000000000000001527052371200243435ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/batching/000077500000000000000000000000001527052371200261225ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/batching/1/000077500000000000000000000000001527052371200262625ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/batching/1/.gitkeep000066400000000000000000000000001527052371200277010ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/batching/config.pbtxt000066400000000000000000000004521527052371200304530ustar00rootroot00000000000000backend: "minimal" max_batch_size: 8 dynamic_batching { max_queue_delay_microseconds: 5000000 } input [ { name: "IN0" data_type: TYPE_INT32 dims: [ 4 ] } ] output [ { name: "OUT0" data_type: TYPE_INT32 dims: [ 4 ] } ] instance_group [ { kind: KIND_CPU } ] kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/nonbatching/000077500000000000000000000000001527052371200266355ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/nonbatching/1/000077500000000000000000000000001527052371200267755ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/nonbatching/1/.gitkeep000066400000000000000000000000001527052371200304140ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/minimal_models/nonbatching/config.pbtxt000066400000000000000000000003551527052371200311700ustar00rootroot00000000000000backend: "minimal" max_batch_size: 0 input [ { name: "IN0" data_type: TYPE_INT32 dims: [ 4 ] } ] output [ { name: "OUT0" data_type: TYPE_INT32 dims: [ 4 ] } ] instance_group [ { kind: KIND_CPU } ] kytensor-2.49.0.7/backend/examples/model_repos/recommended_models/000077500000000000000000000000001527052371200251775ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/recommended_models/batching/000077500000000000000000000000001527052371200267565ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/recommended_models/batching/1/000077500000000000000000000000001527052371200271165ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/recommended_models/batching/1/.gitkeep000066400000000000000000000000001527052371200305350ustar00rootroot00000000000000kytensor-2.49.0.7/backend/examples/model_repos/recommended_models/batching/config.pbtxt000066400000000000000000000004661527052371200313140ustar00rootroot00000000000000backend: "recommended" max_batch_size: 8 dynamic_batching { max_queue_delay_microseconds: 5000000 } input [ { name: "INPUT" data_type: TYPE_FP32 dims: [ 4, 4 ] } ] output [ { name: "OUTPUT" data_type: TYPE_FP32 dims: [ 4, 4 ] } ] instance_group [ { kind: KIND_CPU } ] kytensor-2.49.0.7/backend/include/000077500000000000000000000000001527052371200166475ustar00rootroot00000000000000kytensor-2.49.0.7/backend/include/triton/000077500000000000000000000000001527052371200201665ustar00rootroot00000000000000kytensor-2.49.0.7/backend/include/triton/backend/000077500000000000000000000000001527052371200215555ustar00rootroot00000000000000kytensor-2.49.0.7/backend/include/triton/backend/backend_common.h000066400000000000000000000757611527052371200247050ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include #include "triton/common/error.h" #include "triton/core/tritonbackend.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr #include "triton/common/triton_json.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace backend { #define IGNORE_ERROR(X) \ do { \ TRITONSERVER_Error* ie_err__ = (X); \ if (ie_err__ != nullptr) { \ TRITONSERVER_ErrorDelete(ie_err__); \ } \ } while (false) #define LOG_IF_ERROR(X, MSG) \ do { \ TRITONSERVER_Error* lie_err__ = (X); \ if (lie_err__ != nullptr) { \ IGNORE_ERROR(TRITONSERVER_LogMessage( \ TRITONSERVER_LOG_ERROR, __FILE__, __LINE__, \ (std::string(MSG) + ": " + TRITONSERVER_ErrorCodeString(lie_err__) + \ " - " + TRITONSERVER_ErrorMessage(lie_err__)) \ .c_str())); \ TRITONSERVER_ErrorDelete(lie_err__); \ } \ } while (false) #define LOG_MESSAGE(LEVEL, MSG) \ do { \ LOG_IF_ERROR( \ TRITONSERVER_LogMessage(LEVEL, __FILE__, __LINE__, MSG), \ ("failed to log message: ")); \ } while (false) #define RETURN_ERROR_IF_FALSE(P, C, MSG) \ do { \ if (!(P)) { \ return TRITONSERVER_ErrorNew(C, (MSG).c_str()); \ } \ } while (false) #define RETURN_ERROR_IF_TRUE(P, C, MSG) \ do { \ if ((P)) { \ return TRITONSERVER_ErrorNew(C, (MSG).c_str()); \ } \ } while (false) #define RETURN_IF_ERROR(X) \ do { \ TRITONSERVER_Error* rie_err__ = (X); \ if (rie_err__ != nullptr) { \ return rie_err__; \ } \ } while (false) #ifdef TRITON_ENABLE_GPU #define LOG_IF_CUDA_ERROR(X, MSG) \ do { \ cudaError_t lice_err__ = (X); \ if (lice_err__ != cudaSuccess) { \ IGNORE_ERROR(TRITONSERVER_LogMessage( \ TRITONSERVER_LOG_INFO, __FILE__, __LINE__, \ (std::string(MSG) + ": " + cudaGetErrorString(lice_err__)) \ .c_str())); \ } \ } while (false) #define RETURN_IF_CUDA_ERROR(X, C, MSG) \ do { \ cudaError_t rice_err__ = (X); \ if (rice_err__ != cudaSuccess) { \ return TRITONSERVER_ErrorNew( \ C, ((MSG) + ": " + cudaGetErrorString(rice_err__)).c_str()); \ } \ } while (false) #endif // TRITON_ENABLE_GPU #define RESPOND_AND_SET_NULL_IF_ERROR(RESPONSE_PTR, X) \ do { \ TRITONSERVER_Error* rarie_err__ = (X); \ if (rarie_err__ != nullptr) { \ if (*RESPONSE_PTR != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ *RESPONSE_PTR, TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ rarie_err__), \ "failed to send error response"); \ *RESPONSE_PTR = nullptr; \ } \ TRITONSERVER_ErrorDelete(rarie_err__); \ } \ } while (false) #define RESPOND_ALL_AND_SET_NULL_IF_ERROR(RESPONSES, RESPONSES_COUNT, X) \ do { \ TRITONSERVER_Error* raasnie_err__ = (X); \ if (raasnie_err__ != nullptr) { \ for (size_t ridx = 0; ridx < RESPONSES_COUNT; ++ridx) { \ if (RESPONSES[ridx] != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ RESPONSES[ridx], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ raasnie_err__), \ "failed to send error response"); \ RESPONSES[ridx] = nullptr; \ } \ } \ TRITONSERVER_ErrorDelete(raasnie_err__); \ } \ } while (false) #define RESPOND_ALL_AND_SET_TRUE_IF_ERROR(RESPONSES, RESPONSES_COUNT, BOOL, X) \ do { \ TRITONSERVER_Error* raasnie_err__ = (X); \ if (raasnie_err__ != nullptr) { \ BOOL = true; \ for (size_t ridx = 0; ridx < RESPONSES_COUNT; ++ridx) { \ if (RESPONSES[ridx] != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ RESPONSES[ridx], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ raasnie_err__), \ "failed to send error response"); \ RESPONSES[ridx] = nullptr; \ } \ } \ TRITONSERVER_ErrorDelete(raasnie_err__); \ } \ } while (false) #ifdef TRITON_ENABLE_STATS #define TIMESPEC_TO_NANOS(TS) ((TS).tv_sec * 1000000000 + (TS).tv_nsec) #define SET_TIMESTAMP(TS_NS) \ { \ TS_NS = std::chrono::duration_cast( \ std::chrono::steady_clock::now().time_since_epoch()) \ .count(); \ } #define DECL_TIMESTAMP(TS_NS) \ uint64_t TS_NS; \ SET_TIMESTAMP(TS_NS); #else #define DECL_TIMESTAMP(TS_NS) #define SET_TIMESTAMP(TS_NS) #endif // TRITON_ENABLE_STATS #ifndef TRITON_ENABLE_GPU using cudaStream_t = void*; #endif // !TRITON_ENABLE_GPU /// Convenience deleter for TRITONBACKEND_ResponseFactory. struct ResponseFactoryDeleter { void operator()(TRITONBACKEND_ResponseFactory* f) { LOG_IF_ERROR( TRITONBACKEND_ResponseFactoryDelete(f), "failed deleting response factory"); } }; // A representation of the BatchInput message in model config class BatchInput { public: enum class Kind { BATCH_ELEMENT_COUNT, BATCH_ACCUMULATED_ELEMENT_COUNT, BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO, BATCH_MAX_ELEMENT_COUNT_AS_SHAPE, BATCH_ITEM_SHAPE, BATCH_ITEM_SHAPE_FLATTEN }; static TRITONSERVER_Error* ParseFromModelConfig( triton::common::TritonJson::Value& config, std::vector* batch_inputs); const std::vector& TargetNames() const { return target_names_; } TRITONSERVER_DataType DataType() const { return data_type_; } Kind BatchInputKind() const { return kind_; } std::string BatchInputKindString() const { return kind_str_; } const std::vector& SourceInputs() const { return source_inputs_; } private: TRITONSERVER_Error* Init(triton::common::TritonJson::Value& bi_config); Kind kind_; std::string kind_str_; std::vector target_names_; TRITONSERVER_DataType data_type_; std::vector source_inputs_; }; // A representation of the BatchOutput message in model config class BatchOutput { public: enum class Kind { BATCH_SCATTER_WITH_INPUT_SHAPE }; static TRITONSERVER_Error* ParseFromModelConfig( triton::common::TritonJson::Value& config, std::vector* batch_outputs); const std::vector& TargetNames() const { return target_names_; } TRITONSERVER_DataType DataType() const { return data_type_; } const std::vector& OutputShape() const { return shape_; } Kind BatchOutputKind() const { return kind_; } const std::vector& SourceInputs() const { return source_inputs_; } private: Kind kind_; std::vector target_names_; TRITONSERVER_DataType data_type_; std::vector shape_; std::vector source_inputs_; }; struct CopyParams { CopyParams(void* dst, const void* src, const size_t byte_size) : dst_(dst), src_(src), byte_size_(byte_size) { } void* dst_; const void* src_; const size_t byte_size_; }; /// The value for a dimension in a shape that indicates that that /// dimension can take on any size. constexpr int WILDCARD_DIM = -1; constexpr char kTensorRTExecutionAccelerator[] = "tensorrt"; constexpr char kOpenVINOExecutionAccelerator[] = "openvino"; constexpr char kCUDAExecutionAccelerator[] = "cuda"; constexpr char kGPUIOExecutionAccelerator[] = "gpu_io"; constexpr char kAutoMixedPrecisionExecutionAccelerator[] = "auto_mixed_precision"; TRITONSERVER_MemoryType GetUsePinnedMemoryType( TRITONSERVER_MemoryType ref_buffer_type); TRITONSERVER_Error* CommonErrorToTritonError(triton::common::Error error); TRITONSERVER_Error_Code StatusCodeToTritonCode( triton::common::Error::Code error_code); /// Parse an array in a JSON object into the corresponding shape. The /// array must be composed of integers. /// /// \param io The JSON object containing the member array. /// \param name The name of the array member in the JSON object. /// \param shape Returns the shape. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ParseShape( common::TritonJson::Value& io, const std::string& name, std::vector* shape); /// Return the string representation of a shape. /// /// \param dims The shape dimensions. /// \param dims_count The number of dimensions. /// \return The string representation. std::string ShapeToString(const int64_t* dims, const size_t dims_count); /// Return the string representation of a shape. /// /// \param shape The shape as a vector of dimensions. /// \return The string representation. std::string ShapeToString(const std::vector& shape); /// Return the number of elements of a shape. /// /// \param dims The shape dimensions. /// \param dims_count The number of dimensions. /// \return The number of elements. int64_t GetElementCount(const int64_t* dims, const size_t dims_count); /// Return the number of elements of a shape. /// /// \param shape The shape as a vector of dimensions. /// \return The number of elements. int64_t GetElementCount(const std::vector& shape); /// Get the size, in bytes, of a tensor based on datatype and /// shape. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize( const TRITONSERVER_DataType& dtype, const std::vector& dims); /// Get an input tensor's contents into a buffer. This overload expects /// both 'buffer' and buffers of the input to be in CPU. /// /// \param request The inference request. /// \param input_name The name of the input buffer. /// \param buffer The buffer where the input tensor content is copied into. /// \param buffer_byte_size Acts as both input and output. On input /// gives the size of 'buffer', in bytes. The function will fail if /// the buffer is not large enough to hold the input tensor /// contents. Returns the size of the input tensor data returned in /// 'buffer'. /// \param host_policy_name The host policy name to look up the input buffer. /// Default input buffer will be used if nullptr is provided. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ReadInputTensor( TRITONBACKEND_Request* request, const std::string& input_name, char* buffer, size_t* buffer_byte_size, const char* host_policy_name = nullptr); /// Get an input tensor's contents into a buffer. This overload of /// 'ReadInputTensor' supports input buffers that can be in any memory. /// /// \param request The inference request. /// \param input_name The name of the input buffer. /// \param buffer The buffer where the input tensor content is copied into. /// \param buffer_byte_size Acts as both input and output. On input /// gives the size of 'buffer', in bytes. The function will fail if /// the buffer is not large enough to hold the input tensor /// contents. Returns the size of the input tensor data returned in /// 'buffer'. /// \param host_policy_name The host policy name to look up the input buffer. /// Default input buffer will be used if nullptr is provided. /// \param memory_type The memory type of the buffer provided. /// \param memory_type_id The memory type id of the buffer provided. /// \param cuda_stream specifies the stream to be associated with, and 0 can be /// passed for default stream. /// \param cuda_used returns whether a CUDA memory copy is initiated. If true, /// the caller should synchronize on the given 'cuda_stream' to ensure data copy /// is completed. /// \param copy_on_stream whether the memory copies should be performed in cuda /// host functions on the 'cuda_stream'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ReadInputTensor( TRITONBACKEND_Request* request, const std::string& input_name, char* buffer, size_t* buffer_byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, cudaStream_t cuda_stream, bool* cuda_used, const char* host_policy_name = nullptr, const bool copy_on_stream = false); /// Validate that an input matches one of the allowed input names. /// \param io The model input. /// \param allowed The set of allowed input names. /// \return The error status. A non-OK status indicates the input /// is not valid. TRITONSERVER_Error* CheckAllowedModelInput( common::TritonJson::Value& io, const std::set& allowed); /// Validate that an output matches one of the allowed output names. /// \param io The model output. /// \param allowed The set of allowed output names. /// \return The error status. A non-OK status indicates the output /// is not valid. TRITONSERVER_Error* CheckAllowedModelOutput( common::TritonJson::Value& io, const std::set& allowed); /// Get the tensor name, false value, and true value for a boolean /// sequence batcher control kind. If 'required' is true then must /// find a tensor for the control. If 'required' is false, return /// 'tensor_name' as empty-string if the control is not mapped to any /// tensor. /// /// \param batcher The JSON object of the sequence batcher. /// \param model_name The name of the model. /// \param control_kind The kind of control tensor to look for. /// \param required Whether the tensor must be specified. /// \param tensor_name Returns the name of the tensor. /// \param tensor_datatype Returns the data type of the tensor. /// \param fp32_false_value Returns the float value for false if /// the tensor type is FP32. /// \param fp32_true_value Returns the float value for true if /// the tensor type is FP32. /// \param int32_false_value Returns the int value for false if /// the tensor type is INT32. /// \param int32_true_value Returns the int value for true if /// the tensor type is INT32. /// \param bool_false_value Returns the bool value for false if /// the tensor type is BOOL. /// \param bool_true_value Returns the bool value for true if /// the tensor type is BOOL. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* GetBooleanSequenceControlProperties( common::TritonJson::Value& batcher, const std::string& model_name, const std::string& control_kind, const bool required, std::string* tensor_name, std::string* tensor_datatype, float* fp32_false_value, float* fp32_true_value, int32_t* int32_false_value, int32_t* int32_true_value, bool* bool_false_value, bool* bool_true_value); /// Get the tensor name and datatype for a non-boolean sequence /// batcher control kind. If 'required' is true then must find a /// tensor for the control. If 'required' is false, return /// 'tensor_name' as empty-string if the control is not mapped to any /// tensor. 'tensor_datatype' returns the required datatype for the /// control. /// /// \param batcher The JSON object of the sequence batcher. /// \param model_name The name of the model. /// \param control_kind The kind of control tensor to look for. /// \param required Whether the tensor must be specified. /// \param tensor_name Returns the name of the tensor. /// \param tensor_datatype Returns the data type of the tensor. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* GetTypedSequenceControlProperties( common::TritonJson::Value& batcher, const std::string& model_name, const std::string& control_kind, const bool required, std::string* tensor_name, std::string* tensor_datatype); /// Create and send an error response for a set of requests. This /// function takes ownership of 'response_err' and so the caller must /// not access or delete it after this call returns. /// /// \param requests The requests. /// \param request_count The number of 'requests'. /// \param response_err The error to send to each request. /// \param release_request If true, the requests will be released after /// sending the error responses and the request pointers are set to /// nullptr. void RequestsRespondWithError( TRITONBACKEND_Request** requests, const uint32_t request_count, TRITONSERVER_Error* response_err, const bool release_request = true); /// Send an error response for a set of responses. This function takes /// ownership of 'response_err' and so the caller must not access or /// delete it after this call returns. /// /// \param responses The responses. /// \param response_count The number of 'responses'. /// \param response_err The error to send. void SendErrorForResponses( std::vector* responses, const uint32_t response_count, TRITONSERVER_Error* response_err); /// Copy buffer from 'src' to 'dst' for given 'byte_size'. The buffer location /// is identified by the memory type and id, and the corresponding copy will be /// initiated. /// \param msg The message to be prepended in error message. /// \param src_memory_type The memory type of the source buffer. /// \param src_memory_type_id The memory type id of the source buffer. /// \param dst_memory_type The memory type of the destination buffer. /// \param dst_memory_type_id The memory type id of the destination buffer. /// \param byte_size The byte size of the source buffer. /// \param src The pointer to the source buffer. /// \param dst The pointer to the destination buffer. /// \param cuda_stream specifies the stream to be associated with, and 0 can be /// passed for default stream. /// \param cuda_used returns whether a CUDA memory copy is initiated. If true, /// the caller should synchronize on the given 'cuda_stream' to ensure data copy /// is completed. /// \param copy_on_stream whether the memory copies should be performed in cuda /// host functions on the 'cuda_stream'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* CopyBuffer( const std::string& msg, const TRITONSERVER_MemoryType src_memory_type, const int64_t src_memory_type_id, const TRITONSERVER_MemoryType dst_memory_type, const int64_t dst_memory_type_id, const size_t byte_size, const void* src, void* dst, cudaStream_t cuda_stream, bool* cuda_used, const bool copy_on_stream = false); /// Does a file or directory exist? /// \param path The path to check for existence. /// \param exists Returns true if file/dir exists /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* FileExists(const std::string& path, bool* exists); /// Read a text file into a string. /// \param path The path of the file. /// \param contents Returns the contents of the file. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ReadTextFile( const std::string& path, std::string* contents); /// Is a path a directory? /// \param path The path to check. /// \param is_dir Returns true if path represents a directory /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* IsDirectory(const std::string& path, bool* is_dir); /// Join path segments into a longer path /// \param segments The path segments. /// \return the path formed by joining the segments. std::string JoinPath(std::initializer_list segments); /// Returns the content in the model version path and the path to the content as /// key-value pair. /// \param model_repository_path The path to the model repository. /// \param version The version of the model. /// \param ignore_directories Whether the directories will be ignored. /// \param ignore_files Whether the files will be ignored. /// \param model_paths Returns the content in the model version path and /// the path to the content. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ModelPaths( const std::string& model_repository_path, uint64_t version, const bool ignore_directories, const bool ignore_files, std::unordered_map* model_paths); /// Create a CUDA stream appropriate for GPU<->CPU data transfer /// operations for a given GPU device. The caller takes ownership of /// the stream. 'stream' returns nullptr if GPU support is disabled. /// /// \param device_id The ID of the GPU. /// \param priority The stream priority. Use 0 for normal priority. /// \param stream Returns the created stream. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* CreateCudaStream( const int device_id, const int cuda_stream_priority, cudaStream_t* stream); /// Parse the string as long long integer. /// /// \param value The string. /// \param parse_value The long long integral value of the string. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ParseLongLongValue( const std::string& value, int64_t* parsed_value); /// Parse the string as unsigned long long integer. /// /// \param value The string. /// \param parse_value The unsigned long long integral value of the string. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ParseUnsignedLongLongValue( const std::string& value, uint64_t* parsed_value); /// Parse the string as boolean. /// /// \param value The string. /// \param parse_value The boolean value of the string. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ParseBoolValue( const std::string& value, bool* parsed_value); /// Parse the string as integer. /// /// \param value The string. /// \param parse_value The integral value of the string. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ParseIntValue(const std::string& value, int* parsed_value); /// Parse the string as double. /// /// \param value The string. /// \param parse_value The double value of the string. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ParseDoubleValue( const std::string& value, double* parsed_value); /// Return the value of the specified key in a JSON object. /// /// \param params The JSON object containing the key-value mapping. /// \param key The key to look up the value in the JSON object. /// \param value Returns the value. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* GetParameterValue( triton::common::TritonJson::Value& params, const std::string& key, std::string* value); /// Return the Triton server data type of the data type string specified /// in model config JSON. /// /// \param data_type_str The string representation of the data type. /// \return the Triton server data type. TRITONSERVER_DataType ModelConfigDataTypeToTritonServerDataType( const std::string& data_type_str); /// Try to parse the requested parameter. /// /// \param params The param in model config /// \param mkey Key in the model config. /// \param value The parsed string value. /// \param default_value Default value to use when key is not found. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TryParseModelStringParameter( triton::common::TritonJson::Value& params, const std::string& mkey, std::string* value, const std::string& default_value); /// Try to parse the requested parameter. /// /// \param params The param in model config /// \param mkey Key in the model config. /// \param value The parsed int value. /// \param default_value Default value to use when key is not found. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TryParseModelStringParameter( triton::common::TritonJson::Value& params, const std::string& mkey, int* value, const int& default_value); /// Try to parse the requested parameter. /// /// \param params The param in model config /// \param mkey Key in the model config. /// \param value The parsed bool value. /// \param default_value Default value to use when key is not found. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TryParseModelStringParameter( triton::common::TritonJson::Value& params, const std::string& mkey, bool* value, const bool& default_value); /// Try to parse the requested parameter. /// /// \param params The param in model config /// \param mkey Key in the model config. /// \param value The parsed uint64 value. /// \param default_value Default value to use when key is not found. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* TryParseModelStringParameter( triton::common::TritonJson::Value& params, const std::string& mkey, uint64_t* value, const uint64_t& default_value); /// Get a string representation of a tensor buffer. /// /// \param str Returns the string. /// \param buffer The base pointer to the tensor buffer. /// \param buffer_byte_size The size of the buffer in bytes. /// \param datatype The type of the tensor /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* BufferAsTypedString( std::string& str, const char* buffer, size_t buffer_byte_size, TRITONSERVER_DataType datatype); /// Get the ID of the request as a string formatted for logging. /// /// \param request Request of which to get the ID. /// \return a formatted string for logging the request ID. std::string GetRequestId(TRITONBACKEND_Request* request); /// Validate the contiguous string buffer with correct format /// ... and parse string /// elements into list of pairs of memory address and length. /// Note the returned list of pairs points to valid memory as long /// as memory pointed by buffer remains allocated. /// /// \param buffer The pointer to the contiguous string buffer. /// \param buffer_byte_size The size of the buffer in bytes. /// \param expected_element_cnt The number of expected string elements. /// \param input_name The name of the input buffer. /// \param str_list Returns pairs of address and length of parsed strings. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* ValidateStringBuffer( const char* buffer, size_t buffer_byte_size, const size_t expected_element_cnt, const char* input_name, std::vector>* str_list); }} // namespace triton::backend kytensor-2.49.0.7/backend/include/triton/backend/backend_input_collector.h000066400000000000000000000325261527052371200266120ustar00rootroot00000000000000// Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "triton/backend/backend_common.h" #include "triton/backend/backend_memory.h" #include "triton/common/async_work_queue.h" #include "triton/common/sync_queue.h" #include "triton/core/tritonbackend.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace backend { #ifndef TRITON_ENABLE_GPU using cudaStream_t = void*; using cudaEvent_t = void*; #endif // !TRITON_ENABLE_GPU // // BackendInputCollector // class BackendInputCollector { public: // The caller can optionally provide 'event' for internal synchronization // instead of using 'stream'. If 'host_policy_name' is provided, it must be // valid for the lifetime of the collector explicit BackendInputCollector( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, TRITONBACKEND_MemoryManager* memory_manager, const bool pinned_enabled, cudaStream_t stream, cudaEvent_t event = nullptr, cudaEvent_t buffer_ready_event = nullptr, const size_t kernel_buffer_threshold = 0, const char* host_policy_name = nullptr, const bool copy_on_stream = false, const bool coalesce_request_input = false) : need_sync_(false), requests_(requests), request_count_(request_count), responses_(responses), memory_manager_(memory_manager), pinned_enabled_(pinned_enabled), use_async_cpu_copy_(triton::common::AsyncWorkQueue::WorkerCount() > 1), stream_(stream), #ifdef TRITON_ENABLE_GPU event_(event), buffer_ready_event_(buffer_ready_event), #endif // TRITON_ENABLE_GPU kernel_buffer_threshold_(kernel_buffer_threshold), pending_pinned_byte_size_(0), pending_pinned_offset_(0), pending_copy_kernel_buffer_byte_size_(0), pending_copy_kernel_buffer_offset_(0), pending_copy_kernel_input_buffer_counts_(0), async_task_count_(0), host_policy_cstr_(host_policy_name), copy_on_stream_(copy_on_stream), coalesce_request_input_(coalesce_request_input) { } ~BackendInputCollector() = default; // Process all requests for a named input tensor and return the // concatenated values of those requests in a single contiguous // buffer. This overload of the function can avoid data copy if the // tensor values are already contiguous and the caller doesn't // provide a destination 'buffer'. // // 'buffer' is used to determine whether the input should be placed at the // 'buffer' provided by the caller. If 'buffer' == nullptr, the returned // buffer will be managed by the BackendInputCollector object and // has the same lifecycle as the BackendInputCollector object. // 'buffer_byte_size' is the byte size of 'buffer' if it is not nullptr. // 'allowed_input_types' is the ordered list of the memory type and id pairs // that the returned buffer can be. It must only contain the memory type // and id of 'buffer' if 'buffer' is not nullptr. // 'dst_buffer' returns the contiguous buffer of the input tensor. // 'dst_buffer_byte_size' the byte size of 'dst_buffer'. // 'dst_memory_type' returns the memory type of 'dst_buffer'. // 'dst_memory_type_id' returns the memory type id of 'dst_buffer'. TRITONSERVER_Error* ProcessTensor( const char* input_name, char* buffer, const size_t buffer_byte_size, const std::vector>& allowed_input_types, const char** dst_buffer, size_t* dst_buffer_byte_size, TRITONSERVER_MemoryType* dst_memory_type, int64_t* dst_memory_type_id); // Process all requests for a named input tensor and return the // concatenated values of those requests in a single contiguous // 'buffer'. // // 'buffer' The buffer to hold the concatenates tensor value. Must // be large enough to hold all tensor value. // 'buffer_byte_size' is the byte size of 'buffer'. // 'dst_memory_type' The memory type of 'buffer'. // 'dst_memory_type_id' The memory type id of 'buffer'. void ProcessTensor( const char* input_name, char* buffer, const size_t buffer_byte_size, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id); // Process the batch input and return its shape. Returning error indicates // that the batch input can't be formed properly and the caller should abort // the whole batch. TRITONSERVER_Error* BatchInputShape( const BatchInput& batch_input, std::vector* shape); // Process the batch input and derive its value into 'buffer'. Returning // error indicates that the batch input can't be formed properly and // the caller should abort the whole batch. // 'buffer' is used to determine whether the input should be placed at the // 'buffer' provided by the caller. If 'buffer' == nullptr, the returned // buffer will be managed by the BackendInputCollector object and // has the same lifecycle as the BackendInputCollector object. // 'buffer_byte_size' is the byte size of 'buffer' if it is not nullptr. // 'allowed_input_types' is the ordered list of the memory type and id pairs // that the returned buffer can be. It must only contain the memory type // and id of 'buffer' if it is not nullptr. // 'dst_buffer' returns the contiguous buffer of the input tensor. // 'dst_memory_type' returns the memory type of 'dst_buffer'. // 'dst_memory_type_id' returns the memory type id of 'dst_buffer'. TRITONSERVER_Error* ProcessBatchInput( const BatchInput& batch_input, char* buffer, const size_t buffer_byte_size, const std::vector>& allowed_input_types, const char** dst_buffer, size_t* dst_buffer_byte_size, TRITONSERVER_MemoryType* dst_memory_type, int64_t* dst_memory_type_id); // Finalize processing of all requests for all input tensors. Return // true if cudaMemcpyAsync is called, and the caller should call // cudaStreamSynchronize (or cudaEventSynchronize on 'event') before // using the data. bool Finalize(); private: struct ContiguousBuffer { ContiguousBuffer() : start_request_idx_(0), end_request_idx_(0) {} MemoryDesc memory_desc_; size_t start_request_idx_; size_t end_request_idx_; }; class InputIterator { public: InputIterator( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, const char* input_name, const char* host_policy_name, const bool coalesce_request_input); // Return false if iterator reaches the end of inputs, 'input' is not set. bool GetNextContiguousInput(ContiguousBuffer* input); private: TRITONBACKEND_Request** requests_; const uint32_t request_count_; std::vector* responses_; const char* input_name_; const char* host_policy_; const bool coalesce_request_input_; TRITONBACKEND_Input* curr_input_; size_t curr_request_idx_; size_t curr_buffer_idx_; uint32_t curr_buffer_cnt_; bool reach_end_; }; // Return whether the entire input is in a contiguous buffer. If returns true, // the properties of the contiguous input buffer will also be returned. // Otherwise, only 'buffer_byte_size' will be set and return the total byte // size of the input. bool GetInputBufferIfContiguous( const char* input_name, const char** buffer, size_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); bool FlushPendingPinned( char* tensor_buffer, const size_t tensor_buffer_byte_size, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id); bool FlushPendingCopyKernel( char* tensor_buffer, const size_t tensor_buffer_byte_size, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id); TRITONSERVER_Error* LaunchCopyKernel( char* tensor_buffer, const size_t tensor_buffer_byte_size, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id); bool SetInputTensor( const char* input_name, const ContiguousBuffer& input, char* tensor_buffer, const size_t tensor_buffer_byte_size, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id, const size_t tensor_buffer_offset, const TRITONSERVER_MemoryType use_pinned_memory_type, const bool use_kernel, const bool wait_buffer); template TRITONSERVER_Error* SetElementCount( const std::string& source_input, char* buffer, const size_t buffer_byte_size); template TRITONSERVER_Error* SetAccumulatedElementCount( const std::string& source_input, char* buffer, const size_t buffer_byte_size); template TRITONSERVER_Error* SetBatchItemShape( const std::string& source_input, char* buffer, const size_t buffer_byte_size); bool need_sync_; TRITONBACKEND_Request** requests_; const uint32_t request_count_; std::vector* responses_; TRITONBACKEND_MemoryManager* memory_manager_; const bool pinned_enabled_; const bool use_async_cpu_copy_; cudaStream_t stream_; #ifdef TRITON_ENABLE_GPU cudaEvent_t event_; cudaEvent_t buffer_ready_event_; #endif // TRITON_ENABLE_GPU const size_t kernel_buffer_threshold_; size_t pending_pinned_byte_size_; size_t pending_pinned_offset_; std::list pending_pinned_input_buffers_; // managed memories that need to live over the lifetime of this // BackendInputCollector object. std::list> in_use_memories_; size_t pending_copy_kernel_buffer_byte_size_; size_t pending_copy_kernel_buffer_offset_; size_t pending_copy_kernel_input_buffer_counts_; std::list pending_copy_kernel_input_buffers_; std::vector>> input_ptr_buffer_host_; std::vector>> byte_size_buffer_host_; std::vector>> byte_size_offset_buffer_host_; // Pinned memory buffers and the corresponding request_inputs where // the final copy to the tensor is deferred until Finalize() after // waiting for all in-flight copies. struct DeferredPinned { DeferredPinned( char* pinned_memory, const size_t pinned_memory_size, char* tensor_buffer, const size_t tensor_buffer_offset, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_id, std::list&& request_buffers, std::vector* responses) : finalized_(false), pinned_memory_(pinned_memory), pinned_memory_size_(pinned_memory_size), tensor_buffer_(tensor_buffer), tensor_buffer_offset_(tensor_buffer_offset), tensor_memory_type_(tensor_memory_type), tensor_memory_id_(tensor_memory_id), requests_(std::move(request_buffers)), responses_(responses) { } bool Finalize(cudaStream_t stream); bool finalized_; // Holding reference to the pinned memory buffer, which is managed // by BackendInputCollector as 'pinned_memory' char* pinned_memory_; const size_t pinned_memory_size_; char* tensor_buffer_; const size_t tensor_buffer_offset_; const TRITONSERVER_MemoryType tensor_memory_type_; const int64_t tensor_memory_id_; std::list requests_; std::vector* responses_; }; std::list deferred_pinned_; // FIXME use future to maintain an issue-order queue to drop task count triton::common::SyncQueue completion_queue_; size_t async_task_count_; const char* host_policy_cstr_; const bool copy_on_stream_; const bool coalesce_request_input_; }; }} // namespace triton::backend kytensor-2.49.0.7/backend/include/triton/backend/backend_memory.h000066400000000000000000000127011527052371200247060ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "triton/core/tritonbackend.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { // Collection of common properties that describes a buffer in Triton struct MemoryDesc { MemoryDesc() : buffer_(nullptr), byte_size_(0), memory_type_(TRITONSERVER_MEMORY_CPU), memory_type_id_(0) { } MemoryDesc( const char* buffer, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) : buffer_(buffer), byte_size_(byte_size), memory_type_(memory_type), memory_type_id_(memory_type_id) { } const char* buffer_; size_t byte_size_; TRITONSERVER_MemoryType memory_type_; int64_t memory_type_id_; }; // // BackendMemory // // Utility class for allocating and deallocating memory using both // TRITONBACKEND_MemoryManager and direct GPU and CPU malloc/free. // class BackendMemory { public: enum class AllocationType { CPU, CPU_PINNED, GPU, CPU_PINNED_POOL, GPU_POOL }; // Allocate a contiguous block of 'alloc_type' memory. 'mem' // returns the pointer to the allocated memory. // // CPU, CPU_PINNED_POOL and GPU_POOL are allocated using // TRITONBACKEND_MemoryManagerAllocate. Note that CPU_PINNED and GPU // allocations can be much slower than the POOL variants. // // Two error codes have specific interpretations for this function: // // TRITONSERVER_ERROR_UNSUPPORTED: Indicates that function is // incapable of allocating the requested memory type and memory // type ID. Requests for the memory type and ID will always fail // no matter 'byte_size' of the request. // // TRITONSERVER_ERROR_UNAVAILABLE: Indicates that function can // allocate the memory type and ID but that currently it cannot // allocate a contiguous block of memory of the requested // 'byte_size'. static TRITONSERVER_Error* Create( TRITONBACKEND_MemoryManager* manager, const AllocationType alloc_type, const int64_t memory_type_id, const size_t byte_size, BackendMemory** mem); // Allocate a contiguous block of memory by attempting the // allocation using 'alloc_types' in order until one is successful. // See BackendMemory::Create() above for details. static TRITONSERVER_Error* Create( TRITONBACKEND_MemoryManager* manager, const std::vector& alloc_types, const int64_t memory_type_id, const size_t byte_size, BackendMemory** mem); // Creates a BackendMemory object from a pre-allocated buffer. The buffer // is not owned by the object created with this function. Hence, for // proper operation, the lifetime of the buffer should at least extend till // the corresponding BackendMemory. static TRITONSERVER_Error* Create( TRITONBACKEND_MemoryManager* manager, const AllocationType alloc_type, const int64_t memory_type_id, void* buffer, const size_t byte_size, BackendMemory** mem); ~BackendMemory(); AllocationType AllocType() const { return alloctype_; } int64_t MemoryTypeId() const { return memtype_id_; } char* MemoryPtr() { return buffer_; } size_t ByteSize() const { return byte_size_; } TRITONSERVER_MemoryType MemoryType() const { return AllocTypeToMemoryType(alloctype_); } static TRITONSERVER_MemoryType AllocTypeToMemoryType(const AllocationType a); static const char* AllocTypeString(const AllocationType a); private: BackendMemory( TRITONBACKEND_MemoryManager* manager, const AllocationType alloctype, const int64_t memtype_id, char* buffer, const size_t byte_size, const bool owns_buffer = true) : manager_(manager), alloctype_(alloctype), memtype_id_(memtype_id), buffer_(buffer), byte_size_(byte_size), owns_buffer_(owns_buffer) { } TRITONBACKEND_MemoryManager* manager_; AllocationType alloctype_; int64_t memtype_id_; char* buffer_; size_t byte_size_; bool owns_buffer_; }; }} // namespace triton::backend kytensor-2.49.0.7/backend/include/triton/backend/backend_model.h000066400000000000000000000127111527052371200244770ustar00rootroot00000000000000// Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include "triton/backend/backend_common.h" #include "triton/core/tritonbackend.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { // // BackendModel // // Common functionality for a backend model. This class is provided as // a convenience; backends are not required to use this class. // class BackendModel { public: BackendModel( TRITONBACKEND_Model* triton_model, const bool allow_optional = false); virtual ~BackendModel() = default; // Get the handle to the TRITONBACKEND server hosting this model. TRITONSERVER_Server* TritonServer() { return triton_server_; } // Get the handle to the memory manager for this model. TRITONBACKEND_MemoryManager* TritonMemoryManager() { return triton_memory_manager_; } // Get the handle to the TRITONBACKEND model. TRITONBACKEND_Model* TritonModel() { return triton_model_; } // Get the name and version of the model. const std::string& Name() const { return name_; } uint64_t Version() const { return version_; } const std::string& RepositoryPath() const { return repository_path_; } // The model configuration. common::TritonJson::Value& ModelConfig() { return model_config_; } // Sets the updated model configuration to the core. TRITONSERVER_Error* SetModelConfig(); // Parses information out of the model configuration. TRITONSERVER_Error* ParseModelConfig(); // Maximum batch size supported by the model. A value of 0 // indicates that the model does not support batching. int MaxBatchSize() const { return max_batch_size_; } // Set the max batch size for the model. When a backend // auto-completes a configuration it may set or change the maximum // batch size. void SetMaxBatchSize(const int b) { max_batch_size_ = b; } // Does this model support batching in the first dimension? TRITONSERVER_Error* SupportsFirstDimBatching(bool* supports); // Use indirect pinned memory buffer when copying an input or output // tensor to/from the model. bool EnablePinnedInput() const { return enable_pinned_input_; } bool EnablePinnedOutput() const { return enable_pinned_output_; } const std::vector& BatchInputs() const { return batch_inputs_; } const std::vector& BatchOutputs() const { return batch_outputs_; } const BatchOutput* FindBatchOutput(const std::string& output_name) const; bool IsInputRagged(const std::string& input_name) const { return (ragged_inputs_.find(input_name) != ragged_inputs_.end()); } bool IsInputOptional(const std::string& input_name) const { return (optional_inputs_.find(input_name) != optional_inputs_.end()); } protected: TRITONSERVER_Server* triton_server_; TRITONBACKEND_MemoryManager* triton_memory_manager_; TRITONBACKEND_Model* triton_model_; std::string name_; uint64_t version_; std::string repository_path_; bool allow_optional_; common::TritonJson::Value model_config_; int max_batch_size_; bool enable_pinned_input_; bool enable_pinned_output_; std::vector batch_inputs_; std::vector batch_outputs_; std::map batch_output_map_; std::set ragged_inputs_; std::set optional_inputs_; }; // // BackendModelException // // Exception thrown if error occurs while constructing an // BackendModel. // struct BackendModelException { BackendModelException(TRITONSERVER_Error* err) : err_(err) {} TRITONSERVER_Error* err_; }; #define THROW_IF_BACKEND_MODEL_ERROR(X) \ do { \ TRITONSERVER_Error* tie_err__ = (X); \ if (tie_err__ != nullptr) { \ throw triton::backend::BackendModelException(tie_err__); \ } \ } while (false) }} // namespace triton::backend kytensor-2.49.0.7/backend/include/triton/backend/backend_model_instance.h000066400000000000000000000105461527052371200263670ustar00rootroot00000000000000// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "triton/core/tritonbackend.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace backend { #ifndef TRITON_ENABLE_GPU using cudaStream_t = void*; #endif // !TRITON_ENABLE_GPU class BackendModel; // // BackendModelInstance // // Common functionality for a backend model instance. This class is // provided as a convenience; backends are not required to use this // class. // class BackendModelInstance { public: BackendModelInstance( BackendModel* backend_model, TRITONBACKEND_ModelInstance* triton_model_instance); virtual ~BackendModelInstance(); // Get the name, kind and device ID of the instance. const std::string& Name() const { return name_; } TRITONSERVER_InstanceGroupKind Kind() const { return kind_; } int32_t DeviceId() const { return device_id_; } // Get the handle to the TRITONBACKEND model instance. TRITONBACKEND_ModelInstance* TritonModelInstance() { return triton_model_instance_; } // Get the BackendModel representing the model that corresponds to // this instance. BackendModel* Model() const { return backend_model_; } // The model configuration 'default_model_filename' value, or the // value in model configuration 'cc_model_filenames' for the GPU // targeted by this instance. If neither are specified in the model // configuration, the return empty string. const std::string& ArtifactFilename() const { return artifact_filename_; } // Returns the stream associated with this instance that can be used // for GPU<->CPU memory transfers. Returns nullptr if GPU support is // disabled or if this instance is not executing on a GPU. cudaStream_t CudaStream() { return stream_; } const std::string& HostPolicyName() const { return host_policy_name_; } protected: BackendModel* backend_model_; TRITONBACKEND_ModelInstance* triton_model_instance_; std::string name_; TRITONSERVER_InstanceGroupKind kind_; int32_t device_id_; std::string artifact_filename_; cudaStream_t stream_; std::string host_policy_name_; }; // // BackendModelInstanceException // // Exception thrown if error occurs while constructing an // BackendModelInstance. // struct BackendModelInstanceException { BackendModelInstanceException(TRITONSERVER_Error* err) : err_(err) {} TRITONSERVER_Error* err_; }; #define THROW_IF_BACKEND_INSTANCE_ERROR(X) \ do { \ TRITONSERVER_Error* tie_err__ = (X); \ if (tie_err__ != nullptr) { \ throw triton::backend::BackendModelInstanceException(tie_err__); \ } \ } while (false) }} // namespace triton::backend kytensor-2.49.0.7/backend/include/triton/backend/backend_output_responder.h000066400000000000000000000176101527052371200270230ustar00rootroot00000000000000// Copyright 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include "triton/backend/backend_common.h" #include "triton/common/async_work_queue.h" #include "triton/core/tritonbackend.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace backend { #ifndef TRITON_ENABLE_GPU using cudaStream_t = void*; using cudaEvent_t = void*; #endif // !TRITON_ENABLE_GPU // // BackendOutputResponder // class BackendOutputResponder { public: // The caller can optionally provide 'event' for internal synchronization // instead of using 'stream'. explicit BackendOutputResponder( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, TRITONBACKEND_MemoryManager* memory_manager, const bool first_dim_batching, const bool pinned_enabled, cudaStream_t stream, cudaEvent_t event = nullptr, bool copy_on_stream = false) : need_sync_(false), requests_(requests), request_count_(request_count), responses_(responses), memory_manager_(memory_manager), first_dim_batching_(first_dim_batching), pinned_enabled_(pinned_enabled), use_async_cpu_copy_(triton::common::AsyncWorkQueue::WorkerCount() > 1), stream_(stream), event_(event), pending_pinned_byte_size_(0), copy_on_stream_(copy_on_stream) { } // Legacy constructor for backwards compatibility. The above // constructor should be used for all new cases. The responder needs // to know if the model is batching along the first dimension. With // this constructor we derive that information from the // max_batch_size value instead of having it provided directly as in // the above constructor. explicit BackendOutputResponder( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, const int max_batch_size, TRITONBACKEND_MemoryManager* memory_manager, const bool pinned_enabled, cudaStream_t stream, cudaEvent_t event = nullptr, bool copy_on_stream = false) : need_sync_(false), requests_(requests), request_count_(request_count), responses_(responses), memory_manager_(memory_manager), first_dim_batching_(max_batch_size >= 1), pinned_enabled_(pinned_enabled), use_async_cpu_copy_(triton::common::AsyncWorkQueue::WorkerCount() > 1), stream_(stream), event_(event), pending_pinned_byte_size_(0), copy_on_stream_(copy_on_stream) { } ~BackendOutputResponder(); // Process all responses for a named output tensor. // 'batchn_shape' may be modified by the call. void ProcessTensor( const std::string& name, const TRITONSERVER_DataType datatype, std::vector& batchn_shape, const char* buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id); // Process all responses for a named state tensor. Returns a vector of // TRITONBACKEND_State objects that the backend can use to update the state. // If TRITONBACKEND_StateUpdate is not called on the vector elements, the // state will not be updated. // 'batchn_shape' may be modified by the call. std::vector ProcessStateTensor( const std::string& name, const TRITONSERVER_DataType datatype, std::vector& batchn_shape, const char* buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id); // Process all responses for a batch output and derive its value from // 'buffer'. void ProcessBatchOutput( const std::string& name, const BatchOutput& batch_output, const char* buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id); // Finalize processing of all responses for all output // tensors. Return true if cudaMemcpyAsync is called, and the caller // should call cudaStreamSynchronize (or cudaEventSynchronize on 'event') // before using the data. bool Finalize(); private: bool FlushPendingPinned( const char* tensor_buffer, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id); bool SetFixedSizeBuffer( TRITONBACKEND_Response** response, void* response_state_or_output, const std::string& output_name, const size_t tensor_byte_size, const size_t tensor_offset, const char* tensor_buffer, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id, const TRITONSERVER_MemoryType use_pinned_memory_type, bool state); struct OutputData { OutputData( const std::string& name, void* buffer, const size_t buffer_byte_size, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id) : name_(name), buffer_(buffer), buffer_byte_size_(buffer_byte_size), memory_type_(memory_type), memory_type_id_(memory_type_id) { } const std::string name_; void* buffer_; const size_t buffer_byte_size_; const TRITONSERVER_MemoryType memory_type_; const int64_t memory_type_id_; }; bool need_sync_; TRITONBACKEND_Request** requests_; const uint32_t request_count_; std::vector* responses_; TRITONBACKEND_MemoryManager* memory_manager_; const bool first_dim_batching_; const bool pinned_enabled_; const bool use_async_cpu_copy_; cudaStream_t stream_; cudaEvent_t event_; using ResponsesList = std::list>; size_t pending_pinned_byte_size_; size_t pending_pinned_offset_; ResponsesList pending_pinned_outputs_; const bool copy_on_stream_; // Pinned memories that need to live over the lifetime of this // BackendOutputResponder object. std::list pinned_memories_; // Pinned memory buffers and the corresponding response outputs // where the final copy to the response is deferred until Finalize() // after waiting for all in-flight copies. struct DeferredPinned { DeferredPinned( char* pinned_memory, const size_t pinned_memory_size, ResponsesList&& responses) : pinned_memory_(pinned_memory), pinned_memory_size_(pinned_memory_size), responses_(std::move(responses)) { } char* pinned_memory_; const size_t pinned_memory_size_; ResponsesList responses_; }; std::list deferred_pinned_; }; }} // namespace triton::backend kytensor-2.49.0.7/backend/include/triton/backend/device_memory_tracker.h000066400000000000000000000311071527052371200262720ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include "triton/backend/backend_common.h" #include "triton/common/logging.h" #include "triton/core/tritonserver.h" #if defined(TRITON_ENABLE_GPU) && defined(TRITON_ENABLE_MEMORY_TRACKER) #include #endif static_assert( sizeof(uint64_t) >= sizeof(uintptr_t), "The implementation is storing address pointer as uint64_t, " "must ensure the space for pointer is <= sizeof(uint64_t)."); namespace triton { namespace backend { /// DeviceMemoryTracker is a backend utility provided to track the memory /// allocated for a particular model and associated model instances. /// This utility is often used for backend to report memory usage through /// TRITONBACKEND_ModelReportMemoryUsage and /// TRITONBACKEND_ModelInstanceReportMemoryUsage, which provides /// additional information to Triton for making decision on model scaling and /// deployment. /// /// Caveat: The memory tracker is implemented with CUPTI library which currently /// only supports single client/subscriber. This is an known limitation and as a /// result, the memory tracker can cause unexpected application failure if other /// component of the Triton process also uses CUPTI with a different /// configuration, for example, the framework used by the backend may have /// implemented similar profiler with CUPTI. Therefore, before enabling this /// memory tracker utilities, you should make sure that there is no other CUPTI /// client in the process. This tracker is implemented with the assumption that /// all other CUPTI clients are using the same implementation so that /// as long as all backends are compiled with this memory tracker, they may /// interact with an externally-initialized CUPTI to the backend without issues. /// /// Typical usage: /// /// On TRITONBACKEND_Initialize /// - Call DeviceMemoryTracker::Init /// /// If DeviceMemoryTracker::Init returns true, /// DeviceMemoryTracker::TrackThreadMemoryUsage and /// DeviceMemoryTracker::UntrackThreadMemoryUsage can be called accordingly to /// track memory allocation in the scope between the two calls. The memory usage /// will be recorded in MemoryUsage object and may be reported through /// TRITONBACKEND_ModelReportMemoryUsage or /// TRITONBACKEND_ModelInstanceReportMemoryUsage based on the entity of the /// memory usage. /// /// On reporting memory usage /// - Call MemoryUsage::SerializeToBufferAttributes to prepare the usage /// in the desired format. The BufferAttributes will be owned by MemoryUsage. extern "C" { typedef struct TRITONBACKEND_CuptiTracker_t { // C struct require extra implementation for dynamic array, for simplicity, // the following assumptions are made to pre-allocate the array with max // possible length: // - system / pinned memory allocation should only be on deviceId 0 // - CUDA allocation will only be on visible CUDA devices int64_t* system_memory_usage_byte_; int64_t* pinned_memory_usage_byte_; int64_t* cuda_memory_usage_byte_; uint32_t system_array_len_; uint32_t pinned_array_len_; uint32_t cuda_array_len_; // only set to false if somehow the CUPTI activity occurs on index out of // range. In that case, user should invalidate the whole tracker. bool valid_; } TRITONBACKEND_CuptiTracker; } class DeviceMemoryTracker { public: struct MemoryUsage { MemoryUsage() { cuda_memory_usage_byte_.resize(CudaDeviceCount(), 0); cupti_tracker_.system_memory_usage_byte_ = system_memory_usage_byte_.data(); cupti_tracker_.pinned_memory_usage_byte_ = pinned_memory_usage_byte_.data(); cupti_tracker_.cuda_memory_usage_byte_ = cuda_memory_usage_byte_.data(); cupti_tracker_.system_array_len_ = system_memory_usage_byte_.size(); cupti_tracker_.pinned_array_len_ = pinned_memory_usage_byte_.size(); cupti_tracker_.cuda_array_len_ = cuda_memory_usage_byte_.size(); cupti_tracker_.valid_ = true; } ~MemoryUsage() { // Make sure all C struct reference are dropped before clearing. if (tracked_) { UntrackThreadMemoryUsage(this); } for (auto& ba : buffer_attributes_) { if (ba) { LOG_IF_ERROR( TRITONSERVER_BufferAttributesDelete(ba), "Releasing buffer attributes in MemoryUsage object"); } } } // Disable copy and assign to better manage C struct lifecycle MemoryUsage(const MemoryUsage&) = delete; void operator=(const MemoryUsage&) = delete; // merge record from another MemoryUsage object MemoryUsage& operator+=(const MemoryUsage& rhs) { std::transform( rhs.system_memory_usage_byte_.begin(), rhs.system_memory_usage_byte_.end(), system_memory_usage_byte_.begin(), system_memory_usage_byte_.begin(), std::plus()); std::transform( rhs.pinned_memory_usage_byte_.begin(), rhs.pinned_memory_usage_byte_.end(), pinned_memory_usage_byte_.begin(), pinned_memory_usage_byte_.begin(), std::plus()); std::transform( rhs.cuda_memory_usage_byte_.begin(), rhs.cuda_memory_usage_byte_.end(), cuda_memory_usage_byte_.begin(), cuda_memory_usage_byte_.begin(), std::plus()); return *this; } // Serialize the MemoryUsage into an array of TRITONSERVER_BufferAttributes, // the buffer attributes object are owned by the MemoryUsage object. // Empty usage will be returned if the MemoryUsage object is invalid. TRITONSERVER_Error* SerializeToBufferAttributes( TRITONSERVER_BufferAttributes*** usage, uint32_t* usage_size) { if (!cupti_tracker_.valid_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "MemoryUsage record is invalid."); } uint32_t usage_idx = 0; // Define lambda to convert an vector of memory usage of the same type of // device into buffer attributes and set in 'usage' auto set_attributes_for_device_fn = [&](const std::vector& devices, const TRITONSERVER_MemoryType mem_type) -> TRITONSERVER_Error* { for (size_t idx = 0; idx < devices.size(); ++idx) { // skip if no allocation if (devices[idx] == 0) { continue; } // there is space in usage array if (usage_idx >= buffer_attributes_.size()) { buffer_attributes_.emplace_back(nullptr); RETURN_IF_ERROR( TRITONSERVER_BufferAttributesNew(&buffer_attributes_.back())); } auto entry = buffer_attributes_[usage_idx]; RETURN_IF_ERROR( TRITONSERVER_BufferAttributesSetMemoryType(entry, mem_type)); RETURN_IF_ERROR( TRITONSERVER_BufferAttributesSetMemoryTypeId(entry, idx)); RETURN_IF_ERROR( TRITONSERVER_BufferAttributesSetByteSize(entry, devices[idx])); ++usage_idx; } return nullptr; // success }; RETURN_IF_ERROR(set_attributes_for_device_fn( system_memory_usage_byte_, TRITONSERVER_MEMORY_CPU)); RETURN_IF_ERROR(set_attributes_for_device_fn( pinned_memory_usage_byte_, TRITONSERVER_MEMORY_CPU_PINNED)); RETURN_IF_ERROR(set_attributes_for_device_fn( cuda_memory_usage_byte_, TRITONSERVER_MEMORY_GPU)); *usage_size = usage_idx; *usage = buffer_attributes_.data(); return nullptr; } // Byte size of allocated memory tracked, // 'system_memory_usage_byte_' is likely to be empty as system memory // allocation is not controlled by CUDA driver. But keeping it for // completeness. std::vector system_memory_usage_byte_{0}; std::vector pinned_memory_usage_byte_{0}; std::vector cuda_memory_usage_byte_{0}; bool tracked_{false}; std::vector buffer_attributes_; TRITONBACKEND_CuptiTracker cupti_tracker_; }; // Simple scope guard to make sure memory usage is untracked without coupling // with MemoryUsage lifecycle struct ScopeGuard { ScopeGuard(MemoryUsage* usage) : usage_(usage) {} ~ScopeGuard() { if (usage_ && usage_->tracked_) { UntrackThreadMemoryUsage(usage_); } } MemoryUsage* usage_{nullptr}; }; #if defined(TRITON_ENABLE_GPU) && defined(TRITON_ENABLE_MEMORY_TRACKER) static bool Init(); static void Fini(); static int CudaDeviceCount(); // The memory usage will be tracked and modified until it's untracked, 'usage' // must be valid and not to be modified externally until untrack is called. // Currently can distinguish activity by correlation id which is thread // specific, which implies that there will be missing records if tracking // region switching threads to handle other activities. // This function takes no affect if 'usage' is nullptr. static void TrackThreadMemoryUsage(MemoryUsage* usage); // Note that CUPTI always pop from the top of the thread-wise stack, must be // careful on the untrack order if there is need to use multiple MemoryUsage // objects. // This function takes no affect if 'usage' is nullptr. static void UntrackThreadMemoryUsage(MemoryUsage* usage); static bool EnableFromBackendConfig( triton::common::TritonJson::Value& backend_config) { triton::common::TritonJson::Value cmdline; if (backend_config.Find("cmdline", &cmdline)) { triton::common::TritonJson::Value value; std::string value_str; if (cmdline.Find("triton-backend-memory-tracker", &value)) { bool lvalue = false; auto err = value.AsString(&value_str); if (err != nullptr) { LOG_IF_ERROR(err, "Error parsing backend config: "); return false; } err = ParseBoolValue(value_str, &lvalue); if (err != nullptr) { LOG_IF_ERROR(err, "Error parsing backend config: "); return false; } return lvalue; } } return false; } ~DeviceMemoryTracker(); static void TrackActivity(CUpti_Activity* record) { if (tracker_) { tracker_->TrackActivityInternal(record); } } private: DeviceMemoryTracker(); void TrackActivityInternal(CUpti_Activity* record); bool UpdateMemoryTypeUsage( CUpti_ActivityMemory3* memory_record, const bool is_allocation, int64_t* memory_usage, uint32_t usage_len); std::mutex mtx_; std::unordered_map activity_to_memory_usage_; CUpti_SubscriberHandle subscriber_{nullptr}; int device_cnt_{0}; static std::unique_ptr tracker_; #else // no-ops static bool Init() { return false; } static void Fini() {} static int CudaDeviceCount() { return 0; } static void TrackThreadMemoryUsage(MemoryUsage* usage) {} static void UntrackThreadMemoryUsage(MemoryUsage* usage) {} static bool EnableFromBackendConfig( const triton::common::TritonJson::Value& backend_config) { return false; } #endif // TRITON_ENABLE_GPU && TRITON_ENABLE_MEMORY_TRACKER }; }} // namespace triton::backend kytensor-2.49.0.7/backend/pyproject.toml000066400000000000000000000043021527052371200201370ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [tool.codespell] # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - # this is only to allow you to run codespell interactively skip = "./.git,./.github" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" # use the 'clear' dictionary for unambiguous spelling mistakes builtin = "clear" # disable warnings about binary files and wrong encoding quiet-level = 3 [tool.isort] profile = "black" use_parentheses = true multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 ensure_newline_before_comments = true line_length = 88 balanced_wrapping = true indent = " " skip = ["build"] kytensor-2.49.0.7/backend/src/000077500000000000000000000000001527052371200160135ustar00rootroot00000000000000kytensor-2.49.0.7/backend/src/backend_common.cc000066400000000000000000001333021527052371200212630ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/backend_common.h" #ifdef _WIN32 // suppress the min and max definitions in Windef.h. #define NOMINMAX #include // _CRT_INTERNAL_NONSTDC_NAMES 1 before including Microsoft provided C Runtime // library to expose declarations without "_" prefix to match POSIX style. #define _CRT_INTERNAL_NONSTDC_NAMES 1 #include #include #else #include #include #endif #include #include #include #include #include #include #ifdef _WIN32 // in Windows doesn't define S_ISDIR macro #if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) #define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) #endif #define F_OK 0 #endif namespace triton { namespace backend { #ifdef TRITON_ENABLE_GPU void CUDART_CB MemcpyHost(void* args) { auto* copy_params = reinterpret_cast(args); memcpy(copy_params->dst_, copy_params->src_, copy_params->byte_size_); delete copy_params; } #endif // TRITON_ENABLE_GPU TRITONSERVER_MemoryType GetUsePinnedMemoryType(TRITONSERVER_MemoryType ref_buffer_type) { // The following matrix is used for both input and output. // src \ dest | non-pinned | pinned | device // non-pinned | memcpy | memcpy | buffer needed // pinned | memcpy | memcpy | cudaMemcpy // device | buffer needed | cudaMemcpy | cudaMemcpy if (ref_buffer_type == TRITONSERVER_MEMORY_CPU_PINNED) { return TRITONSERVER_MEMORY_CPU_PINNED; } return (ref_buffer_type == TRITONSERVER_MEMORY_CPU) ? TRITONSERVER_MEMORY_GPU : TRITONSERVER_MEMORY_CPU; } TRITONSERVER_Error_Code StatusCodeToTritonCode(triton::common::Error::Code error_code) { switch (error_code) { case triton::common::Error::Code::UNKNOWN: return TRITONSERVER_ERROR_UNKNOWN; case triton::common::Error::Code::INTERNAL: return TRITONSERVER_ERROR_INTERNAL; case triton::common::Error::Code::NOT_FOUND: return TRITONSERVER_ERROR_NOT_FOUND; case triton::common::Error::Code::INVALID_ARG: return TRITONSERVER_ERROR_INVALID_ARG; case triton::common::Error::Code::UNAVAILABLE: return TRITONSERVER_ERROR_UNAVAILABLE; case triton::common::Error::Code::UNSUPPORTED: return TRITONSERVER_ERROR_UNSUPPORTED; case triton::common::Error::Code::ALREADY_EXISTS: return TRITONSERVER_ERROR_ALREADY_EXISTS; default: break; } return TRITONSERVER_ERROR_UNKNOWN; } TRITONSERVER_Error* CommonErrorToTritonError(triton::common::Error error) { return TRITONSERVER_ErrorNew( StatusCodeToTritonCode(error.ErrorCode()), error.Message().c_str()); } TRITONSERVER_Error* ParseShape( common::TritonJson::Value& io, const std::string& name, std::vector* shape) { common::TritonJson::Value shape_array; RETURN_IF_ERROR(io.MemberAsArray(name.c_str(), &shape_array)); for (size_t i = 0; i < shape_array.ArraySize(); ++i) { int64_t d = 0; RETURN_IF_ERROR(shape_array.IndexAsInt(i, &d)); shape->push_back(d); } return nullptr; // success } std::string ShapeToString(const int64_t* dims, const size_t dims_count) { bool first = true; std::string str("["); for (size_t i = 0; i < dims_count; ++i) { const int64_t dim = dims[i]; if (!first) { str += ","; } str += std::to_string(dim); first = false; } str += "]"; return str; } std::string ShapeToString(const std::vector& shape) { return ShapeToString(shape.data(), shape.size()); } int64_t GetElementCount(const int64_t* dims, const size_t dims_count) { bool first = true; int64_t cnt = 0; for (size_t i = 0; i < dims_count; i++) { if (dims[i] == WILDCARD_DIM) { return -1; } if (first) { cnt = dims[i]; first = false; } else { cnt *= dims[i]; } } return cnt; } int64_t GetElementCount(const std::vector& shape) { return GetElementCount(shape.data(), shape.size()); } int64_t GetByteSize( const TRITONSERVER_DataType& dtype, const std::vector& dims) { size_t dt_size = TRITONSERVER_DataTypeByteSize(dtype); if (dt_size == 0) { return -1; } int64_t cnt = GetElementCount(dims); if (cnt == -1) { return -1; } return cnt * dt_size; } TRITONSERVER_Error* ReadInputTensor( TRITONBACKEND_Request* request, const std::string& input_name, char* buffer, size_t* buffer_byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, cudaStream_t cuda_stream, bool* cuda_used, const char* host_policy_name, const bool copy_on_stream) { TRITONBACKEND_Input* input; RETURN_IF_ERROR( TRITONBACKEND_RequestInput(request, input_name.c_str(), &input)); uint64_t input_byte_size; uint32_t input_buffer_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( input, host_policy_name, nullptr, nullptr, nullptr, nullptr, &input_byte_size, &input_buffer_count)); RETURN_ERROR_IF_FALSE( input_byte_size <= *buffer_byte_size, TRITONSERVER_ERROR_INVALID_ARG, std::string( GetRequestId(request) + "buffer too small for input tensor '" + input_name + "', " + std::to_string(*buffer_byte_size) + " < " + std::to_string(input_byte_size))); size_t output_buffer_offset = 0; for (uint32_t b = 0; b < input_buffer_count; ++b) { const void* input_buffer = nullptr; uint64_t input_buffer_byte_size = 0; TRITONSERVER_MemoryType input_memory_type = TRITONSERVER_MEMORY_CPU; int64_t input_memory_type_id = 0; RETURN_IF_ERROR(TRITONBACKEND_InputBufferForHostPolicy( input, host_policy_name, b, &input_buffer, &input_buffer_byte_size, &input_memory_type, &input_memory_type_id)); RETURN_IF_ERROR(CopyBuffer( "Failed to copy buffer", input_memory_type, input_memory_type_id, memory_type, memory_type_id, input_buffer_byte_size, input_buffer, buffer + output_buffer_offset, cuda_stream, cuda_used, copy_on_stream)); output_buffer_offset += input_buffer_byte_size; } *buffer_byte_size = input_byte_size; return nullptr; // success } TRITONSERVER_Error* ReadInputTensor( TRITONBACKEND_Request* request, const std::string& input_name, char* buffer, size_t* buffer_byte_size, const char* host_policy_name) { bool cuda_used; return ReadInputTensor( request, input_name, buffer, buffer_byte_size, TRITONSERVER_MEMORY_CPU /* memory_type */, 0 /* memory_type_id */, 0 /* cuda_stream */, &cuda_used); } TRITONSERVER_Error* CheckAllowedModelInput( common::TritonJson::Value& io, const std::set& allowed) { std::string io_name; RETURN_IF_ERROR(io.MemberAsString("name", &io_name)); if (allowed.find(io_name) == allowed.end()) { std::string astr; for (const auto& a : allowed) { if (!astr.empty()) { astr.append(", "); } astr.append(a); } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected inference input '" + io_name + "', allowed inputs are: " + astr) .c_str()); } return nullptr; // success } TRITONSERVER_Error* CheckAllowedModelOutput( common::TritonJson::Value& io, const std::set& allowed) { std::string io_name; RETURN_IF_ERROR(io.MemberAsString("name", &io_name)); if (allowed.find(io_name) == allowed.end()) { std::string astr; for (const auto& a : allowed) { if (!astr.empty()) { astr.append(", "); } astr.append(a); } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected inference output '" + io_name + "', allowed outputs are: " + astr) .c_str()); } return nullptr; // success } TRITONSERVER_Error* GetBooleanSequenceControlProperties( common::TritonJson::Value& batcher, const std::string& model_name, const std::string& control_kind, const bool required, std::string* tensor_name, std::string* tensor_datatype, float* fp32_false_value, float* fp32_true_value, int32_t* int32_false_value, int32_t* int32_true_value, bool* bool_false_value, bool* bool_true_value) { // Make sure same tensor is not configured for multiple controls std::set seen_tensors; // Make sure the control kind is not mentioned multiple times. bool seen_control = false; common::TritonJson::Value control_inputs; if (batcher.Find("control_input", &control_inputs)) { for (size_t ci_idx = 0; ci_idx < control_inputs.ArraySize(); ci_idx++) { common::TritonJson::Value control_input; RETURN_IF_ERROR(control_inputs.IndexAsObject(ci_idx, &control_input)); std::string input_name; RETURN_IF_ERROR(control_input.MemberAsString("name", &input_name)); if (input_name.empty()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching control tensor must have a name for ") + model_name) .c_str()); } if (seen_tensors.find(input_name) != seen_tensors.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("sequence batching control tensor '") + input_name + "' is specified for multiple control kinds for " + model_name) .c_str()); } seen_tensors.insert(input_name); common::TritonJson::Value controls; if (control_input.Find("control", &controls)) { for (size_t c_idx = 0; c_idx < controls.ArraySize(); c_idx++) { common::TritonJson::Value c; RETURN_IF_ERROR(controls.IndexAsObject(c_idx, &c)); std::string kind_str; RETURN_IF_ERROR(c.MemberAsString("kind", &kind_str)); if (kind_str == control_kind) { if (seen_control) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching specifies multiple " + control_kind + " tensors for " + model_name) .c_str())); } *tensor_name = input_name; seen_control = true; common::TritonJson::Value int32_false_true, fp32_false_true, bool_false_true; bool found_int32 = (c.Find("int32_false_true", &int32_false_true) && (int32_false_true.ArraySize() > 0)); bool found_fp32 = (c.Find("fp32_false_true", &fp32_false_true) && (fp32_false_true.ArraySize() > 0)); bool found_bool = (c.Find("bool_false_true", &bool_false_true) && (bool_false_true.ArraySize() > 0)); // Make sure only one of int, float, or bool type is specified. if (!(found_int32 || found_fp32 || found_bool)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching must specify either " "'int32_false_true', 'fp32_false_true' or " "'bool_false_true' for " + control_kind + " for " + model_name)) .c_str()); } else if ( (found_fp32 && found_int32) || (found_fp32 && found_bool) || (found_int32 && found_bool)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching specifies more than one from " "'int32_false_true', 'fp32_false_true' and " "'bool_false_true' for " + control_kind + " for " + model_name)) .c_str()); } if (found_int32) { if (int32_false_true.ArraySize() != 2) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching control 'int32_false_true' must " "have " "exactly 2 entries for " + control_kind + " for " + model_name)) .c_str()); } if (tensor_datatype != nullptr) { *tensor_datatype = "TYPE_INT32"; } if (int32_false_value != nullptr) { int64_t value; RETURN_IF_ERROR(int32_false_true.IndexAsInt(0, &value)); *int32_false_value = value; } if (int32_true_value != nullptr) { int64_t value; RETURN_IF_ERROR(int32_false_true.IndexAsInt(1, &value)); *int32_true_value = value; } } else if (found_fp32) { if (fp32_false_true.ArraySize() != 2) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching control 'fp32_false_true' must " "have exactly " "2 entries for " + control_kind + " for " + model_name)) .c_str()); } if (tensor_datatype != nullptr) { *tensor_datatype = "TYPE_FP32"; } if (fp32_false_value != nullptr) { double value = 0.0; RETURN_IF_ERROR(fp32_false_true.IndexAsDouble(0, &value)); *fp32_false_value = value; } if (fp32_true_value != nullptr) { double value = 0.0; RETURN_IF_ERROR(fp32_false_true.IndexAsDouble(1, &value)); *fp32_true_value = value; } } else { if (bool_false_true.ArraySize() != 2) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching control 'bool_false_true' must " "have exactly " "2 entries for " + control_kind + " for " + model_name)) .c_str()); } if (tensor_datatype != nullptr) { *tensor_datatype = "TYPE_BOOL"; } if (bool_false_value != nullptr) { bool value; RETURN_IF_ERROR(bool_false_true.IndexAsBool(0, &value)); *bool_false_value = value; } if (bool_true_value != nullptr) { bool value; RETURN_IF_ERROR(bool_false_true.IndexAsBool(1, &value)); *bool_true_value = value; } } } } } } } if (!seen_control) { if (required) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching control tensor must specify a " + control_kind + " value for " + model_name)) .c_str()); } tensor_name->clear(); } return nullptr; // success } TRITONSERVER_Error* GetTypedSequenceControlProperties( common::TritonJson::Value& batcher, const std::string& model_name, const std::string& control_kind, const bool required, std::string* tensor_name, std::string* tensor_datatype) { // Make sure same tensor is not configured for multiple controls std::set seen_tensors; // Make sure the control kind is not mentioned multiple times. bool seen_control = false; common::TritonJson::Value control_inputs; if (batcher.Find("control_input", &control_inputs)) { for (size_t ci_idx = 0; ci_idx < control_inputs.ArraySize(); ci_idx++) { common::TritonJson::Value control_input; RETURN_IF_ERROR(control_inputs.IndexAsObject(ci_idx, &control_input)); std::string input_name; RETURN_IF_ERROR(control_input.MemberAsString("name", &input_name)); if (input_name.empty()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching control tensor must have a name for ") + model_name) .c_str()); } if (seen_tensors.find(input_name) != seen_tensors.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("sequence batching control tensor '") + input_name + "' is specified for multiple control kinds for " + model_name) .c_str()); } seen_tensors.insert(input_name); common::TritonJson::Value controls; if (control_input.Find("control", &controls)) { for (size_t c_idx = 0; c_idx < controls.ArraySize(); c_idx++) { common::TritonJson::Value c; RETURN_IF_ERROR(controls.IndexAsObject(c_idx, &c)); std::string kind_str; RETURN_IF_ERROR(c.MemberAsString("kind", &kind_str)); if (kind_str == control_kind) { if (seen_control) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching specifies multiple " + control_kind + " tensors for " + model_name) .c_str())); } *tensor_name = input_name; if (tensor_datatype != nullptr) { RETURN_IF_ERROR(c.MemberAsString("data_type", tensor_datatype)); } seen_control = true; common::TritonJson::Value int32_false_true, fp32_false_true, bool_false_true; bool found_int32 = (c.Find("int32_false_true", &int32_false_true) && (int32_false_true.ArraySize() > 0)); bool found_fp32 = (c.Find("fp32_false_true", &fp32_false_true) && (fp32_false_true.ArraySize() > 0)); bool found_bool = (c.Find("bool_false_true", &bool_false_true) && (bool_false_true.ArraySize() > 0)); if (found_fp32 || found_int32 || found_bool) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching must not specify either " "'int32_false_true', 'fp32_false_true' or " "'bool_false_true' for " + control_kind + " for " + model_name)) .c_str()); } } } } } } if (!seen_control) { if (required) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "sequence batching control tensor must specify a " + control_kind + " value for " + model_name)) .c_str()); } tensor_name->clear(); } return nullptr; // success } void RequestsRespondWithError( TRITONBACKEND_Request** requests, const uint32_t request_count, TRITONSERVER_Error* response_err, const bool release_request) { for (size_t i = 0; i < request_count; i++) { TRITONBACKEND_Response* response; auto err = TRITONBACKEND_ResponseNew(&response, requests[i]); if (err != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (GetRequestId(requests[i]) + "fail to create response").c_str()); TRITONSERVER_ErrorDelete(err); } else { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, response_err), (GetRequestId(requests[i]) + "fail to send error response").c_str()); } if (release_request) { LOG_IF_ERROR( TRITONBACKEND_RequestRelease( requests[i], TRITONSERVER_REQUEST_RELEASE_ALL), "fail to release request"); requests[i] = nullptr; } } TRITONSERVER_ErrorDelete(response_err); } void SendErrorForResponses( std::vector* responses, const uint32_t response_count, TRITONSERVER_Error* response_err) { for (size_t i = 0; i < response_count; i++) { TRITONBACKEND_Response* response = (*responses)[i]; if (response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, response_err), "fail to send error response"); (*responses)[i] = nullptr; } } TRITONSERVER_ErrorDelete(response_err); } TRITONSERVER_Error* CopyBuffer( const std::string& msg, const TRITONSERVER_MemoryType src_memory_type, const int64_t src_memory_type_id, const TRITONSERVER_MemoryType dst_memory_type, const int64_t dst_memory_type_id, const size_t byte_size, const void* src, void* dst, cudaStream_t cuda_stream, bool* cuda_used, const bool copy_on_stream) { *cuda_used = false; if (byte_size > 0) { if (src == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( msg + ": attempted a copy of " + std::to_string(byte_size) + " Bytes from an uninitialized memory") .c_str()); } if (dst == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( msg + ": attempted a copy of " + std::to_string(byte_size) + " Bytes to an uninitialized memory") .c_str()); } } // For CUDA memcpy, if copy_on_stream is false, all host to host copy will be // blocked in respect to the host, so use memcpy() directly. In this case, // need to be careful on whether the src buffer is valid. if ((src_memory_type != TRITONSERVER_MEMORY_GPU) && (dst_memory_type != TRITONSERVER_MEMORY_GPU)) { #ifdef TRITON_ENABLE_GPU if (copy_on_stream) { auto params = new CopyParams(dst, src, byte_size); cudaLaunchHostFunc( cuda_stream, MemcpyHost, reinterpret_cast(params)); *cuda_used = true; } else { memcpy(dst, src, byte_size); } #else memcpy(dst, src, byte_size); #endif // TRITON_ENABLE_GPU } else { #ifdef TRITON_ENABLE_GPU // [TODO] use cudaMemcpyDefault if UVM is supported for the device auto copy_kind = cudaMemcpyDeviceToDevice; if (src_memory_type != TRITONSERVER_MEMORY_GPU) { copy_kind = cudaMemcpyHostToDevice; } else if (dst_memory_type != TRITONSERVER_MEMORY_GPU) { copy_kind = cudaMemcpyDeviceToHost; } if ((src_memory_type_id != dst_memory_type_id) && (copy_kind == cudaMemcpyDeviceToDevice)) { RETURN_IF_CUDA_ERROR( cudaMemcpyPeerAsync( dst, dst_memory_type_id, src, src_memory_type_id, byte_size, cuda_stream), TRITONSERVER_ERROR_INTERNAL, msg + ": failed to perform CUDA copy"); } else { RETURN_IF_CUDA_ERROR( cudaMemcpyAsync(dst, src, byte_size, copy_kind, cuda_stream), TRITONSERVER_ERROR_INTERNAL, msg + ": failed to perform CUDA copy"); } *cuda_used = true; #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string(msg + ": try to use CUDA copy while GPU is not supported") .c_str()); #endif // TRITON_ENABLE_GPU } return nullptr; // success } TRITONSERVER_Error* GetDirectoryContents(const std::string& path, std::set* contents) { #ifdef _WIN32 WIN32_FIND_DATA entry; HANDLE dir = FindFirstFile(path.c_str(), &entry); if (dir == INVALID_HANDLE_VALUE) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("failed to open directory: ") + path).c_str()); } if ((entry.cFileName != ".") && (entry.cFileName != "..")) { contents->insert(entry.cFileName); } while (FindNextFileA(dir, &entry)) { if ((entry.cFileName != ".") && (entry.cFileName != "..")) { contents->insert(entry.cFileName); } } FindClose(dir); #else DIR* dir = opendir(path.c_str()); if (dir == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("failed to open directory: ") + path).c_str()); } struct dirent* entry; while ((entry = readdir(dir)) != nullptr) { std::string entryname = entry->d_name; if ((entryname != ".") && (entryname != "..")) { contents->insert(entryname); } } closedir(dir); #endif return nullptr; // success } TRITONSERVER_Error* FileExists(const std::string& path, bool* exists) { *exists = (access(path.c_str(), F_OK) == 0); return nullptr; // success } TRITONSERVER_Error* ReadTextFile(const std::string& path, std::string* contents) { std::ifstream in(path, std::ios::in | std::ios::binary); if (!in) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, ("failed to open/read file '" + path + "': " + strerror(errno)) .c_str()); } in.seekg(0, std::ios::end); contents->resize(in.tellg()); in.seekg(0, std::ios::beg); in.read(&(*contents)[0], contents->size()); in.close(); return nullptr; // success } TRITONSERVER_Error* IsDirectory(const std::string& path, bool* is_dir) { *is_dir = false; struct stat st; if (stat(path.c_str(), &st) != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("failed to stat file ") + path).c_str()); } *is_dir = S_ISDIR(st.st_mode); return nullptr; // success } std::string JoinPath(std::initializer_list segments) { std::string joined; for (const auto& seg : segments) { if (joined.empty()) { joined = seg; } else if (!seg.empty() && (seg[0] == '/')) { // IsAbsolutePath(seg) if (joined[joined.size() - 1] == '/') { joined.append(seg.substr(1)); } else { joined.append(seg); } } else { // !IsAbsolutePath(seg) if (joined[joined.size() - 1] != '/') { joined.append("/"); } joined.append(seg); } } return joined; } TRITONSERVER_Error* ModelPaths( const std::string& model_repository_path, uint64_t version, const bool ignore_directories, const bool ignore_files, std::unordered_map* model_paths) { std::set model_files; // Read all the files in 'path' and filter by type for different requirements auto path = JoinPath({model_repository_path, std::to_string(version)}); RETURN_IF_ERROR(GetDirectoryContents(path, &model_files)); if (ignore_directories) { // Erase directory entries... for (auto iter = model_files.begin(); iter != model_files.end();) { bool is_dir; RETURN_IF_ERROR(IsDirectory(JoinPath({path, *iter}), &is_dir)); if (is_dir) { iter = model_files.erase(iter); } else { ++iter; } } } if (ignore_files) { // Erase non-directory entries... for (auto iter = model_files.begin(); iter != model_files.end();) { bool is_dir; RETURN_IF_ERROR(IsDirectory(JoinPath({path, *iter}), &is_dir)); if (!is_dir) { iter = model_files.erase(iter); } else { ++iter; } } } for (const auto& filename : model_files) { const auto model_path = JoinPath({path, filename}); model_paths->emplace( std::piecewise_construct, std::make_tuple(filename), std::make_tuple(model_path)); } return nullptr; // success } TRITONSERVER_Error* CreateCudaStream( const int device_id, const int cuda_stream_priority, cudaStream_t* stream) { *stream = nullptr; #ifdef TRITON_ENABLE_GPU // Make sure that correct device is set before creating stream and // then restore the device to what was set by the caller. int current_device; auto cuerr = cudaGetDevice(¤t_device); bool overridden = false; if (cuerr == cudaSuccess) { overridden = (current_device != device_id); if (overridden) { cuerr = cudaSetDevice(device_id); } } if (cuerr == cudaSuccess) { cuerr = cudaStreamCreateWithPriority( stream, cudaStreamDefault, cuda_stream_priority); } if (overridden) { cudaSetDevice(current_device); } if (cuerr != cudaSuccess) { *stream = nullptr; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("unable to create stream: ") + cudaGetErrorString(cuerr)) .c_str()); } #endif // TRITON_ENABLE_GPU return nullptr; // success } TRITONSERVER_Error* ParseLongLongValue(const std::string& value, int64_t* parsed_value) { try { *parsed_value = std::stoll(value); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("failed to convert '") + value + "' to long long integral number") .c_str()); } return nullptr; // success } TRITONSERVER_Error* ParseUnsignedLongLongValue(const std::string& value, uint64_t* parsed_value) { try { *parsed_value = std::stoull(value); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("failed to convert '") + value + "' to unsigned long long integral number") .c_str()); } return nullptr; // success } TRITONSERVER_Error* ParseBoolValue(const std::string& value, bool* parsed_value) { std::string lvalue = value; std::transform( lvalue.begin(), lvalue.end(), lvalue.begin(), [](unsigned char c) { return std::tolower(c); }); if ((lvalue == "true") || (lvalue == "on") || (lvalue == "1")) { *parsed_value = true; return nullptr; // success } if ((lvalue == "false") || (lvalue == "off") || (lvalue == "0")) { *parsed_value = false; return nullptr; // success } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("failed to convert '") + value + "' to boolean").c_str()); } TRITONSERVER_Error* ParseIntValue(const std::string& value, int* parsed_value) { try { *parsed_value = std::stoi(value); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("failed to convert '") + value + "' to integral number") .c_str()); } return nullptr; // success } TRITONSERVER_Error* ParseDoubleValue(const std::string& value, double* parsed_value) { try { *parsed_value = std::stod(value); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("failed to convert '") + value + "' to double number") .c_str()); } return nullptr; // success } TRITONSERVER_Error* GetParameterValue( triton::common::TritonJson::Value& params, const std::string& key, std::string* value) { triton::common::TritonJson::Value json_value; RETURN_ERROR_IF_FALSE( params.Find(key.c_str(), &json_value), TRITONSERVER_ERROR_NOT_FOUND, std::string("model configuration is missing the parameter ") + key); RETURN_IF_ERROR(json_value.MemberAsString("string_value", value)); return nullptr; // success } TRITONSERVER_Error* BatchInput::ParseFromModelConfig( triton::common::TritonJson::Value& config, std::vector* batch_inputs) { batch_inputs->clear(); triton::common::TritonJson::Value bis; RETURN_IF_ERROR(config.MemberAsArray("batch_input", &bis)); for (size_t i = 0; i < bis.ArraySize(); ++i) { triton::common::TritonJson::Value bi; RETURN_IF_ERROR(bis.IndexAsObject(i, &bi)); batch_inputs->emplace_back(); RETURN_IF_ERROR(batch_inputs->back().Init(bi)); } return nullptr; // success } TRITONSERVER_Error* BatchInput::Init(triton::common::TritonJson::Value& bi_config) { { triton::common::TritonJson::Value bi_target_names; RETURN_IF_ERROR(bi_config.MemberAsArray("target_name", &bi_target_names)); for (size_t i = 0; i < bi_target_names.ArraySize(); ++i) { std::string tn; RETURN_IF_ERROR(bi_target_names.IndexAsString(i, &tn)); target_names_.emplace_back(std::move(tn)); } } { RETURN_IF_ERROR(bi_config.MemberAsString("kind", &kind_str_)); if (kind_str_ == "BATCH_ELEMENT_COUNT") { kind_ = Kind::BATCH_ELEMENT_COUNT; } else if (kind_str_ == "BATCH_ACCUMULATED_ELEMENT_COUNT") { kind_ = Kind::BATCH_ACCUMULATED_ELEMENT_COUNT; } else if (kind_str_ == "BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO") { kind_ = Kind::BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO; } else if (kind_str_ == "BATCH_MAX_ELEMENT_COUNT_AS_SHAPE") { kind_ = Kind::BATCH_MAX_ELEMENT_COUNT_AS_SHAPE; } else if (kind_str_ == "BATCH_ITEM_SHAPE") { kind_ = Kind::BATCH_ITEM_SHAPE; } else if (kind_str_ == "BATCH_ITEM_SHAPE_FLATTEN") { kind_ = Kind::BATCH_ITEM_SHAPE_FLATTEN; } else { RETURN_ERROR_IF_FALSE( false, TRITONSERVER_ERROR_INVALID_ARG, std::string("unexpected batch input kind '" + kind_str_ + "'")); } } { std::string bi_dtype; RETURN_IF_ERROR(bi_config.MemberAsString("data_type", &bi_dtype)); data_type_ = ModelConfigDataTypeToTritonServerDataType(bi_dtype); RETURN_ERROR_IF_TRUE( data_type_ == TRITONSERVER_TYPE_INVALID, TRITONSERVER_ERROR_INVALID_ARG, std::string("unexpected batch input data type '" + bi_dtype + "'")); } { triton::common::TritonJson::Value bi_source_inputs; RETURN_IF_ERROR(bi_config.MemberAsArray("source_input", &bi_source_inputs)); for (size_t i = 0; i < bi_source_inputs.ArraySize(); ++i) { std::string si; RETURN_IF_ERROR(bi_source_inputs.IndexAsString(i, &si)); source_inputs_.emplace_back(std::move(si)); } } return nullptr; // success } TRITONSERVER_DataType ModelConfigDataTypeToTritonServerDataType(const std::string& data_type_str) { // Must start with "TYPE_". if (data_type_str.rfind("TYPE_", 0) != 0) { return TRITONSERVER_TYPE_INVALID; } const std::string dtype = data_type_str.substr(strlen("TYPE_")); if (dtype == "BOOL") { return TRITONSERVER_TYPE_BOOL; } else if (dtype == "UINT8") { return TRITONSERVER_TYPE_UINT8; } else if (dtype == "UINT16") { return TRITONSERVER_TYPE_UINT16; } else if (dtype == "UINT32") { return TRITONSERVER_TYPE_UINT32; } else if (dtype == "UINT64") { return TRITONSERVER_TYPE_UINT64; } else if (dtype == "INT8") { return TRITONSERVER_TYPE_INT8; } else if (dtype == "INT16") { return TRITONSERVER_TYPE_INT16; } else if (dtype == "INT32") { return TRITONSERVER_TYPE_INT32; } else if (dtype == "INT64") { return TRITONSERVER_TYPE_INT64; } else if (dtype == "FP16") { return TRITONSERVER_TYPE_FP16; } else if (dtype == "FP32") { return TRITONSERVER_TYPE_FP32; } else if (dtype == "FP64") { return TRITONSERVER_TYPE_FP64; } else if (dtype == "STRING") { return TRITONSERVER_TYPE_BYTES; } else if (dtype == "BF16") { return TRITONSERVER_TYPE_BF16; } return TRITONSERVER_TYPE_INVALID; } TRITONSERVER_Error* BatchOutput::ParseFromModelConfig( triton::common::TritonJson::Value& config, std::vector* batch_outputs) { batch_outputs->clear(); triton::common::TritonJson::Value bos; RETURN_IF_ERROR(config.MemberAsArray("batch_output", &bos)); for (size_t i = 0; i < bos.ArraySize(); ++i) { batch_outputs->emplace_back(); auto& batch_output = batch_outputs->back(); triton::common::TritonJson::Value bo; RETURN_IF_ERROR(bos.IndexAsObject(i, &bo)); { triton::common::TritonJson::Value bo_target_names; RETURN_IF_ERROR(bo.MemberAsArray("target_name", &bo_target_names)); for (size_t i = 0; i < bo_target_names.ArraySize(); ++i) { std::string tn; RETURN_IF_ERROR(bo_target_names.IndexAsString(i, &tn)); batch_output.target_names_.emplace_back(std::move(tn)); } } { std::string bo_kind; RETURN_IF_ERROR(bo.MemberAsString("kind", &bo_kind)); if (bo_kind == "BATCH_SCATTER_WITH_INPUT_SHAPE") { batch_output.kind_ = Kind::BATCH_SCATTER_WITH_INPUT_SHAPE; // Keep track of the output info for later cross reference with input int64_t mbs = 0; RETURN_IF_ERROR(config.MemberAsInt("max_batch_size", &mbs)); if (mbs != 0) { batch_output.shape_.push_back(-1); } triton::common::TritonJson::Value ios; RETURN_IF_ERROR(config.MemberAsArray("output", &ios)); for (size_t i = 0; i < ios.ArraySize(); i++) { triton::common::TritonJson::Value io; RETURN_IF_ERROR(ios.IndexAsObject(i, &io)); std::string io_name; RETURN_IF_ERROR(io.MemberAsString("name", &io_name)); if (io_name == batch_output.target_names_[0]) { std::string io_dtype; RETURN_IF_ERROR(io.MemberAsString("data_type", &io_dtype)); batch_output.data_type_ = ModelConfigDataTypeToTritonServerDataType(io_dtype); // If a reshape is provided for the input then use that when // validating that the model matches what is expected. triton::common::TritonJson::Value reshape; if (io.Find("reshape", &reshape)) { RETURN_IF_ERROR( ParseShape(reshape, "shape", &batch_output.shape_)); } else { RETURN_IF_ERROR(ParseShape(io, "dims", &batch_output.shape_)); } break; } } } else { RETURN_ERROR_IF_FALSE( false, TRITONSERVER_ERROR_INVALID_ARG, std::string("unexpected batch output kind '" + bo_kind + "'")); } } { triton::common::TritonJson::Value bo_source_inputs; RETURN_IF_ERROR(bo.MemberAsArray("source_input", &bo_source_inputs)); for (size_t i = 0; i < bo_source_inputs.ArraySize(); ++i) { std::string si; RETURN_IF_ERROR(bo_source_inputs.IndexAsString(i, &si)); batch_output.source_inputs_.emplace_back(std::move(si)); } } } return nullptr; // success } TRITONSERVER_Error* TryParseModelStringParameter( triton::common::TritonJson::Value& params, const std::string& mkey, std::string* value, const std::string& default_value) { triton::common::TritonJson::Value json_value; if (params.Find(mkey.c_str(), &json_value)) { RETURN_IF_ERROR(json_value.MemberAsString("string_value", value)); } else { *value = default_value; } return nullptr; // success } TRITONSERVER_Error* TryParseModelStringParameter( triton::common::TritonJson::Value& params, const std::string& mkey, int* value, const int& default_value) { triton::common::TritonJson::Value json_value; if (params.Find(mkey.c_str(), &json_value)) { std::string string_value; RETURN_IF_ERROR(json_value.MemberAsString("string_value", &string_value)); return ParseIntValue(string_value, value); } else { *value = default_value; return nullptr; // success } } TRITONSERVER_Error* TryParseModelStringParameter( triton::common::TritonJson::Value& params, const std::string& mkey, bool* value, const bool& default_value) { triton::common::TritonJson::Value json_value; if (params.Find(mkey.c_str(), &json_value)) { std::string string_value; RETURN_IF_ERROR(json_value.MemberAsString("string_value", &string_value)); return ParseBoolValue(string_value, value); } else { *value = default_value; return nullptr; // success } } TRITONSERVER_Error* TryParseModelStringParameter( triton::common::TritonJson::Value& params, const std::string& mkey, uint64_t* value, const uint64_t& default_value) { triton::common::TritonJson::Value json_value; if (params.Find(mkey.c_str(), &json_value)) { std::string string_value; RETURN_IF_ERROR(json_value.MemberAsString("string_value", &string_value)); return ParseUnsignedLongLongValue(string_value, value); } else { *value = default_value; return nullptr; // success } } namespace { template TRITONSERVER_Error* BufferAsTypedString( std::string& str, const char* buffer, const size_t element_cnt) { const T* vals = reinterpret_cast(buffer); str += "[ "; for (size_t i = 0; i < element_cnt; ++i) { const T& v = vals[i]; if (i != 0) { str += ", "; } str += std::to_string(v); } str += " ]"; return nullptr; // success } } // namespace TRITONSERVER_Error* BufferAsTypedString( std::string& str, const char* buffer, size_t buffer_byte_size, TRITONSERVER_DataType datatype) { const size_t element_cnt = buffer_byte_size / TRITONSERVER_DataTypeByteSize(datatype); switch (datatype) { case TRITONSERVER_TYPE_UINT8: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_UINT16: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_UINT32: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_UINT64: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_INT8: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_INT16: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_INT32: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_INT64: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_FP32: return BufferAsTypedString(str, buffer, element_cnt); case TRITONSERVER_TYPE_FP64: return BufferAsTypedString(str, buffer, element_cnt); default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( std::string("class result not available for output due to " "unsupported type '") + std::string(TRITONSERVER_DataTypeString(datatype)) + "'") .c_str()); } return nullptr; // success } std::string GetRequestId(TRITONBACKEND_Request* request) { const char* request_id = nullptr; LOG_IF_ERROR( TRITONBACKEND_RequestId(request, &request_id), "unable to retrieve request ID string"); if ((request_id == nullptr) || (request_id[0] == '\0')) { request_id = ""; } return std::string("[request id: ") + request_id + "] "; } TRITONSERVER_Error* ValidateStringBuffer( const char* buffer, size_t buffer_byte_size, const size_t expected_element_cnt, const char* input_name, std::vector>* str_list) { size_t element_idx = 0; size_t remaining_bytes = buffer_byte_size; // Each string in 'buffer' is a 4-byte length followed by the string itself // with no null-terminator. while (remaining_bytes >= sizeof(uint32_t)) { // Do not modify this line. str_list->size() must not exceed // expected_element_cnt. if (element_idx >= expected_element_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected number of string elements " + std::to_string(element_idx + 1) + " for inference input '" + input_name + "', expecting " + std::to_string(expected_element_cnt)) .c_str()); } const uint32_t len = *(reinterpret_cast(buffer)); remaining_bytes -= sizeof(uint32_t); buffer += sizeof(uint32_t); if (remaining_bytes < len) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "incomplete string data for inference input '" + std::string(input_name) + "', expecting string of length " + std::to_string(len) + " but only " + std::to_string(remaining_bytes) + " bytes available") .c_str()); } if (str_list) { str_list->push_back({buffer, len}); } buffer += len; remaining_bytes -= len; element_idx++; } if (element_idx != expected_element_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "expected " + std::to_string(expected_element_cnt) + " strings for inference input '" + input_name + "', got " + std::to_string(element_idx)) .c_str()); } return nullptr; } }} // namespace triton::backend kytensor-2.49.0.7/backend/src/backend_input_collector.cc000066400000000000000000001456161527052371200232130ustar00rootroot00000000000000// Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/backend_input_collector.h" #include #include "triton/backend/backend_common.h" #ifdef TRITON_ENABLE_GPU #include "kernel.h" #endif // TRITON_ENABLE_GPU namespace triton { namespace backend { // // BackendInputCollector::InputIterator // BackendInputCollector::InputIterator::InputIterator( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, const char* input_name, const char* host_policy_name, const bool coalesce_request_input) : requests_(requests), request_count_(request_count), responses_(responses), input_name_(input_name), host_policy_(host_policy_name), coalesce_request_input_(coalesce_request_input), curr_request_idx_(0), curr_buffer_idx_(0), reach_end_(false) { auto& response = (*responses_)[curr_request_idx_]; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestInput( requests_[curr_request_idx_], input_name_, &curr_input_)); RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_InputPropertiesForHostPolicy( curr_input_, host_policy_, nullptr, nullptr, nullptr, nullptr, nullptr, &curr_buffer_cnt_)); } bool BackendInputCollector::InputIterator::GetNextContiguousInput( ContiguousBuffer* input) { if (reach_end_ || (curr_buffer_idx_ >= curr_buffer_cnt_)) { return false; } // Get the first buffer TRITONBACKEND_InputBufferForHostPolicy( curr_input_, host_policy_, curr_buffer_idx_, reinterpret_cast(&input->memory_desc_.buffer_), reinterpret_cast(&input->memory_desc_.byte_size_), &input->memory_desc_.memory_type_, &input->memory_desc_.memory_type_id_); ++curr_buffer_idx_; input->start_request_idx_ = curr_request_idx_; input->end_request_idx_ = curr_request_idx_; if (!coalesce_request_input_) { if (curr_buffer_idx_ >= curr_buffer_cnt_) { ++curr_request_idx_; if (curr_request_idx_ < request_count_) { auto& response = (*responses_)[curr_request_idx_]; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestInput( requests_[curr_request_idx_], input_name_, &curr_input_)); RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_InputPropertiesForHostPolicy( curr_input_, host_policy_, nullptr, nullptr, nullptr, nullptr, nullptr, &curr_buffer_cnt_)); // reset buffer idx curr_buffer_idx_ = 0; } else { reach_end_ = true; } } return true; } do { for (; curr_buffer_idx_ < curr_buffer_cnt_; ++curr_buffer_idx_) { const void* next_buffer; size_t next_buffer_byte_size; TRITONSERVER_MemoryType next_memory_type; int64_t next_memory_type_id; TRITONBACKEND_InputBufferForHostPolicy( curr_input_, host_policy_, curr_buffer_idx_, &next_buffer, reinterpret_cast(&next_buffer_byte_size), &next_memory_type, &next_memory_type_id); if (((input->memory_desc_.buffer_ + input->memory_desc_.byte_size_) != next_buffer) || (input->memory_desc_.memory_type_ != next_memory_type) || (input->memory_desc_.memory_type_id_ != next_memory_type_id)) { return true; } input->memory_desc_.byte_size_ += next_buffer_byte_size; input->end_request_idx_ = curr_request_idx_; } // Iterated all buffers for current request, check next ++curr_request_idx_; if (curr_request_idx_ < request_count_) { auto& response = (*responses_)[curr_request_idx_]; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestInput( requests_[curr_request_idx_], input_name_, &curr_input_)); RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_InputPropertiesForHostPolicy( curr_input_, host_policy_, nullptr, nullptr, nullptr, nullptr, nullptr, &curr_buffer_cnt_)); // reset buffer idx curr_buffer_idx_ = 0; } } while (curr_request_idx_ < request_count_); reach_end_ = true; return true; } // // BackendInputCollector // bool BackendInputCollector::GetInputBufferIfContiguous( const char* input_name, const char** buffer, size_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { *buffer = nullptr; *buffer_byte_size = 0; const char* expected_next_buffer = nullptr; bool contiguous = true; for (size_t idx = 0; idx < request_count_; idx++) { auto& request = requests_[idx]; auto& response = (*responses_)[idx]; TRITONBACKEND_Input* input; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestInput(request, input_name, &input)); uint64_t byte_size; uint32_t buffer_count; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_InputPropertiesForHostPolicy( input, host_policy_cstr_, nullptr, nullptr, nullptr, nullptr, &byte_size, &buffer_count)); for (size_t idx = 0; idx < buffer_count; ++idx) { const void* src_buffer; size_t src_byte_size; TRITONSERVER_MemoryType src_memory_type; int64_t src_memory_type_id; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_InputBufferForHostPolicy( input, host_policy_cstr_, idx, &src_buffer, reinterpret_cast(&src_byte_size), &src_memory_type, &src_memory_type_id)); if (*buffer != nullptr) { // If have seen the second buffer while coalescing input is not // requested, treat the inputs are not contiguous if (coalesce_request_input_ && (expected_next_buffer == src_buffer) && (*memory_type == src_memory_type) && (*memory_type_id == src_memory_type_id)) { expected_next_buffer += src_byte_size; } else { contiguous = false; } // Want to know total buffer byte size even if it is not contiguous *buffer_byte_size += src_byte_size; } else { *buffer = reinterpret_cast(src_buffer); *memory_type = src_memory_type; *memory_type_id = src_memory_type_id; *buffer_byte_size = src_byte_size; expected_next_buffer = *buffer + src_byte_size; } } } return contiguous; } void BackendInputCollector::ProcessTensor( const char* input_name, char* buffer, const size_t buffer_byte_size, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id) { // A value of CPU_PINNED indicates that pinned memory buffer is not // needed for this tensor. Any other value indicates that a pinned // memory buffer is needed when the target memory type matches // 'use_pinned_memory_type'. TRITONSERVER_MemoryType use_pinned_memory_type = TRITONSERVER_MEMORY_CPU_PINNED; if (pinned_enabled_) { use_pinned_memory_type = GetUsePinnedMemoryType(memory_type); } const bool use_kernel = (kernel_buffer_threshold_ != 0); size_t buffer_offset = 0; InputIterator ii( requests_, request_count_, responses_, input_name, host_policy_cstr_, coalesce_request_input_); ContiguousBuffer input; while (ii.GetNextContiguousInput(&input)) { // If there are pending copies from tensor buffer that is not // contiguous with 'response's part of that buffer, then need to // go ahead and perform the pending copies so that can start a new // contiguous region if necessary. if ((pending_pinned_byte_size_ > 0) && (buffer_offset != (pending_pinned_byte_size_ + pending_pinned_offset_))) { need_sync_ |= FlushPendingPinned( buffer, buffer_byte_size, memory_type, memory_type_id); } if ((pending_copy_kernel_buffer_byte_size_ > 0) && (buffer_offset != (pending_copy_kernel_buffer_byte_size_ + pending_copy_kernel_buffer_offset_))) { need_sync_ |= FlushPendingCopyKernel( buffer, buffer_byte_size, memory_type, memory_type_id); } need_sync_ |= SetInputTensor( input_name, input, buffer, buffer_byte_size, memory_type, memory_type_id, buffer_offset, use_pinned_memory_type, use_kernel, true); buffer_offset += input.memory_desc_.byte_size_; } // Done with the tensor, flush any pending pinned copies. need_sync_ |= FlushPendingPinned(buffer, buffer_byte_size, memory_type, memory_type_id); need_sync_ |= FlushPendingCopyKernel( buffer, buffer_byte_size, memory_type, memory_type_id); #ifdef TRITON_ENABLE_GPU if (need_sync_ && (event_ != nullptr)) { cudaEventRecord(event_, stream_); } #endif // TRITON_ENABLE_GPU } TRITONSERVER_Error* BackendInputCollector::ProcessTensor( const char* input_name, char* buffer, const size_t buffer_byte_size, const std::vector>& allowed_input_types, const char** dst_buffer, size_t* dst_buffer_byte_size, TRITONSERVER_MemoryType* dst_memory_type, int64_t* dst_memory_type_id) { if (buffer == nullptr) { if (allowed_input_types.size() == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "'allowed_input_types' must contain at least one pair of memory type " "and id"); } if (GetInputBufferIfContiguous( input_name, dst_buffer, dst_buffer_byte_size, dst_memory_type, dst_memory_type_id)) { // zero size buffer will be treated as contiguous as well, // but we want to invoke backend memory to have a valid address. if (*dst_buffer_byte_size != 0) { // If the buffer is contiguous, check if the caller expects its type for (const auto& allowed_type : allowed_input_types) { if ((*dst_memory_type == allowed_type.first) && ((*dst_memory_type_id == allowed_type.second))) { return nullptr; // success } } } } // A separate buffer is needed BackendMemory* backend_memory = nullptr; for (const auto& allowed_type : allowed_input_types) { std::vector alloc_types; const int64_t memory_type_id = allowed_type.second; switch (allowed_type.first) { case TRITONSERVER_MEMORY_GPU: alloc_types = { BackendMemory::AllocationType::GPU_POOL, BackendMemory::AllocationType::GPU}; break; case TRITONSERVER_MEMORY_CPU_PINNED: alloc_types = { BackendMemory::AllocationType::CPU_PINNED_POOL, BackendMemory::AllocationType::CPU_PINNED}; break; case TRITONSERVER_MEMORY_CPU: alloc_types = {BackendMemory::AllocationType::CPU}; break; } auto err = BackendMemory::Create( memory_manager_, alloc_types, memory_type_id, *dst_buffer_byte_size, &backend_memory); if (err != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("unable to create backend memory for type: ") + TRITONSERVER_MemoryTypeString(allowed_type.first) + " id: " + std::to_string(memory_type_id) + ": " + TRITONSERVER_ErrorMessage(err)) .c_str()); TRITONSERVER_ErrorDelete(err); } else { in_use_memories_.emplace_back(backend_memory); break; } } if (backend_memory == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("failed to allocate contiguous buffer for input '") + input_name + "'") .c_str()); } buffer = backend_memory->MemoryPtr(); *dst_buffer = backend_memory->MemoryPtr(); *dst_buffer_byte_size = backend_memory->ByteSize(); *dst_memory_type = backend_memory->MemoryType(); *dst_memory_type_id = backend_memory->MemoryTypeId(); } else { if (allowed_input_types.size() != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "'allowed_input_types' must only contain the memory type and id of " "'buffer'"); } *dst_buffer = buffer; *dst_buffer_byte_size = buffer_byte_size; *dst_memory_type = allowed_input_types[0].first; *dst_memory_type_id = allowed_input_types[0].second; } if (*dst_buffer_byte_size != 0) { ProcessTensor( input_name, buffer, *dst_buffer_byte_size, *dst_memory_type, *dst_memory_type_id); } return nullptr; // success } bool BackendInputCollector::Finalize() { #ifdef TRITON_ENABLE_GPU if ((!deferred_pinned_.empty()) && need_sync_) { if (event_ != nullptr) { cudaEventSynchronize(event_); } else { cudaStreamSynchronize(stream_); } need_sync_ = false; } #endif // TRITON_ENABLE_GPU // After the above sync all the GPU->pinned copies are complete. Any // deferred copies of pinned->CPU can now be done. #ifdef TRITON_ENABLE_GPU if (buffer_ready_event_ != nullptr) { cudaEventSynchronize(buffer_ready_event_); buffer_ready_event_ = nullptr; } #endif // TRITON_ENABLE_GPU for (auto& def : deferred_pinned_) { if (!def.finalized_) { need_sync_ |= def.Finalize(stream_); } } for (size_t i = 0; i < async_task_count_; i++) { need_sync_ |= completion_queue_.Get(); } #ifdef TRITON_ENABLE_GPU // Record the new event location if deferred copies occur if ((!deferred_pinned_.empty()) && need_sync_ && (event_ != nullptr)) { cudaEventRecord(event_, stream_); } #endif // TRITON_ENABLE_GPU return need_sync_; } bool BackendInputCollector::DeferredPinned::Finalize(cudaStream_t stream) { bool cuda_used = false; auto err = CopyBuffer( "pinned buffer", TRITONSERVER_MEMORY_CPU_PINNED, 0, tensor_memory_type_, tensor_memory_id_, pinned_memory_size_, pinned_memory_, tensor_buffer_ + tensor_buffer_offset_, stream, &cuda_used); // If something goes wrong with the copy all the pending // responses fail... if (err != nullptr) { for (auto& pr : requests_) { for (size_t idx = pr.start_request_idx_; idx <= pr.end_request_idx_; ++idx) { if ((*responses_)[idx] != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( (*responses_)[idx], TRITONSERVER_RESPONSE_COMPLETE_FINAL, err), "failed to send error response"); (*responses_)[idx] = nullptr; } } } TRITONSERVER_ErrorDelete(err); } return cuda_used; } bool BackendInputCollector::SetInputTensor( const char* input_name, const ContiguousBuffer& input, char* tensor_buffer, const size_t tensor_buffer_byte_size, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id, const size_t tensor_buffer_offset, const TRITONSERVER_MemoryType use_pinned_memory_type, const bool use_kernel, const bool wait_buffer) { bool cuda_copy = false; if ((tensor_buffer_offset + input.memory_desc_.byte_size_) > tensor_buffer_byte_size) { for (size_t i = input.start_request_idx_; i <= input.end_request_idx_; ++i) { RESPOND_AND_SET_NULL_IF_ERROR( &(*responses_)[i], TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected total byte size " + std::to_string( tensor_buffer_offset + input.memory_desc_.byte_size_) + " for input '" + input_name + "', expecting " + std::to_string(tensor_buffer_byte_size)) .c_str())); } return cuda_copy; } // If the request buffer matches the memory type that should use an // intermediate pinned memory buffer for the transfer, then just // record the input as pending and increase the size required for // the intermediate pinned buffer. We only do this check for the // first buffer of an input and apply the same policy for all // buffers. So if an inputs data is split over different memory // types this may not be ideal but that should be a very rare // situation. if ((use_pinned_memory_type != TRITONSERVER_MEMORY_CPU_PINNED) && (input.memory_desc_.memory_type_ == use_pinned_memory_type)) { if (pending_pinned_byte_size_ == 0) { pending_pinned_offset_ = tensor_buffer_offset; } pending_pinned_byte_size_ += input.memory_desc_.byte_size_; pending_pinned_input_buffers_.push_back(input); return cuda_copy; } // [FIXME] support other direction if prove to be faster, all kernel // handling code in this class assumes the destination buffer is on device // If the request buffer and the destination buffer are accessible by all // GPUs (i.e. pinned, device), initiate the copy via copy CUDA kernel. // We only do this check for the // first buffer of an input and apply the same policy for all // buffers. So if an inputs data is split over different memory // types this may not be ideal but that should be a very rare // situation. // Currently checked direction: // pinned -> device // same device -> device // different device -> device if (use_kernel && (input.memory_desc_.memory_type_ != TRITONSERVER_MEMORY_CPU) && (tensor_memory_type == TRITONSERVER_MEMORY_GPU)) { // [FIXME] Currently not allowing copy between devices as it requires // peer-to-peer access to be enabled. Peer-to-peer is enabled by default, // but server can still runs even if it fails to enable peer-to-peer. // Should provide a utility to check whether a device pair allows direct // access and use gather kernel accordingly if ((input.memory_desc_.memory_type_ != TRITONSERVER_MEMORY_GPU) || (input.memory_desc_.memory_type_id_ == tensor_memory_type_id)) { if (pending_copy_kernel_buffer_byte_size_ == 0) { pending_copy_kernel_buffer_offset_ = tensor_buffer_offset; } pending_copy_kernel_buffer_byte_size_ += input.memory_desc_.byte_size_; ++pending_copy_kernel_input_buffer_counts_; pending_copy_kernel_input_buffers_.push_back(input); return cuda_copy; } } #ifdef TRITON_ENABLE_GPU if (wait_buffer && (buffer_ready_event_ != nullptr)) { cudaEventSynchronize(buffer_ready_event_); buffer_ready_event_ = nullptr; } #endif // TRITON_ENABLE_GPU // Direct copy without intermediate pinned memory. bool cuda_used = false; auto err = CopyBuffer( input_name, input.memory_desc_.memory_type_, input.memory_desc_.memory_type_id_, tensor_memory_type, tensor_memory_type_id, input.memory_desc_.byte_size_, input.memory_desc_.buffer_, tensor_buffer + tensor_buffer_offset, stream_, &cuda_used, copy_on_stream_); if (err != nullptr) { for (size_t i = input.start_request_idx_; i <= input.end_request_idx_; ++i) { RESPOND_AND_SET_NULL_IF_ERROR( &(*responses_)[i], TRITONSERVER_ErrorNew( TRITONSERVER_ErrorCode(err), TRITONSERVER_ErrorMessage(err))); } TRITONSERVER_ErrorDelete(err); } cuda_copy |= cuda_used; return cuda_copy; } bool BackendInputCollector::FlushPendingPinned( char* tensor_buffer, const size_t tensor_buffer_byte_size, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id) { bool cuda_copy = false; // Will be copying from CPU->pinned->GPU or GPU->pinned->CPU // Attempt to allocate a pinned buffer to use for staging the // copy... if we fail to allocated the pinned buffer then we just // directly go CPU->GPU or GPU->CPU. char* pinned_memory = nullptr; int64_t pinned_memory_type_id = 0; TRITONSERVER_MemoryType pinned_memory_type; BackendMemory* backend_memory; if (pending_pinned_byte_size_ > 0) { TRITONSERVER_Error* err = BackendMemory::Create( memory_manager_, {BackendMemory::AllocationType::CPU_PINNED_POOL, BackendMemory::AllocationType::CPU_PINNED}, 0 /* memory_type_id */, pending_pinned_byte_size_, &backend_memory); if (err != nullptr) { TRITONSERVER_ErrorDelete(err); } else { pinned_memory = backend_memory->MemoryPtr(); pinned_memory_type = backend_memory->MemoryType(); pinned_memory_type_id = backend_memory->MemoryTypeId(); } } // If the pinned buffer wasn't actually allocated then just perform // a direct copy. if (pinned_memory == nullptr) { size_t offset = 0; for (auto& pr : pending_pinned_input_buffers_) { cuda_copy |= SetInputTensor( "pinned fallback", pr, tensor_buffer, tensor_buffer_byte_size, tensor_memory_type, tensor_memory_type_id, pending_pinned_offset_ + offset, TRITONSERVER_MEMORY_CPU_PINNED, false, true); offset += pr.memory_desc_.byte_size_; } } // We have a pinned buffer so copy the pending input buffer(s) into // the pinned memory. else { // pinned_memory_type == TRITONSERVER_MEMORY_CPU_PINNED bool cuda_used = false; size_t offset = 0; if (!use_async_cpu_copy_) { for (auto& pr : pending_pinned_input_buffers_) { cuda_used |= SetInputTensor( "pinned H2H", pr, pinned_memory, pending_pinned_byte_size_, TRITONSERVER_MEMORY_CPU_PINNED, 0 /* memory_type_id */, offset, TRITONSERVER_MEMORY_CPU_PINNED, false, true); offset += pr.memory_desc_.byte_size_; } cuda_copy |= cuda_used; // If the copy was not async (i.e. if request input was in CPU so // a CPU->CPU-PINNED copy was performed above), then the pinned // buffer now holds the tensor contents and we can immediately // issue the copies from the pinned buffer to the tensor. // // Otherwise the GPU->CPU-PINNED async copies are in flight and we // simply remember the pinned buffer and the corresponding // request inputs so that we can do the pinned->CPU copies in // finalize after we have waited for all async copies to complete. if (!cuda_used) { #ifdef TRITON_ENABLE_GPU if (buffer_ready_event_ != nullptr) { cudaEventSynchronize(buffer_ready_event_); buffer_ready_event_ = nullptr; } #endif // TRITON_ENABLE_GPU auto err = CopyBuffer( "pinned input buffer H2D", TRITONSERVER_MEMORY_CPU_PINNED, 0 /* memory_type_id */, tensor_memory_type, tensor_memory_type_id, pending_pinned_byte_size_, pinned_memory, tensor_buffer + pending_pinned_offset_, stream_, &cuda_used, copy_on_stream_); cuda_copy |= cuda_used; // If something goes wrong with the copy all the pending // responses fail... if (err != nullptr) { for (auto& pr : pending_pinned_input_buffers_) { for (size_t idx = pr.start_request_idx_; idx <= pr.end_request_idx_; ++idx) { if ((*responses_)[idx] != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( (*responses_)[idx], TRITONSERVER_RESPONSE_COMPLETE_FINAL, err), "failed to send error response"); (*responses_)[idx] = nullptr; } } } TRITONSERVER_ErrorDelete(err); } } else { // cuda_used deferred_pinned_.emplace_back( pinned_memory, pending_pinned_byte_size_, tensor_buffer, pending_pinned_offset_, tensor_memory_type, tensor_memory_type_id, std::move(pending_pinned_input_buffers_), responses_); } } else { async_task_count_++; deferred_pinned_.emplace_back( pinned_memory, pending_pinned_byte_size_, tensor_buffer, pending_pinned_offset_, tensor_memory_type, tensor_memory_type_id, std::move(pending_pinned_input_buffers_), responses_); auto& deferred_pinned = deferred_pinned_.back(); // Mark finalized to avoid duplicated call to DeferredPinned::Finalized() // in BackendInputCollector::Finalize() deferred_pinned_.back().finalized_ = true; auto incomplete_count = new std::atomic(std::min( deferred_pinned_.back().requests_.size(), triton::common::AsyncWorkQueue::WorkerCount())); auto pending_pinned_byte_size = pending_pinned_byte_size_; size_t stride = (deferred_pinned_.back().requests_.size() + triton::common::AsyncWorkQueue::WorkerCount() - 1) / triton::common::AsyncWorkQueue::WorkerCount(); auto pending_it = deferred_pinned_.back().requests_.begin(); while (pending_it != deferred_pinned_.back().requests_.end()) { auto end_it = pending_it; auto next_offset = offset; for (size_t idx = 0; idx < stride; idx++) { next_offset += end_it->memory_desc_.byte_size_; end_it++; if (end_it == deferred_pinned_.back().requests_.end()) { break; } } auto err = CommonErrorToTritonError(triton::common::AsyncWorkQueue::AddTask( [this, offset, pinned_memory, pinned_memory_type, pending_pinned_byte_size, pinned_memory_type_id, pending_it, end_it, incomplete_count, &deferred_pinned]() mutable { for (; pending_it != end_it; pending_it++) { SetInputTensor( "pinned async H2H", *pending_it, pinned_memory, pending_pinned_byte_size, pinned_memory_type, pinned_memory_type_id, offset, TRITONSERVER_MEMORY_CPU_PINNED, false, false); offset += pending_it->memory_desc_.byte_size_; } // The last segmented task will start the next phase of // the internal pinned buffer copy if (incomplete_count->fetch_sub(1) == 1) { #ifdef TRITON_ENABLE_GPU if (buffer_ready_event_ != nullptr) { cudaEventSynchronize(buffer_ready_event_); buffer_ready_event_ = nullptr; } #endif // TRITON_ENABLE_GPU completion_queue_.Put(deferred_pinned.Finalize(stream_)); delete incomplete_count; } })); if (err != nullptr) { for (; pending_it != end_it; pending_it++) { for (size_t idx = pending_it->start_request_idx_; idx <= pending_it->end_request_idx_; ++idx) { if ((*responses_)[idx] != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( (*responses_)[idx], TRITONSERVER_RESPONSE_COMPLETE_FINAL, err), "failed to send error response"); (*responses_)[idx] = nullptr; } } } } TRITONSERVER_ErrorDelete(err); offset = next_offset; pending_it = end_it; } } } // Pending pinned copies are handled... pending_pinned_byte_size_ = 0; pending_pinned_offset_ = 0; pending_pinned_input_buffers_.clear(); // Need to hold on to the allocated pinned buffer as there are still // copies in flight. Will delete it in finalize. if (pinned_memory != nullptr) { in_use_memories_.emplace_back(backend_memory); } return cuda_copy; } TRITONSERVER_Error* BackendInputCollector::BatchInputShape( const BatchInput& batch_input, std::vector* shape) { *shape = std::vector{0}; switch (batch_input.BatchInputKind()) { case BatchInput::Kind::BATCH_ELEMENT_COUNT: case BatchInput::Kind::BATCH_ACCUMULATED_ELEMENT_COUNT: { (*shape)[0] = request_count_; break; } case BatchInput::Kind::BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO: { (*shape)[0] = request_count_ + 1; break; } case BatchInput::Kind::BATCH_MAX_ELEMENT_COUNT_AS_SHAPE: { const auto& source_input = batch_input.SourceInputs()[0]; for (size_t req_idx = 0; req_idx < request_count_; req_idx++) { TRITONBACKEND_Input* input; RETURN_IF_ERROR(TRITONBACKEND_RequestInput( requests_[req_idx], source_input.c_str(), &input)); const int64_t* shape_arr; uint32_t dims_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( input, host_policy_cstr_, nullptr, nullptr, &shape_arr, &dims_count, nullptr, nullptr)); (*shape)[0] = std::max((*shape)[0], GetElementCount(shape_arr, dims_count)); } break; } case BatchInput::Kind::BATCH_ITEM_SHAPE: { shape->emplace_back(0); const auto& source_input = batch_input.SourceInputs()[0]; for (size_t req_idx = 0; req_idx < request_count_; req_idx++) { TRITONBACKEND_Input* input; RETURN_IF_ERROR(TRITONBACKEND_RequestInput( requests_[req_idx], source_input.c_str(), &input)); const int64_t* shape_arr; uint32_t dims_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( input, host_policy_cstr_, nullptr, nullptr, &shape_arr, &dims_count, nullptr, nullptr)); // Assuming first dimension is batch size and ragged input is only set // for batching enabled model. (*shape)[0] += shape_arr[0]; // The batch input tracks the shape without batch dimension for // each batch item (*shape)[1] = (dims_count - 1); } break; } case BatchInput::Kind::BATCH_ITEM_SHAPE_FLATTEN: { const auto& source_input = batch_input.SourceInputs()[0]; for (size_t req_idx = 0; req_idx < request_count_; req_idx++) { TRITONBACKEND_Input* input; RETURN_IF_ERROR(TRITONBACKEND_RequestInput( requests_[req_idx], source_input.c_str(), &input)); const int64_t* shape_arr; uint32_t dims_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( input, host_policy_cstr_, nullptr, nullptr, &shape_arr, &dims_count, nullptr, nullptr)); // Assuming first dimension is batch size and ragged input is only set // for batching enabled model. // The batch input tracks the shape without batch dimension for // each batch item (*shape)[0] += (shape_arr[0] * (dims_count - 1)); } break; } default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unsupported BatchInputKind received"); } return nullptr; // success } TRITONSERVER_Error* BackendInputCollector::ProcessBatchInput( const BatchInput& batch_input, char* buffer, const size_t buffer_byte_size, const std::vector>& allowed_input_types, const char** dst_buffer, size_t* dst_buffer_byte_size, TRITONSERVER_MemoryType* dst_memory_type, int64_t* dst_memory_type_id) { #ifdef TRITON_ENABLE_GPU if (buffer_ready_event_ != nullptr) { cudaEventSynchronize(buffer_ready_event_); buffer_ready_event_ = nullptr; } #endif // TRITON_ENABLE_GPU if (buffer == nullptr) { if (allowed_input_types.size() == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "'allowed_input_types' must contain at least one pair of memory type " "and id"); } // Calculate the byte size of the buffer std::vector shape; RETURN_IF_ERROR(BatchInputShape(batch_input, &shape)); *dst_buffer_byte_size = GetByteSize(batch_input.DataType(), shape); BackendMemory* backend_memory = nullptr; for (const auto& allowed_type : allowed_input_types) { std::vector alloc_types; const int64_t memory_type_id = allowed_type.second; switch (allowed_type.first) { case TRITONSERVER_MEMORY_GPU: alloc_types = { BackendMemory::AllocationType::GPU_POOL, BackendMemory::AllocationType::GPU}; break; case TRITONSERVER_MEMORY_CPU_PINNED: alloc_types = { BackendMemory::AllocationType::CPU_PINNED_POOL, BackendMemory::AllocationType::CPU_PINNED}; break; case TRITONSERVER_MEMORY_CPU: alloc_types = {BackendMemory::AllocationType::CPU}; break; } auto err = BackendMemory::Create( memory_manager_, alloc_types, memory_type_id, *dst_buffer_byte_size, &backend_memory); if (err != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("unable to create backend memory for type: ") + TRITONSERVER_MemoryTypeString(allowed_type.first) + " id: " + std::to_string(memory_type_id) + ": " + TRITONSERVER_ErrorMessage(err)) .c_str()); TRITONSERVER_ErrorDelete(err); } else { in_use_memories_.emplace_back(backend_memory); break; } } if (backend_memory == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string( "failed to allocate contiguous buffer for batch input '") + batch_input.TargetNames()[0] + "'") .c_str()); } buffer = backend_memory->MemoryPtr(); *dst_buffer = backend_memory->MemoryPtr(); *dst_buffer_byte_size = backend_memory->ByteSize(); *dst_memory_type = backend_memory->MemoryType(); *dst_memory_type_id = backend_memory->MemoryTypeId(); } else { if (allowed_input_types.size() != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "'allowed_input_types' must only contain the memory type and id of " "'buffer'"); } *dst_buffer = buffer; *dst_buffer_byte_size = buffer_byte_size; *dst_memory_type = allowed_input_types[0].first; *dst_memory_type_id = allowed_input_types[0].second; } char* input_buffer = buffer; std::unique_ptr internal_buffer; // Need a CPU buffer for modifying the value if (*dst_memory_type == TRITONSERVER_MEMORY_GPU) { BackendMemory* ib = nullptr; RETURN_IF_ERROR(BackendMemory::Create( memory_manager_, {BackendMemory::AllocationType::CPU_PINNED_POOL, BackendMemory::AllocationType::CPU}, 0, *dst_buffer_byte_size, &ib)); internal_buffer.reset(ib); input_buffer = internal_buffer->MemoryPtr(); } const auto& data_type = batch_input.DataType(); switch (batch_input.BatchInputKind()) { case BatchInput::Kind::BATCH_ELEMENT_COUNT: { const auto& source_input = batch_input.SourceInputs()[0]; if (data_type == TRITONSERVER_TYPE_FP32) { RETURN_IF_ERROR(SetElementCount( source_input, input_buffer, *dst_buffer_byte_size)); } else { RETURN_IF_ERROR(SetElementCount( source_input, input_buffer, *dst_buffer_byte_size)); } break; } case BatchInput::Kind::BATCH_ACCUMULATED_ELEMENT_COUNT: { const auto& source_input = batch_input.SourceInputs()[0]; if (data_type == TRITONSERVER_TYPE_FP32) { RETURN_IF_ERROR(SetAccumulatedElementCount( source_input, input_buffer, *dst_buffer_byte_size)); } else { RETURN_IF_ERROR(SetAccumulatedElementCount( source_input, input_buffer, *dst_buffer_byte_size)); } break; } case BatchInput::Kind::BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO: { const auto& source_input = batch_input.SourceInputs()[0]; if (data_type == TRITONSERVER_TYPE_FP32) { *reinterpret_cast(input_buffer) = 0; RETURN_IF_ERROR(SetAccumulatedElementCount( source_input, input_buffer + sizeof(float), *dst_buffer_byte_size - sizeof(float))); } else { *reinterpret_cast(input_buffer) = 0; RETURN_IF_ERROR(SetAccumulatedElementCount( source_input, input_buffer + sizeof(int32_t), *dst_buffer_byte_size - sizeof(int32_t))); } break; } case BatchInput::Kind::BATCH_MAX_ELEMENT_COUNT_AS_SHAPE: { // The batch input is described by the shape, // no data modification is needed return nullptr; // success } case BatchInput::Kind::BATCH_ITEM_SHAPE: case BatchInput::Kind::BATCH_ITEM_SHAPE_FLATTEN: { // Use the same utilities for both types as the data will be the same, // only difference is the shape of the tensor. const auto& source_input = batch_input.SourceInputs()[0]; if (data_type == TRITONSERVER_TYPE_FP32) { *reinterpret_cast(input_buffer) = 0; RETURN_IF_ERROR(SetBatchItemShape( source_input, input_buffer, *dst_buffer_byte_size)); } else { *reinterpret_cast(input_buffer) = 0; RETURN_IF_ERROR(SetBatchItemShape( source_input, input_buffer, *dst_buffer_byte_size)); } break; } } if (*dst_memory_type == TRITONSERVER_MEMORY_GPU) { bool cuda_used; RETURN_IF_ERROR(CopyBuffer( "batch input buffer", internal_buffer->MemoryType(), internal_buffer->MemoryTypeId(), *dst_memory_type, *dst_memory_type_id, *dst_buffer_byte_size, input_buffer, buffer, stream_, &cuda_used, copy_on_stream_)); // Need to keep the backend memory alive in the case of async copy in_use_memories_.emplace_back(std::move(internal_buffer)); need_sync_ |= cuda_used; } return nullptr; // success } template TRITONSERVER_Error* BackendInputCollector::SetElementCount( const std::string& source_input, char* buffer, const size_t buffer_byte_size) { size_t buffer_offset = 0; for (size_t req_idx = 0; req_idx < request_count_; req_idx++) { if (buffer_offset + sizeof(T) > buffer_byte_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "unexpected total byte size for batch input"); } TRITONBACKEND_Input* input; RETURN_IF_ERROR(TRITONBACKEND_RequestInput( requests_[req_idx], source_input.c_str(), &input)); const int64_t* shape; uint32_t dims_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( input, host_policy_cstr_, nullptr, nullptr, &shape, &dims_count, nullptr, nullptr)); *(reinterpret_cast(buffer) + req_idx) = GetElementCount(shape, dims_count); buffer_offset += sizeof(T); } // Set the rest of the buffer to 0 for (; buffer_offset + sizeof(T) <= buffer_byte_size; buffer_offset += sizeof(T)) { *reinterpret_cast(buffer + buffer_offset) = 0; } return nullptr; // success } template TRITONSERVER_Error* BackendInputCollector::SetAccumulatedElementCount( const std::string& source_input, char* buffer, const size_t buffer_byte_size) { size_t accumulated_element_count = 0; size_t buffer_offset = 0; for (size_t req_idx = 0; req_idx < request_count_; req_idx++) { if (buffer_offset + sizeof(T) > buffer_byte_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "unexpected total byte size for batch input"); } TRITONBACKEND_Input* input; RETURN_IF_ERROR(TRITONBACKEND_RequestInput( requests_[req_idx], source_input.c_str(), &input)); const int64_t* shape; uint32_t dims_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( input, host_policy_cstr_, nullptr, nullptr, &shape, &dims_count, nullptr, nullptr)); accumulated_element_count += GetElementCount(shape, dims_count); *(reinterpret_cast(buffer) + req_idx) = accumulated_element_count; buffer_offset += sizeof(T); } // Set the rest of the buffer to 'accumulated_element_count' // (no increase in element count) for (; buffer_offset + sizeof(T) <= buffer_byte_size; buffer_offset += sizeof(T)) { *reinterpret_cast(buffer + buffer_offset) = accumulated_element_count; } return nullptr; // success } template TRITONSERVER_Error* BackendInputCollector::SetBatchItemShape( const std::string& source_input, char* buffer, const size_t buffer_byte_size) { size_t buffer_offset = 0; for (size_t req_idx = 0; req_idx < request_count_; req_idx++) { TRITONBACKEND_Input* input; RETURN_IF_ERROR(TRITONBACKEND_RequestInput( requests_[req_idx], source_input.c_str(), &input)); const int64_t* shape; uint32_t dims_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( input, host_policy_cstr_, nullptr, nullptr, &shape, &dims_count, nullptr, nullptr)); // Assuming first dimension is batch size and ragged input is only set // for batching enabled model. size_t batch_1_size = sizeof(T) * (dims_count - 1); if (buffer_offset + (size_t)shape[0] * batch_1_size > buffer_byte_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (GetRequestId(requests_[req_idx]) + "unexpected total byte size for batch input") .c_str()); } // The batch input tracks the shape without batch dimension for // each batch item for (size_t idx = 1; idx < dims_count; ++idx) { // Need to set the element explicitly for type conversion *(reinterpret_cast(buffer + buffer_offset) + (idx - 1)) = shape[idx]; } // memcpy the data repeatedly if the request has batch size > 1 for (int64_t idx = 1; idx < shape[0]; ++idx) { memcpy( buffer + buffer_offset + idx * batch_1_size, buffer + buffer_offset, batch_1_size); } buffer_offset += batch_1_size * (size_t)shape[0]; } return nullptr; // success } bool BackendInputCollector::FlushPendingCopyKernel( char* tensor_buffer, const size_t tensor_buffer_byte_size, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id) { if (pending_copy_kernel_input_buffers_.size() == 0) { return false; } bool cuda_copy = false; TRITONSERVER_Error* error = nullptr; // Only try to launch kernel if buffer count is large enough for // good GPU utilization if (pending_copy_kernel_input_buffer_counts_ >= kernel_buffer_threshold_) { error = LaunchCopyKernel( tensor_buffer, tensor_buffer_byte_size, tensor_memory_type, tensor_memory_type_id); cuda_copy = (error == nullptr); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("gather kernel launched with status: ") + ((error == nullptr) ? "Success" : TRITONSERVER_ErrorMessage(error))) .c_str()); } // If kernel can't be launched then just perform a direct copy. if ((pending_copy_kernel_input_buffer_counts_ < kernel_buffer_threshold_) || (error != nullptr)) { size_t offset = 0; for (auto& pr : pending_copy_kernel_input_buffers_) { cuda_copy |= SetInputTensor( "gather kernel fallback", pr, tensor_buffer, tensor_buffer_byte_size, tensor_memory_type, tensor_memory_type_id, pending_copy_kernel_buffer_offset_ + offset, TRITONSERVER_MEMORY_CPU_PINNED, false, true); offset += pr.memory_desc_.byte_size_; } } TRITONSERVER_ErrorDelete(error); // Pending kernel copies are handled... pending_copy_kernel_buffer_byte_size_ = 0; pending_copy_kernel_buffer_offset_ = 0; pending_copy_kernel_input_buffer_counts_ = 0; pending_copy_kernel_input_buffers_.clear(); return cuda_copy; } TRITONSERVER_Error* BackendInputCollector::LaunchCopyKernel( char* tensor_buffer, const size_t tensor_buffer_byte_size, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id) { #ifdef TRITON_ENABLE_GPU input_ptr_buffer_host_.emplace_back(new std::vector()); byte_size_buffer_host_.emplace_back(new std::vector()); byte_size_offset_buffer_host_.emplace_back(new std::vector()); auto& input_ptr_buffer_host = *input_ptr_buffer_host_.back(); auto& byte_size_buffer_host = *byte_size_buffer_host_.back(); auto& byte_size_offset_buffer_host = *byte_size_offset_buffer_host_.back(); input_ptr_buffer_host.reserve(pending_copy_kernel_input_buffer_counts_); byte_size_buffer_host.reserve(pending_copy_kernel_input_buffer_counts_); byte_size_offset_buffer_host.reserve( pending_copy_kernel_input_buffer_counts_); size_t byte_size_offset = 0; for (const auto& response_input : pending_copy_kernel_input_buffers_) { const auto& input = response_input.memory_desc_; input_ptr_buffer_host.emplace_back( const_cast(reinterpret_cast(input.buffer_))); byte_size_buffer_host.emplace_back(input.byte_size_); byte_size_offset_buffer_host.emplace_back(byte_size_offset); byte_size_offset += input.byte_size_; } BackendMemory* backend_memory = nullptr; std::vector alloc_types; switch (tensor_memory_type) { case TRITONSERVER_MEMORY_GPU: alloc_types = { BackendMemory::AllocationType::GPU_POOL, BackendMemory::AllocationType::GPU}; break; case TRITONSERVER_MEMORY_CPU_PINNED: alloc_types = { BackendMemory::AllocationType::CPU_PINNED_POOL, BackendMemory::AllocationType::CPU_PINNED}; break; case TRITONSERVER_MEMORY_CPU: alloc_types = {BackendMemory::AllocationType::CPU}; break; } // input_ptr_buffer size_t input_ptr_buffer_byte_size = pending_copy_kernel_input_buffer_counts_ * sizeof(int8_t*); auto err = BackendMemory::Create( memory_manager_, alloc_types, tensor_memory_type_id, input_ptr_buffer_byte_size, &backend_memory); if (err != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("unable to create backend memory for type: ") + TRITONSERVER_MemoryTypeString(tensor_memory_type) + " id: " + std::to_string(tensor_memory_type_id) + ": " + TRITONSERVER_ErrorMessage(err)) .c_str()); TRITONSERVER_ErrorDelete(err); } else { in_use_memories_.emplace_back(backend_memory); } if (backend_memory == nullptr || (backend_memory->MemoryType() != tensor_memory_type) || (backend_memory->MemoryTypeId() != tensor_memory_type_id)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to obtain memory buffer for copy kernel input"); } char* input_ptr_buffer = backend_memory->MemoryPtr(); // byte_size_buffer size_t byte_size_buffer_byte_size = pending_copy_kernel_input_buffer_counts_ * sizeof(size_t); err = BackendMemory::Create( memory_manager_, alloc_types, tensor_memory_type_id, byte_size_buffer_byte_size, &backend_memory); if (err != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("unable to create backend memory for type: ") + TRITONSERVER_MemoryTypeString(tensor_memory_type) + " id: " + std::to_string(tensor_memory_type_id) + ": " + TRITONSERVER_ErrorMessage(err)) .c_str()); TRITONSERVER_ErrorDelete(err); } else { in_use_memories_.emplace_back(backend_memory); } if (backend_memory == nullptr || (backend_memory->MemoryType() != tensor_memory_type) || (backend_memory->MemoryTypeId() != tensor_memory_type_id)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to obtain memory buffer for copy kernel input"); } char* byte_size_buffer = backend_memory->MemoryPtr(); // byte_size_offset_buffer size_t byte_size_offset_buffer_byte_size = pending_copy_kernel_input_buffer_counts_ * sizeof(size_t); err = BackendMemory::Create( memory_manager_, alloc_types, tensor_memory_type_id, byte_size_offset_buffer_byte_size, &backend_memory); if (err != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("unable to create backend memory for type: ") + TRITONSERVER_MemoryTypeString(tensor_memory_type) + " id: " + std::to_string(tensor_memory_type_id) + ": " + TRITONSERVER_ErrorMessage(err)) .c_str()); TRITONSERVER_ErrorDelete(err); } else { in_use_memories_.emplace_back(backend_memory); } if (backend_memory == nullptr || (backend_memory->MemoryType() != tensor_memory_type) || (backend_memory->MemoryTypeId() != tensor_memory_type_id)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to obtain memory buffer for copy kernel input"); } char* byte_size_offset_buffer = backend_memory->MemoryPtr(); cudaMemcpyAsync( input_ptr_buffer, input_ptr_buffer_host.data(), pending_copy_kernel_input_buffer_counts_ * sizeof(int8_t*), cudaMemcpyDefault, stream_); cudaMemcpyAsync( byte_size_buffer, byte_size_buffer_host.data(), pending_copy_kernel_input_buffer_counts_ * sizeof(size_t), cudaMemcpyDefault, stream_); cudaMemcpyAsync( byte_size_offset_buffer, byte_size_offset_buffer_host.data(), pending_copy_kernel_input_buffer_counts_ * sizeof(size_t), cudaMemcpyDefault, stream_); if (buffer_ready_event_ != nullptr) { cudaEventSynchronize(buffer_ready_event_); buffer_ready_event_ = nullptr; } RETURN_IF_CUDA_ERROR( RunGatherKernel( (const int8_t**)input_ptr_buffer, (const size_t*)byte_size_buffer, (const size_t*)byte_size_offset_buffer, (int8_t*)tensor_buffer + pending_copy_kernel_buffer_offset_, pending_copy_kernel_input_buffer_counts_, stream_), TRITONSERVER_ERROR_INTERNAL, std::string("Failed to launch gather kernel")); return nullptr; #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Copy kernel can not be launched with TRITON_ENABLE_GPU=OFF"); #endif // TRITON_ENABLE_GPU } }} // namespace triton::backend kytensor-2.49.0.7/backend/src/backend_memory.cc000066400000000000000000000170201527052371200213010ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/backend_memory.h" #include #include "triton/backend/backend_common.h" namespace triton { namespace backend { TRITONSERVER_Error* BackendMemory::Create( TRITONBACKEND_MemoryManager* manager, const AllocationType alloc_type, const int64_t memory_type_id, const size_t byte_size, BackendMemory** mem) { *mem = nullptr; void* ptr = nullptr; switch (alloc_type) { case AllocationType::CPU_PINNED: { #ifdef TRITON_ENABLE_GPU RETURN_IF_CUDA_ERROR( cudaHostAlloc(&ptr, byte_size, cudaHostAllocPortable), TRITONSERVER_ERROR_UNAVAILABLE, std::string("failed to allocate pinned system memory")); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "pinned-memory allocation not supported"); #endif // TRITON_ENABLE_GPU break; } case AllocationType::GPU: { #ifdef TRITON_ENABLE_GPU int current_device; RETURN_IF_CUDA_ERROR( cudaGetDevice(¤t_device), TRITONSERVER_ERROR_INTERNAL, std::string("failed to get device")); bool overridden = (current_device != memory_type_id); if (overridden) { RETURN_IF_CUDA_ERROR( cudaSetDevice(memory_type_id), TRITONSERVER_ERROR_INTERNAL, std::string("failed to set device")); } auto err = cudaMalloc(&ptr, byte_size); if (overridden) { LOG_IF_CUDA_ERROR( cudaSetDevice(current_device), "failed to set CUDA device"); } RETURN_ERROR_IF_FALSE( err == cudaSuccess, TRITONSERVER_ERROR_UNAVAILABLE, std::string("failed to allocate GPU memory: ") + cudaGetErrorString(err)); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "GPU allocation not supported"); #endif // TRITON_ENABLE_GPU break; } case AllocationType::CPU: case AllocationType::CPU_PINNED_POOL: case AllocationType::GPU_POOL: RETURN_IF_ERROR(TRITONBACKEND_MemoryManagerAllocate( manager, &ptr, AllocTypeToMemoryType(alloc_type), memory_type_id, byte_size)); break; } *mem = new BackendMemory( manager, alloc_type, memory_type_id, reinterpret_cast(ptr), byte_size); return nullptr; // success } TRITONSERVER_Error* BackendMemory::Create( TRITONBACKEND_MemoryManager* manager, const std::vector& alloc_types, const int64_t memory_type_id, const size_t byte_size, BackendMemory** mem) { *mem = nullptr; RETURN_ERROR_IF_TRUE( alloc_types.size() == 0, TRITONSERVER_ERROR_INVALID_ARG, std::string("BackendMemory::Create, at least one allocation type must be " "specified")); bool success = false; std::unordered_map errors; for (const AllocationType alloc_type : alloc_types) { TRITONSERVER_Error* err = Create(manager, alloc_type, memory_type_id, byte_size, mem); if (err == nullptr) { success = true; break; } errors.insert({alloc_type, err}); } // If allocation failed for all allocation types then display all // the error messages and show the entire allocation request as // failing. if (!success) { std::string msg = "BackendMemory::Create, all allocation types failed:"; for (const auto& pr : errors) { const AllocationType alloc_type = pr.first; TRITONSERVER_Error* err = pr.second; msg += std::string("\n\t") + AllocTypeString(alloc_type) + ": " + TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, msg.c_str()); } // If it succeeded we might have to clean up errors associated with // attempts that failed for (const auto& pr : errors) { TRITONSERVER_ErrorDelete(pr.second); } return nullptr; // success } TRITONSERVER_Error* BackendMemory::Create( TRITONBACKEND_MemoryManager* manager, const AllocationType alloc_type, const int64_t memory_type_id, void* buffer, const size_t byte_size, BackendMemory** mem) { *mem = new BackendMemory( manager, alloc_type, memory_type_id, reinterpret_cast(buffer), byte_size, false /* owns_buffer */); return nullptr; // success } BackendMemory::~BackendMemory() { if (owns_buffer_) { switch (alloctype_) { case AllocationType::CPU_PINNED: #ifdef TRITON_ENABLE_GPU if (buffer_ != nullptr) { LOG_IF_CUDA_ERROR( cudaFreeHost(buffer_), "failed to free pinned memory"); } #endif // TRITON_ENABLE_GPU break; case AllocationType::GPU: #ifdef TRITON_ENABLE_GPU if (buffer_ != nullptr) { LOG_IF_CUDA_ERROR(cudaFree(buffer_), "failed to free CUDA memory"); } #endif // TRITON_ENABLE_GPU break; case AllocationType::CPU: case AllocationType::CPU_PINNED_POOL: case AllocationType::GPU_POOL: LOG_IF_ERROR( TRITONBACKEND_MemoryManagerFree( manager_, buffer_, AllocTypeToMemoryType(alloctype_), memtype_id_), "failed to free memory buffer"); break; } } } TRITONSERVER_MemoryType BackendMemory::AllocTypeToMemoryType(const AllocationType a) { switch (a) { case AllocationType::CPU: return TRITONSERVER_MEMORY_CPU; case AllocationType::CPU_PINNED: case AllocationType::CPU_PINNED_POOL: return TRITONSERVER_MEMORY_CPU_PINNED; case AllocationType::GPU: case AllocationType::GPU_POOL: return TRITONSERVER_MEMORY_GPU; } return TRITONSERVER_MEMORY_CPU; // unreachable } const char* BackendMemory::AllocTypeString(const AllocationType a) { switch (a) { case AllocationType::CPU: return "CPU"; case AllocationType::CPU_PINNED: return "CPU_PINNED"; case AllocationType::GPU: return "GPU"; case AllocationType::CPU_PINNED_POOL: return "CPU_PINNED_POOL"; case AllocationType::GPU_POOL: return "GPU_POOL"; } return ""; } }} // namespace triton::backend kytensor-2.49.0.7/backend/src/backend_model.cc000066400000000000000000000160101527052371200210670ustar00rootroot00000000000000// Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/backend_model.h" #include "triton/backend/backend_common.h" namespace triton { namespace backend { // // BackendModel // BackendModel::BackendModel( TRITONBACKEND_Model* triton_model, const bool allow_optional) : triton_model_(triton_model), allow_optional_(allow_optional) { const char* model_name; THROW_IF_BACKEND_MODEL_ERROR( TRITONBACKEND_ModelName(triton_model, &model_name)); name_ = model_name; THROW_IF_BACKEND_MODEL_ERROR( TRITONBACKEND_ModelVersion(triton_model, &version_)); const char* repository_path = nullptr; TRITONBACKEND_ArtifactType repository_artifact_type; THROW_IF_BACKEND_MODEL_ERROR(TRITONBACKEND_ModelRepository( triton_model, &repository_artifact_type, &repository_path)); if (repository_artifact_type != TRITONBACKEND_ARTIFACT_FILESYSTEM) { throw BackendModelException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, (std::string("unsupported repository artifact type for model '") + model_name + "'") .c_str())); } repository_path_ = repository_path; THROW_IF_BACKEND_MODEL_ERROR( TRITONBACKEND_ModelServer(triton_model, &triton_server_)); TRITONBACKEND_Backend* backend; THROW_IF_BACKEND_MODEL_ERROR( TRITONBACKEND_ModelBackend(triton_model, &backend)); THROW_IF_BACKEND_MODEL_ERROR( TRITONBACKEND_BackendMemoryManager(backend, &triton_memory_manager_)); THROW_IF_BACKEND_MODEL_ERROR(ParseModelConfig()); } TRITONSERVER_Error* BackendModel::ParseModelConfig() { TRITONSERVER_Message* config_message; RETURN_IF_ERROR(TRITONBACKEND_ModelConfig( triton_model_, 1 /* config_version */, &config_message)); // Get the model configuration as a json string from // config_message. We use TritonJson, which is a wrapper that // returns nice errors (currently the underlying implementation is // rapidjson... but others could be added). const char* buffer; size_t byte_size; RETURN_IF_ERROR( TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size)); TRITONSERVER_Error* err = model_config_.Parse(buffer, byte_size); RETURN_IF_ERROR(TRITONSERVER_MessageDelete(config_message)); RETURN_IF_ERROR(err); int64_t mbs = 0; RETURN_IF_ERROR(model_config_.MemberAsInt("max_batch_size", &mbs)); max_batch_size_ = mbs; enable_pinned_input_ = false; enable_pinned_output_ = false; { common::TritonJson::Value optimization; if (model_config_.Find("optimization", &optimization)) { common::TritonJson::Value pinned_memory; if (optimization.Find("input_pinned_memory", &pinned_memory)) { RETURN_IF_ERROR( pinned_memory.MemberAsBool("enable", &enable_pinned_input_)); } if (optimization.Find("output_pinned_memory", &pinned_memory)) { RETURN_IF_ERROR( pinned_memory.MemberAsBool("enable", &enable_pinned_output_)); } } } RETURN_IF_ERROR( BatchInput::ParseFromModelConfig(model_config_, &batch_inputs_)); RETURN_IF_ERROR( BatchOutput::ParseFromModelConfig(model_config_, &batch_outputs_)); for (const auto& batch_output : batch_outputs_) { for (const auto& name : batch_output.TargetNames()) { batch_output_map_.emplace(name, &batch_output); } } triton::common::TritonJson::Value config_inputs; RETURN_IF_ERROR(model_config_.MemberAsArray("input", &config_inputs)); for (size_t i = 0; i < config_inputs.ArraySize(); i++) { triton::common::TritonJson::Value io; RETURN_IF_ERROR(config_inputs.IndexAsObject(i, &io)); std::string io_name; RETURN_IF_ERROR(io.MemberAsString("name", &io_name)); triton::common::TritonJson::Value input_property_json; bool allow_ragged_batch = false; if (io.Find("allow_ragged_batch", &input_property_json)) { RETURN_IF_ERROR(input_property_json.AsBool(&allow_ragged_batch)); } if (allow_ragged_batch) { ragged_inputs_.emplace(io_name); } bool optional = false; if (io.Find("optional", &input_property_json)) { RETURN_IF_ERROR(input_property_json.AsBool(&optional)); } if (optional) { if (allow_optional_) { optional_inputs_.emplace(io_name); } else { RETURN_IF_ERROR(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("'optional' is set to true for input '") + io_name + "' while the backend model doesn't support optional input") .c_str())); } } } return nullptr; } TRITONSERVER_Error* BackendModel::SetModelConfig() { triton::common::TritonJson::WriteBuffer json_buffer; RETURN_IF_ERROR(ModelConfig().Write(&json_buffer)); TRITONSERVER_Message* message; RETURN_IF_ERROR(TRITONSERVER_MessageNewFromSerializedJson( &message, json_buffer.Base(), json_buffer.Size())); RETURN_IF_ERROR(TRITONBACKEND_ModelSetConfig( triton_model_, 1 /* config_version */, message)); RETURN_IF_ERROR(TRITONSERVER_MessageDelete(message)); // Triton core can normalize the missing config settings // in the above call. We must retrieve the updated model // configuration from the core. RETURN_IF_ERROR(ParseModelConfig()); return nullptr; } TRITONSERVER_Error* BackendModel::SupportsFirstDimBatching(bool* supports) { *supports = max_batch_size_ > 0; return nullptr; } const BatchOutput* BackendModel::FindBatchOutput(const std::string& output_name) const { const auto it = batch_output_map_.find(output_name); return ((it == batch_output_map_.end()) ? nullptr : it->second); } }} // namespace triton::backend kytensor-2.49.0.7/backend/src/backend_model_instance.cc000066400000000000000000000151541527052371200227630ustar00rootroot00000000000000// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/backend_model_instance.h" #include #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" namespace triton { namespace backend { // // BackendModelInstance // BackendModelInstance::BackendModelInstance( BackendModel* backend_model, TRITONBACKEND_ModelInstance* triton_model_instance) : backend_model_(backend_model), triton_model_instance_(triton_model_instance) { const char* instance_name; THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_ModelInstanceName(triton_model_instance, &instance_name)); name_ = instance_name; THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_ModelInstanceKind(triton_model_instance, &kind_)); THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_ModelInstanceDeviceId(triton_model_instance, &device_id_)); common::TritonJson::Value& model_config = backend_model->ModelConfig(); // If the model configuration specifies a 'default_model_filename' // and/or specifies 'cc_model_filenames' then determine the // appropriate 'artifact_filename' value. If model configuration // does not specify then just leave 'artifact_filename' empty and // the backend can then provide its own logic for determine the // filename if that is appropriate. THROW_IF_BACKEND_INSTANCE_ERROR(model_config.MemberAsString( "default_model_filename", &artifact_filename_)); switch (kind_) { case TRITONSERVER_INSTANCEGROUPKIND_CPU: { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Creating instance ") + name_ + " on CPU using artifact '" + artifact_filename_ + "'") .c_str()); break; } case TRITONSERVER_INSTANCEGROUPKIND_MODEL: { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Creating instance ") + name_ + " on model-specified devices using artifact '" + artifact_filename_ + "'") .c_str()); break; } case TRITONSERVER_INSTANCEGROUPKIND_GPU: { #if defined(TRITON_ENABLE_GPU) cudaDeviceProp cuprops; cudaError_t cuerr = cudaGetDeviceProperties(&cuprops, device_id_); if (cuerr != cudaSuccess) { throw BackendModelInstanceException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("unable to get CUDA device properties for ") + name_ + ": " + cudaGetErrorString(cuerr)) .c_str())); } const std::string cc = std::to_string(cuprops.major) + "." + std::to_string(cuprops.minor); common::TritonJson::Value cc_names; common::TritonJson::Value cc_name; if ((model_config.Find("cc_model_filenames", &cc_names)) && (cc_names.Find(cc.c_str(), &cc_name))) { cc_name.AsString(&artifact_filename_); } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Creating instance ") + name_ + " on GPU " + std::to_string(device_id_) + " (" + cc + ") using artifact '" + artifact_filename_ + "'") .c_str()); #elif !defined(TRITON_ENABLE_MALI_GPU) throw BackendModelInstanceException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "GPU instances not supported")); #endif // TRITON_ENABLE_GPU break; } default: { throw BackendModelInstanceException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("unexpected instance kind for ") + name_).c_str())); } } stream_ = nullptr; if (kind_ == TRITONSERVER_INSTANCEGROUPKIND_GPU) { THROW_IF_BACKEND_INSTANCE_ERROR( CreateCudaStream(device_id_, 0 /* cuda_stream_priority */, &stream_)); } // Get the host policy setting as a json string from message, // and extract the host policy name for the instance. TRITONSERVER_Message* message = nullptr; THROW_IF_BACKEND_MODEL_ERROR( TRITONBACKEND_ModelInstanceHostPolicy(triton_model_instance_, &message)); const char* buffer; size_t byte_size; THROW_IF_BACKEND_MODEL_ERROR( TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size)); common::TritonJson::Value host_policy; TRITONSERVER_Error* err = host_policy.Parse(buffer, byte_size); THROW_IF_BACKEND_MODEL_ERROR(err); std::vector host_policy_name; THROW_IF_BACKEND_MODEL_ERROR(host_policy.Members(&host_policy_name)); if (host_policy_name.size() != 1) { throw BackendModelInstanceException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("unexpected no host policy for ") + name_).c_str())); } host_policy_name_ = host_policy_name[0]; } BackendModelInstance::~BackendModelInstance() { #ifdef TRITON_ENABLE_GPU if (stream_ != nullptr) { cudaError_t err = cudaStreamDestroy(stream_); if (err != cudaSuccess) { TRITONSERVER_LogMessage( TRITONSERVER_LOG_ERROR, __FILE__, __LINE__, (std::string("~BackendModelInstance: ") + name_ + " failed to destroy cuda stream: " + cudaGetErrorString(err)) .c_str()); } stream_ = nullptr; } #endif // TRITON_ENABLE_GPU } }} // namespace triton::backend kytensor-2.49.0.7/backend/src/backend_output_responder.cc000066400000000000000000000547621527052371200234300ustar00rootroot00000000000000// Copyright 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/backend_output_responder.h" #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" namespace triton { namespace backend { // // BackendOutputResponder // BackendOutputResponder::~BackendOutputResponder() { for (auto& pinned_memory : pinned_memories_) { LOG_IF_ERROR( TRITONBACKEND_MemoryManagerFree( memory_manager_, reinterpret_cast(pinned_memory), TRITONSERVER_MEMORY_CPU_PINNED, 0), "failed to free pinned memory"); } } void BackendOutputResponder::ProcessTensor( const std::string& output_name, const TRITONSERVER_DataType datatype, std::vector& batchn_shape, const char* buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id) { // A value of CPU_PINNED indicates that pinned memory buffer is not // needed for this tensor. Any other value indicates that a pinned // memory buffer is needed when the target memory type matches // 'use_pinned_memory_type'. TRITONSERVER_MemoryType use_pinned_memory_type = TRITONSERVER_MEMORY_CPU_PINNED; if (pinned_enabled_) { use_pinned_memory_type = GetUsePinnedMemoryType(memory_type); } const int64_t batchn_batch_size = batchn_shape[0]; int64_t batch_size_offset = 0; size_t tensor_offset = 0; for (size_t idx = 0; idx < responses_->size(); idx++) { auto& request = requests_[idx]; auto& response = (*responses_)[idx]; // If then pending copies are from tensor buffer that is not // contiguous with 'response's part of that buffer, then need to // go ahead and perform the pending copies so that can start a // new contiguous region if necessary. if ((pending_pinned_byte_size_ > 0) && (tensor_offset != (pending_pinned_byte_size_ + pending_pinned_offset_))) { need_sync_ |= FlushPendingPinned(buffer, memory_type, memory_type_id); } // Override shape to be correct for this response. if (first_dim_batching_) { TRITONBACKEND_Input* input; TRITONBACKEND_RequestInputByIndex(request, 0, &input); const int64_t* shape; TRITONBACKEND_InputProperties( input, nullptr, nullptr, &shape, nullptr, nullptr, nullptr); if ((batchn_batch_size != -1) && ((batch_size_offset + shape[0]) > batchn_batch_size)) { if (response != nullptr) { RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string( GetRequestId(request) + "failed to split the output tensor '" + output_name + "' in responses: expected batch size of at least " + std::to_string(batch_size_offset + shape[0]) + " in model output, got " + std::to_string(batchn_batch_size)) .c_str())); } } batchn_shape[0] = shape[0]; batch_size_offset += shape[0]; } const size_t tensor_byte_size = GetByteSize(datatype, batchn_shape); TRITONBACKEND_Output* response_output; if (response != nullptr) { uint32_t output_count; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestOutputCount(request, &output_count)); if (response != nullptr) { for (uint32_t output_idx = 0; output_idx < output_count; output_idx++) { const char* name; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestOutputName(request, output_idx, &name)); if ((response != nullptr) && (output_name == name)) { RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_ResponseOutput( response, &response_output, name, datatype, batchn_shape.data(), batchn_shape.size())); if (response != nullptr) { need_sync_ |= SetFixedSizeBuffer( &response, response_output, output_name, tensor_byte_size, tensor_offset, buffer, memory_type, memory_type_id, use_pinned_memory_type, false /* state */); } break; } } } } tensor_offset += tensor_byte_size; } // Done with the tensor, flush any pending pinned copies. need_sync_ |= FlushPendingPinned(buffer, memory_type, memory_type_id); #ifdef TRITON_ENABLE_GPU if (need_sync_ && (event_ != nullptr)) { cudaEventRecord(event_, stream_); } #endif // TRITON_ENABLE_GPU } std::vector BackendOutputResponder::ProcessStateTensor( const std::string& output_state_name, const TRITONSERVER_DataType datatype, std::vector& batchn_shape, const char* buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id) { // A value of CPU_PINNED indicates that pinned memory buffer is not // needed for this tensor. Any other value indicates that a pinned // memory buffer is needed when the target memory type matches // 'use_pinned_memory_type'. TRITONSERVER_MemoryType use_pinned_memory_type = TRITONSERVER_MEMORY_CPU_PINNED; if (pinned_enabled_) { use_pinned_memory_type = GetUsePinnedMemoryType(memory_type); } std::vector states; const int64_t batchn_batch_size = batchn_shape[0]; int64_t batch_size_offset = 0; size_t tensor_offset = 0; for (size_t idx = 0; idx < responses_->size(); idx++) { auto& request = requests_[idx]; auto& response = (*responses_)[idx]; // If then pending copies are from tensor buffer that is not // contiguous with 'response's part of that buffer, then need to // go ahead and perform the pending copies so that can start a // new contiguous region if necessary. if ((pending_pinned_byte_size_ > 0) && (tensor_offset != (pending_pinned_byte_size_ + pending_pinned_offset_))) { need_sync_ |= FlushPendingPinned(buffer, memory_type, memory_type_id); } // Override shape to be correct for this response. if (first_dim_batching_) { TRITONBACKEND_Input* input; TRITONBACKEND_RequestInputByIndex(request, 0, &input); const int64_t* shape; TRITONBACKEND_InputProperties( input, nullptr, nullptr, &shape, nullptr, nullptr, nullptr); if ((batchn_batch_size != -1) && ((batch_size_offset + shape[0]) > batchn_batch_size)) { if (response != nullptr) { RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string( GetRequestId(request) + "failed to split the output state tensor '" + output_state_name + "' in responses: expected batch size of at least " + std::to_string(batch_size_offset + shape[0]) + " in model output, got " + std::to_string(batchn_batch_size)) .c_str())); } } batchn_shape[0] = shape[0]; batch_size_offset += shape[0]; } const size_t tensor_byte_size = GetByteSize(datatype, batchn_shape); TRITONBACKEND_State* output_state; if (response != nullptr) { RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_StateNew( &output_state, request, output_state_name.c_str(), datatype, batchn_shape.data(), batchn_shape.size())); if (response != nullptr) { states.push_back(output_state); need_sync_ |= SetFixedSizeBuffer( &response, output_state, output_state_name, tensor_byte_size, tensor_offset, buffer, memory_type, memory_type_id, use_pinned_memory_type, true /* state */); } } tensor_offset += tensor_byte_size; } // Done with the tensor, flush any pending pinned copies. need_sync_ |= FlushPendingPinned(buffer, memory_type, memory_type_id); #ifdef TRITON_ENABLE_GPU if (need_sync_ && (event_ != nullptr)) { cudaEventRecord(event_, stream_); } #endif // TRITON_ENABLE_GPU return states; } bool BackendOutputResponder::Finalize() { #ifdef TRITON_ENABLE_GPU if ((!deferred_pinned_.empty()) && need_sync_) { if (event_ != nullptr) { cudaEventSynchronize(event_); } else { cudaStreamSynchronize(stream_); } need_sync_ = false; } #endif // TRITON_ENABLE_GPU // After the above sync all the GPU->pinned copies are complete. Any // deferred copies of pinned->CPU can now be done. for (auto& def : deferred_pinned_) { auto pinned_memory_type = TRITONSERVER_MEMORY_CPU_PINNED; int64_t pinned_memory_id = 0; char* pinned_buffer = def.pinned_memory_; size_t offset = 0; for (auto& pr : def.responses_) { auto& response = pr.first; auto& response_output = pr.second; bool cuda_used = false; RESPOND_AND_SET_NULL_IF_ERROR( response, CopyBuffer( response_output.name_, pinned_memory_type, pinned_memory_id, response_output.memory_type_, response_output.memory_type_id_, response_output.buffer_byte_size_, pinned_buffer + offset, const_cast(response_output.buffer_), stream_, &cuda_used, copy_on_stream_)); need_sync_ |= cuda_used; offset += response_output.buffer_byte_size_; } } #ifdef TRITON_ENABLE_GPU // Record the new event location if deferred copies occur if ((!deferred_pinned_.empty()) && need_sync_ && (event_ != nullptr)) { cudaEventRecord(event_, stream_); } #endif // TRITON_ENABLE_GPU deferred_pinned_.clear(); return need_sync_; } bool BackendOutputResponder::SetFixedSizeBuffer( TRITONBACKEND_Response** response, void* response_output_or_state, const std::string& output_name, const size_t tensor_byte_size, const size_t tensor_offset, const char* tensor_buffer, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id, const TRITONSERVER_MemoryType use_pinned_memory_type, bool state) { void* buffer = nullptr; bool cuda_copy = false; TRITONSERVER_MemoryType actual_memory_type = tensor_memory_type; int64_t actual_memory_type_id = tensor_memory_type_id; if (state) { TRITONBACKEND_State* response_state = reinterpret_cast(response_output_or_state); auto err = TRITONBACKEND_StateBuffer( response_state, &buffer, tensor_byte_size, &actual_memory_type, &actual_memory_type_id); if (err != nullptr) { RESPOND_AND_SET_NULL_IF_ERROR(response, err); return cuda_copy; } } else { TRITONBACKEND_Output* response_output = reinterpret_cast(response_output_or_state); auto err = TRITONBACKEND_OutputBuffer( response_output, &buffer, tensor_byte_size, &actual_memory_type, &actual_memory_type_id); if (err != nullptr) { RESPOND_AND_SET_NULL_IF_ERROR(response, err); return cuda_copy; } } // If the response buffer matches the memory type that should use an // intermediate pinned memory buffer for the transfer, then just // record the response as pending and increase the size required for // the intermediate pinned buffer. if ((use_pinned_memory_type != TRITONSERVER_MEMORY_CPU_PINNED) && (actual_memory_type == use_pinned_memory_type)) { if (pending_pinned_byte_size_ == 0) { pending_pinned_offset_ = tensor_offset; } pending_pinned_byte_size_ += tensor_byte_size; pending_pinned_outputs_.push_back(std::make_pair( response, OutputData( output_name, buffer, tensor_byte_size, actual_memory_type, actual_memory_type_id))); } else { // Direct copy without intermediate pinned memory. bool cuda_used = false; auto err = CopyBuffer( output_name, tensor_memory_type, tensor_memory_type_id, actual_memory_type, actual_memory_type_id, tensor_byte_size, tensor_buffer + tensor_offset, buffer, stream_, &cuda_used, copy_on_stream_); cuda_copy |= cuda_used; if (err != nullptr) { RESPOND_AND_SET_NULL_IF_ERROR(response, err); return cuda_copy; } } return cuda_copy; } bool BackendOutputResponder::FlushPendingPinned( const char* tensor_buffer, const TRITONSERVER_MemoryType tensor_memory_type, const int64_t tensor_memory_type_id) { bool cuda_copy = false; // Will be copying from CPU->pinned->GPU or GPU->pinned->CPU // Attempt to allocate a pinned buffer to use for staging the // copy... if we fail to allocated the pinned buffer then we just // directly go CPU->GPU or GPU->CPU. char* pinned_memory = nullptr; if (pending_pinned_byte_size_ > 0) { TRITONSERVER_Error* err = TRITONBACKEND_MemoryManagerAllocate( memory_manager_, reinterpret_cast(&pinned_memory), TRITONSERVER_MEMORY_CPU_PINNED, 0 /* memory_type_id */, pending_pinned_byte_size_); if (err != nullptr) { pinned_memory = nullptr; TRITONSERVER_ErrorDelete(err); } } // If the pinned buffer wasn't actually allocated then just perform // a direct copy. if (pinned_memory == nullptr) { size_t offset = 0; for (auto& pr : pending_pinned_outputs_) { auto& response = pr.first; auto& response_output = pr.second; bool cuda_used = false; RESPOND_AND_SET_NULL_IF_ERROR( response, CopyBuffer( response_output.name_, tensor_memory_type, tensor_memory_type_id, response_output.memory_type_, response_output.memory_type_id_, response_output.buffer_byte_size_, tensor_buffer + pending_pinned_offset_ + offset, const_cast(response_output.buffer_), stream_, &cuda_used, copy_on_stream_)); cuda_copy |= cuda_used; offset += response_output.buffer_byte_size_; } } // We have a pinned buffer so do a single copy of a block of tensor // data to the pinned buffer. else { // pinned_memory_type == TRITONSERVER_MEMORY_CPU_PINNED bool cuda_used = false; auto err = CopyBuffer( "pinned buffer", tensor_memory_type, tensor_memory_type_id, TRITONSERVER_MEMORY_CPU_PINNED, 0 /* memory_type_id */, pending_pinned_byte_size_, tensor_buffer + pending_pinned_offset_, pinned_memory, stream_, &cuda_used, copy_on_stream_); cuda_copy |= cuda_used; // If something goes wrong with the copy all the pending // responses fail... if (err != nullptr) { for (auto& pr : pending_pinned_outputs_) { auto& response = pr.first; if (*response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( *response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err), "failed to send TensorFlow error response"); *response = nullptr; } } TRITONSERVER_ErrorDelete(err); } // If the copy was not async (i.e. if tensor was in CPU so a // CPU->CPU-PINNED copy was performed above), then the pinned // buffer now holds the tensor contents and we can immediately // issue the copies from the pinned buffer to the // responses. // // Otherwise the GPU->CPU-PINNED async copies are in flight and we // simply remember the pinned buffer and the corresponding // response outputs so that we can do the pinned->CPU copies in // finalize after we have waited for all async copies to complete. if (!cuda_used) { size_t offset = 0; for (auto& pr : pending_pinned_outputs_) { auto& response = pr.first; auto& response_output = pr.second; bool cuda_used = false; RESPOND_AND_SET_NULL_IF_ERROR( response, CopyBuffer( response_output.name_, TRITONSERVER_MEMORY_CPU_PINNED, 0 /* memory_type_id */, response_output.memory_type_, response_output.memory_type_id_, response_output.buffer_byte_size_, pinned_memory + offset, const_cast(response_output.buffer_), stream_, &cuda_used, copy_on_stream_)); cuda_copy |= cuda_used; offset += response_output.buffer_byte_size_; } } else { deferred_pinned_.emplace_back( pinned_memory, pending_pinned_byte_size_, std::move(pending_pinned_outputs_)); } } // Pending pinned copies are handled... pending_pinned_byte_size_ = 0; pending_pinned_offset_ = 0; pending_pinned_outputs_.clear(); // Need to hold on to the allocated pinned buffer as there are still // copies in flight. Will delete it in finalize. if (pinned_memory != nullptr) { pinned_memories_.push_back(pinned_memory); } return cuda_copy; } void BackendOutputResponder::ProcessBatchOutput( const std::string& name, const BatchOutput& batch_output, const char* buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id) { // A value of CPU_PINNED indicates that pinned memory buffer is not // needed for this tensor. Any other value indicates that a pinned // memory buffer is needed when the target memory type matches // 'use_pinned_memory_type'. TRITONSERVER_MemoryType use_pinned_memory_type = TRITONSERVER_MEMORY_CPU_PINNED; if (pinned_enabled_) { use_pinned_memory_type = GetUsePinnedMemoryType(memory_type); } // Batch output may be processed differently based on the kind switch (batch_output.BatchOutputKind()) { case BatchOutput::Kind::BATCH_SCATTER_WITH_INPUT_SHAPE: { const auto& output_name = batch_output.TargetNames()[0]; const auto& input_name = batch_output.SourceInputs()[0]; const auto& datatype = batch_output.DataType(); size_t tensor_offset = 0; for (size_t idx = 0; idx < responses_->size(); idx++) { auto& request = requests_[idx]; auto& response = (*responses_)[idx]; // If then pending copies are from tensor buffer that is not // contiguous with 'response's part of that buffer, then need to // go ahead and perform the pending copies so that can start a // new contiguous region if necessary. if ((pending_pinned_byte_size_ > 0) && (tensor_offset != (pending_pinned_byte_size_ + pending_pinned_offset_))) { need_sync_ |= FlushPendingPinned(buffer, memory_type, memory_type_id); } // Override shape to be correct for this response, with a naive // assumption that the dynamic dimension in output is mapped to the same // dimension in the input auto output_batchn_shape = batch_output.OutputShape(); { TRITONBACKEND_Input* input; TRITONBACKEND_RequestInput(request, input_name.c_str(), &input); const int64_t* shape; TRITONBACKEND_InputProperties( input, nullptr, nullptr, &shape, nullptr, nullptr, nullptr); for (size_t dim_idx = 0; dim_idx < output_batchn_shape.size(); dim_idx++) { if (output_batchn_shape[dim_idx] == -1) { output_batchn_shape[dim_idx] = shape[dim_idx]; } } } const size_t tensor_byte_size = GetByteSize(datatype, output_batchn_shape); TRITONBACKEND_Output* response_output; if (response != nullptr) { uint32_t output_count; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestOutputCount(request, &output_count)); if (response != nullptr) { for (uint32_t output_idx = 0; output_idx < output_count; output_idx++) { const char* name; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestOutputName(request, output_idx, &name)); if ((response != nullptr) && (output_name == name)) { RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_ResponseOutput( response, &response_output, name, datatype, output_batchn_shape.data(), output_batchn_shape.size())); if (response != nullptr) { need_sync_ |= SetFixedSizeBuffer( &response, response_output, output_name, tensor_byte_size, tensor_offset, buffer, memory_type, memory_type_id, use_pinned_memory_type, false /* state */); } break; } } } } tensor_offset += tensor_byte_size; } break; } } // Done with the tensor, flush any pending pinned copies. need_sync_ |= FlushPendingPinned(buffer, memory_type, memory_type_id); #ifdef TRITON_ENABLE_GPU if (need_sync_ && (event_ != nullptr)) { cudaEventRecord(event_, stream_); } #endif // TRITON_ENABLE_GPU } }} // namespace triton::backend kytensor-2.49.0.7/backend/src/device_memory_tracker.cc000066400000000000000000000257041527052371200226740ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/backend/device_memory_tracker.h" #include #include #include "triton/core/tritonserver.h" namespace triton { namespace backend { std::unique_ptr DeviceMemoryTracker::tracker_{nullptr}; // Boilerplate from CUPTI examples namespace { #define LOG_IF_CUPTI_ERR(call) \ do { \ CUptiResult _status = call; \ if (_status != CUPTI_SUCCESS) { \ const char* errstr; \ cuptiGetResultString(_status, &errstr); \ LOG_ERROR << #call << " failed with error: " << errstr; \ } \ } while (0) #define THROW_IF_CUPTI_ERR(call) \ do { \ CUptiResult _status = call; \ if (_status != CUPTI_SUCCESS) { \ const char* errstr; \ cuptiGetResultString(_status, &errstr); \ throw std::runtime_error( \ std::string(#call) + " failed with error: " + errstr); \ } \ } while (0) #define BUF_SIZE (32 * 1024) #define ALIGN_SIZE (8) #define ALIGN_BUFFER(buffer, align) \ (((uintptr_t)(buffer) & ((align)-1)) \ ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) \ : (buffer)) void bufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords) { uint8_t* bfr = (uint8_t*)malloc(BUF_SIZE + ALIGN_SIZE); if (bfr != nullptr) { *size = BUF_SIZE; *buffer = ALIGN_BUFFER(bfr, ALIGN_SIZE); *maxNumRecords = 0; } else { LOG_ERROR << "Failed to allocate buffer for CUPTI: out of memory"; } } void bufferCompleted( CUcontext ctx, uint32_t streamId, uint8_t* buffer, size_t size, size_t validSize) { CUptiResult status; CUpti_Activity* record = nullptr; if (validSize > 0) { do { status = cuptiActivityGetNextRecord(buffer, validSize, &record); if (status == CUPTI_SUCCESS) { DeviceMemoryTracker::TrackActivity(record); } else if (status == CUPTI_ERROR_MAX_LIMIT_REACHED) break; else { LOG_IF_CUPTI_ERR(status); } } while (1); // report any records dropped from the queue size_t dropped = 0; LOG_IF_CUPTI_ERR( cuptiActivityGetNumDroppedRecords(ctx, streamId, &dropped)); if (dropped != 0) { LOG_WARNING << "Dropped " << dropped << " activity records"; } } free(buffer); } } // namespace DeviceMemoryTracker::DeviceMemoryTracker() { cudaError_t cuerr = cudaGetDeviceCount(&device_cnt_); if ((cuerr == cudaErrorNoDevice) || (cuerr == cudaErrorInsufficientDriver)) { device_cnt_ = 0; } else if (cuerr != cudaSuccess) { throw std::runtime_error( "Unexpected failure on getting CUDA device count."); } // Use 'cuptiSubscribe' to check if the cupti has been initialized // elsewhere. Due to cupti limitation, there can only be one cupti client // within the process, so in the case of per-backend memory tracking, we // have to make the assumption that the other cupti client is using the same // memory tracker implementation so that the backend may use the cupti // configuration that is external to the backend without issue. auto cupti_res = cuptiSubscribe(&subscriber_, nullptr, nullptr); switch (cupti_res) { case CUPTI_SUCCESS: { THROW_IF_CUPTI_ERR( cuptiActivityRegisterCallbacks(bufferRequested, bufferCompleted)); THROW_IF_CUPTI_ERR(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME)); THROW_IF_CUPTI_ERR(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMORY2)); THROW_IF_CUPTI_ERR( cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION)); break; } case CUPTI_ERROR_MULTIPLE_SUBSCRIBERS_NOT_SUPPORTED: { LOG_WARNING << "CUPTI has been initialized elsewhere, assuming the " "implementation is the same"; break; } default: { // other error, should propagate and disable memory tracking for the // backend const char* errstr; cuptiGetResultString(cupti_res, &errstr); throw std::runtime_error( std::string("Unexpected failure on configuring CUPTI: ") + errstr); } } } DeviceMemoryTracker::~DeviceMemoryTracker() { if (subscriber_) { cuptiUnsubscribe(subscriber_); } } int DeviceMemoryTracker::CudaDeviceCount() { if (tracker_) { return tracker_->device_cnt_; } throw std::runtime_error( "DeviceMemoryTracker::Init() must be called before using any " "DeviceMemoryTracker features."); } bool DeviceMemoryTracker::Init() { if (tracker_ == nullptr) { try { tracker_.reset(new DeviceMemoryTracker()); } catch (const std::runtime_error& ex) { // Fail initialization LOG_ERROR << ex.what(); return false; } } return true; } void DeviceMemoryTracker::Fini() { tracker_.reset(); } void DeviceMemoryTracker::TrackThreadMemoryUsage(MemoryUsage* usage) { if (!usage) { return; } if (tracker_) { THROW_IF_CUPTI_ERR(cuptiActivityPushExternalCorrelationId( CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, reinterpret_cast(&usage->cupti_tracker_))); usage->tracked_ = true; } else { throw std::runtime_error( "DeviceMemoryTracker::Init() must be called before using any " "DeviceMemoryTracker features."); } } void DeviceMemoryTracker::UntrackThreadMemoryUsage(MemoryUsage* usage) { if (!usage) { return; } if (tracker_) { THROW_IF_CUPTI_ERR(cuptiActivityFlushAll(0)); uint64_t id = 0; THROW_IF_CUPTI_ERR(cuptiActivityPopExternalCorrelationId( CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &id)); usage->tracked_ = false; } else { throw std::runtime_error( "DeviceMemoryTracker::Init() must be called before using any " "DeviceMemoryTracker features."); } } void DeviceMemoryTracker::TrackActivityInternal(CUpti_Activity* record) { switch (record->kind) { case CUPTI_ACTIVITY_KIND_MEMORY2: { CUpti_ActivityMemory3* memory_record = (CUpti_ActivityMemory3*)record; TRITONBACKEND_CuptiTracker* usage = nullptr; { std::lock_guard lk(mtx_); auto it = activity_to_memory_usage_.find(memory_record->correlationId); if (it != activity_to_memory_usage_.end()) { usage = reinterpret_cast(it->second); activity_to_memory_usage_.erase(it); } } const bool is_allocation = (memory_record->memoryOperationType == CUPTI_ACTIVITY_MEMORY_OPERATION_TYPE_ALLOCATION); const bool is_release = (memory_record->memoryOperationType == CUPTI_ACTIVITY_MEMORY_OPERATION_TYPE_RELEASE); // Ignore memory record that is not associated with a // TRITONBACKEND_CuptiTracker object or not related to allocations if ((usage == nullptr) || (!usage->valid_) || (!is_allocation && !is_release)) { break; } switch (memory_record->memoryKind) { case CUPTI_ACTIVITY_MEMORY_KIND_DEVICE: { usage->valid_ = UpdateMemoryTypeUsage( memory_record, is_allocation, usage->cuda_memory_usage_byte_, usage->cuda_array_len_); break; } case CUPTI_ACTIVITY_MEMORY_KIND_PINNED: { usage->valid_ = UpdateMemoryTypeUsage( memory_record, is_allocation, usage->pinned_memory_usage_byte_, usage->pinned_array_len_); break; } case CUPTI_ACTIVITY_MEMORY_KIND_PAGEABLE: { usage->valid_ = UpdateMemoryTypeUsage( memory_record, is_allocation, usage->system_memory_usage_byte_, usage->system_array_len_); break; } default: LOG_WARNING << "Unrecognized type of memory is allocated, kind " << memory_record->memoryKind; usage->valid_ = false; break; } break; } case CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION: { CUpti_ActivityExternalCorrelation* corr = (CUpti_ActivityExternalCorrelation*)record; if (CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN == corr->externalKind) { std::lock_guard lk(mtx_); activity_to_memory_usage_[corr->correlationId] = static_cast(corr->externalId); } break; } case CUPTI_ACTIVITY_KIND_RUNTIME: { // DO NOTHING, runtime API will be captured and reported to properly // initialize records for CUPTI_ACTIVITY_KIND_MEMORY2. break; } default: LOG_ERROR << "Unexpected capture of cupti record, kind: " << record->kind; break; } } inline bool DeviceMemoryTracker::UpdateMemoryTypeUsage( CUpti_ActivityMemory3* memory_record, const bool is_allocation, int64_t* memory_usage, uint32_t usage_len) { if (memory_record->deviceId >= usage_len) { return false; } if (is_allocation) { memory_usage[memory_record->deviceId] += memory_record->bytes; } else { memory_usage[memory_record->deviceId] -= memory_record->bytes; } return true; } }} // namespace triton::backend kytensor-2.49.0.7/backend/src/kernel.cu000066400000000000000000000063751527052371200176370ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include "kernel.h" #define THREADBLOCK_SIZE 512 __launch_bounds__(THREADBLOCK_SIZE) __global__ void TritonGatherKernel( const int8_t** __restrict input_ptr_buffer, const size_t* __restrict byte_size_buffer, const size_t* __restrict byte_size_offset_buffer, int8_t* __restrict output_buffer) { int request_idx = blockIdx.x; int lane_id = threadIdx.x; const int8_t* request_input_buffer = input_ptr_buffer[request_idx]; int byte_size = byte_size_buffer[request_idx]; int byte_size_offset = byte_size_offset_buffer[request_idx]; int8_t* output_buffer_with_offset = output_buffer + byte_size_offset; if (((byte_size % 4) == 0) && (((uint64_t)request_input_buffer % 4) == 0) && (((uint64_t)output_buffer_with_offset % 4) == 0)) { int32_t* input_4 = (int32_t*)request_input_buffer; int32_t* output_4 = (int32_t*)output_buffer_with_offset; int element_count = byte_size / 4; for (int elem_id = lane_id; elem_id < element_count; elem_id += THREADBLOCK_SIZE) { output_4[elem_id] = input_4[elem_id]; } } else { for (int elem_id = lane_id; elem_id < byte_size; elem_id += THREADBLOCK_SIZE) { output_buffer_with_offset[elem_id] = __ldg(request_input_buffer + elem_id); } } } #ifdef __cplusplus extern "C" { #endif cudaError_t RunGatherKernel( const int8_t** input_ptr_buffer, const size_t* byte_size_buffer, const size_t* byte_size_offset_buffer, int8_t* output_buffer, size_t request_count, cudaStream_t stream) { TritonGatherKernel<<>>( input_ptr_buffer, byte_size_buffer, byte_size_offset_buffer, output_buffer); return cudaGetLastError(); } #ifdef __cplusplus } #endif kytensor-2.49.0.7/backend/src/kernel.h000066400000000000000000000035601527052371200174500ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #ifdef __cplusplus extern "C" { #endif cudaError_t RunGatherKernel( const int8_t** input_ptr_buffer, const size_t* byte_size_buffer, const size_t* byte_size_offset_buffer, int8_t* output_buffer, size_t request_count, cudaStream_t stream); #ifdef __cplusplus } #endif kytensor-2.49.0.7/client/000077500000000000000000000000001527052371200151135ustar00rootroot00000000000000kytensor-2.49.0.7/client/.clang-format000066400000000000000000000014271527052371200174720ustar00rootroot00000000000000--- BasedOnStyle: Google IndentWidth: 2 ColumnLimit: 80 ContinuationIndentWidth: 4 UseTab: Never MaxEmptyLinesToKeep: 2 SortIncludes: true CompactNamespaces: true ReflowComments: true DerivePointerAlignment: false PointerAlignment: Left AllowShortIfStatementsOnASingleLine: false AllowShortBlocksOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AlwaysBreakAfterReturnType: TopLevelDefinitions AlignAfterOpenBracket: AlwaysBreak BreakBeforeBraces: Custom BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: true AfterNamespace: false AfterStruct: false AfterUnion: false BeforeCatch: true BinPackArguments: true BinPackParameters: true ConstructorInitializerAllOnOneLineOrOnePerLine: false IndentCaseLabels: true kytensor-2.49.0.7/client/.github/000077500000000000000000000000001527052371200164535ustar00rootroot00000000000000kytensor-2.49.0.7/client/.github/workflows/000077500000000000000000000000001527052371200205105ustar00rootroot00000000000000kytensor-2.49.0.7/client/.github/workflows/codeql.yml000066400000000000000000000071451527052371200225110ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "CodeQL" on: pull_request: jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to: # https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs queries: +security-and-quality # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 # Command-line programs to run using the OS shell. # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | # echo "Run, Build Application using script" # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: category: "/language:${{matrix.language}}" kytensor-2.49.0.7/client/.github/workflows/pre-commit.yml000066400000000000000000000033201527052371200233050ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: pre-commit on: pull_request: jobs: pre-commit: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - uses: pre-commit/action@v3.0.0 kytensor-2.49.0.7/client/.pre-commit-config.yaml000066400000000000000000000062621527052371200214020ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. exclude: ^src/grpc_generated repos: - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black rev: 24.4.0 hooks: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] types_or: [python, cython] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.5 hooks: - id: clang-format types_or: [c, c++, cuda, proto, textproto, java] args: ["-fallback-style=none", "-style=file", "-i"] - repo: https://github.com/codespell-project/codespell rev: v2.2.4 hooks: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$) # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-json - id: check-toml - id: check-yaml - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.9.0 hooks: - id: mypy types_or: [python, cython] additional_dependencies: [tokenize-rt==3.2.0, types-requests] exclude: >- (?x) ^(?!(src/c\+\+/perf_analyzer/genai-perf/)).*\.py$ kytensor-2.49.0.7/client/CMakeLists.txt000066400000000000000000000231661527052371200176630ustar00rootroot00000000000000# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonclient LANGUAGES C CXX) # Use C++17 standard as Triton's minimum required. set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.") set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) # # Options # set(TRITON_VERSION "0.0.0" CACHE STRING "Version for the clients") option(TRITON_ENABLE_CC_HTTP "Build C++ HTTP client libraries" OFF) option(TRITON_ENABLE_CC_GRPC "Build C++ GRPC client libraries" OFF) option(TRITON_ENABLE_PYTHON_HTTP "Enable Python HTTP client libraries" OFF) option(TRITON_ENABLE_PYTHON_GRPC "Enable Python GRPC client libraries" OFF) option(TRITON_ENABLE_JAVA_HTTP "Enable JAVA HTTP client libraries" OFF) option(TRITON_ENABLE_EXAMPLES "Include examples in build" OFF) option(TRITON_ENABLE_TESTS "Include tests in build" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in libraries" OFF) option(TRITON_ENABLE_ZLIB "Include ZLIB library in build" ON) # Package Perf Analyzer with the Python Client -- Intended to be used by PA repo option(TRITON_PACKAGE_PERF_ANALYZER "Include Perf Analyzer in pip wheel" OFF) set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_THIRD_PARTY_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/third_party repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() if(NOT TRITON_ENABLE_PYTHON_GRPC) set(TRITON_COMMON_ENABLE_PROTOBUF_PYTHON OFF) endif() # # Dependencies # include(FetchContent) FetchContent_Declare( repo-third-party GIT_REPOSITORY ${TRITON_REPO_ORGANIZATION}/third_party.git GIT_TAG ${TRITON_THIRD_PARTY_REPO_TAG} GIT_SHALLOW ON ) set(TRITON_THIRD_PARTY_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/third-party) FetchContent_MakeAvailable(repo-third-party) # Some libs are installed to ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib64 instead # of ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib on Centos set (LIB_DIR "lib") # /etc/os-release does not exist on Windows if(EXISTS "/etc/os-release") file(STRINGS /etc/os-release DISTRO REGEX "^NAME=") string(REGEX REPLACE "NAME=\"(.*)\"" "\\1" DISTRO "${DISTRO}") message(STATUS "Distro Name: ${DISTRO}") if(DISTRO MATCHES "CentOS.*") set (LIB_DIR "lib64") endif() endif() # Need to use ExternalProject for our builds so that we can get the # correct dependencies between our components and the ExternalProject # dependencies (found in the third_party repo) include(ExternalProject) if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(TRITON_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/cc-clients/install) else() set(TRITON_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) endif() set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "") if (OPENSSL_ROOT_DIR) set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") endif() set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "") if (CMAKE_TOOLCHAIN_FILE) set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "-DCMAKE_TOOLCHAIN_FILE:PATH=${CMAKE_TOOLCHAIN_FILE}") endif() set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "") if (VCPKG_TARGET_TRIPLET) set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "-DVCPKG_TARGET_TRIPLET:STRING=${VCPKG_TARGET_TRIPLET}") endif() # Location where protobuf-config.cmake will be installed varies by # platform if (WIN32) set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/cmake") else() set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf") endif() if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC) set(_cc_client_depends re2) if(${TRITON_ENABLE_CC_HTTP}) set(_cc_client_depends ${_cc_client_depends} curl) endif() # TRITON_ENABLE_CC_HTTP if(${TRITON_ENABLE_CC_GRPC}) set(_cc_client_depends ${_cc_client_depends} grpc protobuf) endif() # TRITON_ENABLE_CC_GRPC ExternalProject_Add(cc-clients PREFIX cc-clients SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/c++" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/cc-clients" CMAKE_CACHE_ARGS ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DCURL_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/curl/${LIB_DIR}/cmake/CURL -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} -DgRPC_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc/lib/cmake/grpc -Dabsl_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl/${LIB_DIR}/cmake/absl -Dre2_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/re2/${LIB_DIR}/cmake/re2 -Dc-ares_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares/${LIB_DIR}/cmake/c-ares -DGTEST_ROOT:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/googletest -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} -DTRITON_ENABLE_CC_HTTP:BOOL=${TRITON_ENABLE_CC_HTTP} -DTRITON_ENABLE_CC_GRPC:BOOL=${TRITON_ENABLE_CC_GRPC} -DTRITON_ENABLE_EXAMPLES:BOOL=${TRITON_ENABLE_EXAMPLES} -DTRITON_ENABLE_TESTS:BOOL=${TRITON_ENABLE_TESTS} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DTRITON_ENABLE_ZLIB:BOOL=${TRITON_ENABLE_ZLIB} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} DEPENDS ${_cc_client_depends} ) endif() # TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC if(TRITON_ENABLE_PYTHON_HTTP OR TRITON_ENABLE_PYTHON_GRPC) set(_py_client_depends re2) if(${TRITON_ENABLE_PYTHON_GRPC}) set(_py_client_depends ${_py_client_depends} grpc protobuf) endif() # TRITON_ENABLE_PYTHON_GRPC ExternalProject_Add(python-clients PREFIX python-clients SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/python" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/python-clients" CMAKE_CACHE_ARGS ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} -DgRPC_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc/lib/cmake/grpc -Dabsl_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl/${LIB_DIR}/cmake/absl -Dc-ares_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares/${LIB_DIR}/cmake/c-ares -Dre2_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/re2/${LIB_DIR}/cmake/re2 -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} -DTRITON_VERSION:STRING=${TRITON_VERSION} -DTRITON_ENABLE_PYTHON_HTTP:BOOL=${TRITON_ENABLE_PYTHON_HTTP} -DTRITON_ENABLE_PYTHON_GRPC:BOOL=${TRITON_ENABLE_PYTHON_GRPC} -DTRITON_ENABLE_EXAMPLES:BOOL=${TRITON_ENABLE_EXAMPLES} -DTRITON_ENABLE_TESTS:BOOL=${TRITON_ENABLE_TESTS} -DTRITON_PACKAGE_PERF_ANALYZER:BOOL=${TRITON_PACKAGE_PERF_ANALYZER} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} DEPENDS ${_py_client_depends} ) endif() # TRITON_ENABLE_PYTHON_HTTP OR TRITON_ENABLE_PYTHON_GRPC if(TRITON_ENABLE_JAVA_HTTP) ExternalProject_Add(java-clients PREFIX java-clients SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/java" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/java-clients" CMAKE_CACHE_ARGS ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_VERSION:STRING=${TRITON_VERSION} -DTRITON_ENABLE_JAVA_HTTP:BOOL=${TRITON_ENABLE_JAVA_HTTP} -DTRITON_ENABLE_EXAMPLES:BOOL=${TRITON_ENABLE_EXAMPLES} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} INSTALL_COMMAND "" ) endif() # TRITON_ENABLE_JAVA_HTTP kytensor-2.49.0.7/client/LICENSE000066400000000000000000000027151527052371200161250ustar00rootroot00000000000000Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/client/README.md000066400000000000000000001002641527052371200163750ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Triton Client Libraries and Examples To simplify communication with Triton, the Triton project provides several client libraries and examples of how to use those libraries. Ask questions or report problems in the main Triton [issues page](https://github.com/triton-inference-server/server/issues). The provided client libraries are: * [C++ and Python APIs](#client-library-apis) that make it easy to communicate with Triton from your C++ or Python application. Using these libraries you can send either HTTP/REST or GRPC requests to Triton to access all its capabilities: inferencing, status and health, statistics and metrics, model repository management, etc. These libraries also support using system and CUDA shared memory for passing inputs to and receiving outputs from Triton. * [Java API](#client-library-apis) (contributed by Alibaba Cloud PAI Team) that makes it easy to communicate with Triton from your Java application using HTTP/REST requests. For now, only a limited feature subset is supported. * The [protoc compiler](https://developers.google.com/protocol-buffers/docs/tutorials) can generate a GRPC API in a large number of programming languages. * See [src/grpc_generated/go](src/grpc_generated/go) for an example for the [Go programming language](https://golang.org/). * See [src/grpc_generated/java](src/grpc_generated/java) for an example for the Java and Scala programming languages. * See [src/grpc_generated/javascript](src/grpc_generated/javascript) for an example with JavaScript programming language. There are also many example applications that show how to use these libraries. Many of these examples use models from the [example model repository](https://github.com/triton-inference-server/server/blob/main/docs/getting_started/quickstart.md#create-a-model-repository). * C++ and Python versions of *image_client*, an example application that uses the C++ or Python client library to execute image classification models on Triton. See [Image Classification Example](#image-classification-example). * Several simple [C++ examples](src/c%2B%2B/examples) show how to use the C++ library to communicate with Triton to perform inferencing and other task. The C++ examples demonstrating the HTTP/REST client are named with a *simple_http_* prefix and the examples demonstrating the GRPC client are named with a *simple_grpc_* prefix. See [Simple Example Applications](#simple-example-applications). * Several simple [Python examples](src/python/examples) show how to use the Python library to communicate with Triton to perform inferencing and other task. The Python examples demonstrating the HTTP/REST client are named with a *simple_http_* prefix and the examples demonstrating the GRPC client are named with a *simple_grpc_* prefix. See [Simple Example Applications](#simple-example-applications). * Several simple [Java examples](src/java/src/main/java/triton/client/examples) show how to use the Java API to communicate with Triton to perform inferencing and other task. * A couple of [Python examples that communicate with Triton using a Python GRPC API](src/python/examples) generated by the [protoc compiler](https://grpc.io/docs/guides/). *grpc_client.py* is a simple example that shows simple API usage. *grpc_image_client.py* is functionally equivalent to *image_client* but that uses a generated GRPC client stub to communicate with Triton. ## Getting the Client Libraries And Examples The easiest way to get the Python client library is to [use pip to install the tritonclient module](#download-using-python-package-installer-pip). You can also download the C++, Python and Java client libraries from [Triton GitHub release](#download-from-github), or [download a pre-built Docker image containing the client libraries](#download-docker-image-from-ngc) from [NVIDIA GPU Cloud (NGC)](https://ngc.nvidia.com). It is also possible to build the client libraries with [cmake](#build-using-cmake). ### Download Using Python Package Installer (pip) The GRPC and HTTP client libraries are available as a Python package that can be installed using a recent version of pip. ``` $ pip install tritonclient[all] ``` Using *all* installs both the HTTP/REST and GRPC client libraries. There are two optional packages available, *grpc* and *http* that can be used to install support specifically for the protocol. For example, to install only the HTTP/REST client library use, ``` $ pip install tritonclient[http] ``` There is another optional package namely *cuda*, that must be installed in order to use cuda_shared_memory utilities. *all* specification will install the *cuda* package by default but in other cases *cuda* needs to be explicitly specified for installing client with cuda_shared_memory support. ``` $ pip install tritonclient[http, cuda] ``` The components of the install packages are: * http * grpc [ `service_pb2`, `service_pb2_grpc`, `model_config_pb2` ] * utils [ linux distribution will include `shared_memory` and `cuda_shared_memory`] ### Download From GitHub The client libraries can be downloaded from the [Triton GitHub release page](https://github.com/triton-inference-server/server/releases) corresponding to the release you are interested in. The client libraries are found in the "Assets" section of the release page in a tar file named after the version of the release and the OS, for example, v2.3.0_ubuntu2004.clients.tar.gz. The pre-built libraries can be used on the corresponding host system or you can install them into the Triton container to have both the clients and server in the same container. ```bash $ mkdir clients $ cd clients $ wget https://github.com/triton-inference-server/server/releases/download/ $ tar xzf ``` After installing, the libraries can be found in lib/, the headers in include/, the Python wheel files in python/, and the jar files in java/. The bin/ and python/ directories contain the built examples that you can learn more about below. ### Download Docker Image From NGC A Docker image containing the client libraries and examples is available from [NVIDIA GPU Cloud (NGC)](https://ngc.nvidia.com). Before attempting to pull the container ensure you have access to NGC. For step-by-step instructions, see the [NGC Getting Started Guide](http://docs.nvidia.com/ngc/ngc-getting-started-guide/index.html). Use docker pull to get the client libraries and examples container from NGC. ```bash $ docker pull nvcr.io/nvidia/tritonserver:-py3-sdk ``` Where \ is the version that you want to pull. Within the container the client libraries are in /workspace/install/lib, the corresponding headers in /workspace/install/include, and the Python wheel files in /workspace/install/python. The image will also contain the built client examples. **Important Note:** When running either the server or the client using Docker containers and using the [CUDA shared memory feature](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_shared_memory.md#cuda-shared-memory) you need to add `--pid host` flag when launching the containers. The reason is that CUDA IPC APIs require the PID of the source and destination of the exported pointer to be different. Otherwise, Docker enables PID namespace which may result in equality between the source and destination PIDs. The error will be always observed when both of the containers are started in the non-interactive mode. ### Build Using CMake The client library build is performed using CMake. To build the client libraries and examples with all features, first change directory to the root of this repo and checkout the release version of the branch that you want to build (or the *main* branch if you want to build the under-development version). ```bash $ git checkout main ``` If building the Java client you must first install Maven and a JDK appropriate for your OS. For example, for Ubuntu you should install the `default-jdk` package: ``` $ apt-get install default-jdk maven ``` Building on Windows vs. non-Windows requires different invocations because Triton on Windows does not yet support all the build options. #### Non-Windows Use *cmake* to configure the build. You should adjust the flags depending on the components of Triton Client you are working and would like to build. If you are building on a release branch (or on a development branch that is based off of a release branch), then you must also use additional cmake arguments to point to that release branch for repos that the client build depends on. For example, if you are building the r21.10 client branch then you need to use the following additional cmake flags: ``` -DTRITON_COMMON_REPO_TAG=r21.10 -DTRITON_THIRD_PARTY_REPO_TAG=r21.10 -DTRITON_CORE_REPO_TAG=r21.10 ``` Then use *make* to build the clients and examples. ``` $ make cc-clients python-clients java-clients ``` When the build completes the libraries and examples can be found in the install directory. #### Windows To build the clients you must install an appropriate C++ compiler and other dependencies required for the build. The easiest way to do this is to create the [Windows min Docker image](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/build.md#windows-10-min-image) and the perform the build within a container launched from that image. ``` > docker run -it --rm win10-py3-min powershell ``` It is not necessary to use Docker or the win10-py3-min container for the build, but if you do not you must install the appropriate dependencies onto your host system. Next use *cmake* to configure the build. If you are not building within the win10-py3-min container then you will likely need to adjust the CMAKE_TOOLCHAIN_FILE location in the following command. ``` $ mkdir build $ cd build $ cmake -DVCPKG_TARGET_TRIPLET=x64-windows -DCMAKE_TOOLCHAIN_FILE='/vcpkg/scripts/buildsystems/vcpkg.cmake' -DCMAKE_INSTALL_PREFIX=install -DTRITON_ENABLE_CC_GRPC=ON -DTRITON_ENABLE_PYTHON_GRPC=ON -DTRITON_ENABLE_GPU=OFF -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_ENABLE_TESTS=ON .. ``` If you are building on a release branch (or on a development branch that is based off of a release branch), then you must also use additional cmake arguments to point to that release branch for repos that the client build depends on. For example, if you are building the r21.10 client branch then you need to use the following additional cmake flags: ``` -DTRITON_COMMON_REPO_TAG=r21.10 -DTRITON_THIRD_PARTY_REPO_TAG=r21.10 -DTRITON_CORE_REPO_TAG=r21.10 ``` Then use msbuild.exe to build. ``` $ msbuild.exe cc-clients.vcxproj -p:Configuration=Release -clp:ErrorsOnly $ msbuild.exe python-clients.vcxproj -p:Configuration=Release -clp:ErrorsOnly ``` When the build completes the libraries and examples can be found in the install directory. ## Client Library APIs The C++ client API exposes a class-based interface. The commented interface is available in [grpc_client.h](src/c%2B%2B/library/grpc_client.h), [http_client.h](src/c%2B%2B/library/http_client.h), [common.h](src/c%2B%2B/library/common.h). The Python client API provides similar capabilities as the C++ API. The commented interface is available in [grpc](src/python/library/tritonclient/grpc/__init__.py) and [http](src/python/library/tritonclient/http/__init__.py). The Java client API provides similar capabilities as the Python API with similar classes and methods. For more information please refer to the [Java client directory](src/java). ### HTTP Options #### SSL/TLS The client library allows communication across a secured channel using HTTPS protocol. Just setting these SSL options do not ensure the secure communication. Triton server should be running behind `https://` proxy such as nginx. The client can then establish a secure channel to the proxy. The [`qa/L0_https`](https://github.com/triton-inference-server/server/blob/main/qa/L0_https/test.sh) in the server repository demonstrates how this can be achieved. For C++ client, see `HttpSslOptions` struct that encapsulates these options in [http_client.h](src/c%2B%2B/library/http_client.h). For Python client, look for the following options in [http/\_\_init\_\_.py](src/python/library/tritonclient/http/__init__.py): * ssl * ssl_options * ssl_context_factory * insecure The [C++](src/c%2B%2B/examples/simple_http_infer_client.cc) and [Python](src/python/examples/simple_http_infer_client.py) examples demonstrates how to use SSL/TLS settings on client side. #### Compression The client library enables on-wire compression for HTTP transactions. For C++ client, see `request_compression_algorithm` and `response_compression_algorithm` parameters in the `Infer` and `AsyncInfer` functions in [http_client.h](src/c%2B%2B/library/http_client.h). By default, the parameter is set as `CompressionType::NONE`. Similarly, for Python client, see `request_compression_algorithm` and `response_compression_algorithm` parameters in `infer` and `async_infer` functions in [http/\_\_init\_\_.py](src/python/library/tritonclient/http/__init__.py). The [C++](src/c%2B%2B/examples/simple_http_infer_client.cc) and [Python](src/python/examples/simple_http_infer_client.py) examples demonstrates how to use compression options. #### Python AsyncIO Support (Beta) *This feature is currently in beta and may be subject to change.* Advanced users may call the Python client via `async` and `await` syntax. The [infer](src/python/examples/simple_http_aio_infer_client.py) example demonstrates how to infer with AsyncIO. If using SSL/TLS with AsyncIO, look for the `ssl` and `ssl_context` options in [http/aio/\_\_init\_\_.py](src/python/library/tritonclient/http/aio/__init__.py) #### Python Client Plugin API (Beta) *This feature is currently in beta and may be subject to change.* The Triton Client Plugin API lets you register custom plugins to add or modify request headers. This is useful if you have gateway in front of Triton Server that requires extra headers for each request, such as HTTP Authorization. By registering the plugin, your gateway will work with Python clients without additional configuration. Note that Triton Server does not implement authentication or authorization mechanisms and similarly, Triton Server is not the direct consumer of the additional headers. The plugin must implement the `__call__` method. The signature of the `__call__` method should look like below: ```python class MyPlugin: def __call__(self, request): """This method will be called for every HTTP request. Currently, the only field that can be accessed by the request object is the `request.headers` field. This field must be updated in-place. """ request.headers['my-header-key'] = 'my-header-value' ``` After the plugin implementation is complete, you can register the plugin by calling `register` on the `InferenceServerClient` object. ```python from tritonclient.http import InferenceServerClient client = InferenceServerClient(...) # Register the plugin my_plugin = MyPlugin() client.register_plugin(my_plugin) # All the method calls will update the headers according to the plugin # implementation. client.infer(...) ``` To unregister the plugin, you can call the `client.unregister_plugin()` function. ##### Basic Auth You can register the `BasicAuth` plugin that implements [Basic Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication). ```python from tritonclient.grpc.auth import BasicAuth from tritonclient.grpc import InferenceServerClient basic_auth = BasicAuth('username', 'password') client = InferenceServerClient('...') client.register_plugin(basic_auth) ``` The example above shows how to register the plugin for gRPC client. The `BasicAuth` plugin can be registered similarly for HTTP and [AsyncIO](#python-asyncio-support-beta) clients. ### GRPC Options #### SSL/TLS The client library allows communication across a secured channel using gRPC protocol. For C++ client, see `SslOptions` struct that encapsulates these options in [grpc_client.h](src/c%2B%2B/library/grpc_client.h). For Python client, look for the following options in [grpc/\_\_init\_\_.py](src/python/library/tritonclient/grpc/__init__.py): * ssl * root_certificates * private_key * certificate_chain The [C++](src/c%2B%2B/examples/simple_grpc_infer_client.cc) and [Python](src/python/examples/simple_grpc_infer_client.py) examples demonstrates how to use SSL/TLS settings on client side. For information on the corresponding server-side parameters, refer to the [server documentation](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/inference_protocols.md#ssltls) #### Compression The client library also exposes options to use on-wire compression for gRPC transactions. For C++ client, see `compression_algorithm` parameter in the `Infer`, `AsyncInfer` and `StartStream` functions in [grpc_client.h](src/c%2B%2B/library/grpc_client.h). By default, the parameter is set as `GRPC_COMPRESS_NONE`. Similarly, for Python client, see `compression_algorithm` parameter in `infer`, `async_infer` and `start_stream` functions in [grpc/\_\_init\_\_.py](src/python/library/tritonclient/grpc/__init__.py). The [C++](src/c%2B%2B/examples/simple_grpc_infer_client.cc) and [Python](src/python/examples/simple_grpc_infer_client.py) examples demonstrates how to configure compression for clients. For information on the corresponding server-side parameters, refer to the [server documentation](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/inference_protocols.md#compression). #### GRPC KeepAlive Triton exposes GRPC KeepAlive parameters with the default values for both client and server described [here](https://github.com/grpc/grpc/blob/master/doc/keepalive.md). You can find a `KeepAliveOptions` struct/class that encapsulates these parameters in both the [C++](src/c%2B%2B/library/grpc_client.h) and [Python](src/python/library/tritonclient/grpc/__init__.py) client libraries. There is also a [C++](src/c%2B%2B/examples/simple_grpc_keepalive_client.cc) and [Python](src/python/examples/simple_grpc_keepalive_client.py) example demonstrating how to setup these parameters on the client-side. For information on the corresponding server-side parameters, refer to the [server documentation](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/inference_protocols.md#grpc-keepalive) #### Custom GRPC Channel Arguments Advanced users may require specific client-side GRPC Channel Arguments that are not currently exposed by Triton through direct means. To support this, Triton allows users to pass custom channel arguments upon creating a GRPC client. When using this option, it is up to the user to pass a valid combination of arguments for their use case; Triton cannot feasibly test every possible combination of channel arguments. There is a [C++](src/c%2B%2B/examples/simple_grpc_custom_args_client.cc) and [Python](src/python/examples/simple_grpc_custom_args_client.py) example demonstrating how to construct and pass these custom arguments upon creating a GRPC client. You can find a comprehensive list of possible GRPC Channel Arguments [here](https://grpc.github.io/grpc/core/group__grpc__arg__keys.html). #### Python AsyncIO Support (Beta) *This feature is currently in beta and may be subject to change.* Advanced users may call the Python client via `async` and `await` syntax. The [infer](src/python/examples/simple_grpc_aio_infer_client.py) and [stream](src/python/examples/simple_grpc_aio_sequence_stream_infer_client.py) examples demonstrate how to infer with AsyncIO. ### Request Cancellation Starting from r23.10, triton python gRPC client can issue cancellation to inflight requests. This can be done by calling `cancel()` on the CallContext object returned by `async_infer()` API. ```python ctx = client.async_infer(...) ctx.cancel() ``` For streaming requests, `cancel_requests=True` can be sent to `stop_stream()` API to terminate all the inflight requests sent via this stream. ```python client.start_stream() for _ in range(10): client.async_stream_infer(...) # Cancels all pending requests on stream closure rather than blocking until requests complete client.stop_stream(cancel_requests=True) ``` See more details about these APIs in [grpc/\_client.py](src/python/library/tritonclient/grpc/_client.py). For gRPC AsyncIO requests, an AsyncIO task wrapping an `infer()` coroutine can be safely cancelled. ```python infer_task = asyncio.create_task(aio_client.infer(...)) infer_task.cancel() ``` For gRPC AsyncIO streaming requests, `cancel()` can be called on the asynchronous iterator returned by `stream_infer()` API. ```python responses_iterator = aio_client.stream_infer(...) responses_iterator.cancel() ``` See more details about these APIs in [grpc/aio/\__init__.py](src/python/library/tritonclient/grpc/aio/__init__.py). See [request_cancellation](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/request_cancellation.md) in the server user-guide to learn about how this is handled on the server side. If writing your own gRPC clients in the language of choice consult gRPC guide on [cancellation](https://grpc.io/docs/guides/cancellation/#cancelling-an-rpc-call-on-the-client-side). ### GRPC Status Codes Starting from release 24.08, Triton server introduces support for gRPC error codes in streaming mode for all clients enhancing error reporting capabilities. When this feature is enabled, the Triton server will return standard gRPC error codes and subsequently close the stream after delivering the error. This feature is optional can be enabled by adding header with `triton_grpc_error` key and `true` as value. See [grpc error codes](https://github.com/triton-inference-server/server/tree/main#GRPC-Status-Codes) in the server to learn about how this is handled on the server side. See gRPC guide on [status-codes](https://grpc.io/docs/guides/status-codes/) for more details. Below is a Python snippet to enable the feature. Without this header Triton server will continue streaming in default mode returning error message and status inside `InferenceServerException` object within the callback provided. ```python triton_client = grpcclient.InferenceServerClient(triton_server_url) # New added header key value metadata = {"triton_grpc_error": "true"} triton_client.start_stream( callback=partial(callback, user_data), headers=metadata ) ``` ## Simple Example Applications This section describes several of the simple example applications and the features that they illustrate. ### Bytes/String Datatype Some frameworks support tensors where each element in the tensor is variable-length binary data. Each element can hold a string or an arbitrary sequence of bytes. On the client this datatype is BYTES (see [Datatypes](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#datatypes) for information on supported datatypes). The Python client library uses numpy to represent input and output tensors. For BYTES tensors the dtype of the numpy array should be 'np.object_' as shown in the examples. For backwards compatibility with previous versions of the client library, 'np.bytes_' can also be used for BYTES tensors. However, using 'np.bytes_' is not recommended because using this dtype will cause numpy to remove all trailing zeros from each array element. As a result, binary sequences ending in zero(s) will not be represented correctly. BYTES tensors are demonstrated in the C++ example applications simple_http_string_infer_client.cc and simple_grpc_string_infer_client.cc. String tensors are demonstrated in the Python example application simple_http_string_infer_client.py and simple_grpc_string_infer_client.py. ### System Shared Memory Using system shared memory to communicate tensors between the client library and Triton can significantly improve performance in some cases. Using system shared memory is demonstrated in the C++ example applications simple_http_shm_client.cc and simple_grpc_shm_client.cc. Using system shared memory is demonstrated in the Python example application simple_http_shm_client.py and simple_grpc_shm_client.py. Python does not have a standard way of allocating and accessing shared memory so as an example a simple [system shared memory module](src/python/library/tritonclient/utils/shared_memory) is provided that can be used with the Python client library to create, set and destroy system shared memory. ### CUDA Shared Memory Using CUDA shared memory to communicate tensors between the client library and Triton can significantly improve performance in some cases. Using CUDA shared memory is demonstrated in the C++ example applications simple_http_cudashm_client.cc and simple_grpc_cudashm_client.cc. Using CUDA shared memory is demonstrated in the Python example application simple_http_cudashm_client.py and simple_grpc_cudashm_client.py. Python does not have a standard way of allocating and accessing shared memory so as an example a simple [CUDA shared memory module](src/python/library/tritonclient/utils/cuda_shared_memory) is provided that can be used with the Python client library to create, set and destroy CUDA shared memory. The module currently supports numpy arrays ([example usage](src/python/examples/simple_http_cudashm_client.py)) and DLPack tensors ([example usage](src/python/library/tests/test_dlpack.py)). ### Client API for Stateful Models When performing inference using a [stateful model](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/architecture.md#stateful-models), a client must identify which inference requests belong to the same sequence and also when a sequence starts and ends. Each sequence is identified with a sequence ID that is provided when an inference request is made. It is up to the clients to create a unique sequence ID. For each sequence the first inference request should be marked as the start of the sequence and the last inference requests should be marked as the end of the sequence. The use of sequence ID and start and end flags are demonstrated in the C++ example applications simple_grpc_sequence_stream_infer_client.cc. The use of sequence ID and start and end flags are demonstrated in the Python example application simple_grpc_sequence_stream_infer_client.py. ## Image Classification Example The image classification example that uses the C++ client API is available at [src/c++/examples/image_client.cc](src/c%2B%2B/examples/image_client.cc). The Python version of the image classification client is available at [src/python/examples/image_client.py](src/python/examples/image_client.py). To use image_client (or image_client.py) you must first have a running Triton that is serving one or more image classification models. The image_client application requires that the model have a single image input and produce a single classification output. If you don't have a model repository with image classification models see [QuickStart](https://github.com/triton-inference-server/server/blob/main/docs/getting_started/quickstart.md) for instructions on how to create one. Once Triton is running you can use the image_client application to send inference requests. You can specify a single image or a directory holding images. Here we send a request for the inception_graphdef model for an image from the [qa/images](https://github.com/triton-inference-server/server/tree/main/qa/images). ```bash $ image_client -m inception_graphdef -s INCEPTION qa/images/mug.jpg Request 0, batch size 1 Image 'qa/images/mug.jpg': 0.754130 (505) = COFFEE MUG ``` The Python version of the application accepts the same command-line arguments. ```bash $ python image_client.py -m inception_graphdef -s INCEPTION qa/images/mug.jpg Request 0, batch size 1 Image 'qa/images/mug.jpg': 0.826384 (505) = COFFEE MUG ``` The image_client and image_client.py applications use the client libraries to talk to Triton. By default image_client instructs the client library to use HTTP/REST protocol, but you can use the GRPC protocol by providing the -i flag. You must also use the -u flag to point at the GRPC endpoint on Triton. ```bash $ image_client -i grpc -u localhost:8001 -m inception_graphdef -s INCEPTION qa/images/mug.jpg Request 0, batch size 1 Image 'qa/images/mug.jpg': 0.754130 (505) = COFFEE MUG ``` By default the client prints the most probable classification for the image. Use the -c flag to see more classifications. ```bash $ image_client -m inception_graphdef -s INCEPTION -c 3 qa/images/mug.jpg Request 0, batch size 1 Image 'qa/images/mug.jpg': 0.754130 (505) = COFFEE MUG 0.157077 (969) = CUP 0.002880 (968) = ESPRESSO ``` The -b flag allows you to send a batch of images for inferencing. The image_client application will form the batch from the image or images that you specified. If the batch is bigger than the number of images then image_client will just repeat the images to fill the batch. ```bash $ image_client -m inception_graphdef -s INCEPTION -c 3 -b 2 qa/images/mug.jpg Request 0, batch size 2 Image 'qa/images/mug.jpg': 0.754130 (505) = COFFEE MUG 0.157077 (969) = CUP 0.002880 (968) = ESPRESSO Image 'qa/images/mug.jpg': 0.754130 (505) = COFFEE MUG 0.157077 (969) = CUP 0.002880 (968) = ESPRESSO ``` Provide a directory instead of a single image to perform inferencing on all images in the directory. ``` $ image_client -m inception_graphdef -s INCEPTION -c 3 -b 2 qa/images Request 0, batch size 2 Image '/opt/tritonserver/qa/images/car.jpg': 0.819196 (818) = SPORTS CAR 0.033457 (437) = BEACH WAGON 0.031232 (480) = CAR WHEEL Image '/opt/tritonserver/qa/images/mug.jpg': 0.754130 (505) = COFFEE MUG 0.157077 (969) = CUP 0.002880 (968) = ESPRESSO Request 1, batch size 2 Image '/opt/tritonserver/qa/images/vulture.jpeg': 0.977632 (24) = VULTURE 0.000613 (9) = HEN 0.000560 (137) = EUROPEAN GALLINULE Image '/opt/tritonserver/qa/images/car.jpg': 0.819196 (818) = SPORTS CAR 0.033457 (437) = BEACH WAGON 0.031232 (480) = CAR WHEEL ``` The [grpc_image_client.py](src/python/examples/grpc_image_client.py) application behaves the same as the image_client except that instead of using the client library it uses the GRPC generated library to communicate with Triton. ## Ensemble Image Classification Example Application In comparison to the image classification example above, this example uses an ensemble of an image-preprocessing model implemented as a [DALI backend](https://github.com/triton-inference-server/dali_backend) and a TensorFlow Inception model. The ensemble model allows you to send the raw image binaries in the request and receive classification results without preprocessing the images on the client. To try this example you should follow the [DALI ensemble example instructions](https://github.com/triton-inference-server/dali_backend/tree/main/docs/examples/inception_ensemble). kytensor-2.49.0.7/client/TritonClientTargets.cmake000066400000000000000000000051641527052371200220730ustar00rootroot00000000000000add_library(TritonClient::httpclient UNKNOWN IMPORTED) set_target_properties(TritonClient::httpclient PROPERTIES IMPORTED_LOCATION "${TRITONCLIENT_CMAKE_DIR}/../../libhttpclient.so" INTERFACE_INCLUDE_DIRECTORIES "/usr/include/triton/client" ) add_library(TritonClient::grpcclient UNKNOWN IMPORTED) set_target_properties(TritonClient::grpcclient PROPERTIES IMPORTED_LOCATION "${TRITONCLIENT_CMAKE_DIR}/../../libgrpcclient.so" INTERFACE_INCLUDE_DIRECTORIES "/usr/include/triton/client" ) add_library(TritonClient::grpcclient_static STATIC IMPORTED) set_target_properties(TritonClient::grpcclient_static PROPERTIES IMPORTED_LOCATION "${TRITONCLIENT_CMAKE_DIR}/../../libgrpcclient_static.a" INTERFACE_INCLUDE_DIRECTORIES "/usr/include/triton/client" INTERFACE_LINK_LIBRARIES "libgrpc++.a;libgrpc.a;libcares_static.a;libaddress_sorting.a;libupb.a;libabsl_statusor.a;libgpr.a;libabsl_status.a;libabsl_strerror.a;libabsl_flags_internal.a;libabsl_flags_reflection.a;libabsl_hash.a;libabsl_city.a;libabsl_bad_variant_access.a;libabsl_low_level_hash.a;libabsl_raw_hash_set.a;libabsl_hashtablez_sampler.a;libabsl_flags_config.a;libabsl_flags_program_name.a;libabsl_flags_private_handle_accessor.a;libabsl_flags_commandlineflag.a;libabsl_flags_commandlineflag_internal.a;libabsl_flags_marshalling.a;libabsl_random_distributions.a;libabsl_random_seed_sequences.a;libabsl_random_internal_pool_urbg.a;libabsl_random_internal_randen.a;libabsl_random_internal_randen_hwaes.a;libabsl_random_internal_randen_hwaes_impl.a;libabsl_random_internal_randen_slow.a;libabsl_random_internal_platform.a;libabsl_random_internal_seed_material.a;libabsl_random_seed_gen_exception.a;libabsl_cord.a;libabsl_bad_optional_access.a;libabsl_cordz_info.a;libabsl_cord_internal.a;libabsl_cordz_functions.a;libabsl_exponential_biased.a;libabsl_cordz_handle.a;libabsl_str_format_internal.a;libabsl_synchronization.a;libabsl_stacktrace.a;libabsl_symbolize.a;libabsl_debugging_internal.a;libabsl_demangle_internal.a;libabsl_graphcycles_internal.a;libabsl_malloc_internal.a;libabsl_time.a;libabsl_strings.a;libabsl_throw_delegate.a;libabsl_int128.a;libabsl_strings_internal.a;libabsl_base.a;libabsl_spinlock_wait.a;libabsl_raw_logging_internal.a;libabsl_log_severity.a;libabsl_civil_time.a;libabsl_time_zone.a;-lssl;-lcrypto;-lpthread;-lz;libprotobuf.a;libre2.a" ) add_library(TritonClient::httpclient_static STATIC IMPORTED) set_target_properties(TritonClient::httpclient_static PROPERTIES IMPORTED_LOCATION "${TRITONCLIENT_CMAKE_DIR}/../../libhttpclient_static.a" INTERFACE_INCLUDE_DIRECTORIES "/usr/include/triton/client" INTERFACE_LINK_LIBRARIES "-lcurl;-lpthread" ) kytensor-2.49.0.7/client/pyproject.toml000066400000000000000000000046261527052371200200370ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [tool.codespell] # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - # this is only to allow you to run codespell interactively # this also overrides the grpc_generated folder, since it is generated skip = "./.git,./.github,./src/grpc_generated" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" # ignore allowed words # ignoring atleast to avoid testing::AtLeast from getting flagged ignore-words-list = "atleast" # use the 'clear' dictionary for unambiguous spelling mistakes builtin = "clear" # disable warnings about binary files and wrong encoding quiet-level = 3 [tool.isort] profile = "black" use_parentheses = true multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 ensure_newline_before_comments = true line_length = 88 balanced_wrapping = true indent = " " skip = ["build"] kytensor-2.49.0.7/client/src/000077500000000000000000000000001527052371200157025ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/c++/000077500000000000000000000000001527052371200162525ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/c++/CMakeLists.txt000066400000000000000000000111561527052371200210160ustar00rootroot00000000000000# Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(cc-clients LANGUAGES C CXX) # Use C++17 standard as Triton's minimum required. set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.") set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) # # Options # option(TRITON_ENABLE_CC_HTTP "Build C++ HTTP client libraries" OFF) option(TRITON_ENABLE_CC_GRPC "Build C++ GRPC client libraries" OFF) option(TRITON_ENABLE_EXAMPLES "Include examples in build" OFF) option(TRITON_ENABLE_TESTS "Include tests in build" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in libraries" OFF) option(TRITON_USE_THIRD_PARTY "Use local version of third party libraries" ON) option(TRITON_KEEP_TYPEINFO "Keep typeinfo symbols by disabling ldscript" OFF) option(TRITON_ENABLE_ZLIB "Include ZLIB library in build" ON) set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) # FetchContent_Declare( # googletest # SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/third_party/grpc/third_party/googletest # #URL https://github.com/google/googletest/archive/9406a60c7839052e4944ea4dbc8344762a89f9bd.zip # ) if(TRITON_ENABLE_CC_GRPC) set(TRITON_COMMON_ENABLE_PROTOBUF ON) set(TRITON_COMMON_ENABLE_GRPC ON) endif() # TRITON_ENABLE_CC_GRPC if(NOT TRITON_ENABLE_CC_HTTP AND NOT TRITON_ENABLE_EXAMPLES) set(TRITON_COMMON_ENABLE_JSON OFF) endif() # if(TRITON_ENABLE_TESTS) # FetchContent_MakeAvailable(googletest) # endif() FetchContent_MakeAvailable(repo-common) if(TRITON_ENABLE_TESTS) include_directories( ${repo-common_SOURCE_DIR}/include ) endif() # TRITON_ENABLE_TESTS # # CUDA # if(TRITON_ENABLE_GPU) find_package(CUDAToolkit REQUIRED) endif() # TRITON_ENABLE_GPU # # libcurl # if(TRITON_ENABLE_CC_HTTP) find_package(CURL REQUIRED) message(STATUS "Using curl ${CURL_VERSION}") endif() # TRITON_ENABLE_CC_HTTP # # Protobuf # if(TRITON_ENABLE_CC_GRPC) set(protobuf_MODULE_COMPATIBLE TRUE CACHE BOOL "protobuf_MODULE_COMPATIBLE" FORCE) find_package(Protobuf REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") include_directories(${Protobuf_INCLUDE_DIRS}) endif() # TRITON_ENABLE_CC_GRPC # # GRPC # if(TRITON_ENABLE_CC_GRPC) find_package(gRPC CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") include_directories($) endif() # TRITON_ENABLE_CC_GRPC if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC) add_subdirectory(library) endif() # TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC) if(TRITON_ENABLE_EXAMPLES) add_subdirectory(examples) endif() # TRITON_ENABLE_EXAMPLES if(TRITON_ENABLE_TESTS) add_subdirectory(tests) endif() # TRITON_ENABLE_TESTS endif() # TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC kytensor-2.49.0.7/client/src/c++/examples/000077500000000000000000000000001527052371200200705ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/c++/examples/CMakeLists.txt000066400000000000000000000237241527052371200226400ustar00rootroot00000000000000# Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) if(WIN32) message("C++ examples are not currently supported on Windows because " "they require functionalities that are UNIX specific.") else() if(TRITON_ENABLE_CC_HTTP AND TRITON_ENABLE_CC_GRPC) # # image_client # find_package(OpenCV REQUIRED) add_executable( image_client image_client.cc $ ) target_include_directories( image_client PRIVATE ${OpenCV_INCLUDE_DIRS} ) target_link_libraries( image_client PRIVATE grpcclient_static httpclient_static ${OpenCV_LIBS} ) install( TARGETS image_client RUNTIME DESTINATION bin ) # # ensemble_image_client # add_executable( ensemble_image_client ensemble_image_client.cc $ ) target_link_libraries( ensemble_image_client PRIVATE grpcclient_static httpclient_static ) install( TARGETS ensemble_image_client RUNTIME DESTINATION bin ) # # reuse_infer_objects_client # add_executable( reuse_infer_objects_client reuse_infer_objects_client.cc $ ) target_link_libraries( reuse_infer_objects_client PRIVATE grpcclient_static httpclient_static -pthread rt ) install( TARGETS reuse_infer_objects_client RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_CC_HTTP AND TRITON_ENABLE_CC_GRPC if(TRITON_ENABLE_CC_GRPC) # # simple_grpc_health_metadata # add_executable(simple_grpc_health_metadata simple_grpc_health_metadata.cc) target_link_libraries( simple_grpc_health_metadata PRIVATE grpcclient_static ) install( TARGETS simple_grpc_health_metadata RUNTIME DESTINATION bin ) # # simple_grpc_model_control # add_executable(simple_grpc_model_control simple_grpc_model_control.cc) target_link_libraries( simple_grpc_model_control PRIVATE grpcclient_static ) install( TARGETS simple_grpc_model_control RUNTIME DESTINATION bin ) # # simple_grpc_infer_client # add_executable(simple_grpc_infer_client simple_grpc_infer_client.cc) target_link_libraries( simple_grpc_infer_client PRIVATE grpcclient_static ) install( TARGETS simple_grpc_infer_client RUNTIME DESTINATION bin ) # # simple_grpc_keepalive_client # add_executable(simple_grpc_keepalive_client simple_grpc_keepalive_client.cc) target_link_libraries( simple_grpc_keepalive_client PRIVATE grpcclient_static ) install( TARGETS simple_grpc_keepalive_client RUNTIME DESTINATION bin ) # # simple_grpc_custom_args_client # add_executable(simple_grpc_custom_args_client simple_grpc_custom_args_client.cc) target_link_libraries( simple_grpc_custom_args_client PRIVATE grpcclient_static ) install( TARGETS simple_grpc_custom_args_client RUNTIME DESTINATION bin ) # # simple_grpc_string_infer_client # add_executable(simple_grpc_string_infer_client simple_grpc_string_infer_client.cc) target_link_libraries( simple_grpc_string_infer_client PRIVATE grpcclient_static ) install( TARGETS simple_grpc_string_infer_client RUNTIME DESTINATION bin ) # # simple_grpc_async_infer_client # add_executable(simple_grpc_async_infer_client simple_grpc_async_infer_client.cc) target_link_libraries( simple_grpc_async_infer_client PRIVATE grpcclient_static ) install( TARGETS simple_grpc_async_infer_client RUNTIME DESTINATION bin ) # # simple_grpc_sequence_stream_infer_client # add_executable(simple_grpc_sequence_stream_infer_client simple_grpc_sequence_stream_infer_client.cc) target_link_libraries( simple_grpc_sequence_stream_infer_client PRIVATE grpcclient_static ) install( TARGETS simple_grpc_sequence_stream_infer_client RUNTIME DESTINATION bin ) # # simple_grpc_sequence_sync_infer_client # add_executable(simple_grpc_sequence_sync_infer_client simple_grpc_sequence_sync_infer_client.cc) target_link_libraries( simple_grpc_sequence_sync_infer_client PRIVATE grpcclient_static ) install( TARGETS simple_grpc_sequence_sync_infer_client RUNTIME DESTINATION bin ) # # simple_grpc_shm_client # add_executable( simple_grpc_shm_client simple_grpc_shm_client.cc $ ) target_link_libraries( simple_grpc_shm_client PRIVATE grpcclient_static ) install( TARGETS simple_grpc_shm_client RUNTIME DESTINATION bin ) # # simple_grpc_custom_repeat # add_executable(simple_grpc_custom_repeat simple_grpc_custom_repeat.cc) target_link_libraries( simple_grpc_custom_repeat PRIVATE grpcclient_static ) install( TARGETS simple_grpc_custom_repeat RUNTIME DESTINATION bin ) if(${TRITON_ENABLE_GPU}) # # simple_grpc_cudashm_client # set( SIMPLE_GRPC_CUDA_SHM_SRCS simple_grpc_cudashm_client.cc ) set( SIMPLE_GRPC_CUDA_SHM_HDRS ) add_executable(simple_grpc_cudashm_client ${SIMPLE_GRPC_CUDA_SHM_SRCS} ${SIMPLE_GRPC_CUDA_SHM_HDRS}) target_include_directories(simple_grpc_cudashm_client PRIVATE ${CUDA_INCLUDE_DIRS}) target_link_libraries( simple_grpc_cudashm_client PRIVATE grpcclient_static ${CUDA_LIBRARIES} ) install( TARGETS simple_grpc_cudashm_client RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_GPU endif() # TRITON_ENABLE_CC_GRPC if(TRITON_ENABLE_CC_HTTP) # # simple_http_health_metadata # add_executable( simple_http_health_metadata simple_http_health_metadata.cc $ ) target_link_libraries( simple_http_health_metadata PRIVATE httpclient_static ) install( TARGETS simple_http_health_metadata RUNTIME DESTINATION bin ) # # simple_http_model_control # add_executable( simple_http_model_control simple_http_model_control.cc $ ) target_link_libraries( simple_http_model_control PRIVATE httpclient_static ) install( TARGETS simple_http_model_control RUNTIME DESTINATION bin ) # # simple_http_infer_client # add_executable(simple_http_infer_client simple_http_infer_client.cc) target_link_libraries( simple_http_infer_client PRIVATE httpclient_static ) install( TARGETS simple_http_infer_client RUNTIME DESTINATION bin ) # # simple_http_string_infer_client # add_executable(simple_http_string_infer_client simple_http_string_infer_client.cc) target_link_libraries( simple_http_string_infer_client PRIVATE httpclient_static ) install( TARGETS simple_http_string_infer_client RUNTIME DESTINATION bin ) # # simple_http_async_infer_client # add_executable(simple_http_async_infer_client simple_http_async_infer_client.cc) target_link_libraries( simple_http_async_infer_client PRIVATE httpclient_static ) install( TARGETS simple_http_async_infer_client RUNTIME DESTINATION bin ) # # simple_http_sequence_sync_infer_client # add_executable(simple_http_sequence_sync_infer_client simple_http_sequence_sync_infer_client.cc) target_link_libraries( simple_http_sequence_sync_infer_client PRIVATE httpclient_static ) install( TARGETS simple_http_sequence_sync_infer_client RUNTIME DESTINATION bin ) # # simple_http_shm_client # add_executable( simple_http_shm_client simple_http_shm_client.cc $ ) target_link_libraries( simple_http_shm_client PRIVATE httpclient_static rt ) install( TARGETS simple_http_shm_client RUNTIME DESTINATION bin ) if(${TRITON_ENABLE_GPU}) # # simple_http_cudashm_client # set( SIMPLE_HTTP_CUDA_SHM_SRCS simple_http_cudashm_client.cc ) set( SIMPLE_HTTP_CUDA_SHM_HDRS ) add_executable(simple_http_cudashm_client ${SIMPLE_HTTP_CUDA_SHM_SRCS} ${SIMPLE_HTTP_CUDA_SHM_HDRS}) target_include_directories(simple_http_cudashm_client PRIVATE ${CUDA_INCLUDE_DIRS}) target_link_libraries( simple_http_cudashm_client PRIVATE httpclient_static ${CUDA_LIBRARIES} ) install( TARGETS simple_http_cudashm_client RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_GPU endif() # TRITON_ENABLE_CC_HTTP endif() # WIN32 kytensor-2.49.0.7/client/src/c++/examples/ensemble_image_client.cc000066400000000000000000000271731527052371200247030ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include #include #include #include "grpc_client.h" #include "http_client.h" #include "json_utils.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Postprocess( const std::unique_ptr result, const std::vector& filenames, const size_t batch_size, const size_t topk) { std::string output_name("OUTPUT"); if (!result->RequestStatus().IsOk()) { std::cerr << "inference failed with error: " << result->RequestStatus() << std::endl; exit(1); } if (filenames.size() != batch_size) { std::cerr << "expected " << batch_size << " filenames, got " << filenames.size() << std::endl; exit(1); } // Get and validate the shape and datatype std::vector shape; tc::Error err = result->Shape(output_name, &shape); if (!err.IsOk()) { std::cerr << "unable to get shape for " << output_name << std::endl; exit(1); } // Validate shape if ((shape.size() != 2) || (shape[0] != (int)batch_size) || (shape[1] != (int)topk)) { std::cerr << "received incorrect shapes for " << output_name << std::endl; exit(1); } std::string datatype; err = result->Datatype(output_name, &datatype); if (!err.IsOk()) { std::cerr << "unable to get datatype for " << output_name << std::endl; exit(1); } // Validate datatype if (datatype.compare("BYTES") != 0) { std::cerr << "received incorrect datatype for " << output_name << ": " << datatype << std::endl; exit(1); } std::vector result_data; err = result->StringData(output_name, &result_data); if (!err.IsOk()) { std::cerr << "unable to get data for " << output_name << std::endl; exit(1); } if (result_data.size() != (topk * batch_size)) { std::cerr << "unexpected number of strings in the result, expected " << (topk * batch_size) << ", got " << result_data.size() << std::endl; exit(1); } size_t index = 0; for (size_t b = 0; b < batch_size; ++b) { std::cout << "Image '" << filenames[b] << "':" << std::endl; for (size_t c = 0; c < topk; ++c) { std::istringstream is(result_data[index]); int count = 0; std::string token; while (getline(is, token, ':')) { if (count == 0) { std::cout << " " << token; } else if (count == 1) { std::cout << " (" << token << ")"; } else if (count == 2) { std::cout << " = " << token; } count++; } std::cout << std::endl; index++; } } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options] " << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-c " << std::endl; std::cerr << "\t-i " << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << std::endl; std::cerr << "For -c, the classes will be returned, default is 1." << std::endl; std::cerr << "For -i, available protocols are 'grpc' and 'http'. Default is 'http." << std::endl; exit(1); } union TritonClient { TritonClient() { new (&http_client_) std::unique_ptr{}; } ~TritonClient() {} std::unique_ptr http_client_; std::unique_ptr grpc_client_; }; } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); std::string protocol = "http"; size_t topk = 1; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vi:u:p:c:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'i': protocol = optarg; break; case 'u': url = optarg; break; case 'c': topk = std::atoi(optarg); break; case '?': Usage(argv); break; } } if (topk <= 0) { Usage(argv, "topk must be > 0"); } // The ensemble model takes 1 input tensor with shape [ 1 ] and STRING // data type and returns 1 output tensor as top k (see '-c' flag) // classification result of the input. std::string model_name = "preprocess_inception_ensemble"; // Create the inference client for the model. TritonClient triton_client; tc::Error err; if (protocol == "http") { err = tc::InferenceServerHttpClient::Create( &triton_client.http_client_, url, verbose); } else { err = tc::InferenceServerGrpcClient::Create( &triton_client.grpc_client_, url, verbose); } if (!err.IsOk()) { std::cerr << "error: unable to create client for inference: " << err << std::endl; exit(1); } if (optind >= argc) { Usage(argv, "image file or image folder must be specified"); } if (!err.IsOk()) { std::cerr << "error: unable to create inference context: " << err << std::endl; exit(1); } // Obtain a list of the image names to be processed std::vector image_filenames; struct stat name_stat; if (stat(argv[optind], &name_stat) != 0) { std::cerr << "Failed to find '" << std::string(argv[optind]) << "': " << strerror(errno) << std::endl; exit(1); } if (name_stat.st_mode & S_IFDIR) { const std::string dirname = argv[optind]; DIR* dir_ptr = opendir(dirname.c_str()); struct dirent* d_ptr; while ((d_ptr = readdir(dir_ptr)) != NULL) { const std::string filename = d_ptr->d_name; if ((filename != ".") && (filename != "..")) { image_filenames.push_back(dirname + "/" + filename); } } closedir(dir_ptr); } else { image_filenames.push_back(argv[optind]); } // Sort the filenames so that we always visit them in the same order // (readdir does not guarantee any particular order). std::sort(image_filenames.begin(), image_filenames.end()); // Read the raw image as string std::vector> images; for (const auto& fn : image_filenames) { images.emplace_back(); auto& image_str = images.back(); std::ifstream file(fn); file >> std::noskipws; image_str.emplace_back( (std::istreambuf_iterator(file)), std::istreambuf_iterator()); if (image_str.back().empty()) { std::cerr << "error: unable to read image file " << fn << std::endl; exit(1); } } // this client only send one request for simplicity. So the maximum number // of the images to be processed is limited by the maximum batch size size_t batch_size = 0; if (protocol == "http") { std::string model_config; err = triton_client.http_client_->ModelConfig(&model_config, model_name); if (!err.IsOk()) { std::cerr << "error: failed to get model config: " << err << std::endl; } rapidjson::Document model_config_json; err = tc::ParseJson(&model_config_json, model_config); if (!err.IsOk()) { std::cerr << "error: failed to parse model config: " << err << std::endl; } const auto bs_itr = model_config_json.FindMember("max_batch_size"); if (bs_itr != model_config_json.MemberEnd()) { batch_size = bs_itr->value.GetInt(); } } else { inference::ModelConfigResponse model_config; err = triton_client.grpc_client_->ModelConfig(&model_config, model_name); if (!err.IsOk()) { std::cerr << "error: failed to get model config: " << err << std::endl; } batch_size = model_config.config().max_batch_size(); } if (images.size() > batch_size) { std::cerr << "The number of images exceeds maximum batch size, only the" << " first " << batch_size << " images, sorted by name" << " alphabetically, will be processed" << std::endl; } batch_size = (images.size() < batch_size) ? images.size() : batch_size; // Initialize the inputs with the data. tc::InferInput* input; std::vector shape{(int64_t)batch_size, 1}; err = tc::InferInput::Create(&input, "INPUT", shape, "BYTES"); if (!err.IsOk()) { std::cerr << "unable to get input: " << err << std::endl; exit(1); } std::shared_ptr input_ptr(input); tc::InferRequestedOutput* output; // Set the number of classification expected err = tc::InferRequestedOutput::Create(&output, "OUTPUT", topk); if (!err.IsOk()) { std::cerr << "unable to get output: " << err << std::endl; exit(1); } std::shared_ptr output_ptr(output); std::vector inputs = {input_ptr.get()}; std::vector outputs = {output_ptr.get()}; tc::InferOptions options(model_name); FAIL_IF_ERR(input_ptr->Reset(), "unable to reset INPUT"); for (size_t i = 0; i < batch_size; i++) { FAIL_IF_ERR( input_ptr->AppendFromString(images[i]), "unable to set data for INPUT"); } // Send inference request to the inference server. tc::InferResult* results; if (protocol == "http") { FAIL_IF_ERR( triton_client.http_client_->Infer(&results, options, inputs, outputs), "unable to run model"); } else { FAIL_IF_ERR( triton_client.grpc_client_->Infer(&results, options, inputs, outputs), "unable to run model"); } std::unique_ptr results_ptr; results_ptr.reset(results); // Print classification results Postprocess(std::move(results_ptr), image_filenames, batch_size, topk); return 0; } kytensor-2.49.0.7/client/src/c++/examples/image_client.cc000066400000000000000000001122211527052371200230160ustar00rootroot00000000000000// Copyright 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "grpc_client.h" #include "http_client.h" #include "json_utils.h" #if CV_MAJOR_VERSION == 2 #include #include #include #elif CV_MAJOR_VERSION >= 3 #include #include #include #endif #if CV_MAJOR_VERSION == 4 #define GET_TRANSFORMATION_CODE(x) cv::COLOR_##x #else #define GET_TRANSFORMATION_CODE(x) CV_##x #endif namespace tc = triton::client; namespace { enum ScaleType { NONE = 0, VGG = 1, INCEPTION = 2 }; enum ProtocolType { HTTP = 0, GRPC = 1 }; struct ModelInfo { std::string output_name_; std::string input_name_; std::string input_datatype_; // The shape of the input int input_c_; int input_h_; int input_w_; // The format of the input std::string input_format_; int type1_; int type3_; int max_batch_size_; }; void Preprocess( const cv::Mat& img, const std::string& format, int img_type1, int img_type3, size_t img_channels, const cv::Size& img_size, const ScaleType scale, std::vector* input_data) { // Image channels are in BGR order. Currently model configuration // data doesn't provide any information as to the expected channel // orderings (like RGB, BGR). We are going to assume that RGB is the // most likely ordering and so change the channels to that ordering. cv::Mat sample; if ((img.channels() == 3) && (img_channels == 1)) { cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(BGR2GRAY)); } else if ((img.channels() == 4) && (img_channels == 1)) { cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(BGRA2GRAY)); } else if ((img.channels() == 3) && (img_channels == 3)) { cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(BGR2RGB)); } else if ((img.channels() == 4) && (img_channels == 3)) { cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(BGRA2RGB)); } else if ((img.channels() == 1) && (img_channels == 3)) { cv::cvtColor(img, sample, GET_TRANSFORMATION_CODE(GRAY2RGB)); } else { std::cerr << "unexpected number of channels " << img.channels() << " in input image, model expects " << img_channels << "." << std::endl; exit(1); } cv::Mat sample_resized; if (sample.size() != img_size) { cv::resize(sample, sample_resized, img_size); } else { sample_resized = sample; } cv::Mat sample_type; sample_resized.convertTo( sample_type, (img_channels == 3) ? img_type3 : img_type1); cv::Mat sample_final; if (scale == ScaleType::INCEPTION) { if (img_channels == 1) { sample_final = sample_type.mul(cv::Scalar(1 / 127.5)); sample_final = sample_final - cv::Scalar(1.0); } else { sample_final = sample_type.mul(cv::Scalar(1 / 127.5, 1 / 127.5, 1 / 127.5)); sample_final = sample_final - cv::Scalar(1.0, 1.0, 1.0); } } else if (scale == ScaleType::VGG) { if (img_channels == 1) { sample_final = sample_type - cv::Scalar(128); } else { sample_final = sample_type - cv::Scalar(123, 117, 104); } } else { sample_final = sample_type; } // Allocate a buffer to hold all image elements. size_t img_byte_size = sample_final.total() * sample_final.elemSize(); size_t pos = 0; input_data->resize(img_byte_size); // For NHWC format Mat is already in the correct order but need to // handle both cases of data being contiguous or not. if (format.compare("FORMAT_NHWC") == 0) { if (sample_final.isContinuous()) { memcpy(&((*input_data)[0]), sample_final.datastart, img_byte_size); pos = img_byte_size; } else { size_t row_byte_size = sample_final.cols * sample_final.elemSize(); for (int r = 0; r < sample_final.rows; ++r) { memcpy( &((*input_data)[pos]), sample_final.ptr(r), row_byte_size); pos += row_byte_size; } } } else { // (format.compare("FORMAT_NCHW") == 0) // // For CHW formats must split out each channel from the matrix and // order them as BBBB...GGGG...RRRR. To do this split the channels // of the image directly into 'input_data'. The BGR channels are // backed by the 'input_data' vector so that ends up with CHW // order of the data. std::vector input_bgr_channels; for (size_t i = 0; i < img_channels; ++i) { input_bgr_channels.emplace_back( img_size.height, img_size.width, img_type1, &((*input_data)[pos])); pos += input_bgr_channels.back().total() * input_bgr_channels.back().elemSize(); } cv::split(sample_final, input_bgr_channels); } if (pos != img_byte_size) { std::cerr << "unexpected total size of channels " << pos << ", expecting " << img_byte_size << std::endl; exit(1); } } void Postprocess( const std::unique_ptr result, const std::vector& filenames, const size_t batch_size, const std::string& output_name, const size_t topk, const bool batching) { if (!result->RequestStatus().IsOk()) { std::cerr << "inference failed with error: " << result->RequestStatus() << std::endl; exit(1); } if (filenames.size() != batch_size) { std::cerr << "expected " << batch_size << " filenames, got " << filenames.size() << std::endl; exit(1); } // Get and validate the shape and datatype std::vector shape; tc::Error err = result->Shape(output_name, &shape); if (!err.IsOk()) { std::cerr << "unable to get shape for " << output_name << std::endl; exit(1); } // Validate shape. Special handling for non-batch model if (!batching) { if ((shape.size() != 1) || (shape[0] != (int)topk)) { std::cerr << "received incorrect shape for " << output_name << std::endl; exit(1); } } else { if ((shape.size() != 2) || (shape[0] != (int)batch_size) || (shape[1] != (int)topk)) { std::cerr << "received incorrect shape for " << output_name << std::endl; exit(1); } } std::string datatype; err = result->Datatype(output_name, &datatype); if (!err.IsOk()) { std::cerr << "unable to get datatype for " << output_name << std::endl; exit(1); } // Validate datatype if (datatype.compare("BYTES") != 0) { std::cerr << "received incorrect datatype for " << output_name << ": " << datatype << std::endl; exit(1); } std::vector result_data; err = result->StringData(output_name, &result_data); if (!err.IsOk()) { std::cerr << "unable to get data for " << output_name << std::endl; exit(1); } if (result_data.size() != (topk * batch_size)) { std::cerr << "unexpected number of strings in the result, expected " << (topk * batch_size) << ", got " << result_data.size() << std::endl; exit(1); } size_t index = 0; for (size_t b = 0; b < batch_size; ++b) { std::cout << "Image '" << filenames[b] << "':" << std::endl; for (size_t c = 0; c < topk; ++c) { std::istringstream is(result_data[index]); int count = 0; std::string token; while (getline(is, token, ':')) { if (count == 0) { std::cout << " " << token; } else if (count == 1) { std::cout << " (" << token << ")"; } else if (count == 2) { std::cout << " = " << token; } count++; } std::cout << std::endl; index++; } } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options] " << std::endl; std::cerr << " Note that image folder should only contain image files." << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-a" << std::endl; std::cerr << "\t--streaming" << std::endl; std::cerr << "\t-b " << std::endl; std::cerr << "\t-c " << std::endl; std::cerr << "\t-s " << std::endl; std::cerr << "\t-p " << std::endl; std::cerr << "\t-m " << std::endl; std::cerr << "\t-x " << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-i " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "If -a is specified then asynchronous client API will be used. " << "Default is to use the synchronous API." << std::endl; std::cerr << "The --streaming flag is only valid with gRPC protocol." << std::endl; std::cerr << "For -b, a single image will be replicated and sent in a batch" << std::endl << " of the specified size. A directory of images will be grouped" << std::endl << " into batches. Default is 1." << std::endl; std::cerr << "For -c, the classes will be returned, default is 1." << std::endl; std::cerr << "For -s, specify the type of pre-processing scaling that" << std::endl << " should be performed on the image, default is NONE." << std::endl << " INCEPTION: scale each pixel RGB value to [-1.0, 1.0)." << std::endl << " VGG: subtract mean BGR value (123, 117, 104) from" << std::endl << " each pixel." << std::endl; std::cerr << "If -x is not specified the most recent version (that is, the highest " << "numbered version) of the model will be used." << std::endl; std::cerr << "For -p, it generates file only if image file is specified." << std::endl; std::cerr << "For -u, the default server URL is localhost:8000." << std::endl; std::cerr << "For -i, available protocols are gRPC and HTTP. Default is HTTP." << std::endl; std::cerr << "For -H, the header will be added to HTTP requests (ignored for GRPC " "requests). The header must be specified as 'Header:Value'. -H may be " "specified multiple times to add multiple headers." << std::endl; std::cerr << std::endl; exit(1); } ScaleType ParseScale(const std::string& str) { if (str == "NONE") { return ScaleType::NONE; } else if (str == "INCEPTION") { return ScaleType::INCEPTION; } else if (str == "VGG") { return ScaleType::VGG; } std::cerr << "unexpected scale type \"" << str << "\", expecting NONE, INCEPTION or VGG" << std::endl; exit(1); return ScaleType::NONE; } ProtocolType ParseProtocol(const std::string& str) { std::string protocol(str); std::transform(protocol.begin(), protocol.end(), protocol.begin(), ::tolower); if (protocol == "http") { return ProtocolType::HTTP; } else if (protocol == "grpc") { return ProtocolType::GRPC; } std::cerr << "unexpected protocol type \"" << str << "\", expecting HTTP or gRPC" << std::endl; exit(1); return ProtocolType::HTTP; } bool ParseType(const std::string& dtype, int* type1, int* type3) { if (dtype.compare("UINT8") == 0) { *type1 = CV_8UC1; *type3 = CV_8UC3; } else if (dtype.compare("INT8") == 0) { *type1 = CV_8SC1; *type3 = CV_8SC3; } else if (dtype.compare("UINT16") == 0) { *type1 = CV_16UC1; *type3 = CV_16UC3; } else if (dtype.compare("INT16") == 0) { *type1 = CV_16SC1; *type3 = CV_16SC3; } else if (dtype.compare("INT32") == 0) { *type1 = CV_32SC1; *type3 = CV_32SC3; } else if (dtype.compare("FP32") == 0) { *type1 = CV_32FC1; *type3 = CV_32FC3; } else if (dtype.compare("FP64") == 0) { *type1 = CV_64FC1; *type3 = CV_64FC3; } else { return false; } return true; } void ParseModelGrpc( const inference::ModelMetadataResponse& model_metadata, const inference::ModelConfigResponse& model_config, const size_t batch_size, ModelInfo* model_info) { if (model_metadata.inputs().size() != 1) { std::cerr << "expecting 1 input, got " << model_metadata.inputs().size() << std::endl; exit(1); } if (model_metadata.outputs().size() != 1) { std::cerr << "expecting 1 output, got " << model_metadata.outputs().size() << std::endl; exit(1); } if (model_config.config().input().size() != 1) { std::cerr << "expecting 1 input in model configuration, got " << model_config.config().input().size() << std::endl; exit(1); } auto input_metadata = model_metadata.inputs(0); auto input_config = model_config.config().input(0); auto output_metadata = model_metadata.outputs(0); if (output_metadata.datatype().compare("FP32") != 0) { std::cerr << "expecting output datatype to be FP32, model '" << model_metadata.name() << "' output type is '" << output_metadata.datatype() << "'" << std::endl; exit(1); } model_info->max_batch_size_ = model_config.config().max_batch_size(); // Model specifying maximum batch size of 0 indicates that batching // is not supported and so the input tensors do not expect a "N" // dimension (and 'batch_size' should be 1 so that only a single // image instance is inferred at a time). if (model_info->max_batch_size_ == 0) { if (batch_size != 1) { std::cerr << "batching not supported for model \"" << model_metadata.name() << "\"" << std::endl; exit(1); } } else { // model_info->max_batch_size_ > 0 if (batch_size > (size_t)model_info->max_batch_size_) { std::cerr << "expecting batch size <= " << model_info->max_batch_size_ << " for model '" << model_metadata.name() << "'" << std::endl; exit(1); } } // Output is expected to be a vector. But allow any number of // dimensions as long as all but 1 is size 1 (e.g. { 10 }, { 1, 10 // }, { 10, 1, 1 } are all ok). bool output_batch_dim = (model_info->max_batch_size_ > 0); size_t non_one_cnt = 0; for (const auto dim : output_metadata.shape()) { if (output_batch_dim) { output_batch_dim = false; } else if (dim == -1) { std::cerr << "variable-size dimension in model output not supported" << std::endl; exit(1); } else if (dim > 1) { non_one_cnt += 1; if (non_one_cnt > 1) { std::cerr << "expecting model output to be a vector" << std::endl; exit(1); } } } // Model input must have 3 dims, either CHW or HWC (not counting the // batch dimension), either CHW or HWC const bool input_batch_dim = (model_info->max_batch_size_ > 0); const int expected_input_dims = 3 + (input_batch_dim ? 1 : 0); if (input_metadata.shape().size() != expected_input_dims) { std::cerr << "expecting input to have " << expected_input_dims << " dimensions, model '" << model_metadata.name() << "' input has " << input_metadata.shape().size() << std::endl; exit(1); } if ((input_config.format() != inference::ModelInput::FORMAT_NCHW) && (input_config.format() != inference::ModelInput::FORMAT_NHWC)) { std::cerr << "unexpected input format " << inference::ModelInput_Format_Name(input_config.format()) << ", expecting " << inference::ModelInput_Format_Name(inference::ModelInput::FORMAT_NHWC) << " or " << inference::ModelInput_Format_Name(inference::ModelInput::FORMAT_NCHW) << std::endl; exit(1); } model_info->output_name_ = output_metadata.name(); model_info->input_name_ = input_metadata.name(); model_info->input_datatype_ = input_metadata.datatype(); if (input_config.format() == inference::ModelInput::FORMAT_NHWC) { model_info->input_format_ = "FORMAT_NHWC"; model_info->input_h_ = input_metadata.shape(input_batch_dim ? 1 : 0); model_info->input_w_ = input_metadata.shape(input_batch_dim ? 2 : 1); model_info->input_c_ = input_metadata.shape(input_batch_dim ? 3 : 2); } else { model_info->input_format_ = "FORMAT_NCHW"; model_info->input_c_ = input_metadata.shape(input_batch_dim ? 1 : 0); model_info->input_h_ = input_metadata.shape(input_batch_dim ? 2 : 1); model_info->input_w_ = input_metadata.shape(input_batch_dim ? 3 : 2); } if (!ParseType( model_info->input_datatype_, &(model_info->type1_), &(model_info->type3_))) { std::cerr << "unexpected input datatype '" << model_info->input_datatype_ << "' for model \"" << model_metadata.name() << std::endl; exit(1); } } void ParseModelHttp( const rapidjson::Document& model_metadata, const rapidjson::Document& model_config, const size_t batch_size, ModelInfo* model_info) { const auto& input_itr = model_metadata.FindMember("inputs"); size_t input_count = 0; if (input_itr != model_metadata.MemberEnd()) { input_count = input_itr->value.Size(); } if (input_count != 1) { std::cerr << "expecting 1 input, got " << input_count << std::endl; exit(1); } const auto& output_itr = model_metadata.FindMember("outputs"); size_t output_count = 0; if (output_itr != model_metadata.MemberEnd()) { output_count = output_itr->value.Size(); } if (output_count != 1) { std::cerr << "expecting 1 output, got " << output_count << std::endl; exit(1); } const auto& input_config_itr = model_config.FindMember("input"); input_count = 0; if (input_config_itr != model_config.MemberEnd()) { input_count = input_config_itr->value.Size(); } if (input_count != 1) { std::cerr << "expecting 1 input in model configuration, got " << input_count << std::endl; exit(1); } const auto& input_metadata = *input_itr->value.Begin(); const auto& input_config = *input_config_itr->value.Begin(); const auto& output_metadata = *output_itr->value.Begin(); const auto& output_dtype_itr = output_metadata.FindMember("datatype"); if (output_dtype_itr == output_metadata.MemberEnd()) { std::cerr << "output missing datatype in the metadata for model'" << model_metadata["name"].GetString() << "'" << std::endl; exit(1); } auto datatype = std::string( output_dtype_itr->value.GetString(), output_dtype_itr->value.GetStringLength()); if (datatype.compare("FP32") != 0) { std::cerr << "expecting output datatype to be FP32, model '" << model_metadata["name"].GetString() << "' output type is '" << datatype << "'" << std::endl; exit(1); } int max_batch_size = 0; const auto bs_itr = model_config.FindMember("max_batch_size"); if (bs_itr != model_config.MemberEnd()) { max_batch_size = bs_itr->value.GetUint(); } model_info->max_batch_size_ = max_batch_size; // Model specifying maximum batch size of 0 indicates that batching // is not supported and so the input tensors do not expect a "N" // dimension (and 'batch_size' should be 1 so that only a single // image instance is inferred at a time). if (max_batch_size == 0) { if (batch_size != 1) { std::cerr << "batching not supported for model '" << model_metadata["name"].GetString() << "'" << std::endl; exit(1); } } else { // max_batch_size > 0 if (batch_size > (size_t)max_batch_size) { std::cerr << "expecting batch size <= " << max_batch_size << " for model '" << model_metadata["name"].GetString() << "'" << std::endl; exit(1); } } // Output is expected to be a vector. But allow any number of // dimensions as long as all but 1 is size 1 (e.g. { 10 }, { 1, 10 // }, { 10, 1, 1 } are all ok). bool output_batch_dim = (max_batch_size > 0); size_t non_one_cnt = 0; const auto output_shape_itr = output_metadata.FindMember("shape"); if (output_shape_itr != output_metadata.MemberEnd()) { const rapidjson::Value& shape_json = output_shape_itr->value; for (rapidjson::SizeType i = 0; i < shape_json.Size(); i++) { if (output_batch_dim) { output_batch_dim = false; } else if (shape_json[i].GetInt() == -1) { std::cerr << "variable-size dimension in model output not supported" << std::endl; exit(1); } else if (shape_json[i].GetInt() > 1) { non_one_cnt += 1; if (non_one_cnt > 1) { std::cerr << "expecting model output to be a vector" << std::endl; exit(1); } } } } else { std::cerr << "output missing shape in the metadata for model'" << model_metadata["name"].GetString() << "'" << std::endl; exit(1); } // Model input must have 3 dims, either CHW or HWC (not counting the // batch dimension), either CHW or HWC const bool input_batch_dim = (max_batch_size > 0); const size_t expected_input_dims = 3 + (input_batch_dim ? 1 : 0); const auto input_shape_itr = input_metadata.FindMember("shape"); if (input_shape_itr != input_metadata.MemberEnd()) { if (input_shape_itr->value.Size() != expected_input_dims) { std::cerr << "expecting input to have " << expected_input_dims << " dimensions, model '" << model_metadata["name"].GetString() << "' input has " << input_shape_itr->value.Size() << std::endl; exit(1); } } else { std::cerr << "input missing shape in the metadata for model'" << model_metadata["name"].GetString() << "'" << std::endl; exit(1); } model_info->input_format_ = std::string( input_config["format"].GetString(), input_config["format"].GetStringLength()); if ((model_info->input_format_.compare("FORMAT_NCHW") != 0) && (model_info->input_format_.compare("FORMAT_NHWC") != 0)) { std::cerr << "unexpected input format " << model_info->input_format_ << ", expecting FORMAT_NCHW or FORMAT_NHWC" << std::endl; exit(1); } model_info->output_name_ = std::string( output_metadata["name"].GetString(), output_metadata["name"].GetStringLength()); model_info->input_name_ = std::string( input_metadata["name"].GetString(), input_metadata["name"].GetStringLength()); model_info->input_datatype_ = std::string( input_metadata["datatype"].GetString(), input_metadata["datatype"].GetStringLength()); if (model_info->input_format_.compare("FORMAT_NHWC") == 0) { model_info->input_h_ = input_shape_itr->value[input_batch_dim ? 1 : 0].GetInt(); model_info->input_w_ = input_shape_itr->value[input_batch_dim ? 2 : 1].GetInt(); model_info->input_c_ = input_shape_itr->value[input_batch_dim ? 3 : 2].GetInt(); } else { model_info->input_c_ = input_shape_itr->value[input_batch_dim ? 1 : 0].GetInt(); model_info->input_h_ = input_shape_itr->value[input_batch_dim ? 2 : 1].GetInt(); model_info->input_w_ = input_shape_itr->value[input_batch_dim ? 3 : 2].GetInt(); } if (!ParseType( model_info->input_datatype_, &(model_info->type1_), &(model_info->type3_))) { std::cerr << "unexpected input datatype '" << model_info->input_datatype_ << "' for model \"" << model_metadata["name"].GetString() << std::endl; exit(1); } } void FileToInputData( const std::string& filename, size_t c, size_t h, size_t w, const std::string& format, int type1, int type3, ScaleType scale, std::vector* input_data) { // Load the specified image. std::ifstream file(filename); std::vector data; file >> std::noskipws; std::copy( std::istream_iterator(file), std::istream_iterator(), std::back_inserter(data)); if (data.empty()) { std::cerr << "error: unable to read image file " << filename << std::endl; exit(1); } cv::Mat img = imdecode(cv::Mat(data), 1); if (img.empty()) { std::cerr << "error: unable to decode image " << filename << std::endl; exit(1); } // Pre-process the image to match input size expected by the model. Preprocess(img, format, type1, type3, c, cv::Size(w, h), scale, input_data); } union TritonClient { TritonClient() { new (&http_client_) std::unique_ptr{}; } ~TritonClient() {} std::unique_ptr http_client_; std::unique_ptr grpc_client_; }; } // namespace int main(int argc, char** argv) { bool verbose = false; bool async = false; bool streaming = false; int batch_size = 1; int topk = 1; ScaleType scale = ScaleType::NONE; std::string preprocess_output_filename; std::string model_name; std::string model_version = ""; std::string url("localhost:8000"); ProtocolType protocol = ProtocolType::HTTP; tc::Headers http_headers; static struct option long_options[] = {{"streaming", 0, 0, 0}, {0, 0, 0, 0}}; // Parse commandline... int opt; while ((opt = getopt_long( argc, argv, "vau:m:x:b:c:s:p:i:H:", long_options, NULL)) != -1) { switch (opt) { case 0: streaming = true; break; case 'v': verbose = true; break; case 'a': async = true; break; case 'u': url = optarg; break; case 'm': model_name = optarg; break; case 'x': model_version = optarg; break; case 'b': batch_size = std::atoi(optarg); break; case 'c': topk = std::atoi(optarg); break; case 's': scale = ParseScale(optarg); break; case 'p': preprocess_output_filename = optarg; break; case 'i': protocol = ParseProtocol(optarg); break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } if (model_name.empty()) { Usage(argv, "-m flag must be specified"); } if (batch_size <= 0) { Usage(argv, "batch size must be > 0"); } if (topk <= 0) { Usage(argv, "topk must be > 0"); } if (optind >= argc) { Usage(argv, "image file or image folder must be specified"); } if (streaming && (protocol != ProtocolType::GRPC)) { Usage(argv, "Streaming is only allowed with gRPC protocol"); } if (streaming && (!async)) { Usage(argv, "Only async operation is supported in streaming"); } if (!http_headers.empty() && (protocol != ProtocolType::HTTP)) { std::cerr << "WARNING: HTTP headers specified with -H are ignored when " "using non-HTTP protocol." << std::endl; } // Create the inference client for the server. From it // extract and validate that the model meets the requirements for // image classification. TritonClient triton_client; tc::Error err; if (protocol == ProtocolType::HTTP) { err = tc::InferenceServerHttpClient::Create( &triton_client.http_client_, url, verbose); } else { err = tc::InferenceServerGrpcClient::Create( &triton_client.grpc_client_, url, verbose); } if (!err.IsOk()) { std::cerr << "error: unable to create client for inference: " << err << std::endl; exit(1); } ModelInfo model_info; if (protocol == ProtocolType::HTTP) { std::string model_metadata; err = triton_client.http_client_->ModelMetadata( &model_metadata, model_name, model_version, http_headers); if (!err.IsOk()) { std::cerr << "error: failed to get model metadata: " << err << std::endl; } rapidjson::Document model_metadata_json; err = tc::ParseJson(&model_metadata_json, model_metadata); if (!err.IsOk()) { std::cerr << "error: failed to parse model metadata: " << err << std::endl; } std::string model_config; err = triton_client.http_client_->ModelConfig( &model_config, model_name, model_version, http_headers); if (!err.IsOk()) { std::cerr << "error: failed to get model config: " << err << std::endl; } rapidjson::Document model_config_json; err = tc::ParseJson(&model_config_json, model_config); if (!err.IsOk()) { std::cerr << "error: failed to parse model config: " << err << std::endl; } ParseModelHttp( model_metadata_json, model_config_json, batch_size, &model_info); } else { inference::ModelMetadataResponse model_metadata; err = triton_client.grpc_client_->ModelMetadata( &model_metadata, model_name, model_version, http_headers); if (!err.IsOk()) { std::cerr << "error: failed to get model metadata: " << err << std::endl; } inference::ModelConfigResponse model_config; err = triton_client.grpc_client_->ModelConfig( &model_config, model_name, model_version, http_headers); if (!err.IsOk()) { std::cerr << "error: failed to get model config: " << err << std::endl; } ParseModelGrpc(model_metadata, model_config, batch_size, &model_info); } // Collect the names of the image(s). std::vector image_filenames; struct stat name_stat; if (stat(argv[optind], &name_stat) != 0) { std::cerr << "Failed to find '" << std::string(argv[optind]) << "': " << strerror(errno) << std::endl; exit(1); } if (name_stat.st_mode & S_IFDIR) { const std::string dirname = argv[optind]; DIR* dir_ptr = opendir(dirname.c_str()); struct dirent* d_ptr; while ((d_ptr = readdir(dir_ptr)) != NULL) { const std::string filename = d_ptr->d_name; if ((filename != ".") && (filename != "..")) { image_filenames.push_back(dirname + "/" + filename); } } closedir(dir_ptr); } else { image_filenames.push_back(argv[optind]); } // Sort the filenames so that we always visit them in the same order // (readdir does not guarantee any particular order). std::sort(image_filenames.begin(), image_filenames.end()); // Preprocess the images into input data according to model // requirements std::vector> image_data; for (const auto& fn : image_filenames) { image_data.emplace_back(); FileToInputData( fn, model_info.input_c_, model_info.input_h_, model_info.input_w_, model_info.input_format_, model_info.type1_, model_info.type3_, scale, &(image_data.back())); if ((image_data.size() == 1) && !preprocess_output_filename.empty()) { std::ofstream output_file(preprocess_output_filename); std::ostream_iterator output_iterator(output_file); std::copy(image_data[0].begin(), image_data[0].end(), output_iterator); } } std::vector shape; // Include the batch dimension if required if (model_info.max_batch_size_ != 0) { shape.push_back(batch_size); } if (model_info.input_format_.compare("FORMAT_NHWC") == 0) { shape.push_back(model_info.input_h_); shape.push_back(model_info.input_w_); shape.push_back(model_info.input_c_); } else { shape.push_back(model_info.input_c_); shape.push_back(model_info.input_h_); shape.push_back(model_info.input_w_); } // Initialize the inputs with the data. tc::InferInput* input; err = tc::InferInput::Create( &input, model_info.input_name_, shape, model_info.input_datatype_); if (!err.IsOk()) { std::cerr << "unable to get input: " << err << std::endl; exit(1); } std::shared_ptr input_ptr(input); tc::InferRequestedOutput* output; // Set the number of classification expected err = tc::InferRequestedOutput::Create(&output, model_info.output_name_, topk); if (!err.IsOk()) { std::cerr << "unable to get output: " << err << std::endl; exit(1); } std::shared_ptr output_ptr(output); std::vector inputs = {input_ptr.get()}; std::vector outputs = {output_ptr.get()}; // Configure context for 'batch_size' and 'topk' tc::InferOptions options(model_name); options.model_version_ = model_version; // Send requests of 'batch_size' images. If the number of images // isn't an exact multiple of 'batch_size' then just start over with // the first images until the batch is filled. // // Number of requests sent = ceil(number of images / batch_size) std::vector> results; std::vector> result_filenames; size_t image_idx = 0; size_t done_cnt = 0; size_t sent_count = 0; bool last_request = false; std::mutex mtx; std::condition_variable cv; auto callback_func = [&](tc::InferResult* result) { { // Defer the response retrieval to main thread std::lock_guard lk(mtx); results.emplace_back(result); done_cnt++; } cv.notify_all(); }; if (streaming) { err = triton_client.grpc_client_->StartStream( callback_func, true /* enable_stats */, 0 /* stream_timeout */, http_headers); if (!err.IsOk()) { std::cerr << "failed to establish the stream: " << err << std::endl; } } while (!last_request) { // Reset the input for new request. err = input_ptr->Reset(); if (!err.IsOk()) { std::cerr << "failed resetting input: " << err << std::endl; exit(1); } // Set input to be the next 'batch_size' images (preprocessed). std::vector input_filenames; for (int idx = 0; idx < batch_size; ++idx) { input_filenames.push_back(image_filenames[image_idx]); err = input_ptr->AppendRaw(image_data[image_idx]); if (!err.IsOk()) { std::cerr << "failed setting input: " << err << std::endl; exit(1); } image_idx = (image_idx + 1) % image_data.size(); if (image_idx == 0) { last_request = true; } } result_filenames.emplace_back(std::move(input_filenames)); options.request_id_ = std::to_string(sent_count); // Send request. if (!async) { tc::InferResult* result; if (protocol == ProtocolType::HTTP) { err = triton_client.http_client_->Infer( &result, options, inputs, outputs, http_headers); } else { err = triton_client.grpc_client_->Infer( &result, options, inputs, outputs, http_headers); } if (!err.IsOk()) { std::cerr << "failed sending synchronous infer request: " << err << std::endl; exit(1); } results.emplace_back(result); } else { if (streaming) { err = triton_client.grpc_client_->AsyncStreamInfer( options, inputs, outputs); } else { if (protocol == ProtocolType::HTTP) { err = triton_client.http_client_->AsyncInfer( callback_func, options, inputs, outputs, http_headers); } else { err = triton_client.grpc_client_->AsyncInfer( callback_func, options, inputs, outputs, http_headers); } } if (!err.IsOk()) { std::cerr << "failed sending asynchronous infer request: " << err << std::endl; exit(1); } } sent_count++; } // For async, retrieve results according to the send order if (async) { // Wait until all callbacks are invoked { std::unique_lock lk(mtx); cv.wait(lk, [&]() { if (done_cnt >= sent_count) { return true; } else { return false; } }); } } // Post-process the results to make prediction(s) for (size_t idx = 0; idx < results.size(); idx++) { std::cout << "Request " << idx << ", batch size " << batch_size << std::endl; Postprocess( std::move(results[idx]), result_filenames[idx], batch_size, model_info.output_name_, topk, model_info.max_batch_size_ != 0); } return 0; } kytensor-2.49.0.7/client/src/c++/examples/reuse_infer_objects_client.cc000066400000000000000000000407201527052371200257570ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "grpc_client.h" #include "http_client.h" #include "shm_utils.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { union TritonClient { TritonClient() { new (&http_client_) std::unique_ptr{}; } ~TritonClient() {} std::unique_ptr http_client_; std::unique_ptr grpc_client_; }; void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void InferAndValidate( const bool use_shared_memory, TritonClient& triton_client, const std::string& protocol, const tc::InferOptions& options, const tc::Headers& http_headers, std::vector& inputs, const size_t input_byte_size, std::vector& outputs, const size_t output_byte_size, std::vector& shm_ptrs) { std::vector input0_data(16); std::vector input1_data(16); int32_t* input0_data_ptr; int32_t* input1_data_ptr; int32_t* output0_data_ptr; int32_t* output1_data_ptr; FAIL_IF_ERR(inputs[0]->Reset(), "unable to reset input 'INPUT0'"); FAIL_IF_ERR(inputs[1]->Reset(), "unable to reset input 'INPUT1'"); if (use_shared_memory) { input0_data_ptr = shm_ptrs[0]; input1_data_ptr = shm_ptrs[1]; FAIL_IF_ERR( inputs[0]->SetSharedMemory( "input_data", input_byte_size, 0 /* offset */), "unable to set shared memory for INPUT0"); FAIL_IF_ERR( inputs[1]->SetSharedMemory( "input_data", input_byte_size, input_byte_size /* offset */), "unable to set shared memory for INPUT1"); FAIL_IF_ERR( outputs[0]->SetSharedMemory( "output_data", output_byte_size, 0 /* offset */), "unable to set shared memory for 'OUTPUT0'"); FAIL_IF_ERR( outputs[1]->SetSharedMemory( "output_data", output_byte_size, output_byte_size /* offset */), "unable to set shared memory for 'OUTPUT1'"); } else { input0_data_ptr = &input0_data[0]; input1_data_ptr = &input1_data[0]; // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all twos. We use twos instead // of ones in input1_data to validate whether inputs were set correctly. for (size_t i = 0; i < 16; ++i) { input0_data[i] = i; input1_data[i] = 2; } FAIL_IF_ERR( inputs[0]->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for 'INPUT0'"); FAIL_IF_ERR( inputs[1]->AppendRaw( reinterpret_cast(&input1_data[0]), input1_data.size() * sizeof(int32_t)), "unable to set data for 'INPUT1'"); FAIL_IF_ERR( outputs[0]->UnsetSharedMemory(), "unable to unset shared memory for 'OUTPUT0'"); FAIL_IF_ERR( outputs[1]->UnsetSharedMemory(), "unable to unset shared memory for 'OUTPUT1'"); } std::vector routputs = { outputs[0], outputs[1]}; tc::InferResult* results; if (protocol == "http") { FAIL_IF_ERR( triton_client.http_client_->Infer( &results, options, inputs, routputs, http_headers), "unable to run model"); } else { FAIL_IF_ERR( triton_client.grpc_client_->Infer( &results, options, inputs, routputs, http_headers), "unable to run model"); } std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); if (use_shared_memory) { std::cout << "\n\n======== SHARED_MEMORY ========\n"; output0_data_ptr = shm_ptrs[2]; output1_data_ptr = shm_ptrs[3]; } else { std::cout << "\n\n======== NO_SHARED_MEMORY ========\n"; // Get pointers to the result returned... size_t recv_output0_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT0", (const uint8_t**)&output0_data_ptr, &recv_output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (recv_output0_byte_size != output_byte_size) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << recv_output0_byte_size << std::endl; exit(1); } size_t recv_output1_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT1", (const uint8_t**)&output1_data_ptr, &recv_output1_byte_size), "unable to get result data for 'OUTPUT1'"); if (recv_output1_byte_size != output_byte_size) { std::cerr << "error: received incorrect byte size for 'OUTPUT1': " << recv_output1_byte_size << std::endl; exit(1); } } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data_ptr[i] << " + " << input1_data_ptr[i] << " = " << output0_data_ptr[i] << std::endl; std::cout << input0_data_ptr[i] << " - " << input1_data_ptr[i] << " = " << output1_data_ptr[i] << std::endl; if ((input0_data_ptr[i] + input1_data_ptr[i]) != output0_data_ptr[i]) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_data_ptr[i] - input1_data_ptr[i]) != output1_data_ptr[i]) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } std::cout << "\n======== END ========\n\n"; } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace // Tests whether the same InferInput and InferRequestedOutput objects can be // successfully used repeatedly for different inferences using/not-using // shared memory. int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); bool url_specified = false; tc::Headers http_headers; std::string protocol("http"); // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:i:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; url_specified = true; break; case 'i': protocol = optarg; std::transform( protocol.begin(), protocol.end(), protocol.begin(), ::tolower); break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create the inference client for the server. From it // extract and validate that the model meets the requirements for // image classification. TritonClient triton_client; tc::Error err; if (protocol == "http") { err = tc::InferenceServerHttpClient::Create( &triton_client.http_client_, url, verbose); } else if (protocol == "grpc") { if (!url_specified) { url = "localhost:8001"; } err = tc::InferenceServerGrpcClient::Create( &triton_client.grpc_client_, url, verbose); } else { std::cerr << "error: unsupported protocol provided: only supports grpc or http." << std::endl; exit(1); } if (!err.IsOk()) { std::cerr << "error: unable to create client for inference: " << err << std::endl; exit(1); } // Unregistering all shared memory regions for a clean // start. if (protocol == "http") { FAIL_IF_ERR( triton_client.http_client_->UnregisterSystemSharedMemory(), "unable to unregister all system shared memory regions"); FAIL_IF_ERR( triton_client.http_client_->UnregisterCudaSharedMemory(), "unable to unregister all cuda shared memory regions"); } else { FAIL_IF_ERR( triton_client.grpc_client_->UnregisterSystemSharedMemory(), "unable to unregister all system shared memory regions"); FAIL_IF_ERR( triton_client.grpc_client_->UnregisterCudaSharedMemory(), "unable to unregister all cuda shared memory regions"); } std::vector shape{1, 16}; size_t input_byte_size = 64; size_t output_byte_size = 64; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); // Create Input0 and Input1 in Shared Memory. Initialize Input0 to unique // integers and Input1 to all ones. std::string shm_key = "/input_simple"; int shm_fd_ip, *input0_shm; FAIL_IF_ERR( tc::CreateSharedMemoryRegion(shm_key, input_byte_size * 2, &shm_fd_ip), ""); FAIL_IF_ERR( tc::MapSharedMemory( shm_fd_ip, 0, input_byte_size * 2, (void**)&input0_shm), ""); FAIL_IF_ERR(tc::CloseSharedMemory(shm_fd_ip), ""); int* input1_shm = (int*)(input0_shm + 16); for (size_t i = 0; i < 16; ++i) { *(input0_shm + i) = i; *(input1_shm + i) = 1; } if (protocol == "http") { FAIL_IF_ERR( triton_client.http_client_->RegisterSystemSharedMemory( "input_data", "/input_simple", input_byte_size * 2), "failed to register input shared memory region"); } else { FAIL_IF_ERR( triton_client.grpc_client_->RegisterSystemSharedMemory( "input_data", "/input_simple", input_byte_size * 2), "failed to register input shared memory region"); } // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // Create Output0 and Output1 in Shared Memory shm_key = "/output_simple"; int shm_fd_op; int* output0_shm; FAIL_IF_ERR( tc::CreateSharedMemoryRegion(shm_key, output_byte_size * 2, &shm_fd_op), ""); FAIL_IF_ERR( tc::MapSharedMemory( shm_fd_op, 0, output_byte_size * 2, (void**)&output0_shm), ""); FAIL_IF_ERR(tc::CloseSharedMemory(shm_fd_op), ""); int* output1_shm = (int*)(output0_shm + 16); if (protocol == "http") { FAIL_IF_ERR( triton_client.http_client_->RegisterSystemSharedMemory( "output_data", "/output_simple", output_byte_size * 2), "failed to register output shared memory region"); } else { FAIL_IF_ERR( triton_client.grpc_client_->RegisterSystemSharedMemory( "output_data", "/output_simple", output_byte_size * 2), "failed to register output shared memory region"); } std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; std::vector shm_ptrs = { input0_shm, input1_shm, output0_shm, output1_shm}; // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; // Issue inference using shared memory InferAndValidate( true /* use_shared_memory */, triton_client, protocol, options, http_headers, inputs, input_byte_size, outputs, output_byte_size, shm_ptrs); // Issue inference without using shared memory InferAndValidate( false /* use_shared_memory */, triton_client, protocol, options, http_headers, inputs, input_byte_size, outputs, output_byte_size, shm_ptrs); // Issue inference using shared memory InferAndValidate( true /* use_shared_memory */, triton_client, protocol, options, http_headers, inputs, input_byte_size, outputs, output_byte_size, shm_ptrs); // Unregister shared memory if (protocol == "http") { FAIL_IF_ERR( triton_client.http_client_->UnregisterSystemSharedMemory("input_data"), "unable to unregister shared memory input region"); FAIL_IF_ERR( triton_client.http_client_->UnregisterSystemSharedMemory("output_data"), "unable to unregister shared memory output region"); } else { FAIL_IF_ERR( triton_client.grpc_client_->UnregisterSystemSharedMemory("input_data"), "unable to unregister shared memory input region"); FAIL_IF_ERR( triton_client.grpc_client_->UnregisterSystemSharedMemory("output_data"), "unable to unregister shared memory output region"); } // Cleanup shared memory FAIL_IF_ERR(tc::UnmapSharedMemory(input0_shm, input_byte_size * 2), ""); FAIL_IF_ERR(tc::UnlinkSharedMemoryRegion("/input_simple"), ""); FAIL_IF_ERR(tc::UnmapSharedMemory(output0_shm, output_byte_size * 2), ""); FAIL_IF_ERR(tc::UnlinkSharedMemoryRegion("/output_simple"), ""); return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_async_infer_client.cc000066400000000000000000000273571527052371200266370ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include "grpc_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void ValidateResult( const std::shared_ptr result, std::vector& input0_data, std::vector& input1_data) { // Validate the results... ValidateShapeAndDatatype("OUTPUT0", result); ValidateShapeAndDatatype("OUTPUT1", result); // Get pointers to the result returned... int32_t* output0_data; size_t output0_byte_size; FAIL_IF_ERR( result->RawData( "OUTPUT0", (const uint8_t**)&output0_data, &output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << output0_byte_size << std::endl; exit(1); } int32_t* output1_data; size_t output1_byte_size; FAIL_IF_ERR( result->RawData( "OUTPUT1", (const uint8_t**)&output1_data, &output1_byte_size), "unable to get result data for 'OUTPUT1'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT1': " << output0_byte_size << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data[i] << " + " << input1_data[i] << " = " << *(output0_data + i) << std::endl; std::cout << input0_data[i] << " - " << input1_data[i] << " = " << *(output1_data + i) << std::endl; if ((input0_data[i] + input1_data[i]) != *(output0_data + i)) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_data[i] - input1_data[i]) != *(output1_data + i)) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get full response std::cout << result->DebugString() << std::endl; } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; uint32_t client_timeout = 0; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:t:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 't': client_timeout = std::stoi(optarg); break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. std::vector input0_data(16); std::vector input1_data(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = i; input1_data[i] = 1; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); FAIL_IF_ERR( input0_ptr->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for INPUT0"); FAIL_IF_ERR( input1_ptr->AppendRaw( reinterpret_cast(&input1_data[0]), input1_data.size() * sizeof(int32_t)), "unable to set data for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; options.client_timeout_ = client_timeout; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; // Send inference request to the inference server. std::mutex mtx; std::condition_variable cv; size_t repeat_cnt = 2; size_t done_cnt = 0; for (size_t i = 0; i < repeat_cnt; i++) { FAIL_IF_ERR( client->AsyncInfer( [&, i](tc::InferResult* result) { { std::shared_ptr result_ptr; result_ptr.reset(result); std::lock_guard lk(mtx); std::cout << "Callback no." << i << " is called" << std::endl; done_cnt++; if (result_ptr->RequestStatus().IsOk()) { ValidateResult(result_ptr, input0_data, input1_data); } else { std::cerr << "error: Inference failed: " << result_ptr->RequestStatus() << std::endl; exit(1); } } cv.notify_all(); }, options, inputs, outputs, http_headers), "unable to run model"); } // Wait until all callbacks are invoked { std::unique_lock lk(mtx); cv.wait(lk, [&]() { if (done_cnt >= repeat_cnt) { return true; } else { return false; } }); } if (done_cnt == repeat_cnt) { std::cout << "All done" << std::endl; } else { std::cerr << "Done cnt: " << done_cnt << " does not match repeat cnt: " << repeat_cnt << std::endl; exit(1); } // Send another AsyncInfer whose callback defers the completed request // to another thread (main thread) to handle bool callback_invoked = false; std::shared_ptr result_placeholder; FAIL_IF_ERR( client->AsyncInfer( [&](tc::InferResult* result) { { std::shared_ptr result_ptr; result_ptr.reset(result); // Defer the response retrieval to main thread std::lock_guard lk(mtx); callback_invoked = true; result_placeholder = std::move(result_ptr); } cv.notify_all(); }, options, inputs, outputs, http_headers), "unable to run model"); // Ensure callback is completed { std::unique_lock lk(mtx); cv.wait(lk, [&]() { return callback_invoked; }); } // Get deferred response std::cout << "Getting results from deferred response" << std::endl; if (result_placeholder->RequestStatus().IsOk()) { ValidateResult(result_placeholder, input0_data, input1_data); } else { std::cerr << "error: Inference failed: " << result_placeholder->RequestStatus() << std::endl; exit(1); } tc::InferStat infer_stat; client->ClientInferStat(&infer_stat); std::cout << "completed_request_count " << infer_stat.completed_request_count << std::endl; std::cout << "cumulative_total_request_time_ns " << infer_stat.cumulative_total_request_time_ns << std::endl; std::cout << "cumulative_send_time_ns " << infer_stat.cumulative_send_time_ns << std::endl; std::cout << "cumulative_receive_time_ns " << infer_stat.cumulative_receive_time_ns << std::endl; std::cout << "PASS : Async Infer" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_cudashm_client.cc000066400000000000000000000261031527052371200257470ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include "grpc_client.h" #include "shm_utils.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace #define FAIL_IF_CUDA_ERR(FUNC) \ { \ const cudaError_t result = FUNC; \ if (result != cudaSuccess) { \ std::cerr << "CUDA exception (line " << __LINE__ \ << "): " << cudaGetErrorName(result) << " (" \ << cudaGetErrorString(result) << ")" << std::endl; \ exit(1); \ } \ } void CreateCUDAIPCHandle( cudaIpcMemHandle_t* cuda_handle, void* input_d_ptr, int device_id = 0) { // Set the GPU device to the desired GPU FAIL_IF_CUDA_ERR(cudaSetDevice(device_id)); // Create IPC handle for data on the gpu FAIL_IF_CUDA_ERR(cudaIpcGetMemHandle(cuda_handle, input_d_ptr)); } int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); // Unregistering all shared memory regions for a clean // start. FAIL_IF_ERR( client->UnregisterSystemSharedMemory(), "unable to unregister all system shared memory regions"); FAIL_IF_ERR( client->UnregisterCudaSharedMemory(), "unable to unregister all cuda shared memory regions"); std::vector shape{1, 16}; size_t input_byte_size = 64; size_t output_byte_size = 64; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); // Create Input0 and Input1 in CUDA Shared Memory. Initialize Input0 to // unique integers and Input1 to all ones. int input_data[32]; for (size_t i = 0; i < 16; ++i) { input_data[i] = i; input_data[16 + i] = 1; } // copy INPUT0 and INPUT1 data in GPU shared memory int* input_d_ptr; cudaMalloc((void**)&input_d_ptr, input_byte_size * 2); cudaMemcpy( (void*)input_d_ptr, (void*)input_data, input_byte_size * 2, cudaMemcpyHostToDevice); cudaIpcMemHandle_t input_cuda_handle; CreateCUDAIPCHandle(&input_cuda_handle, (void*)input_d_ptr); FAIL_IF_ERR( client->RegisterCudaSharedMemory( "input_data", input_cuda_handle, 0 /* device_id */, input_byte_size * 2), "failed to register input shared memory region"); FAIL_IF_ERR( input0_ptr->SetSharedMemory( "input_data", input_byte_size, 0 /* offset */), "unable to set shared memory for INPUT0"); FAIL_IF_ERR( input1_ptr->SetSharedMemory( "input_data", input_byte_size, input_byte_size /* offset */), "unable to set shared memory for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // Create Output0 and Output1 in CUDA Shared Memory int *output0_d_ptr, *output1_d_ptr; cudaMalloc((void**)&output0_d_ptr, output_byte_size * 2); output1_d_ptr = (int*)output0_d_ptr + 16; cudaIpcMemHandle_t output_cuda_handle; CreateCUDAIPCHandle(&output_cuda_handle, (void*)output0_d_ptr); FAIL_IF_ERR( client->RegisterCudaSharedMemory( "output_data", output_cuda_handle, 0 /* device_id */, output_byte_size * 2), "failed to register output shared memory region"); FAIL_IF_ERR( output0_ptr->SetSharedMemory( "output_data", output_byte_size, 0 /* offset */), "unable to set shared memory for 'OUTPUT0'"); FAIL_IF_ERR( output1_ptr->SetSharedMemory( "output_data", output_byte_size, output_byte_size /* offset */), "unable to set shared memory for 'OUTPUT1'"); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer(&results, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); // Copy input and output data back to the CPU int output0_data[16], output1_data[16]; cudaMemcpy( output0_data, output0_d_ptr, output_byte_size, cudaMemcpyDeviceToHost); cudaMemcpy( output1_data, output1_d_ptr, output_byte_size, cudaMemcpyDeviceToHost); for (size_t i = 0; i < 16; ++i) { std::cout << input_data[i] << " + " << input_data[16 + i] << " = " << output0_data[i] << std::endl; std::cout << input_data[i] << " + " << input_data[16 + i] << " = " << output1_data[i] << std::endl; if ((input_data[i] + input_data[16 + i]) != output0_data[i]) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input_data[i] - input_data[16 + i]) != output1_data[i]) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get shared memory regions active/registered within triton inference::CudaSharedMemoryStatusResponse status; FAIL_IF_ERR( client->CudaSharedMemoryStatus(&status), "failed to get shared memory status"); std::cout << "Shared Memory Status:\n" << status.DebugString() << "\n"; // Unregister shared memory FAIL_IF_ERR( client->UnregisterCudaSharedMemory("input_data"), "unable to unregister shared memory input region"); FAIL_IF_ERR( client->UnregisterCudaSharedMemory("output_data"), "unable to unregister shared memory output region"); // Free GPU memory FAIL_IF_CUDA_ERR(cudaFree(input_d_ptr)); FAIL_IF_CUDA_ERR(cudaFree(output0_d_ptr)); std::cout << "PASS : Cuda Shared Memory " << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_custom_args_client.cc000066400000000000000000000227471527052371200266630ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include "grpc_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-m " << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; uint32_t client_timeout = 0; bool use_ssl = false; tc::SslOptions ssl_options; grpc::ChannelArguments channel_args; // Set any valid grpc::ChannelArguments here based on use case channel_args.SetMaxSendMessageSize(1024 * 1024); channel_args.SetMaxReceiveMessageSize(1024 * 1024); // Setting KeepAlive options using new generic channel arguments option // https://grpc.github.io/grpc/cpp/md_doc_keepalive.html channel_args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, INT_MAX); channel_args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, 20000); channel_args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, false); channel_args.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 2); // Example arg requested for the feature channel_args.SetInt(GRPC_ARG_DNS_ENABLE_SRV_QUERIES, 1); // Parse commandline... int opt; while ((opt = getopt_long(argc, argv, "vu:t:H:C:", NULL, NULL)) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 't': client_timeout = std::stoi(optarg); break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create( &client, url, channel_args, verbose, use_ssl, ssl_options), "unable to create grpc client"); // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. std::vector input0_data(16); std::vector input1_data(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = i; input1_data[i] = 1; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); FAIL_IF_ERR( input0_ptr->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for INPUT0"); FAIL_IF_ERR( input1_ptr->AppendRaw( reinterpret_cast(&input1_data[0]), input1_data.size() * sizeof(int32_t)), "unable to set data for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; options.client_timeout_ = client_timeout; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer(&results, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); // Get pointers to the result returned... int32_t* output0_data; size_t output0_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT0", (const uint8_t**)&output0_data, &output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << output0_byte_size << std::endl; exit(1); } int32_t* output1_data; size_t output1_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT1", (const uint8_t**)&output1_data, &output1_byte_size), "unable to get result data for 'OUTPUT1'"); if (output1_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT1': " << output1_byte_size << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data[i] << " + " << input1_data[i] << " = " << *(output0_data + i) << std::endl; std::cout << input0_data[i] << " - " << input1_data[i] << " = " << *(output1_data + i) << std::endl; if ((input0_data[i] + input1_data[i]) != *(output0_data + i)) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_data[i] - input1_data[i]) != *(output1_data + i)) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get full response std::cout << results_ptr->DebugString() << std::endl; std::cout << "PASS : CustomArgs" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_custom_repeat.cc000066400000000000000000000366231527052371200256470ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include #include #include "grpc_client.h" namespace tc = triton::client; using ResultMap = std::map>>; using ResultList = std::vector>; // Global mutex to synchronize the threads std::mutex mutex_; std::condition_variable cv_; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; std::cerr << "\t-r " << std::endl; std::cerr << "\t-s " << std::endl; std::cerr << "\t-o " << std::endl; std::cerr << "\t-d " << std::endl; std::cerr << "\t-w " << std::endl; exit(1); } } // namespace int main_async(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; int request_count = 1; int repeat_count = 1; int data_offset = 100; uint32_t delay_time = 1000; uint32_t wait_time = 500; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:r:s:o:d:w:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case 'r': request_count = std::stoi(optarg); break; case 's': repeat_count = std::stoi(optarg); break; case 'o': data_offset = std::stoi(optarg); break; case 'd': delay_time = std::stoi(optarg); break; case 'w': wait_time = std::stoi(optarg); break; case '?': Usage(argv); break; } } tc::Error err; // We use the custom "repeat_int32" model which takes 3 inputs and // 1 output. For a single request the model will generate 'repeat_count' // responses. See is src/backends/backend/examples/repeat.cc. std::string model_name = "repeat_int32"; std::atomic received_response(0); // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); ResultMap result_map; // Note that client side statistics should be disabled in case of // of decoupled model. FAIL_IF_ERR( client->StartStream( [&](tc::InferResult* result) { { std::shared_ptr result_ptr(result); std::lock_guard lk(mutex_); std::string request_id; result->Id(&request_id); auto it = result_map.find(request_id); if (it == result_map.end()) { result_map[request_id] = ResultList(); } result_map[request_id].push_back(result_ptr); received_response++; } cv_.notify_all(); }, false /*enable_stats*/, 0 /* stream_timeout */, http_headers), "unable to establish a streaming connection to server"); // Prepare the data for the tensors std::vector in_data; std::vector delay_data; std::vector wait_data; for (int i = 0; i < repeat_count; i++) { in_data.push_back(data_offset + i); delay_data.push_back(delay_time); } wait_data.push_back(wait_time); // Initialize the inputs with the data. tc::InferInput* in; std::vector shape{repeat_count}; FAIL_IF_ERR( tc::InferInput::Create(&in, "IN", shape, "INT32"), "unable to create 'IN'"); std::shared_ptr in_ptr(in); FAIL_IF_ERR(in_ptr->Reset(), "unable to reset 'IN'"); FAIL_IF_ERR( in_ptr->AppendRaw( reinterpret_cast(&in_data[0]), sizeof(int32_t) * repeat_count), "unable to set data for 'IN'"); tc::InferInput* delay; FAIL_IF_ERR( tc::InferInput::Create(&delay, "DELAY", shape, "UINT32"), "unable to create 'DELAY'"); std::shared_ptr delay_ptr(delay); FAIL_IF_ERR(delay_ptr->Reset(), "unable to reset 'DELAY'"); FAIL_IF_ERR( delay_ptr->AppendRaw( reinterpret_cast(&delay_data[0]), sizeof(uint32_t) * repeat_count), "unable to set data for 'DELAY'"); tc::InferInput* wait; shape[0] = 1; FAIL_IF_ERR( tc::InferInput::Create(&wait, "WAIT", shape, "UINT32"), "unable to create 'WAIT'"); std::shared_ptr wait_ptr(wait); FAIL_IF_ERR(wait_ptr->Reset(), "unable to reset 'WAIT'"); FAIL_IF_ERR( wait_ptr->AppendRaw( reinterpret_cast(&wait_data[0]), sizeof(uint32_t)), "unable to set data for 'WAIT'"); std::vector inputs = { in_ptr.get(), delay_ptr.get(), wait_ptr.get()}; tc::InferOptions options(model_name); auto start_total = std::chrono::high_resolution_clock::now(); for (int id = 0; id < request_count; id++) { options.request_id_ = std::to_string(id); // Send inference request to the inference server. FAIL_IF_ERR( client->AsyncStreamInfer(options, inputs), "unable to run model"); } // Wait until all callbacks are invoked { std::unique_lock lk(mutex_); cv_.wait(lk, [&]() { if (received_response >= (repeat_count * request_count)) { return true; } else { return false; } }); } auto end_total = std::chrono::high_resolution_clock::now(); double total_time = std::chrono::duration(end_total - start_total).count(); std::cout << "Total time: " << total_time << " seconds" << std::endl; for (int i = 0; i < request_count; i++) { std::string id(std::to_string(i)); if (repeat_count == 0) { auto it = result_map.find(id); if (it != result_map.end()) { std::cerr << "received unexpected response for request id " << id << std::endl; exit(1); } } else { int32_t expected_output = data_offset; auto it = result_map.find(id); if (it == result_map.end()) { std::cerr << "response for request id " << id << " not received" << std::endl; exit(1); } if (it->second.size() != (uint32_t)repeat_count) { std::cerr << "expected " << repeat_count << " many responses, got " << it->second.size() << std::endl; exit(1); } for (auto this_result : it->second) { int32_t* output_data; size_t output_byte_size; FAIL_IF_ERR( this_result->RawData( "OUT", (const uint8_t**)&output_data, &output_byte_size), "unable to get result data for 'OUT'"); if (output_byte_size != 4) { std::cerr << "error: received incorrect byte size for 'OUT': " << output_byte_size << std::endl; exit(1); } if (*output_data != expected_output) { std::cerr << "error: incorrect result returned, expected " << expected_output << ", got " << *output_data << std::endl; exit(1); } expected_output++; } } } return 0; } int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; grpc_compression_algorithm compression_algorithm = grpc_compression_algorithm::GRPC_COMPRESS_NONE; int request_count = 1; int repeat_count = 1; int data_offset = 100; uint32_t delay_time = 1000; uint32_t wait_time = 500; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:r:s:o:d:w:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case 'r': request_count = std::stoi(optarg); break; case 's': repeat_count = std::stoi(optarg); break; case 'o': data_offset = std::stoi(optarg); break; case 'd': delay_time = std::stoi(optarg); break; case 'w': wait_time = std::stoi(optarg); break; case '?': Usage(argv); break; } } tc::Error err; // We use the custom "repeat_int32" model which takes 3 inputs and // 1 output. For a single request the model will generate 'repeat_count' // responses. See is src/backends/backend/examples/repeat.cc. std::string model_name = "repeat_int32"; std::atomic received_response(0); // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); ResultMap result_map; // Prepare the data for the tensors std::vector in_data; std::vector delay_data; std::vector wait_data; for (int i = 0; i < repeat_count; i++) { in_data.push_back(data_offset + i); delay_data.push_back(delay_time); } wait_data.push_back(wait_time); // Initialize the inputs with the data. tc::InferInput* in; std::vector shape{repeat_count}; FAIL_IF_ERR( tc::InferInput::Create(&in, "IN", shape, "INT32"), "unable to create 'IN'"); std::shared_ptr in_ptr(in); FAIL_IF_ERR(in_ptr->Reset(), "unable to reset 'IN'"); FAIL_IF_ERR( in_ptr->AppendRaw( reinterpret_cast(&in_data[0]), sizeof(int32_t) * repeat_count), "unable to set data for 'IN'"); tc::InferInput* delay; FAIL_IF_ERR( tc::InferInput::Create(&delay, "DELAY", shape, "UINT32"), "unable to create 'DELAY'"); std::shared_ptr delay_ptr(delay); FAIL_IF_ERR(delay_ptr->Reset(), "unable to reset 'DELAY'"); FAIL_IF_ERR( delay_ptr->AppendRaw( reinterpret_cast(&delay_data[0]), sizeof(uint32_t) * repeat_count), "unable to set data for 'DELAY'"); tc::InferInput* wait; shape[0] = 1; FAIL_IF_ERR( tc::InferInput::Create(&wait, "WAIT", shape, "UINT32"), "unable to create 'WAIT'"); std::shared_ptr wait_ptr(wait); FAIL_IF_ERR(wait_ptr->Reset(), "unable to reset 'WAIT'"); FAIL_IF_ERR( wait_ptr->AppendRaw( reinterpret_cast(&wait_data[0]), sizeof(uint32_t)), "unable to set data for 'WAIT'"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUT"), "unable to get 'OUT'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); std::vector inputs = { in_ptr.get(), delay_ptr.get(), wait_ptr.get()}; std::vector outputs = { output0_ptr.get()}; tc::InferOptions options(model_name); //预热连接 tc::InferResult* results; FAIL_IF_ERR( client->Infer( &results, options, inputs, outputs, http_headers, compression_algorithm), "unable to run model"); std::vector latencies; latencies.reserve(request_count); auto start_total = std::chrono::high_resolution_clock::now(); for (int id = 0; id < request_count; id++) { auto start = std::chrono::high_resolution_clock::now(); options.request_id_ = std::to_string(id); // Send inference request to the inference server. tc::InferResult* results; FAIL_IF_ERR( client->Infer( &results, options, inputs, outputs, http_headers, compression_algorithm), "unable to run model"); auto end = std::chrono::high_resolution_clock::now(); double latency = std::chrono::duration(end - start).count(); latencies.push_back(latency); } auto end_total = std::chrono::high_resolution_clock::now(); double total_time = std::chrono::duration(end_total - start_total).count(); // 计算统计信息 double avg_latency = 0; double min_latency = INFINITY, max_latency = 0; for (double lat : latencies) { avg_latency += lat; if (lat < min_latency) min_latency = lat; if (lat > max_latency) max_latency = lat; } avg_latency /= request_count; double throughput = request_count / total_time; std::cout << "Latency (ms):\n" << " Avg: " << avg_latency << "\n" << " Min: " << min_latency << "\n" << " Max: " << max_latency << "\n" << "Throughput: " << throughput << " requests/sec" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_health_metadata.cc000066400000000000000000000136161527052371200260770ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "grpc_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); bool live; FAIL_IF_ERR( client->IsServerLive(&live, http_headers), "unable to get server liveness"); if (!live) { std::cerr << "error: server is not live" << std::endl; exit(1); } bool ready; FAIL_IF_ERR( client->IsServerReady(&ready, http_headers), "unable to get server readiness"); if (!ready) { std::cerr << "error: server is not live" << std::endl; exit(1); } bool model_ready; FAIL_IF_ERR( client->IsModelReady( &model_ready, model_name, model_version, http_headers), "unable to get model readiness"); if (!model_ready) { std::cerr << "error: model " << model_name << " is not live" << std::endl; exit(1); } inference::ServerMetadataResponse server_metadata; FAIL_IF_ERR( client->ServerMetadata(&server_metadata, http_headers), "unable to get server metadata"); if (server_metadata.name().compare("triton") != 0) { std::cerr << "error: unexpected server metadata: " << server_metadata.DebugString() << std::endl; exit(1); } inference::ModelMetadataResponse model_metadata; FAIL_IF_ERR( client->ModelMetadata( &model_metadata, model_name, model_version, http_headers), "unable to get model metadata"); if (model_metadata.name().compare(model_name) != 0) { std::cerr << "error: unexpected model metadata: " << model_metadata.DebugString() << std::endl; exit(1); } inference::ModelConfigResponse model_config; FAIL_IF_ERR( client->ModelConfig( &model_config, model_name, model_version, http_headers), "unable to get model config"); if (model_config.config().name().compare(model_name) != 0) { std::cerr << "error: unexpected model config: " << model_config.DebugString() << std::endl; exit(1); } tc::Error err = client->ModelMetadata( &model_metadata, "wrong_model_name", model_version, http_headers); if (err.IsOk()) { std::cerr << "error: expected an error but got: " << err << std::endl; exit(1); } std::cout << err << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_infer_client.cc000066400000000000000000000316231527052371200254310ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include "grpc_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-m " << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << "\tFor -H, header must be 'Header:Value'. May be given multiple times." << std::endl; std::cerr << "\t-C . \'deflate\', " "\'gzip\' and \'none\' are supported" << std::endl; std::cerr << "\t-c . " " Use cached channel when creating new client. " " Specify 'true' or 'false'. True by default" << std::endl; std::cerr << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; uint32_t client_timeout = 0; bool use_ssl = false; std::string root_certificates; std::string private_key; std::string certificate_chain; grpc_compression_algorithm compression_algorithm = grpc_compression_algorithm::GRPC_COMPRESS_NONE; bool test_use_cached_channel = false; bool use_cached_channel = true; // {name, has_arg, *flag, val} static struct option long_options[] = { {"ssl", 0, 0, 0}, {"root-certificates", 1, 0, 1}, {"private-key", 1, 0, 2}, {"certificate-chain", 1, 0, 3}}; // Parse commandline... int opt; while ((opt = getopt_long(argc, argv, "vu:t:H:C:c:", long_options, NULL)) != -1) { switch (opt) { case 0: use_ssl = true; break; case 1: root_certificates = optarg; break; case 2: private_key = optarg; break; case 3: certificate_chain = optarg; break; case 'v': verbose = true; break; case 'u': url = optarg; break; case 't': client_timeout = std::stoi(optarg); break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); if (header.size() == arg.size() || header.empty()) { Usage( argv, "HTTP header specified incorrectly. Must be formmated as " "'Header:Value'"); } else { http_headers[header] = arg.substr(header.size() + 1); } break; } case 'C': { std::string algorithm_str{optarg}; if (algorithm_str.compare("deflate") == 0) { compression_algorithm = grpc_compression_algorithm::GRPC_COMPRESS_DEFLATE; } else if (algorithm_str.compare("gzip") == 0) { compression_algorithm = grpc_compression_algorithm::GRPC_COMPRESS_GZIP; } else if (algorithm_str.compare("none") == 0) { compression_algorithm = grpc_compression_algorithm::GRPC_COMPRESS_NONE; } else { Usage( argv, "unsupported compression algorithm specified... only " "\'deflate\', " "\'gzip\' and \'none\' are supported."); } break; } case 'c': { test_use_cached_channel = true; std::string arg = optarg; if (arg.find("false") != std::string::npos) { use_cached_channel = false; } else if (arg.find("true") != std::string::npos) { use_cached_channel = true; } else { Usage(argv, "need to specify true or false for use_cached_channel"); } break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; tc::SslOptions ssl_options = tc::SslOptions(); std::string err; if (use_ssl) { ssl_options.root_certificates = root_certificates; ssl_options.private_key = private_key; ssl_options.certificate_chain = certificate_chain; err = "unable to create secure grpc client"; } else { err = "unable to create grpc client"; } // Run with the same name to ensure cached channel is not used int numRuns = test_use_cached_channel ? 2 : 1; for (int i = 0; i < numRuns; ++i) { FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create( &client, url, verbose, use_ssl, ssl_options, tc::KeepAliveOptions(), use_cached_channel), err); if (verbose) { std::cout << "There are " << client->GetNumCachedChannels() << " cached channels" << std::endl; } // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. std::vector input0_data(16); std::vector input1_data(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = i; input1_data[i] = 1; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); FAIL_IF_ERR( input0_ptr->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for INPUT0"); FAIL_IF_ERR( input1_ptr->AppendRaw( reinterpret_cast(&input1_data[0]), input1_data.size() * sizeof(int32_t)), "unable to set data for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; options.client_timeout_ = client_timeout; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer( &results, options, inputs, outputs, http_headers, compression_algorithm), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); // Get pointers to the result returned... int32_t* output0_data; size_t output0_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT0", (const uint8_t**)&output0_data, &output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << output0_byte_size << std::endl; exit(1); } int32_t* output1_data; size_t output1_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT1", (const uint8_t**)&output1_data, &output1_byte_size), "unable to get result data for 'OUTPUT1'"); if (output1_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT1': " << output1_byte_size << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data[i] << " + " << input1_data[i] << " = " << *(output0_data + i) << std::endl; std::cout << input0_data[i] << " - " << input1_data[i] << " = " << *(output1_data + i) << std::endl; if ((input0_data[i] + input1_data[i]) != *(output0_data + i)) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_data[i] - input1_data[i]) != *(output1_data + i)) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get full response std::cout << results_ptr->DebugString() << std::endl; tc::InferStat infer_stat; client->ClientInferStat(&infer_stat); std::cout << "======Client Statistics======" << std::endl; std::cout << "completed_request_count " << infer_stat.completed_request_count << std::endl; std::cout << "cumulative_total_request_time_ns " << infer_stat.cumulative_total_request_time_ns << std::endl; std::cout << "cumulative_send_time_ns " << infer_stat.cumulative_send_time_ns << std::endl; std::cout << "cumulative_receive_time_ns " << infer_stat.cumulative_receive_time_ns << std::endl; inference::ModelStatisticsResponse model_stat; client->ModelInferenceStatistics(&model_stat, model_name); std::cout << "======Model Statistics======" << std::endl; std::cout << model_stat.DebugString() << std::endl; std::cout << "PASS : Infer" << std::endl; } return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_keepalive_client.cc000066400000000000000000000240431527052371200262710ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include "grpc_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-m " << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << "\t--grpc-keepalive-time " << std::endl; std::cerr << "\t--grpc-keepalive-timeout " << std::endl; std::cerr << "\t--grpc-keepalive-permit-without-calls" << std::endl; std::cerr << "\t--grpc-http2-max-pings-without-data " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; uint32_t client_timeout = 0; bool use_ssl = false; tc::SslOptions ssl_options; tc::KeepAliveOptions keepalive_options; // GRPC KeepAlive: https://grpc.github.io/grpc/cpp/md_doc_keepalive.html int keepalive_time_ms = INT_MAX; int keepalive_timeout_ms = 20000; bool keepalive_permit_without_calls = false; int http2_max_pings_without_data = 2; // {name, has_arg, *flag, val} static struct option long_options[] = { {"grpc-keepalive-time", 1, 0, 0}, {"grpc-keepalive-timeout", 1, 0, 1}, {"grpc-keepalive-permit-without-calls", 0, 0, 2}, {"grpc-http2-max-pings-without-data", 1, 0, 3}}; // Parse commandline... int opt; while ((opt = getopt_long(argc, argv, "vu:t:H:C:", long_options, NULL)) != -1) { switch (opt) { case 0: keepalive_options.keepalive_time_ms = std::stoi(optarg); break; case 1: keepalive_options.keepalive_timeout_ms = std::stoi(optarg); break; case 2: keepalive_options.keepalive_permit_without_calls = true; break; case 3: keepalive_options.http2_max_pings_without_data = std::stoi(optarg); break; case 'v': verbose = true; break; case 'u': url = optarg; break; case 't': client_timeout = std::stoi(optarg); break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create( &client, url, verbose, use_ssl, ssl_options, keepalive_options), "unable to create grpc client"); // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. std::vector input0_data(16); std::vector input1_data(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = i; input1_data[i] = 1; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); FAIL_IF_ERR( input0_ptr->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for INPUT0"); FAIL_IF_ERR( input1_ptr->AppendRaw( reinterpret_cast(&input1_data[0]), input1_data.size() * sizeof(int32_t)), "unable to set data for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; options.client_timeout_ = client_timeout; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer(&results, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); // Get pointers to the result returned... int32_t* output0_data; size_t output0_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT0", (const uint8_t**)&output0_data, &output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << output0_byte_size << std::endl; exit(1); } int32_t* output1_data; size_t output1_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT1", (const uint8_t**)&output1_data, &output1_byte_size), "unable to get result data for 'OUTPUT1'"); if (output1_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT1': " << output1_byte_size << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data[i] << " + " << input1_data[i] << " = " << *(output0_data + i) << std::endl; std::cout << input0_data[i] << " - " << input1_data[i] << " = " << *(output1_data + i) << std::endl; if ((input0_data[i] + input1_data[i]) != *(output0_data + i)) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_data[i] - input1_data[i]) != *(output1_data + i)) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get full response std::cout << results_ptr->DebugString() << std::endl; std::cout << "PASS : KeepAlive" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_model_control.cc000066400000000000000000000114111527052371200256210ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "grpc_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } std::string model_name = "simple"; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); inference::RepositoryIndexResponse repository_index; FAIL_IF_ERR( client->ModelRepositoryIndex(&repository_index, http_headers), "Failed to get repository index"); if (repository_index.models().size() != 8) { std::cerr << "expected number of models 8, got " << repository_index.models().size() << std::endl; exit(1); } FAIL_IF_ERR( client->LoadModel(model_name, http_headers), "Failed to load model"); bool model_ready; FAIL_IF_ERR( client->IsModelReady(&model_ready, model_name), "unable to get model readiness"); if (!model_ready) { std::cerr << "error: model " << model_name << " is not live" << std::endl; exit(1); } FAIL_IF_ERR( client->UnloadModel(model_name, http_headers), "Failed to unload model"); FAIL_IF_ERR( client->IsModelReady(&model_ready, model_name), "unable to get model readiness"); if (model_ready) { std::cerr << "error: model " << model_name << " is live after unloading" << std::endl; exit(1); } tc::Error err = client->LoadModel("wrong_model_name", http_headers); if (err.IsOk()) { std::cerr << "error: wrong model name was successfully loaded" << std::endl; } return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_sequence_stream_infer_client.cc000066400000000000000000000312031527052371200306660ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include "grpc_client.h" namespace tc = triton::client; using ResultList = std::vector>; // Global mutex to synchronize the threads std::mutex mutex_; std::condition_variable cv_; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << "\t-o " << std::endl; std::cerr << std::endl; std::cerr << "For -o, the client will use sequence ID <1 + 2 * offset> " << "and <2 + 2 * offset>. Default offset is 0." << std::endl; exit(1); } void StreamSend( const std::unique_ptr& client, tc::InferOptions& options, int32_t value, const int32_t index) { // Initialize the inputs with the data. tc::InferInput* input; std::vector shape{1, 1}; FAIL_IF_ERR( tc::InferInput::Create(&input, "INPUT", shape, "INT32"), "unable to create 'INPUT'"); std::shared_ptr ivalue(input); FAIL_IF_ERR(ivalue->Reset(), "unable to reset 'INPUT'"); FAIL_IF_ERR( ivalue->AppendRaw(reinterpret_cast(&value), sizeof(int32_t)), "unable to set data for 'INPUT'"); std::vector inputs = {ivalue.get()}; // Send inference request to the inference server. FAIL_IF_ERR(client->AsyncStreamInfer(options, inputs), "unable to run model"); } void StreamSend( const std::unique_ptr& client, const std::string& model_name, int32_t value, const uint64_t sequence_id, bool start_of_sequence, bool end_of_sequence, const int32_t index) { // Stream send for unsigned int sequence IDs tc::InferOptions options(model_name); options.sequence_id_ = sequence_id; options.sequence_start_ = start_of_sequence; options.sequence_end_ = end_of_sequence; options.request_id_ = std::to_string(sequence_id) + "_" + std::to_string(index); StreamSend(client, options, value, index); } void StreamSend( const std::unique_ptr& client, const std::string& model_name, int32_t value, const std::string& sequence_id, bool start_of_sequence, bool end_of_sequence, const int32_t index) { // Stream send for string sequence IDs tc::InferOptions options(model_name); options.sequence_id_str_ = sequence_id; options.sequence_start_ = start_of_sequence; options.sequence_end_ = end_of_sequence; options.request_id_ = sequence_id + "_" + std::to_string(index); StreamSend(client, options, value, index); } } // namespace int main(int argc, char** argv) { bool verbose = false; bool dyna_sequence = false; std::string url("localhost:8001"); tc::Headers http_headers; int sequence_id_offset = 0; uint32_t stream_timeout = 0; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vdu:H:t:o:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'd': dyna_sequence = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case 't': stream_timeout = std::stoi(optarg); break; case 'o': sequence_id_offset = std::stoi(optarg); break; case '?': Usage(argv); break; } } tc::Error err; // We use the custom "sequence" model which takes 1 input value. The // output is the accumulated value of the inputs. See // src/custom/sequence. std::string int_model_name = dyna_sequence ? "simple_dyna_sequence" : "simple_sequence"; std::string string_model_name = dyna_sequence ? "simple_string_dyna_sequence" : "simple_sequence"; const uint64_t int_sequence_id0 = 1 + sequence_id_offset * 2; const uint64_t int_sequence_id1 = 2 + sequence_id_offset * 2; // For string sequence IDs, the dyna backend requires that the // sequence id be decodable into an integer, otherwise we'll use // a test string sequence id and a model that doesn't require corrid // control. const std::string string_sequence_id0 = dyna_sequence ? std::to_string(3 + sequence_id_offset * 2) : "SEQ-3"; std::cout << "sequence ID " << int_sequence_id0 << " : " << "sequence ID " << int_sequence_id1 << " : " << "sequence ID " << string_sequence_id0 << std::endl; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); // Now send the inference sequences.. // std::vector values{11, 7, 5, 3, 2, 0, 1}; ResultList result_list; FAIL_IF_ERR( client->StartStream( [&](tc::InferResult* result) { { std::shared_ptr result_ptr(result); std::lock_guard lk(mutex_); result_list.push_back(result_ptr); } cv_.notify_all(); }, false /*ship_stats*/, stream_timeout, http_headers), "unable to establish a streaming connection to server"); // Send requests, first reset accumulator for the sequence. int32_t index = 0; StreamSend( client, int_model_name, 0, int_sequence_id0, true /* start-of-sequence */, false /* end-of-sequence */, index++); StreamSend( client, int_model_name, 100, int_sequence_id1, true /* start-of-sequence */, false /* end-of-sequence */, index++); StreamSend( client, string_model_name, 20, string_sequence_id0, true /* start-of-sequence */, false /* end-of-sequence */, index++); // Now send a sequence of values... for (int32_t v : values) { StreamSend( client, int_model_name, v, int_sequence_id0, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, index++); StreamSend( client, int_model_name, -v, int_sequence_id1, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, index++); StreamSend( client, string_model_name, -v, string_sequence_id0, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, index++); } if (stream_timeout == 0) { // Wait until all callbacks are invoked { std::unique_lock lk(mutex_); cv_.wait(lk, [&]() { if (result_list.size() > (3 * values.size() + 2)) { return true; } else { return false; } }); } } else { auto timeout = std::chrono::microseconds(stream_timeout); // Wait until all callbacks are invoked or the timeout expires { std::unique_lock lk(mutex_); if (!cv_.wait_for(lk, timeout, [&]() { return (result_list.size() > (3 * values.size() + 2)); })) { std::cerr << "Stream has been closed" << std::endl; exit(1); } } } // Extract data from the result std::vector int_result0_data; std::vector int_result1_data; std::vector string_result0_data; for (const auto& this_result : result_list) { auto err = this_result->RequestStatus(); if (!err.IsOk()) { std::cerr << "The inference failed: " << err << std::endl; exit(1); } // Get pointers to the result returned... int32_t* output_data; size_t output_byte_size; FAIL_IF_ERR( this_result->RawData( "OUTPUT", (const uint8_t**)&output_data, &output_byte_size), "unable to get result data for 'OUTPUT'"); if (output_byte_size != 4) { std::cerr << "error: received incorrect byte size for 'OUTPUT': " << output_byte_size << std::endl; exit(1); } std::string request_id; FAIL_IF_ERR( this_result->Id(&request_id), "unable to get request id for response"); try { std::string this_sequence_id = std::string(request_id, 0, request_id.find("_")); if (std::stoi(this_sequence_id) == int_sequence_id0) { int_result0_data.push_back(*output_data); } else if (std::stoi(this_sequence_id) == int_sequence_id1) { int_result1_data.push_back(*output_data); } else if (this_sequence_id == string_sequence_id0) { string_result0_data.push_back(*output_data); } else { std::cerr << "error: received incorrect sequence id in response: " << this_sequence_id << std::endl; exit(1); } } catch (std::invalid_argument& e) { // stoi will throw this when called with the test sequence SEQ3 string_result0_data.push_back(*output_data); } } for (size_t i = 0; i < int_result0_data.size(); i++) { int32_t int_seq0_expected = (i == 0) ? 1 : values[i - 1]; int32_t int_seq1_expected = (i == 0) ? 101 : values[i - 1] * -1; int32_t string_seq0_expected; // For string sequence ID case we are testing two different backends if ((i == 0) && dyna_sequence) { string_seq0_expected = 20; } else if ((i == 0) && !dyna_sequence) { string_seq0_expected = 21; } else if ((i != 0) && dyna_sequence) { string_seq0_expected = values[i - 1] * -1 + string_result0_data[i - 1]; } else { string_seq0_expected = values[i - 1] * -1; } // The dyna_sequence custom backend adds the sequence ID to // the last request in a sequence. if (dyna_sequence && (i != 0) && (values[i - 1] == 1)) { int_seq0_expected += int_sequence_id0; int_seq1_expected += int_sequence_id1; string_seq0_expected += std::stoi(string_sequence_id0); } std::cout << "[" << i << "] " << int_result0_data[i] << " : " << int_result1_data[i] << " : " << string_result0_data[i] << std::endl; if ((int_seq0_expected != int_result0_data[i]) || (int_seq1_expected != int_result1_data[i]) || (string_seq0_expected != string_result0_data[i])) { std::cout << "[ expected ] " << int_seq0_expected << " : " << int_seq1_expected << " : " << string_seq0_expected << std::endl; return 1; } } return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_sequence_sync_infer_client.cc000066400000000000000000000252321527052371200303540ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include "grpc_client.h" namespace tc = triton::client; using ResultList = std::vector>; // Global mutex to synchronize the threads std::mutex mutex_; std::condition_variable cv_; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; std::cerr << "\t-o " << std::endl; std::cerr << std::endl; std::cerr << "For -o, the client will use sequence ID <1 + 2 * offset> " << "and <2 + 2 * offset>. Default offset is 0." << std::endl; exit(1); } void SyncSend( const std::unique_ptr& client, tc::InferOptions& options, int32_t value, std::vector& result_data, tc::Headers& http_headers) { // Initialize the inputs with the data. tc::InferInput* input; std::vector shape{1, 1}; FAIL_IF_ERR( tc::InferInput::Create(&input, "INPUT", shape, "INT32"), "unable to create 'INPUT'"); std::shared_ptr ivalue(input); FAIL_IF_ERR(ivalue->Reset(), "unable to reset 'INPUT'"); FAIL_IF_ERR( ivalue->AppendRaw(reinterpret_cast(&value), sizeof(int32_t)), "unable to set data for 'INPUT'"); std::vector inputs = {ivalue.get()}; tc::InferRequestedOutput* output; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output, "OUTPUT"), "unable to get 'OUTPUT'"); std::shared_ptr routput; routput.reset(output); std::vector outputs = {routput.get()}; tc::InferResult* result; // Send inference request to the inference server. FAIL_IF_ERR( client->Infer(&result, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr this_result(result); // Get pointers to the result returned... int32_t* output_data; size_t output_byte_size; FAIL_IF_ERR( this_result->RawData( "OUTPUT", (const uint8_t**)&output_data, &output_byte_size), "unable to get result data for 'OUTPUT'"); if (output_byte_size != 4) { std::cerr << "error: received incorrect byte size for 'OUTPUT': " << output_byte_size << std::endl; exit(1); } result_data.push_back(*output_data); } void SyncSend( const std::unique_ptr& client, const std::string& model_name, int32_t value, const uint64_t sequence_id, bool start_of_sequence, bool end_of_sequence, std::vector& result_data, tc::Headers& http_headers) { tc::InferOptions options(model_name); options.sequence_id_ = sequence_id; options.sequence_start_ = start_of_sequence; options.sequence_end_ = end_of_sequence; SyncSend(client, options, value, result_data, http_headers); } void SyncSend( const std::unique_ptr& client, const std::string& model_name, int32_t value, const std::string& sequence_id, bool start_of_sequence, bool end_of_sequence, std::vector& result_data, tc::Headers& http_headers) { tc::InferOptions options(model_name); options.sequence_id_str_ = sequence_id; options.sequence_start_ = start_of_sequence; options.sequence_end_ = end_of_sequence; SyncSend(client, options, value, result_data, http_headers); } } // namespace int main(int argc, char** argv) { bool verbose = false; bool dyna_sequence = false; std::string url("localhost:8001"); int sequence_id_offset = 0; tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vdu:H:o:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'd': dyna_sequence = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case 'o': sequence_id_offset = std::stoi(optarg); break; case '?': Usage(argv); break; } } tc::Error err; // We use the custom "sequence" model which takes 1 input value. The // output is the accumulated value of the inputs. See // src/custom/sequence. std::string int_model_name = dyna_sequence ? "simple_dyna_sequence" : "simple_sequence"; std::string string_model_name = dyna_sequence ? "simple_string_dyna_sequence" : "simple_sequence"; const uint64_t int_sequence_id0 = 1 + sequence_id_offset * 2; const uint64_t int_sequence_id1 = 2 + sequence_id_offset * 2; // For string sequence IDs, the dyna backend requires that the // sequence id be decodable into an integer, otherwise we'll use // a test string sequence id and a model that doesn't require corrid // control. const std::string string_sequence_id0 = dyna_sequence ? std::to_string(3 + sequence_id_offset * 2) : "SEQ-3"; std::cout << "sequence ID " << int_sequence_id0 << " : " << "sequence ID " << int_sequence_id1 << " : " << "sequence ID " << string_sequence_id0 << std::endl; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); // Now send the inference sequences.. // std::vector values{11, 7, 5, 3, 2, 0, 1}; std::vector int_result0_data; std::vector int_result1_data; std::vector string_result0_data; // Send requests, first reset accumulator for the sequence. SyncSend( client, int_model_name, 0, int_sequence_id0, true /* start-of-sequence */, false /* end-of-sequence */, int_result0_data, http_headers); SyncSend( client, int_model_name, 100, int_sequence_id1, true /* start-of-sequence */, false /* end-of-sequence */, int_result1_data, http_headers); SyncSend( client, string_model_name, 20, string_sequence_id0, true /* start-of-sequence */, false /* end-of-sequence */, string_result0_data, http_headers); // Now send a sequence of values... for (int32_t v : values) { SyncSend( client, int_model_name, v, int_sequence_id0, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, int_result0_data, http_headers); SyncSend( client, int_model_name, -v, int_sequence_id1, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, int_result1_data, http_headers); SyncSend( client, string_model_name, -v, string_sequence_id0, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, string_result0_data, http_headers); } for (size_t i = 0; i < int_result0_data.size(); i++) { int32_t int_seq0_expected = (i == 0) ? 1 : values[i - 1]; int32_t int_seq1_expected = (i == 0) ? 101 : values[i - 1] * -1; int32_t string_seq0_expected; // For string sequence ID case we are testing two different backends if ((i == 0) && dyna_sequence) { string_seq0_expected = 20; } else if ((i == 0) && !dyna_sequence) { string_seq0_expected = 21; } else if ((i != 0) && dyna_sequence) { string_seq0_expected = values[i - 1] * -1 + string_result0_data[i - 1]; } else { string_seq0_expected = values[i - 1] * -1; } // The dyna_sequence custom backend adds the sequence ID to // the last request in a sequence. if (dyna_sequence && (i != 0) && (values[i - 1] == 1)) { int_seq0_expected += int_sequence_id0; int_seq1_expected += int_sequence_id1; string_seq0_expected += std::stoi(string_sequence_id0); } std::cout << "[" << i << "] " << int_result0_data[i] << " : " << int_result1_data[i] << " : " << string_result0_data[i] << std::endl; if ((int_seq0_expected != int_result0_data[i]) || (int_seq1_expected != int_result1_data[i]) || (string_seq0_expected != string_result0_data[i])) { std::cout << "[ expected ] " << int_seq0_expected << " : " << int_seq1_expected << " : " << string_seq0_expected << std::endl; return 1; } } return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_shm_client.cc000066400000000000000000000242221527052371200251120ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "grpc_client.h" #include "shm_utils.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); // Unregistering all shared memory regions for a clean // start. FAIL_IF_ERR( client->UnregisterSystemSharedMemory(), "unable to unregister all system shared memory regions"); FAIL_IF_ERR( client->UnregisterCudaSharedMemory(), "unable to unregister all cuda shared memory regions"); std::vector shape{1, 16}; size_t input_byte_size = 64; size_t output_byte_size = 64; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); // Create Input0 and Input1 in Shared Memory. Initialize Input0 to unique // integers and Input1 to all ones. std::string shm_key = "/input_simple"; int shm_fd_ip, *input0_shm; FAIL_IF_ERR( tc::CreateSharedMemoryRegion(shm_key, input_byte_size * 2, &shm_fd_ip), ""); FAIL_IF_ERR( tc::MapSharedMemory( shm_fd_ip, 0, input_byte_size * 2, (void**)&input0_shm), ""); FAIL_IF_ERR(tc::CloseSharedMemory(shm_fd_ip), ""); int* input1_shm = (int*)(input0_shm + 16); for (size_t i = 0; i < 16; ++i) { *(input0_shm + i) = i; *(input1_shm + i) = 1; } FAIL_IF_ERR( client->RegisterSystemSharedMemory( "input_data", "/input_simple", input_byte_size * 2), "failed to register input shared memory region"); FAIL_IF_ERR( input0_ptr->SetSharedMemory( "input_data", input_byte_size, 0 /* offset */), "unable to set shared memory for INPUT0"); FAIL_IF_ERR( input1_ptr->SetSharedMemory( "input_data", input_byte_size, input_byte_size /* offset */), "unable to set shared memory for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // Create Output0 and Output1 in Shared Memory shm_key = "/output_simple"; int shm_fd_op; int* output0_shm; FAIL_IF_ERR( tc::CreateSharedMemoryRegion(shm_key, output_byte_size * 2, &shm_fd_op), ""); FAIL_IF_ERR( tc::MapSharedMemory( shm_fd_op, 0, output_byte_size * 2, (void**)&output0_shm), ""); FAIL_IF_ERR(tc::CloseSharedMemory(shm_fd_op), ""); int* output1_shm = (int*)(output0_shm + 16); FAIL_IF_ERR( client->RegisterSystemSharedMemory( "output_data", "/output_simple", output_byte_size * 2), "failed to register output shared memory region"); FAIL_IF_ERR( output0_ptr->SetSharedMemory( "output_data", output_byte_size, 0 /* offset */), "unable to set shared memory for 'OUTPUT0'"); FAIL_IF_ERR( output1_ptr->SetSharedMemory( "output_data", output_byte_size, output_byte_size /* offset */), "unable to set shared memory for 'OUTPUT1'"); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer(&results, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); for (size_t i = 0; i < 16; ++i) { std::cout << input0_shm[i] << " + " << input1_shm[i] << " = " << output0_shm[i] << std::endl; std::cout << input0_shm[i] << " - " << input1_shm[i] << " = " << output1_shm[i] << std::endl; if ((input0_shm[i] + input1_shm[i]) != output0_shm[i]) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_shm[i] - input1_shm[i]) != output1_shm[i]) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get shared memory regions active/registered within triton inference::SystemSharedMemoryStatusResponse status; FAIL_IF_ERR( client->SystemSharedMemoryStatus(&status), "failed to get shared memory status"); std::cout << "Shared Memory Status:\n" << status.DebugString() << "\n"; // Unregister shared memory FAIL_IF_ERR( client->UnregisterSystemSharedMemory("input_data"), "unable to unregister shared memory input region"); FAIL_IF_ERR( client->UnregisterSystemSharedMemory("output_data"), "unable to unregister shared memory output region"); // Cleanup shared memory FAIL_IF_ERR(tc::UnmapSharedMemory(input0_shm, input_byte_size * 2), ""); FAIL_IF_ERR(tc::UnlinkSharedMemoryRegion("/input_simple"), ""); FAIL_IF_ERR(tc::UnmapSharedMemory(output0_shm, output_byte_size * 2), ""); FAIL_IF_ERR(tc::UnlinkSharedMemoryRegion("/output_simple"), ""); std::cout << "PASS : System Shared Memory " << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_grpc_string_infer_client.cc000066400000000000000000000205021527052371200270110ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "grpc_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR(result->Shape(name, &shape), "unable to get shape for " + name); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for " << name << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for " + name); // Validate datatype if (datatype.compare("BYTES") != 0) { std::cerr << "error: received incorrect datatype for " << name << ": " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8001"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 strings // each and returns 2 output tensors of 16 strings each. The input // strings must represent integers. One output tensor is the // element-wise sum of the inputs and one output is the element-wise // difference. std::string model_name = "simple_string"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&client, url, verbose), "unable to create grpc client"); // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. The input tensors // are the string representation of these values. std::vector input0_data(16); std::vector input1_data(16); std::vector expected_sum(16); std::vector expected_diff(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = std::to_string(i); input1_data[i] = std::to_string(1); expected_sum[i] = i + 1; expected_diff[i] = i - 1; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "BYTES"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "BYTES"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); FAIL_IF_ERR( input0_ptr->AppendFromString(input0_data), "unable to set data for INPUT0"); FAIL_IF_ERR( input1_ptr->AppendFromString(input1_data), "unable to set data for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get OUTPUT0"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get OUTPUT1"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer(&results, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); // Get the result data std::vector result0_data; std::vector result1_data; FAIL_IF_ERR( results_ptr->StringData("OUTPUT0", &result0_data), "unable to get data for OUTPUT0"); if (result0_data.size() != 16) { std::cerr << "error: received incorrect number of strings for OUTPUT0: " << result0_data.size() << std::endl; exit(1); } FAIL_IF_ERR( results_ptr->StringData("OUTPUT1", &result1_data), "unable to get data for OUTPUT1"); if (result1_data.size() != 16) { std::cerr << "error: received incorrect number of strings for OUTPUT1: " << result1_data.size() << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data[i] << " + " << input0_data[i] << " = " << result0_data[i] << std::endl; std::cout << input0_data[i] << " - " << input1_data[i] << " = " << result1_data[i] << std::endl; if (expected_sum[i] != std::stoi(result0_data[i])) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if (expected_diff[i] != std::stoi(result1_data[i])) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get full response std::cout << results_ptr->DebugString() << std::endl; std::cout << "PASS : String Infer" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_http_async_infer_client.cc000066400000000000000000000273571527052371200266630ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include "http_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void ValidateResult( const std::shared_ptr result, std::vector& input0_data, std::vector& input1_data) { // Validate the results... ValidateShapeAndDatatype("OUTPUT0", result); ValidateShapeAndDatatype("OUTPUT1", result); // Get pointers to the result returned... int32_t* output0_data; size_t output0_byte_size; FAIL_IF_ERR( result->RawData( "OUTPUT0", (const uint8_t**)&output0_data, &output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << output0_byte_size << std::endl; exit(1); } int32_t* output1_data; size_t output1_byte_size; FAIL_IF_ERR( result->RawData( "OUTPUT1", (const uint8_t**)&output1_data, &output1_byte_size), "unable to get result data for 'OUTPUT1'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT1': " << output0_byte_size << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data[i] << " + " << input1_data[i] << " = " << *(output0_data + i) << std::endl; std::cout << input0_data[i] << " - " << input1_data[i] << " = " << *(output1_data + i) << std::endl; if ((input0_data[i] + input1_data[i]) != *(output0_data + i)) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_data[i] - input1_data[i]) != *(output1_data + i)) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get full response std::cout << result->DebugString() << std::endl; } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); tc::Headers http_headers; uint32_t client_timeout = 0; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:t:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 't': client_timeout = std::stoi(optarg); break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerHttpClient instance to communicate with the // server using HTTP protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&client, url, verbose), "unable to create http client"); // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. std::vector input0_data(16); std::vector input1_data(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = i; input1_data[i] = 1; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); FAIL_IF_ERR( input0_ptr->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for INPUT0"); FAIL_IF_ERR( input1_ptr->AppendRaw( reinterpret_cast(&input1_data[0]), input1_data.size() * sizeof(int32_t)), "unable to set data for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; options.client_timeout_ = client_timeout; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; // Send inference request to the inference server. std::mutex mtx; std::condition_variable cv; size_t repeat_cnt = 2; size_t done_cnt = 0; for (size_t i = 0; i < repeat_cnt; i++) { FAIL_IF_ERR( client->AsyncInfer( [&, i](tc::InferResult* result) { { std::shared_ptr result_ptr; result_ptr.reset(result); std::lock_guard lk(mtx); std::cout << "Callback no." << i << " is called" << std::endl; done_cnt++; if (result_ptr->RequestStatus().IsOk()) { ValidateResult(result_ptr, input0_data, input1_data); } else { std::cerr << "error: Inference failed: " << result_ptr->RequestStatus() << std::endl; exit(1); } } cv.notify_all(); }, options, inputs, outputs, http_headers), "unable to run model"); } // Wait until all callbacks are invoked { std::unique_lock lk(mtx); cv.wait(lk, [&]() { if (done_cnt >= repeat_cnt) { return true; } else { return false; } }); } if (done_cnt == repeat_cnt) { std::cout << "All done" << std::endl; } else { std::cerr << "Done cnt: " << done_cnt << " does not match repeat cnt: " << repeat_cnt << std::endl; exit(1); } // Send another AsyncInfer whose callback defers the completed request // to another thread (main thread) to handle bool callback_invoked = false; std::shared_ptr result_placeholder; FAIL_IF_ERR( client->AsyncInfer( [&](tc::InferResult* result) { { std::shared_ptr result_ptr; result_ptr.reset(result); // Defer the response retrieval to main thread std::lock_guard lk(mtx); callback_invoked = true; result_placeholder = std::move(result_ptr); } cv.notify_all(); }, options, inputs, outputs, http_headers), "unable to run model"); // Ensure callback is completed { std::unique_lock lk(mtx); cv.wait(lk, [&]() { return callback_invoked; }); } // Get deferred response std::cout << "Getting results from deferred response" << std::endl; if (result_placeholder->RequestStatus().IsOk()) { ValidateResult(result_placeholder, input0_data, input1_data); } else { std::cerr << "error: Inference failed: " << result_placeholder->RequestStatus() << std::endl; exit(1); } tc::InferStat infer_stat; client->ClientInferStat(&infer_stat); std::cout << "completed_request_count " << infer_stat.completed_request_count << std::endl; std::cout << "cumulative_total_request_time_ns " << infer_stat.cumulative_total_request_time_ns << std::endl; std::cout << "cumulative_send_time_ns " << infer_stat.cumulative_send_time_ns << std::endl; std::cout << "cumulative_receive_time_ns " << infer_stat.cumulative_receive_time_ns << std::endl; std::cout << "PASS : Async Infer" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_http_cudashm_client.cc000066400000000000000000000260431527052371200257760ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include "http_client.h" #include "shm_utils.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace #define FAIL_IF_CUDA_ERR(FUNC) \ { \ const cudaError_t result = FUNC; \ if (result != cudaSuccess) { \ std::cerr << "CUDA exception (line " << __LINE__ \ << "): " << cudaGetErrorName(result) << " (" \ << cudaGetErrorString(result) << ")" << std::endl; \ exit(1); \ } \ } void CreateCUDAIPCHandle( cudaIpcMemHandle_t* cuda_handle, void* input_d_ptr, int device_id = 0) { // Set the GPU device to the desired GPU FAIL_IF_CUDA_ERR(cudaSetDevice(device_id)); // Create IPC handle for data on the gpu FAIL_IF_CUDA_ERR(cudaIpcGetMemHandle(cuda_handle, input_d_ptr)); } int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerHttpClient instance to communicate with the // server using http protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&client, url, verbose), "unable to create http client"); // Unregistering all shared memory regions for a clean // start. FAIL_IF_ERR( client->UnregisterSystemSharedMemory(), "unable to unregister all system shared memory regions"); FAIL_IF_ERR( client->UnregisterCudaSharedMemory(), "unable to unregister all cuda shared memory regions"); std::vector shape{1, 16}; size_t input_byte_size = 64; size_t output_byte_size = 64; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); // Create Input0 and Input1 in CUDA Shared Memory. Initialize Input0 to // unique integers and Input1 to all ones. int input_data[32]; for (size_t i = 0; i < 16; ++i) { input_data[i] = i; input_data[16 + i] = 1; } // copy INPUT0 and INPUT1 data in GPU shared memory int* input_d_ptr; cudaMalloc((void**)&input_d_ptr, input_byte_size * 2); cudaMemcpy( (void*)input_d_ptr, (void*)input_data, input_byte_size * 2, cudaMemcpyHostToDevice); cudaIpcMemHandle_t input_cuda_handle; CreateCUDAIPCHandle(&input_cuda_handle, (void*)input_d_ptr); FAIL_IF_ERR( client->RegisterCudaSharedMemory( "input_data", input_cuda_handle, 0 /* device_id */, input_byte_size * 2), "failed to register input shared memory region"); FAIL_IF_ERR( input0_ptr->SetSharedMemory( "input_data", input_byte_size, 0 /* offset */), "unable to set shared memory for INPUT0"); FAIL_IF_ERR( input1_ptr->SetSharedMemory( "input_data", input_byte_size, input_byte_size /* offset */), "unable to set shared memory for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // Create Output0 and Output1 in CUDA Shared Memory int *output0_d_ptr, *output1_d_ptr; cudaMalloc((void**)&output0_d_ptr, output_byte_size * 2); output1_d_ptr = (int*)output0_d_ptr + 16; cudaIpcMemHandle_t output_cuda_handle; CreateCUDAIPCHandle(&output_cuda_handle, (void*)output0_d_ptr); FAIL_IF_ERR( client->RegisterCudaSharedMemory( "output_data", output_cuda_handle, 0 /* device_id */, output_byte_size * 2), "failed to register output shared memory region"); FAIL_IF_ERR( output0_ptr->SetSharedMemory( "output_data", output_byte_size, 0 /* offset */), "unable to set shared memory for 'OUTPUT0'"); FAIL_IF_ERR( output1_ptr->SetSharedMemory( "output_data", output_byte_size, output_byte_size /* offset */), "unable to set shared memory for 'OUTPUT1'"); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer(&results, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); // Copy input and output data back to the CPU int output0_data[16], output1_data[16]; cudaMemcpy( output0_data, output0_d_ptr, output_byte_size, cudaMemcpyDeviceToHost); cudaMemcpy( output1_data, output1_d_ptr, output_byte_size, cudaMemcpyDeviceToHost); for (size_t i = 0; i < 16; ++i) { std::cout << input_data[i] << " + " << input_data[16 + i] << " = " << output0_data[i] << std::endl; std::cout << input_data[i] << " + " << input_data[16 + i] << " = " << output1_data[i] << std::endl; if ((input_data[i] + input_data[16 + i]) != output0_data[i]) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input_data[i] - input_data[16 + i]) != output1_data[i]) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get shared memory regions active/registered within triton std::string shm_status; FAIL_IF_ERR( client->CudaSharedMemoryStatus(&shm_status), "failed to get shared memory status"); std::cout << "Shared Memory Status:\n" << shm_status << "\n"; // Unregister shared memory FAIL_IF_ERR( client->UnregisterCudaSharedMemory("input_data"), "unable to unregister shared memory input region"); FAIL_IF_ERR( client->UnregisterCudaSharedMemory("output_data"), "unable to unregister shared memory output region"); // Free GPU memory FAIL_IF_CUDA_ERR(cudaFree(input_d_ptr)); FAIL_IF_CUDA_ERR(cudaFree(output0_d_ptr)); std::cout << "PASS : Cuda Shared Memory " << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_http_health_metadata.cc000066400000000000000000000147161527052371200261250ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "http_client.h" #include "json_utils.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerHttpClient instance to communicate with the // server using http protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&client, url, verbose), "unable to create http client"); bool live; FAIL_IF_ERR( client->IsServerLive(&live, http_headers), "unable to get server liveness"); if (!live) { std::cerr << "error: server is not live" << std::endl; exit(1); } bool ready; FAIL_IF_ERR( client->IsServerReady(&ready, http_headers), "server is not live"); bool model_ready; FAIL_IF_ERR( client->IsModelReady( &model_ready, model_name, model_version, http_headers), "unable to get model readiness"); if (!model_ready) { std::cerr << "error: model " << model_name << " is not live" << std::endl; exit(1); } { std::string server_metadata; FAIL_IF_ERR( client->ServerMetadata(&server_metadata, http_headers), "unable to get server metadata"); rapidjson::Document server_metadata_json; FAIL_IF_ERR( tc::ParseJson(&server_metadata_json, server_metadata), "failed to parse server metadata"); if ((std::string(server_metadata_json["name"].GetString())) .compare("triton") != 0) { std::cerr << "error: unexpected server metadata: " << server_metadata << std::endl; exit(1); } } { std::string model_metadata; FAIL_IF_ERR( client->ModelMetadata( &model_metadata, model_name, model_version, http_headers), "unable to get model metadata"); rapidjson::Document model_metadata_json; FAIL_IF_ERR( tc::ParseJson(&model_metadata_json, model_metadata), "failed to parse model metadata"); if ((std::string(model_metadata_json["name"].GetString())) .compare(model_name) != 0) { std::cerr << "error: unexpected model metadata: " << model_metadata << std::endl; exit(1); } } { std::string model_config; FAIL_IF_ERR( client->ModelConfig( &model_config, model_name, model_version, http_headers), "unable to get model config"); rapidjson::Document model_config_json; FAIL_IF_ERR( tc::ParseJson(&model_config_json, model_config), "failed to parse model config"); if ((std::string(model_config_json["name"].GetString())) .compare(model_name) != 0) { std::cerr << "error: unexpected model config: " << model_config << std::endl; exit(1); } } { std::string model_metadata; tc::Error err = client->ModelMetadata( &model_metadata, "wrong_model_name", model_version, http_headers); if (err.IsOk()) { std::cerr << "error: expected an error but got: " << err << std::endl; exit(1); } } std::cout << "SUCCESS" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_http_infer_client.cc000066400000000000000000000267051527052371200254620ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include "http_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << "\t-i " << std::endl; std::cerr << "\t-o " << std::endl; std::cerr << std::endl; std::cerr << "\t--verify-peer" << std::endl; std::cerr << "\t--verify-host" << std::endl; std::cerr << "\t--ca-certs" << std::endl; std::cerr << "\t--cert-file" << std::endl; std::cerr << "\t--key-file" << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl << "For -i, it sets the compression algorithm used for sending request " "body." << "For -o, it sets the compression algorithm used for receiving " "response body." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); tc::Headers http_headers; uint32_t client_timeout = 0; auto request_compression_algorithm = tc::InferenceServerHttpClient::CompressionType::NONE; auto response_compression_algorithm = tc::InferenceServerHttpClient::CompressionType::NONE; long verify_peer = 1; long verify_host = 2; std::string cacerts; std::string certfile; std::string keyfile; // {name, has_arg, *flag, val} static struct option long_options[] = { {"verify-peer", 1, 0, 0}, {"verify-host", 1, 0, 1}, {"ca-certs", 1, 0, 2}, {"cert-file", 1, 0, 3}, {"key-file", 1, 0, 4}, {0, 0, 0, 0}}; // Parse commandline... int opt; while ((opt = getopt_long(argc, argv, "vu:t:H:i:o:", long_options, NULL)) != -1) { switch (opt) { case 0: verify_peer = std::atoi(optarg); break; case 1: verify_host = std::atoi(optarg); break; case 2: cacerts = optarg; break; case 3: certfile = optarg; break; case 4: keyfile = optarg; break; case 'v': verbose = true; break; case 'u': url = optarg; break; case 't': client_timeout = std::stoi(optarg); break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case 'i': { std::string arg = optarg; if (arg == "gzip") { request_compression_algorithm = tc::InferenceServerHttpClient::CompressionType::GZIP; } else if (arg == "deflate") { request_compression_algorithm = tc::InferenceServerHttpClient::CompressionType::DEFLATE; } break; } case 'o': { std::string arg = optarg; if (arg == "gzip") { response_compression_algorithm = tc::InferenceServerHttpClient::CompressionType::GZIP; } else if (arg == "deflate") { response_compression_algorithm = tc::InferenceServerHttpClient::CompressionType::DEFLATE; } break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; tc::HttpSslOptions ssl_options; ssl_options.verify_peer = verify_peer; ssl_options.verify_host = verify_host; ssl_options.ca_info = cacerts; ssl_options.cert = certfile; ssl_options.key = keyfile; // Create a InferenceServerHttpClient instance to communicate with the // server using HTTP protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&client, url, verbose, ssl_options), "unable to create http client"); // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. std::vector input0_data(16); std::vector input1_data(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = i; input1_data[i] = 1; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); FAIL_IF_ERR( input0_ptr->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for INPUT0"); FAIL_IF_ERR( input1_ptr->AppendRaw( reinterpret_cast(&input1_data[0]), input1_data.size() * sizeof(int32_t)), "unable to set data for INPUT1"); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; options.client_timeout_ = client_timeout; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; // Empty output vector will request data for all the output tensors from // the server. std::vector outputs = {}; tc::InferResult* results; FAIL_IF_ERR( client->Infer( &results, options, inputs, outputs, http_headers, tc::Parameters(), request_compression_algorithm, response_compression_algorithm), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); // Get pointers to the result returned... int32_t* output0_data; size_t output0_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT0", (const uint8_t**)&output0_data, &output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << output0_byte_size << std::endl; exit(1); } int32_t* output1_data; size_t output1_byte_size; FAIL_IF_ERR( results_ptr->RawData( "OUTPUT1", (const uint8_t**)&output1_data, &output1_byte_size), "unable to get result data for 'OUTPUT1'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT1': " << output0_byte_size << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data[i] << " + " << input1_data[i] << " = " << *(output0_data + i) << std::endl; std::cout << input0_data[i] << " - " << input1_data[i] << " = " << *(output1_data + i) << std::endl; if ((input0_data[i] + input1_data[i]) != *(output0_data + i)) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_data[i] - input1_data[i]) != *(output1_data + i)) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get full response std::cout << results_ptr->DebugString() << std::endl; tc::InferStat infer_stat; client->ClientInferStat(&infer_stat); std::cout << "======Client Statistics======" << std::endl; std::cout << "completed_request_count " << infer_stat.completed_request_count << std::endl; std::cout << "cumulative_total_request_time_ns " << infer_stat.cumulative_total_request_time_ns << std::endl; std::cout << "cumulative_send_time_ns " << infer_stat.cumulative_send_time_ns << std::endl; std::cout << "cumulative_receive_time_ns " << infer_stat.cumulative_receive_time_ns << std::endl; std::string model_stat; FAIL_IF_ERR( client->ModelInferenceStatistics(&model_stat, model_name), "unable to get model statistics"); std::cout << "======Model Statistics======" << std::endl; std::cout << model_stat << std::endl; std::cout << "PASS : Infer" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_http_model_control.cc000066400000000000000000000117071527052371200256550ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "http_client.h" #include "json_utils.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } std::string model_name = "simple"; // Create a InferenceServerHttpClient instance to communicate with the // server using http protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&client, url, verbose), "unable to create http client"); { std::string repository_index; FAIL_IF_ERR( client->ModelRepositoryIndex(&repository_index, http_headers), "Failed to get repository index"); rapidjson::Document repository_index_json; FAIL_IF_ERR( tc::ParseJson(&repository_index_json, repository_index), "failed to parse model config"); if (repository_index_json.Size() != 7) { std::cerr << "expected number of models 7, got " << repository_index_json.Size() << std::endl; exit(1); } } FAIL_IF_ERR( client->LoadModel(model_name, http_headers), "Failed to load model"); bool model_ready; FAIL_IF_ERR( client->IsModelReady(&model_ready, model_name), "unable to get model readiness"); if (!model_ready) { std::cerr << "error: model " << model_name << " is not live" << std::endl; exit(1); } FAIL_IF_ERR( client->UnloadModel(model_name, http_headers), "Failed to unload model"); FAIL_IF_ERR( client->IsModelReady(&model_ready, model_name), "unable to get model readiness"); if (model_ready) { std::cerr << "error: model " << model_name << " is live after unloading" << std::endl; exit(1); } tc::Error err = client->LoadModel("wrong_model_name", http_headers); if (err.IsOk()) { std::cerr << "error: wrong model name was successfully loaded" << std::endl; } return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_http_sequence_sync_infer_client.cc000066400000000000000000000252321527052371200304000ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include "http_client.h" namespace tc = triton::client; using ResultList = std::vector>; // Global mutex to synchronize the threads std::mutex mutex_; std::condition_variable cv_; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; std::cerr << "\t-o " << std::endl; std::cerr << std::endl; std::cerr << "For -o, the client will use sequence ID <1 + 2 * offset> " << "and <2 + 2 * offset>. Default offset is 0." << std::endl; exit(1); } void SyncSend( const std::unique_ptr& client, tc::InferOptions& options, int32_t value, std::vector& result_data, tc::Headers& http_headers) { // Initialize the inputs with the data. tc::InferInput* input; std::vector shape{1, 1}; FAIL_IF_ERR( tc::InferInput::Create(&input, "INPUT", shape, "INT32"), "unable to create 'INPUT'"); std::shared_ptr ivalue(input); FAIL_IF_ERR(ivalue->Reset(), "unable to reset 'INPUT'"); FAIL_IF_ERR( ivalue->AppendRaw(reinterpret_cast(&value), sizeof(int32_t)), "unable to set data for 'INPUT'"); std::vector inputs = {ivalue.get()}; tc::InferRequestedOutput* output; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output, "OUTPUT"), "unable to get 'OUTPUT'"); std::shared_ptr routput; routput.reset(output); std::vector outputs = {routput.get()}; tc::InferResult* result; // Send inference request to the inference server. FAIL_IF_ERR( client->Infer(&result, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr this_result(result); // Get pointers to the result returned... int32_t* output_data; size_t output_byte_size; FAIL_IF_ERR( this_result->RawData( "OUTPUT", (const uint8_t**)&output_data, &output_byte_size), "unable to get result data for 'OUTPUT'"); if (output_byte_size != 4) { std::cerr << "error: received incorrect byte size for 'OUTPUT': " << output_byte_size << std::endl; exit(1); } result_data.push_back(*output_data); } void SyncSend( const std::unique_ptr& client, const std::string& model_name, int32_t value, const uint64_t sequence_id, bool start_of_sequence, bool end_of_sequence, std::vector& result_data, tc::Headers& http_headers) { tc::InferOptions options(model_name); options.sequence_id_ = sequence_id; options.sequence_start_ = start_of_sequence; options.sequence_end_ = end_of_sequence; SyncSend(client, options, value, result_data, http_headers); } void SyncSend( const std::unique_ptr& client, const std::string& model_name, int32_t value, const std::string& sequence_id, bool start_of_sequence, bool end_of_sequence, std::vector& result_data, tc::Headers& http_headers) { tc::InferOptions options(model_name); options.sequence_id_str_ = sequence_id; options.sequence_start_ = start_of_sequence; options.sequence_end_ = end_of_sequence; SyncSend(client, options, value, result_data, http_headers); } } // namespace int main(int argc, char** argv) { bool verbose = false; bool dyna_sequence = false; std::string url("localhost:8000"); int sequence_id_offset = 0; tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vdu:H:o:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case 'd': dyna_sequence = true; break; case 'u': url = optarg; break; case 'o': sequence_id_offset = std::stoi(optarg); break; case '?': Usage(argv); break; } } tc::Error err; // We use the custom "sequence" model which takes 1 input value. The // output is the accumulated value of the inputs. See // src/custom/sequence. std::string int_model_name = dyna_sequence ? "simple_dyna_sequence" : "simple_sequence"; std::string string_model_name = dyna_sequence ? "simple_string_dyna_sequence" : "simple_sequence"; const uint64_t int_sequence_id0 = 1 + sequence_id_offset * 2; const uint64_t int_sequence_id1 = 2 + sequence_id_offset * 2; // For string sequence IDs, the dyna backend requires that the // sequence id be decodable into an integer, otherwise we'll use // a test string sequence id and a model that doesn't require corrid // control. const std::string string_sequence_id0 = dyna_sequence ? std::to_string(3 + sequence_id_offset * 2) : "SEQ-3"; std::cout << "sequence ID " << int_sequence_id0 << " : " << "sequence ID " << int_sequence_id1 << " : " << "sequence ID " << string_sequence_id0 << std::endl; // Create a InferenceServerHttpClient instance to communicate with the // server using http protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&client, url, verbose), "unable to create http client"); // Now send the inference sequences.. // std::vector values{11, 7, 5, 3, 2, 0, 1}; std::vector int_result0_data; std::vector int_result1_data; std::vector string_result0_data; // Send requests, first reset accumulator for the sequence. SyncSend( client, int_model_name, 0, int_sequence_id0, true /* start-of-sequence */, false /* end-of-sequence */, int_result0_data, http_headers); SyncSend( client, int_model_name, 100, int_sequence_id1, true /* start-of-sequence */, false /* end-of-sequence */, int_result1_data, http_headers); SyncSend( client, string_model_name, 20, string_sequence_id0, true /* start-of-sequence */, false /* end-of-sequence */, string_result0_data, http_headers); // Now send a sequence of values... for (int32_t v : values) { SyncSend( client, int_model_name, v, int_sequence_id0, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, int_result0_data, http_headers); SyncSend( client, int_model_name, -v, int_sequence_id1, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, int_result1_data, http_headers); SyncSend( client, string_model_name, -v, string_sequence_id0, false /* start-of-sequence */, (v == 1) /* end-of-sequence */, string_result0_data, http_headers); } for (size_t i = 0; i < int_result0_data.size(); i++) { int32_t int_seq0_expected = (i == 0) ? 1 : values[i - 1]; int32_t int_seq1_expected = (i == 0) ? 101 : values[i - 1] * -1; int32_t string_seq0_expected; // For string sequence ID case we are testing two different backends if ((i == 0) && dyna_sequence) { string_seq0_expected = 20; } else if ((i == 0) && !dyna_sequence) { string_seq0_expected = 21; } else if ((i != 0) && dyna_sequence) { string_seq0_expected = values[i - 1] * -1 + string_result0_data[i - 1]; } else { string_seq0_expected = values[i - 1] * -1; } // The dyna_sequence custom backend adds the sequence ID to // the last request in a sequence. if (dyna_sequence && (i != 0) && (values[i - 1] == 1)) { int_seq0_expected += int_sequence_id0; int_seq1_expected += int_sequence_id1; string_seq0_expected += std::stoi(string_sequence_id0); } std::cout << "[" << i << "] " << int_result0_data[i] << " : " << int_result1_data[i] << " : " << string_result0_data[i] << std::endl; if ((int_seq0_expected != int_result0_data[i]) || (int_seq1_expected != int_result1_data[i]) || (string_seq0_expected != string_result0_data[i])) { std::cout << "[ expected ] " << int_seq0_expected << " : " << int_seq1_expected << " : " << string_seq0_expected << std::endl; return 1; } } return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_http_shm_client.cc000066400000000000000000000241611527052371200251400ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "http_client.h" #include "shm_utils.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); tc::Headers http_headers; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vu:H:")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 integers // each and returns 2 output tensors of 16 integers each. One output // tensor is the element-wise sum of the inputs and one output is // the element-wise difference. std::string model_name = "simple"; std::string model_version = ""; // Create a InferenceServerHttpClient instance to communicate with the // server using http protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&client, url, verbose), "unable to create http client"); // Unregistering all shared memory regions for a clean // start. FAIL_IF_ERR( client->UnregisterSystemSharedMemory(), "unable to unregister all system shared memory regions"); FAIL_IF_ERR( client->UnregisterCudaSharedMemory(), "unable to unregister all cuda shared memory regions"); std::vector shape{1, 16}; size_t input_byte_size = 64; size_t output_byte_size = 64; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "INT32"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); // Create Input0 and Input1 in Shared Memory. Initialize Input0 to unique // integers and Input1 to all ones. std::string shm_key = "/input_simple"; int shm_fd_ip, *input0_shm; FAIL_IF_ERR( tc::CreateSharedMemoryRegion(shm_key, input_byte_size * 2, &shm_fd_ip), ""); FAIL_IF_ERR( tc::MapSharedMemory( shm_fd_ip, 0, input_byte_size * 2, (void**)&input0_shm), ""); FAIL_IF_ERR(tc::CloseSharedMemory(shm_fd_ip), ""); int* input1_shm = (int*)(input0_shm + 16); for (size_t i = 0; i < 16; ++i) { *(input0_shm + i) = i; *(input1_shm + i) = 1; } FAIL_IF_ERR( client->RegisterSystemSharedMemory( "input_data", "/input_simple", input_byte_size * 2), "failed to register input shared memory region"); FAIL_IF_ERR( input0_ptr->SetSharedMemory( "input_data", input_byte_size, 0 /* offset */), "unable to set shared memory for INPUT0"); FAIL_IF_ERR( input1_ptr->SetSharedMemory( "input_data", input_byte_size, input_byte_size /* offset */), "unable to set shared memory for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get 'OUTPUT1'"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); // Create Output0 and Output1 in Shared Memory shm_key = "/output_simple"; int shm_fd_op; int* output0_shm; FAIL_IF_ERR( tc::CreateSharedMemoryRegion(shm_key, output_byte_size * 2, &shm_fd_op), ""); FAIL_IF_ERR( tc::MapSharedMemory( shm_fd_op, 0, output_byte_size * 2, (void**)&output0_shm), ""); FAIL_IF_ERR(tc::CloseSharedMemory(shm_fd_op), ""); int* output1_shm = (int*)(output0_shm + 16); FAIL_IF_ERR( client->RegisterSystemSharedMemory( "output_data", "/output_simple", output_byte_size * 2), "failed to register output shared memory region"); FAIL_IF_ERR( output0_ptr->SetSharedMemory( "output_data", output_byte_size, 0 /* offset */), "unable to set shared memory for 'OUTPUT0'"); FAIL_IF_ERR( output1_ptr->SetSharedMemory( "output_data", output_byte_size, output_byte_size /* offset */), "unable to set shared memory for 'OUTPUT1'"); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer(&results, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); for (size_t i = 0; i < 16; ++i) { std::cout << input0_shm[i] << " + " << input1_shm[i] << " = " << output0_shm[i] << std::endl; std::cout << input0_shm[i] << " - " << input1_shm[i] << " = " << output1_shm[i] << std::endl; if ((input0_shm[i] + input1_shm[i]) != output0_shm[i]) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if ((input0_shm[i] - input1_shm[i]) != output1_shm[i]) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get shared memory regions active/registered within triton std::string shm_status; FAIL_IF_ERR( client->SystemSharedMemoryStatus(&shm_status), "failed to get shared memory status"); std::cout << "Shared Memory Status:\n" << shm_status << "\n"; // Unregister shared memory FAIL_IF_ERR( client->UnregisterSystemSharedMemory("input_data"), "unable to unregister shared memory input region"); FAIL_IF_ERR( client->UnregisterSystemSharedMemory("output_data"), "unable to unregister shared memory output region"); // Cleanup shared memory FAIL_IF_ERR(tc::UnmapSharedMemory(input0_shm, input_byte_size * 2), ""); FAIL_IF_ERR(tc::UnlinkSharedMemoryRegion("/input_simple"), ""); FAIL_IF_ERR(tc::UnmapSharedMemory(output0_shm, output_byte_size * 2), ""); FAIL_IF_ERR(tc::UnlinkSharedMemoryRegion("/output_simple"), ""); std::cout << "PASS : System Shared Memory " << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/examples/simple_http_string_infer_client.cc000066400000000000000000000220031527052371200270330ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include "http_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } namespace { void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR(result->Shape(name, &shape), "unable to get shape for " + name); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for " << name << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for " + name); // Validate datatype if (datatype.compare("BYTES") != 0) { std::cerr << "error: received incorrect datatype for " << name << ": " << datatype << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-H " << std::endl; std::cerr << "\t--json-input-data" << std::endl; std::cerr << "\t--json-output-data" << std::endl; std::cerr << std::endl; std::cerr << "For -H, header must be 'Header:Value'. May be given multiple times." << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string url("localhost:8000"); tc::Headers http_headers; bool json_input_data{false}; bool json_output_data{false}; // {name, has_arg, *flag, val} static struct option long_options[] = { {"json-input-data", 0, 0, 0}, {"json-output-data", 0, 0, 1}, {0, 0, 0, 0}}; // Parse commandline... int opt; while ((opt = getopt_long(argc, argv, "vu:H:", long_options, NULL)) != -1) { switch (opt) { case 0: json_input_data = true; break; case 1: json_output_data = true; break; case 'v': verbose = true; break; case 'u': url = optarg; break; case 'H': { std::string arg = optarg; std::string header = arg.substr(0, arg.find(":")); http_headers[header] = arg.substr(header.size() + 1); break; } case '?': Usage(argv); break; } } // We use a simple model that takes 2 input tensors of 16 strings // each and returns 2 output tensors of 16 strings each. The input // strings must represent integers. One output tensor is the // element-wise sum of the inputs and one output is the element-wise // difference. std::string model_name = "simple_string"; std::string model_version = ""; // Create a InferenceServerHttpClient instance to communicate with the // server using http protocol. std::unique_ptr client; FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&client, url, verbose), "unable to create http client"); // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. The input tensors // are the string representation of these values. std::vector input0_data(16); std::vector input1_data(16); std::vector expected_sum(16); std::vector expected_diff(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = std::to_string(i); input1_data[i] = std::to_string(1); expected_sum[i] = i + 1; expected_diff[i] = i - 1; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; tc::InferInput* input1; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "BYTES"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); input0_ptr->SetBinaryData(!json_input_data); FAIL_IF_ERR( tc::InferInput::Create(&input1, "INPUT1", shape, "BYTES"), "unable to get INPUT1"); std::shared_ptr input1_ptr; input1_ptr.reset(input1); input1_ptr->SetBinaryData(!json_input_data); FAIL_IF_ERR( input0_ptr->AppendFromString(input0_data), "unable to set data for INPUT0"); FAIL_IF_ERR( input1_ptr->AppendFromString(input1_data), "unable to set data for INPUT1"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; tc::InferRequestedOutput* output1; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get OUTPUT0"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); output0_ptr->SetBinaryData(!json_output_data); FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output1, "OUTPUT1"), "unable to get OUTPUT1"); std::shared_ptr output1_ptr; output1_ptr.reset(output1); output1_ptr->SetBinaryData(!json_output_data); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; std::vector outputs = { output0_ptr.get(), output1_ptr.get()}; tc::InferResult* results; FAIL_IF_ERR( client->Infer(&results, options, inputs, outputs, http_headers), "unable to run model"); std::shared_ptr results_ptr; results_ptr.reset(results); // Validate the results... ValidateShapeAndDatatype("OUTPUT0", results_ptr); ValidateShapeAndDatatype("OUTPUT1", results_ptr); // Get the result data std::vector result0_data; std::vector result1_data; FAIL_IF_ERR( results_ptr->StringData("OUTPUT0", &result0_data), "unable to get data for OUTPUT0"); if (result0_data.size() != 16) { std::cerr << "error: received incorrect number of strings for OUTPUT0: " << result0_data.size() << std::endl; exit(1); } FAIL_IF_ERR( results_ptr->StringData("OUTPUT1", &result1_data), "unable to get data for OUTPUT1"); if (result1_data.size() != 16) { std::cerr << "error: received incorrect number of strings for OUTPUT1: " << result1_data.size() << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { std::cout << input0_data[i] << " + " << input0_data[i] << " = " << result0_data[i] << std::endl; std::cout << input0_data[i] << " - " << input1_data[i] << " = " << result1_data[i] << std::endl; if (expected_sum[i] != std::stoi(result0_data[i])) { std::cerr << "error: incorrect sum" << std::endl; exit(1); } if (expected_diff[i] != std::stoi(result1_data[i])) { std::cerr << "error: incorrect difference" << std::endl; exit(1); } } // Get full response std::cout << results_ptr->DebugString() << std::endl; std::cout << "PASS : String Infer" << std::endl; return 0; } kytensor-2.49.0.7/client/src/c++/library/000077500000000000000000000000001527052371200177165ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/c++/library/CMakeLists.txt000066400000000000000000000361711527052371200224660ustar00rootroot00000000000000# Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) execute_process( COMMAND lsb_release -d OUTPUT_VARIABLE LINUX_DISTRO OUTPUT_STRIP_TRAILING_WHITESPACE ) message(STATUS "System: ${LINUX_DISTRO}") find_package(Threads REQUIRED) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) # # common # add_library( client-common-library INTERFACE ) target_include_directories( client-common-library INTERFACE $ ) # # json_utils # if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_EXAMPLES) find_package(RapidJSON CONFIG REQUIRED) add_library( json-utils-library EXCLUDE_FROM_ALL OBJECT json_utils.h json_utils.cc ) target_include_directories( json-utils-library PUBLIC ${RapidJSON_INCLUDE_DIRS} ) target_link_libraries( json-utils-library PRIVATE client-common-library ) add_library( json_utils_static STATIC $ ) add_library( TritonClient::json_utils_static ALIAS json_utils_static ) foreach(_json_target json-utils-library json_utils_static) target_compile_features(${_json_target} PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( ${_json_target} PRIVATE $<$,$,$>: -Wall -Wextra -Werror> $<$:/W0 /D_WIN32_WINNT=0x0A00 /EHsc> ) set_target_properties( ${_json_target} PROPERTIES POSITION_INDEPENDENT_CODE ON ) target_include_directories( ${_json_target} PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) install( FILES ${CMAKE_CURRENT_SOURCE_DIR}/json_utils.h DESTINATION include ) endforeach() install( TARGETS json_utils_static EXPORT triton-client-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) endif() # TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_EXAMPLES # # shm_utils # add_library( shm-utils-library EXCLUDE_FROM_ALL OBJECT shm_utils.h shm_utils.cc ) target_link_libraries( shm-utils-library PRIVATE client-common-library ) if(NOT WIN32) add_library( shm_utils_static STATIC $ ) add_library( TritonClient::shm_utils_static ALIAS shm_utils_static ) foreach(_shm_target shm-utils-library shm_utils_static) target_compile_features(${_shm_target} PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( ${_shm_target} PRIVATE $<$,$,$>: -Wall -Wextra -Werror> $<$:/W0 /D_WIN32_WINNT=0x0A00 /EHsc> ) set_target_properties( ${_shm_target} PROPERTIES POSITION_INDEPENDENT_CODE ON ) target_include_directories( ${_shm_target} PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) install( FILES ${CMAKE_CURRENT_SOURCE_DIR}/shm_utils.h DESTINATION include ) endforeach() install( TARGETS shm_utils_static EXPORT triton-client-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) endif(NOT WIN32) if(TRITON_ENABLE_CC_GRPC) # # libgrpcclient.so and libgrpcclient_static.a # configure_file(libgrpcclient.ldscript libgrpcclient.ldscript COPYONLY) # libgrpcclient object build set( REQUEST_SRCS grpc_client.cc common.cc ) set( REQUEST_HDRS grpc_client.h common.h ipc.h ) add_library( grpc-client-library EXCLUDE_FROM_ALL OBJECT ${REQUEST_SRCS} ${REQUEST_HDRS} ) add_dependencies( grpc-client-library grpc-service-library proto-library ) # libgrpcclient_static.a add_library( grpcclient_static STATIC $ $ $ ) add_library( TritonClient::grpcclient_static ALIAS grpcclient_static ) target_include_directories( grpcclient_static PUBLIC $ $ ) target_link_libraries( grpcclient_static PRIVATE gRPC::grpc++ PRIVATE gRPC::grpc PUBLIC protobuf::libprotobuf PUBLIC Threads::Threads ) # libgrpcclient.so add_library( grpcclient SHARED $ $ $ ) add_library( TritonClient::grpcclient ALIAS grpcclient ) if (NOT WIN32 AND NOT TRITON_KEEP_TYPEINFO) set_target_properties( grpcclient PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libgrpcclient.ldscript LINK_FLAGS "-Wl,--version-script=libgrpcclient.ldscript" ) endif() # NOT WIN32 AND NOT TRITON_KEEP_TYPEINFO target_link_libraries( grpcclient PRIVATE gRPC::grpc++ PRIVATE gRPC::grpc PUBLIC protobuf::libprotobuf PUBLIC Threads::Threads ) foreach(_client_target grpc-client-library grpcclient_static grpcclient) target_compile_features(${_client_target} PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( ${_client_target} PRIVATE $<$,$,$>: -Wall -Wextra -Werror> $<$:/W0 /D_WIN32_WINNT=0x0A00 /EHsc> ) set_target_properties( ${_client_target} PROPERTIES POSITION_INDEPENDENT_CODE ON ) target_include_directories( ${_client_target} PUBLIC $ $ $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) if(TRITON_ENABLE_GPU) target_compile_definitions( ${_client_target} PUBLIC TRITON_ENABLE_GPU=1 ) endif() # TRITON_ENABLE_GPU if(TRITON_ENABLE_GPU) target_link_libraries( ${_client_target} PUBLIC CUDA::cudart ) endif() # TRITON_ENABLE_GPU endforeach() if (TRITON_USE_THIRD_PARTY) if (NOT WIN32) install( DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/curl/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/grpc/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/protobuf/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/c-ares/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/absl/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/re2/lib/ DESTINATION ${CMAKE_INSTALL_LIBDIR} FILES_MATCHING PATTERN "*\.a" PATTERN "CMakeFiles" EXCLUDE PATTERN "cmake" EXCLUDE PATTERN "gens" EXCLUDE PATTERN "libs" EXCLUDE PATTERN "third_party" EXCLUDE ) else() install( DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/curl/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/grpc/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/protobuf/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/c-ares/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/absl/lib/ ${CMAKE_CURRENT_BINARY_DIR}/../../third-party/re2/lib/ DESTINATION ${CMAKE_INSTALL_LIBDIR} FILES_MATCHING PATTERN "*\.lib" PATTERN "CMakeFiles" EXCLUDE PATTERN "cmake" EXCLUDE PATTERN "gens" EXCLUDE PATTERN "libs" EXCLUDE PATTERN "third_party" EXCLUDE ) endif() # NOT WIN32 install( DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/grpc/include/ ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/absl/include/ ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/protobuf/include/ ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/re2/include/ DESTINATION include ) endif() install( FILES ${CMAKE_CURRENT_SOURCE_DIR}/grpc_client.h DESTINATION include ) endif() # TRITON_ENABLE_CC_GRPC if(TRITON_ENABLE_CC_HTTP) if(${TRITON_ENABLE_ZLIB}) find_package(ZLIB REQUIRED) endif() # TRITON_ENABLE_ZLIB # # libhttpclient.so and libhttpclient_static.a # configure_file(libhttpclient.ldscript libhttpclient.ldscript COPYONLY) # libhttpclient object build set( REQUEST_SRCS http_client.cc common.cc cencode.c ) set( REQUEST_HDRS http_client.h common.h ipc.h cencode.h ) add_library( http-client-library EXCLUDE_FROM_ALL OBJECT ${REQUEST_SRCS} ${REQUEST_HDRS} ) if (NOT WIN32) set_property( SOURCE cencode.c PROPERTY COMPILE_FLAGS -Wno-implicit-fallthrough ) endif() # NOT WIN32 target_link_libraries( http-client-library PUBLIC triton-common-json # from repo-common ) # libhttpclient_static.a add_library( httpclient_static STATIC $ ) add_library( TritonClient::httpclient_static ALIAS httpclient_static ) target_link_libraries( httpclient_static PRIVATE triton-common-json PUBLIC CURL::libcurl PUBLIC Threads::Threads ) if(${TRITON_ENABLE_ZLIB}) target_link_libraries( httpclient_static PRIVATE ZLIB::ZLIB ) endif() # TRITON_ENABLE_ZLIB # libhttpclient.so add_library( httpclient SHARED $ ) add_library( TritonClient::httpclient ALIAS httpclient ) if (NOT WIN32) set_target_properties( httpclient PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libhttpclient.ldscript LINK_FLAGS "-Wl,--version-script=libhttpclient.ldscript" ) endif() # NOT WIN32 target_link_libraries( httpclient PRIVATE triton-common-json PUBLIC CURL::libcurl PUBLIC Threads::Threads ) foreach(_client_target http-client-library httpclient_static httpclient) target_compile_features(${_client_target} PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( ${_client_target} PRIVATE $<$,$,$>: -Wall -Wextra -Werror> $<$:/W0 /D_WIN32_WINNT=0x0A00 /EHsc> ) set_target_properties( ${_client_target} PROPERTIES POSITION_INDEPENDENT_CODE ON ) target_include_directories( ${_client_target} PUBLIC $ $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) target_compile_definitions( ${_client_target} PRIVATE CURL_STATICLIB=1 ) if(TRITON_ENABLE_GPU) target_compile_definitions( ${_client_target} PUBLIC TRITON_ENABLE_GPU=1 ) endif() # TRITON_ENABLE_GPU if(TRITON_ENABLE_GPU) target_link_libraries( ${_client_target} PUBLIC CUDA::cudart ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_ZLIB}) target_compile_definitions( ${_client_target} PUBLIC TRITON_ENABLE_ZLIB=1 ) endif() # TRITON_ENABLE_ZLIB endforeach() install( FILES ${CMAKE_CURRENT_SOURCE_DIR}/http_client.h DESTINATION include ) endif() # TRITON_ENABLE_CC_HTTP if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC) install( FILES ${CMAKE_CURRENT_SOURCE_DIR}/common.h ${CMAKE_CURRENT_SOURCE_DIR}/ipc.h ${CMAKE_CURRENT_SOURCE_DIR}/cencode.h DESTINATION include ) include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonClient) if(TRITON_ENABLE_CC_GRPC) install( TARGETS #grpcclient grpcclient_static EXPORT triton-client-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) endif() # TRITON_ENABLE_CC_GRPC if(TRITON_ENABLE_CC_HTTP) install( TARGETS httpclient httpclient_static EXPORT triton-client-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) endif() # TRITON_ENABLE_CC_HTTP install( EXPORT triton-client-targets FILE TritonClientTargets.cmake NAMESPACE TritonClient:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonClientConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonClientConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonClientConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-client-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonClientTargets.cmake NAMESPACE TritonClient:: ) export(PACKAGE TritonClient) endif() # TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_CC_GRPC kytensor-2.49.0.7/client/src/c++/library/cencode.c000066400000000000000000000053661527052371200214740ustar00rootroot00000000000000/* cencoder.c - c source to a base64 encoding algorithm implementation This is part of the libb64 project, and has been placed in the public domain. For details, see http://sourceforge.net/projects/libb64 */ #include "cencode.h" const int CHARS_PER_LINE = 72; void base64_init_encodestate(base64_encodestate* state_in) { state_in->step = step_A; state_in->result = 0; state_in->stepcount = 0; } char base64_encode_value(char value_in) { static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; if (value_in > 63) return '='; return encoding[(int)value_in]; } int base64_encode_block( const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in) { const char* plainchar = plaintext_in; const char* const plaintextend = plaintext_in + length_in; char* codechar = code_out; char result; char fragment; result = state_in->result; switch (state_in->step) { while (1) { case step_A: if (plainchar == plaintextend) { state_in->result = result; state_in->step = step_A; return codechar - code_out; } fragment = *plainchar++; result = (fragment & 0x0fc) >> 2; *codechar++ = base64_encode_value(result); result = (fragment & 0x003) << 4; case step_B: if (plainchar == plaintextend) { state_in->result = result; state_in->step = step_B; return codechar - code_out; } fragment = *plainchar++; result |= (fragment & 0x0f0) >> 4; *codechar++ = base64_encode_value(result); result = (fragment & 0x00f) << 2; case step_C: if (plainchar == plaintextend) { state_in->result = result; state_in->step = step_C; return codechar - code_out; } fragment = *plainchar++; result |= (fragment & 0x0c0) >> 6; *codechar++ = base64_encode_value(result); result = (fragment & 0x03f) >> 0; *codechar++ = base64_encode_value(result); ++(state_in->stepcount); if (state_in->stepcount == CHARS_PER_LINE / 4) { *codechar++ = '\n'; state_in->stepcount = 0; } } } /* control should not reach here */ return codechar - code_out; } int base64_encode_blockend(char* code_out, base64_encodestate* state_in) { char* codechar = code_out; switch (state_in->step) { case step_B: *codechar++ = base64_encode_value(state_in->result); *codechar++ = '='; *codechar++ = '='; break; case step_C: *codechar++ = base64_encode_value(state_in->result); *codechar++ = '='; break; case step_A: break; } *codechar++ = '\n'; return codechar - code_out; } kytensor-2.49.0.7/client/src/c++/library/cencode.h000066400000000000000000000013361527052371200214720ustar00rootroot00000000000000/* cencode.h - c header for a base64 encoding algorithm This is part of the libb64 project, and has been placed in the public domain. For details, see http://sourceforge.net/projects/libb64 */ #ifndef BASE64_CENCODE_H #define BASE64_CENCODE_H typedef enum { step_A, step_B, step_C } base64_encodestep; typedef struct { base64_encodestep step; char result; int stepcount; } base64_encodestate; void base64_init_encodestate(base64_encodestate* state_in); char base64_encode_value(char value_in); int base64_encode_block( const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in); int base64_encode_blockend(char* code_out, base64_encodestate* state_in); #endif /* BASE64_CENCODE_H */ kytensor-2.49.0.7/client/src/c++/library/cmake/000077500000000000000000000000001527052371200207765ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/c++/library/cmake/TRITONConfig.cmake.in000066400000000000000000000043721527052371200245600ustar00rootroot00000000000000# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # specific version required for protobuf if(NOT PROTOBUF_FOUND AND NOT Protobuf_FOUND) set(protobuf_MODULE_COMPATIBLE TRUE CACHE BOOL "protobuf_MODULE_COMPATIBLE" FORCE) find_package(Protobuf @Protobuf_VERSION@ CONFIG REQUIRED) endif() # specific version required for grpc if(NOT GRPC_FOUND AND NOT gRPC_FOUND) find_package(gRPC @gRPC_VERSION@ CONFIG REQUIRED) endif() if(NOT CURL_FOUND) if(${TRITON_CURL_WITHOUT_CONFIG}) find_package(CURL REQUIRED) else() find_package(CURL CONFIG REQUIRED) endif() # TRITON_CURL_WITHOUT_CONFIG endif() if(NOT Threads_FOUND) find_package(Threads REQUIRED) endif() if (NOT (TARGET TRITON::grpcclient OR TARGET TRITON::httpclient)) include ("${CMAKE_CURRENT_LIST_DIR}/TRITONTargets.cmake") endif () kytensor-2.49.0.7/client/src/c++/library/cmake/TritonClientConfig.cmake.in000066400000000000000000000034451527052371200261570ustar00rootroot00000000000000# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONCLIENT_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONCLIENT_CMAKE_DIR}) if(NOT TARGET TritonClient::triton-client) include("${TRITONCLIENT_CMAKE_DIR}/TritonClientTargets.cmake") endif() kytensor-2.49.0.7/client/src/c++/library/common.cc000066400000000000000000000234151527052371200215220ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common.h" namespace triton { namespace client { //============================================================================== const Error Error::Success(""); Error::Error(const std::string& msg) : msg_(msg) {} std::ostream& operator<<(std::ostream& out, const Error& err) { if (!err.msg_.empty()) { out << err.msg_; } return out; } //============================================================================== Error InferenceServerClient::ClientInferStat(InferStat* infer_stat) const { *infer_stat = infer_stat_; return Error::Success; } Error InferenceServerClient::UpdateInferStat(const RequestTimers& timer) { const uint64_t request_time_ns = timer.Duration( RequestTimers::Kind::REQUEST_START, RequestTimers::Kind::REQUEST_END); const uint64_t send_time_ns = timer.Duration( RequestTimers::Kind::SEND_START, RequestTimers::Kind::SEND_END); const uint64_t recv_time_ns = timer.Duration( RequestTimers::Kind::RECV_START, RequestTimers::Kind::RECV_END); if ((request_time_ns == std::numeric_limits::max()) || (send_time_ns == std::numeric_limits::max()) || (recv_time_ns == std::numeric_limits::max())) { return Error( "Timer not set correctly." + ((timer.Timestamp(RequestTimers::Kind::REQUEST_START) > timer.Timestamp(RequestTimers::Kind::REQUEST_END)) ? (" Request time from " + std::to_string( timer.Timestamp(RequestTimers::Kind::REQUEST_START)) + " to " + std::to_string( timer.Timestamp(RequestTimers::Kind::REQUEST_END)) + ".") : "") + ((timer.Timestamp(RequestTimers::Kind::SEND_START) > timer.Timestamp(RequestTimers::Kind::SEND_END)) ? (" Send time from " + std::to_string( timer.Timestamp(RequestTimers::Kind::SEND_START)) + " to " + std::to_string(timer.Timestamp(RequestTimers::Kind::SEND_END)) + ".") : "") + ((timer.Timestamp(RequestTimers::Kind::RECV_START) > timer.Timestamp(RequestTimers::Kind::RECV_END)) ? (" Receive time from " + std::to_string( timer.Timestamp(RequestTimers::Kind::RECV_START)) + " to " + std::to_string(timer.Timestamp(RequestTimers::Kind::RECV_END)) + ".") : "")); } infer_stat_.completed_request_count++; infer_stat_.cumulative_total_request_time_ns += request_time_ns; infer_stat_.cumulative_send_time_ns += send_time_ns; infer_stat_.cumulative_receive_time_ns += recv_time_ns; return Error::Success; } //============================================================================== Error InferInput::Create( InferInput** infer_input, const std::string& name, const std::vector& dims, const std::string& datatype) { *infer_input = new InferInput(name, dims, datatype); return Error::Success; } Error InferInput::SetShape(const std::vector& shape) { shape_ = shape; return Error::Success; } Error InferInput::Reset() { bufs_.clear(); buf_byte_sizes_.clear(); str_bufs_.clear(); bufs_idx_ = 0; byte_size_ = 0; io_type_ = NONE; return Error::Success; } Error InferInput::AppendRaw(const std::vector& input) { return AppendRaw(&input[0], input.size()); } Error InferInput::AppendRaw(const uint8_t* input, size_t input_byte_size) { byte_size_ += input_byte_size; bufs_.push_back(input); buf_byte_sizes_.push_back(input_byte_size); io_type_ = RAW; return Error::Success; } Error InferInput::SetSharedMemory( const std::string& name, size_t byte_size, size_t offset) { shm_name_ = name; shm_offset_ = offset; byte_size_ = byte_size; io_type_ = SHARED_MEMORY; return Error::Success; } Error InferInput::AppendFromString(const std::vector& input) { // Serialize the strings into a "raw" buffer. The first 4-bytes are // the length of the string length. Next are the actual string // characters. There is *not* a null-terminator on the string. str_bufs_.emplace_back(); std::string& sbuf = str_bufs_.back(); for (const auto& str : input) { uint32_t len = str.size(); sbuf.append(reinterpret_cast(&len), sizeof(uint32_t)); sbuf.append(str); } return AppendRaw(reinterpret_cast(&sbuf[0]), sbuf.size()); } Error InferInput::RawData(const uint8_t** buf, size_t* byte_size) { if (bufs_.size()) { // TMA-1775 - handle multi-batch case *buf = bufs_[0]; *byte_size = buf_byte_sizes_[0]; } else { *buf = nullptr; *byte_size = 0; } return Error::Success; } Error InferInput::ByteSize(size_t* byte_size) const { *byte_size = byte_size_; return Error::Success; } InferInput::InferInput( const std::string& name, const std::vector& shape, const std::string& datatype) : name_(name), shape_(shape), datatype_(datatype), byte_size_(0), bufs_idx_(0), buf_pos_(0), io_type_(NONE), shm_name_(""), shm_offset_(0) { } Error InferInput::SharedMemoryInfo( std::string* name, size_t* byte_size, size_t* offset) const { if (io_type_ != SHARED_MEMORY) { return Error("The input has not been set with the shared memory."); } *name = shm_name_; *offset = shm_offset_; *byte_size = byte_size_; return Error::Success; } Error InferInput::SetBinaryData(const bool binary_data) { binary_data_ = binary_data; return Error::Success; } Error InferInput::PrepareForRequest() { // Reset position so request sends entire input. bufs_idx_ = 0; buf_pos_ = 0; return Error::Success; } Error InferInput::GetNext( uint8_t* buf, size_t size, size_t* input_bytes, bool* end_of_input) { size_t total_size = 0; while ((bufs_idx_ < bufs_.size()) && (size > 0)) { const size_t buf_byte_size = buf_byte_sizes_[bufs_idx_]; const size_t csz = (std::min)(buf_byte_size - buf_pos_, size); if (csz > 0) { const uint8_t* input_ptr = bufs_[bufs_idx_] + buf_pos_; std::copy(input_ptr, input_ptr + csz, buf); buf_pos_ += csz; buf += csz; size -= csz; total_size += csz; } if (buf_pos_ == buf_byte_size) { bufs_idx_++; buf_pos_ = 0; } } *input_bytes = total_size; *end_of_input = (bufs_idx_ >= bufs_.size()); return Error::Success; } Error InferInput::GetNext( const uint8_t** buf, size_t* input_bytes, bool* end_of_input) { if (bufs_idx_ < bufs_.size()) { *buf = bufs_[bufs_idx_]; *input_bytes = buf_byte_sizes_[bufs_idx_]; bufs_idx_++; } else { *buf = nullptr; *input_bytes = 0; } *end_of_input = (bufs_idx_ >= bufs_.size()); return Error::Success; } //============================================================================== Error InferRequestedOutput::Create( InferRequestedOutput** infer_output, const std::string& name, const size_t class_count, const std::string& datatype) { *infer_output = new InferRequestedOutput(name, datatype, class_count); return Error::Success; } Error InferRequestedOutput::SetSharedMemory( const std::string& region_name, const size_t byte_size, const size_t offset) { shm_name_ = region_name; shm_byte_size_ = byte_size; shm_offset_ = offset; io_type_ = SHARED_MEMORY; return Error::Success; } Error InferRequestedOutput::UnsetSharedMemory() { shm_name_ = ""; shm_byte_size_ = 0; shm_offset_ = 0; io_type_ = NONE; return Error::Success; } InferRequestedOutput::InferRequestedOutput( const std::string& name, const std::string& datatype, const size_t class_count) : name_(name), datatype_(datatype), class_count_(class_count), io_type_(NONE) { } Error InferRequestedOutput::SharedMemoryInfo( std::string* name, size_t* byte_size, size_t* offset) const { if (io_type_ != SHARED_MEMORY) { return Error("The input has not been set with the shared memory."); } *name = shm_name_; *byte_size = shm_byte_size_; *offset = shm_offset_; return Error::Success; } Error InferRequestedOutput::SetBinaryData(const bool binary_data) { binary_data_ = binary_data; return Error::Success; } //============================================================================== }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/common.h000066400000000000000000000642461527052371200213730ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef TRITON_INFERENCE_SERVER_CLIENT_CLASS namespace triton { namespace perfanalyzer { namespace clientbackend { namespace tritoncapi { class TritonLoader; }}}} // namespace triton::perfanalyzer::clientbackend::tritoncapi #endif namespace triton { namespace client { constexpr char kInferHeaderContentLengthHTTPHeader[] = "Inference-Header-Content-Length"; constexpr int MAX_GRPC_MESSAGE_SIZE = INT32_MAX; class InferResult; class InferRequest; class RequestTimers; //============================================================================== /// Error status reported by client API. /// class Error { public: /// Create an error with the specified message. /// \param msg The message for the error explicit Error(const std::string& msg = ""); /// Accessor for the message of this error. /// \return The message for the error. Empty if no error. const std::string& Message() const { return msg_; } /// Does this error indicate OK status? /// \return True if this error indicates "ok"/"success", false if /// error indicates a failure. bool IsOk() const { return msg_.empty(); } /// Convenience "success" value. Can be used as Error::Success to /// indicate no error. static const Error Success; private: friend std::ostream& operator<<(std::ostream&, const Error&); std::string msg_; }; //============================================================================== /// Cumulative inference statistics. /// /// \note /// For GRPC protocol, 'cumulative_send_time_ns' represents the /// time for marshaling infer request. /// 'cumulative_receive_time_ns' represents the time for /// unmarshaling infer response. struct InferStat { /// Total number of requests completed. size_t completed_request_count; /// Time from the request start until the response is completely /// received. uint64_t cumulative_total_request_time_ns; /// Time from the request start until the last byte is sent. uint64_t cumulative_send_time_ns; /// Time from receiving first byte of the response until the /// response is completely received. uint64_t cumulative_receive_time_ns; /// Create a new InferStat object with zero-ed statistics. InferStat() : completed_request_count(0), cumulative_total_request_time_ns(0), cumulative_send_time_ns(0), cumulative_receive_time_ns(0) { } }; //============================================================================== /// The base class for InferenceServerClients /// class InferenceServerClient { public: using OnCompleteFn = std::function; using OnMultiCompleteFn = std::function)>; explicit InferenceServerClient(bool verbose) : verbose_(verbose), exiting_(false) { } virtual ~InferenceServerClient() = default; /// Obtain the cumulative inference statistics of the client. /// \param Returns the InferStat object holding current statistics. /// \return Error object indicating success or failure. Error ClientInferStat(InferStat* infer_stat) const; protected: // Update the infer stat with the given timer Error UpdateInferStat(const RequestTimers& timer); // Enables verbose operation in the client. bool verbose_; // worker thread that will perform the asynchronous transfer std::thread worker_; // Avoid race condition between main thread and worker thread std::mutex mutex_; // Condition variable used for waiting on asynchronous request std::condition_variable cv_; // signal for worker thread to stop bool exiting_; // The inference statistic of the current client InferStat infer_stat_; }; struct RequestParameter { std::string name; std::string value; std::string type; }; //============================================================================== /// Structure to hold options for Inference Request. /// struct InferOptions { explicit InferOptions(const std::string& model_name) : model_name_(model_name), model_version_(""), request_id_(""), sequence_id_(0), sequence_id_str_(""), sequence_start_(false), sequence_end_(false), priority_(0), server_timeout_(0), client_timeout_(0), triton_enable_empty_final_response_(false) { } /// The name of the model to run inference. std::string model_name_; /// The version of the model to use while running inference. The default /// value is an empty string which means the server will select the /// version of the model based on its internal policy. std::string model_version_; /// An identifier for the request. If specified will be returned /// in the response. Default value is an empty string which means no /// request_id will be used. std::string request_id_; /// The unique identifier for the sequence being represented by the /// object. Default value is 0 which means that the request does not /// belong to a sequence. If this value is non-zero, then sequence_id_str_ /// MUST be set to "". uint64_t sequence_id_; /// The unique identifier for the sequence being represented by the /// object. Default value is "" which means that the request does not /// belong to a sequence. If this value is non-empty, then sequence_id_ /// MUST be set to 0. std::string sequence_id_str_; /// Indicates whether the request being added marks the start of the /// sequence. Default value is False. This argument is ignored if /// 'sequence_id' is 0. bool sequence_start_; /// Indicates whether the request being added marks the end of the /// sequence. Default value is False. This argument is ignored if /// 'sequence_id' is 0. bool sequence_end_; /// Indicates the priority of the request. Priority value zero /// indicates that the default priority level should be used /// (i.e. same behavior as not specifying the priority parameter). /// Lower value priorities indicate higher priority levels. Thus /// the highest priority level is indicated by setting the parameter /// to 1, the next highest is 2, etc. If not provided, the server /// will handle the request using default setting for the model. uint64_t priority_; /// The timeout value for the request, in microseconds. If the request /// cannot be completed within the time by the server. The server can take a /// model-specific action such as terminating the request. If not /// provided, the server will handle the request using default setting /// for the model. This option is only respected by the model that is /// configured with dynamic batching. See here for more details: /// https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher uint64_t server_timeout_; // The maximum end-to-end time, in microseconds, the request is allowed // to take. The client will abort request when the specified time elapses. // The request will return error with message "Deadline Exceeded". // The default value is 0 which means client will wait for the // response from the server. This option is not supported for streaming // requests. Instead see 'stream_timeout' argument in // InferenceServerGrpcClient::StartStream(). // NOTE: the HTTP client library only offers millisecond precision, so a // timeout < 1000 microseconds will be rounded down to 0 milliseconds and have // no effect. uint64_t client_timeout_; /// Whether to tell Triton to enable an empty final response. bool triton_enable_empty_final_response_; /// Additional parameters to pass to the model std::unordered_map request_parameters; }; //============================================================================== /// An interface for InferInput object to describe the model input for /// inference. /// class InferInput { public: /// Create a InferInput instance that describes a model input. /// \param infer_input Returns a new InferInput object. /// \param name The name of input whose data will be described by this object. /// \param dims The shape of the input. /// \param datatype The datatype of the input. /// \return Error object indicating success or failure. static Error Create( InferInput** infer_input, const std::string& name, const std::vector& dims, const std::string& datatype); /// Gets name of the associated input tensor. /// \return The name of the tensor. const std::string& Name() const { return name_; } /// Gets datatype of the associated input tensor. /// \return The datatype of the tensor. const std::string& Datatype() const { return datatype_; } /// Gets the shape of the input tensor. /// \return The shape of the tensor. const std::vector& Shape() const { return shape_; } /// Set the shape of input associated with this object. /// \param dims the vector of dims representing the new shape /// of input. /// \return Error object indicating success or failure of the /// request. Error SetShape(const std::vector& dims); /// Prepare this input to receive new tensor values. Forget any /// existing values that were set by previous calls to SetSharedMemory() /// or AppendRaw(). /// \return Error object indicating success or failure. Error Reset(); /// Append tensor values for this input from a byte vector. The vector /// is not copied and so it must not be modified or destroyed /// until this input is no longer needed (that is until the Infer() /// call(s) that use the input have completed). Multiple calls can /// be made to this API to keep adding tensor data for this input. /// The data will be delivered in the order it was added. /// \param input The vector holding tensor values. /// \return Error object indicating success or failure. Error AppendRaw(const std::vector& input); /// Append tensor values for this input from a byte array. The array /// is not copied and so it must not be modified or destroyed /// until this input is no longer needed (that is until the Infer() /// call(s) that use the input have completed). Multiple calls can /// be made to this API to keep adding tensor data for this input. /// The data will be delivered in the order it was added. /// \param input The pointer to the array holding the tensor value. /// \param input_byte_size The size of the array in bytes. /// \return Error object indicating success or failure. Error AppendRaw(const uint8_t* input, size_t input_byte_size); /// Set tensor values for this input by reference into a shared memory /// region. The values are not copied and so the shared memory region and /// its contents must not be modified or destroyed until this input is no /// longer needed (that is until the Infer() call(s) that use the input have /// completed. This function must be called a single time for an input that /// is using shared memory. The entire tensor data required by this input /// must be contiguous in a single shared memory region. /// \param name The user-given name for the registered shared memory region /// where the tensor values for this input is stored. /// \param byte_size The size, in bytes of the input tensor data. Must /// match the size expected for the input shape. /// \param offset The offset into the shared memory region upto the start /// of the input tensor values. The default value is 0. /// \return Error object indicating success or failure Error SetSharedMemory( const std::string& name, size_t byte_size, size_t offset = 0); /// \return true if this input is being provided in shared memory. bool IsSharedMemory() const { return (io_type_ == SHARED_MEMORY); } /// Get information about the shared memory being used for this /// input. /// \param name Returns the name of the shared memory region. /// \param byte_size Returns the size, in bytes, of the shared /// memory region. /// \param offset Returns the offset within the shared memory /// region. /// \return Error object indicating success or failure. Error SharedMemoryInfo( std::string* name, size_t* byte_size, size_t* offset) const; /// Append tensor values for this input from a vector or /// strings. This method can only be used for tensors with BYTES /// data-type. The strings are assigned in row-major order to the /// elements of the tensor. The strings are copied and so the /// 'input' does not need to be preserved as with AppendRaw(). Multiple /// calls can be made to this API to keep adding tensor data for /// this input. The data will be delivered in the order it was added. /// \param input The vector holding tensor string values. /// \return Error object indicating success or failure. Error AppendFromString(const std::vector& input); /// Get access to the buffer holding raw input. Note the buffer is owned by /// InferInput instance. Users can copy out the data if required to extend /// the lifetime. /// \param buf Returns the pointer to the start of the buffer. /// \param byte_size Returns the size of buffer in bytes. /// \return Error object indicating success or failure of the /// request. Error RawData(const uint8_t** buf, size_t* byte_size); /// Gets the size of data added into this input in bytes. /// \param byte_size The size of data added in bytes. /// \return Error object indicating success or failure. Error ByteSize(size_t* byte_size) const; /// \return true if this input should be sent in binary format. bool BinaryData() const { return binary_data_; } /// \return Error object indicating success or failure. Error SetBinaryData(const bool binary_data); private: #ifdef TRITON_INFERENCE_SERVER_CLIENT_CLASS friend class TRITON_INFERENCE_SERVER_CLIENT_CLASS; #endif friend class HttpInferRequest; InferInput( const std::string& name, const std::vector& dims, const std::string& datatype); Error PrepareForRequest(); Error GetNext( uint8_t* buf, size_t size, size_t* input_bytes, bool* end_of_input); Error GetNext(const uint8_t** buf, size_t* input_bytes, bool* end_of_input); std::string name_; std::vector shape_; std::string datatype_; size_t byte_size_; size_t bufs_idx_, buf_pos_; std::vector bufs_; std::vector buf_byte_sizes_; // Used only for STRING type tensors set with SetFromString(). Hold // the "raw" serialization of the string values for each index // that are then referenced by 'bufs_'. A std::list is used to avoid // reallocs that could invalidate the pointer references into the // std::string objects. std::list str_bufs_; // Used only if working with Shared Memory enum IOType { NONE, RAW, SHARED_MEMORY }; IOType io_type_; std::string shm_name_; size_t shm_offset_; bool binary_data_{true}; }; //============================================================================== /// An InferRequestedOutput object is used to describe the requested model /// output for inference. /// class InferRequestedOutput { public: /// Create a InferRequestedOutput instance that describes a model output being /// requested. /// \param infer_output Returns a new InferOutputGrpc object. /// \param name The name of output being requested. /// \param class_count The number of classifications to be requested. The /// default value is 0 which means the classification results are not /// requested. /// \return Error object indicating success or failure. static Error Create( InferRequestedOutput** infer_output, const std::string& name, const size_t class_count = 0, const std::string& datatype = ""); /// Gets name of the associated output tensor. /// \return The name of the tensor. const std::string& Name() const { return name_; } /// Get the number of classifications requested for this output, or /// 0 if the output is not being returned as classifications. size_t ClassificationCount() const { return class_count_; } /// Set the output tensor data to be written to specified shared /// memory region. /// \param region_name The name of the shared memory region. /// \param byte_size The size of data in bytes. /// \param offset The offset in shared memory region. Default value is 0. /// \return Error object indicating success or failure of the /// request. Error SetSharedMemory( const std::string& region_name, const size_t byte_size, const size_t offset = 0); /// Clears the shared memory option set by the last call to /// InferRequestedOutput::SetSharedMemory(). After call to this /// function requested output will no longer be returned in a /// shared memory region. /// \return Error object indicating success or failure of the /// request. Error UnsetSharedMemory(); /// \return true if this output is being returned in shared memory. bool IsSharedMemory() const { return (io_type_ == SHARED_MEMORY); } /// Get information about the shared memory being used for this /// output. /// \param name Returns the name of the shared memory region. /// \param byte_size Returns the size, in bytes, of the shared /// memory region. /// \param offset Returns the offset within the shared memory /// region. /// \return Error object indicating success or failure. Error SharedMemoryInfo( std::string* name, size_t* byte_size, size_t* offset) const; /// \return true if this output should be received in binary format. bool BinaryData() const { return binary_data_; } /// \return Error object indicating success or failure. Error SetBinaryData(const bool binary_data); private: #ifdef TRITON_INFERENCE_SERVER_CLIENT_CLASS friend class TRITON_INFERENCE_SERVER_CLIENT_CLASS; #endif explicit InferRequestedOutput( const std::string& name, const std::string& datatype, const size_t class_count = 0); std::string name_; std::string datatype_; size_t class_count_; // Used only if working with Shared Memory enum IOType { NONE, RAW, SHARED_MEMORY }; IOType io_type_; std::string shm_name_; size_t shm_byte_size_; size_t shm_offset_; bool binary_data_{true}; }; //============================================================================== /// An interface for InferResult object to interpret the response to an /// inference request. /// class InferResult { public: virtual ~InferResult() = default; /// Get the name of the model which generated this response. /// \param name Returns the name of the model. /// \return Error object indicating success or failure. virtual Error ModelName(std::string* name) const = 0; /// Get the version of the model which generated this response. /// \param version Returns the version of the model. /// \return Error object indicating success or failure. virtual Error ModelVersion(std::string* version) const = 0; /// Get the id of the request which generated this response. /// \param version Returns the version of the model. /// \return Error object indicating success or failure. virtual Error Id(std::string* id) const = 0; /// Get the shape of output result returned in the response. /// \param output_name The name of the output to get shape. /// \param shape Returns the shape of result for specified output name. /// \return Error object indicating success or failure. virtual Error Shape( const std::string& output_name, std::vector* shape) const = 0; /// Get the datatype of output result returned in the response. /// \param output_name The name of the output to get datatype. /// \param shape Returns the datatype of result for specified output name. /// \return Error object indicating success or failure. virtual Error Datatype( const std::string& output_name, std::string* datatype) const = 0; /// Get access to the buffer holding raw results of specified output /// returned by the server. Note the buffer is owned by InferResult /// instance. Users can copy out the data if required to extend the /// lifetime. /// \param output_name The name of the output to get result data. /// \param buf Returns the pointer to the start of the buffer. /// \param byte_size Returns the size of buffer in bytes. /// \return Error object indicating success or failure of the /// request. virtual Error RawData( const std::string& output_name, const uint8_t** buf, size_t* byte_size) const = 0; /// Get final response bool for this response. /// \return Error object indicating the success or failure. virtual Error IsFinalResponse(bool* is_final_response) const = 0; /// Get null response bool for this response. /// \return Error object indicating the success or failure. virtual Error IsNullResponse(bool* is_null_response) const = 0; /// Get the result data as a vector of strings. The vector will /// receive a copy of result data. An error will be generated if /// the datatype of output is not 'BYTES'. /// \param output_name The name of the output to get result data. /// \param string_result Returns the result data represented as /// a vector of strings. The strings are stored in the /// row-major order. /// \return Error object indicating success or failure of the /// request. virtual Error StringData( const std::string& output_name, std::vector* string_result) const = 0; /// Returns the complete response as a user friendly string. /// \return The string describing the complete response. virtual std::string DebugString() const = 0; /// Returns the status of the request. /// \return Error object indicating the success or failure of the /// request. virtual Error RequestStatus() const = 0; }; //============================================================================== /// Records timestamps for different stages of request handling. /// class RequestTimers { public: /// Timestamp kinds. enum class Kind { /// The start of request handling. REQUEST_START, /// The end of request handling. REQUEST_END, /// The start of sending request bytes to the server (i.e. first /// byte). SEND_START, /// The end of sending request bytes to the server (i.e. last /// byte). SEND_END, /// The start of receiving response bytes from the server /// (i.e. first byte). RECV_START, /// The end of receiving response bytes from the server (i.e. last /// byte). RECV_END, COUNT__ }; /// Construct a timer with zero-ed timestamps. RequestTimers() : timestamps_((size_t)Kind::COUNT__) { Reset(); } /// Reset all timestamp values to zero. Must be called before /// re-using the timer. void Reset() { memset(×tamps_[0], 0, sizeof(uint64_t) * timestamps_.size()); } /// Get the timestamp, in nanoseconds, for a kind. /// \param kind The timestamp kind. /// \return The timestamp in nanoseconds. uint64_t Timestamp(Kind kind) const { return timestamps_[(size_t)kind]; } /// Set a timestamp to the current time, in nanoseconds. /// \param kind The timestamp kind. /// \return The timestamp in nanoseconds. uint64_t CaptureTimestamp(Kind kind) { uint64_t& ts = timestamps_[(size_t)kind]; ts = std::chrono::duration_cast( std::chrono::high_resolution_clock::now().time_since_epoch()) .count(); return ts; } /// Return the duration between start time point and end timepoint /// in nanosecond. /// \param start The start time point. /// \param end The end time point. /// \return Duration in nanosecond, or /// std::numeric_limits::max to indicate that duration /// could not be calculated. uint64_t Duration(Kind start, Kind end) const { const uint64_t stime = timestamps_[(size_t)start]; const uint64_t etime = timestamps_[(size_t)end]; // If the start or end timestamp is 0 then can't calculate the // duration, so return max to indicate error. if ((stime == 0) || (etime == 0)) { return (std::numeric_limits::max)(); } return (stime > etime) ? (std::numeric_limits::max)() : etime - stime; } private: std::vector timestamps_; }; //============================================================================== /// The base class to describe an inflight inference request. /// class InferRequest { public: InferRequest( InferenceServerClient::OnCompleteFn callback = nullptr, const bool verbose = false) : callback_(callback), verbose_(verbose) { } virtual ~InferRequest() = default; RequestTimers& Timer() { return timer_; } protected: InferenceServerClient::OnCompleteFn callback_; const bool verbose_; private: // The timers for infer request. RequestTimers timer_; }; }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/grpc_client.cc000066400000000000000000001501351527052371200225230ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Include this first to make sure we are a friend of common classes. #define TRITON_INFERENCE_SERVER_CLIENT_CLASS InferenceServerGrpcClient #include "grpc_client.h" #include #include #include #include #include #include #include #include #include "common.h" namespace triton { namespace client { namespace { //============================================================================== // Use map to keep track of GRPC channels. : > If context is created on url that has established Channel // and hasn't reached max shared count, then reuse it. std::map< std::string, std::tuple< size_t, std::shared_ptr, std::shared_ptr>> grpc_channel_stub_map_; std::mutex grpc_channel_stub_map_mtx_; std::string GetEnvironmentVariableOrDefault( const std::string& variable_name, const std::string& default_value) { const char* value = getenv(variable_name.c_str()); return value ? value : default_value; } void ReadFile(const std::string& filename, std::string& data) { data.clear(); if (!filename.empty()) { std::ifstream file(filename.c_str(), std::ios::in); if (file.is_open()) { std::stringstream ss; ss << file.rdbuf(); file.close(); data = ss.str(); } } } std::shared_ptr GetStub( const std::string& url, bool use_ssl, const SslOptions& ssl_options, const grpc::ChannelArguments& channel_args, const bool use_cached_channel, bool verbose) { std::lock_guard lock(grpc_channel_stub_map_mtx_); // Limit the number of sharing for each channel connects to the url, // distributing clients to different channels relieves // the pressure of reaching max connection concurrency // https://grpc.io/docs/guides/performance/ (4th point) static const size_t max_share_count = std::stoul(GetEnvironmentVariableOrDefault( "TRITON_CLIENT_GRPC_CHANNEL_MAX_SHARE_COUNT", "6")); const auto& channel_itr = grpc_channel_stub_map_.find(url); // Reuse cached channel if the channel is found in the map and // use_cached_channel flag is true if ((channel_itr != grpc_channel_stub_map_.end()) && use_cached_channel) { // check if NewStub should be created const auto& shared_count = std::get<0>(channel_itr->second); if (shared_count % max_share_count != 0) { std::get<0>(channel_itr->second)++; return std::get<2>(channel_itr->second); } } if (verbose) { std::cout << "Creating new channel with url:" << url << std::endl; } // Start with a copy of channel_args param, then modify our copy as needed. grpc::ChannelArguments arguments(channel_args); static std::atomic channel_count{0}; // Explicitly avoid channel re-use // "channels must have different channel args to prevent re-use // so define a use-specific channel arg such as channel number" // https://grpc.io/docs/guides/performance/ // NOTE: The argument name "triton_client_channel_idx" is arbitrary. arguments.SetInt("triton_client_channel_idx", channel_count.fetch_add(1)); std::shared_ptr credentials; if (use_ssl) { std::string root; std::string key; std::string cert; ReadFile(ssl_options.root_certificates, root); ReadFile(ssl_options.private_key, key); ReadFile(ssl_options.certificate_chain, cert); grpc::SslCredentialsOptions opts = {root, key, cert}; credentials = grpc::SslCredentials(opts); } else { credentials = grpc::InsecureChannelCredentials(); } std::shared_ptr channel = grpc::CreateCustomChannel(url, credentials, arguments); std::shared_ptr stub = inference::GRPCInferenceService::NewStub(channel); // If `use_cached_channel` is true, create no new channels even if there // are no cached channels. if (use_cached_channel) { // Replace if channel / stub have been in the map if (channel_itr != grpc_channel_stub_map_.end()) { channel_itr->second = std::make_tuple(1, channel, stub); } else { grpc_channel_stub_map_.insert( std::make_pair(url, std::make_tuple(1, channel, stub))); } } return stub; } /// Set client timeout /// /// \param client_timeout_ms Deadline for timeout in microseconds /// \param context Client context to add deadline to void SetTimeout(const uint64_t& client_timeout_ms, grpc::ClientContext* context) { if (client_timeout_ms != 0) { auto deadline = std::chrono::system_clock::now() + std::chrono::microseconds(client_timeout_ms); context->set_deadline(deadline); } } } // namespace //============================================================================== // An GrpcInferRequest represents an inflght inference request on gRPC. // class GrpcInferRequest : public InferRequest { public: GrpcInferRequest(InferenceServerClient::OnCompleteFn callback = nullptr) : InferRequest(callback), grpc_status_(), grpc_response_(std::make_shared()) { } friend InferenceServerGrpcClient; private: // Variables for GRPC call grpc::ClientContext grpc_context_; grpc::Status grpc_status_; std::shared_ptr grpc_response_; }; //============================================================================== class InferResultGrpc : public InferResult { public: static Error Create( InferResult** infer_result, std::shared_ptr response, Error& request_status); static Error Create( InferResult** infer_result, std::shared_ptr response); Error RequestStatus() const override; Error ModelName(std::string* name) const override; Error ModelVersion(std::string* version) const override; Error Id(std::string* id) const override; Error Shape(const std::string& output_name, std::vector* shape) const override; Error Datatype( const std::string& output_name, std::string* datatype) const override; Error RawData( const std::string& output_name, const uint8_t** buf, size_t* byte_size) const override; Error IsFinalResponse(bool* is_final_response) const override; Error IsNullResponse(bool* is_null_response) const override; Error StringData( const std::string& output_name, std::vector* string_result) const override; std::string DebugString() const override { return response_->DebugString(); } private: InferResultGrpc( std::shared_ptr response, Error& request_status); InferResultGrpc( std::shared_ptr response); std::map output_name_to_tensor_map_; std::map> output_name_to_buffer_map_; std::shared_ptr response_; std::shared_ptr stream_response_; Error request_status_; bool is_final_response_{true}; bool is_null_response_{false}; }; Error InferResultGrpc::Create( InferResult** infer_result, std::shared_ptr response, Error& request_status) { *infer_result = reinterpret_cast( new InferResultGrpc(response, request_status)); return Error::Success; } Error InferResultGrpc::Create( InferResult** infer_result, std::shared_ptr response) { *infer_result = reinterpret_cast(new InferResultGrpc(response)); return Error::Success; } Error InferResultGrpc::RequestStatus() const { return request_status_; } Error InferResultGrpc::ModelName(std::string* name) const { *name = response_->model_name(); return Error::Success; } Error InferResultGrpc::ModelVersion(std::string* version) const { *version = response_->model_version(); return Error::Success; } Error InferResultGrpc::Id(std::string* id) const { *id = response_->id(); return Error::Success; } Error InferResultGrpc::Shape( const std::string& output_name, std::vector* shape) const { shape->clear(); auto it = output_name_to_tensor_map_.find(output_name); if (it != output_name_to_tensor_map_.end()) { for (const auto dim : it->second->shape()) { shape->push_back(dim); } } else { return Error( "The response does not contain shape for output name '" + output_name + "'"); } return Error::Success; } Error InferResultGrpc::Datatype( const std::string& output_name, std::string* datatype) const { auto it = output_name_to_tensor_map_.find(output_name); if (it != output_name_to_tensor_map_.end()) { *datatype = it->second->datatype(); } else { return Error( "The response does not contain datatype for output name '" + output_name + "'"); } return Error::Success; } Error InferResultGrpc::RawData( const std::string& output_name, const uint8_t** buf, size_t* byte_size) const { auto it = output_name_to_buffer_map_.find(output_name); if (it != output_name_to_buffer_map_.end()) { *buf = it->second.first; *byte_size = it->second.second; } else { return Error( "The response does not contain results for output name '" + output_name + "'"); } return Error::Success; } Error InferResultGrpc::IsFinalResponse(bool* is_final_response) const { if (is_final_response == nullptr) { return Error("is_final_response cannot be nullptr"); } *is_final_response = is_final_response_; return Error::Success; } Error InferResultGrpc::IsNullResponse(bool* is_null_response) const { if (is_null_response == nullptr) { return Error("is_null_response cannot be nullptr"); } *is_null_response = is_null_response_; return Error::Success; } Error InferResultGrpc::StringData( const std::string& output_name, std::vector* string_result) const { std::string datatype; Error err = Datatype(output_name, &datatype); if (!err.IsOk()) { return err; } if (datatype.compare("BYTES") != 0) { return Error( "This function supports tensors with datatype 'BYTES', requested " "output tensor '" + output_name + "' with datatype '" + datatype + "'"); } const uint8_t* buf; size_t byte_size; err = RawData(output_name, &buf, &byte_size); string_result->clear(); if (byte_size != 0) { size_t buf_offset = 0; while (byte_size > buf_offset) { const uint32_t element_size = *(reinterpret_cast(buf + buf_offset)); string_result->emplace_back( reinterpret_cast( buf + buf_offset + sizeof(element_size)), element_size); buf_offset += (sizeof(element_size) + element_size); } } else { auto it = output_name_to_tensor_map_.find(output_name); for (const auto& element : it->second->contents().bytes_contents()) { string_result->push_back(element); } } return Error::Success; } InferResultGrpc::InferResultGrpc( std::shared_ptr response, Error& request_status) : response_(response), request_status_(request_status) { uint32_t index = 0; for (const auto& output : response_->outputs()) { output_name_to_tensor_map_[output.name()] = &output; const uint8_t* buf = (uint8_t*)&(response_->raw_output_contents()[index][0]); const uint32_t byte_size = response_->raw_output_contents()[index].size(); output_name_to_buffer_map_.insert( std::make_pair(output.name(), std::make_pair(buf, byte_size))); index++; } const auto& is_final_response_itr{ response_->parameters().find("triton_final_response")}; if (is_final_response_itr != response_->parameters().end()) { is_final_response_ = is_final_response_itr->second.bool_param(); } is_null_response_ = response_->outputs().empty() && is_final_response_; } InferResultGrpc::InferResultGrpc( std::shared_ptr stream_response) : stream_response_(stream_response) { request_status_ = Error(stream_response_->error_message()); response_.reset( stream_response->mutable_infer_response(), [](inference::ModelInferResponse*) {}); uint32_t index = 0; for (const auto& output : response_->outputs()) { output_name_to_tensor_map_[output.name()] = &output; const uint8_t* buf = (uint8_t*)&(response_->raw_output_contents()[index][0]); const uint32_t byte_size = response_->raw_output_contents()[index].size(); output_name_to_buffer_map_.insert( std::make_pair(output.name(), std::make_pair(buf, byte_size))); index++; } const auto& is_final_response_itr{ response_->parameters().find("triton_final_response")}; if (is_final_response_itr != response_->parameters().end()) { is_final_response_ = is_final_response_itr->second.bool_param(); } is_null_response_ = response_->outputs().empty() && is_final_response_; } //============================================================================== // Advanced users can generically pass any channel arguments // through the channel_args parameter, including KeepAlive options. // Channel arguments provided by the user are at the user's own risk and // are assumed to be correct/complete. Error InferenceServerGrpcClient::Create( std::unique_ptr* client, const std::string& server_url, const grpc::ChannelArguments& channel_args, bool verbose, bool use_ssl, const SslOptions& ssl_options, const bool use_cached_channel) { client->reset(new InferenceServerGrpcClient( server_url, verbose, use_ssl, ssl_options, channel_args, use_cached_channel)); return Error::Success; } // Most users should use this method of creating a client unless // they have an advanced use case that is not supported. Error InferenceServerGrpcClient::Create( std::unique_ptr* client, const std::string& server_url, bool verbose, bool use_ssl, const SslOptions& ssl_options, const KeepAliveOptions& keepalive_options, const bool use_cached_channel) { // Construct channel channel_args specific to Triton grpc::ChannelArguments channel_args; channel_args.SetMaxSendMessageSize(MAX_GRPC_MESSAGE_SIZE); channel_args.SetMaxReceiveMessageSize(MAX_GRPC_MESSAGE_SIZE); // GRPC KeepAlive: https://github.com/grpc/grpc/blob/master/doc/keepalive.md channel_args.SetInt( GRPC_ARG_KEEPALIVE_TIME_MS, keepalive_options.keepalive_time_ms); channel_args.SetInt( GRPC_ARG_KEEPALIVE_TIMEOUT_MS, keepalive_options.keepalive_timeout_ms); channel_args.SetInt( GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, keepalive_options.keepalive_permit_without_calls); channel_args.SetInt( GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, keepalive_options.http2_max_pings_without_data); client->reset(new InferenceServerGrpcClient( server_url, verbose, use_ssl, ssl_options, channel_args, use_cached_channel)); return Error::Success; } Error InferenceServerGrpcClient::IsServerLive( bool* live, const Headers& headers, const uint64_t timeout_ms) { Error err; inference::ServerLiveRequest request; inference::ServerLiveResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } grpc::Status grpc_status = stub_->ServerLive(&context, request, &response); if (grpc_status.ok()) { *live = response.live(); if (verbose_) { std::cout << "Server Live : " << *live << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::IsServerReady( bool* ready, const Headers& headers, const uint64_t timeout_ms) { Error err; inference::ServerReadyRequest request; inference::ServerReadyResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } grpc::Status grpc_status = stub_->ServerReady(&context, request, &response); if (grpc_status.ok()) { *ready = response.ready(); if (verbose_) { std::cout << "Server Ready : " << *ready << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::IsModelReady( bool* ready, const std::string& model_name, const std::string& model_version, const Headers& headers, const uint64_t timeout_ms) { Error err; inference::ModelReadyRequest request; inference::ModelReadyResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(model_name); request.set_version(model_version); grpc::Status grpc_status = stub_->ModelReady(&context, request, &response); if (grpc_status.ok()) { *ready = response.ready(); if (verbose_) { std::cout << "Model Ready : name: " << model_name; if (!model_version.empty()) { std::cout << "(version: " << model_version << ") "; } std::cout << ": " << *ready << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::ServerMetadata( inference::ServerMetadataResponse* server_metadata, const Headers& headers, const uint64_t timeout_ms) { server_metadata->Clear(); Error err; inference::ServerMetadataRequest request; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } grpc::Status grpc_status = stub_->ServerMetadata(&context, request, server_metadata); if (grpc_status.ok()) { if (verbose_) { std::cout << server_metadata->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::ModelMetadata( inference::ModelMetadataResponse* model_metadata, const std::string& model_name, const std::string& model_version, const Headers& headers, const uint64_t timeout_ms) { model_metadata->Clear(); Error err; inference::ModelMetadataRequest request; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(model_name); request.set_version(model_version); grpc::Status grpc_status = stub_->ModelMetadata(&context, request, model_metadata); if (grpc_status.ok()) { if (verbose_) { std::cout << model_metadata->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::ModelConfig( inference::ModelConfigResponse* model_config, const std::string& model_name, const std::string& model_version, const Headers& headers, const uint64_t timeout_ms) { model_config->Clear(); Error err; inference::ModelConfigRequest request; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(model_name); request.set_version(model_version); grpc::Status grpc_status = stub_->ModelConfig(&context, request, model_config); if (grpc_status.ok()) { if (verbose_) { std::cout << model_config->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::ModelRepositoryIndex( inference::RepositoryIndexResponse* repository_index, const Headers& headers, const uint64_t timeout_ms) { repository_index->Clear(); Error err; inference::RepositoryIndexRequest request; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } grpc::Status grpc_status = stub_->RepositoryIndex(&context, request, repository_index); if (grpc_status.ok()) { if (verbose_) { std::cout << repository_index->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::LoadModel( const std::string& model_name, const Headers& headers, const std::string& config, const std::map>& files, const uint64_t timeout_ms) { Error err; inference::RepositoryModelLoadRequest request; inference::RepositoryModelLoadResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_model_name(model_name); if (!config.empty()) { (*request.mutable_parameters())["config"].set_string_param(config); } for (const auto& file : files) { (*request.mutable_parameters())[file.first].set_bytes_param( file.second.data(), file.second.size()); } grpc::Status grpc_status = stub_->RepositoryModelLoad(&context, request, &response); if (!grpc_status.ok()) { err = Error(grpc_status.error_message()); } else { if (verbose_) { std::cout << "Loaded model '" << model_name << "'" << std::endl; } } return err; } Error InferenceServerGrpcClient::UnloadModel( const std::string& model_name, const Headers& headers, const uint64_t timeout_ms) { Error err; inference::RepositoryModelUnloadRequest request; inference::RepositoryModelUnloadResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_model_name(model_name); grpc::Status grpc_status = stub_->RepositoryModelUnload(&context, request, &response); if (!grpc_status.ok()) { err = Error(grpc_status.error_message()); } else { if (verbose_) { std::cout << "Unloaded model '" << model_name << "'" << std::endl; } } return err; } Error InferenceServerGrpcClient::ModelInferenceStatistics( inference::ModelStatisticsResponse* infer_stat, const std::string& model_name, const std::string& model_version, const Headers& headers, const uint64_t timeout_ms) { infer_stat->Clear(); Error err; inference::ModelStatisticsRequest request; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(model_name); request.set_version(model_version); grpc::Status grpc_status = stub_->ModelStatistics(&context, request, infer_stat); if (grpc_status.ok()) { if (verbose_) { std::cout << infer_stat->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::UpdateTraceSettings( inference::TraceSettingResponse* response, const std::string& model_name, const std::map>& settings, const Headers& headers, const uint64_t timeout_ms) { inference::TraceSettingRequest request; grpc::ClientContext context; Error err; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } if (!model_name.empty()) { request.set_model_name(model_name); } if (!settings.empty()) { for (const auto& pr : settings) { if (pr.second.empty()) { (*request.mutable_settings())[pr.first].clear_value(); } else { for (const auto& v : pr.second) { (*request.mutable_settings())[pr.first].add_value(v); } } } } grpc::Status grpc_status = stub_->TraceSetting(&context, request, response); if (grpc_status.ok()) { if (verbose_) { std::cout << "Update trace settings " << response->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::GetTraceSettings( inference::TraceSettingResponse* settings, const std::string& model_name, const Headers& headers, const uint64_t timeout_ms) { settings->Clear(); Error err; inference::TraceSettingRequest request; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } if (!model_name.empty()) { request.set_model_name(model_name); } grpc::Status grpc_status = stub_->TraceSetting(&context, request, settings); if (grpc_status.ok()) { if (verbose_) { std::cout << settings->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::SystemSharedMemoryStatus( inference::SystemSharedMemoryStatusResponse* status, const std::string& region_name, const Headers& headers, const uint64_t timeout_ms) { status->Clear(); Error err; inference::SystemSharedMemoryStatusRequest request; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(region_name); grpc::Status grpc_status = stub_->SystemSharedMemoryStatus(&context, request, status); if (grpc_status.ok()) { if (verbose_) { std::cout << status->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::RegisterSystemSharedMemory( const std::string& name, const std::string& key, const size_t byte_size, const size_t offset, const Headers& headers, const uint64_t timeout_ms) { Error err; inference::SystemSharedMemoryRegisterRequest request; inference::SystemSharedMemoryRegisterResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(name); request.set_key(key); request.set_offset(offset); request.set_byte_size(byte_size); grpc::Status grpc_status = stub_->SystemSharedMemoryRegister(&context, request, &response); if (!grpc_status.ok()) { err = Error(grpc_status.error_message()); } else { if (verbose_) { std::cout << "Registered system shared memory with name '" << name << "'" << std::endl; } } return err; } Error InferenceServerGrpcClient::UnregisterSystemSharedMemory( const std::string& name, const Headers& headers, const uint64_t timeout_ms) { Error err; inference::SystemSharedMemoryUnregisterRequest request; inference::SystemSharedMemoryUnregisterResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(name); grpc::Status grpc_status = stub_->SystemSharedMemoryUnregister(&context, request, &response); if (!grpc_status.ok()) { err = Error(grpc_status.error_message()); } else { if (verbose_) { if (name.size() != 0) { std::cout << "Unregistered system shared memory with name '" << name << "'" << std::endl; } else { std::cout << "Unregistered all system shared memory regions" << std::endl; } } } return err; } Error InferenceServerGrpcClient::CudaSharedMemoryStatus( inference::CudaSharedMemoryStatusResponse* status, const std::string& region_name, const Headers& headers, const uint64_t timeout_ms) { status->Clear(); Error err; inference::CudaSharedMemoryStatusRequest request; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(region_name); grpc::Status grpc_status = stub_->CudaSharedMemoryStatus(&context, request, status); if (grpc_status.ok()) { if (verbose_) { std::cout << status->DebugString() << std::endl; } } else { err = Error(grpc_status.error_message()); } return err; } Error InferenceServerGrpcClient::RegisterCudaSharedMemory( const std::string& name, const cudaIpcMemHandle_t& cuda_shm_handle, const size_t device_id, const size_t byte_size, const Headers& headers, const uint64_t timeout_ms) { Error err; inference::CudaSharedMemoryRegisterRequest request; inference::CudaSharedMemoryRegisterResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(name); request.set_raw_handle((char*)&cuda_shm_handle, sizeof(cudaIpcMemHandle_t)); request.set_device_id(device_id); request.set_byte_size(byte_size); grpc::Status grpc_status = stub_->CudaSharedMemoryRegister(&context, request, &response); if (!grpc_status.ok()) { err = Error(grpc_status.error_message()); } else { if (verbose_) { std::cout << "Registered cuda shared memory with name '" << name << "'" << std::endl; } } return err; } Error InferenceServerGrpcClient::UnregisterCudaSharedMemory( const std::string& name, const Headers& headers, const uint64_t timeout_ms) { Error err; inference::CudaSharedMemoryUnregisterRequest request; inference::CudaSharedMemoryUnregisterResponse response; grpc::ClientContext context; SetTimeout(timeout_ms, &context); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } request.set_name(name); grpc::Status grpc_status = stub_->CudaSharedMemoryUnregister(&context, request, &response); if (!grpc_status.ok()) { err = Error(grpc_status.error_message()); } else { if (verbose_) { if (name.size() != 0) { std::cout << "Unregistered system shared memory with name '" << name << "'" << std::endl; } else { std::cout << "Unregistered all system shared memory regions" << std::endl; } } } return err; } Error InferenceServerGrpcClient::Infer( InferResult** result, const InferOptions& options, const std::vector& inputs, const std::vector& outputs, const Headers& headers, grpc_compression_algorithm compression_algorithm) { Error err; grpc::ClientContext context; std::shared_ptr sync_request(new GrpcInferRequest()); sync_request->Timer().Reset(); sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::REQUEST_START); // Use send timer to measure time for marshalling infer request sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::SEND_START); for (const auto& it : headers) { context.AddMetadata(it.first, it.second); } if (options.client_timeout_ != 0) { SetTimeout(options.client_timeout_, &context); } context.set_compression_algorithm(compression_algorithm); err = PreRunProcessing(options, inputs, outputs); sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::SEND_END); if (!err.IsOk()) { return err; } sync_request->grpc_response_->Clear(); sync_request->grpc_status_ = stub_->ModelInfer( &context, infer_request_, sync_request->grpc_response_.get()); if (!sync_request->grpc_status_.ok()) { err = Error(sync_request->grpc_status_.error_message()); } sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::RECV_START); InferResultGrpc::Create(result, sync_request->grpc_response_, err); sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::RECV_END); sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::REQUEST_END); err = UpdateInferStat(sync_request->Timer()); if (!err.IsOk()) { std::cerr << "Failed to update context stat: " << err << std::endl; } if (sync_request->grpc_status_.ok()) { if (verbose_) { std::cout << sync_request->grpc_response_->DebugString() << std::endl; } } return (*result)->RequestStatus(); } Error InferenceServerGrpcClient::AsyncInfer( OnCompleteFn callback, const InferOptions& options, const std::vector& inputs, const std::vector& outputs, const Headers& headers, grpc_compression_algorithm compression_algorithm) { if (callback == nullptr) { return Error( "Callback function must be provided along with AsyncInfer() call."); } if (!worker_.joinable()) { worker_ = std::thread(&InferenceServerGrpcClient::AsyncTransfer, this); } GrpcInferRequest* async_request; async_request = new GrpcInferRequest(std::move(callback)); async_request->Timer().CaptureTimestamp(RequestTimers::Kind::REQUEST_START); async_request->Timer().CaptureTimestamp(RequestTimers::Kind::SEND_START); for (const auto& it : headers) { async_request->grpc_context_.AddMetadata(it.first, it.second); } if (options.client_timeout_ != 0) { SetTimeout(options.client_timeout_, &(async_request->grpc_context_)); } async_request->grpc_context_.set_compression_algorithm(compression_algorithm); Error err = PreRunProcessing(options, inputs, outputs); if (!err.IsOk()) { delete async_request; return err; } async_request->Timer().CaptureTimestamp(RequestTimers::Kind::SEND_END); std::unique_ptr< grpc::ClientAsyncResponseReader> rpc(stub_->PrepareAsyncModelInfer( &async_request->grpc_context_, infer_request_, &async_request_completion_queue_)); rpc->StartCall(); rpc->Finish( async_request->grpc_response_.get(), &async_request->grpc_status_, (void*)async_request); if (verbose_) { std::cout << "Sent request"; if (options.request_id_.size() != 0) { std::cout << " '" << options.request_id_ << "'"; } std::cout << std::endl; } return Error::Success; } Error InferenceServerGrpcClient::InferMulti( std::vector* results, const std::vector& options, const std::vector>& inputs, const std::vector>& outputs, const Headers& headers, grpc_compression_algorithm compression_algorithm) { Error err; // Sanity check if ((inputs.size() != options.size()) && (options.size() != 1)) { return Error( "'options' must either contain 1 element or match size of 'inputs'"); } if ((inputs.size() != outputs.size()) && ((outputs.size() != 1) && (outputs.size() != 0))) { return Error( "'outputs' must either contain 0/1 element or match size of 'inputs'"); } int64_t max_option_idx = options.size() - 1; // value of '-1' means no output is specified int64_t max_output_idx = outputs.size() - 1; static std::vector empty_outputs{}; for (int64_t i = 0; i < (int64_t)inputs.size(); ++i) { const auto& request_options = options[std::min(max_option_idx, i)]; const auto& request_output = (max_output_idx == -1) ? empty_outputs : outputs[std::min(max_output_idx, i)]; results->emplace_back(); err = Infer( &results->back(), request_options, inputs[i], request_output, headers, compression_algorithm); if (!err.IsOk()) { return err; } } return Error::Success; } Error InferenceServerGrpcClient::AsyncInferMulti( OnMultiCompleteFn callback, const std::vector& options, const std::vector>& inputs, const std::vector>& outputs, const Headers& headers, grpc_compression_algorithm compression_algorithm) { // Sanity check if ((inputs.size() != options.size()) && (options.size() != 1)) { return Error( "'options' must either contain 1 element or match size of 'inputs'"); } if ((inputs.size() != outputs.size()) && ((outputs.size() != 1) && (outputs.size() != 0))) { return Error( "'outputs' must either contain 0/1 element or match size of 'inputs'"); } if (callback == nullptr) { return Error( "Callback function must be provided along with AsyncInferMulti() " "call."); } if (!worker_.joinable()) { worker_ = std::thread(&InferenceServerGrpcClient::AsyncTransfer, this); } int64_t max_option_idx = options.size() - 1; // value of '-1' means no output is specified int64_t max_output_idx = outputs.size() - 1; static std::vector empty_outputs{}; std::shared_ptr> response_counter( new std::atomic(inputs.size())); std::shared_ptr> responses( new std::vector(inputs.size())); for (int64_t i = 0; i < (int64_t)inputs.size(); ++i) { const auto& request_options = options[std::min(max_option_idx, i)]; const auto& request_output = (max_output_idx == -1) ? empty_outputs : outputs[std::min(max_output_idx, i)]; OnCompleteFn cb = [response_counter, responses, i, callback](InferResult* result) { (*responses)[i] = result; // last response if (response_counter->fetch_sub(1) == 1) { std::vector results; results.swap(*responses); callback(results); } }; auto err = AsyncInfer( cb, request_options, inputs[i], request_output, headers, compression_algorithm); if (!err.IsOk()) { // Create response with error as other requests may be sent and their // responses may not be accessed outside the callback. InferResult* err_res; std::shared_ptr empty_response( new inference::ModelInferResponse()); InferResultGrpc::Create(&err_res, empty_response, err); cb(err_res); continue; } } return Error::Success; } Error InferenceServerGrpcClient::StartStream( OnCompleteFn callback, bool enable_stats, uint32_t stream_timeout, const Headers& headers, grpc_compression_algorithm compression_algorithm) { if (stream_worker_.joinable()) { return Error( "cannot start another stream with one already running. " "'InferenceServerClient' supports only a single active " "stream at a given time."); } if (callback == nullptr) { return Error( "Callback function must be provided along with StartStream() call."); } stream_callback_ = callback; enable_stream_stats_ = enable_stats; for (const auto& it : headers) { grpc_context_.AddMetadata(it.first, it.second); } if (stream_timeout != 0) { SetTimeout(stream_timeout, &grpc_context_); } grpc_context_.set_compression_algorithm(compression_algorithm); grpc_stream_ = stub_->ModelStreamInfer(&grpc_context_); stream_worker_ = std::thread(&InferenceServerGrpcClient::AsyncStreamTransfer, this); if (verbose_) { std::cout << "Started stream..." << std::endl; } return Error::Success; } Error InferenceServerGrpcClient::StopStream() { if (stream_worker_.joinable()) { grpc_stream_->WritesDone(); // The reader thread will drain the stream properly stream_worker_.join(); if (verbose_) { std::cout << "Stopped stream..." << std::endl; } } return Error::Success; } Error InferenceServerGrpcClient::AsyncStreamInfer( const InferOptions& options, const std::vector& inputs, const std::vector& outputs) { std::unique_ptr timer; if (enable_stream_stats_) { timer.reset(new RequestTimers()); timer->CaptureTimestamp(RequestTimers::Kind::REQUEST_START); timer->CaptureTimestamp(RequestTimers::Kind::SEND_START); } Error err = PreRunProcessing(options, inputs, outputs); if (!err.IsOk()) { return err; } if (enable_stream_stats_) { timer->CaptureTimestamp(RequestTimers::Kind::SEND_END); } if (enable_stream_stats_) { std::lock_guard lock(stream_mutex_); ongoing_stream_request_timers_.push(std::move(timer)); } bool ok = grpc_stream_->Write(infer_request_); if (ok) { if (verbose_) { std::cout << "Sent request"; if (options.request_id_.size() != 0) { std::cout << " '" << options.request_id_ << "'"; } std::cout << " to the stream" << std::endl; } return Error::Success; } else { return Error("Stream has been closed."); } } Error InferenceServerGrpcClient::PreRunProcessing( const InferOptions& options, const std::vector& inputs, const std::vector& outputs) { // Populate the request protobuf infer_request_.set_model_name(options.model_name_); infer_request_.set_model_version(options.model_version_); infer_request_.set_id(options.request_id_); infer_request_.mutable_parameters()->clear(); (*infer_request_.mutable_parameters())["triton_enable_empty_final_response"] .set_bool_param(options.triton_enable_empty_final_response_); if ((options.sequence_id_ != 0) || (options.sequence_id_str_ != "")) { if (options.sequence_id_ != 0) { (*infer_request_.mutable_parameters())["sequence_id"].set_int64_param( options.sequence_id_); } else { (*infer_request_.mutable_parameters())["sequence_id"].set_string_param( options.sequence_id_str_); } (*infer_request_.mutable_parameters())["sequence_start"].set_bool_param( options.sequence_start_); (*infer_request_.mutable_parameters())["sequence_end"].set_bool_param( options.sequence_end_); } if (options.priority_ != 0) { (*infer_request_.mutable_parameters())["priority"].set_uint64_param( options.priority_); } if (options.server_timeout_ != 0) { (*infer_request_.mutable_parameters())["timeout"].set_int64_param( options.server_timeout_); } for (auto& param : options.request_parameters) { if (param.second.type == "string") { (*infer_request_.mutable_parameters())[param.first].set_string_param( param.second.value); } else if (param.second.type == "int") { (*infer_request_.mutable_parameters())[param.first].set_int64_param( std::stoi(param.second.value)); } else if (param.second.type == "bool") { bool val = false; if (param.second.value == "true") { val = true; } (*infer_request_.mutable_parameters())[param.first].set_bool_param(val); } } int index = 0; infer_request_.mutable_raw_input_contents()->Clear(); for (const auto input : inputs) { // Add new InferInputTensor submessages only if required, otherwise // reuse the submessages already available. auto grpc_input = (infer_request_.inputs().size() <= index) ? infer_request_.add_inputs() : infer_request_.mutable_inputs()->Mutable(index); if (input->IsSharedMemory()) { // The input contents must be cleared when using shared memory. grpc_input->Clear(); } grpc_input->set_name(input->Name()); grpc_input->mutable_shape()->Clear(); for (const auto dim : input->Shape()) { grpc_input->mutable_shape()->Add(dim); } grpc_input->set_datatype(input->Datatype()); input->PrepareForRequest(); grpc_input->mutable_parameters()->clear(); if (input->IsSharedMemory()) { std::string region_name; size_t offset; size_t byte_size; input->SharedMemoryInfo(®ion_name, &byte_size, &offset); (*grpc_input->mutable_parameters())["shared_memory_region"] .set_string_param(region_name); (*grpc_input->mutable_parameters())["shared_memory_byte_size"] .set_int64_param(byte_size); if (offset != 0) { (*grpc_input->mutable_parameters())["shared_memory_offset"] .set_int64_param(offset); } } else { bool end_of_input = false; std::string* raw_contents = infer_request_.add_raw_input_contents(); size_t content_size; input->ByteSize(&content_size); raw_contents->reserve(content_size); raw_contents->clear(); while (!end_of_input) { const uint8_t* buf; size_t buf_size; input->GetNext(&buf, &buf_size, &end_of_input); if (buf != nullptr) { raw_contents->append(reinterpret_cast(buf), buf_size); } } } index++; } // Remove extra InferInputTensor submessages, that are not required for // this request. while (index < infer_request_.inputs().size()) { infer_request_.mutable_inputs()->RemoveLast(); } index = 0; for (const auto routput : outputs) { // Add new InferRequestedOutputTensor submessage only if required, otherwise // reuse the submessages already available. auto grpc_output = (infer_request_.outputs().size() <= index) ? infer_request_.add_outputs() : infer_request_.mutable_outputs()->Mutable(index); grpc_output->Clear(); grpc_output->set_name(routput->Name()); size_t class_count = routput->ClassificationCount(); if (class_count != 0) { (*grpc_output->mutable_parameters())["classification"].set_int64_param( class_count); } if (routput->IsSharedMemory()) { std::string region_name; size_t offset; size_t byte_size; routput->SharedMemoryInfo(®ion_name, &byte_size, &offset); (*grpc_output->mutable_parameters())["shared_memory_region"] .set_string_param(region_name); (*grpc_output->mutable_parameters())["shared_memory_byte_size"] .set_int64_param(byte_size); if (offset != 0) { (*grpc_output->mutable_parameters())["shared_memory_offset"] .set_int64_param(offset); } } index++; } // Remove extra InferRequestedOutputTensor submessages, that are not required // for this request. while (index < infer_request_.outputs().size()) { infer_request_.mutable_outputs()->RemoveLast(); } if (infer_request_.ByteSizeLong() > INT_MAX) { size_t request_size = infer_request_.ByteSizeLong(); infer_request_.Clear(); return Error( "Request has byte size " + std::to_string(request_size) + " which exceed gRPC's byte size limit " + std::to_string(INT_MAX) + "."); } return Error::Success; } void InferenceServerGrpcClient::AsyncTransfer() { while (!exiting_) { // GRPC async APIs are thread-safe https://github.com/grpc/grpc/issues/4486 GrpcInferRequest* raw_async_request; bool ok = true; bool status = async_request_completion_queue_.Next((void**)(&raw_async_request), &ok); std::shared_ptr async_request; if (!ok) { fprintf(stderr, "Unexpected not ok on client side.\n"); } if (!status) { if (!exiting_) { fprintf(stderr, "Completion queue is closed.\n"); } } else if (raw_async_request == nullptr) { fprintf(stderr, "Unexpected null tag received at client.\n"); } else { async_request.reset(raw_async_request); InferResult* async_result; Error err; if (!async_request->grpc_status_.ok()) { err = Error(async_request->grpc_status_.error_message()); } async_request->Timer().CaptureTimestamp(RequestTimers::Kind::RECV_START); InferResultGrpc::Create( &async_result, async_request->grpc_response_, err); async_request->Timer().CaptureTimestamp(RequestTimers::Kind::RECV_END); async_request->Timer().CaptureTimestamp(RequestTimers::Kind::REQUEST_END); err = UpdateInferStat(async_request->Timer()); if (!err.IsOk()) { std::cerr << "Failed to update context stat: " << err << std::endl; } if (async_request->grpc_status_.ok()) { if (verbose_) { std::cout << async_request->grpc_response_->DebugString() << std::endl; } } async_request->callback_(async_result); } } } void InferenceServerGrpcClient::AsyncStreamTransfer() { std::shared_ptr response = std::make_shared(); // End loop if Read() returns false // (stream ended and all responses are drained) while (grpc_stream_->Read(response.get())) { if (exiting_) { continue; } std::unique_ptr timer; if (enable_stream_stats_) { std::lock_guard lock(stream_mutex_); if (!ongoing_stream_request_timers_.empty()) { timer = std::move(ongoing_stream_request_timers_.front()); ongoing_stream_request_timers_.pop(); } } InferResult* stream_result; // FIXME, DLIS-1263 there is no 1:1 mapping between // requests and response for decoupled streaming case // hence, this method will record incorrect statistics // for decoupled case. if (timer.get() != nullptr) { timer->CaptureTimestamp(RequestTimers::Kind::RECV_START); } InferResultGrpc::Create(&stream_result, response); if (timer.get() != nullptr) { timer->CaptureTimestamp(RequestTimers::Kind::RECV_END); timer->CaptureTimestamp(RequestTimers::Kind::REQUEST_END); Error err = UpdateInferStat(*timer); if (!err.IsOk()) { std::cerr << "Failed to update context stat: " << err << std::endl; } } if (verbose_) { std::cout << response->DebugString() << std::endl; } stream_callback_(stream_result); response = std::make_shared(); } grpc_stream_->Finish(); } InferenceServerGrpcClient::InferenceServerGrpcClient( const std::string& url, bool verbose, bool use_ssl, const SslOptions& ssl_options, const grpc::ChannelArguments& channel_args, const bool use_cached_channel) : InferenceServerClient(verbose) { stub_ = GetStub( url, use_ssl, ssl_options, channel_args, use_cached_channel, verbose); } InferenceServerGrpcClient::~InferenceServerGrpcClient() { exiting_ = true; // Close complete queue and wait for the worker thread to return async_request_completion_queue_.Shutdown(); // thread not joinable if AsyncInfer() is not called // (it is default constructed thread before the first AsyncInfer() call) if (worker_.joinable()) { worker_.join(); } bool has_next = true; GrpcInferRequest* async_request; bool ok; do { has_next = async_request_completion_queue_.Next((void**)&async_request, &ok); if (has_next && async_request != nullptr) { delete async_request; } } while (has_next); StopStream(); } size_t InferenceServerGrpcClient::GetNumCachedChannels() const { std::lock_guard lock(grpc_channel_stub_map_mtx_); return grpc_channel_stub_map_.size(); } //============================================================================== }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/grpc_client.h000066400000000000000000001004571527052371200223670ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "common.h" #include "grpc_service.grpc.pb.h" #include "ipc.h" #include "model_config.pb.h" namespace triton { namespace client { /// The key-value map type to be included in the request /// metadata typedef std::map Headers; struct SslOptions { explicit SslOptions() {} // File containing the PEM encoding of the server root certificates. // If this parameter is empty, the default roots will be used. The // default roots can be overridden using the // GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment variable pointing // to a file on the file system containing the roots. std::string root_certificates; // File containing the PEM encoding of the client's private key. // This parameter can be empty if the client does not have a // private key. std::string private_key; // File containing the PEM encoding of the client's certificate chain. // This parameter can be empty if the client does not have a // certificate chain. std::string certificate_chain; }; // GRPC KeepAlive: https://grpc.github.io/grpc/cpp/md_doc_keepalive.html struct KeepAliveOptions { explicit KeepAliveOptions() : keepalive_time_ms(INT_MAX), keepalive_timeout_ms(20000), keepalive_permit_without_calls(false), http2_max_pings_without_data(2) { } // The period (in milliseconds) after which a keepalive ping is sent on the // transport int keepalive_time_ms; // The amount of time (in milliseconds) the sender of the keepalive ping waits // for an acknowledgement. If it does not receive an acknowledgment within // this time, it will close the connection. int keepalive_timeout_ms; // If true, allow keepalive pings to be sent even if there are no calls in // flight. bool keepalive_permit_without_calls; // The maximum number of pings that can be sent when there is no data/header // frame to be sent. gRPC Core will not continue sending pings if we run over // the limit. Setting it to 0 allows sending pings without such a restriction. int http2_max_pings_without_data; }; //============================================================================== /// An InferenceServerGrpcClient object is used to perform any kind of /// communication with the InferenceServer using gRPC protocol. Most /// of the methods are thread-safe except Infer, AsyncInfer, StartStream /// StopStream and AsyncStreamInfer. Calling these functions from different /// threads will cause undefined behavior. /// /// \code /// std::unique_ptr client; /// InferenceServerGrpcClient::Create(&client, "localhost:8001"); /// bool live; /// client->IsServerLive(&live); /// ... /// ... /// \endcode /// class InferenceServerGrpcClient : public InferenceServerClient { public: ~InferenceServerGrpcClient(); /// Create a client that can be used to communicate with the server. /// This is the expected method for most users to create a GRPC client with /// the options directly exposed Triton. /// \param client Returns a new InferenceServerGrpcClient object. /// \param server_url The inference server name and port. /// \param verbose If true generate verbose output when contacting /// the inference server. /// \param use_ssl If true use encrypted channel to the server. /// \param ssl_options Specifies the files required for /// SSL encryption and authorization. /// \param keepalive_options Specifies the GRPC KeepAlive options described /// in https://grpc.github.io/grpc/cpp/md_doc_keepalive.html /// \param use_cached_channel If false, a new channel is created for each /// new client instance. When true, re-use old channels from cache for new /// client instances. The default value is true. /// \return Error object indicating success or failure. static Error Create( std::unique_ptr* client, const std::string& server_url, bool verbose = false, bool use_ssl = false, const SslOptions& ssl_options = SslOptions(), const KeepAliveOptions& keepalive_options = KeepAliveOptions(), const bool use_cached_channel = true); /// Create a client that can be used to communicate with the server. /// This method is available for advanced users who need to specify custom /// grpc::ChannelArguments not exposed by Triton, at their own risk. /// \param client Returns a new InferenceServerGrpcClient object. /// \param channel_args Exposes user-defined grpc::ChannelArguments to /// be set for the client. Triton assumes that the "channel_args" passed /// to this method are correct and complete, and are set at the user's /// own risk. For example, GRPC KeepAlive options may be specified directly /// in this argument rather than passing a KeepAliveOptions object. /// \param server_url The inference server name and port. /// \param verbose If true generate verbose output when contacting /// the inference server. /// \param use_ssl If true use encrypted channel to the server. /// \param ssl_options Specifies the files required for /// SSL encryption and authorization. /// \param use_cached_channel If false, a new channel is created for each /// new client instance. When true, re-use old channels from cache for new /// client instances. The default value is true. /// \return Error object indicating success or failure. static Error Create( std::unique_ptr* client, const std::string& server_url, const grpc::ChannelArguments& channel_args, bool verbose = false, bool use_ssl = false, const SslOptions& ssl_options = SslOptions(), const bool use_cached_channel = true); /// Contact the inference server and get its liveness. /// \param live Returns whether the server is live or not. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error IsServerLive( bool* live, const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get its readiness. /// \param ready Returns whether the server is ready or not. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error IsServerReady( bool* ready, const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get the readiness of specified model. /// \param ready Returns whether the specified model is ready or not. /// \param model_name The name of the model to check for readiness. /// \param model_version The version of the model to check for readiness. /// The default value is an empty string which means then the server will /// choose a version based on the model and internal policy. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error IsModelReady( bool* ready, const std::string& model_name, const std::string& model_version = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get its metadata. /// \param server_metadata Returns the server metadata as /// SeverMetadataResponse message. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error ServerMetadata( inference::ServerMetadataResponse* server_metadata, const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get the metadata of specified model. /// \param model_metadata Returns model metadata as ModelMetadataResponse /// message. /// \param model_name The name of the model to get metadata. /// \param model_version The version of the model to get metadata. /// The default value is an empty string which means then the server will /// choose a version based on the model and internal policy. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error ModelMetadata( inference::ModelMetadataResponse* model_metadata, const std::string& model_name, const std::string& model_version = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get the configuration of specified model. /// \param model_config Returns model config as ModelConfigResponse /// message. /// \param model_name The name of the model to get configuration. /// \param model_version The version of the model to get configuration. /// The default value is an empty string which means then the server will /// choose a version based on the model and internal policy. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error ModelConfig( inference::ModelConfigResponse* model_config, const std::string& model_name, const std::string& model_version = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get the index of model repository /// contents. /// \param repository_index Returns the repository index as /// RepositoryIndexRequestResponse /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error ModelRepositoryIndex( inference::RepositoryIndexResponse* repository_index, const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Request the inference server to load or reload specified model. /// \param model_name The name of the model to be loaded or reloaded. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param config Optional JSON representation of a model config provided for /// the load request, if provided, this config will be used for /// loading the model. /// \param files Optional map specifying file path (with "file:" /// prefix) in the override model directory to the file content. /// The files will form the model directory that the model /// will be loaded from. If specified, 'config' must be provided to be /// the model configuration of the override model directory. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error LoadModel( const std::string& model_name, const Headers& headers = Headers(), const std::string& config = std::string(), const std::map>& files = {}, const uint64_t timeout_ms = 0); /// Request the inference server to unload specified model. /// \param model_name The name of the model to be unloaded. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error UnloadModel( const std::string& model_name, const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get the inference statistics for the /// specified model name and version. /// \param infer_stat The inference statistics of requested model name and /// version. /// \param model_name The name of the model to get inference statistics. The /// default value is an empty string which means statistics of all models will /// be returned in the response. /// \param model_version The version of the model to get inference statistics. /// The default value is an empty string which means then the server will /// choose a version based on the model and internal policy. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error ModelInferenceStatistics( inference::ModelStatisticsResponse* infer_stat, const std::string& model_name = "", const std::string& model_version = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Update the trace settings for the specified model name, or global trace /// settings if model name is not given. /// \param response The updated settings as TraceSettingResponse. /// \param model_name The name of the model to update trace settings. The /// default value is an empty string which means the global trace settings /// will be updated. /// \param settings The new trace setting values. Only the settings listed /// will be updated. If a trace setting is listed in the map with an empty /// string, that setting will be cleared. /// \param config Optional JSON representation of a model config provided for /// the load request, if provided, this config will be used for /// loading the model. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error UpdateTraceSettings( inference::TraceSettingResponse* response, const std::string& model_name = "", const std::map>& settings = std::map>(), const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Get the trace settings for the specified model name, or global trace /// settings if model name is not given. /// \param settings The trace settings as TraceSettingResponse. /// \param model_name The name of the model to get trace settings. The /// default value is an empty string which means the global trace settings /// will be returned. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error GetTraceSettings( inference::TraceSettingResponse* settings, const std::string& model_name = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get the status for requested system /// shared memory. /// \param status The system shared memory status as /// SystemSharedMemoryStatusResponse /// \param region_name The name of the region to query status. The default /// value is an empty string, which means that the status of all active system /// shared memory will be returned. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error SystemSharedMemoryStatus( inference::SystemSharedMemoryStatusResponse* status, const std::string& region_name = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Request the server to register a system shared memory with the provided /// details. /// \param name The name of the region to register. /// \param key The key of the underlying memory object that contains the /// system shared memory region. /// \param byte_size The size of the system shared memory region, in bytes. /// \param offset Offset, in bytes, within the underlying memory object to /// the start of the system shared memory region. The default value is zero. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request Error RegisterSystemSharedMemory( const std::string& name, const std::string& key, const size_t byte_size, const size_t offset = 0, const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Request the server to unregister a system shared memory with the /// specified name. /// \param name The name of the region to unregister. The default value is /// empty string which means all the system shared memory regions will be /// unregistered. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request Error UnregisterSystemSharedMemory( const std::string& name = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Contact the inference server and get the status for requested CUDA /// shared memory. /// \param status The CUDA shared memory status as /// CudaSharedMemoryStatusResponse /// \param region_name The name of the region to query status. The default /// value is an empty string, which means that the status of all active CUDA /// shared memory will be returned. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request. Error CudaSharedMemoryStatus( inference::CudaSharedMemoryStatusResponse* status, const std::string& region_name = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Request the server to register a CUDA shared memory with the provided /// details. /// \param name The name of the region to register. /// \param cuda_shm_handle The cudaIPC handle for the memory object. /// \param device_id The GPU device ID on which the cudaIPC handle was /// created. /// \param byte_size The size of the CUDA shared memory region, in /// bytes. /// \param headers Optional map specifying additional HTTP headers to /// include in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request Error RegisterCudaSharedMemory( const std::string& name, const cudaIpcMemHandle_t& cuda_shm_handle, const size_t device_id, const size_t byte_size, const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Request the server to unregister a CUDA shared memory with the /// specified name. /// \param name The name of the region to unregister. The default value is /// empty string which means all the CUDA shared memory regions will be /// unregistered. /// \param headers Optional map specifying additional HTTP headers to /// include in the metadata of gRPC request. /// \param timeout_ms Optional timeout for API call, in microseconds, the /// request is allowed to take. /// \return Error object indicating success or /// failure of the request Error UnregisterCudaSharedMemory( const std::string& name = "", const Headers& headers = Headers(), const uint64_t timeout_ms = 0); /// Run synchronous inference on server. /// \param result Returns the result of inference. /// \param options The options for inference request. /// \param inputs The vector of InferInput describing the model inputs. /// \param outputs Optional vector of InferRequestedOutput describing how the /// output must be returned. If not provided then all the outputs in the model /// config will be returned as default settings. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param compression_algorithm The compression algorithm to be used /// by gRPC when sending requests. By default compression is not used. /// \return Error object indicating success or failure of the /// request. Error Infer( InferResult** result, const InferOptions& options, const std::vector& inputs, const std::vector& outputs = std::vector(), const Headers& headers = Headers(), grpc_compression_algorithm compression_algorithm = GRPC_COMPRESS_NONE); /// Run asynchronous inference on server. /// Once the request is completed, the InferResult pointer will be passed to /// the provided 'callback' function. Upon the invocation of callback /// function, the ownership of InferResult object is transferred to the /// function caller. It is then the caller's choice on either retrieving the /// results inside the callback function or deferring it to a different thread /// so that the client is unblocked. In order to prevent memory leak, user /// must ensure this object gets deleted. /// \param callback The callback function to be invoked on request completion. /// \param options The options for inference request. /// \param inputs The vector of InferInput describing the model inputs. /// \param outputs Optional vector of InferRequestedOutput describing how the /// output must be returned. If not provided then all the outputs in the model /// config will be returned as default settings. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param compression_algorithm The compression algorithm to be used /// by gRPC when sending requests. By default compression is not used. /// \return Error object indicating success or failure of the request. Error AsyncInfer( OnCompleteFn callback, const InferOptions& options, const std::vector& inputs, const std::vector& outputs = std::vector(), const Headers& headers = Headers(), grpc_compression_algorithm compression_algorithm = GRPC_COMPRESS_NONE); /// Run multiple synchronous inferences on server. /// \param results Returns the results of the inferences. /// \param options The options for each inference request, one set of /// options may be provided and it will be used for all inference requests. /// \param inputs The vector of InferInput objects describing the model inputs /// for each inference request. /// \param outputs Optional vector of InferRequestedOutput objects describing /// how the output must be returned. If not provided then all the outputs in /// the model config will be returned as default settings. And one set of /// outputs may be provided and it will be used for all inference requests. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param compression_algorithm The compression algorithm to be used /// by gRPC when sending requests. By default compression is not used. /// \return Error object indicating success or failure of the /// request. Error InferMulti( std::vector* results, const std::vector& options, const std::vector>& inputs, const std::vector>& outputs = std::vector>(), const Headers& headers = Headers(), grpc_compression_algorithm compression_algorithm = GRPC_COMPRESS_NONE); /// Run multiple asynchronous inferences on server. /// Once all the requests are completed, the vector of InferResult pointers /// will be passed to the provided 'callback' function. Upon the invocation /// of callback function, the ownership of the InferResult objects are /// transferred to the function caller. It is then the caller's choice on /// either retrieving the results inside the callback function or deferring it /// to a different thread so that the client is unblocked. In order to /// prevent memory leak, user must ensure these objects get deleted. /// \param callback The callback function to be invoked on the completion of /// all requests. /// \param options The options for each inference request, one set of /// option may be provided and it will be used for all inference requests. /// \param inputs The vector of InferInput objects describing the model inputs /// for each inference request. /// \param outputs Optional vector of InferRequestedOutput objects describing /// how the output must be returned. If not provided then all the outputs in /// the model config will be returned as default settings. And one set of /// outputs may be provided and it will be used for all inference requests. /// \param headers Optional map specifying additional HTTP headers to include /// in the metadata of gRPC request. /// \param compression_algorithm The compression algorithm to be used /// by gRPC when sending requests. By default compression is not used. /// \return Error object indicating success or failure of the request. Error AsyncInferMulti( OnMultiCompleteFn callback, const std::vector& options, const std::vector>& inputs, const std::vector>& outputs = std::vector>(), const Headers& headers = Headers(), grpc_compression_algorithm compression_algorithm = GRPC_COMPRESS_NONE); /// Starts a grpc bi-directional stream to send streaming inferences. /// \param callback The callback function to be invoked on receiving a /// response at the stream. /// \param enable_stats Indicates whether client library should record the /// the client-side statistics for inference requests on stream or not. /// The library does not support client side statistics for decoupled /// streaming. Set this option false when there is no 1:1 mapping between /// request and response on the stream. /// \param stream_timeout Specifies the end-to-end timeout for the streaming /// connection in microseconds. The default value is 0 which means that /// there is no limitation on deadline. The stream will be closed once /// the specified time elapses. /// \param headers Optional map specifying additional HTTP headers to /// include in the metadata of gRPC request. /// \param compression_algorithm The compression algorithm to be used /// by gRPC when sending requests. By default compression is not used. /// \return Error object indicating success or failure of the request. Error StartStream( OnCompleteFn callback, bool enable_stats = true, uint32_t stream_timeout = 0, const Headers& headers = Headers(), grpc_compression_algorithm compression_algorithm = GRPC_COMPRESS_NONE); /// Stops an active grpc bi-directional stream, if one available. /// \return Error object indicating success or failure of the request. Error StopStream(); /// Runs an asynchronous inference over gRPC bi-directional streaming /// API. A stream must be established with a call to StartStream() /// before calling this function. All the results will be provided to the /// callback function provided when starting the stream. /// \param options The options for inference request. /// \param inputs The vector of InferInput describing the model inputs. /// \param outputs Optional vector of InferRequestedOutput describing how the /// output must be returned. If not provided then all the outputs in the model /// config will be returned as default settings. /// \return Error object indicating success or failure of the request. Error AsyncStreamInfer( const InferOptions& options, const std::vector& inputs, const std::vector& outputs = std::vector()); // Number of Cached Channels size_t GetNumCachedChannels() const; private: InferenceServerGrpcClient( const std::string& url, bool verbose, bool use_ssl, const SslOptions& ssl_options, const grpc::ChannelArguments& channel_args, const bool use_cached_channel); Error PreRunProcessing( const InferOptions& options, const std::vector& inputs, const std::vector& outputs); void AsyncTransfer(); void AsyncStreamTransfer(); // The producer-consumer queue used to communicate asynchronously with // the GRPC runtime. grpc::CompletionQueue async_request_completion_queue_; // Required to support the grpc bi-directional streaming API. InferenceServerClient::OnCompleteFn stream_callback_; std::thread stream_worker_; std::shared_ptr> grpc_stream_; grpc::ClientContext grpc_context_; bool enable_stream_stats_; std::queue> ongoing_stream_request_timers_; std::mutex stream_mutex_; // GRPC end point. std::shared_ptr stub_; // request for GRPC call, one request object can be used for multiple calls // since it can be overwritten as soon as the GRPC send finishes. inference::ModelInferRequest infer_request_; }; }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/http_client.cc000066400000000000000000002353501527052371200225520ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Include this first to make sure we are a friend of common classes. #define TRITON_INFERENCE_SERVER_CLIENT_CLASS InferenceServerHttpClient #include "http_client.h" #include #include #include #include #include #include #include #include #include "common.h" #ifdef TRITON_ENABLE_ZLIB #include #endif extern "C" { #include "cencode.h" } #define TRITONJSON_STATUSTYPE triton::client::Error #define TRITONJSON_STATUSRETURN(M) return triton::client::Error(M) #define TRITONJSON_STATUSSUCCESS triton::client::Error::Success #include "triton/common/triton_json.h" #ifdef _WIN32 #define strncasecmp(x, y, z) _strnicmp(x, y, z) #undef min // NOMINMAX did not resolve std::min compile error #endif //_WIN32 namespace triton { namespace client { namespace { constexpr char kContentLengthHTTPHeader[] = "Content-Length"; //============================================================================== // Global initialization for libcurl. Libcurl requires global // initialization before any other threads are created and before any // curl methods are used. The curl_global static object is used to // perform this initialization. class CurlGlobal { public: ~CurlGlobal(); const Error& Status() const { return err_; } static const CurlGlobal& Get() { static CurlGlobal* curl_global = new CurlGlobal(); return *curl_global; } private: CurlGlobal(); Error err_; }; CurlGlobal::CurlGlobal() : err_(Error::Success) { if (curl_global_init(CURL_GLOBAL_ALL) != 0) { err_ = Error("global initialization failed"); } } CurlGlobal::~CurlGlobal() { curl_global_cleanup(); } class CurlGlobalDestroyer { public: ~CurlGlobalDestroyer() { delete &CurlGlobal::Get(); } }; static CurlGlobalDestroyer curl_global_destroyer_; std::string GetQueryString(const Headers& query_params) { std::string query_string; bool first = true; for (const auto& pr : query_params) { if (first) { first = false; } else { query_string += "&"; } query_string += pr.first + "=" + pr.second; } return query_string; } // Encodes the contents of the provided buffer into base64 string. Note the // string is not guaranteed to be null-terminated. Must rely on the returned // encoded size to get the right contents. void Base64Encode( const char* raw_ptr, const size_t raw_size, char** encoded_ptr, int* encoded_size) { // Encode the handle object to base64 base64_encodestate es; base64_init_encodestate(&es); *encoded_ptr = (char*)malloc(raw_size * 2); /* ~4/3 x raw_size */ *encoded_size = base64_encode_block(raw_ptr, raw_size, *encoded_ptr, &es); int padding_size = base64_encode_blockend(*encoded_ptr + *encoded_size, &es); *encoded_size += padding_size; } #ifdef TRITON_ENABLE_ZLIB // libcurl provides automatic decompression, so only implement compression Error CompressData( const InferenceServerHttpClient::CompressionType type, const std::deque>& source, const size_t source_byte_size, std::vector, size_t>>* compressed_data) { // nothing to be compressed if (source_byte_size == 0) { return Error("nothing to be compressed"); } z_stream stream; stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; switch (type) { case InferenceServerHttpClient::CompressionType::GZIP: if (deflateInit2( &stream, Z_DEFAULT_COMPRESSION /* level */, Z_DEFLATED /* method */, 15 | 16 /* windowBits */, 8 /* memLevel */, Z_DEFAULT_STRATEGY /* strategy */) != Z_OK) { return Error("failed to initialize state for gzip data compression"); } break; case InferenceServerHttpClient::CompressionType::DEFLATE: { if (deflateInit(&stream, Z_DEFAULT_COMPRESSION /* level */) != Z_OK) { return Error("failed to initialize state for deflate data compression"); } break; } case InferenceServerHttpClient::CompressionType::NONE: return Error("can't compress data with NONE type"); break; } // ensure the internal state are cleaned up on function return std::unique_ptr managed_stream( &stream, deflateEnd); // Reserve the same size as source for compressed data, it is less likely // that a negative compression happens. std::unique_ptr current_reserved_space(new char[source_byte_size]); stream.next_out = reinterpret_cast(current_reserved_space.get()); stream.avail_out = source_byte_size; // Compress until end of 'source' for (auto it = source.begin(); it != source.end(); ++it) { stream.next_in = reinterpret_cast(it->first); stream.avail_in = it->second; // run deflate() on input until source has been read in do { // Need additional buffer if (stream.avail_out == 0) { compressed_data->emplace_back( std::move(current_reserved_space), source_byte_size); current_reserved_space.reset(new char[source_byte_size]); stream.next_out = reinterpret_cast(current_reserved_space.get()); stream.avail_out = source_byte_size; } auto flush = (std::next(it) == source.end()) ? Z_FINISH : Z_NO_FLUSH; auto ret = deflate(&stream, flush); if (ret == Z_STREAM_ERROR) { return Error( "encountered inconsistent stream state during compression"); } } while (stream.avail_out == 0); } // Make sure the last buffer is committed if (current_reserved_space != nullptr) { compressed_data->emplace_back( std::move(current_reserved_space), source_byte_size - stream.avail_out); } return Error::Success; } #else Error CompressData( const InferenceServerHttpClient::CompressionType type, const std::deque>& source, const size_t source_byte_size, std::vector, size_t>>* compressed_data) { return Error("Cannot compress data as ZLIB is not included in this build"); } #endif Error ParseSslCertType( HttpSslOptions::CERTTYPE cert_type, std::string* curl_cert_type) { switch (cert_type) { case HttpSslOptions::CERTTYPE::CERT_PEM: *curl_cert_type = "PEM"; break; case HttpSslOptions::CERTTYPE::CERT_DER: *curl_cert_type = "DER"; break; default: return Error( "unsupported ssl certificate type encountered. Only PEM and DER are " "supported."); } return Error::Success; } Error ParseSslKeyType(HttpSslOptions::KEYTYPE key_type, std::string* curl_key_type) { switch (key_type) { case HttpSslOptions::KEYTYPE::KEY_PEM: *curl_key_type = "PEM"; break; case HttpSslOptions::KEYTYPE::KEY_DER: *curl_key_type = "DER"; break; default: return Error( "unsupported ssl key type encountered. Only PEM and DER are " "supported."); } return Error::Success; } Error SetSSLCurlOptions(CURL** curl, const HttpSslOptions& ssl_options) { curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, ssl_options.verify_peer); curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST, ssl_options.verify_host); if (!ssl_options.ca_info.empty()) { curl_easy_setopt(*curl, CURLOPT_CAINFO, ssl_options.ca_info.c_str()); } std::string curl_cert_type; Error err = ParseSslCertType(ssl_options.cert_type, &curl_cert_type); if (!err.IsOk()) { return err; } curl_easy_setopt(*curl, CURLOPT_SSLCERTTYPE, curl_cert_type.c_str()); if (!ssl_options.cert.empty()) { curl_easy_setopt(*curl, CURLOPT_SSLCERT, ssl_options.cert.c_str()); } std::string curl_key_type; err = ParseSslKeyType(ssl_options.key_type, &curl_key_type); if (!err.IsOk()) { return err; } curl_easy_setopt(*curl, CURLOPT_SSLKEYTYPE, curl_key_type.c_str()); if (!ssl_options.key.empty()) { curl_easy_setopt(*curl, CURLOPT_SSLKEY, ssl_options.key.c_str()); } return Error::Success; } } // namespace //============================================================================== class HttpInferRequest : public InferRequest { public: HttpInferRequest( InferenceServerClient::OnCompleteFn callback = nullptr, const bool verbose = false); ~HttpInferRequest(); // Initialize the request for HTTP transfer. */ Error InitializeRequest( const InferOptions& options, const std::vector& inputs, const std::vector& outputs); // Adds the input data to be delivered to the server Error AddInput(uint8_t* buf, size_t byte_size); // Copy into 'buf' up to 'size' bytes of input data. Return the // actual amount copied in 'input_bytes'. Error GetNextInput(uint8_t* buf, size_t size, size_t* input_bytes); Error CompressInput(const InferenceServerHttpClient::CompressionType type); private: friend class InferenceServerHttpClient; friend class InferResultHttp; Error PrepareRequestJson( const InferOptions& options, const std::vector& inputs, const std::vector& outputs, triton::common::TritonJson::Value* request_json); protected: virtual Error ConvertBinaryInputsToJSON( InferInput& input, triton::common::TritonJson::Value& data_json) const; virtual Error ConvertBinaryInputToJSON( const uint8_t* buf, const size_t buf_size, const std::string& datatype, triton::common::TritonJson::Value& data_json) const; private: // Pointer to the list of the HTTP request header, keep it such that it will // be valid during the transfer and can be freed once transfer is completed. struct curl_slist* header_list_; // HTTP response code for the inference request long http_code_; size_t total_input_byte_size_; triton::common::TritonJson::WriteBuffer request_json_; // Buffer that accumulates the response body. std::unique_ptr infer_response_buffer_; // The pointers to the input data. std::deque> data_buffers_; // Placeholder for the compressed data std::vector, size_t>> compressed_data_; size_t response_json_size_; }; HttpInferRequest::HttpInferRequest( InferenceServerClient::OnCompleteFn callback, const bool verbose) : InferRequest(callback, verbose), header_list_(nullptr), total_input_byte_size_(0), response_json_size_(0) { } HttpInferRequest::~HttpInferRequest() { if (header_list_ != nullptr) { curl_slist_free_all(header_list_); header_list_ = nullptr; } } Error HttpInferRequest::InitializeRequest( const InferOptions& options, const std::vector& inputs, const std::vector& outputs) { data_buffers_ = {}; total_input_byte_size_ = 0; http_code_ = 400; triton::common::TritonJson::Value request_json( triton::common::TritonJson::ValueType::OBJECT); Error err = PrepareRequestJson(options, inputs, outputs, &request_json); if (!err.IsOk()) { return err; } request_json_.Clear(); request_json.Write(&request_json_); // Add the buffer holding the json to be delivered first AddInput((uint8_t*)request_json_.Base(), request_json_.Size()); // Prepare buffer to record the response infer_response_buffer_.reset(new std::string()); return Error::Success; } Error HttpInferRequest::PrepareRequestJson( const InferOptions& options, const std::vector& inputs, const std::vector& outputs, triton::common::TritonJson::Value* request_json) { // Can use string-ref because json is serialized before end of // 'options', 'inputs' and 'outputs' lifetime. request_json->AddStringRef( "id", options.request_id_.c_str(), options.request_id_.size()); if ((options.sequence_id_ != 0) || (options.sequence_id_str_ != "") || (options.priority_ != 0) || (options.server_timeout_ != 0) || outputs.empty()) { triton::common::TritonJson::Value parameters_json( *request_json, triton::common::TritonJson::ValueType::OBJECT); { if ((options.sequence_id_ != 0) || (options.sequence_id_str_ != "")) { if (options.sequence_id_ != 0) { parameters_json.AddUInt("sequence_id", options.sequence_id_); } else { parameters_json.AddString( "sequence_id", options.sequence_id_str_.c_str(), options.sequence_id_str_.size()); } parameters_json.AddBool("sequence_start", options.sequence_start_); parameters_json.AddBool("sequence_end", options.sequence_end_); } if (options.priority_ != 0) { parameters_json.AddUInt("priority", options.priority_); } if (options.server_timeout_ != 0) { parameters_json.AddUInt("timeout", options.server_timeout_); } // If no outputs are provided then set the request parameter // to return all outputs as binary data. if (outputs.empty()) { parameters_json.AddBool("binary_data_output", true); } for (auto& param : options.request_parameters) { if (param.second.type == "string") { parameters_json.AddString( param.first.c_str(), param.second.value.c_str(), param.second.value.size()); } else if (param.second.type == "int") { parameters_json.AddInt( param.first.c_str(), std::stoi(param.second.value)); } else if (param.second.type == "bool") { bool val = false; if (param.second.value == "true") { val = true; } parameters_json.AddBool(param.first.c_str(), val); } } } request_json->Add("parameters", std::move(parameters_json)); } if (!inputs.empty()) { triton::common::TritonJson::Value inputs_json( *request_json, triton::common::TritonJson::ValueType::ARRAY); for (const auto io : inputs) { triton::common::TritonJson::Value io_json( *request_json, triton::common::TritonJson::ValueType::OBJECT); io_json.AddStringRef("name", io->Name().c_str(), io->Name().size()); io_json.AddStringRef( "datatype", io->Datatype().c_str(), io->Datatype().size()); triton::common::TritonJson::Value shape_json( *request_json, triton::common::TritonJson::ValueType::ARRAY); for (const auto dim : io->Shape()) { shape_json.AppendUInt(dim); } io_json.Add("shape", std::move(shape_json)); triton::common::TritonJson::Value ioparams_json( *request_json, triton::common::TritonJson::ValueType::OBJECT); if (io->IsSharedMemory()) { std::string region_name; size_t offset; size_t byte_size; Error err = io->SharedMemoryInfo(®ion_name, &byte_size, &offset); if (!err.IsOk()) { return err; } ioparams_json.AddString( "shared_memory_region", region_name.c_str(), region_name.size()); ioparams_json.AddUInt("shared_memory_byte_size", byte_size); if (offset != 0) { ioparams_json.AddUInt("shared_memory_offset", offset); } io_json.Add("parameters", std::move(ioparams_json)); } else if (io->BinaryData()) { size_t byte_size; Error err = io->ByteSize(&byte_size); if (!err.IsOk()) { return err; } ioparams_json.AddUInt("binary_data_size", byte_size); io_json.Add("parameters", std::move(ioparams_json)); } else { triton::common::TritonJson::Value data_json( *request_json, triton::common::TritonJson::ValueType::ARRAY); Error err = ConvertBinaryInputsToJSON(*io, data_json); if (!err.IsOk()) { return err; } io_json.Add("data", std::move(data_json)); } inputs_json.Append(std::move(io_json)); } request_json->Add("inputs", std::move(inputs_json)); } if (!outputs.empty()) { triton::common::TritonJson::Value outputs_json( *request_json, triton::common::TritonJson::ValueType::ARRAY); for (const auto io : outputs) { triton::common::TritonJson::Value io_json( *request_json, triton::common::TritonJson::ValueType::OBJECT); io_json.AddStringRef("name", io->Name().c_str(), io->Name().size()); triton::common::TritonJson::Value ioparams_json( *request_json, triton::common::TritonJson::ValueType::OBJECT); if (io->ClassificationCount() > 0) { ioparams_json.AddUInt("classification", io->ClassificationCount()); } if (io->IsSharedMemory()) { std::string region_name; size_t offset; size_t byte_size; Error err = io->SharedMemoryInfo(®ion_name, &byte_size, &offset); if (!err.IsOk()) { return err; } ioparams_json.AddString( "shared_memory_region", region_name.c_str(), region_name.size()); ioparams_json.AddUInt("shared_memory_byte_size", byte_size); if (offset != 0) { ioparams_json.AddUInt("shared_memory_offset", offset); } } else { ioparams_json.AddBool("binary_data", io->BinaryData()); } io_json.Add("parameters", std::move(ioparams_json)); outputs_json.Append(std::move(io_json)); } request_json->Add("outputs", std::move(outputs_json)); } return Error::Success; } Error HttpInferRequest::ConvertBinaryInputsToJSON( InferInput& input, triton::common::TritonJson::Value& data_json) const { input.PrepareForRequest(); bool end_of_input{false}; while (!end_of_input) { const uint8_t* buf{nullptr}; size_t buf_size{0}; input.GetNext(&buf, &buf_size, &end_of_input); size_t element_count{1}; for (size_t i = 1; i < input.Shape().size(); i++) { element_count *= input.Shape()[i]; } if (buf != nullptr) { Error err = ConvertBinaryInputToJSON( buf, element_count, input.Datatype(), data_json); if (!err.IsOk()) { return err; } } } return Error::Success; } Error HttpInferRequest::ConvertBinaryInputToJSON( const uint8_t* buf, const size_t element_count, const std::string& datatype, triton::common::TritonJson::Value& data_json) const { if (datatype == "BOOL") { for (size_t i = 0; i < element_count; i++) { data_json.AppendBool(reinterpret_cast(buf)[i]); } } else if (datatype == "UINT8") { for (size_t i = 0; i < element_count; i++) { data_json.AppendUInt(reinterpret_cast(buf)[i]); } } else if (datatype == "UINT16") { for (size_t i = 0; i < element_count; i++) { data_json.AppendUInt(reinterpret_cast(buf)[i]); } } else if (datatype == "UINT32") { for (size_t i = 0; i < element_count; i++) { data_json.AppendUInt(reinterpret_cast(buf)[i]); } } else if (datatype == "UINT64") { for (size_t i = 0; i < element_count; i++) { data_json.AppendUInt(reinterpret_cast(buf)[i]); } } else if (datatype == "INT8") { for (size_t i = 0; i < element_count; i++) { data_json.AppendInt(reinterpret_cast(buf)[i]); } } else if (datatype == "INT16") { for (size_t i = 0; i < element_count; i++) { data_json.AppendInt(reinterpret_cast(buf)[i]); } } else if (datatype == "INT32") { for (size_t i = 0; i < element_count; i++) { data_json.AppendInt(reinterpret_cast(buf)[i]); } } else if (datatype == "INT64") { for (size_t i = 0; i < element_count; i++) { data_json.AppendInt(reinterpret_cast(buf)[i]); } } else if (datatype == "FP16") { return Error( "datatype '" + datatype + "' is not supported with JSON. Please use the binary data format"); } else if (datatype == "FP32") { for (size_t i = 0; i < element_count; i++) { data_json.AppendDouble(reinterpret_cast(buf)[i]); } } else if (datatype == "FP64") { for (size_t i = 0; i < element_count; i++) { data_json.AppendDouble(reinterpret_cast(buf)[i]); } } else if (datatype == "BYTES") { size_t offset{0}; for (size_t i = 0; i < element_count; i++) { const size_t len{*reinterpret_cast(buf + offset)}; data_json.AppendStringRef( reinterpret_cast(buf + offset + sizeof(const uint32_t)), len); offset += sizeof(const uint32_t) + len; } } else if (datatype == "BF16") { return Error( "datatype '" + datatype + "' is not supported with JSON. Please use the binary data format"); } else { return Error("datatype '" + datatype + "' is invalid"); } return Error::Success; } Error HttpInferRequest::AddInput(uint8_t* buf, size_t byte_size) { data_buffers_.push_back(std::pair(buf, byte_size)); total_input_byte_size_ += byte_size; return Error::Success; } Error HttpInferRequest::GetNextInput(uint8_t* buf, size_t size, size_t* input_bytes) { *input_bytes = 0; while (!data_buffers_.empty() && size > 0) { const size_t csz = std::min(data_buffers_.front().second, size); if (csz > 0) { const uint8_t* input_ptr = data_buffers_.front().first; std::copy(input_ptr, input_ptr + csz, buf); size -= csz; buf += csz; *input_bytes += csz; data_buffers_.front().first += csz; data_buffers_.front().second -= csz; } if (data_buffers_.front().second == 0) { data_buffers_.pop_front(); } } // Set end timestamp if all inputs have been sent. if (data_buffers_.empty()) { Timer().CaptureTimestamp(RequestTimers::Kind::SEND_END); } return Error::Success; } Error HttpInferRequest::CompressInput( const InferenceServerHttpClient::CompressionType type) { auto err = CompressData( type, data_buffers_, total_input_byte_size_, &compressed_data_); if (!err.IsOk()) { return err; } data_buffers_.clear(); total_input_byte_size_ = 0; for (const auto& data : compressed_data_) { data_buffers_.push_back(std::pair( reinterpret_cast(data.first.get()), data.second)); total_input_byte_size_ += data.second; } return Error::Success; } //============================================================================== class InferResultHttp : public InferResult { public: static void Create( InferResult** infer_result, std::shared_ptr infer_request); static Error Create(InferResult** infer_result, const Error err); Error RequestStatus() const override; Error ModelName(std::string* name) const override; Error ModelVersion(std::string* version) const override; Error Id(std::string* id) const override; Error Shape(const std::string& output_name, std::vector* shape) const override; Error Datatype( const std::string& output_name, std::string* datatype) const override; Error RawData( const std::string& output_name, const uint8_t** buf, size_t* byte_size) const override; Error IsFinalResponse(bool* is_final_response) const override; Error IsNullResponse(bool* is_null_response) const override; Error StringData( const std::string& output_name, std::vector* string_result) const override; std::string DebugString() const override; private: InferResultHttp(std::shared_ptr infer_request); InferResultHttp(const Error err) : status_(err) {} protected: InferResultHttp() {} ~InferResultHttp(); virtual Error ConvertJSONOutputToBinary( triton::common::TritonJson::Value& data_json, const std::string& datatype, const uint8_t** buf, size_t* buf_size) const; private: std::map output_name_to_result_map_; std::map> output_name_to_buffer_map_; Error status_; triton::common::TritonJson::Value response_json_; std::shared_ptr infer_request_; bool binary_data_{true}; bool is_final_response_{true}; bool is_null_response_{false}; }; void InferResultHttp::Create( InferResult** infer_result, std::shared_ptr infer_request) { *infer_result = reinterpret_cast(new InferResultHttp(infer_request)); } Error InferResultHttp::Create(InferResult** infer_result, const Error err) { if (err.IsOk()) { return Error( "Error is not provided for error reporting override of " "InferResultHttp::Create()"); } *infer_result = reinterpret_cast(new InferResultHttp(err)); return Error::Success; } Error InferResultHttp::ModelName(std::string* name) const { if (!status_.IsOk()) { return status_; } const char* name_str; size_t name_strlen; Error err = response_json_.MemberAsString("model_name", &name_str, &name_strlen); if (!err.IsOk()) { return Error("model name was not returned in the response"); } name->assign(name_str, name_strlen); return Error::Success; } Error InferResultHttp::ModelVersion(std::string* version) const { if (!status_.IsOk()) { return status_; } const char* version_str; size_t version_strlen; Error err = response_json_.MemberAsString( "model_version", &version_str, &version_strlen); if (!err.IsOk()) { return Error("model version was not returned in the response"); } version->assign(version_str, version_strlen); return Error::Success; } Error InferResultHttp::Id(std::string* id) const { if (!status_.IsOk()) { return status_; } const char* id_str; size_t id_strlen; Error err = response_json_.MemberAsString("id", &id_str, &id_strlen); if (!err.IsOk()) { return Error("model id was not returned in the response"); } id->assign(id_str, id_strlen); return Error::Success; } namespace { Error ShapeHelper( const std::string& result_name, const triton::common::TritonJson::Value& result_json, std::vector* shape) { triton::common::TritonJson::Value shape_json; if (!const_cast(result_json) .Find("shape", &shape_json)) { return Error( "The response does not contain shape for output name " + result_name); } for (size_t i = 0; i < shape_json.ArraySize(); i++) { int64_t dim; Error err = shape_json.IndexAsInt(i, &dim); if (!err.IsOk()) { return err; } shape->push_back(dim); } return Error::Success; } } // namespace Error InferResultHttp::Shape( const std::string& output_name, std::vector* shape) const { if (!status_.IsOk()) { return status_; } shape->clear(); auto itr = output_name_to_result_map_.find(output_name); if (itr == output_name_to_result_map_.end()) { return Error( "The response does not contain results for output name " + output_name); } return ShapeHelper(output_name, itr->second, shape); } Error InferResultHttp::Datatype( const std::string& output_name, std::string* datatype) const { if (!status_.IsOk()) { return status_; } auto itr = output_name_to_result_map_.find(output_name); if (itr == output_name_to_result_map_.end()) { return Error( "The response does not contain results for output name " + output_name); } const char* dtype_str; size_t dtype_strlen; Error err = itr->second.MemberAsString("datatype", &dtype_str, &dtype_strlen); if (!err.IsOk()) { return Error( "The response does not contain datatype for output name " + output_name); } datatype->assign(dtype_str, dtype_strlen); return Error::Success; } Error InferResultHttp::RawData( const std::string& output_name, const uint8_t** buf, size_t* byte_size) const { if (!status_.IsOk()) { return status_; } auto itr = output_name_to_buffer_map_.find(output_name); if (itr != output_name_to_buffer_map_.end()) { *buf = itr->second.first; *byte_size = itr->second.second; } else { return Error( "The response does not contain results for output name " + output_name); } return Error::Success; } Error InferResultHttp::IsFinalResponse(bool* is_final_response) const { if (is_final_response == nullptr) { return Error("is_final_response cannot be nullptr"); } *is_final_response = is_final_response_; return Error::Success; } Error InferResultHttp::IsNullResponse(bool* is_null_response) const { if (is_null_response == nullptr) { return Error("is_null_response cannot be nullptr"); } *is_null_response = is_null_response_; return Error::Success; } Error InferResultHttp::StringData( const std::string& output_name, std::vector* string_result) const { if (!status_.IsOk()) { return status_; } std::string datatype; Error err = Datatype(output_name, &datatype); if (!err.IsOk()) { return err; } if (datatype.compare("BYTES") != 0) { return Error( "This function supports tensors with datatype 'BYTES', requested " "output tensor '" + output_name + "' with datatype '" + datatype + "'"); } const uint8_t* buf; size_t byte_size; err = RawData(output_name, &buf, &byte_size); string_result->clear(); size_t buf_offset = 0; while (byte_size > buf_offset) { const uint32_t element_size = *(reinterpret_cast(buf + buf_offset)); string_result->emplace_back( reinterpret_cast(buf + buf_offset + sizeof(element_size)), element_size); buf_offset += (sizeof(element_size) + element_size); } return Error::Success; } std::string InferResultHttp::DebugString() const { if (!status_.IsOk()) { return status_.Message(); } triton::common::TritonJson::WriteBuffer buffer; Error err = response_json_.Write(&buffer); if (!err.IsOk()) { return ""; } return buffer.Contents(); } Error InferResultHttp::RequestStatus() const { return status_; } InferResultHttp::InferResultHttp( std::shared_ptr infer_request) : infer_request_(infer_request) { size_t offset = infer_request->response_json_size_; if (infer_request->http_code_ == 499) { status_ = Error("Deadline Exceeded"); } else { if (offset != 0) { if (infer_request->verbose_) { std::cout << "inference response: " << infer_request->infer_response_buffer_->substr(0, offset) << std::endl; } status_ = response_json_.Parse( (char*)infer_request->infer_response_buffer_.get()->c_str(), offset); } else { if (infer_request->verbose_) { std::cout << "inference response: " << *infer_request->infer_response_buffer_ << std::endl; } status_ = response_json_.Parse( (char*)infer_request->infer_response_buffer_.get()->c_str()); } } // There should be a valid JSON response in all cases. Either the // successful infer response or an error response. if (status_.IsOk()) { if (infer_request->http_code_ != 200) { const char* err_str; size_t err_strlen; if (!response_json_.MemberAsString("error", &err_str, &err_strlen) .IsOk()) { status_ = Error("inference failed with unknown error"); } else { status_ = Error(std::string(err_str, err_strlen)); } } else { triton::common::TritonJson::Value outputs_json; if (response_json_.Find("outputs", &outputs_json)) { for (size_t i = 0; i < outputs_json.ArraySize(); i++) { triton::common::TritonJson::Value output_json; status_ = outputs_json.IndexAsObject(i, &output_json); if (!status_.IsOk()) { break; } const char* name_str; size_t name_strlen; status_ = output_json.MemberAsString("name", &name_str, &name_strlen); if (!status_.IsOk()) { break; } std::string output_name(name_str, name_strlen); triton::common::TritonJson::Value param_json, data_json; if (output_json.Find("parameters", ¶m_json)) { uint64_t data_size = 0; status_ = param_json.MemberAsUInt("binary_data_size", &data_size); if (!status_.IsOk()) { break; } output_name_to_buffer_map_.emplace( output_name, std::pair( (uint8_t*)(infer_request->infer_response_buffer_.get() ->c_str()) + offset, data_size)); offset += data_size; } else if (output_json.Find("data", &data_json)) { binary_data_ = false; std::string datatype; status_ = output_json.MemberAsString("datatype", &datatype); if (!status_.IsOk()) { break; } const uint8_t* buf{nullptr}; size_t buf_size{0}; status_ = ConvertJSONOutputToBinary(data_json, datatype, &buf, &buf_size); if (!status_.IsOk()) { break; } output_name_to_buffer_map_.emplace( output_name, std::pair(buf, buf_size)); } output_name_to_result_map_[output_name] = std::move(output_json); } } } } } InferResultHttp::~InferResultHttp() { if (binary_data_) { return; } for (auto& buf_pair : output_name_to_buffer_map_) { const uint8_t* buf{buf_pair.second.first}; delete buf; } } Error InferResultHttp::ConvertJSONOutputToBinary( triton::common::TritonJson::Value& data_json, const std::string& datatype, const uint8_t** buf, size_t* buf_size) const { const size_t element_count{data_json.ArraySize()}; if (datatype == "BOOL") { *buf = reinterpret_cast(new bool[element_count]); *buf_size = sizeof(bool) * element_count; for (size_t i = 0; i < element_count; i++) { bool value{false}; data_json.IndexAsBool(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "UINT8") { *buf = reinterpret_cast(new uint8_t[element_count]); *buf_size = sizeof(uint8_t) * element_count; for (size_t i = 0; i < element_count; i++) { uint64_t value{0}; data_json.IndexAsUInt(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "UINT16") { *buf = reinterpret_cast(new uint16_t[element_count]); *buf_size = sizeof(uint16_t) * element_count; for (size_t i = 0; i < element_count; i++) { uint64_t value{0}; data_json.IndexAsUInt(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "UINT32") { *buf = reinterpret_cast(new uint32_t[element_count]); *buf_size = sizeof(uint32_t) * element_count; for (size_t i = 0; i < element_count; i++) { uint64_t value{0}; data_json.IndexAsUInt(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "UINT64") { *buf = reinterpret_cast(new uint64_t[element_count]); *buf_size = sizeof(uint64_t) * element_count; for (size_t i = 0; i < element_count; i++) { uint64_t value{0}; data_json.IndexAsUInt(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "INT8") { *buf = reinterpret_cast(new int8_t[element_count]); *buf_size = sizeof(int8_t) * element_count; for (size_t i = 0; i < element_count; i++) { int64_t value{0}; data_json.IndexAsInt(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "INT16") { *buf = reinterpret_cast(new int16_t[element_count]); *buf_size = sizeof(int16_t) * element_count; for (size_t i = 0; i < element_count; i++) { int64_t value{0}; data_json.IndexAsInt(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "INT32") { *buf = reinterpret_cast(new int32_t[element_count]); *buf_size = sizeof(int32_t) * element_count; for (size_t i = 0; i < element_count; i++) { int64_t value{0}; data_json.IndexAsInt(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "INT64") { *buf = reinterpret_cast(new int64_t[element_count]); *buf_size = sizeof(int64_t) * element_count; for (size_t i = 0; i < element_count; i++) { int64_t value{0}; data_json.IndexAsInt(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "FP16") { return Error("datatype '" + datatype + "' is not supported with JSON."); } else if (datatype == "FP32") { *buf = reinterpret_cast(new float[element_count]); *buf_size = sizeof(float) * element_count; for (size_t i = 0; i < element_count; i++) { double value{0.0}; data_json.IndexAsDouble(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "FP64") { *buf = reinterpret_cast(new float[element_count]); *buf_size = sizeof(double) * element_count; for (size_t i = 0; i < element_count; i++) { double value{0.0}; data_json.IndexAsDouble(i, &value); const_cast(reinterpret_cast(*buf))[i] = value; } } else if (datatype == "BYTES") { size_t total_buf_size{0}; std::vector> bytes_pairs{}; bytes_pairs.resize(element_count); for (size_t i = 0; i < element_count; i++) { data_json.IndexAsString(i, &bytes_pairs[i].first, &bytes_pairs[i].second); total_buf_size += sizeof(const uint32_t) + bytes_pairs[i].second; } *buf = reinterpret_cast(new uint8_t[total_buf_size]); *buf_size = total_buf_size; size_t offset{0}; for (const auto& bytes_pair : bytes_pairs) { const char* bytes{bytes_pair.first}; size_t bytes_size{bytes_pair.second}; std::memcpy( const_cast(*buf + offset), &bytes_size, sizeof(const uint32_t)); std::memcpy( const_cast(*buf + offset + sizeof(const uint32_t)), bytes, bytes_size); offset += sizeof(const uint32_t) + bytes_size; } } else if (datatype == "BF16") { return Error("datatype '" + datatype + "' is not supported with JSON."); } else { return Error("datatype '" + datatype + "' is invalid"); } return Error::Success; } //============================================================================== Error InferenceServerHttpClient::GenerateRequestBody( std::vector* request_body, size_t* header_length, const InferOptions& options, const std::vector& inputs, const std::vector& outputs) { auto infer_request = std::unique_ptr( new HttpInferRequest(nullptr /* callback */, false)); // Prepare the request object to provide the data for inference. Error err = infer_request->InitializeRequest(options, inputs, outputs); if (!err.IsOk()) { return err; } // Add the buffers holding input tensor data for (const auto this_input : inputs) { if (!this_input->IsSharedMemory()) { this_input->PrepareForRequest(); bool end_of_input = false; while (!end_of_input) { const uint8_t* buf; size_t buf_size; this_input->GetNext(&buf, &buf_size, &end_of_input); if (buf != nullptr) { infer_request->AddInput(const_cast(buf), buf_size); } } } } *header_length = infer_request->request_json_.Size(); *request_body = std::vector(infer_request->total_input_byte_size_); size_t remaining_bytes = infer_request->total_input_byte_size_; size_t actual_copied_bytes = 0; char* current_pos = request_body->data(); while (true) { err = infer_request->GetNextInput( reinterpret_cast(current_pos), remaining_bytes, &actual_copied_bytes); if (!err.IsOk()) { return err; } if (actual_copied_bytes == remaining_bytes) { break; } else { current_pos += actual_copied_bytes; remaining_bytes -= actual_copied_bytes; } } return Error::Success; } Error InferenceServerHttpClient::ParseResponseBody( InferResult** result, const std::vector& response_body, const size_t header_length) { // Result data is actually stored in request object auto infer_request = std::shared_ptr( new HttpInferRequest(nullptr /* callback */, false)); infer_request->http_code_ = 200; infer_request->response_json_size_ = header_length; infer_request->infer_response_buffer_.reset( new std::string(response_body.data(), response_body.size())); InferResultHttp::Create(result, infer_request); return Error::Success; } Error InferenceServerHttpClient::Create( std::unique_ptr* client, const std::string& server_url, bool verbose, const HttpSslOptions& ssl_options) { client->reset( new InferenceServerHttpClient(server_url, verbose, ssl_options)); return Error::Success; } InferenceServerHttpClient::InferenceServerHttpClient( const std::string& url, bool verbose, const HttpSslOptions& ssl_options) : InferenceServerClient(verbose), url_(url), ssl_options_(ssl_options), easy_handle_(reinterpret_cast(curl_easy_init())), multi_handle_(curl_multi_init()) { } InferenceServerHttpClient::~InferenceServerHttpClient() { { std::lock_guard lock(mutex_); exiting_ = true; } curl_multi_wakeup(multi_handle_); // thread not joinable if AsyncInfer() is not called // (it is default constructed thread before the first AsyncInfer() call) if (worker_.joinable()) { worker_.join(); } if (easy_handle_ != nullptr) { curl_easy_cleanup(reinterpret_cast(easy_handle_)); } curl_multi_cleanup(multi_handle_); } Error InferenceServerHttpClient::IsServerLive( bool* live, const Headers& headers, const Parameters& query_params) { Error err; std::string request_uri(url_ + "/v2/health/live"); long http_code; std::string response; err = Get(request_uri, headers, query_params, &response, &http_code); *live = (http_code == 200) ? true : false; return err; } Error InferenceServerHttpClient::IsServerReady( bool* ready, const Headers& headers, const Parameters& query_params) { Error err; std::string request_uri(url_ + "/v2/health/live"); long http_code; std::string response; err = Get(request_uri, headers, query_params, &response, &http_code); *ready = (http_code == 200) ? true : false; return err; } Error InferenceServerHttpClient::IsModelReady( bool* ready, const std::string& model_name, const std::string& model_version, const Headers& headers, const Parameters& query_params) { Error err; std::string request_uri(url_ + "/v2/models/" + model_name); if (!model_version.empty()) { request_uri = request_uri + "/versions/" + model_version; } request_uri = request_uri + "/ready"; long http_code; std::string response; err = Get(request_uri, headers, query_params, &response, &http_code); *ready = (http_code == 200) ? true : false; return err; } Error InferenceServerHttpClient::ServerMetadata( std::string* server_metadata, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2"); return Get(request_uri, headers, query_params, server_metadata); } Error InferenceServerHttpClient::ModelMetadata( std::string* model_metadata, const std::string& model_name, const std::string& model_version, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2/models/" + model_name); if (!model_version.empty()) { request_uri = request_uri + "/versions/" + model_version; } return Get(request_uri, headers, query_params, model_metadata); } Error InferenceServerHttpClient::ModelConfig( std::string* model_config, const std::string& model_name, const std::string& model_version, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2/models/" + model_name); if (!model_version.empty()) { request_uri = request_uri + "/versions/" + model_version; } request_uri = request_uri + "/config"; return Get(request_uri, headers, query_params, model_config); } Error InferenceServerHttpClient::ModelRepositoryIndex( std::string* repository_index, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2/repository/index"); std::string request; // empty request body return Post(request_uri, request, headers, query_params, repository_index); } Error InferenceServerHttpClient::LoadModel( const std::string& model_name, const Headers& headers, const Parameters& query_params, const std::string& config, const std::map>& files) { std::string request_uri( url_ + "/v2/repository/models/" + model_name + "/load"); triton::common::TritonJson::Value request_json( triton::common::TritonJson::ValueType::OBJECT); bool has_param = false; triton::common::TritonJson::Value parameters_json( request_json, triton::common::TritonJson::ValueType::OBJECT); if (!config.empty()) { has_param = true; parameters_json.AddStringRef("config", config.c_str()); } for (const auto& file : files) { // base64 encode the file content for HTTP protocol requirement // Must free encoded_handle after use to prevent memory leak char* encoded_handle = nullptr; int encoded_size; Base64Encode( file.second.data(), file.second.size(), &encoded_handle, &encoded_size); if (encoded_handle == nullptr) { return Error("Failed to base64 encode the file content"); } has_param = true; parameters_json.AddString(file.first.c_str(), encoded_handle, encoded_size); free(encoded_handle); } if (has_param) { request_json.Add("parameters", std::move(parameters_json)); } triton::common::TritonJson::WriteBuffer buffer; Error err = request_json.Write(&buffer); if (!err.IsOk()) { return err; } std::string response; return Post(request_uri, buffer.Contents(), headers, query_params, &response); } Error InferenceServerHttpClient::UnloadModel( const std::string& model_name, const Headers& headers, const Parameters& query_params) { std::string request_uri( url_ + "/v2/repository/models/" + model_name + "/unload"); std::string request; // empty request body std::string response; return Post(request_uri, request, headers, query_params, &response); } Error InferenceServerHttpClient::ModelInferenceStatistics( std::string* infer_stat, const std::string& model_name, const std::string& model_version, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2/models"); if (!model_name.empty()) { request_uri += "/" + model_name; } if (!model_version.empty()) { request_uri += "/versions/" + model_version; } request_uri += "/stats"; return Get(request_uri, headers, query_params, infer_stat); } Error InferenceServerHttpClient::UpdateTraceSettings( std::string* response, const std::string& model_name, const std::map>& settings, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2"); if (!model_name.empty()) { request_uri += "/models/" + model_name; } request_uri += "/trace/setting"; triton::common::TritonJson::Value request_json( triton::common::TritonJson::ValueType::OBJECT); { for (const auto& pr : settings) { if (pr.second.empty()) { request_json.Add(pr.first.c_str(), triton::common::TritonJson::Value()); } else { if (pr.first == "trace_level") { triton::common::TritonJson::Value level_json( triton::common::TritonJson::ValueType::ARRAY); for (const auto& v : pr.second) { level_json.AppendStringRef(v.c_str()); } request_json.Add(pr.first.c_str(), std::move(level_json)); } else { request_json.AddStringRef(pr.first.c_str(), pr.second[0].c_str()); } } } } triton::common::TritonJson::WriteBuffer buffer; Error err = request_json.Write(&buffer); if (!err.IsOk()) { return err; } return Post(request_uri, buffer.Contents(), headers, query_params, response); } Error InferenceServerHttpClient::GetTraceSettings( std::string* settings, const std::string& model_name, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2"); if (!model_name.empty()) { request_uri += "/models/" + model_name; } request_uri += "/trace/setting"; return Get(request_uri, headers, query_params, settings); } Error InferenceServerHttpClient::SystemSharedMemoryStatus( std::string* status, const std::string& name, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2/systemsharedmemory"); if (!name.empty()) { request_uri = request_uri + "/region/" + name; } request_uri = request_uri + "/status"; return Get(request_uri, headers, query_params, status); } Error InferenceServerHttpClient::RegisterSystemSharedMemory( const std::string& name, const std::string& key, const size_t byte_size, const size_t offset, const Headers& headers, const Parameters& query_params) { std::string request_uri( url_ + "/v2/systemsharedmemory/region/" + name + "/register"); triton::common::TritonJson::Value request_json( triton::common::TritonJson::ValueType::OBJECT); { request_json.AddStringRef("key", key.c_str(), key.size()); request_json.AddUInt("offset", offset); request_json.AddUInt("byte_size", byte_size); } triton::common::TritonJson::WriteBuffer buffer; Error err = request_json.Write(&buffer); if (!err.IsOk()) { return err; } std::string response; return Post(request_uri, buffer.Contents(), headers, query_params, &response); } Error InferenceServerHttpClient::UnregisterSystemSharedMemory( const std::string& region_name, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2/systemsharedmemory"); if (!region_name.empty()) { request_uri = request_uri + "/region/" + region_name; } request_uri = request_uri + "/unregister"; std::string request; // empty request body std::string response; return Post(request_uri, request, headers, query_params, &response); } Error InferenceServerHttpClient::CudaSharedMemoryStatus( std::string* status, const std::string& region_name, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2/cudasharedmemory"); if (!region_name.empty()) { request_uri = request_uri + "/region/" + region_name; } request_uri = request_uri + "/status"; return Get(request_uri, headers, query_params, status); } Error InferenceServerHttpClient::RegisterCudaSharedMemory( const std::string& name, const cudaIpcMemHandle_t& raw_handle, const size_t device_id, const size_t byte_size, const Headers& headers, const Parameters& query_params) { std::string request_uri( url_ + "/v2/cudasharedmemory/region/" + name + "/register"); triton::common::TritonJson::Value request_json( triton::common::TritonJson::ValueType::OBJECT); { triton::common::TritonJson::Value raw_handle_json( request_json, triton::common::TritonJson::ValueType::OBJECT); { // Must free encoded_handle after use to prevent memory leak char* encoded_handle = nullptr; int encoded_size; Base64Encode( (char*)((void*)&raw_handle), sizeof(cudaIpcMemHandle_t), &encoded_handle, &encoded_size); if (encoded_handle == nullptr) { return Error("Failed to base64 encode the cudaIpcMemHandle_t"); } raw_handle_json.AddString("b64", encoded_handle, encoded_size); free(encoded_handle); } request_json.Add("raw_handle", std::move(raw_handle_json)); request_json.AddUInt("device_id", device_id); request_json.AddUInt("byte_size", byte_size); } triton::common::TritonJson::WriteBuffer buffer; Error err = request_json.Write(&buffer); if (!err.IsOk()) { return err; } std::string response; return Post(request_uri, buffer.Contents(), headers, query_params, &response); } Error InferenceServerHttpClient::UnregisterCudaSharedMemory( const std::string& name, const Headers& headers, const Parameters& query_params) { std::string request_uri(url_ + "/v2/cudasharedmemory"); if (!name.empty()) { request_uri = request_uri + "/region/" + name; } request_uri = request_uri + "/unregister"; std::string request; // empty request body std::string response; return Post(request_uri, request, headers, query_params, &response); } Error InferenceServerHttpClient::Infer( InferResult** result, const InferOptions& options, const std::vector& inputs, const std::vector& outputs, const Headers& headers, const Parameters& query_params, const CompressionType request_compression_algorithm, const CompressionType response_compression_algorithm) { Error err; std::string request_uri(url_ + "/v2/models/" + options.model_name_); if (!options.model_version_.empty()) { request_uri = request_uri + "/versions/" + options.model_version_; } request_uri = request_uri + "/infer"; std::shared_ptr sync_request( new HttpInferRequest(nullptr /* callback */, verbose_)); sync_request->Timer().Reset(); sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::REQUEST_START); if (!CurlGlobal::Get().Status().IsOk()) { return CurlGlobal::Get().Status(); } err = PreRunProcessing( easy_handle_, request_uri, options, inputs, outputs, headers, query_params, request_compression_algorithm, response_compression_algorithm, sync_request); if (!err.IsOk()) { return err; } sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::SEND_START); // Set SEND_END when content length is 0 (because // CURLOPT_READFUNCTION will not be called). In that case, we can't // measure SEND_END properly (send ends after sending request // header). if (sync_request->total_input_byte_size_ == 0) { sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::SEND_END); } // During this call SEND_END (except in above case), RECV_START, and // RECV_END will be set. auto curl_status = curl_easy_perform(easy_handle_); if (curl_status == CURLE_OPERATION_TIMEDOUT) { return Error( "HTTP client failed (Deadline Exceeded): " + std::string(curl_easy_strerror(curl_status))); } else if (curl_status != CURLE_OK) { return Error( "HTTP client failed: " + std::string(curl_easy_strerror(curl_status))); } else { // Success curl_easy_getinfo( easy_handle_, CURLINFO_RESPONSE_CODE, &sync_request->http_code_); } InferResultHttp::Create(result, sync_request); sync_request->Timer().CaptureTimestamp(RequestTimers::Kind::REQUEST_END); err = UpdateInferStat(sync_request->Timer()); if (!err.IsOk()) { std::cerr << "Failed to update context stat: " << err << std::endl; } err = (*result)->RequestStatus(); return err; } Error InferenceServerHttpClient::AsyncInfer( OnCompleteFn callback, const InferOptions& options, const std::vector& inputs, const std::vector& outputs, const Headers& headers, const Parameters& query_params, const CompressionType request_compression_algorithm, const CompressionType response_compression_algorithm) { if (callback == nullptr) { return Error( "Callback function must be provided along with AsyncInfer() call."); } std::shared_ptr async_request; if (!multi_handle_) { return Error("failed to start HTTP asynchronous client"); } else if (!worker_.joinable()) { worker_ = std::thread(&InferenceServerHttpClient::AsyncTransfer, this); } std::string request_uri(url_ + "/v2/models/" + options.model_name_); if (!options.model_version_.empty()) { request_uri = request_uri + "/versions/" + options.model_version_; } request_uri = request_uri + "/infer"; HttpInferRequest* raw_async_request = new HttpInferRequest(std::move(callback), verbose_); async_request.reset(raw_async_request); async_request->Timer().CaptureTimestamp(RequestTimers::Kind::REQUEST_START); CURL* multi_easy_handle = curl_easy_init(); Error err = PreRunProcessing( reinterpret_cast(multi_easy_handle), request_uri, options, inputs, outputs, headers, query_params, request_compression_algorithm, response_compression_algorithm, async_request); if (!err.IsOk()) { curl_easy_cleanup(multi_easy_handle); return err; } { std::lock_guard lock(mutex_); if (exiting_) { return Error("Client is exiting."); } auto insert_result = new_async_requests_.emplace(std::make_pair( reinterpret_cast(multi_easy_handle), async_request)); if (!insert_result.second) { curl_easy_cleanup(multi_easy_handle); return Error("Failed to insert new asynchronous request context."); } } async_request->Timer().CaptureTimestamp(RequestTimers::Kind::SEND_START); curl_multi_wakeup(multi_handle_); if (async_request->total_input_byte_size_ == 0) { // Set SEND_END here because CURLOPT_READFUNCTION will not be called if // content length is 0. In that case, we can't measure SEND_END properly // (send ends after sending request header). async_request->Timer().CaptureTimestamp(RequestTimers::Kind::SEND_END); } return Error::Success; } Error InferenceServerHttpClient::InferMulti( std::vector* results, const std::vector& options, const std::vector>& inputs, const std::vector>& outputs, const Headers& headers, const Parameters& query_params, const CompressionType request_compression_algorithm, const CompressionType response_compression_algorithm) { Error err; // Sanity check if ((inputs.size() != options.size()) && (options.size() != 1)) { return Error( "'options' must either contain 1 element or match size of 'inputs'"); } if ((inputs.size() != outputs.size()) && ((outputs.size() != 1) && (outputs.size() != 0))) { return Error( "'outputs' must either contain 0/1 element or match size of 'inputs'"); } int64_t max_option_idx = options.size() - 1; // value of '-1' means no output is specified int64_t max_output_idx = outputs.size() - 1; static std::vector empty_outputs{}; for (int64_t i = 0; i < (int64_t)inputs.size(); ++i) { const auto& request_options = options[std::min(max_option_idx, i)]; const auto& request_output = (max_output_idx == -1) ? empty_outputs : outputs[std::min(max_output_idx, i)]; results->emplace_back(); err = Infer( &results->back(), request_options, inputs[i], request_output, headers, query_params, request_compression_algorithm, response_compression_algorithm); if (!err.IsOk()) { return err; } } return Error::Success; } Error InferenceServerHttpClient::AsyncInferMulti( OnMultiCompleteFn callback, const std::vector& options, const std::vector>& inputs, const std::vector>& outputs, const Headers& headers, const Parameters& query_params, const CompressionType request_compression_algorithm, const CompressionType response_compression_algorithm) { // Sanity check if ((inputs.size() != options.size()) && (options.size() != 1)) { return Error( "'options' must either contain 1 element or match size of 'inputs'"); } if ((inputs.size() != outputs.size()) && ((outputs.size() != 1) && (outputs.size() != 0))) { return Error( "'outputs' must either contain 0/1 element or match size of 'inputs'"); } if (callback == nullptr) { return Error( "Callback function must be provided along with AsyncInfer() call."); } int64_t max_option_idx = options.size() - 1; // value of '-1' means no output is specified int64_t max_output_idx = outputs.size() - 1; static std::vector empty_outputs{}; std::shared_ptr> response_counter( new std::atomic(inputs.size())); std::shared_ptr> responses( new std::vector(inputs.size())); for (int64_t i = 0; i < (int64_t)inputs.size(); ++i) { const auto& request_options = options[std::min(max_option_idx, i)]; const auto& request_output = (max_output_idx == -1) ? empty_outputs : outputs[std::min(max_output_idx, i)]; OnCompleteFn cb = [response_counter, responses, i, callback](InferResult* result) { (*responses)[i] = result; // last response if (response_counter->fetch_sub(1) == 1) { std::vector results; results.swap(*responses); callback(results); } }; auto err = AsyncInfer( cb, request_options, inputs[i], request_output, headers, query_params, request_compression_algorithm, response_compression_algorithm); if (!err.IsOk()) { // Create response with error as other requests may be sent and their // responses may not be accessed outside the callback. InferResult* err_res; err = InferResultHttp::Create(&err_res, err); if (!err.IsOk()) { std::cerr << "Failed to create result for error: " << err.Message() << std::endl; } cb(err_res); continue; } } return Error::Success; } size_t InferenceServerHttpClient::InferRequestProvider( void* contents, size_t size, size_t nmemb, void* userp) { HttpInferRequest* request = reinterpret_cast(userp); size_t input_bytes = 0; Error err = request->GetNextInput( reinterpret_cast(contents), size * nmemb, &input_bytes); if (!err.IsOk()) { std::cerr << "RequestProvider: " << err << std::endl; return CURL_READFUNC_ABORT; } return input_bytes; } size_t InferenceServerHttpClient::InferResponseHeaderHandler( void* contents, size_t size, size_t nmemb, void* userp) { HttpInferRequest* request = reinterpret_cast(userp); char* buf = reinterpret_cast(contents); size_t byte_size = size * nmemb; size_t idx = strlen(kInferHeaderContentLengthHTTPHeader); size_t length_idx = strlen(kContentLengthHTTPHeader); if ((idx < byte_size) && !strncasecmp(buf, kInferHeaderContentLengthHTTPHeader, idx)) { while ((idx < byte_size) && (buf[idx] != ':')) { ++idx; } if (idx < byte_size) { std::string hdr(buf + idx + 1, byte_size - idx - 1); request->response_json_size_ = std::stoi(hdr); } } else if ( (length_idx < byte_size) && !strncasecmp(buf, kContentLengthHTTPHeader, length_idx)) { while ((length_idx < byte_size) && (buf[length_idx] != ':')) { ++length_idx; } if (length_idx < byte_size) { std::string hdr(buf + length_idx + 1, byte_size - length_idx - 1); request->infer_response_buffer_->reserve(std::stoi(hdr)); } } return byte_size; } size_t InferenceServerHttpClient::InferResponseHandler( void* contents, size_t size, size_t nmemb, void* userp) { HttpInferRequest* request = reinterpret_cast(userp); if (request->Timer().Timestamp(RequestTimers::Kind::RECV_START) == 0) { request->Timer().CaptureTimestamp(RequestTimers::Kind::RECV_START); } char* buf = reinterpret_cast(contents); size_t result_bytes = size * nmemb; request->infer_response_buffer_->append(buf, result_bytes); // InferResponseHandler may be called multiple times so we overwrite // RECV_END so that we always have the time of the last. request->Timer().CaptureTimestamp(RequestTimers::Kind::RECV_END); return result_bytes; } Error InferenceServerHttpClient::PreRunProcessing( void* vcurl, std::string& request_uri, const InferOptions& options, const std::vector& inputs, const std::vector& outputs, const Headers& headers, const Parameters& query_params, const CompressionType request_compression_algorithm, const CompressionType response_compression_algorithm, std::shared_ptr& http_request) { CURL* curl = reinterpret_cast(vcurl); // Prepare the request object to provide the data for inference. Error err = http_request->InitializeRequest(options, inputs, outputs); if (!err.IsOk()) { return err; } // Add the buffers holding input tensor data bool all_inputs_are_json{true}; for (const auto this_input : inputs) { if (this_input->BinaryData()) { all_inputs_are_json = false; } if (!this_input->IsSharedMemory() && this_input->BinaryData()) { this_input->PrepareForRequest(); bool end_of_input = false; while (!end_of_input) { const uint8_t* buf; size_t buf_size; this_input->GetNext(&buf, &buf_size, &end_of_input); if (buf != nullptr) { http_request->AddInput(const_cast(buf), buf_size); } } } } // Compress data if requested switch (request_compression_algorithm) { case CompressionType::NONE: break; case CompressionType::DEFLATE: case CompressionType::GZIP: #ifdef TRITON_ENABLE_ZLIB http_request->CompressInput(request_compression_algorithm); break; #else return Error( "Compression type needs to be CompressionType::NONE since ZLIB is " "not included in client build"); #endif } // Prepare curl if (!query_params.empty()) { request_uri = request_uri + "?" + GetQueryString(query_params); } curl_easy_setopt(curl, CURLOPT_URL, request_uri.c_str()); curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0"); curl_easy_setopt(curl, CURLOPT_POST, 1L); curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 1L); if (options.client_timeout_ != 0) { uint64_t timeout_ms = (options.client_timeout_ / 1000); curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout_ms); } if (verbose_) { curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); } const long buffer_byte_size = 16 * 1024 * 1024; curl_easy_setopt(curl, CURLOPT_UPLOAD_BUFFERSIZE, buffer_byte_size); curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, buffer_byte_size); // request data provided by InferRequestProvider() curl_easy_setopt(curl, CURLOPT_READFUNCTION, InferRequestProvider); curl_easy_setopt(curl, CURLOPT_READDATA, http_request.get()); // response headers handled by InferResponseHeaderHandler() curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, InferResponseHeaderHandler); curl_easy_setopt(curl, CURLOPT_HEADERDATA, http_request.get()); // response data handled by InferResponseHandler() curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, InferResponseHandler); curl_easy_setopt(curl, CURLOPT_WRITEDATA, http_request.get()); const curl_off_t post_byte_size = http_request->total_input_byte_size_; curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, post_byte_size); err = SetSSLCurlOptions(&curl, ssl_options_); if (!err.IsOk()) { return err; } struct curl_slist* list = nullptr; std::string infer_hdr{ std::string(kInferHeaderContentLengthHTTPHeader) + ": " + std::to_string(http_request->request_json_.Size())}; list = curl_slist_append(list, infer_hdr.c_str()); list = curl_slist_append(list, "Expect:"); if (all_inputs_are_json) { list = curl_slist_append(list, "Content-Type: application/json"); } else { list = curl_slist_append(list, "Content-Type: application/octet-stream"); } for (const auto& pr : headers) { std::string hdr = pr.first + ": " + pr.second; list = curl_slist_append(list, hdr.c_str()); } // Compress data if requested switch (request_compression_algorithm) { case CompressionType::NONE: break; case CompressionType::DEFLATE: list = curl_slist_append(list, "Content-Encoding: deflate"); break; case CompressionType::GZIP: list = curl_slist_append(list, "Content-Encoding: gzip"); break; } switch (response_compression_algorithm) { case CompressionType::NONE: break; case CompressionType::DEFLATE: curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate"); break; case CompressionType::GZIP: curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip"); break; } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); // The list will be freed when the request is destructed http_request->header_list_ = list; if (verbose_) { std::cout << "inference request: " << http_request->request_json_.Contents() << std::endl; } return Error::Success; } void InferenceServerHttpClient::AsyncTransfer() { int messages_in_queue = 0; int still_running = 0; int numfds = 0; CURLMsg* msg = nullptr; AsyncReqMap ongoing_async_requests; do { // Check for new requests and add them to ongoing requests { std::lock_guard lock(mutex_); for (auto& pair : new_async_requests_) { curl_multi_add_handle( multi_handle_, reinterpret_cast(pair.first)); ongoing_async_requests[pair.first] = std::move(pair.second); } new_async_requests_.clear(); } CURLMcode mc = curl_multi_perform(multi_handle_, &still_running); if (mc != CURLM_OK) { std::cerr << "Unexpected error: curl_multi failed. Code:" << mc << std::endl; continue; } while ((msg = curl_multi_info_read(multi_handle_, &messages_in_queue))) { if (msg->msg != CURLMSG_DONE) { // Something wrong happened. std::cerr << "Unexpected error: received CURLMsg=" << msg->msg << std::endl; continue; } uintptr_t identifier = reinterpret_cast(msg->easy_handle); auto itr = ongoing_async_requests.find(identifier); // This shouldn't happen if (itr == ongoing_async_requests.end()) { std::cerr << "Unexpected error: received completed request that is not " "in the list of asynchronous requests" << std::endl; curl_multi_remove_handle(multi_handle_, msg->easy_handle); curl_easy_cleanup(msg->easy_handle); continue; } auto async_request = itr->second; uint32_t http_code = 400; if (msg->data.result == CURLE_OK) { curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &http_code); async_request->Timer().CaptureTimestamp( RequestTimers::Kind::REQUEST_END); Error err = UpdateInferStat(async_request->Timer()); if (!err.IsOk()) { std::cerr << "Failed to update context stat: " << err << std::endl; } } else if (msg->data.result == CURLE_OPERATION_TIMEDOUT) { http_code = 499; } async_request->http_code_ = http_code; InferResult* result; InferResultHttp::Create(&result, async_request); async_request->callback_(result); ongoing_async_requests.erase(itr); curl_multi_remove_handle(multi_handle_, msg->easy_handle); curl_easy_cleanup(msg->easy_handle); } // Wait for activity on existing requests or // explicit curl_multi_wakeup call // // If there are no descriptors in the multi_handle_ // then curl_multi_poll will wait until curl_multi_wakeup // is called // // curl_multi_wakeup is called when adding a new request // or exiting mc = curl_multi_poll(multi_handle_, NULL, 0, INT_MAX, &numfds); if (mc != CURLM_OK) { std::cerr << "Unexpected error: curl_multi_poll failed. Code:" << mc << std::endl; } } while (!exiting_); for (auto& request : ongoing_async_requests) { CURL* easy_handle = reinterpret_cast(request.first); curl_multi_remove_handle(multi_handle_, easy_handle); curl_easy_cleanup(easy_handle); } for (auto& request : new_async_requests_) { CURL* easy_handle = reinterpret_cast(request.first); curl_easy_cleanup(easy_handle); } } size_t InferenceServerHttpClient::ResponseHandler( void* contents, size_t size, size_t nmemb, void* userp) { std::string* response_string = reinterpret_cast(userp); uint8_t* buf = reinterpret_cast(contents); size_t result_bytes = size * nmemb; std::copy(buf, buf + result_bytes, std::back_inserter(*response_string)); return result_bytes; } namespace { Error ParseErrorJson(const std::string& json_str) { triton::common::TritonJson::Value json; Error err = json.Parse(json_str.c_str(), json_str.size()); if (!err.IsOk()) { return err; } const char* errstr; size_t errlen; err = json.MemberAsString("error", &errstr, &errlen); if (!err.IsOk()) { return err; } return Error(std::move(std::string(errstr, errlen))); } } // namespace Error InferenceServerHttpClient::Get( std::string& request_uri, const Headers& headers, const Parameters& query_params, std::string* response, long* http_code) { if (!query_params.empty()) { request_uri = request_uri + "?" + GetQueryString(query_params); } if (!CurlGlobal::Get().Status().IsOk()) { return CurlGlobal::Get().Status(); } CURL* curl = curl_easy_init(); if (!curl) { return Error("failed to initialize HTTP client"); } curl_easy_setopt(curl, CURLOPT_URL, request_uri.c_str()); curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0"); curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 1L); if (verbose_) { curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); } // Response data handled by ResponseHandler() response->clear(); response->reserve(1024); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, ResponseHandler); curl_easy_setopt(curl, CURLOPT_WRITEDATA, response); Error err = SetSSLCurlOptions(&curl, ssl_options_); if (!err.IsOk()) { return err; } // Add user provided headers... struct curl_slist* header_list = nullptr; for (const auto& pr : headers) { std::string hdr = pr.first + ": " + pr.second; header_list = curl_slist_append(header_list, hdr.c_str()); } if (header_list != nullptr) { curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list); } CURLcode res = curl_easy_perform(curl); if (res != CURLE_OK) { curl_slist_free_all(header_list); curl_easy_cleanup(curl); return Error("HTTP client failed: " + std::string(curl_easy_strerror(res))); } long lhttp_code; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &lhttp_code); curl_slist_free_all(header_list); curl_easy_cleanup(curl); if (verbose_) { std::cout << *response << std::endl; } // If http code was requested for return, then just return it, // otherwise flag an error if the http code is not 200. if (http_code != nullptr) { *http_code = lhttp_code; } else if (lhttp_code != 200) { return ParseErrorJson(*response); } return Error::Success; } Error InferenceServerHttpClient::Post( std::string& request_uri, const std::string& request, const Headers& headers, const Parameters& query_params, std::string* response) { if (!query_params.empty()) { request_uri = request_uri + "?" + GetQueryString(query_params); } if (!CurlGlobal::Get().Status().IsOk()) { return CurlGlobal::Get().Status(); } CURL* curl = curl_easy_init(); if (!curl) { return Error("failed to initialize HTTP client"); } curl_easy_setopt(curl, CURLOPT_URL, request_uri.c_str()); curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0"); curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 1L); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, request.size()); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request.c_str()); if (verbose_) { curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); } // Response data handled by ResponseHandler() response->clear(); response->reserve(1024); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, ResponseHandler); curl_easy_setopt(curl, CURLOPT_WRITEDATA, response); Error err = SetSSLCurlOptions(&curl, ssl_options_); if (!err.IsOk()) { return err; } // Add user provided headers... struct curl_slist* header_list = nullptr; for (const auto& pr : headers) { std::string hdr = pr.first + ": " + pr.second; header_list = curl_slist_append(header_list, hdr.c_str()); } if (header_list != nullptr) { curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list); } CURLcode res = curl_easy_perform(curl); if (res != CURLE_OK) { curl_slist_free_all(header_list); curl_easy_cleanup(curl); return Error("HTTP client failed: " + std::string(curl_easy_strerror(res))); } long http_code; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); curl_slist_free_all(header_list); curl_easy_cleanup(curl); if (verbose_) { std::cout << *response << std::endl; } if (http_code != 200) { return ParseErrorJson(*response); } return Error::Success; } //============================================================================== }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/http_client.h000066400000000000000000001016121527052371200224050ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "common.h" #include "ipc.h" namespace triton { namespace client { class HttpInferRequest; /// The key-value map type to be included in the request /// as custom headers. typedef std::map Headers; /// The key-value map type to be included as URL parameters. typedef std::map Parameters; // The options for authorizing and authenticating SSL/TLS connections. struct HttpSslOptions { enum CERTTYPE { CERT_PEM = 0, CERT_DER = 1 }; enum KEYTYPE { KEY_PEM = 0, KEY_DER = 1 // TODO TMA-1645: Support loading private key from crypto engine // KEY_ENG = 2 }; explicit HttpSslOptions() : verify_peer(1), verify_host(2), cert_type(CERTTYPE::CERT_PEM), key_type(KEYTYPE::KEY_PEM) { } // This option determines whether curl verifies the authenticity of the peer's // certificate. A value of 1 means curl verifies; 0 (zero) means it does not. // Default value is 1. See here for more details: // https://curl.se/libcurl/c/CURLOPT_SSL_VERIFYPEER.html long verify_peer; // This option determines whether libcurl verifies that the server cert is for // the server it is known as. The default value for this option is 2 which // means that certificate must indicate that the server is the server to which // you meant to connect, or the connection fails. See here for more details: // https://curl.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html long verify_host; // File holding one or more certificates to verify the peer with. If not // specified, client will look for the system path where cacert bundle is // assumed to be stored, as established at build time. See here for more // information: https://curl.se/libcurl/c/CURLOPT_CAINFO.html std::string ca_info; // The format of client certificate. By default it is CERT_PEM. See here for // more details: https://curl.se/libcurl/c/CURLOPT_SSLCERTTYPE.html CERTTYPE cert_type; // The file name of your client certificate. See here for more details: // https://curl.se/libcurl/c/CURLOPT_SSLCERT.html std::string cert; // The format of the private key. By default it is KEY_PEM. See here for more // details: https://curl.se/libcurl/c/CURLOPT_SSLKEYTYPE.html. KEYTYPE key_type; // The private key. See here for more details: // https://curl.se/libcurl/c/CURLOPT_SSLKEY.html. std::string key; }; //============================================================================== /// An InferenceServerHttpClient object is used to perform any kind of /// communication with the InferenceServer using HTTP protocol. None /// of the methods of InferenceServerHttpClient are thread safe. The /// class is intended to be used by a single thread and simultaneously /// calling different methods with different threads is not supported /// and will cause undefined behavior. /// /// \code /// std::unique_ptr client; /// InferenceServerHttpClient::Create(&client, "localhost:8000"); /// bool live; /// client->IsServerLive(&live); /// ... /// ... /// \endcode /// class InferenceServerHttpClient : public InferenceServerClient { public: enum class CompressionType { NONE, DEFLATE, GZIP }; ~InferenceServerHttpClient(); /// Generate a request body for inference using the supplied 'inputs' and /// requesting the outputs specified by 'outputs'. /// \param request_body Returns the generated inference request body /// \param header_length Returns the length of the inference header. /// \param options The options for inference request. /// \param inputs The vector of InferInput describing the model inputs. /// \param outputs The vector of InferRequestedOutput describing how the /// output must be returned. If not provided then all the outputs in the /// model config will be returned as default settings. /// \return Error object indicating success or failure of the /// request. static Error GenerateRequestBody( std::vector* request_body, size_t* header_length, const InferOptions& options, const std::vector& inputs, const std::vector& outputs = std::vector()); /// Generate a InferResult object from the given 'response_body'. /// \param result Returns the generated InferResult object. /// \param response_body The inference response from the server /// \param header_length The length of the inference header if the header /// does not occupy the whole response body. 0 indicates that /// the whole response body is the inference response header. /// \return Error object indicating success or failure of the /// request. static Error ParseResponseBody( InferResult** result, const std::vector& response_body, const size_t header_length = 0); /// Create a client that can be used to communicate with the server. /// \param client Returns a new InferenceServerHttpClient object. /// \param server_url The inference server name, port, optional /// scheme and optional base path in the following format: /// host:port/. /// \param verbose If true generate verbose output when contacting /// the inference server. /// \param ssl_options Specifies the settings for configuring /// SSL encryption and authorization. Providing these options /// do not ensure that SSL/TLS will be used in communication. /// The use of SSL/TLS depends entirely on the server endpoint. /// These options will be ignored if the server_url does not /// expose `https://` scheme. /// \return Error object indicating success or failure. static Error Create( std::unique_ptr* client, const std::string& server_url, bool verbose = false, const HttpSslOptions& ssl_options = HttpSslOptions()); /// Contact the inference server and get its liveness. /// \param live Returns whether the server is live or not. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \return Error object indicating success or failure of the request. Error IsServerLive( bool* live, const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get its readiness. /// \param ready Returns whether the server is ready or not. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error IsServerReady( bool* ready, const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get the readiness of specified model. /// \param ready Returns whether the specified model is ready or not. /// \param model_name The name of the model to check for readiness. /// \param model_version The version of the model to check for readiness. /// The default value is an empty string which means then the server will /// choose a version based on the model and internal policy. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error IsModelReady( bool* ready, const std::string& model_name, const std::string& model_version = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get its metadata. /// \param server_metadata Returns JSON representation of the /// metadata as a string. /// \param headers Optional map specifying additional HTTP headers to /// include in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error ServerMetadata( std::string* server_metadata, const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get the metadata of specified model. /// \param model_metadata Returns JSON representation of model /// metadata as a string. /// \param model_name The name of the model to get metadata. /// \param model_version The version of the model to get metadata. /// The default value is an empty string which means then the server will /// choose a version based on the model and internal policy. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error ModelMetadata( std::string* model_metadata, const std::string& model_name, const std::string& model_version = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get the configuration of specified model. /// \param model_config Returns JSON representation of model /// configuration as a string. /// \param model_name The name of the model to get configuration. /// \param model_version The version of the model to get configuration. /// The default value is an empty string which means then the server will /// choose a version based on the model and internal policy. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error ModelConfig( std::string* model_config, const std::string& model_name, const std::string& model_version = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get the index of model repository /// contents. /// \param repository_index Returns JSON representation of the /// repository index as a string. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error ModelRepositoryIndex( std::string* repository_index, const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Request the inference server to load or reload specified model. /// \param model_name The name of the model to be loaded or reloaded. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \param config Optional JSON representation of a model config provided for /// the load request, if provided, this config will be used for /// loading the model. /// \param files Optional map specifying file path (with "file:" /// prefix) in the override model directory to the file content. /// The files will form the model directory that the model /// will be loaded from. If specified, 'config' must be provided to be /// the model configuration of the override model directory. /// \return Error object indicating success or failure of the request. Error LoadModel( const std::string& model_name, const Headers& headers = Headers(), const Parameters& query_params = Parameters(), const std::string& config = std::string(), const std::map>& files = {}); /// Request the inference server to unload specified model. /// \param model_name The name of the model to be unloaded. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error UnloadModel( const std::string& model_name, const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get the inference statistics for the /// specified model name and version. /// \param infer_stat Returns the JSON representation of the /// inference statistics as a string. /// \param model_name The name of the model to get inference statistics. The /// default value is an empty string which means statistics of all models will /// be returned in the response. /// \param model_version The version of the model to get inference statistics. /// The default value is an empty string which means then the server will /// choose a version based on the model and internal policy. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error ModelInferenceStatistics( std::string* infer_stat, const std::string& model_name = "", const std::string& model_version = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Update the trace settings for the specified model name, or global trace /// settings if model name is not given. /// \param response Returns the JSON representation of the updated trace /// settings as a string. /// \param model_name The name of the model to update trace settings. The /// default value is an empty string which means the global trace settings /// will be updated. /// \param settings The new trace setting values. Only the settings listed /// will be updated. If a trace setting is listed in the map with an empty /// string, that setting will be cleared. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error UpdateTraceSettings( std::string* response, const std::string& model_name = "", const std::map>& settings = std::map>(), const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Get the trace settings for the specified model name, or global trace /// settings if model name is not given. /// \param settings Returns the JSON representation of the trace /// settings as a string. /// \param model_name The name of the model to get trace settings. The /// default value is an empty string which means the global trace settings /// will be returned. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error GetTraceSettings( std::string* settings, const std::string& model_name = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get the status for requested system /// shared memory. /// \param status Returns the JSON representation of the system /// shared memory status as a string. /// \param region_name The name of the region to query status. The default /// value is an empty string, which means that the status of all active system /// shared memory will be returned. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error SystemSharedMemoryStatus( std::string* status, const std::string& region_name = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Request the server to register a system shared memory with the provided /// details. /// \param name The name of the region to register. /// \param key The key of the underlying memory object that contains the /// system shared memory region. /// \param byte_size The size of the system shared memory region, in bytes. /// \param offset Offset, in bytes, within the underlying memory object to /// the start of the system shared memory region. The default value is zero. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request Error RegisterSystemSharedMemory( const std::string& name, const std::string& key, const size_t byte_size, const size_t offset = 0, const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Request the server to unregister a system shared memory with the /// specified name. /// \param name The name of the region to unregister. The default value is /// empty string which means all the system shared memory regions will be /// unregistered. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request Error UnregisterSystemSharedMemory( const std::string& name = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Contact the inference server and get the status for requested CUDA /// shared memory. /// \param status Returns the JSON representation of the CUDA shared /// memory status as a string. /// \param region_name The name of the region to query status. The default /// value is an empty string, which means that the status of all active CUDA /// shared memory will be returned. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request. Error CudaSharedMemoryStatus( std::string* status, const std::string& region_name = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Request the server to register a CUDA shared memory with the provided /// details. /// \param name The name of the region to register. /// \param cuda_shm_handle The cudaIPC handle for the memory object. /// \param device_id The GPU device ID on which the cudaIPC handle was /// created. /// \param byte_size The size of the CUDA shared memory region, in /// bytes. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request Error RegisterCudaSharedMemory( const std::string& name, const cudaIpcMemHandle_t& cuda_shm_handle, const size_t device_id, const size_t byte_size, const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Request the server to unregister a CUDA shared memory with the /// specified name. /// \param name The name of the region to unregister. The default value is /// empty string which means all the CUDA shared memory regions will be /// unregistered. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \return Error object indicating success or failure of the request Error UnregisterCudaSharedMemory( const std::string& name = "", const Headers& headers = Headers(), const Parameters& query_params = Parameters()); /// Run synchronous inference on server. /// \param result Returns the result of inference. /// \param options The options for inference request. /// \param inputs The vector of InferInput describing the model inputs. /// \param outputs The vector of InferRequestedOutput describing how the /// output must be returned. If not provided then all the outputs in the /// model config will be returned as default settings. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \param request_compression_algorithm Optional HTTP compression algorithm /// to use for the request body on client side. Currently supports DEFLATE, /// GZIP and NONE. By default, no compression is used. /// \param response_compression_algorithm Optional HTTP compression algorithm /// to request for the response body. Note that the response may not be /// compressed if the server does not support the specified algorithm. /// Currently supports DEFLATE, GZIP and NONE. By default, no compression /// is used. /// \return Error object indicating success or failure of the /// request. Error Infer( InferResult** result, const InferOptions& options, const std::vector& inputs, const std::vector& outputs = std::vector(), const Headers& headers = Headers(), const Parameters& query_params = Parameters(), const CompressionType request_compression_algorithm = CompressionType::NONE, const CompressionType response_compression_algorithm = CompressionType::NONE); /// Run asynchronous inference on server. /// Once the request is completed, the InferResult pointer will be passed to /// the provided 'callback' function. Upon the invocation of callback /// function, the ownership of InferResult object is transferred to the /// function caller. It is then the caller's choice on either retrieving the /// results inside the callback function or deferring it to a different thread /// so that the client is unblocked. In order to prevent memory leak, user /// must ensure this object gets deleted. /// Note: InferInput::AppendRaw() or InferInput::SetSharedMemory() calls do /// not copy the data buffers but hold the pointers to the data directly. /// It is advisable to not to disturb the buffer contents until the respective /// callback is invoked. /// \param callback The callback function to be invoked on request completion. /// \param options The options for inference request. /// \param inputs The vector of InferInput describing the model inputs. /// \param outputs The vector of InferRequestedOutput describing how the /// output must be returned. If not provided then all the outputs in the /// model config will be returned as default settings. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \param request_compression_algorithm Optional HTTP compression algorithm /// to use for the request body on client side. Currently supports DEFLATE, /// GZIP and NONE. By default, no compression is used. /// \param response_compression_algorithm Optional HTTP compression algorithm /// to request for the response body. Note that the response may not be /// compressed if the server does not support the specified algorithm. /// Currently supports DEFLATE, GZIP and NONE. By default, no compression /// is used. /// \return Error object indicating success /// or failure of the request. Error AsyncInfer( OnCompleteFn callback, const InferOptions& options, const std::vector& inputs, const std::vector& outputs = std::vector(), const Headers& headers = Headers(), const Parameters& query_params = Parameters(), const CompressionType request_compression_algorithm = CompressionType::NONE, const CompressionType response_compression_algorithm = CompressionType::NONE); /// Run multiple synchronous inferences on server. /// \param results Returns the results of the inferences. /// \param options The options for each inference request, one set of /// options may be provided and it will be used for all inference requests. /// \param inputs The vector of InferInput objects describing the model inputs /// for each inference request. /// \param outputs Optional vector of InferRequestedOutput objects describing /// how the output must be returned. If not provided then all the outputs in /// the model config will be returned as default settings. And one set of /// outputs may be provided and it will be used for all inference requests. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \param request_compression_algorithm Optional HTTP compression algorithm /// to use for the request body on client side. Currently supports DEFLATE, /// GZIP and NONE. By default, no compression is used. /// \param response_compression_algorithm Optional HTTP compression algorithm /// to request for the response body. Note that the response may not be /// compressed if the server does not support the specified algorithm. /// Currently supports DEFLATE, GZIP and NONE. By default, no compression /// is used. /// \return Error object indicating success or failure of the /// request. Error InferMulti( std::vector* results, const std::vector& options, const std::vector>& inputs, const std::vector>& outputs = std::vector>(), const Headers& headers = Headers(), const Parameters& query_params = Parameters(), const CompressionType request_compression_algorithm = CompressionType::NONE, const CompressionType response_compression_algorithm = CompressionType::NONE); /// Run multiple asynchronous inferences on server. /// Once all the requests are completed, the vector of InferResult pointers /// will be passed to the provided 'callback' function. Upon the invocation /// of callback function, the ownership of the InferResult objects are /// transferred to the function caller. It is then the caller's choice on /// either retrieving the results inside the callback function or deferring it /// to a different thread so that the client is unblocked. In order to /// prevent memory leak, user must ensure these objects get deleted. /// Note: InferInput::AppendRaw() or InferInput::SetSharedMemory() calls do /// not copy the data buffers but hold the pointers to the data directly. /// It is advisable to not to disturb the buffer contents until the respective /// callback is invoked. /// \param callback The callback function to be invoked on the completion of /// all requests. /// \param options The options for each inference request, one set of /// option may be provided and it will be used for all inference requests. /// \param inputs The vector of InferInput objects describing the model inputs /// for each inference request. /// \param outputs Optional vector of InferRequestedOutput objects describing /// how the output must be returned. If not provided then all the outputs in /// the model config will be returned as default settings. And one set of /// outputs may be provided and it will be used for all inference requests. /// \param headers Optional map specifying additional HTTP headers to include /// in request. /// \param query_params Optional map specifying parameters that must be /// included with URL query. /// \param request_compression_algorithm Optional HTTP compression algorithm /// to use for the request body on client side. Currently supports DEFLATE, /// GZIP and NONE. By default, no compression is used. /// \param response_compression_algorithm Optional HTTP compression algorithm /// to request for the response body. Note that the response may not be /// compressed if the server does not support the specified algorithm. /// Currently supports DEFLATE, GZIP and NONE. By default, no compression /// is used. /// \return Error object indicating success /// or failure of the request. Error AsyncInferMulti( OnMultiCompleteFn callback, const std::vector& options, const std::vector>& inputs, const std::vector>& outputs = std::vector>(), const Headers& headers = Headers(), const Parameters& query_params = Parameters(), const CompressionType request_compression_algorithm = CompressionType::NONE, const CompressionType response_compression_algorithm = CompressionType::NONE); private: InferenceServerHttpClient( const std::string& url, bool verbose, const HttpSslOptions& ssl_options); Error PreRunProcessing( void* curl, std::string& request_uri, const InferOptions& options, const std::vector& inputs, const std::vector& outputs, const Headers& headers, const Parameters& query_params, const CompressionType request_compression_algorithm, const CompressionType response_compression_algorithm, std::shared_ptr& request); void AsyncTransfer(); Error Get( std::string& request_uri, const Headers& headers, const Parameters& query_params, std::string* response, long* http_code = nullptr); Error Post( std::string& request_uri, const std::string& request, const Headers& headers, const Parameters& query_params, std::string* response); static size_t ResponseHandler( void* contents, size_t size, size_t nmemb, void* userp); static size_t InferRequestProvider( void* contents, size_t size, size_t nmemb, void* userp); static size_t InferResponseHeaderHandler( void* contents, size_t size, size_t nmemb, void* userp); static size_t InferResponseHandler( void* contents, size_t size, size_t nmemb, void* userp); // The server url const std::string url_; // The options for authorizing and authenticating SSL/TLS connections HttpSslOptions ssl_options_; using AsyncReqMap = std::map>; // curl easy handle shared for all synchronous requests void* easy_handle_; // curl multi handle for processing asynchronous requests void* multi_handle_; // map to record new asynchronous requests with pointer to easy handle // or tag id as key AsyncReqMap new_async_requests_; }; }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/ipc.h000066400000000000000000000032341527052371200206440ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef TRITON_ENABLE_GPU #include #else struct cudaIpcMemHandle_t {}; #endif // TRITON_ENABLE_GPU kytensor-2.49.0.7/client/src/c++/library/json_utils.cc000066400000000000000000000041401527052371200224150ustar00rootroot00000000000000// Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "json_utils.h" #include namespace triton { namespace client { Error ParseJson(rapidjson::Document* document, const std::string& json_str) { const unsigned int parseFlags = rapidjson::kParseNanAndInfFlag; document->Parse(json_str.c_str(), json_str.size()); if (document->HasParseError()) { return Error( "failed to parse JSON at" + std::to_string(document->GetErrorOffset()) + ": " + std::string(GetParseError_En(document->GetParseError()))); } return Error::Success; } }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/json_utils.h000066400000000000000000000034431527052371200222640ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include "common.h" namespace triton { namespace client { Error ParseJson(rapidjson::Document* document, const std::string& json_str); }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/libgrpcclient.ldscript000066400000000000000000000031401527052371200243030ustar00rootroot00000000000000# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: extern "C++" { triton::client*; inference*; }; local: *; }; kytensor-2.49.0.7/client/src/c++/library/libhttpclient.ldscript000066400000000000000000000031151527052371200243310ustar00rootroot00000000000000# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: extern "C++" { triton::client* }; local: *; }; kytensor-2.49.0.7/client/src/c++/library/shm_utils.cc000066400000000000000000000067161527052371200222460ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "shm_utils.h" #include #include #include #include #include namespace triton { namespace client { Error CreateSharedMemoryRegion(std::string shm_key, size_t byte_size, int* shm_fd) { // get shared memory region descriptor *shm_fd = shm_open(shm_key.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (*shm_fd == -1) { return Error( "unable to get shared memory descriptor for shared-memory key '" + shm_key + "'"); } // extend shared memory object as by default it's initialized with size 0 int res = ftruncate(*shm_fd, byte_size); if (res == -1) { return Error( "unable to initialize shared-memory key '" + shm_key + "' to requested size: " + std::to_string(byte_size) + " bytes"); } return Error::Success; } Error MapSharedMemory(int shm_fd, size_t offset, size_t byte_size, void** shm_addr) { // map shared memory to process address space *shm_addr = mmap(NULL, byte_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, offset); if (*shm_addr == MAP_FAILED) { return Error( "unable to process address space or shared-memory descriptor: " + std::to_string(shm_fd)); } return Error::Success; } Error CloseSharedMemory(int shm_fd) { // close shared memory descriptor if (close(shm_fd) == -1) { return Error( "unable to close shared-memory descriptor: " + std::to_string(shm_fd)); } return Error::Success; } Error UnlinkSharedMemoryRegion(std::string shm_key) { int shm_fd = shm_unlink(shm_key.c_str()); if (shm_fd == -1) { return Error("unable to unlink shared memory for key '" + shm_key + "'"); } return Error::Success; } Error UnmapSharedMemory(void* shm_addr, size_t byte_size) { int tmp_fd = munmap(shm_addr, byte_size); if (tmp_fd == -1) { return Error("unable to munmap shared memory region"); } return Error::Success; } }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/library/shm_utils.h000066400000000000000000000064531527052371200221060ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "common.h" namespace triton { namespace client { // Create a shared memory region of the size 'byte_size' and return the unique // identifier. // \param shm_key The string identifier of the shared memory region // \param byte_size The size in bytes of the shared memory region // \param shm_fd Returns an int descriptor of the created shared memory region // \return error Returns an error if unable to open shared memory region. Error CreateSharedMemoryRegion( std::string shm_key, size_t byte_size, int* shm_fd); // Mmap the shared memory region with the given 'offset' and 'byte_size' and // return the base address of the region. // \param shm_fd The int descriptor of the created shared memory region // \param offset The offset of the shared memory block from the start of the // shared memory region // \param byte_size The size in bytes of the shared memory region // \param shm_addr Returns the base address of the shared memory region // \return error Returns an error if unable to mmap shared memory region. Error MapSharedMemory( int shm_fd, size_t offset, size_t byte_size, void** shm_addr); // Close the shared memory descriptor. // \param shm_fd The int descriptor of the created shared memory region // \return error Returns an error if unable to close shared memory descriptor. Error CloseSharedMemory(int shm_fd); // Destroy the shared memory region with the given name. // \return error Returns an error if unable to unlink shared memory region. Error UnlinkSharedMemoryRegion(std::string shm_key); // Munmap the shared memory region from the base address with the given // byte_size. // \return error Returns an error if unable to unmap shared memory region. Error UnmapSharedMemory(void* shm_addr, size_t byte_size); }} // namespace triton::client kytensor-2.49.0.7/client/src/c++/perf_analyzer/000077500000000000000000000000001527052371200211135ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/c++/perf_analyzer/README.md000066400000000000000000000032301527052371200223700ustar00rootroot00000000000000 Perf Analyzer documentation has been relocated to [here](https://github.com/triton-inference-server/perf_analyzer/blob/main/README.md). kytensor-2.49.0.7/client/src/c++/perf_analyzer/genai-perf/000077500000000000000000000000001527052371200231305ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/c++/perf_analyzer/genai-perf/README.md000066400000000000000000000032401527052371200244060ustar00rootroot00000000000000 GenAI-Perf documentation has been relocated to [here](https://github.com/triton-inference-server/perf_analyzer/blob/main/genai-perf/README.md). kytensor-2.49.0.7/client/src/c++/tests/000077500000000000000000000000001527052371200174145ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/c++/tests/CMakeLists.txt000066400000000000000000000052271527052371200221620ustar00rootroot00000000000000# Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) if(WIN32) message("C++ tests are not currently supported on Windows because " "they require functionalities that are UNIX specific.") else() if(TRITON_ENABLE_CC_HTTP AND TRITON_ENABLE_CC_GRPC) # # client_timeout_test # add_executable( client_timeout_test client_timeout_test.cc ) target_link_libraries( client_timeout_test PRIVATE grpcclient_static httpclient_static ) install( TARGETS client_timeout_test RUNTIME DESTINATION bin ) # # memory_leak_test # add_executable( memory_leak_test memory_leak_test.cc ) target_link_libraries( memory_leak_test PRIVATE grpcclient_static httpclient_static ) install( TARGETS memory_leak_test RUNTIME DESTINATION bin ) add_executable( cc_client_test cc_client_test.cc ) target_include_directories(cc_client_test PRIVATE ${GTEST_INCLUDE_DIRS}) target_link_libraries( cc_client_test PRIVATE grpcclient_static httpclient_static gtest ${GTEST_LIBRARY} ${GTEST_MAIN_LIBRARY} ) install( TARGETS cc_client_test RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_CC_HTTP AND TRITON_ENABLE_CC_GRPC endif() kytensor-2.49.0.7/client/src/c++/tests/README.md000066400000000000000000000157401527052371200207020ustar00rootroot00000000000000## 推理服务层API单元测试 libtriton-client-unittest用于测试推理服务API接口，目前完成主要接口测试，接口说明参考《统一推理服务层API》。 ## 单元测试情况 ### 推理接口 | 推理接口 | GRPC | HTTP | 测试用例 | |------------------------------|------|------|---------------------| | Create | 支持 | 支持 | client_timeout_test | | IsServerLive | 支持 | 暂不支持 | client_timeout_test | | IsServerReady | 支持 | 暂不支持 | client_timeout_test | | IsModelReady | 支持 | 暂不支持 | client_timeout_test | | ServerMetadata | 支持 | 暂不支持 | client_timeout_test | | ModelMetadata | 支持 | 暂不支持 | client_timeout_test | | ModelConfig | 支持 | 暂不支持 | client_timeout_test | | UnloadModel | 支持 | 暂不支持 | client_timeout_test | | ModelInferenceStatistics | 支持 | 暂不支持 | client_timeout_test | | SystemSharedMemoryStatus | 支持 | 暂不支持 | client_timeout_test | | RegisterSystemSharedMemory | 暂不支持 | 暂不支持 | | | UnregisterSystemSharedMemory | 支持 | 暂不支持 | client_timeout_test | | Infer | 支持 | 支持 | client_timeout_test | | AsyncInfer | 支持 | 支持 | client_timeout_test | | InferMulti | 支持 | 支持 | cc_client_test | | AsyncInferMulti | 支持 | 支持 | cc_client_test | | StartStream | 支持 | 暂不支持 | cc_client_test | | StopStream | 支持 | 暂不支持 | cc_client_test | | AsyncStreamInfer | 支持 | 支持 | cc_client_test | ### 推理输入接口 | 推理输入接口 | GRPC | HTTP | 测试用例 | |------------------|------|------|----------------| | Create | 支持 | 支持 | cc_client_test | | Name | 暂不支持 | 暂不支持 | | | Datatype | 暂不支持 | 暂不支持 | | | Shape | 暂不支持 | 暂不支持 | | | SetShape | 暂不支持 | 暂不支持 | | | Reset | 暂不支持 | 暂不支持 | | | AppendRaw | 支持 | 支持 | cc_client_test | | SetSharedMemory | 暂不支持 | 暂不支持 | | | IsSharedMemory | 暂不支持 | 暂不支持 | | | SharedMemoryInfo | 暂不支持 | 暂不支持 | | | RawData | 暂不支持 | 暂不支持 | | | ByteSize | 暂不支持 | 暂不支持 | | | BinaryData | 暂不支持 | 暂不支持 | | | SetBinaryData | 暂不支持 | 暂不支持 | | ### 推理输出接口 | 推理输出接口 | GRPC | HTTP | 测试用例 | |------------------|------|------|----------------| | Create | 支持 | 支持 | cc_client_test | | Name | 暂不支持 | 暂不支持 | | | SetSharedMemory | 暂不支持 | 暂不支持 | | | IsSharedMemory | 暂不支持 | 暂不支持 | | | SharedMemoryInfo | 暂不支持 | 暂不支持 | | | BinaryData | 暂不支持 | 暂不支持 | | | SetBinaryData | 暂不支持 | 暂不支持 | | ### 推理输出结果接口 | 推理输出结果接口 | GRPC | HTTP | 测试用例 | |-----------------|------|------|---------------------| | ModelName | 暂不支持 | 暂不支持 | | | ModelVersion | 暂不支持 | 暂不支持 | | | Id | 暂不支持 | 暂不支持 | | | Shape | 暂不支持 | 暂不支持 | | | Datatype | 支持 | 支持 | client_timeout_test | | RawData | 支持 | 支持 | cc_client_test | | IsFinalResponse | 暂不支持 | 暂不支持 | | | IsNullResponse | 暂不支持 | 暂不支持 | | | StringData | 暂不支持 | 暂不支持 | | | RequestStatus | 支持 | 支持 | cc_client_test | ## 安装 ``` ## 安装测试包 sudo dpkg -i libtriton-client-unittest_2.49.0_amd64.deb ## 进入安装目录 cd /opt/appdata/kylin-ai/libtriton-client-unittest/ ``` ## 测试使用说明 ### cc_client_test测试 ``` ## 首先杀掉系统已有的推理服务，再起推理服务 killall tritonserver tritonserver --model-repository=/opt/appdata/kylin-ai/libtriton-client-unittest/unittest_models ## 然后运行测试程序 ./cc_client_test ``` 测试结果如下： ``` [ RUN ] HTTP/ClientTest/0.AsyncInferMulti [ OK ] HTTP/ClientTest/0.AsyncInferMulti (1 ms) [ RUN ] HTTP/ClientTest/0.AsyncInferMultiDifferentOutputs [ OK ] HTTP/ClientTest/0.AsyncInferMultiDifferentOutputs (1 ms) [ RUN ] HTTP/ClientTest/0.AsyncInferMultiDifferentOptions [ OK ] HTTP/ClientTest/0.AsyncInferMultiDifferentOptions (1 ms) [ RUN ] HTTP/ClientTest/0.AsyncInferMultiOneOption [ OK ] HTTP/ClientTest/0.AsyncInferMultiOneOption (1 ms) [ RUN ] HTTP/ClientTest/0.AsyncInferMultiOneOutput [ OK ] HTTP/ClientTest/0.AsyncInferMultiOneOutput (0 ms) [ RUN ] HTTP/ClientTest/0.AsyncInferMultiNoOutput [ OK ] HTTP/ClientTest/0.AsyncInferMultiNoOutput (1 ms) [ RUN ] HTTP/ClientTest/0.AsyncInferMultiMismatchOptions [ OK ] HTTP/ClientTest/0.AsyncInferMultiMismatchOptions (0 ms) [ RUN ] HTTP/ClientTest/0.AsyncInferMultiMismatchOutputs [ OK ] HTTP/ClientTest/0.AsyncInferMultiMismatchOutputs (0 ms) [----------] 16 tests from HTTP/ClientTest/0 (14 ms total) [----------] Global test environment tear-down [==========] 35 tests from 3 test suites ran. (28 ms total) [ PASSED ] 35 tests. ``` 所有测试都通过。 ### client_timeout_test测试 ``` ## 首先杀掉系统已有的推理服务，再起推理服务 killall tritonserver tritonserver --model-repository=/opt/appdata/kylin-ai/libtriton-client-unittest/unittest_models --model-control-mode=explicit --load-model=custom_identity_int32 ## 然后运行测试程序 ./client_timeout_test -i grpc -p ``` 测试结果如下： ``` testing other apis OK: Success on IsServerLive: OK: Success on IsServerReady: OK: Success on IsModelReady: OK: Success on ServerMetadata: OK: Success on ModelMetadata: OK: Success on ModelConfig: OK: Success on ModelRepositoryIndex: OK: Success on ModelInferenceStatistics: OK: Success on LoadModel: OK: Success on UnloadModel: OK: Success on SystemSharedMemoryStatus: error: Failed on RegisterSystemSharedMemory: Unable to open shared memory region: '' OK: Success on UnregisterSystemSharedMemory: error count: 1 which is not 0 ``` 有一个错误，但RegisterSystemSharedMemory暂不支持测试，可以忽略。 kytensor-2.49.0.7/client/src/c++/tests/cc_client_test.cc000066400000000000000000002427761527052371200227270ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #define TRITON_INFERENCE_SERVER_CLIENT_CLASS InferenceServerHttpClient #include "grpc_client.h" #include "gtest/gtest.h" #include "http_client.cc" #include "http_client.h" namespace tc = triton::client; namespace { // This test must be run with a running Triton server, // check L0_grpc in server repo for the setup. template class ClientTest : public ::testing::Test { public: ClientTest() : model_name_("onnx_int32_int32_int32"), shape_{1, 16}, dtype_("INT32") { } void SetUp() override { std::string url; if (std::is_same::value) { url = "localhost:8001"; } else if (std::is_same::value) { url = "localhost:8000"; } else { ASSERT_TRUE(false) << "Unrecognized client class type '" << typeid(ClientType).name() << "'"; } auto err = ClientType::Create(&this->client_, url); ASSERT_TRUE(err.IsOk()) << "failed to create GRPC client: " << err.Message(); // Initialize 3 sets of inputs, each with 16 elements for (size_t i = 0; i < 3; ++i) { this->input_data_.emplace_back(); for (size_t j = 0; j < 16; ++j) { this->input_data_.back().emplace_back(i * 16 + j); } } } tc::Error PrepareInputs( const std::vector& input_0, const std::vector& input_1, std::vector* inputs) { inputs->emplace_back(); auto err = tc::InferInput::Create( &inputs->back(), "INPUT0", this->shape_, this->dtype_); if (!err.IsOk()) { return err; } err = inputs->back()->AppendRaw( reinterpret_cast(input_0.data()), input_0.size() * sizeof(int32_t)); if (!err.IsOk()) { return err; } inputs->emplace_back(); err = tc::InferInput::Create( &inputs->back(), "INPUT1", this->shape_, this->dtype_); if (!err.IsOk()) { return err; } err = inputs->back()->AppendRaw( reinterpret_cast(input_1.data()), input_1.size() * sizeof(int32_t)); if (!err.IsOk()) { return err; } return tc::Error::Success; } void ValidateOutput( const std::vector& results, const std::vector>>& expected_outputs) { ASSERT_EQ(results.size(), expected_outputs.size()) << "unexpected number of results"; for (size_t i = 0; i < results.size(); ++i) { ASSERT_TRUE(results[i]->RequestStatus().IsOk()); for (const auto& expected : expected_outputs[i]) { const uint8_t* buf = nullptr; size_t byte_size = 0; auto err = results[i]->RawData(expected.first, &buf, &byte_size); ASSERT_TRUE(err.IsOk()) << "failed to retrieve output '" << expected.first << "' for result " << i << ": " << err.Message(); ASSERT_EQ(byte_size, (expected.second.size() * sizeof(int32_t))); EXPECT_EQ(memcmp(buf, expected.second.data(), byte_size), 0); } } } tc::Error LoadModel( const std::string& model_name, const std::string& config, const std::map>& files = {}); std::string model_name_; std::unique_ptr client_; std::vector> input_data_; std::vector shape_; std::string dtype_; }; template <> tc::Error ClientTest::LoadModel( const std::string& model_name, const std::string& config, const std::map>& files) { return this->client_->LoadModel(model_name, tc::Headers(), config, files); } template <> tc::Error ClientTest::LoadModel( const std::string& model_name, const std::string& config, const std::map>& files) { return this->client_->LoadModel( model_name, tc::Headers(), tc::Parameters(), config, files); } class HTTPTraceTest : public ::testing::Test { public: HTTPTraceTest() : model_name_("simple") {} void SetUp() override { std::string url; url = "localhost:8000"; auto err = tc::InferenceServerHttpClient::Create(&client_, url); ASSERT_TRUE(err.IsOk()) << "failed to create HTTP client: " << err.Message(); } // Helper function to clear all the trace settings to initial state. void TearDown() { tc::Error err = tc::Error::Success; std::string response; std::map> clear_settings = { {"trace_level", {}}, {"trace_rate", {}}, {"trace_count", {}}, {"log_frequency", {}}}; err = client_->UpdateTraceSettings(&response, model_name_, clear_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); err = client_->UpdateTraceSettings(&response, "", clear_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); } // Helper function to make sure the trace setting is properly initialized / // reset before actually running the test case. void CheckServerInitialState() { tc::Error err = tc::Error::Success; std::string trace_settings; std::string initial_settings = "{\"trace_level\":[\"TIMESTAMPS\"],\"trace_rate\":\"1\",\"trace_" "count\":\"-1\",\"log_frequency\":\"0\",\"trace_file\":\"global_" "unittest.log\",\"trace_mode\":\"triton\"}"; err = client_->GetTraceSettings(&trace_settings, model_name_); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); ASSERT_EQ(trace_settings, initial_settings) << "error: trace settings is not properly initialized for model'" << model_name_ << "'" << std::endl; err = client_->GetTraceSettings(&trace_settings, ""); ASSERT_TRUE(err.IsOk()) << "unable to get default trace settings: " << err.Message(); ASSERT_EQ(trace_settings, initial_settings) << "error: default trace settings is not properly initialized" << std::endl; } std::string model_name_; std::unique_ptr client_; }; class GRPCTraceTest : public ::testing::Test { public: GRPCTraceTest() : model_name_("simple") {} void SetUp() override { std::string url; url = "localhost:8001"; auto err = tc::InferenceServerGrpcClient::Create(&this->client_, url); ASSERT_TRUE(err.IsOk()) << "failed to create GRPC client: " << err.Message(); } // Helper function to convert 'inference::TraceSettingResponse response' to a // string void ConvertResponse( const inference::TraceSettingResponse& response, std::string* str) { *str = response.DebugString(); str->erase(std::remove(str->begin(), str->end(), ' '), str->end()); str->erase(std::remove(str->begin(), str->end(), '\n'), str->end()); } // Helper function to clear all the trace settings to initial state. void TearDown() { tc::Error err = tc::Error::Success; inference::TraceSettingResponse response; std::map> clear_settings = { {"trace_level", {}}, {"trace_rate", {}}, {"trace_count", {}}, {"log_frequency", {}}}; err = client_->UpdateTraceSettings(&response, model_name_, clear_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); err = client_->UpdateTraceSettings(&response, "", clear_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); } // Helper function to make sure the trace setting is properly initialized / // reset before actually running the test case. void CheckServerInitialState() { tc::Error err = tc::Error::Success; inference::TraceSettingResponse response; std::string trace_settings; std::string initial_settings = "settings{key:\"log_frequency\"value{value:\"0\"}}settings{key:\"trace_" "count\"value{value:\"-1\"}}settings{key:\"trace_file\"value{value:" "\"global_unittest.log\"}}settings{key:\"trace_level\"value{value:" "\"TIMESTAMPS\"}}settings{key:\"trace_mode\"value{value:\"triton\"}}" "settings{key:\"trace_rate\"value{value:\"1\"}}"; err = client_->GetTraceSettings(&response, model_name_); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, initial_settings) << "error: trace settings is not properly initialized for model'" << model_name_ << "'" << std::endl; err = client_->GetTraceSettings(&response, ""); ASSERT_TRUE(err.IsOk()) << "unable to get default trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, initial_settings) << "error: default trace settings is not properly initialized" << std::endl; } std::string model_name_; std::unique_ptr client_; }; TYPED_TEST_SUITE_P(ClientTest); TYPED_TEST_P(ClientTest, InferMulti) { tc::Error err = tc::Error::Success; // Create 3 sets of 'options', 'inputs', 'outputs', technically // only InferInput can not be reused for requests that are sent // concurrently, here use distinct objects for all 'options', // 'inputs', and 'outputs' for simplicity. std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // Not swap options.back().model_version_ = "1"; const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()["OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } { auto& expected = expected_outputs.back()["OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; err = this->client_->InferMulti(&results, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, InferMultiDifferentOutputs) { tc::Error err = tc::Error::Success; // Create 3 sets of 'options', 'inputs', 'outputs', technically // only InferInput can not be reused for requests that are sent // concurrently, here use distinct objects for all 'options', // 'inputs', and 'outputs' for simplicity. std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // Not swap options.back().model_version_ = "1"; const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); // Explicitly request different output for different request // 0: request 0 // 1: request 1 // 2: no request (both will be returned) tc::InferRequestedOutput* output; outputs.emplace_back(); expected_outputs.emplace_back(); if (i != 1) { if (i != 2) { err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); } auto& expected = expected_outputs.back()["OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } if (i != 0) { if (i != 2) { err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); } auto& expected = expected_outputs.back()["OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; err = this->client_->InferMulti(&results, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, InferMultiDifferentOptions) { tc::Error err = tc::Error::Success; // Create 3 sets of 'options', 'inputs', 'outputs', technically // only InferInput can not be reused for requests that are sent // concurrently, here use distinct objects for all 'options', // 'inputs', and 'outputs' for simplicity. std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // output will be different based on version // v1 : not swap // v2 : swap // v3 : swap size_t version = (i % 3) + 1; options.back().model_version_ = std::to_string(version); const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()[version == 1 ? "OUTPUT0" : "OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } { auto& expected = expected_outputs.back()[version == 1 ? "OUTPUT1" : "OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; err = this->client_->InferMulti(&results, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, InferMultiOneOption) { // Create only 1 sets of 'options'. tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; options.emplace_back(this->model_name_); // Not swap options.back().model_version_ = "1"; for (size_t i = 0; i < 3; ++i) { const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()["OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } { auto& expected = expected_outputs.back()["OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; err = this->client_->InferMulti(&results, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, InferMultiOneOutput) { // Create only 1 sets of 'outputs', but combine with different 'options' tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // output will be different based on version // v1 : not swap // v2 : swap // v3 : swap size_t version = (i % 3) + 1; options.back().model_version_ = std::to_string(version); const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()["OUTPUT0"]; if (version == 1) { for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } else { for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } } std::vector results; err = this->client_->InferMulti(&results, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, InferMultiNoOutput) { // Not specifying 'outputs' at all, but combine with different 'options' tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // output will be different based on version // v1 : not swap // v2 : swap // v3 : swap size_t version = (i % 3) + 1; options.back().model_version_ = std::to_string(version); const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()[version == 1 ? "OUTPUT0" : "OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } { auto& expected = expected_outputs.back()[version == 1 ? "OUTPUT1" : "OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; err = this->client_->InferMulti(&results, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, InferMultiMismatchOptions) { // Create mismatch number of 'options'. tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; options.emplace_back(this->model_name_); options.emplace_back(this->model_name_); for (size_t i = 0; i < 3; ++i) { const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); } std::vector results; err = this->client_->InferMulti(&results, options, inputs, outputs); ASSERT_FALSE(err.IsOk()) << "Expect InferMulti() to fail"; } TYPED_TEST_P(ClientTest, InferMultiMismatchOutputs) { // Create mismatch number of 'outputs'. tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); if (i != 2) { tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); } } std::vector results; err = this->client_->InferMulti(&results, options, inputs, outputs); ASSERT_FALSE(err.IsOk()) << "Expect InferMulti() to fail"; } TYPED_TEST_P(ClientTest, AsyncInferMulti) { tc::Error err = tc::Error::Success; // Create 3 sets of 'options', 'inputs', 'outputs', technically // only InferInput can not be reused for requests that are sent // concurrently, here use distinct objects for all 'options', // 'inputs', and 'outputs' for simplicity. std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // Not swap options.back().model_version_ = "1"; const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()["OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } { auto& expected = expected_outputs.back()["OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; std::condition_variable cv; std::mutex mu; err = this->client_->AsyncInferMulti( [&results, &cv, &mu](std::vector res) { { std::lock_guard lk(mu); results.swap(res); } cv.notify_one(); }, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); std::unique_lock lk(mu); cv.wait(lk, [this, &results] { return !results.empty(); }); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, AsyncInferMultiDifferentOutputs) { tc::Error err = tc::Error::Success; // Create 3 sets of 'options', 'inputs', 'outputs', technically // only InferInput can not be reused for requests that are sent // concurrently, here use distinct objects for all 'options', // 'inputs', and 'outputs' for simplicity. std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // Not swap options.back().model_version_ = "1"; const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); // Explicitly request different output for different request // 0: request 0 // 1: request 1 // 2: no request (both will be returned) tc::InferRequestedOutput* output; outputs.emplace_back(); expected_outputs.emplace_back(); if (i != 1) { if (i != 2) { err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); } auto& expected = expected_outputs.back()["OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } if (i != 0) { if (i != 2) { err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); } auto& expected = expected_outputs.back()["OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; std::condition_variable cv; std::mutex mu; err = this->client_->AsyncInferMulti( [&results, &cv, &mu](std::vector res) { { std::lock_guard lk(mu); results.swap(res); } cv.notify_one(); }, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); std::unique_lock lk(mu); cv.wait(lk, [this, &results] { return !results.empty(); }); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, AsyncInferMultiDifferentOptions) { tc::Error err = tc::Error::Success; // Create 3 sets of 'options', 'inputs', 'outputs', technically // only InferInput can not be reused for requests that are sent // concurrently, here use distinct objects for all 'options', // 'inputs', and 'outputs' for simplicity. std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // output will be different based on version // v1 : not swap // v2 : swap // v3 : swap size_t version = (i % 3) + 1; options.back().model_version_ = std::to_string(version); const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()[version == 1 ? "OUTPUT0" : "OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } { auto& expected = expected_outputs.back()[version == 1 ? "OUTPUT1" : "OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; std::condition_variable cv; std::mutex mu; err = this->client_->AsyncInferMulti( [&results, &cv, &mu](std::vector res) { { std::lock_guard lk(mu); results.swap(res); } cv.notify_one(); }, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); std::unique_lock lk(mu); cv.wait(lk, [this, &results] { return !results.empty(); }); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, AsyncInferMultiOneOption) { // Create only 1 sets of 'options'. tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; options.emplace_back(this->model_name_); // Not swap options.back().model_version_ = "1"; for (size_t i = 0; i < 3; ++i) { const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()["OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } { auto& expected = expected_outputs.back()["OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; std::condition_variable cv; std::mutex mu; err = this->client_->AsyncInferMulti( [&results, &cv, &mu](std::vector res) { { std::lock_guard lk(mu); results.swap(res); } cv.notify_one(); }, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); std::unique_lock lk(mu); cv.wait(lk, [this, &results] { return !results.empty(); }); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, AsyncInferMultiOneOutput) { // Create only 1 sets of 'outputs', but combine with different 'options' tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // output will be different based on version // v1 : not swap // v2 : swap // v3 : swap size_t version = (i % 3) + 1; options.back().model_version_ = std::to_string(version); const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()["OUTPUT0"]; if (version == 1) { for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } else { for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } } std::vector results; std::condition_variable cv; std::mutex mu; err = this->client_->AsyncInferMulti( [&results, &cv, &mu](std::vector res) { { std::lock_guard lk(mu); results.swap(res); } cv.notify_one(); }, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); std::unique_lock lk(mu); cv.wait(lk, [this, &results] { return !results.empty(); }); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, AsyncInferMultiNoOutput) { // Not specifying 'outputs' at all, but combine with different 'options' tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); // output will be different based on version // v1 : not swap // v2 : swap // v3 : swap size_t version = (i % 3) + 1; options.back().model_version_ = std::to_string(version); const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); expected_outputs.emplace_back(); { auto& expected = expected_outputs.back()[version == 1 ? "OUTPUT0" : "OUTPUT1"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] + input_1[i]); } } { auto& expected = expected_outputs.back()[version == 1 ? "OUTPUT1" : "OUTPUT0"]; for (size_t i = 0; i < 16; ++i) { expected.emplace_back(input_0[i] - input_1[i]); } } } std::vector results; std::condition_variable cv; std::mutex mu; err = this->client_->AsyncInferMulti( [&results, &cv, &mu](std::vector res) { { std::lock_guard lk(mu); results.swap(res); } cv.notify_one(); }, options, inputs, outputs); ASSERT_TRUE(err.IsOk()) << "failed to perform multiple inferences: " << err.Message(); std::unique_lock lk(mu); cv.wait(lk, [this, &results] { return !results.empty(); }); EXPECT_NO_FATAL_FAILURE(this->ValidateOutput(results, expected_outputs)); } TYPED_TEST_P(ClientTest, AsyncInferMultiMismatchOptions) { // Create mismatch number of 'options'. tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; options.emplace_back(this->model_name_); options.emplace_back(this->model_name_); for (size_t i = 0; i < 3; ++i) { const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); } std::vector results; std::condition_variable cv; std::mutex mu; err = this->client_->AsyncInferMulti( [&results, &cv, &mu](std::vector res) { { std::lock_guard lk(mu); results.swap(res); } cv.notify_one(); }, options, inputs, outputs); ASSERT_FALSE(err.IsOk()) << "Expect AsyncInferMulti() to fail"; } TYPED_TEST_P(ClientTest, AsyncInferMultiMismatchOutputs) { // Create mismatch number of 'outputs'. tc::Error err = tc::Error::Success; std::vector options; std::vector> inputs; std::vector> outputs; std::vector>> expected_outputs; for (size_t i = 0; i < 3; ++i) { options.emplace_back(this->model_name_); const auto& input_0 = this->input_data_[i % this->input_data_.size()]; const auto& input_1 = this->input_data_[(i + 1) % this->input_data_.size()]; inputs.emplace_back(); err = this->PrepareInputs(input_0, input_1, &inputs.back()); if (i != 2) { tc::InferRequestedOutput* output; outputs.emplace_back(); err = tc::InferRequestedOutput::Create(&output, "OUTPUT0"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); err = tc::InferRequestedOutput::Create(&output, "OUTPUT1"); ASSERT_TRUE(err.IsOk()) << "failed to create inference output: " << err.Message(); outputs.back().emplace_back(output); } } std::vector results; std::condition_variable cv; std::mutex mu; err = this->client_->AsyncInferMulti( [&results, &cv, &mu](std::vector res) { { std::lock_guard lk(mu); results.swap(res); } cv.notify_one(); }, options, inputs, outputs); ASSERT_FALSE(err.IsOk()) << "Expect AsyncInferMulti() to fail"; } #if 0 TYPED_TEST_P(ClientTest, LoadWithFileOverride) { std::vector content; { std::string path("unit_test_models/onnx_int32_int32_int32/3/model.onnx"); std::ifstream in(path, std::ios::in | std::ios::binary); if (!in) { ASSERT_TRUE(false) << "failed to open file for testing"; } in.seekg(0, std::ios::end); content = std::vector(in.tellg()); in.seekg(0, std::ios::beg); in.read(content.data(), content.size()); in.close(); } std::string config("{\"backend\":\"onnxruntime\"}"); std::string model_name("onnx_int32_int32_int32"); std::string override_name("override_model"); std::vector> expected_version_ready{ {"1", false}, {"3", true}}; std::vector> expected_override_version_ready{ {"1", true}, {"3", false}}; tc::Error err = tc::Error::Success; for (const auto& vr : expected_version_ready) { bool ready = false; err = this->client_->IsModelReady(&ready, model_name, vr.first); ASSERT_TRUE(err.IsOk()) << "failed to get version readiness: " << err.Message(); ASSERT_EQ(ready, vr.second) << "expect model " << model_name << " version " << vr.first << " readiness: " << vr.second; } // Request to load the model with override file, should fail // without providing override config. The config requirement // serves as an reminder that the existing model directory will // not be used. err = this->LoadModel( model_name, std::string(), {{"file:1/model.onnx", content}}); ASSERT_FALSE(err.IsOk()) << "Expect LoadModel() to fail"; // Sanity check that the model is unchanged for (const auto& vr : expected_version_ready) { bool ready = false; err = this->client_->IsModelReady(&ready, model_name, vr.first); ASSERT_TRUE(err.IsOk()) << "failed to get version readiness: " << err.Message(); ASSERT_EQ(ready, vr.second) << "expect model " << model_name << " version " << vr.first << " readiness: " << vr.second; } // Request to load the model with override file and config in // a different name err = this->LoadModel(override_name, config, {{"file:1/model.onnx", content}}); ASSERT_TRUE(err.IsOk()) << "Expect LoadModel() succeed: " << err.Message(); // Sanity check that the model with original name is unchanged for (const auto& vr : expected_version_ready) { bool ready = false; err = this->client_->IsModelReady(&ready, model_name, vr.first); ASSERT_TRUE(err.IsOk()) << "failed to get version readiness: " << err.Message(); ASSERT_EQ(ready, vr.second) << "expect model " << model_name << " version " << vr.first << " readiness: " << vr.second; } // Check override model readiness for (const auto& vr : expected_override_version_ready) { bool ready = false; err = this->client_->IsModelReady(&ready, override_name, vr.first); ASSERT_TRUE(err.IsOk()) << "failed to get version readiness: " << err.Message(); ASSERT_EQ(ready, vr.second) << "expect model " << override_name << " version " << vr.first << " readiness: " << vr.second; } // Request to load the model with override file and config in // original name err = this->LoadModel(model_name, config, {{"file:1/model.onnx", content}}); ASSERT_TRUE(err.IsOk()) << "Expect LoadModel() succeed: " << err.Message(); // check that the model with original name is changed for (const auto& vr : expected_override_version_ready) { bool ready = false; err = this->client_->IsModelReady(&ready, model_name, vr.first); ASSERT_TRUE(err.IsOk()) << "failed to get version readiness: " << err.Message(); ASSERT_EQ(ready, vr.second) << "expect model " << model_name << " version " << vr.first << " readiness: " << vr.second; } // Sanity check readiness of the different named model for (const auto& vr : expected_override_version_ready) { bool ready = false; err = this->client_->IsModelReady(&ready, override_name, vr.first); ASSERT_TRUE(err.IsOk()) << "failed to get version readiness: " << err.Message(); ASSERT_EQ(ready, vr.second) << "expect model " << override_name << " version " << vr.first << " readiness: " << vr.second; } } TYPED_TEST_P(ClientTest, LoadWithConfigOverride) { // Request to load the model with override config std::string model_name("onnx_int32_int32_int32"); std::vector> original_version_ready{ {"2", true}, {"3", true}}; std::vector> expected_version_ready{ {"2", true}, {"3", false}}; tc::Error err = tc::Error::Success; // Send the config with wrong format std::string config( "\"parameters\": {\"config\": {{\"backend\":\"onnxruntime\", " "\"version_policy\":{\"specific\":{\"versions\":[2]}}}}}"); err = this->LoadModel(model_name, config); ASSERT_FALSE(err.IsOk()) << "Expect LoadModel() to fail"; // The model should not be changed after a failed LoadModel request for (const auto& vr : original_version_ready) { bool ready = false; err = this->client_->IsModelReady(&ready, model_name, vr.first); ASSERT_TRUE(err.IsOk()) << "failed to get version readiness: " << err.Message(); ASSERT_EQ(ready, vr.second) << "expect model " << model_name << " version " << vr.first << " readiness: " << vr.second; } // Send the config with correct format config = "{\"backend\":\"onnxruntime\", " "\"version_policy\":{\"specific\":{\"versions\":[2]}}}"; err = this->LoadModel(model_name, config); // The model should be changed after a successful LoadModel request for (const auto& vr : expected_version_ready) { bool ready = false; err = this->client_->IsModelReady(&ready, model_name, vr.first); ASSERT_TRUE(err.IsOk()) << "failed to get version readiness: " << err.Message(); ASSERT_EQ(ready, vr.second) << "expect model " << model_name << " version " << vr.first << " readiness: " << vr.second; } } #endif #if 0 TEST_F(HTTPTraceTest, HTTPUpdateTraceSettings) { // Update model and global trace settings in order, and expect the global // trace settings will only reflect to the model setting fields that haven't // been specified. tc::Error err = tc::Error::Success; std::string trace_settings; EXPECT_NO_FATAL_FAILURE(this->TearDown()); EXPECT_NO_FATAL_FAILURE(this->CheckServerInitialState()); std::string expected_first_model_settings = "{\"trace_level\":[\"TIMESTAMPS\"],\"trace_rate\":\"1\",\"trace_count\":" "\"-1\",\"log_frequency\":\"0\",\"trace_file\":\"global_unittest.log\"," "\"trace_" "mode\":\"triton\"}"; std::string expected_second_model_settings = "{\"trace_level\":[\"TIMESTAMPS\",\"TENSORS\"],\"trace_rate\":\"1\"," "\"trace_count\":\"-1\",\"log_frequency\":\"0\",\"trace_file\":\"global_" "unittest." "log\",\"trace_mode\":\"triton\"}"; std::string expected_global_settings = "{\"trace_level\":[\"TIMESTAMPS\",\"TENSORS\"],\"trace_rate\":\"1\"," "\"trace_count\":\"-1\",\"log_frequency\":\"0\",\"trace_file\":\"global_" "unittest." "log\",\"trace_mode\":\"triton\"}"; std::map> model_update_settings = { {"trace_file", {"model.log"}}}; std::map> global_update_settings = { {"trace_level", {"TIMESTAMPS", "TENSORS"}}}; err = this->client_->UpdateTraceSettings( &trace_settings, this->model_name_, model_update_settings); ASSERT_FALSE(err.IsOk()) << "update disabled settings: trace_file"; ASSERT_TRUE( err.Message() == "trace file location can not be updated through network protocol") << "error: Unexpected error message: " << err.Message(); err = this->client_->GetTraceSettings(&trace_settings, this->model_name_); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); ASSERT_EQ(trace_settings, expected_first_model_settings) << "error: Unexpected updated model trace settings" << std::endl; // Note that 'trace_level' may be mismatch due to the order of the levels // listed, currently we assume the order is the same for simplicity. But the // order shouldn't be enforced and this checking needs to be improved when // this kind of failure is reported err = this->client_->UpdateTraceSettings( &trace_settings, "", global_update_settings); ASSERT_EQ(trace_settings, expected_global_settings) << "error: Unexpected updated global trace settings" << std::endl; err = this->client_->GetTraceSettings(&trace_settings, this->model_name_); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); ASSERT_EQ(trace_settings, expected_second_model_settings) << "error: Unexpected model trace settings after global update" << std::endl; } TEST_F(HTTPTraceTest, HTTPClearTraceSettings) { // Clear global and model trace settings in order, and expect the default / // global trace settings are propagated properly. tc::Error err = tc::Error::Success; std::string trace_settings; EXPECT_NO_FATAL_FAILURE(this->TearDown()); EXPECT_NO_FATAL_FAILURE(this->CheckServerInitialState()); // First set up the model / global trace setting that: model 'simple' has // 'trace_rate' and 'log_frequency' specified global has 'trace_level', // 'trace_count' and 'trace_rate' specified std::map> model_update_settings = { {"trace_rate", {"12"}}, {"log_frequency", {"34"}}}; std::map> global_update_settings = { {"trace_rate", {"56"}}, {"trace_count", {"78"}}, {"trace_level", {"OFF"}}}; err = this->client_->UpdateTraceSettings( &trace_settings, "", global_update_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); err = this->client_->UpdateTraceSettings( &trace_settings, this->model_name_, model_update_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); std::string expected_global_settings = "{\"trace_level\":[\"OFF\"],\"trace_rate\":\"1\",\"trace_count\":\"-1\"," "\"log_frequency\":\"0\",\"trace_file\":\"global_unittest.log\",\"trace_" "mode\":\"triton\"}"; std::string expected_first_model_settings = "{\"trace_level\":[\"OFF\"],\"trace_rate\":\"12\",\"trace_count\":\"-1\"," "\"log_frequency\":\"34\",\"trace_file\":\"global_unittest.log\",\"trace_" "mode\":\"triton\"}"; std::string expected_second_model_settings = "{\"trace_level\":[\"OFF\"],\"trace_rate\":\"1\",\"trace_count\":\"-1\"," "\"log_frequency\":\"34\",\"trace_file\":\"global_unittest.log\",\"trace_" "mode\":\"triton\"}"; std::map> global_clear_settings = { {"trace_rate", {}}, {"trace_count", {}}}; std::map> model_clear_settings = { {"trace_rate", {}}, {"trace_level", {}}}; // Clear global err = this->client_->UpdateTraceSettings( &trace_settings, "", global_clear_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); ASSERT_EQ(trace_settings, expected_global_settings) << "error: Unexpected updated global trace settings" << std::endl; err = this->client_->GetTraceSettings(&trace_settings, this->model_name_); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); ASSERT_EQ(trace_settings, expected_first_model_settings) << "error: Unexpected model trace settings after global clear" << std::endl; // Clear model err = this->client_->UpdateTraceSettings( &trace_settings, this->model_name_, model_clear_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); ASSERT_EQ(trace_settings, expected_second_model_settings) << "error: Unexpected model trace settings after model clear" << std::endl; err = this->client_->GetTraceSettings(&trace_settings, ""); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); ASSERT_EQ(trace_settings, expected_global_settings) << "error: Unexpected global trace settings after model clear" << std::endl; } TEST_F(GRPCTraceTest, GRPCUpdateTraceSettings) { // Update model and global trace settings in order, and expect the global // trace settings will only reflect to the model setting fields that haven't // been specified. tc::Error err = tc::Error::Success; inference::TraceSettingResponse response; std::string trace_settings; EXPECT_NO_FATAL_FAILURE(this->TearDown()); EXPECT_NO_FATAL_FAILURE(this->CheckServerInitialState()); std::string expected_first_model_settings = "settings{key:\"log_frequency\"value{value:\"0\"}}settings{key:\"trace_" "count\"value{value:\"-1\"}}settings{key:\"trace_file\"value{value:" "\"global_unittest.log\"}}settings{key:\"trace_level\"value{value:" "\"TIMESTAMPS\"}}" "settings{key:\"trace_mode\"value{value:\"triton\"}}" "settings{key:\"trace_rate\"value{value:\"1\"}}"; std::string expected_second_model_settings = "settings{key:\"log_frequency\"value{value:\"0\"}}settings{key:\"trace_" "count\"value{value:\"-1\"}}settings{key:\"trace_file\"value{value:" "\"global_unittest.log\"}}settings{key:\"trace_level\"value{value:" "\"TIMESTAMPS\"value:\"TENSORS\"}}settings{key:\"trace_mode\"value{value:" "\"triton\"}}" "settings{key:\"trace_rate\"value{value:\"1\"}}"; std::string expected_global_settings = "settings{key:\"log_frequency\"value{value:\"0\"}}settings{key:\"trace_" "count\"value{value:\"-1\"}}settings{key:\"trace_file\"value{value:" "\"global_unittest.log\"}}settings{key:\"trace_level\"value{value:" "\"TIMESTAMPS\"value:\"TENSORS\"}}settings{key:\"trace_mode\"value{value:" "\"triton\"}}" "settings{key:\"trace_rate\"value{value:\"1\"}}"; std::map> model_update_settings = { {"trace_file", {"model.log"}}}; std::map> global_update_settings = { {"trace_level", {"TIMESTAMPS", "TENSORS"}}}; err = this->client_->UpdateTraceSettings( &response, this->model_name_, model_update_settings); ASSERT_FALSE(err.IsOk()) << "update disabled settings: trace_file"; ASSERT_TRUE( err.Message() == "trace file location can not be updated through network protocol") << "error: Unexpected error message: " << err.Message(); err = this->client_->GetTraceSettings(&response, this->model_name_); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, expected_first_model_settings) << "error: Unexpected updated model trace settings" << std::endl; // Note that 'trace_level' may be mismatch due to the order of the levels // listed, currently we assume the order is the same for simplicity. But the // order shouldn't be enforced and this checking needs to be improved when // this kind of failure is reported err = this->client_->UpdateTraceSettings(&response, "", global_update_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, expected_global_settings) << "error: Unexpected updated global trace settings" << std::endl; err = this->client_->GetTraceSettings(&response, this->model_name_); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, expected_second_model_settings) << "error: Unexpected model trace settings after global update" << std::endl; } TEST_F(GRPCTraceTest, GRPCClearTraceSettings) { // Clear global and model trace settings in order, and expect the default / // global trace settings are propagated properly. tc::Error err = tc::Error::Success; inference::TraceSettingResponse response; std::string trace_settings; EXPECT_NO_FATAL_FAILURE(this->TearDown()); EXPECT_NO_FATAL_FAILURE(this->CheckServerInitialState()); // First set up the model / global trace setting that: model 'simple' has // 'trace_rate' and 'log_frequency' specified global has 'trace_level', // 'trace_count' and 'trace_rate' specified std::map> model_update_settings = { {"trace_rate", {"12"}}, {"log_frequency", {"34"}}}; std::map> global_update_settings = { {"trace_rate", {"56"}}, {"trace_count", {"78"}}, {"trace_level", {"OFF"}}}; err = this->client_->UpdateTraceSettings(&response, "", global_update_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); err = this->client_->UpdateTraceSettings( &response, this->model_name_, model_update_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); std::string expected_global_settings = "settings{key:\"log_frequency\"value{value:\"0\"}}settings{key:\"trace_" "count\"value{value:\"-1\"}}settings{key:\"trace_file\"value{value:" "\"global_unittest.log\"}}settings{key:\"trace_level\"value{value:" "\"OFF\"}}settings{key:\"trace_mode\"value{value:\"triton\"}}" "settings{key:\"trace_rate\"value{value:\"1\"}}"; std::string expected_first_model_settings = "settings{key:\"log_frequency\"value{value:\"34\"}}settings{key:\"trace_" "count\"value{value:\"-1\"}}settings{key:\"trace_file\"value{value:" "\"global_unittest.log\"}}settings{key:\"trace_level\"value{value:" "\"OFF\"}}settings{key:\"trace_mode\"value{value:\"triton\"}}" "settings{key:\"trace_rate\"value{value:\"12\"}}"; std::string expected_second_model_settings = "settings{key:\"log_frequency\"value{value:\"34\"}}settings{key:\"trace_" "count\"value{value:\"-1\"}}settings{key:\"trace_file\"value{value:" "\"global_unittest.log\"}}settings{key:\"trace_level\"value{value:" "\"OFF\"}}settings{key:\"trace_mode\"value{value:\"triton\"}}" "settings{key:\"trace_rate\"value{value:\"1\"}}"; std::map> global_clear_settings = { {"trace_rate", {}}, {"trace_count", {}}}; std::map> model_clear_settings = { {"trace_rate", {}}, {"trace_level", {}}}; // Clear global err = this->client_->UpdateTraceSettings(&response, "", global_clear_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, expected_global_settings) << "error: Unexpected updated global trace settings" << std::endl; err = client_->GetTraceSettings(&response, this->model_name_); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, expected_first_model_settings) << "error: Unexpected model trace settings after global clear" << std::endl; // Clear model err = this->client_->UpdateTraceSettings( &response, this->model_name_, model_clear_settings); ASSERT_TRUE(err.IsOk()) << "unable to update trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, expected_second_model_settings) << "error: Unexpected model trace settings after model clear" << std::endl; err = client_->GetTraceSettings(&response, ""); ASSERT_TRUE(err.IsOk()) << "unable to get trace settings: " << err.Message(); EXPECT_NO_FATAL_FAILURE(ConvertResponse(response, &trace_settings)); ASSERT_EQ(trace_settings, expected_global_settings) << "error: Unexpected global trace settings after model clear" << std::endl; } #endif class TestHttpInferRequest : public tc::HttpInferRequest { public: tc::Error ConvertBinaryInputsToJSON( tc::InferInput& input, triton::common::TritonJson::Value& data_json) const override { return tc::HttpInferRequest::ConvertBinaryInputsToJSON(input, data_json); } tc::Error ConvertBinaryInputToJSON( const uint8_t* buf, const size_t element_count, const std::string& datatype, triton::common::TritonJson::Value& data_json) const override { return tc::HttpInferRequest::ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); } }; class HTTPJSONDataTest : public ::testing::Test {}; TEST_F(HTTPJSONDataTest, ConvertBinaryInputsToJSON) { // This tests the HttpInferRequest::ConvertBinaryInputsToJSON() function, // which basically cycles through all the inputs that added to an InferInput // via InferInput::AppendRaw(). This test confirms that an InferInput with two // calls to InferInput::AppendRaw() correctly has all contents from the // AppendRaw() calls correctly converted into a flattened JSON array. TestHttpInferRequest test_http_infer_request{}; tc::InferInput* input{}; tc::InferInput::Create(&input, "INPUT", {1, 2, 2}, "INT32"); int32_t input_raw_1[4] = {1, 3, 5, 7}; size_t input_raw_byte_size_1 = sizeof(input_raw_1); int32_t input_raw_2[4] = {2, 4, 6, 8}; size_t input_raw_byte_size_2 = sizeof(input_raw_2); input->AppendRaw( reinterpret_cast(input_raw_1), input_raw_byte_size_1); input->AppendRaw( reinterpret_cast(input_raw_2), input_raw_byte_size_2); triton::common::TritonJson::Value data_json( triton::common::TritonJson::ValueType::ARRAY); tc::Error err{ test_http_infer_request.ConvertBinaryInputsToJSON(*input, data_json)}; EXPECT_TRUE(err.IsOk()); EXPECT_TRUE(data_json.ArraySize() == 8); int64_t value{0}; data_json.IndexAsInt(0, &value); EXPECT_TRUE(value == 1); data_json.IndexAsInt(1, &value); EXPECT_TRUE(value == 3); data_json.IndexAsInt(2, &value); EXPECT_TRUE(value == 5); data_json.IndexAsInt(3, &value); EXPECT_TRUE(value == 7); data_json.IndexAsInt(4, &value); EXPECT_TRUE(value == 2); data_json.IndexAsInt(5, &value); EXPECT_TRUE(value == 4); data_json.IndexAsInt(6, &value); EXPECT_TRUE(value == 6); data_json.IndexAsInt(7, &value); EXPECT_TRUE(value == 8); } TEST_F(HTTPJSONDataTest, ConvertBinaryInputToJSON) { // This tests the HttpInferRequest::ConvertBinaryInputToJSON() function, // which converts one binary buffer into a corresponding JSON array of // specified data type. This test tests each valid and invalid data type. TestHttpInferRequest test_http_infer_request{}; tc::Error err{}; const uint8_t* buf{nullptr}; size_t element_count{2}; std::string datatype{""}; triton::common::TritonJson::Value data_json{}; // BOOL datatype = "BOOL"; std::array bool_array({false, true}); buf = reinterpret_cast(bool_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == bool_array.size()); bool bool_value{false}; data_json.IndexAsBool(0, &bool_value); EXPECT_TRUE(bool_value == bool_array[0]); data_json.IndexAsBool(1, &bool_value); EXPECT_TRUE(bool_value == bool_array[1]); // UINT8 datatype = "UINT8"; std::array uint8_array({1, UINT8_MAX}); buf = reinterpret_cast(uint8_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == uint8_array.size()); uint64_t uint8_value{0}; data_json.IndexAsUInt(0, &uint8_value); EXPECT_TRUE(uint8_value == uint8_array[0]); data_json.IndexAsUInt(1, &uint8_value); EXPECT_TRUE(uint8_value == uint8_array[1]); // UINT16 datatype = "UINT16"; std::array uint16_array({1, UINT16_MAX}); buf = reinterpret_cast(uint16_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == uint16_array.size()); uint64_t uint16_value{0}; data_json.IndexAsUInt(0, &uint16_value); EXPECT_TRUE(uint16_value == uint16_array[0]); data_json.IndexAsUInt(1, &uint16_value); EXPECT_TRUE(uint16_value == uint16_array[1]); // UINT32 datatype = "UINT32"; std::array uint32_array({1, UINT32_MAX}); buf = reinterpret_cast(uint32_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == uint32_array.size()); uint64_t uint32_value{0}; data_json.IndexAsUInt(0, &uint32_value); EXPECT_TRUE(uint32_value == uint32_array[0]); data_json.IndexAsUInt(1, &uint32_value); EXPECT_TRUE(uint32_value == uint32_array[1]); // UINT64 datatype = "UINT64"; std::array uint64_array({1, UINT64_MAX}); buf = reinterpret_cast(uint64_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == uint64_array.size()); uint64_t uint64_value{0}; data_json.IndexAsUInt(0, &uint64_value); EXPECT_TRUE(uint64_value == uint64_array[0]); data_json.IndexAsUInt(1, &uint64_value); EXPECT_TRUE(uint64_value == uint64_array[1]); // INT8 datatype = "INT8"; std::array int8_array({INT8_MIN, INT8_MAX}); buf = reinterpret_cast(int8_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == int8_array.size()); int64_t int8_value{0}; data_json.IndexAsInt(0, &int8_value); EXPECT_TRUE(int8_value == int8_array[0]); data_json.IndexAsInt(1, &int8_value); EXPECT_TRUE(int8_value == int8_array[1]); // INT16 datatype = "INT16"; std::array int16_array({INT16_MIN, INT16_MAX}); buf = reinterpret_cast(int16_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == int16_array.size()); int64_t int16_value{0}; data_json.IndexAsInt(0, &int16_value); EXPECT_TRUE(int16_value == int16_array[0]); data_json.IndexAsInt(1, &int16_value); EXPECT_TRUE(int16_value == int16_array[1]); // INT32 datatype = "INT32"; std::array int32_array({INT32_MIN, INT32_MAX}); buf = reinterpret_cast(int32_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == int32_array.size()); int64_t int32_value{0}; data_json.IndexAsInt(0, &int32_value); EXPECT_TRUE(int32_value == int32_array[0]); data_json.IndexAsInt(1, &int32_value); EXPECT_TRUE(int32_value == int32_array[1]); // INT64 datatype = "INT64"; std::array int64_array({INT64_MIN, INT64_MAX}); buf = reinterpret_cast(int64_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == int64_array.size()); int64_t int64_value{0}; data_json.IndexAsInt(0, &int64_value); EXPECT_TRUE(int64_value == int64_array[0]); data_json.IndexAsInt(1, &int64_value); EXPECT_TRUE(int64_value == int64_array[1]); // FP16 - invalid data type datatype = "FP16"; err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == false); // FP32 datatype = "FP32"; std::array fp32_array({-1000.0, 1000.0}); buf = reinterpret_cast(fp32_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == fp32_array.size()); double fp32_value{0.0}; data_json.IndexAsDouble(0, &fp32_value); EXPECT_NEAR(fp32_value, fp32_array[0], 1.0); data_json.IndexAsDouble(1, &fp32_value); EXPECT_NEAR(fp32_value, fp32_array[1], 1.0); // FP64 datatype = "FP64"; std::array fp64_array({-1000.0, 1000.0}); buf = reinterpret_cast(fp64_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == fp64_array.size()); double fp64_value{0.0}; data_json.IndexAsDouble(0, &fp64_value); EXPECT_NEAR(fp64_value, fp64_array[0], 1.0); data_json.IndexAsDouble(1, &fp64_value); EXPECT_NEAR(fp64_value, fp64_array[1], 1.0); // BYTES datatype = "BYTES"; std::array bytes_array( {2, 0, 0, 0, 1, INT8_MAX, 2, 0, 0, 0, 2, INT8_MAX}); buf = reinterpret_cast(bytes_array.data()); data_json = triton::common::TritonJson::Value( triton::common::TritonJson::ValueType::ARRAY); err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(data_json.ArraySize() == 2); const char* bytes_value{nullptr}; size_t bytes_len{0}; data_json.IndexAsString(0, &bytes_value, &bytes_len); EXPECT_TRUE(bytes_len == 2); EXPECT_TRUE(bytes_value[0] == bytes_array[4]); EXPECT_TRUE(bytes_value[1] == bytes_array[5]); data_json.IndexAsString(1, &bytes_value, &bytes_len); EXPECT_TRUE(bytes_len == 2); EXPECT_TRUE(bytes_value[0] == bytes_array[10]); EXPECT_TRUE(bytes_value[1] == bytes_array[11]); // BF16 - invalid data type datatype = "BF16"; err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == false); // invaliddatatype - invalid data type datatype = "invaliddatatype"; err = test_http_infer_request.ConvertBinaryInputToJSON( buf, element_count, datatype, data_json); EXPECT_TRUE(err.IsOk() == false); } class TestInferResultHttp : public tc::InferResultHttp { public: TestInferResultHttp() {} tc::Error ConvertJSONOutputToBinary( triton::common::TritonJson::Value& data_json, const std::string& datatype, const uint8_t** buf, size_t* buf_size) const override { return tc::InferResultHttp::ConvertJSONOutputToBinary( data_json, datatype, buf, buf_size); } }; TEST_F(HTTPJSONDataTest, ConvertJSONOutputToBinary) { // This tests the InferResultHttp::ConvertJSONOutputToBinary() function, // which converts one JSON array into a binary buffer of specified data type. // This test tests each valid and invalid data type. TestInferResultHttp test_infer_result_http{}; tc::Error err{}; triton::common::TritonJson::Value data_json{}; std::string datatype{""}; const uint8_t* buf{nullptr}; size_t buf_size{0}; // BOOL datatype = "BOOL"; data_json.Parse(R"([false, true])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(bool) * 2); EXPECT_TRUE(reinterpret_cast(buf)[0] == false); EXPECT_TRUE(reinterpret_cast(buf)[1] == true); // UINT8 datatype = "UINT8"; data_json.Parse(R"([1, 255])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(uint8_t) * 2); EXPECT_TRUE(reinterpret_cast(buf)[0] == 1); EXPECT_TRUE(reinterpret_cast(buf)[1] == 255); // UINT16 datatype = "UINT16"; data_json.Parse(R"([1, 65535])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(uint16_t) * 2); EXPECT_TRUE(reinterpret_cast(buf)[0] == 1); EXPECT_TRUE(reinterpret_cast(buf)[1] == 65535); // UINT32 datatype = "UINT32"; data_json.Parse(R"([1, 4294967295])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(uint32_t) * 2); EXPECT_TRUE(reinterpret_cast(buf)[0] == 1); EXPECT_TRUE(reinterpret_cast(buf)[1] == 4294967295); // UINT64 datatype = "UINT64"; data_json.Parse(R"([1, 18446744073709551615])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(uint64_t) * 2); EXPECT_TRUE(reinterpret_cast(buf)[0] == 1); EXPECT_TRUE( reinterpret_cast(buf)[1] == 18446744073709551615ULL); // INT8 datatype = "INT8"; data_json.Parse(R"([-128, 127])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(int8_t) * 2); EXPECT_TRUE(reinterpret_cast(buf)[0] == -128); EXPECT_TRUE(reinterpret_cast(buf)[1] == 127); // INT16 datatype = "INT16"; data_json.Parse(R"([-32768, 32767])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(int16_t) * 2); EXPECT_TRUE(reinterpret_cast(buf)[0] == -32768); EXPECT_TRUE(reinterpret_cast(buf)[1] == 32767); // INT32 datatype = "INT32"; data_json.Parse(R"([-2147483648, 2147483647])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(int32_t) * 2); EXPECT_TRUE(reinterpret_cast(buf)[0] == -2147483648); EXPECT_TRUE(reinterpret_cast(buf)[1] == 2147483647); // INT64 datatype = "INT64"; data_json.Parse(R"([-9223372036854775808, 9223372036854775807])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(int64_t) * 2); EXPECT_TRUE( reinterpret_cast(buf)[0] == -9223372036854775807L - 1); EXPECT_TRUE( reinterpret_cast(buf)[1] == 9223372036854775807LL); // FP16 - invalid data type datatype = "FP16"; err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == false); // FP32 datatype = "FP32"; data_json.Parse(R"([-1000.0, 1000.0])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(float) * 2); EXPECT_NEAR(reinterpret_cast(buf)[0], -1000.0, 1.0); EXPECT_NEAR(reinterpret_cast(buf)[1], 1000.0, 1.0); // FP64 datatype = "FP64"; data_json.Parse(R"([-1000.0, 1000.0])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == sizeof(double) * 2); EXPECT_NEAR(reinterpret_cast(buf)[0], -1000.0, 1.0); EXPECT_NEAR(reinterpret_cast(buf)[1], 1000.0, 1.0); // BYTES datatype = "BYTES"; data_json.Parse(R"(["\u0001\u007F", "\u0002\u007F"])"); err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == true); EXPECT_TRUE(buf_size == 12); EXPECT_TRUE(*reinterpret_cast(buf) == 2); EXPECT_TRUE(reinterpret_cast(buf)[4] == 1); EXPECT_TRUE(reinterpret_cast(buf)[5] == 127); EXPECT_TRUE(*reinterpret_cast(buf + 6) == 2); EXPECT_TRUE(reinterpret_cast(buf)[10] == 2); EXPECT_TRUE(reinterpret_cast(buf)[11] == 127); // BF16 - invalid data type datatype = "BF16"; err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == false); // invaliddatatype - invalid data type datatype = "invaliddatatype"; err = test_infer_result_http.ConvertJSONOutputToBinary( data_json, datatype, &buf, &buf_size); EXPECT_TRUE(err.IsOk() == false); } REGISTER_TYPED_TEST_SUITE_P( ClientTest, InferMulti, InferMultiDifferentOutputs, InferMultiDifferentOptions, InferMultiOneOption, InferMultiOneOutput, InferMultiNoOutput, InferMultiMismatchOptions, InferMultiMismatchOutputs, AsyncInferMulti, AsyncInferMultiDifferentOutputs, AsyncInferMultiDifferentOptions, AsyncInferMultiOneOption, AsyncInferMultiOneOutput, AsyncInferMultiNoOutput, AsyncInferMultiMismatchOptions, AsyncInferMultiMismatchOutputs/*, LoadWithFileOverride, LoadWithConfigOverride*/); INSTANTIATE_TYPED_TEST_SUITE_P(GRPC, ClientTest, tc::InferenceServerGrpcClient); INSTANTIATE_TYPED_TEST_SUITE_P(HTTP, ClientTest, tc::InferenceServerHttpClient); } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/client/src/c++/tests/client_timeout_test.cc000066400000000000000000000420731527052371200240140ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include "grpc_client.h" #include "http_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } #define COUNT_ERROR_MSGS(X, MSG, MSG_SUCESS, CNT) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cout << "error: " << (MSG) << ": " << err << std::endl; \ ++CNT; \ } else { \ std::cout << "OK: " << (MSG_SUCESS) << ": " << err << std::endl; \ } \ } namespace { void TestTimeoutAPIs( const uint64_t timeout_ms, const std::string& model_name, std::unique_ptr& grpc_client) { std::cout << "testing other apis" << std::endl; bool success = false; std::map headers; inference::ServerMetadataResponse server_metadata; inference::ModelMetadataResponse model_metadata; inference::ModelConfigResponse model_config; inference::RepositoryIndexResponse repository_index; inference::ModelStatisticsResponse infer_stat; inference::TraceSettingResponse response; std::map> settings; inference::TraceSettingResponse trace_settings; inference::SystemSharedMemoryStatusResponse shmstatus; size_t byte_size; std::string memory_name = ""; inference::CudaSharedMemoryStatusResponse cuda_shmstatus; cudaIpcMemHandle_t cuda_shm_handle; size_t count = 0; COUNT_ERROR_MSGS( grpc_client->IsServerLive(&success, headers, timeout_ms), "Failed on IsServerLive", "Success on IsServerLive", count); COUNT_ERROR_MSGS( grpc_client->IsServerReady(&success, headers, timeout_ms), "Failed on IsServerReady", "Success on IsServerReady", count); COUNT_ERROR_MSGS( grpc_client->IsModelReady(&success, model_name, "", headers, timeout_ms), "Failed on IsModelReady", "Success on IsModelReady", count); COUNT_ERROR_MSGS( grpc_client->ServerMetadata(&server_metadata, headers, timeout_ms), "Failed on ServerMetadata", "Success on ServerMetadata", count); COUNT_ERROR_MSGS( grpc_client->ModelMetadata( &model_metadata, model_name, "", headers, timeout_ms), "Failed on ModelMetadata", "Success on ModelMetadata", count); COUNT_ERROR_MSGS( grpc_client->ModelConfig( &model_config, model_name, "", headers, timeout_ms), "Failed on ModelConfig", "Success on ModelConfig", count); COUNT_ERROR_MSGS( grpc_client->ModelRepositoryIndex(&repository_index, headers, timeout_ms), "Failed on ModelRepositoryIndex", "Success on ModelRepositoryIndex", count); COUNT_ERROR_MSGS( grpc_client->ModelInferenceStatistics( &infer_stat, model_name, "", headers, timeout_ms), "Failed on ModelInferenceStatistics", "Success on ModelInferenceStatistics", count); COUNT_ERROR_MSGS( grpc_client->LoadModel(model_name, headers, "", {}, timeout_ms), "Failed on LoadModel", "Success on LoadModel", count); COUNT_ERROR_MSGS( grpc_client->UnloadModel(model_name, headers, timeout_ms), "Failed on UnloadModel", "Success on UnloadModel", count); #if 0 COUNT_ERROR_MSGS( grpc_client->UpdateTraceSettings( &response, model_name, settings, headers, timeout_ms), "Failed on UpdateTraceSettings", "Success on IsServerLive", count); COUNT_ERROR_MSGS( grpc_client->GetTraceSettings( &trace_settings, model_name, headers, timeout_ms), "Failed on GetTraceSettings", "Success on IsServerLive", count); #endif COUNT_ERROR_MSGS( grpc_client->SystemSharedMemoryStatus( &shmstatus, memory_name, headers, timeout_ms), "Failed on SystemSharedMemoryStatus", "Success on SystemSharedMemoryStatus", count); COUNT_ERROR_MSGS( grpc_client->RegisterSystemSharedMemory( memory_name, memory_name, byte_size, 0, headers, timeout_ms), "Failed on RegisterSystemSharedMemory", "Success on RegisterSystemSharedMemory", count); COUNT_ERROR_MSGS( grpc_client->UnregisterSystemSharedMemory( memory_name, headers, timeout_ms), "Failed on UnregisterSystemSharedMemory", "Success on UnregisterSystemSharedMemory", count); #if 0 COUNT_ERROR_MSGS( grpc_client->CudaSharedMemoryStatus( &cuda_shmstatus, "", headers, timeout_ms), "Failed on CudaSharedMemoryStatus", "Success on IsServerLive", count); COUNT_ERROR_MSGS( grpc_client->RegisterCudaSharedMemory( model_name, cuda_shm_handle, 0, byte_size, headers, timeout_ms), "Failed on RegisterCudaSharedMemory", "Success on IsServerLive", count); COUNT_ERROR_MSGS( grpc_client->UnregisterCudaSharedMemory(memory_name, headers, timeout_ms), "Failed on UnregisterSystemSharedMemory", "Success on IsServerLive", count); #endif if (count != 0) { std::cerr << "error count: " << count << " which is not 0 " << std::endl; exit(1); } } void ValidateShapeAndDatatype( const std::string& name, std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 16)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void ValidateResult( const std::shared_ptr result, std::vector& input0_data) { // Validate the results... ValidateShapeAndDatatype("OUTPUT0", result); // Get pointers to the result returned... int32_t* output0_data; size_t output0_byte_size; FAIL_IF_ERR( result->RawData( "OUTPUT0", (const uint8_t**)&output0_data, &output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (output0_byte_size != 64) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << output0_byte_size << std::endl; exit(1); } for (size_t i = 0; i < 16; ++i) { if ((input0_data[i]) != *(output0_data + i)) { std::cerr << "error: incorrect output" << std::endl; exit(1); } } // Get full response std::cout << result->DebugString() << std::endl; } void RunSynchronousInference( std::unique_ptr& grpc_client, std::unique_ptr& http_client, uint32_t client_timeout_ms, std::vector& inputs, std::vector& outputs, tc::InferOptions& options, std::vector& input0_data) { options.client_timeout_ = client_timeout_ms; tc::InferResult* results; if (grpc_client.get() != nullptr) { FAIL_IF_ERR( grpc_client->Infer(&results, options, inputs, outputs), "unable to run model"); } else { FAIL_IF_ERR( http_client->Infer(&results, options, inputs, outputs), "unable to run model"); } std::shared_ptr results_ptr; results_ptr.reset(results); // Validate results if (results_ptr->RequestStatus().IsOk()) { ValidateResult(results_ptr, input0_data); } else { std::cerr << "error: Inference failed: " << results_ptr->RequestStatus() << std::endl; exit(1); } } void RunAsynchronousInference( std::unique_ptr& grpc_client, std::unique_ptr& http_client, uint32_t client_timeout_ms, std::vector& inputs, std::vector& outputs, tc::InferOptions& options, std::vector& input0_data) { std::mutex mtx; std::condition_variable cv; bool done = false; auto callback = [&](tc::InferResult* result) { { std::shared_ptr result_ptr; result_ptr.reset(result); std::lock_guard lk(mtx); std::cout << "Callback called" << std::endl; done = true; if (result_ptr->RequestStatus().IsOk()) { ValidateResult(result_ptr, input0_data); } else { std::cerr << "error: Inference failed: " << result_ptr->RequestStatus() << std::endl; exit(1); } } cv.notify_all(); }; options.client_timeout_ = client_timeout_ms; if (grpc_client.get() != nullptr) { FAIL_IF_ERR( grpc_client->AsyncInfer(callback, options, inputs, outputs), "unable to run model"); } else { FAIL_IF_ERR( http_client->AsyncInfer(callback, options, inputs, outputs), "unable to run model"); } // Wait until all callbacks are invoked { std::unique_lock lk(mtx); cv.wait(lk, [&]() { return done; }); } } void RunStreamingInference( std::unique_ptr& grpc_client, uint32_t client_timeout_ms, std::vector& inputs, std::vector& outputs, tc::InferOptions& options, std::vector& input0_data) { std::mutex mtx; std::condition_variable cv; std::vector> result_list; FAIL_IF_ERR( grpc_client->StartStream( [&](tc::InferResult* result) { { std::shared_ptr result_ptr(result); std::lock_guard lk(mtx); result_list.push_back(result_ptr); } cv.notify_all(); }, false /*ship_stats*/, client_timeout_ms), "Failed to start the stream"); FAIL_IF_ERR( grpc_client->AsyncStreamInfer(options, inputs), "unable to run model"); auto timeout = std::chrono::microseconds(client_timeout_ms); // Wait until all callbacks are invoked or the timeout expires { std::unique_lock lk(mtx); if (!cv.wait_for(lk, timeout, [&]() { return (result_list.size() > 0); })) { std::cerr << "Stream has been closed" << std::endl; exit(1); } } // Validate results if (result_list.size() != 1) { std::cerr << "error: expected a single response, got " << result_list.size() << std::endl; exit(1); } if (result_list[0]->RequestStatus().IsOk()) { ValidateResult(result_list[0], input0_data); } else { std::cerr << "error: Inference failed: " << result_list[0]->RequestStatus() << std::endl; exit(1); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string protocol = "http"; std::string url; bool async = false; bool streaming = false; uint32_t client_timeout_ms = 0; bool test_client_apis = false; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vi:u:ast:p")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'i': { std::string p(optarg); std::transform(p.begin(), p.end(), p.begin(), ::tolower); if (p == "grpc" || p == "http") { protocol = p; } else { protocol = "unknown"; } break; } case 'u': url = optarg; break; case 'a': async = true; break; case 's': streaming = true; break; case 't': client_timeout_ms = std::stoi(optarg); break; case 'p': test_client_apis = true; break; case '?': Usage(argv); break; } } // Option validations if (protocol == "unknown") { std::cerr << "Supports only http and grpc protocols" << std::endl; Usage(argv); } if (streaming && (protocol != "grpc")) { std::cerr << "Supports only http and grpc protocols" << std::endl; Usage(argv); } std::string model_name = "custom_identity_int32"; std::string model_version = ""; // Create a InferenceServerGrpcClient instance to communicate with the // server using gRPC protocol. std::unique_ptr grpc_client; std::unique_ptr http_client; if (protocol == "grpc") { if (url.empty()) { url = "localhost:8001"; } FAIL_IF_ERR( tc::InferenceServerGrpcClient::Create(&grpc_client, url, verbose), "unable to create grpc client"); } else { if (url.empty()) { url = "localhost:8000"; } FAIL_IF_ERR( tc::InferenceServerHttpClient::Create(&http_client, url, verbose), "unable to create grpc client"); } // Test server timeouts for grpc client if (protocol == "grpc" && test_client_apis) { TestTimeoutAPIs(client_timeout_ms, model_name, grpc_client); return 0; } // Initialize the tensor data std::vector input0_data(16); for (size_t i = 0; i < 16; ++i) { input0_data[i] = i; } std::vector shape{1, 16}; // Initialize the inputs with the data. tc::InferInput* input0; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( input0_ptr->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for INPUT0"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; options.client_timeout_ = client_timeout_ms; std::vector inputs = {input0_ptr.get()}; std::vector outputs = {output0_ptr.get()}; // Send inference request to the inference server. if (streaming) { RunStreamingInference( grpc_client, client_timeout_ms, inputs, outputs, options, input0_data); } else if (async) { RunAsynchronousInference( grpc_client, http_client, client_timeout_ms, inputs, outputs, options, input0_data); } else { RunSynchronousInference( grpc_client, http_client, client_timeout_ms, inputs, outputs, options, input0_data); } return 0; } kytensor-2.49.0.7/client/src/c++/tests/memory_leak_test.cc000066400000000000000000000240111527052371200232640ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include "grpc_client.h" #include "http_client.h" namespace tc = triton::client; #define FAIL_IF_ERR(X, MSG) \ { \ tc::Error err = (X); \ if (!err.IsOk()) { \ std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ exit(1); \ } \ } #define INPUT_DIM 16 #define INT32_BYTE_SIZE 4 namespace { void ValidateShapeAndDatatype( const std::string& name, const std::shared_ptr result) { std::vector shape; FAIL_IF_ERR( result->Shape(name, &shape), "unable to get shape for '" + name + "'"); // Validate shape if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != INPUT_DIM)) { std::cerr << "error: received incorrect shapes for '" << name << "'" << std::endl; exit(1); } std::string datatype; FAIL_IF_ERR( result->Datatype(name, &datatype), "unable to get datatype for '" + name + "'"); // Validate datatype if (datatype.compare("INT32") != 0) { std::cerr << "error: received incorrect datatype for '" << name << "': " << datatype << std::endl; exit(1); } } void ValidateResult( const std::shared_ptr result, const std::vector& input0_data) { // Validate the results... ValidateShapeAndDatatype("OUTPUT0", result); // Get pointers to the result returned... int32_t* output0_data; size_t output0_byte_size; FAIL_IF_ERR( result->RawData( "OUTPUT0", (const uint8_t**)&output0_data, &output0_byte_size), "unable to get result data for 'OUTPUT0'"); if (output0_byte_size != INPUT_DIM * INT32_BYTE_SIZE) { std::cerr << "error: received incorrect byte size for 'OUTPUT0': " << output0_byte_size << std::endl; exit(1); } for (size_t i = 0; i < INPUT_DIM; ++i) { if ((input0_data[i]) != *(output0_data + i)) { std::cerr << "error: incorrect output" << std::endl; exit(1); } } // Get full response std::cout << result->DebugString() << std::endl; } void ValidateResponse( const std::shared_ptr results_ptr, const std::vector& input0_data) { // Validate results if (results_ptr->RequestStatus().IsOk()) { ValidateResult(results_ptr, input0_data); } else { std::cerr << "error: Inference failed: " << results_ptr->RequestStatus() << std::endl; exit(1); } } template void InferWithRetries( const std::unique_ptr& client, tc::InferResult** results, tc::InferOptions& options, std::vector& inputs, std::vector& outputs) { auto err = client->Infer(results, options, inputs, outputs); // If the host runs out of available sockets due to TIME_WAIT, sleep and // retry on failure to give time for sockets to become available. int max_retries = 5; int sleep_secs = 60; for (int i = 0; !err.IsOk() && i < max_retries; i++) { std::cerr << "Error: " << err << std::endl; std::cerr << "Sleeping for " << sleep_secs << " seconds and retrying. [Attempt: " << i + 1 << "/" << max_retries << "]" << std::endl; sleep(sleep_secs); // Retry and break from loop on success err = client->Infer(results, options, inputs, outputs); } if (!err.IsOk()) { std::cerr << "error: Exceeded max tries [" << max_retries << "] on inference without success" << std::endl; exit(1); } } // Client should be tc::InferenceServerHttpClient or // tc::InferenceServerGrpcClient template void RunSyncInfer( std::vector& inputs, std::vector& outputs, tc::InferOptions& options, std::vector& input0_data, bool reuse, std::string url, bool verbose, uint32_t repetitions) { // If re-use is enabled then use these client objects else use new objects for // each inference request. std::unique_ptr client; FAIL_IF_ERR(Client::Create(&client, url, verbose), "unable to create client"); for (size_t i = 0; i < repetitions; ++i) { if (!reuse) { // Create new client connection on every request if reuse flag not set FAIL_IF_ERR( Client::Create(&client, url, verbose), "unable to create client"); } tc::InferResult* results; InferWithRetries(client, &results, options, inputs, outputs); std::shared_ptr results_ptr(results); ValidateResponse(results_ptr, input0_data); } } void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << "error: " << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-v" << std::endl; std::cerr << "\t-i " << std::endl; std::cerr << "\t-u " << std::endl; std::cerr << "\t-t " << std::endl; std::cerr << "\t-r default is 100." << std::endl; std::cerr << "\t-R Re-use the same client for each repetition. Without " "this flag, the default is to create a new client on each repetition." << std::endl; std::cerr << std::endl; exit(1); } } // namespace int main(int argc, char** argv) { bool verbose = false; std::string protocol = "http"; std::string url; bool reuse = false; uint32_t repetitions = 100; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vi:u:r:R")) != -1) { switch (opt) { case 'v': verbose = true; break; case 'i': { std::string p(optarg); std::transform(p.begin(), p.end(), p.begin(), ::tolower); if (p == "grpc" || p == "http") { protocol = p; } else { protocol = "unknown"; } break; } case 'u': url = optarg; break; case 'r': repetitions = std::stoi(optarg); break; case 'R': reuse = true; break; case '?': Usage(argv); break; } } // Option validations if (protocol == "unknown") { std::cerr << "Supports only http and grpc protocols" << std::endl; Usage(argv); } std::string model_name = "custom_identity_int32"; std::string model_version = ""; if (protocol == "grpc") { if (url.empty()) { url = "localhost:8001"; } } else { if (url.empty()) { url = "localhost:8000"; } protocol = "http"; } // Initialize the tensor data std::vector input0_data(INPUT_DIM); for (size_t i = 0; i < INPUT_DIM; ++i) { input0_data[i] = i; } std::vector shape{1, INPUT_DIM}; // Initialize the inputs with the data. tc::InferInput* input0; FAIL_IF_ERR( tc::InferInput::Create(&input0, "INPUT0", shape, "INT32"), "unable to get INPUT0"); std::shared_ptr input0_ptr; input0_ptr.reset(input0); FAIL_IF_ERR( input0_ptr->AppendRaw( reinterpret_cast(&input0_data[0]), input0_data.size() * sizeof(int32_t)), "unable to set data for INPUT0"); // Generate the outputs to be requested. tc::InferRequestedOutput* output0; FAIL_IF_ERR( tc::InferRequestedOutput::Create(&output0, "OUTPUT0"), "unable to get 'OUTPUT0'"); std::shared_ptr output0_ptr; output0_ptr.reset(output0); // The inference settings. Will be using default for now. tc::InferOptions options(model_name); options.model_version_ = model_version; std::vector inputs = {input0_ptr.get()}; std::vector outputs = {output0_ptr.get()}; // Send 'repetitions' number of inference requests to the inference server. if (protocol == "http") { RunSyncInfer( inputs, outputs, options, input0_data, reuse, url, verbose, repetitions); } else if (protocol == "grpc") { RunSyncInfer( inputs, outputs, options, input0_data, reuse, url, verbose, repetitions); } else { std::cerr << "Invalid protocol: " << protocol << std::endl; return 1; } return 0; } kytensor-2.49.0.7/client/src/grpc_generated/000077500000000000000000000000001527052371200206535ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/go/000077500000000000000000000000001527052371200212605ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/go/README.rst000066400000000000000000000072411527052371200227530ustar00rootroot00000000000000.. # Copyright (c) 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |License| Generate Go Client stubs ========================== To generate the stubs:: # Clone the repo containing the proto definitions, from within client/src/grpc_generated/go/ git clone https://github.com/triton-inference-server/common.git # Compiles *.proto to *.pb.go # install protoc-gen-go-grpc with # go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest ./gen_go_stubs.sh Example Go Client ================= To run the grpc client example:: # Clone the repo containing the example model, from within client/src/grpc_generated/go/. git clone https://github.com/triton-inference-server/server.git # Setup "simple" model from example model_repository cd server/docs/examples ./fetch_models.sh # Launch (detached) Triton docker run -d -p8000:8000 -p8001:8001 -p8002:8002 -it -v $(pwd)/model_repository:/models nvcr.io/nvidia/tritonserver:22.11-py3 tritonserver --model-store=/models # Use client cd ../../../ go run grpc_simple_client.go Sample Output:: $ go run grpc_simple_client.go FLAGS: {simple 1 localhost:8001} Triton Health - Live: true Triton Health - Ready: true name:"simple" versions:"1" platform:"tensorflow_graphdef" inputs:{name:"INPUT0" datatype:"INT32" shape:-1 shape:16} inputs:{name:"INPUT1" datatype:"INT32" shape:-1 shape:16} outputs:{name:"OUTPUT0" datatype:"INT32" shape:-1 shape:16} outputs:{name:"OUTPUT1" datatype:"INT32" shape:-1 shape:16} Checking Inference Outputs -------------------------- 0 + 1 = 1 0 - 1 = -1 1 + 1 = 2 1 - 1 = 0 2 + 1 = 3 2 - 1 = 1 3 + 1 = 4 3 - 1 = 2 4 + 1 = 5 4 - 1 = 3 5 + 1 = 6 5 - 1 = 4 6 + 1 = 7 6 - 1 = 5 7 + 1 = 8 7 - 1 = 6 8 + 1 = 9 8 - 1 = 7 9 + 1 = 10 9 - 1 = 8 10 + 1 = 11 10 - 1 = 9 11 + 1 = 12 11 - 1 = 10 12 + 1 = 13 12 - 1 = 11 13 + 1 = 14 13 - 1 = 12 14 + 1 = 15 14 - 1 = 13 15 + 1 = 16 15 - 1 = 14 .. |License| image:: https://img.shields.io/badge/License-BSD3-lightgrey.svg :target: https://opensource.org/licenses/BSD-3-Clause kytensor-2.49.0.7/client/src/grpc_generated/go/gen_go_stubs.sh000077500000000000000000000037611527052371200243040ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PACKAGE="grpc-client" # Clean up before rm -rf ./core # Get the proto files from the common repo mkdir core && cp common/protobuf/*.proto core/. for i in ./core/*.proto do # https://developers.google.com/protocol-buffers/docs/reference/go-generated#package echo "option go_package = \"./${PACKAGE}\";" >> $i done # Requires protoc and protoc-gen-go plugin: https://grpc.io/docs/protoc-installation/ protoc -I core --go-grpc_out="./" --go_out="./" core/*.proto kytensor-2.49.0.7/client/src/grpc_generated/go/go.mod000066400000000000000000000005751527052371200223750ustar00rootroot00000000000000module github.com/triton-inference-server/client/src/grpc_generated/go go 1.22.3 require ( google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 ) require ( golang.org/x/net v0.22.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect ) kytensor-2.49.0.7/client/src/grpc_generated/go/go.sum000066400000000000000000000023071527052371200224150ustar00rootroot00000000000000github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= kytensor-2.49.0.7/client/src/grpc_generated/go/grpc_simple_client.go000066400000000000000000000214231527052371200254530ustar00rootroot00000000000000// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package main import ( "bytes" "context" "encoding/binary" "flag" "fmt" "log" "time" triton "github.com/triton-inference-server/client/src/grpc_generated/go/grpc-client" "google.golang.org/grpc" ) const ( inputSize = 16 outputSize = 16 ) type Flags struct { ModelName string ModelVersion string BatchSize int URL string } func parseFlags() Flags { var flags Flags // https://github.com/NVIDIA/triton-inference-server/tree/master/docs/examples/model_repository/simple flag.StringVar(&flags.ModelName, "m", "simple", "Name of model being served. (Required)") flag.StringVar(&flags.ModelVersion, "x", "", "Version of model. Default: Latest Version.") flag.IntVar(&flags.BatchSize, "b", 1, "Batch size. Default: 1.") flag.StringVar(&flags.URL, "u", "localhost:8001", "Inference Server URL. Default: localhost:8001") flag.Parse() return flags } func ServerLiveRequest(client triton.GRPCInferenceServiceClient) *triton.ServerLiveResponse { // Create context for our request with 10 second timeout ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() serverLiveRequest := triton.ServerLiveRequest{} // Submit ServerLive request to server serverLiveResponse, err := client.ServerLive(ctx, &serverLiveRequest) if err != nil { log.Fatalf("Couldn't get server live: %v", err) } return serverLiveResponse } func ServerReadyRequest(client triton.GRPCInferenceServiceClient) *triton.ServerReadyResponse { // Create context for our request with 10 second timeout ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() serverReadyRequest := triton.ServerReadyRequest{} // Submit ServerReady request to server serverReadyResponse, err := client.ServerReady(ctx, &serverReadyRequest) if err != nil { log.Fatalf("Couldn't get server ready: %v", err) } return serverReadyResponse } func ModelMetadataRequest(client triton.GRPCInferenceServiceClient, modelName string, modelVersion string) *triton.ModelMetadataResponse { // Create context for our request with 10 second timeout ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // Create status request for a given model modelMetadataRequest := triton.ModelMetadataRequest{ Name: modelName, Version: modelVersion, } // Submit modelMetadata request to server modelMetadataResponse, err := client.ModelMetadata(ctx, &modelMetadataRequest) if err != nil { log.Fatalf("Couldn't get server model metadata: %v", err) } return modelMetadataResponse } func ModelInferRequest(client triton.GRPCInferenceServiceClient, rawInput [][]byte, modelName string, modelVersion string) *triton.ModelInferResponse { // Create context for our request with 10 second timeout ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // Create request input tensors inferInputs := []*triton.ModelInferRequest_InferInputTensor{ &triton.ModelInferRequest_InferInputTensor{ Name: "INPUT0", Datatype: "INT32", Shape: []int64{1, 16}, }, &triton.ModelInferRequest_InferInputTensor{ Name: "INPUT1", Datatype: "INT32", Shape: []int64{1, 16}, }, } // Create request input output tensors inferOutputs := []*triton.ModelInferRequest_InferRequestedOutputTensor{ &triton.ModelInferRequest_InferRequestedOutputTensor{ Name: "OUTPUT0", }, &triton.ModelInferRequest_InferRequestedOutputTensor{ Name: "OUTPUT1", }, } // Create inference request for specific model/version modelInferRequest := triton.ModelInferRequest{ ModelName: modelName, ModelVersion: modelVersion, Inputs: inferInputs, Outputs: inferOutputs, } modelInferRequest.RawInputContents = append(modelInferRequest.RawInputContents, rawInput[0]) modelInferRequest.RawInputContents = append(modelInferRequest.RawInputContents, rawInput[1]) // Submit inference request to server modelInferResponse, err := client.ModelInfer(ctx, &modelInferRequest) if err != nil { log.Fatalf("Error processing InferRequest: %v", err) } return modelInferResponse } // Convert int32 input data into raw bytes (assumes Little Endian) func Preprocess(inputs [][]int32) [][]byte { inputData0 := inputs[0] inputData1 := inputs[1] var inputBytes0 []byte var inputBytes1 []byte // Temp variable to hold our converted int32 -> []byte bs := make([]byte, 4) for i := 0; i < inputSize; i++ { binary.LittleEndian.PutUint32(bs, uint32(inputData0[i])) inputBytes0 = append(inputBytes0, bs...) binary.LittleEndian.PutUint32(bs, uint32(inputData1[i])) inputBytes1 = append(inputBytes1, bs...) } return [][]byte{inputBytes0, inputBytes1} } // Convert slice of 4 bytes to int32 (assumes Little Endian) func readInt32(fourBytes []byte) int32 { buf := bytes.NewBuffer(fourBytes) var retval int32 binary.Read(buf, binary.LittleEndian, &retval) return retval } // Convert output's raw bytes into int32 data (assumes Little Endian) func Postprocess(inferResponse *triton.ModelInferResponse) [][]int32 { outputBytes0 := inferResponse.RawOutputContents[0] outputBytes1 := inferResponse.RawOutputContents[1] outputData0 := make([]int32, outputSize) outputData1 := make([]int32, outputSize) for i := 0; i < outputSize; i++ { outputData0[i] = readInt32(outputBytes0[i*4 : i*4+4]) outputData1[i] = readInt32(outputBytes1[i*4 : i*4+4]) } return [][]int32{outputData0, outputData1} } func main() { FLAGS := parseFlags() fmt.Println("FLAGS:", FLAGS) // Connect to gRPC server conn, err := grpc.Dial(FLAGS.URL, grpc.WithInsecure()) if err != nil { log.Fatalf("Couldn't connect to endpoint %s: %v", FLAGS.URL, err) } defer conn.Close() // Create client from gRPC server connection client := triton.NewGRPCInferenceServiceClient(conn) serverLiveResponse := ServerLiveRequest(client) fmt.Printf("Triton Health - Live: %v\n", serverLiveResponse.Live) serverReadyResponse := ServerReadyRequest(client) fmt.Printf("Triton Health - Ready: %v\n", serverReadyResponse.Ready) modelMetadataResponse := ModelMetadataRequest(client, FLAGS.ModelName, "") fmt.Println(modelMetadataResponse) inputData0 := make([]int32, inputSize) inputData1 := make([]int32, inputSize) for i := 0; i < inputSize; i++ { inputData0[i] = int32(i) inputData1[i] = 1 } inputs := [][]int32{inputData0, inputData1} rawInput := Preprocess(inputs) /* We use a simple model that takes 2 input tensors of 16 integers each and returns 2 output tensors of 16 integers each. One output tensor is the element-wise sum of the inputs and one output is the element-wise difference. */ inferResponse := ModelInferRequest(client, rawInput, FLAGS.ModelName, FLAGS.ModelVersion) /* We expect there to be 2 results (each with batch-size 1). Walk over all 16 result elements and print the sum and difference calculated by the model. */ outputs := Postprocess(inferResponse) outputData0 := outputs[0] outputData1 := outputs[1] fmt.Println("\nChecking Inference Outputs\n--------------------------") for i := 0; i < outputSize; i++ { fmt.Printf("%d + %d = %d\n", inputData0[i], inputData1[i], outputData0[i]) fmt.Printf("%d - %d = %d\n", inputData0[i], inputData1[i], outputData1[i]) if (inputData0[i]+inputData1[i] != outputData0[i]) || inputData0[i]-inputData1[i] != outputData1[i] { log.Fatalf("Incorrect results from inference") } } } kytensor-2.49.0.7/client/src/grpc_generated/java/000077500000000000000000000000001527052371200215745ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/README.md000066400000000000000000000077131527052371200230630ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Example Java and Scala client Using Generated GRPC API ## Prerequsites Maven 3.3+ and JDK 1.8+ ## Generating java GRPC client stub Clone the [triton-inference-server/common](https://github.com/triton-inference-server/common/) repository: ``` git clone https://github.com/triton-inference-server/common/ -b common-repo ``` \ should be the version of the Triton server that you intend to use (e.g. r21.05). Copy __*.proto__ files to Library src/main/proto ``` $ cd library $ cp ../common-repo/protobuf/*.proto src/main/proto/ ``` After copying the protobuf files, the __library__ dir should look as below. ``` $ mvn compile ``` Once compiled, one should notice the generated *.java files under __target__ folder ## Use the generated files in any project To run the examples clients, copy the above generated stub into __examples__ folder ``` $ cd .. $ cp -R library/target/generated-sources/protobuf/java/inference examples/src/main/java/inference $ cp -R library/target/generated-sources/protobuf/grpc-java/inference/*.java examples/src/main/java/inference/ ``` See the __examples__ project which has __scala__ and __java__ sample client. ## Running java example client ``` $ cd examples $ mvn clean install $ mvn exec:java -Dexec.mainClass=clients.SimpleJavaClient -Dexec.args=" " ``` __host__ where triton inference server is running __port__ default grpc port is 8001 ## Running scala example client ``` $ mvn exec:java -Dexec.mainClass=clients.SimpleClient -Dexec.args=" " ``` Both the examples run inference with respect to __simple__ model. The __scala__ example is more comprehensive and checks APIs like server ready and model ready ### Output of the scala client once run succesfully ``` name: "OUTPUT0" datatype: "INT32" shape: 1 shape: 16 name: "OUTPUT1" datatype: "INT32" shape: 1 shape: 16 1 + 1 = 2 1 - 1 = 0 2 + 2 = 4 2 - 2 = 0 3 + 3 = 6 3 - 3 = 0 4 + 4 = 8 4 - 4 = 0 5 + 5 = 10 5 - 5 = 0 6 + 6 = 12 6 - 6 = 0 7 + 7 = 14 7 - 7 = 0 8 + 8 = 16 8 - 8 = 0 9 + 9 = 18 9 - 9 = 0 10 + 10 = 20 10 - 10 = 0 11 + 11 = 22 11 - 11 = 0 12 + 12 = 24 12 - 12 = 0 13 + 13 = 26 13 - 13 = 0 14 + 14 = 28 14 - 14 = 0 15 + 15 = 30 15 - 15 = 0 16 + 16 = 32 16 - 16 = 0 ``` kytensor-2.49.0.7/client/src/grpc_generated/java/examples/000077500000000000000000000000001527052371200234125ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/examples/pom.xml000066400000000000000000000055551527052371200247410ustar00rootroot00000000000000 4.0.0 nvidia.triton grpc-client-examples 0.0.1-SNAPSHOT 1.8 2.12 ${scala.minor.version} ${scala.minor.version}.11 provided 1.8 1.8 UTF-8 UTF-8 io.grpc grpc-netty-shaded 1.59.1 io.grpc grpc-protobuf 1.59.1 io.grpc grpc-stub 1.59.1 org.apache.tomcat tomcat-annotations-api 11.0.0-M13 provided javax.annotation javax.annotation-api 1.3.2 org.scala-lang scala-library ${scala.complete.version} net.alchim31.maven scala-maven-plugin 4.1.0 org.apache.maven.plugins maven-compiler-plugin 3.8.0 net.alchim31.maven scala-maven-plugin add-source add-source scala-compile-first compile scala-test-compile-first testCompile 2.0 kytensor-2.49.0.7/client/src/grpc_generated/java/examples/src/000077500000000000000000000000001527052371200242015ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/examples/src/main/000077500000000000000000000000001527052371200251255ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/examples/src/main/java/000077500000000000000000000000001527052371200260465ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/examples/src/main/java/clients/000077500000000000000000000000001527052371200275075ustar00rootroot00000000000000SimpleJavaClient.java000066400000000000000000000136431527052371200334740ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/examples/src/main/java/clients// Copyright (c) 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE package clients; import java.nio.ByteOrder; import java.nio.IntBuffer; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; import inference.GRPCInferenceServiceGrpc; import inference.GRPCInferenceServiceGrpc.GRPCInferenceServiceBlockingStub; import inference.GrpcService.InferTensorContents; import inference.GrpcService.ModelInferRequest; import inference.GrpcService.ModelInferResponse; import inference.GrpcService.ServerLiveRequest; import inference.GrpcService.ServerLiveResponse; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; public class SimpleJavaClient { public static void main(String[] args) { String host = args.length > 0 ? args[0] : "localhost"; int port = args.length > 1 ? Integer.parseInt(args[1]) : 8001; String model_name = "simple"; String model_version = ""; // # Create gRPC stub for communicating with the server ManagedChannel channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); GRPCInferenceServiceBlockingStub grpc_stub = GRPCInferenceServiceGrpc.newBlockingStub(channel); // check server is live ServerLiveRequest serverLiveRequest = ServerLiveRequest.getDefaultInstance(); ServerLiveResponse r = grpc_stub.serverLive(serverLiveRequest); System.out.println(r); // Generate the request ModelInferRequest.Builder request = ModelInferRequest.newBuilder(); request.setModelName(model_name); request.setModelVersion(model_version); // Input data List lst_0 = IntStream.rangeClosed(1, 16).boxed().collect(Collectors.toList()); List lst_1 = IntStream.rangeClosed(1, 16).boxed().collect(Collectors.toList()); InferTensorContents.Builder input0_data = InferTensorContents.newBuilder(); InferTensorContents.Builder input1_data = InferTensorContents.newBuilder(); input0_data.addAllIntContents(lst_0); input1_data.addAllIntContents(lst_1); // Populate the inputs in inference request ModelInferRequest.InferInputTensor.Builder input0 = ModelInferRequest.InferInputTensor .newBuilder(); input0.setName("INPUT0"); input0.setDatatype("INT32"); input0.addShape(1); input0.addShape(16); input0.setContents(input0_data); ModelInferRequest.InferInputTensor.Builder input1 = ModelInferRequest.InferInputTensor .newBuilder(); input1.setName("INPUT1"); input1.setDatatype("INT32"); input1.addShape(1); input1.addShape(16); input1.setContents(input1_data); // request.inputs.extend([input0, input1]) request.addInputs(0, input0); request.addInputs(1, input1); // Populate the outputs in the inference request ModelInferRequest.InferRequestedOutputTensor.Builder output0 = ModelInferRequest.InferRequestedOutputTensor .newBuilder(); output0.setName("OUTPUT0"); ModelInferRequest.InferRequestedOutputTensor.Builder output1 = ModelInferRequest.InferRequestedOutputTensor .newBuilder(); output1.setName("OUTPUT1"); // request.outputs.extend([output0, output1]) request.addOutputs(0, output0); request.addOutputs(1, output1); ModelInferResponse response = grpc_stub.modelInfer(request.build()); System.out.println(response); // Get the response outputs int[] op0 = toArray(response.getRawOutputContentsList().get(0).asReadOnlyByteBuffer().order(ByteOrder.LITTLE_ENDIAN).asIntBuffer()); int[] op1 = toArray(response.getRawOutputContentsList().get(1).asReadOnlyByteBuffer().order(ByteOrder.LITTLE_ENDIAN).asIntBuffer()); // Validate response outputs for (int i = 0; i < op0.length; i++) { System.out.println( Integer.toString(lst_0.get(i)) + " + " + Integer.toString(lst_1.get(i)) + " = " + Integer.toString(op0[i])); System.out.println( Integer.toString(lst_0.get(i)) + " - " + Integer.toString(lst_1.get(i)) + " = " + Integer.toString(op1[i])); if (op0[i] != (lst_0.get(i) + lst_1.get(i))) { System.out.println("OUTPUT0 contains incorrect sum"); System.exit(1); } if (op1[i] != (lst_0.get(i) - lst_1.get(i))) { System.out.println("OUTPUT1 contains incorrect difference"); System.exit(1); } } channel.shutdownNow(); } public static int[] toArray(IntBuffer b) { if (b.hasArray()) { if (b.arrayOffset() == 0) return b.array(); return Arrays.copyOfRange(b.array(), b.arrayOffset(), b.array().length); } b.rewind(); int[] tmp = new int[b.remaining()]; b.get(tmp); return tmp; } } kytensor-2.49.0.7/client/src/grpc_generated/java/examples/src/main/scala/000077500000000000000000000000001527052371200262105ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/examples/src/main/scala/clients/000077500000000000000000000000001527052371200276515ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/examples/src/main/scala/clients/SimpleClient.scala000066400000000000000000000202121527052371200332430ustar00rootroot00000000000000// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE package clients import java.nio.ByteOrder import java.nio.IntBuffer import io.grpc.ManagedChannelBuilder import collection.JavaConverters._ import java.util.Arrays import inference.GRPCInferenceServiceGrpc.{ GRPCInferenceServiceBlockingStub => GRPCStub } object SimpleClient { def serverLive(grpc_stub: GRPCStub) = { try { val request = inference.GrpcService.ServerLiveRequest.getDefaultInstance(); val response = grpc_stub.serverLive(request); println(response) response.getLive } catch { case e: Exception => println("server: " + e) false } } def serverReady(grpc_stub: GRPCStub) = { try { val request = inference.GrpcService.ServerReadyRequest.getDefaultInstance(); val response = grpc_stub.serverReady(request); println(response) response.getReady } catch { case e: Exception => println("server: " + e) false } } def serverMetadata(grpc_stub: GRPCStub) = { try { val request = inference.GrpcService.ServerMetadataRequest.getDefaultInstance(); val response = grpc_stub.serverMetadata(request); response.toString() } catch { case e: Exception => println("server: " + e) null } } def modelReady(grpc_stub: GRPCStub, model: String, version: String = "") = { try { val request = inference.GrpcService.ModelReadyRequest.newBuilder(); request.setName(model) request.setVersion(version) val response = grpc_stub.modelReady(request.build()); println(response) response.getReady } catch { case e: Exception => println("server: " + e) false } } def modelMetadata(grpc_stub: GRPCStub, model: String, version: String = "") = { try { val request = inference.GrpcService.ModelMetadataRequest.newBuilder(); request.setName(model) request.setVersion(version) val response = grpc_stub.modelMetadata(request.build()); //println(response) response.toString() } catch { case e: Exception => println("server: " + e) null } } def modelConfig(grpc_stub: GRPCStub, model: String, version: String = "") = { try { val request = inference.GrpcService.ModelConfigRequest.newBuilder(); request.setName(model) request.setVersion(version) val response = grpc_stub.modelConfig(request.build()); //println(response) response.toString() } catch { case e: Exception => println("server: " + e) null } } def testInference(grpc_stub: GRPCStub,model: String, version: String = "") = { val batch_size = 1 val dimension = 16 /** * Generate the request */ val request = inference.GrpcService.ModelInferRequest.newBuilder(); request.setModelName(model); request.setModelVersion(version); /** * Input Data * use input dimension [batch_size, dimension] */ val lst_0 = (1 to batch_size * dimension).toList.map(x => new Integer(x)).asJava; val lst_1 = (1 to batch_size * dimension).toList.map(x => new Integer(x)).asJava; val input0_data = inference.GrpcService.InferTensorContents.newBuilder(); val input1_data = inference.GrpcService.InferTensorContents.newBuilder(); input0_data.addAllIntContents(lst_0); input1_data.addAllIntContents(lst_1); /** * Populate the inputs in inference request */ val input0 = inference.GrpcService.ModelInferRequest.InferInputTensor .newBuilder(); input0.setName("INPUT0"); input0.setDatatype("INT32"); input0.addShape(batch_size); input0.addShape(dimension); input0.setContents(input0_data); val input1 = inference.GrpcService.ModelInferRequest.InferInputTensor .newBuilder(); input1.setName("INPUT1"); input1.setDatatype("INT32"); input1.addShape(batch_size); input1.addShape(dimension); input1.setContents(input1_data); request.addInputs(0, input0); request.addInputs(1, input1); /** * Populate the outputs in the inference request */ val output0 = inference.GrpcService.ModelInferRequest.InferRequestedOutputTensor .newBuilder(); output0.setName("OUTPUT0"); val output1 = inference.GrpcService.ModelInferRequest.InferRequestedOutputTensor .newBuilder(); output1.setName("OUTPUT1"); request.addOutputs(0, output0); request.addOutputs(1, output1); /** * Infer */ val response = grpc_stub.modelInfer(request.build()); //System.out.println(response); /** * check the two output tensors */ println(response.getOutputs(0)) println(response.getOutputs(1)) val op0 = toArray(response.getRawOutputContents(0).asReadOnlyByteBuffer().order(ByteOrder.LITTLE_ENDIAN).asIntBuffer()) val op1 = toArray(response.getRawOutputContents(1).asReadOnlyByteBuffer().order(ByteOrder.LITTLE_ENDIAN).asIntBuffer()) /** * Validate the output tensors */ for (i <- 0 until dimension) { println(lst_0.get(i) + " + " + lst_1.get(i) + " = " + op0(i)) println(lst_0.get(i) + " - " + lst_1.get(i) + " = " + op1(i)) if (op0(i) != (lst_0.get(i) + lst_1.get(i))) { println("OUTPUT0 contains incorrect sum") System.exit(1) } if (op1(i) != (lst_0.get(i) - lst_1.get(i))) { println("OUTPUT1 contains incorrect difference") System.exit(1) } } } /** * converts IntBuffer to Array[Int] */ def toArray(b: IntBuffer) = { // b.flip(); if (b.hasArray()) { if (b.arrayOffset() == 0) b.array(); Arrays.copyOfRange(b.array(), b.arrayOffset(), b.array().length); } b.rewind(); val temp = Array.ofDim[Int](b.remaining()); b.get(temp); temp; } def main(args: Array[String]) { /** * define host port */ val host = if (args.length > 0) args(0) else "localhost" val port = if (args.length > 1) args(1).toInt else 8001 val model = "simple" val version = "" /** * Create gRPC stub for communicating with the server */ val channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); val grpc_stub = inference.GRPCInferenceServiceGrpc.newBlockingStub(channel); /** * check server api, print as needed */ serverLive(grpc_stub) serverReady(grpc_stub) serverMetadata(grpc_stub) /** * check model api */ modelReady(grpc_stub, model, version) modelConfig(grpc_stub, model, version) modelMetadata(grpc_stub, model, version) /** * check simple model inference */ testInference(grpc_stub, model, version) channel.shutdownNow() } }kytensor-2.49.0.7/client/src/grpc_generated/java/images/000077500000000000000000000000001527052371200230415ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/images/grpc-stubs.png000066400000000000000000001503471527052371200256520ustar00rootroot00000000000000PNG  IHDRvaqx biCCPICC ProfileHWXS[RIhP@J-TATBH(1&;E+*ZY "uQe.bCMH`]}{Ο3gS2s :|,@Z(O eKKg* 7@!(? *o E HYBfL^1 mTX LQ*l7@[U$ȁ<ڏ!v %Rt B ^P0YBeg?!k@a,?,|{hbyT*X[yUq4+6NUkIRʨd=j&Ppabw!?,b3#1}V$1\-4I!/I3wH 7\fn_>WeߪKhoEAT`"IJ,(6u;h#W&ⷅ-FlyD^V+Kx\Y(NR-o qHI) "sDdM}Yahfn,?^cE*5ĦD\|T!\j~%Ck>louh,AKb81E :xgl8o{B;!:p{DU,c@deƸ=C@q&n \q/臃CPĭʝok(bD 8~=SY{EU/룎5kܡs_[b$v;5vk.bTxh =XCɃUT׺wԌBѴBNMKrą, XJC PsQ&t ZGh5GOh11+cc\,KDz196+*jkUDpWd\OgK.oů3N0# <8Ba*PAA8L8 wS' Hdp7s3Kv#b/D2!Iq$>TJZGC:AB$#k-r:YJ.!Ww(;?%"L,l4Q.Q:)}T=5DͥΣVR먧wjIjUj:@=ML2hJRNZ36N)};m6O[=GJ^ GgNNAK:ݺ]{]._wnݛz zqzzKv{OҷߦJc0 ;viFgkPnנ͠P0pa1&ƴgeFF#b:+Foˌ_7`2 73Ya`r7u6k:tia ;0W3,l6f2u̻-!-[tY2,,%-OX2dqXJV+*Jiժͪ:ٺz= &fMMۙQvbvg;ا/o`s(vuHw vXx͉vstuv;W9_rA]|\$.]ڇ ~ӕq-ru}tq+qkp{1vD#ΌHGl'3sgK//&[ 1 [?}||m}3}7dKg~~s/?gk@^gFFm(:5#%#*\0&D#)ljy*=ϝm"ÓׇߏȉEZugjx=}GMN^09F43z̪1wcb q *^CƏ$ad̄3I$&-K올LnIIHIy2c܈q]H3M5Swf|gwFiƍ M87tbct&'$dfȏW{xYz\ZsapK(Z)z2YN`Ϊ.qB-JK^Fn}3??5_ T_'ml1yvT1ʚ)=hh,4JG僢wSS7M:t鋧?-(a>C0ey3:5e͜s:F5:/o/%%+K^]|s?ZTT^zsa͋EEm=[LXvܽߏKۖ,۴\ƊV,^h՘UYV^3i͹ kkk;*c*ٮ[zUU6mXF+B6m6\ɖ[[#WWWl#n+d{3?a|ǧҝv6۽Uvsyo:׺Z;0pY=R?Aј~d?ѪcǖDfYsɜZ&95ԵֱmO9Sg8gN <{#.\}/޿ni{Qǯ_9y5x.\~#ƭ7;n o=Eݙ{pfsm_ODZa.>L|xNJ;??xjdz]]{s?š?C3e_K^uKo|7oޖ3y=MHXSw e|9(fgNi0.xo@=uv:υ!I!jQd{zhCxR}?;*v)껦Jn]X_E_@? (eXIfMM*>F(iNxvASCIIScreenshot4 pHYs%%IR$iTXtXML:com.adobe.xmp 630 Screenshot 248 5УmiDOT|(||\:I@IDATx]%i *b _DBRAiT|7~{%qwvwz~3oߛ7K#qg%m%G|V˔A@AC  ӥł  p~" Wi  \;], 'B~V   ӥɂ  p~" Wi  \;], 'B~V   ӥɂ  p~" C(ҫg8gNA@ФxJ#=\rZ,E݆iW9o|Td)/A@A@8s zd:m=_sٮ'?HII'G)܎!}ܹѓq  pbtQs;Et4< ڥZb4CJ?I L)ZW#&P/AS>}tԤI%87nt1N߼K Ghp(! uAx㍀):D5j Ȁ?׮];ҥ 2^}UQae;cGs?<E\+C~;TjU뮻@JrΝ;)%% dصk 6LCr#rC;o;wN & x ȿ,/w]E*X('9Ion淬TJ=J;NՒO;FbE>S8Ox" 4F{hСԩS'|qwoXoU?+h޽#\!v&s X8% fnRlr>`˖-DF ;HKǏ׍vɜ֕,YRKmL-Eŕ<]r}e*T8k[A>X Lɩ&@LߨMyXƄVcư')LMMҼI-.](I^B:HY8}c%\߂͡qn䃶ɓ-K~bN9Wʇ2v&v9ibLM[qgӇrK/&A=1HO'Lbdz g{F-Z֤I詧2 x>[6;L|90xb1n˖-}UM^g=|Aϯk}kFÛ~UGMRǢq}j=굇8{30>4p@u 7bŊ*{Ybbb4]w^,XPy>Sе~yiF͞=[ծ][)RD(Q+K4ȇʊbc^iph"{ʕ++&V(U^qYUxnچ6ꫯ:)lĉ)U(d-_~E]Vk)3aoՓO>Ϊոqc5f̘LqXբXjEZWlO貁O^uv Q}f 8p}q[nEK'L&F:^ws'zqk!TU ͛{1x=u;1KJJR^?-z!յkWŤD:}ؠ0ߏ}7ݬY3˧X,I+n!9J'&~N͜9xN er <[oFXV2}UWi|ntR>Ys+*&~)ƒ}X ""Gr݈Ȝ!Iۭ:R82g "dFܚ? y|oGkU@=!I8C:zG8FB}N2lD؇~[=?щOD ;<|/!s s(nr G*auϜH@7VvVA RkF^}LMNs:_P2[~V&S)xI~xAtPy NU)O MZHY. ʂZny(݁tAEtvbIHx){}N4oWu:SG} T/f/LjBN"~!2N}ri):}lX\ "-ܶmVa<19#>/F7~C#qf,C^~eu7Ⱦ}%ZjVPy SߏoDԳvg~ p?=cCPd)Qo6/_x^۝ Q_~5/ poss^gYHb7PXշjاOCky^ު?Z4Z<%?Y5["< 񖌵nۥ΋ݒ*H:@Oސ"~p&P!REy b}O&>CM_rsFbf'vwNSL-MtjyBhwf2Zx#TӸp /&og^뀺ۉ3;'N ل9b>6cw(;PX?\#j )-кQG!ĸp&o3&OlXipW_+_ 6,_3|DҐKps < tA>Y gl?oWs#v8w>/Ќf=fͲ=Zak 9(g_r$XnȜ!w8p shMMͣ_Gܮ:,zϿjΏϪV3fff򁳾L)5|>&wVSorF6#o TW!]Umƀ2F2bP؛tb~?fY]ve]Xk?ρp4A >A݅36yС,7i 2 @F [<#`lqq뜅@&vNC U B+/V[VPa d#IhsD .º)4 oa-M<`]!xu:6u]nsp&O޻O $}x;.9Ԑ뜅@&vz.cT>uJkvhT*EUՒ iZi*-JWku |n3Z;jMՉh5Ď?)֪p3u7jP&&cMF 7e'o,B#՝q恈tHlYb_0o"&|H&nÆ l'.PGT܈]8嘉6k1a9Y<|;A&&#vS6ᬯ{&>bkaйg&1ıÚiӦ3&BHwom[%I0O3.TCʭ/!Ym?ρp4{{C9r S7~BHݜ؅L7:5[85>GLryxbR CqM{V5C#3::7n.j=Cst=,>Jo9F*>Hv{2 qjj5U 1nհ'2W^yj3&lNt;j,=Vk d'S7 5?@]Ln4DC :05,jZ {|kvX[cXs!6dFPWSYc&Z_D6;Nu0E-_Yo>fXop go8K0.œɭ>u`;&l1LC!vHMӾq7 N0fǎ= { @3P/C_%p;ij @yb|fMs ĮgϞ,<ݜy?ͳ36~ ^Ss9rL4n$njԞ۫HzĎݲimޅժA87%f:TGzȟsmT5>R>~%uUWD$[y?dHpءxb|vJ曵s u93AH]f'LÞzTFRb~%v[#.0~&3_.rD$.X'2ꌺb-^aNbo P #p`!sαaq|΃3a;lutHn 8p0[1ec<"7|~@PUՔ`w`*6~?f7uV/q5 ]vpz ,< mx?/`t _ePe_Mp#&s;]oSZ=GXUKj*v?]UklUqU\e_P[#8 RƿF|rEWg;z4Kf&}L  no,)1_o'Gb2F!(H!cǎu-R6l9`&Z2u[3euR{[`4c$vg-ADam tf+ B=,"C)'~JljDPnzS_cmgn;#>6(ENnI4:HT0!XiKa4 uk9Il#C큌sI1`2 E]+lB>l~ Cas Rڔlt~ps`~ ,@[`5<—s/~( </#(ۗs#v3ݞ7^Pn^s^\r< .XKDA*sUy*}uyt8v߁QȎtxN_Ը%nØg[mfZh9]t5[?P%)howAmVdJo*bև3eÃEĆ) k,^gFv%V/`wcD|V,]wVr/%hQf#bbH&**J8g"LRB_U@cgl@Lu[*N9"&iēfPI1^M0Q)O$ *O|XTPMϐ!C;b Q]G&d qm5gʘ6̋-\OA|D ߏ|"+-/ ^6IkK*vjX))iVY~\nƔC<KE՟<9IchiP)]Z'\   b:C"  tГ  d#eΐ  p:;$  !v٨3*  BN=I+ @6B@]6pB[ӴF:FA@#|-LH]G  p&bw&P< y %,I ]m&-\;m2#MS=@cիsJM?{M7D/b(%   @;v,m޼YT$ː-?&\25lFL&vt"> 6XZ]jS" Nki]Rz߱6U=#gmwk$\'z:?̓'[_Pq vI802'ႀ -[h̘19#ga̹I&TbE{'UǻU b]ՠ'b"'؂6YKR(wtjȀUq>2ҥ 曚4!~۶m)66>믭2bTѣjRK/V^I$}X_Y 5\ 4(QBTH2￧k)$8{h ]пղeK-%qׯ&RN4IGbiA@`b,Rxblu+=P"hlmrE)wD&eYwN2iKMNqhCLZ k=Ђ\^fNI,Kb)Ձw k 0+.]O"e>H-p> DLA H֭[geObWBڱc$`6$̭+Tgz>؝[~'zWJ*z?{|_W^/ B@/d! CAu;f|A'rXȆͤ]NW=щ1I-$V3\YUln:X;M7%:JMj]|Z2)/c͛`C܈]8X¢חCA] <݈,a[nU|d{՞^A@`b J>@ +KfAN`"n*57+\y] ߷c[P+Qk()c/)qr۟_(Dyv',I8Jdlckm&v~Xqh>WĮv4c :0$lҥZXT):|wqbMγ<݈r/Tir/ B@] Xoh"\{J:K\NӖ_tz 5OvaI\,E- h{QABG1RԚ,K~zصkN[N=鰾 _G°\ѢE5^p!w}^YaS/O7b[Peֹ @.Lkիr# /w$xs\}:\{Z=Z?|zgUrAU35CVW'wWh6~BmZ6L8^mv6VCJ jVd c/ 3jU067lX:Vx87h6LP퉕_驧xjr+޽{ukf_#:/ާN-Xʏy?x?ZBA@~w7{{VN)B7oKCD>:?_,U~;:#m]o͚5z/H8-EYRF,Рts21b-ZH7ATZ}罒`lVqϞ=ԡC/cu Xn]P_څ97=ƧH8/fVF΂ @ uoL -+S<ሎ^_uPwGTtq:c7޾KK4C T[%Q7qyҟǒYovZrB M\}T A΄ +_g ظ$1w6LGAF@]9X#+R:h~=2_ؼJgÈO_8A;iUl[<=\E >)v2$W'YHG|靡3R  -؝-dw6s ]V\u7kşbd/_*y@GvОUЬ+u NsقyJjLbAߊ}oة}9^A@r6B첸 jL 1tRD͝]RZvfPDϢXM3wZ 0Dy2]Ǘ?  p! .tIFpիj|Zbwx,>9l<u, Fjނ~5-_R  #؝#}vGt|Zdput4D[hT*tK:H;}jCA@AE@]K鿉mgp`zUJ_{O\A@솀l#6ͧȹD"_Ҽ]SjlPk  d7e  &BN   bzD# @ 8I& @vC@]vlR =kz r3,r-`k,uQO֛?*~1=а8A@A@8w;wX稒5_8E@[d9CLxbrK  !vWȯ=jiT?f3=ѡ]L)4=\K'N$ b<Ƃ @D@]sQY<(Ꝫm[ЕSi5L("za*y˕  i؝iDb~ƍ{J(qKd=+g>HN2twJsQ:Ji)'(j|P͆=U<A@ !v9;t@tӽKy=kbgU)震y 6Ï=/۽[JgbG*%w)=rRfT%wtqmq JLL%K;S:֭[G~i}j*OK/=SM|A@8/bcǎ(,߽{7}|rڴiѕW^I~;nZ'+;B Į{c|z|Իws_ )Qll9ΪAbaɘ9_㏟q_U,-O;a(a6m hpBM'ӈY>n٢;cRyѢESOё#Gxs.)zYcbblٲ٦^ w٪N \ACEA@uRq,vK؁Az)r~/tH,86oݛkhƐ(>ݻhky={^S_i⚑uZ~&z,ԢE +$%@`L?8AzwhoR.55*ڷo}7ڃ u]ezzp)U!MuC <؞vq(clUABɁKT683FijPj`ǚ*&>EEg1köUwWlWGO\~Է+{@zmݺ>g̙:[_[DX1شUfM=S!YD@YLg$qى;y搫ҿC-տ}Mu,6YmGxuL ;7s7 u~k'[n%`\_ Dի5Ⴔ ΐ cZyaMl@{ALDZcS GVH qDS&Њv}vKOXB]s55jJjoruAKzŸ97g+bחI f)ܧx$vF x1&'L_0X6Г(&_ U.]3Ď-MY{; )s:`Bf5R b/jռ΃ Hx2tP#.YhIL-[Xd8&7xo^Ey<󌗿֭{W[$)H&u~EtIDATu~l cp$lPi?{H.2R5kXzN]5kAWv팗A@2el'lvJPDI+;Xm:O "H0a5vuP?l~!3Rz+SA0Wɓ'[~ CNB.S不׮] uMرcUz<7ŊSG)Zab!=2δ[o5^&=fIHQEcxc]WnVGN_TXst yoCQ6dfͲ=Z#-ak *L.A@8 ; ,d;o$z믂SܮjWզԦQ5RdZbNǤHvI eD[d6H&]s@LFD￙!8lOmAca8mz`R7ngH 3mZ0_ " mc2it#QM0W1^h[ݝ~lƭN$ 2 C|#@b4f-& IZj&r4B RrD@;'rNqNxj r(Y1K_nRԌ/]tP.ގbTN\N.:ǟ ׻$ KQYDגLԔbSFR۝KlaJLzjV.˖-#췆4p1K 2xBۨ0{uPo;qD  Do1ԟ"9arɖRV>RCb>~ /ہp&^+ ! |U^zw%^6' te]ga@+ zAK4=Yzp:!?$j`eb>}*m1 ?LufKtZlQ6#kOp#@!R d~m,`Y*-vJ~KjՖ#ԺC u?bvz$_Xnba8bQVs׃nzbZ>fBD\ci+v%n Mt,ć,,Uqc-nۓrHBfĢ`ןu4#ә0U,H߹sggt{lKX榊uMӭNX_2 /(޸Fa!T?lklbw3FAkԈΎ\ @V!i!s8k]Ow:-SZ?5Q&~U)PH-23u2=M%$m{c_+v;[}캏0s;woi b*)c"]j3NC`)p:l9tPMb?;YAa_W_e ڶm`u@nx,IT¸97թS'*I+c!c(fkvO鶸猋{:[\@FNb e66WP_nۣtM:\lr#Y,>b͚:aDrcXZfH#nDS|zgUrAMZՃc[W--[}G'|BunTk  j&S,H@LXk%K*X-](Ďulv Rmظ~ ƿ8XK"ad $?2M8$rD:IPFUVVQ?쇇HG !>qU{975yrڼX`>Vn6aBi #h'Vo2m4v)?f7n,bڊI`1>G:*2PN+X}Ϫy+o!vr!Y,?ԢAՐPpΒc[S{Wz v eڪ ;U@j7}*ؾFj2Bc$ϙ^ӭ᠉ Bd&e|CO RK4$E`:*>C@G$X !*1IW^Ǘ.Fh;ԁ 4[6P{(83D:մF_0e  &SjV'X1CՋ4H6>F$vh@FZ_HP}a1žҢ q +؝+@9gRjMՊϪUC}TZZU~jOM~፳UrJ^[נϪO[1;X9?tm}B$oμ]ﹶal,7}$˚C?$QAM̈́4  6tF kDδ~b3XSb8n _ .P:| B-馦FZXB*gagŠ;HàT 47bXƆ2^_7;ASZ I堂6 ){ q d5bO ٱӂcQ2WxzyOJg ?ӑX9Ҹ=Bf48,[+<M𚶊}*q^Wi#g7 RX61dF[hLl OB6_ߗ,"ʃZXxڕx!'Ir0 V-**pDYEչ{ j bX;KYĒTm ?{se, N섀YPc (rVO*UDl[RۯT--56/Nӵ}L`Mf*v'EK͂zFc]vD 2%jfzJA@*e٤܍RtiT5ܺ}>n:v}Ff)˾ Umu\4$'ؽz$ȳdbUȇ_.#dպn#ޞ%hjJJA@rBrV.ؘRcRD͝|;YZ.jACQMȂVMN 5s\98y$<?u颒bu36k`ΒB/-oXR㞭zH dge9u[ҿ&ju:6BKܢ4mkQ*[bbldJI'I?|=gۈ^6ƮdKP;q GClϚQ5Tjtɵ434͙[(j W)&F i1מ|``YǖBbY˽  ؝srǢ ֱ$lsJ, * Rkﱢ˅  de~Z4OI; _KJႀ  ;w\WA@eRaA@A@b王   bL*, ;Bq}'XȚ3|qNokf{]L7EgFĞ_A@A@@݅!r|; x0N(G'pɞfwBLdrWH]KA@A \؅yqjŕqL~8洧[PRR+*VI,gA@A@8;1{?49.Ï=em $]"E/L5I%UpHA@MJ7{1*QY<!34^)ڹ'LKerwo(Pj6A%/d@A@A # z6СEDDO{/͛7(YA2ʥ"R^қNɓ)/*rR>܁ܕ$瓻;vД)SK.{0?Gu֥ *%>>TR~48q"mܸRSS7ߤ/\WcСn:Oώ1uY]tѹn9-ɓt!*]t~(6"vE| s=ԨQ RHA@# ΎF6قʳdɒOu]wVj=[-V؁q8;oa@-;/] mۺ=GS>s{&M( 5k_ttROu֭d3`0#L<~iiٲeaI%L>l6oLzȑ#:^x|LiΥGJJ~XbUܹs顇"`CVAwn9݋I&я?KHH)RDwDtAGA}wZjߌ &PCSoA E&HDA bESlǎ5w}SIUlg$fv\& C<\06lۏ yDb2u%wXL}T$-[R%dk%ܛ6\7K/ȑ#uDA|;myMJJeܹ3Νx!Cy/_>z뭷Fg'N+QΈW_}FM6zD dhW"řG 4?rs?~'7x9 0@%SEd6ܨ%v!',UMQ6^Fb O"Zrœk:~kFK.Mbh"$θ/+V2ehmݦ~Hb<{=UreRKxU<g*MvZKR -Se˖q;mCm?ngթS'ŤAן5T:uŸ޽{Ŋ#h?^iZgHXլ@ (ETvװaC]浈+q_~e{J*:oVzrYlUZ_W_Yqp@@r ZrTƍ~N/):ٳgڵk4lnEuqCl4c;/z<ތ+^$&&F?'xmmz=ϟ}b5W\s$Ye0YVlfXD[o[nQWfΜauzSXa? SO)&^q!F?\7_p{_['d5D!↳!rnd~P!h؇jLǁzθcƇmkQŚ0nҏء ANA$<ԕN2`U@ 8?QLb&n؅ ze&$U=zԫ(3U_~.Kr;o@xQE^Aᴙ-Yu@ צ,i׮n%۷[uᵒ:9s,?\_c&ب@Ych tPo" ZA 6\ˈg+..E q C[Y t+d 58+ԋEjnk֬U2؈Z7ci@رc Da*T-e'7f[{EyËć" ptp8ظ+~@v@@]v g7~"%-Acqsd3c=m[j5.1WC_(;s^L 7F=0:2H/M]5ܐ@nرV>5~;cYqs-[e͘1CLAi폵zgl" 8qgh5|p%?*H>۶mk䨨(k<;UWYOHxk+ 7b| )D>l="=$,[D_p?d]p A ! .uꀸ†6֦Me֗)|$ -PBj&̓7 u~OA buj%H0iBB҂`zx qiPa;LLfq;琎B5QC K&`C8a( (;7[=^U ؙn&´gp U.pXr&5k!?bBj$Gljc.^|E]xY;%2271;۸b SMA/ X@ zn۶MAlv!>tSNkg<ܻ;R`8rA؂ ] qbcYȎ@IDAT]t X(EQTD ED;@B@AT JBiT@@ PD|K;fݝ71tIuQ<7ߥK_Ԉ۷cv?nT+a[?uGq)j_CTTVT-:P}~-{pgYF*W_ڵkСoذGx=ܹg7U~}~˽+: Fo+BO8Q7߬F}گE_$'ѴaÆe*nԨQ:ꫯ {g|/+WSN/u}i>3Ox n;sL|wmF<߼y…NZmО`:@{݁,%vwp9ѾvTmy iH&rSEʗ/"G> Bi%~~x(znZשL2-c4mF' Do LC5#SjĮw޺xo׬YөG(b˧:5 ի|БbYz뭷H_%=Hۼy2U{W/^) ;SbD쐧 YZ5u]w)P#$vȻq:͗_~j߾tbca-KKOQS:go-R=֎n.VMԡM>)Nlb}0]|:NÝ011]ЁUhŋ|gÇZj={:M"vu rs=AӼ:$H{kH^iӦ|PYfI<_|CT+Efd~&ݍ/T;Wn&WbuMD;w\3Tźu(b8h# 2ATP"OӵkeP cN^L_|~76ڋ g GoV]>-< Z(l^~e)-KuWNȜ)$pƷU}XZ=tu U犢jjj6t6lUnROw#vL`_ծɾcuO֎ء}&aH✦ĴVnky"w}W9p4En!y9L))Ts=Wj@juHK̘1)26#"vCرé7NV\J,0@#-@ 'b]p: &+ DגI)={>7%[^z9m & TgZoə 7LbޫV:>AqK,{R.>G/b'vhۉw;q/>D9A4ظ`- KSo ئ"ymՒ VΟseQ[0ZvvK4PLyϚA+ 'xZnn$^7TQJP!CČB?V!~˖-U||<#uI0H'x©xBH>s 6HQ4;B8`nI4mF'a1a?tPmŋ/8 hk׮~A 0`MWn]|dQl@$؁KTn'm@ qoͦ%/bb ] )3Rl$=Z#e^ P'Lx SԂ>ԒToզ9n.*a}ojAz qSӴ ~P]~ݫcǿc$} ?֚mX+ :yq;1r;H`;'2 뮻yXA(Q%eT"H& s;/5'ۤ J*&"MJiDjCwʔ)*U>6o\Iݡv;3!aW ^yx`# :Pr4 Pv^ta'+*!3WFc`'T@7֒'f1Rl$o{d7B<;M`7%.Ϊx:O-F^ަ2eKѾM[iTt9Ұ'm?Jyo_Oi1}=Բ0*uF9)bʏV aA# ?TJH"V겘H(zʊXZ䇻;u4m$ynbc~b"EmIR/Y0"&;[\LcܹؾJcR%3(9E Ĥ-X8a0HKuf#UZؾ-l'V0 wL3xK 넥$؎tyE~䎧ccWEIbiߙ iBŏf ?^h92mz@V"`]V:~3ʗ?U6W% &_lGP1Sޡån`y]ƑL7Y#l4DRSHPWN-#ކX H,$8b1bi& x,IsIGD7'DR&MzѶ,F,4ej\;ݼν2K*j*~6Ц5E?z.~ *Dɇӡ=c8Nj Tp!JIJC{Ѡ~QbbVZb5%&[oׯO?sXT#,YBթSTWǖvr-4g=z4իW#FO?M麬5j{W˗UW])2,W>XUtX$lB:Hμ3R*]:l9  /~/v#=_>ݫ^ 6wh /Z3)?4?]8m7GrM/ҠsNxK3<\l¹$;{(;XfCENA4vXzᇥ9~Hǀ$5WHH7q,eEsyHMrھtڻgXu f:;qwToQu>6[>ϟ9/L?`;uW%KBܨX喇8\veH"oT[V<:p/<ڵkL؈W/^\j/K4~ȇ'eJAAzu3fKYfW_}U]qɠbՃ_\VU(6B&>cx`ѽ{e 3fU(oT,P?ZlΏm)3aѣzuY55jN&S\xu]:/V(|il6ۑ)E/I9 _ X!XWpgR]4[UbDx=./LtQÇ+QucCURE'A_a͛7 1~z6Btu;1%%%ZGRT%ԝwީ>#lwHAa^9pO7kL *!KWꊋpp?"6vjьc XZyE _㙠偋8%'<9IN Qb7y3Azh{5x~NWI{{\~a7M܁韏9$/ O?U 7LRx?tMM?&ffճJNNvٿvuiR#Lb<-_|BZ z %*URw *.^1đ xn2tP'=1{u 7 #ro/>.?|.@_HȾ< Yw *أ< $N~1}&U?QwAڵkI= pyC:H9F~/cH)b2dܚ7$\tE:w}]\|C?Wf3D'j*ĮiӦ~wl>> gl%ڜwdcI_ޫVc>6]4s]9\=f v6vS:P( @\u^4eE2g-9ر*VA&wB#C^tv?ڹ/qן)~?OEBqjajj};&r~-zњݹ#:b=!-rJaB8qЁr:́Iov\ȃ1ި|h7:R$HW01X]-ZyB 8aCZ]weĘoix#[kh J8Uzu/ڈ;ӽ: DvҤI>t~QA*b:y!?%THS݄4rB P{L壎RvmfyMI~! ¸aU8"bL !;"&&C/_VZ鶃ḅ>p./c:g8oIg6PW5< CG_ }/[޽d>RlPXof_ݻ>wKCMSpE j̙K:/EU'8fl?/phZfH7q4$/b D}&(曺6Ιa4%GEqN>]7S:'R:(67cMᝓ7Y%1qhIH߶7R]?VmYowޖ$-,2ap&DH5Pos"c۶m?4HTHg;bCi:BEE36_xb\+}h@'fK[9zH_xF`cC +/ pΙ%'ɑ;7? Խ?tuZcR=4/aX3[wi4RS{=QOn ;5ԞOi?MM?}`K5kj7lrpy lpDEnx6rJ%^Hini 3w} 5!U΄<%>BM/y9Ďi:,-MtjyBh:y 6缧N).rB>ng 6v"3< ;H:n߾}N=̱rJ-AQ5%ʀ  - BuCh2ƍ'8Gs .$HGH?2&޹sdƽp?)U/; 6fl? WA:@s]9eH@y{o48{ۋؙxy@$j6\93T#@DH&'NY1eZ3YzX3}y:,v\5oݥ&}QWM!uz<&Ïr=>{P΀U pF",$b&D p%apb0n;7?CA򇴰M92r@YH]}[趚$]WQ>OYO97TqX27GLr+OE6`)H 6RNԸmP%xvX0`@y |@L6WE36yKY< hp IF3#H_M^VN>OyE<8s΄69ӗ2ͱ].R;!vۧLVmҺj吧 XsD$To⯢,]|^"%yGX#`U;؎vox` +>`9 @DjwŰB7 @?<Ņ כAbD g;HAPa$"̋ESN~Aa}lMHLWXQ Ps{V[J6m a01U<@FH dF vx\ȝwy] Cl6O!v +^3,1]4c#X7_rF;JQ ag 8ޖ\< .SԱPc>2m8s; M.G;Mᇚ܁Ѻ AMԚQ-CՊ9ѭԊMT&޹*q8o:}Jt~l7vz1Q_OTtQـU pX8tR a#!vO@~IJ F1l.Ԅ(N`d25ܹsu1!p]MTٸ'@cmϰ5;*S)8b96$<#sКj(6_+Nn:rݺuyZ%!^*a's$R썤i0l)# +S ' 2ON""c!v0rX59(/Pɚ.v„ NR@s"J@gN8qh_R S 3Lm#3dI찰@ .PG|aP"vє#qZlr;{ H &{ 96 t =p;!_lw /bCUq̗YXc>,%O<6 ݕ{k[%I0O)br?f_y%$ y!6D;79 G~m4qь ~,@9]s]HnF3m v&v<$`oUfȭ 6g3r7 KD`حՒ !MՂ:Wn.jfj)_3U'9mJ8TcĘ*!fp#|M#UQ-V|5髦*%C!|0qc yGZ?[ Hʓxv Ay>M^>s]<`Q\QMx`LnTYPb#lAuړ|Mb B잤h QN2e>Iw P[p7džC#=ƉI1.O9nCL}4c؎D C$iA<͍ca8b)6o/P.ؽxGy <ء}Ce% b~(+a{9E3J߹3iF:2Cjb#}~Ixe 8*ư_+E;gJ:ki@bdKz^ i ;Eq75v_7P%J؀ 8H-ZWWz4UI 3." y 7Ad<;]5=k~h8P3IID8X${}?A5()G q;,#.0``s!,MX2ꌺ–K/עA;5>,`o)1ǸFrF(䏱>;:aF+RlD8_|~;HiE=DlQ E}؄dឃ}h%a3Z'ؽEHh7O6 `;Tº~jԐwnS~$,R'͠3c ;%塏0)!bCa\Kev>P?&`BX| (H!^|Hٰ$HV0M fׄ"_ܗJ$FW` 6y𣿰lG:6=ݟD!X_<@G')>#|r 쇈`a:\`sε/8> "v%yሗG'D;g"mnqyuhȖV^|!&P>*S$J{cPgSF=)qMcÌye^ϓ H3m\͜Oeι}[*TԱta?*L)͑XCLX} xظX8yb ڕxB%6`cz">,]wV?4f԰ybH<LV8Y,a!&aU/A&/N &Ѻ­GX͈Ķ|4fXI1^M8Q)?IV~9aŏP؄xb{YbbGBTב1%&y9++\i08:+x$ %;hYÙkfUF:"~,I91ۜs<ϯhLb9'vϠuSr[)_|ܥyh5t__Ӛfґ#t}P+/x&rz0a[p\( .rFz`uxwy7?lS&6J%DK4t"d &`Nшm$6͙&y z2&!>J7MN{b7OCJ;|{.a a{0\hwH-; -E7S2x&Mxq՘Ć٦a1*T Ecme+bV䄹\Ӟ-ں*Vι_B*rعn-,ZHλ6^X,E"`8@^Y,E"`Xb :V"`X,@V `]VhX,E" .tE"`X,@@aX,E  `]6[E"`X,Y%vY"`X,@6@l S'Wߪ6sdcZ,E" .w&v}a.-շX,Exx/b 7O7k b/ .GS,E"`ȅXbM;u-fZ\7OQݚuH4ʕh*צLɉI4mR-k}THAʛ7?IK'D.xȲ5kF?,XCJ &XפIJOOjԨAw$6"`X,(. :D={t x@~ H/2qa[7RX9J)IIL2~4krRExGt86o>&T퀐uoW_+-[d1o4"\pm6矧kn`IlE"`X,Q `]?4m4۔ع"%wwy'AJ:zkPũPBNIq8&w|F.'%KPLҏh8L-{XFu>jڵʖ- eX D)B`VZ"`X,E0W^tAGZJCvr~HAT#d\m[n /PK I B>%% q̗/WPPHq BfiiX,鍀%vuhذa--ߨQ#Tx}Y}| b$&n93sB $lRS(!N8۶KG;Y|M$,8p oh"mw7T(996lؠjnݺԭ[7}?g&]6 :D0~xxɄ=z^{MG=)Sh[bŊQZt 64g82m(gԩ;H$zw= e$m5˲E"`KAɈc.fcMʕ+]4nTgv. ("xL쎰:o濫tGMؖE)ȂI /Hdb']ڈzcU,TV@~GGBBEXdR\9Mʔ)XWpa}|'"3g]wݥ#7H@A8UW]E+Vo/_#G^z4n8V 1DZ2زeK'N$p% 'V 3F{@KFy˚ i[%=Z,E \, x9͔!$x -Zp",~jT'9 Z +er*6E3sS)!],݋OC]J(b tU L߾}{c;C~SF0UlLL ߿_sϥ%KhI_͚5Fd*$ RK.DA!i+]41BKye_M;vR4H83poh)!wޚLBJ9vXݻw~[M[~hX,p.\xb_~DBpfQlD_,WȧtJOMnZbWJxyإirlZr*kaibDf1硈]ӦMTȤ]vGi5$TO>ڴi%ӴJOe,NHO?M dBj"*TC!CK/ɘI,W^8|"EVX,@XbX )89a(I۟-aja*s}tFvo-[(ZZP?_ᕱyۓbE RA8^Dy;H a۲ 7nWѣGSA u7|CZr lVZE'N_{ zjuY{n?$O?iܹpۅ]s5"/@ z?''Ѵ5@V"`X,. 4֮]U}~ElEsu!mӏ{{ *~~5; KbbA|I2zaRW7+.ʋ) O)' }[63gN}P2H/+ҦM-qygy&h^R=ٟ/XPb1ٮ}YR/d!cX,KL4"<  xL M:RT]&S2S;6&ivU l;H&,D2jʣz;=þ '8^Xڹf;qQ:~?;/2kHl[4o\#a}vX bH.m_ժUiŁ󏦭~ E"`%va( JY8^gDŴeڗTX s-4*}%wD{$IDATcfMK ;G ^j$)VUlkDIh&vi*@m"v<M4+ X$, 7o$b{I=%1X^d[`/TܣiW]E"`X!`]0tyL[k;J޿K]@vŋ_"q[H 钇PzJm\剱?ۣXBJEq:F};6dNС^] .;a_%Jhm%`yz;l߂r-  ;<:Է 9U^X,E .̺@7?Gb.RK4T+4U'~\QDչZ0Z1ۿv>fjO!UF?F1զECԪvWSQZ>\7 >+,cl &f:ժ~a[ll ^Xyux8kήmOH?Nۦ(V;o߮XMÞ{9' .y>ujƌN~,SxF/4muE"`؅~Cw &QRr:+)9_ zzaDަsK+SRg;WӺ90Rl6oP z6vx/;b,M&=H1׳g|֜ǟ`5S՝E XĂmTu $K.!>\<$vHp؎1vܩXb:o"mGE"`%v"u[=6OM o싣vD_kˤ mw&1P7=N1nt7q'>ڞϽP#Xڅۧ7=Ɗ`}p "iGE"`%v"uAZ߈ήxY=O-Ο:JEeʖ}ލPrTQO;d]}%{V__HMvibעԒR0|m/E݅^%d$ND[)ƱX,܍%vA'wҶJoѪTH/\CGYuX,C,ZlK{iU_߉$7e&b_h9`)n-"`X,%v' 0]5n^J\^ξbjmԡMmcjrOWc{hم+h7}uб}K`,E"` Xbw{qw(!z},+Sӷ=~Į f`nǪtZ2a(Ztܵ U.W*'av9 SrDkX,@CSܥs6ԄT{/Kӈ>[Cg cIcibw??Y)n-"`X,%v' @,ں*VF\Zى&T\b-R_MU}^Uk\-"`X,/؝߿q7)^QS2ȝc]쥷EE"`X,c`Ϛ~Zޠx&pFԯg9Aq۞s jm`X,@vC#>E"`XD(,E"`d7,n=bcX,E J,8"`X,@vC#٤>?[+X+[yQђ_PR%=Z,E"p$ս>V"{a2`cOiRWTԒE"C.v܉kϒ=Ɵ\t|`zZ9] ߶,8UzJGڼ?JNJd]9T u<"`X,%v'O|G^;G\:DhÌ(=56[FIɉTPqAw*SW5|i_E"`XKȑ#ҥKR|Y=!:'0|cr;_M7mShZ*mܰcPTSJ_E"`XN؝XOL;wNz>?1% ]L 89|?H/ֹ=4:z4Q̺ǒbTaw:%w)))tA*_>,HHH 3|ʔ)!8/_NnOr=;lB9r-԰a`QmE!`)>;upqte]yFa ;E'S8o2צrN[> S>r#)q crW#-Sè/Ν;oCҶm۴]w&[B eʯK.>5X`AG7|3{TX13oN7󓋋/jԨAw}7wyf/4ڷoX"tM:z(S)}y<.55U :_tw:$0[n,X@W&.zj۶SM6'|rJ*={J*a/OTڵ딼EXe4D vz&\8UTIV۝קv5 .+ظ9꣨LꎐbQ&vh:Zv:~ao2k,zIg@yeiQj7j(& g#FK7H'N׮]K+*,YRFwcj򖔔I~"~fH[pa 8`j޼&/2լYSҬ'QU.'={˗O?!}Cۗ~i}]@/ Æ ;%g ~WXAYe RbUAԳP(P H}ߥ6 b79;8Ԡ} |Nc?m =MЄQҡU6lkF:,FSl޼~W-9lٲi)}b㫯?+WLՓ̟? 7 $\EaPB I\LL *UfϞMW\qvѢEw^]zi H(kF=zpҚMR!“_+z5V&?m:s [t-[WrH9lgll,}'^{Ϟ GY  ,KcH,ILLT,MqY*_~QW^yj׮bbB)V/RC׈b]/c2C=XbX/fvȉY&iX+Q1f<9^zҸ~a,EB ?.XXBXm9u߱9SP֯__w})-<>S*w'%o޼ym nYqu]ׯW,qV_~ Sq_VX5XʪeFoHɓu6m)Vlpa (Vō_Y xu&_k+&~ $?Lf{=^<0cT۩.I☐;fĮV%^~O Y<01ŋMV aE9x D$GV9wUa5G,^Xx*WEW2@PoɧCA*z~7nP?jp޼1mNׯ^zwfʹ_jHN+r\Ëmu8Wܳ>>?\@څ@&M3$ yì OO*|B'<ʹEKz1Dl?qc,΍koXޚ}qH:>B;!,^qSPxf+N݋/'yH )>}jժbg=YBE8` TF$1_x+3MID&br) $ Diob&O=$" 'O7v}GnD}?ES/2&f !j$R븐\ӡo%”҆yq:_^cfջۣ>Eu8A=+Wɼ8'M۬8 d\\t^G/bonWto 9DүlKɂe;^Xb:_ț(O*PL_J:gGsI>&w UM @}~m*V/@La"6dB$)n"Aʼn ?6Wm H/$ 8\/bRx!.W"vPjV:;℀w<<#i$;y8.zIxÆب%Jre~) !q Wē8]O^ߋ؝ ܲ6n86daxi%ыyŃH$7^>~uHaω1w\{}!`]s!n#n{peZ3YzX3}y:ybR{(b.C;@bw̮,Pd+a3EV~)*$b2$x*;)$X}zν8'@X4,׋؁Tz9.p D~/wkHD*"PND^-dtG@+/Hd ZW+/Wʆ+BxJ/tR]7/bw2pˊ{VH9rg„ Q`1_L"mgH;,nݪy?$v+Rk6Ӌ.X~a>7 Ѫ]Mb&ViҺ|Äv,IQM0}6v"WK=STOkEx_=)!aLjF{U,V!>Ć,$XOd*?ݻwѝsNCU /hO?W\ b u,"*! ȉӋa#R5ܸ^FrS<5O0\đ"+~nYDо}{}+HENnYq i au H\t^@ĎbEx)-`s?Lbי3gߒ^" Hn- <k+`AMԚQ-ECԊ9պѭ_f[49}0Yf/ߚa_[N}a|~6R_*ļVI}qJ qD`%Rc <9,6:c"`/M\!UX"v02G=a壘*C<(fq U$R&$Ad"^up|~w]0#n'aHqtd=+Ď7v7˹8nC /n_헼^u RUD/fE~nb_KQ} =5M'DŞ[bƀ9SZ{ ƷU˾oZխRRչ;Z6Z;Oa{pzp-}>ŤRs\mxT̰TJ]qĂ&ondkKe tHH QEP@cUVEV6-T7RBءLb2^Q bB[??rt$vf!a@^)L/^ 3WsCfPuYgv,HFa'7<ԡ"CMIRSc 9$?ۅ[+r Ծnwqgei yQ$ZHH=^}~ #i?6Ebս9xI7'7t 8rWHNB,c/ #v%_ ܆@4u)l8GlI)vY_He+^HE{{ʔ-I6m7SJ@*'MmkU*Ob lILپ}5"^ؕ<6 Js9+Sz oV{xs'~ UϠF Jo||X` 9z MЎӚ3h޷ptmA,{C®hX7q ԰aCb[B}z'X[E b,ܕ`tp:tu6J;w}.|&h{X}j*]*=>!nǎP=!x@bĮPYb'8٣E">؍?xյU-f1-'KN"ٱ5CTXZWPKަMO=%uhH5kߠy-0 +MEɞO u,$s>+w\_F}^bW8l,*A=bUJV]^C2Z+TOZٶ @Xb)b,> )AL hw`<WeߪKhoEAT`"IJ,(6u;h#W&ⷅ-FlyD^V+Kx\Y(NR-o qHI) "sDdM}Yahfn,?^cE*5ĦD\|T!\j~%Ck>louh,AKb81E :xgl8o{B;!:p{DU,c@deƸ=C@q&n \q/臃CPĭʝok(bD 8~=SY{EU/룎5kܡs_[b$v;5vk.bTxh =XCɃUT׺wԌBѴBNMKrą, XJC PsQ&t ZGh5GOh11+cc\,KDz196+*jkUDpWd\OgK.oů3N0# <8Ba*PAA8L8 wS' Hdp7s3Kv#b/D2!Iq$>TJZGC:AB$#k-r:YJ.!Ww(;?%"L,l4Q.Q:)}T=5DͥΣVR먧wjIjUj:@=ML2hJRNZ36N)};m6O[=GJ^ GgNNAK:ݺ]{]._wnݛz zqzzKv{OҷߦJc0 ;viFgkPnנ͠P0pa1&ƴgeFF#b:+Foˌ_7`2 73Ya`r7u6k:tia ;0W3,l6f2u̻-!-[tY2,,%-OX2dqXJV+*Jiժͪ:ٺz= &fMMۙQvbvg;ا/o`s(vuHw vXx͉vstuv;W9_rA]|\$.]ڇ ~ӕq-ru}tq+qkp{1vD#ΌHGl'3sgK//&[ 1 [?}||m}3}7dKg~~s/?gk@^gFFm(:5#%#*\0&D#)ljy*=ϝm"ÓׇߏȉEZugjx=}GMN^09F43z̪1wcb q *^CƏ$ad̄3I$&-K올LnIIHIy2c܈q]H3M5Swf|gwFiƍ M87tbct&'$dfȏW{xYz\ZsapK(Z)z2YN`Ϊ.qB-JK^Fn}3??5_ T_'ml1yvT1ʚ)=hh,4JG僢wSS7M:t鋧?-(a>C0ey3:5e͜s:F5:/o/%%+K^]|s?ZTT^zsa͋EEm=[LXvܽߏKۖ,۴\ƊV,^h՘UYV^3i͹ kkk;*c*ٮ[zUU6mXF+B6m6\ɖ[[#WWWl#n+d{3?a|ǧҝv6۽Uvsyo:׺Z;0pY=R?Aј~d?ѪcǖDfYsɜZ&95ԵֱmO9Sg8gN <{#.\}/޿ni{Qǯ_9y5x.\~#ƭ7;n o=Eݙ{pfsm_ODZa.>L|xNJ;??xjdz]]{s?š?C3e_K^uKo|7oޖ3y=MHXSw e|9(fgNi0.xo@=uv:υ!I!jQd{zhCxR}?;*v)껦Jn]X_E_@? (eXIfMM*>F(iNxjASCIIScreenshot$ pHYs%%IR$iTXtXML:com.adobe.xmp 362 Screenshot 144 :%#iDOTH(HH%$eo%wIDATx]`~Iл4( R (XAP+(D E.ҤJKBػ]HH`ofo߾yXȈA `0,Aܘ A'>OD4`+'a0{+`8c@ 3(A D] R`.(4+jB_Hx͏A ,QkMD۔-$|48ظXDjunJ}OYkA8wիW'NД)S\7oei "$Fk׮UViZF\N!kժE5jE%ہv/Th#MzC%8=mZ7¡y;*"h|2ɓUWYW1 D D}u>|8]pAZfGd3gNj߾=IF''KwD}/hpJebZְݸEE_gA G/N{NW&|}^=fBCC qqqx-%_^$A H2Qc~ĭ3MֲWdɒ:gC(& ']H;9jK2I{6fQ̵XK9iitOZ_۸qc:p?x n-a/hŊQ aÆԮ];^ztIZ`믿PQ#ۂd3}͛N>-$/_oN#GiTׯ_O+WFR< >ӦMU|5&5+Ub/_~nm߿?uK-d0N1h޽{ݴRJ-كI9gCKR(($NmBs zCkR0Qth_};}knkzR4… ӑ#GⵅƍŻMԾHիWwaMf,ؒBԨs5kVnX*I, wd%j>Z'C@1o÷FVGi2fbU(&:BOnhe*T]Zm=M'l$f4idm%bo}h~Ƭ$%? {I!j <` iІQZ5GԨO_E 4> %9p$]&Mv B#G Yoњ5kȰf͚OϾ6}>=})QJ4GQU3Q  ?{v)YAW.)}Y)OL 'G!> dqLn#jء5>0V7 ^T8^doLl1<NzbL Ze_Bjw^Q ZT$~(YV [ #'= ZZ49&43^Ie}5&r*o~ԨW:Lb viϟ?/!lZ5+V:DإnԨQ+ưuCJp̟?x}7<@2eӒ̏A p[HvaEI8~(v.CWC` 9C*-fC__m'b`w|٬Art2Ѡ/ Y/7oLL+\G8AkI±o߾R޵kEDDIQ_~e5kzx2&4=۽Bm6 SWϕ(ہ?*q7QvP쪧fnfo;ljhOZZv8Ƶ-?fn^l;On:Yh?W:cKY֨SZYÃZ0&XoUW)R$Yxf&eOzLA #(8m*U*+HaغeűBҼGtuڳV,xcZ~C[PPq#A.D ʹcV7KnX1f1<;s٬Qi]  #j{v:zEffY!;fUkEkx7s,A $jA p#`0o0> 8?s@#`:ϑA p#`0o0>S-4Fzhzo1pצyA.Au2("%" ]_IVW4[U46]:ʖ3W24c0ܭN*~Կkx o@zus5gVFN)K"9YӤ5d1A C^awd:yhҽʱg,;&ef$)ә()9KyO9vU'\Kaaa;wnzWYӎA IJ>}:ٳE+̠C6o\ W6E\=EEDsiйP+ֿfh ?w&I*} _ALJy~ &/7ߤS:6#;wf͚,fTjUjڴ)͜i̘,F@$j|5{ذapk/$|z7s~eɀf~'ʘ=SJw7B]'!H[=O?|3C!wP~!U&3iym8r˯ 6ʫQ9rߋ_ާߢLO 3z;~ٳ@:uDm۶6wq֬YC3&d+qޯY&ըQI~S @`)J)35qE]F1WQ^0u& e͙W_pAiV5kWcʅK4ig*^h5jH᧟; On%+V-[ 4-K1bF 5ihЮ!I:/o߾=%tQ1E͹Y1A9fL1ԵëM[!MHikŎݚGi#mv?oW +5={FGGS(@Iҁy+$ /x1lM˸%lgΜumk`'t.aBm&oɼcZb{Si+hbF9GԊS(*ۨ%k#P.͊,[/]dq cԶ{ekJ:**KZr؆AԥJ:P6mH^ztIZ`4믿ݻiȐ!Ge3iҦM?IC?x ϶j-zLCē֭[)[lTrew0ajժ$nL9b6C|Ţ(TR4h@󎴯ǀP߹իiڵR駟ZjQv(C &qp?.߹f(ظq`v3fH'M1I=j M,Z5뛝@Dݡvdys6ђ+y[!$t)[ ڒyHAj̋aN0ߢi=t); 7D@l1ч T8Hs̑,ho6& g͚v#䋯׮][i_&M͐ׄ}ȑ# d ;-.e2eDCD=^hB>4!O O 8v `uО1V"yΝ'nק_~Mu.pC&٘>p)o6r/D*& >JtI>=-_U&' ';Uŋѣ i:ePfeaja*ryf͑ߺnզ^upxu|The7ʑ^U8ӧƪN/RmιS'k`PUH"BU_ŪXvS`AIGָەfSIΝ;K>_jт _jĈ/zcBp#'y IZC'kLH(lk7W[8&KĉUn]I#2m4IgBuK 5suY5tPŞ!RO>qA9r(hVLl|<#?(4QԢpO :X\{RoBjĉ&NGDO Ng'M9eJ !cǫkܕS𣻐&jM$FEZۯFD|4MB s穣ۦKHsձ U_-$~ ׫,?ҫٳg LM-[IDD{ -Oƥ "Kt;vtK!l~[p\y ;q…V!R$o8(Qn`[nn[q38YCw:3fzlpM/7#H=6+WX>.Izc2'M:D>kbWLܼ/kqMEϴPhG[>%4|ʔ>dW;6q5u:yS"y19̙K{=~mGWQŘ#.p*__l}Qyfkoq0Iü*?&YzYyeY O -0m0I.a Xs_,2}L2%ނ#oqNcv݋W9Y~Aj\5d VaT5K~ ߬&̆V_I[y2e`C5l4iZJZVu7 `Ť) K`ҒME2a1iINo'OYtIFز!';Q\#"E d :t3"CQ=GZRϟUWep⛦Ss{TkFtrflK9m<̲-W}jOܞ5=O)ֲAxI1yӏLRnI=W\xyc0^1֋z4 ]mM``ZÞ%!Ӈ7 F m1Qf$o|nѦ~iwܹ:e6PpLL䡌iğCaF&HGt-ۤ =bXI'vъO])IM֙լ/V+~h.YEo8' 9E*'~Q'#O.jjvjC-=8WyĩƘ{c\%5,D&%ug5n]&jpY#XyZ+67oScSm(kD' +kW'\i&UD 1iƖ D멚S }XlHiJ[^V&*qPGfrI,glgs8<:Clse۱iV=:W^#ob/.9B{@~,V 6q m64[o⯿| sn-O0"jL }L(z&j{TٿYxa5Yr(Ĥ y(VD~doWs_䣜ԁ]GC&L)5ʶ'{G-3ګiݪCKG+hZ!lSTġ fewq75M,K&i.mÀ ָ֚iJa-eu]MiM%yA+ 7ƀ-%7+ۓQ7NO++7"A~*ZK֑$_)xά],yt;oT\3QOCGFRwQvk㤞 VU>lÔmWZ?M5KUxPcu0eD͛mkڼIԾDZ=N$τ wl~USkɚ%Ä\`<9%w5\/b_D ?gMh.eW"'*/NqN:k.W5⏌CԼt[J; r%d_^0 j dk.\˶̀ vOBؠqk;4ȦDx &t./ww+7/RQIٿ=ܕDЍ\Ya0qˠԅ*aʾ{8pe;gC Gqkd6A4wxٝL&ɫ.]"ky Ә l^FA *,}V]Mc{'L.HwCXpE 4ܸoL+V:Q᪂* Y&j,a+4<ꀬ#CjxUC ].{&Di֛.>}(=qxj֓'OEg]!HBq?dWy!ڵ}7ȕ5)>xcG NAAXj:\N{i)gL| j-jlpKCnqk'c9Y+OnCZh[;yX&d˛.,g-Z@R '-f !'zXxkR* {4@PgkNN͔%#e̜ѩs 5W[3;;g&mk:+'_Q{vF4'K3̏UwXi}[ x`U"WXLmBwJHqdpH>P~4/ qr4WHBN6$c\TTɆ9GGb**;mdĉ&Hޤ%Ո֖kB^m2q@#`İ]9{+⧔2 )!״n0nCԷ i `0,S_ӺA `e Q2A NY|MA0D} @"`:e5 [FoI$ IDAT] xDɡI(rHDM>51 rr\} FEGԠE(9]vك=N5fgf3stjg42Y ,} 'BhaQGH4wb4FFVy}&8[GlHsRbׁ}MRkuj}K ~:[hG) VC?޺8o,kLhyiuďYy˩4-d86<."J2[f?dOEM..aQ%-[g㝠ն5Y_]巑eԙ7h0@W5Z 9״c RLbRwcYG@lc=6mڤ obE]#~L$@0Z8[m X7Z?0omu)#Mx03Ȋi?| f%0@@1QI^0f}Rwc1X,kSe {܅ZaV0M$9l盟櫁u]v _ͷ3Yj}5\>haÆ,Ւ}&0<|J#^25 ^ :O]bc3S<%\I}tӼkkkUyy:SwG3T-ZP_|EjɾXqjx̖58/5JR`Mqs@,{lb$IgK,]-[|(&}ꨣJ0}I%WU}W &h%a ]z-ܢ?C_YUUU'~E5wwʕ\ʴjՊyu3gNH(**;С[v_]t&0_^X\ }qD r\S/ZO:Uwؑaisҏc}9Xv *8Gp^9)Lg\22a T-&%0n+Y`ܦtksE<+,҂  M޽;tgp\`,.*#<_>묳xB5YwNҥ-su銊 Wt"sd\ hnVD#m̙j^z)3f ~*'WyiV6޾㎳FִkYֻ/ z\E߉܀ʢ7w#y>餓\W0p@vΡT5k5z6: 0.@v_.+Dm8^Obz  '8.Ν;ەW^i_(nM9 ArA#Vtqq1\Cu59!TٳGϞ= oɮ2Df1bǶ.X @K/麺:n}mXB4eM 7;vH1Bwy~@}vo|۵kkr5kkӑO} 8 P~Gn;x =rqtK~Flζ(Fmz_sq adя~x$w]OԽ{v4@KGfpq 'p:\^?~+-]:/JW;_{=?,t<x O3ù%2Ev.sug*u .=Xg5u-.e˖. 'jHGrD _|Yh%/DEBc`Tu ]$}_rÍ/T;- &7'ɽjmS@:/J?W;_͛Jae\n PR!:@OJk5Oڶ<gQ5SܒW%]do9y rAϝN?t`0#_H߿Ιfv-U x̬A2 9sOI;v{F aHҥWqoBk$PjX"s @\ZjźhSF?mD@́_Q|@~EEhݺu#03e _1 N7ڵk9 r~&˗+cPKj1n8/ox\y7u?d?50\M}k*<`,qr&i8J~XНv^9խ[orldPt>}3?0s4}1',(j9ɀȓ׿u;uIiNt022r7@ -7 v3w8]5Na5Et yHM |,K' 0a_wc/;%+0+BڀId\(d`UW]YH{Ǒ ͛7A{rD~@obXs.RE(\m'I?T@Q7nGy~ dp8ud9XԌTt1(-)zo"0QEhP/Yڹs'f RYf.6[~ xKp\dU~~n_ȿE咙JOS>VK+&|yVPkN5cR hKԭ_3@\G; Ȕnztrj#X.'%hE~aEz%VuȲV?Ov^ :U 4ȧ~_U,x Sr d&5@:v â?7cŅϲ7#r5zJ5,|+*}{ nXHiN&W\RVX9r"?yJ^ÐZ>[pq1P[ԴХE.[הnG`O>hʍ\~-j^n6k!"zۛiq.&R5}8LRsԀ9?j3ӌJQ@0jm;e*N \4ט֭[+h/.lje"{[зL684%ʀ#|#e,aڪ  ZsqSрр@@ 0n4`4`4k εM}FFF5`: рр@5`:7   Ԁ 3FFFր\k<`}KYB03tG[ޥWѰ  DMvDQ? w.JŻ::P2\>PQ[#}EՋK_Q6mG܅˶a_0ѲPقfi8R\#ZjXudYEz[%S;wV*o7?-~6^IJm~'-G軐NnJxlaaaԚ?L"&,95ÚD2έ*%KBg8vaȫ02>|>~~i8->ӑAM 挍*9mnw[޷uA+uEŃ, 3h>|~74 i3Y5 Pii})0֭[}z]tnٲ%oGz"Zl&+˦T_z饚]> H[2飻lmOq_>FȣkJQ^ Hݯ_?/k8^~h۶-rchu6m_q^M_N .@s9/ aȐ!v{p[oitRn?}/SCŋƍ J5yр۷c9F\O4IH.kL6ś̙>餓X&:%U( X&"q1a' c<17+@-ǁH@ M_ָXq\EQFq~֬YC6Gm#^pn|loFNj.%.))qUTXwqvN]`083X~=x :={Cq{:ɢryqUViY`-rh %$guEǵ},~Wq/TԩSm z,**sBE"L!Y1@%fJh:eMvYE-גm7*F[Yc$/E-..LGR?.Y ԰<͛g?+ L\"::l٢Of0'4zh.i :|/`Q'BJO7og}6>)ͺ`Z2nVg?l03n N " *8W',PϞ=%9p8Iۻwog#!'%: 2=Ofzu5ay,Z]rвxKOA,lpǨ1A)hT Xaxkv~SOv%0@Q.Ji0@g„ NQ>fyא׽V{n4NX=&2Lj3 t)0` sP7 @ `l)Yj Br% ‚Kb:Z|hZnm \C~r':)@-L7o2|MHM@tRe a׏ӻ/<,a2_z%E c2+ӯ}H3@ -fQa߁WWFq l M )]HDžJR? RuW' sεj PviBIrW&Z 3kջ/<;a_CK_c2+ΆV3(sX#u 0 O6P Pm?+JDe  xE?}t|Ӛ7nlob]@F2$PtM?;W~HR8^2|zLae= jeNR'ݥ8M<$F@|w林ˢFB AH.L afD8 qJ?/-wnTOd+1b ?Zv~Wuqq-'r،>;=a|faLA#W&Z;3qdWhy>=QyZrO_0գ5{)LL PgJY3yp bf%Ws8F`&`ojK^ &\ "#"tLT?Lϒ2o2( 1`ރ>hׅiuB=O>W<1 3@@»%pe%\s15c}Ŋ6r `pF kҮ/.UU Eo^pՌWXt裏;CP+~H8*mKXHP=P8IH Xڂ҄DMP^.O@ ux= Q)tw,@{#]JEsðu{BjL8L0]={TdEl7`Y׊,k^yv:Iwk[-mge0fUlYzM8eqjp2r}ZjYfl5%osbϟqX,h?6Ox( %HESi(qĺDjX} -FBv3Gs+&pYxQ'-[)Ʀ^{F4hӈO}x"IK 1N`OO ˘Ȣٲ)-^dܴh'u֭MS4vI3@99\D-ZjM|ӔiXoi}EjAwfh 0@cZg4`4`4@hhhhh 0@#ce4`4`4׀js*   D\#~L Pshhh 0@dg4`4`4`ڜFFF׀ڲVg6SрY>mU{U;Թ32UQ IENDB`kytensor-2.49.0.7/client/src/grpc_generated/java/library/000077500000000000000000000000001527052371200232405ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/library/pom.xml000066400000000000000000000044511527052371200245610ustar00rootroot00000000000000 4.0.0 triton.inference grpc-client-library 0.0.1-SNAPSHOT 1.8 1.8 1.8 UTF-8 UTF-8 io.grpc grpc-netty-shaded 1.59.1 io.grpc grpc-protobuf 1.59.1 io.grpc grpc-stub 1.59.1 org.apache.tomcat tomcat-annotations-api 11.0.0-M13 provided javax.annotation javax.annotation-api 1.3.2 kr.motd.maven os-maven-plugin 1.6.2 org.xolstice.maven.plugins protobuf-maven-plugin 0.6.1 com.google.protobuf:protoc:3.12.0:exe:${os.detected.classifier} grpc-java io.grpc:protoc-gen-grpc-java:1.59.1:exe:${os.detected.classifier} compile compile-custom kytensor-2.49.0.7/client/src/grpc_generated/java/library/src/000077500000000000000000000000001527052371200240275ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/library/src/main/000077500000000000000000000000001527052371200247535ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/library/src/main/proto/000077500000000000000000000000001527052371200261165ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/java/library/src/main/proto/.keep000066400000000000000000000000001527052371200270310ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/javascript/000077500000000000000000000000001527052371200230215ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/grpc_generated/javascript/README.md000066400000000000000000000051031527052371200242770ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Example Javascript Client Using Generated GRPC API This sample script utilizes `@grpc/proto-loader` to dynamically load `.proto` files at runtime, and `@grpc/grpc-js` to implement gRPC functionality for Node.js. ## Prerequisites Node 12+ is recommended ## Usage Clone the [triton-inference-server/common](https://github.com/triton-inference-server/common/) repository: ``` git clone https://github.com/triton-inference-server/common/ -b common-repo ``` \ should be the version of the Triton server that you intend to use (e.g. r21.05). Copy __*.proto__ files to `./proto` ``` mkdir proto cp common-repo/protobuf/*.proto ./proto/ ``` Install dependencies with ``` npm install ``` Finally, run with ``` node client.js ``` - `host` is the server where triton inference server is running. Default is `localhost`. - `port` is the grpc port of triton inference server. Default is `8001`. kytensor-2.49.0.7/client/src/grpc_generated/javascript/client.js000066400000000000000000000122011527052371200246310ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE const grpc = require('@grpc/grpc-js'); const protoLoader = require('@grpc/proto-loader'); const util = require('util'); const PROTO_PATH = __dirname + '/proto/grpc_service.proto'; const PROTO_IMPORT_PATH = __dirname + '/proto' const GRPCServicePackageDefinition = protoLoader.loadSync(PROTO_PATH, { includeDirs: [PROTO_IMPORT_PATH], keepCase: true, longs: String, enums: String, defaults: true, oneofs: true }); function BufferToInt32Array(buf) { newArray = new Int32Array(buf.byteLength / 4) dv = new DataView(buf.buffer) for (let i = 0; i < newArray.length; i++) { newArray[i] = dv.getInt32(buf.byteOffset + (i * 4), littleEndian = true); } return newArray } const inference = grpc.loadPackageDefinition(GRPCServicePackageDefinition).inference; async function main() { const argv = process.argv.slice(2) const host = argv.length > 0 ? argv[0] : "localhost"; const port = argv.length > 1 ? argv[1] : "8001"; const model_name = "simple"; const model_version = ""; const batch_size = 1 const dimension = 16 const client = new inference.GRPCInferenceService(host + ':' + port, grpc.credentials.createInsecure()); const serverLive = util.promisify(client.serverLive).bind(client); const serverReady = util.promisify(client.serverReady).bind(client); const modelMetadata = util.promisify(client.modelMetadata).bind(client); const modelInfer = util.promisify(client.modelInfer).bind(client); serverLiveResponse = await serverLive({}); console.log("Triton Health - Live:", serverLiveResponse.live); if(!serverLiveResponse.live) { console.error("Triton is not Live") process.exit(1); } serverReadyResponse = await serverReady({}); console.log("Triton Health - Ready:", serverReadyResponse.ready); if(!serverReadyResponse.ready) { console.error("Triton is not Ready") process.exit(1); } modelMetadataResponse = await modelMetadata({ name: model_name, version: model_version }); console.log("\nModel Info:", modelMetadataResponse) // Input Data // Use input dimension [batch_size, dimension] const input0_data = Array(batch_size * dimension).fill().map((element, index) => index) const input1_data = Array(batch_size * dimension).fill().map((element, index) => index) const input0 = { name: "INPUT0", datatype: "INT32", shape: [batch_size, dimension], contents: { int_contents: input0_data } } const input1 = { name: "INPUT1", datatype: "INT32", shape: [batch_size, dimension], contents: { int_contents: input1_data } } const modelInferRequest = { model_name: model_name, model_version: model_version, inputs: [input0, input1], outputs: [ { name: "OUTPUT0" }, { name: "OUTPUT1" } ] } const outputs = await modelInfer(modelInferRequest); output_data = outputs.raw_output_contents.map(BufferToInt32Array); console.log("\nChecking Inference Output") console.log("-------------------------") for (let i = 0; i < dimension; i++) { console.log(input0_data[i] + " + " + input1_data[i] + " = " + output_data[0][i]) console.log(input0_data[i] + " - " + input1_data[i] + " = " + output_data[1][i]) if (((input0_data[i] + input1_data[i]) != output_data[0][i]) || ((input0_data[i] - input1_data[i]) != output_data[1][i])) { console.error("Unexpected results encountered") process.exit(1); } } } main() kytensor-2.49.0.7/client/src/grpc_generated/javascript/package-lock.json000066400000000000000000000252271527052371200262450ustar00rootroot00000000000000{ "name": "simple-grpc-tritonclient", "version": "1.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { "@grpc/grpc-js": { "version": "1.8.20", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.20.tgz", "integrity": "sha512-SPse1wE4PcIFojOISsAnrWXCBsCBwDdcDqz2SS0T8nBSxg9jwmCK70Jy7ypRn2nIspwLy3Ls5TNaKNHo+6dF8A==", "requires": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" } }, "@grpc/proto-loader": { "version": "0.7.8", "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.8.tgz", "integrity": "sha512-GU12e2c8dmdXb7XUlOgYWZ2o2i+z9/VeACkxTA/zzAe2IjclC5PnVL0lpgjhrqfpDYHzM8B1TF6pqWegMYAzlA==", "requires": { "@types/long": "^4.0.1", "lodash.camelcase": "^4.3.0", "long": "^4.0.0", "protobufjs": "^7.2.4", "yargs": "^17.7.2" }, "dependencies": { "cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "protobufjs": { "version": "7.2.4", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz", "integrity": "sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==", "requires": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" }, "dependencies": { "long": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" } } }, "yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "requires": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "yargs-parser": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" } } }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" }, "@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" }, "@protobufjs/codegen": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" }, "@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" }, "@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", "requires": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" } }, "@protobufjs/float": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" }, "@protobufjs/inquire": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" }, "@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" }, "@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" }, "@protobufjs/utf8": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "@types/long": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" }, "@types/node": { "version": "17.0.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.0.tgz", "integrity": "sha512-eMhwJXc931Ihh4tkU+Y7GiLzT/y/DBNpNtr4yU9O2w3SYBsr9NaOPhQlLKRmoWtI54uNwuo0IOUFQjVOTZYRvw==" }, "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "requires": { "color-convert": "^2.0.1" } }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "requires": { "color-name": "~1.1.4" } }, "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, "lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" }, "long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, "string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "requires": { "ansi-regex": "^5.0.1" } }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "requires": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" } } } kytensor-2.49.0.7/client/src/grpc_generated/javascript/package.json000066400000000000000000000003371527052371200253120ustar00rootroot00000000000000{ "name": "simple-grpc-tritonclient", "version": "1.0.0", "description": "Simple gRPC client for NVIDIA Triton in Node.js", "dependencies": { "@grpc/grpc-js": "^1.8.20", "@grpc/proto-loader": "^0.7.8" } } kytensor-2.49.0.7/client/src/java-api-bindings/000077500000000000000000000000001527052371200211655ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java-api-bindings/scripts/000077500000000000000000000000001527052371200226545ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java-api-bindings/scripts/install_dependencies_and_build.sh000077500000000000000000000151741527052371200314000ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. USAGE=" usage: install_dependencies_and_build.sh [options] Installs Maven, Java JDK and builds Tritonserver Java bindings -h|--help Shows usage -t|--triton-home Expected Trition library location, default is: /opt/tritonserver -b|--build-home Expected build location, default is: /tmp/build -v|--maven-version Maven version, default is: "3.8.4" -c|--core-tag Tag for core repo, default is: "main" -j|--jar-install-path Path to install the bindings .jar --javacpp-branch Javacpp-presets git path, default is https://github.com/bytedeco/javacpp-presets.git --javacpp-tag Javacpp-presets branch tag, default "master" --enable-developer-tools-server Include C++ bindings from developer_tools repository --keep-build-dependencies Keep build dependencies instead of deleting " # Get all options: OPTS=$(getopt -l ht:b:v:c:j:,help,triton-home,build-home:,maven-version:,core-tag:,jar-install-path:,javacpp-branch:,javacpp-tag:,enable-developer-tools-server,keep-build-dependencies -- "$@") TRITON_HOME="/opt/tritonserver" BUILD_HOME="/tmp/build" MAVEN_VERSION="3.8.4" export MAVEN_PATH=${BUILD_HOME}/apache-maven-${MAVEN_VERSION}/bin/mvn TRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG:="main"} JAVACPP_BRANCH=${JAVACPP_BRANCH:="https://github.com/bytedeco/javacpp-presets.git"} JAVACPP_BRANCH_TAG=${JAVACPP_BRANCH_TAG:="master"} CMAKE_VERSION=${CMAKE_VERSION:="3.21.1"} export JAR_INSTALL_PATH="/workspace/install/java-api-bindings" # Note: True != 0 and False == 0 export INCLUDE_DEVELOPER_TOOLS_SERVER=0 KEEP_BUILD_DEPENDENCIES=1 for OPTS; do case "$OPTS" in -h|--help) printf "%s\\n" "$USAGE" exit ;; -t|--triton-home) TRITON_HOME=$2 echo "Triton home set to: ${TRITON_HOME}" shift 2 ;; -b|--build-home) BUILD_HOME=$2 export MAVEN_PATH=${BUILD_HOME}/apache-maven-${MAVEN_VERSION}/bin/mvn shift 2 echo "Build home set to: ${BUILD_HOME}" ;; -v|--maven-version) MAVEN_VERSION=$2 export MAVEN_PATH=${BUILD_HOME}/apache-maven-${MAVEN_VERSION}/bin/mvn echo "Maven version is set to: ${MAVEN_VERSION}" shift 2 ;; -c|--core-tag) TRITON_CORE_REPO_TAG=$2 echo "Tritonserver core branch is set to: ${TRITON_CORE_REPO_TAG}" shift 2 ;; -j|--jar-install-path) JAR_INSTALL_PATH=$2 echo "Bindings jar will be set to: ${JAR_INSTALL_PATH}" shift 2 ;; --javacpp-branch) JAVACPP_BRANCH=$2 echo "Javacpp-presets branch set to: ${JAVACPP_BRANCH}" shift 2 ;; --javacpp-tag) JAVACPP_BRANCH_TAG=$2 echo "Javacpp-presets branch tag set to: ${JAVACPP_BRANCH_TAG}" shift 2 ;; --enable-developer-tools-server) export INCLUDE_DEVELOPER_TOOLS_SERVER=1 echo "Including developer tools server C++ bindings" ;; --keep-build-dependencies) KEEP_BUILD_DEPENDENCIES=0 echo "Including developer tools server C++ bindings" ;; esac done set -x if [ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -ne 0 ]; then # install cmake and rapidjson apt-get update && apt-get install -y gpg wget && \ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ gpg --dearmor - | \ tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null && \ . /etc/os-release && \ echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | \ tee /etc/apt/sources.list.d/kitware.list >/dev/null && \ apt-get update && \ apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7* rapidjson-dev fi # Install jdk and maven mkdir -p ${BUILD_HOME} cd ${BUILD_HOME} apt update && apt install -y openjdk-11-jdk wget https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz tar zxvf apache-maven-${MAVEN_VERSION}-bin.tar.gz export PATH=$PATH:$PWD/apache-maven-${MAVEN_VERSION}/bin/ # Clone JavaCPP-presets, build java bindings and copy jar to /opt/tritonserver cd ${BUILD_HOME} git clone --single-branch --depth=1 -b ${JAVACPP_BRANCH_TAG} ${JAVACPP_BRANCH} cd javacpp-presets # Remove developer_tools/server related build if [ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -eq 0 ]; then rm -r tritonserver/src/gen rm tritonserver/src/main/java/org/bytedeco/tritonserver/presets/tritondevelopertoolsserver.java fi mvn clean install --projects .,tritonserver mvn clean install -f platform --projects ../tritonserver/platform -Djavacpp.platform=linux-x86_64 # Copy over the jar to a specific location mkdir -p ${JAR_INSTALL_PATH} cp ${BUILD_HOME}/javacpp-presets/tritonserver/platform/target/tritonserver-platform-*shaded.jar ${JAR_INSTALL_PATH}/tritonserver-java-bindings.jar if [ ${KEEP_BUILD_DEPENDENCIES} -eq 1 ]; then rm -r ${BUILD_HOME}/javacpp-presets/ rm -r /root/.m2/repository fi set +x kytensor-2.49.0.7/client/src/java/000077500000000000000000000000001527052371200166235ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/CMakeLists.txt000066400000000000000000000053621527052371200213710ustar00rootroot00000000000000# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(java-clients LANGUAGES C CXX) # # Options # set(TRITON_VERSION "0.0.0" CACHE STRING "Version for the clients") option(TRITON_ENABLE_JAVA_HTTP "Enable JAVA HTTP client libraries" OFF) option(TRITON_ENABLE_EXAMPLES "Include examples in build" OFF) option(TRITON_ENABLE_TESTS "Include tests in build" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in libraries" OFF) if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() if(TRITON_ENABLE_JAVA_HTTP) find_package(Java REQUIRED) include(UseJava) set(PROJECT_TARGET_DIR "${CMAKE_CURRENT_SOURCE_DIR}/target") set(PROJECT_JAR "java-api-0.0.1.jar") set(EXAMPLES_PATH "src/main/java/triton/client") add_custom_command( OUTPUT "${PROJECT_TARGET_DIR}/${PROJECT_JAR}" COMMAND mkdir -p ${CMAKE_INSTALL_PREFIX}/java COMMAND mvn clean install -Ddir=${PROJECT_TARGET_DIR}/examples COMMAND cp -r ${PROJECT_TARGET_DIR}/examples ${CMAKE_INSTALL_PREFIX}/java COMMAND cp -r ${PROJECT_TARGET_DIR}/${PROJECT_JAR} ${CMAKE_INSTALL_PREFIX}/java WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} VERBATIM ) add_custom_target( JavaProject ALL DEPENDS "${PROJECT_TARGET_DIR}/${PROJECT_JAR}" ) endif() # TRITON_ENABLE_JAVA_HTTP kytensor-2.49.0.7/client/src/java/README.md000066400000000000000000000121621527052371200201040ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Triton Java API This is a Triton Java API contributed by Alibaba Cloud PAI Team. It's based on Triton's HTTP/REST Protocols and for both easy of use and performance. This Java API mimics Triton's official Python API. It has similar classes and methods. - `triton.client.InferInput` describes each input to model. - `triton.client.InferRequestedOutput` describes each output from model. - `triton.client.InferenceServerClient` is the main inference class. Currently the Java API supports only a subset of the entire Triton protocol. A minimal example would be like: ```java package triton.client.example; import java.util.Arrays; import java.util.List; import com.google.common.collect.Lists; import triton.client.InferInput; import triton.client.InferRequestedOutput; import triton.client.InferResult; import triton.client.InferenceServerClient; import triton.client.pojo.DataType; public class MinExample { public static void main(String[] args) throws Exception { boolean isBinary = true; InferInput inputIds = new InferInput("input_ids", new long[] {1L, 32}, DataType.INT32); int[] inputIdsData = new int[32]; Arrays.fill(inputIdsData, 1); // fill with some data. inputIds.setData(inputIdsData, isBinary); InferInput inputMask = new InferInput("input_mask", new long[] {1, 32}, DataType.INT32); int[] inputMaskData = new int[32]; Arrays.fill(inputMaskData, 1); inputMask.setData(inputMaskData, isBinary); InferInput segmentIds = new InferInput("segment_ids", new long[] {1, 32}, DataType.INT32); int[] segmentIdsData = new int[32]; Arrays.fill(segmentIdsData, 0); segmentIds.setData(segmentIdsData, isBinary); List inputs = Lists.newArrayList(inputIds, inputMask, segmentIds); List outputs = Lists.newArrayList(new InferRequestedOutput("logits", isBinary)); InferenceServerClient client = new InferenceServerClient("0.0.0.0:8000", 5000, 5000); InferResult result = client.infer("roberta", inputs, outputs); float[] logits = result.getOutputAsFloat("logits"); System.out.println(Arrays.toString(logits)); } } ``` ## Supported and Unsupported Java client features ### Supported Java client features: HTTP client is supported with limited capability. Currently supported: - Synchronous inference requests GRPC has very limited support. Please see [grpc generated Java client](https://github.com/triton-inference-server/client/tree/main/src/grpc_generated/java) for details ### *Unsupported* Java client features: GRPC client: - Full feature Java GRPC client and corresponding tests HTTP client: 1. Asynchronous inference requests 2. Streaming inference requests 3. SSL or HTTPS protocol communications 4. Requesting/Receiving Server Metadata Information 5. Requesting/Receiving Model Metadata Information 6. Requesting/Receiving Model Inference Statistics 7. Sending inference requests using Shared Memory (System, GPU) 8. Sending multiple synchronous inferences on server 9. [Extensions](https://github.com/triton-inference-server/server/tree/main/docs/protocol) are not supported ## Building Java Examples The Java examples can be found in [examples folder](src/main/java/triton/client/examples/). To compile these examples, simply run: ```bash $ cd client/src/java $ mvn clean install -Ddir=examples ``` Then you will be able to find the examples located in your target folder: `examples` and the compiled jar at `target/java-api-0.0.1.jar`. kytensor-2.49.0.7/client/src/java/pom.xml000066400000000000000000000111651527052371200201440ustar00rootroot00000000000000 4.0.0 triton.client java-api 0.0.1 triton_java_api jar Java API for Triton Inference server. 1.8 1.8 org.apache.httpcomponents httpasyncclient 4.1.5 commons-io commons-io 2.11.0 com.fasterxml.jackson.core jackson-databind 2.15.2 com.google.guava guava 30.1.1-jre maven-assembly-plugin build-SimpleInferClient package single jar-with-dependencies triton.client.examples.SimpleInferClient SimpleInferClient false ${dir} build-SimpleInferPerf package single jar-with-dependencies triton.client.examples.SimpleInferPerf SimpleInferPerf false ${dir} build-MemoryGrowthTest package single jar-with-dependencies triton.client.examples.MemoryGrowthTest MemoryGrowthTest false ${dir} kytensor-2.49.0.7/client/src/java/src/000077500000000000000000000000001527052371200174125ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/src/main/000077500000000000000000000000001527052371200203365ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/src/main/java/000077500000000000000000000000001527052371200212575ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/src/main/java/triton/000077500000000000000000000000001527052371200225765ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/000077500000000000000000000000001527052371200240545ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/BinaryProtocol.java000066400000000000000000000102771527052371200276740ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client; import com.google.common.base.Preconditions; import com.google.common.primitives.Booleans; import com.google.common.primitives.Bytes; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.function.BiConsumer; import triton.client.pojo.DataType; /** * @author xiafei.qiuxf * @date 2021/4/20 */ public class BinaryProtocol { private static byte[] setBinaryDataImpl( DataType dataType, List data, BiConsumer consumer) { byte[] binaryData = new byte[data.size() * dataType.numByte]; ByteBuffer buf = ByteBuffer.wrap(binaryData); buf.order(ByteOrder.LITTLE_ENDIAN); for (T datum : data) { consumer.accept(buf, datum); } Preconditions.checkState(buf.position() == binaryData.length); return binaryData; } public static byte[] toBytes(DataType dataType, boolean[] data) { return setBinaryDataImpl( dataType, Booleans.asList(data), (buf, b) -> buf.put(b ? (byte) 1 : (byte) 0)); } public static byte[] toBytes(DataType dataType, byte[] data) { return setBinaryDataImpl(dataType, Bytes.asList(data), ByteBuffer::put); } public static byte[] toBytes(DataType dataType, short[] data) { return setBinaryDataImpl( dataType, Shorts.asList(data), ByteBuffer::putShort); } public static byte[] toBytes(DataType dataType, int[] data) { return setBinaryDataImpl(dataType, Ints.asList(data), ByteBuffer::putInt); } public static byte[] toBytes(DataType dataType, long[] data) { return setBinaryDataImpl(dataType, Longs.asList(data), ByteBuffer::putLong); } public static byte[] toBytes(DataType dataType, float[] data) { return setBinaryDataImpl( dataType, Floats.asList(data), ByteBuffer::putFloat); } public static byte[] toBytes(DataType dataType, double[] data) { return setBinaryDataImpl( dataType, Doubles.asList(data), ByteBuffer::putDouble); } public static byte[] toBytes(DataType dataType, String[] data) { ByteArrayOutputStream o = new ByteArrayOutputStream(); for (String datum : data) { byte[] bytes = datum.getBytes(StandardCharsets.UTF_8); try { o.write(Util.intToBytes(bytes.length)); o.write(bytes); } catch (Exception e) { throw new RuntimeException(e); } } return o.toByteArray(); } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/InferInput.java000066400000000000000000000311321527052371200270020ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client; import com.google.common.base.Preconditions; import com.google.common.primitives.Booleans; import com.google.common.primitives.Bytes; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; import com.google.common.primitives.UnsignedInteger; import com.google.common.primitives.UnsignedLong; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.List; import java.util.function.BiConsumer; import java.util.function.Function; import triton.client.pojo.DataType; import triton.client.pojo.IOTensor; import triton.client.pojo.Parameters; /** * This class describes an input tensor feeding to inference server, including * it's name, shape, datatype and the actual tensor data. */ public class InferInput { /** * Name of inference input. */ private final String name; /** * Shape of inference input. */ private final long[] shape; /** * Data type of inference input. */ private final DataType dataType; /** * Other parameters on this inference input. */ private final Parameters parameters; /** * Binary representation of tensor data of this input tensor if it's in binary * format. */ private byte[] binaryData; /** * Tensor data of this input tensor if it's in JSON format. */ private Object[] data; /** * Number of elements in this inference input. */ private final long numElement; /** * Create an inference input. * * @param name name of input. * @param shape shape of input. * @param dataType data type of input */ public InferInput(String name, long[] shape, DataType dataType) { this.name = name; this.shape = shape; this.dataType = dataType; this.numElement = Util.elemNumFromShape(shape); this.parameters = new Parameters(); } private void checkShape(int len) { Preconditions.checkArgument( len == this.numElement, "Data tensor's size [%s not consist with shape [%s].", len, this.numElement); } private void setBinaryDataImpl( List data, BiConsumer consumer) { this.binaryData = new byte[data.size() * dataType.numByte]; ByteBuffer buf = ByteBuffer.wrap(this.binaryData); buf.order(ByteOrder.LITTLE_ENDIAN); for (T datum : data) { consumer.accept(buf, datum); } Preconditions.checkState(buf.position() == this.binaryData.length); this.parameters.put( Parameters.KEY_BINARY_DATA_SIZE, this.binaryData.length); } private void setJSONDataImpl(List data) { this.parameters.remove(Parameters.KEY_BINARY_DATA_SIZE); // this.data = new JSONArray(data.size()); // this.data.addAll(data); this.data = data.toArray(new Object[0]); } private void setJSONDataImpl(List data, Function mapper) { this.parameters.remove(Parameters.KEY_BINARY_DATA_SIZE); // this.data = new JSONArray(data.size()); // for (T datum : data) { // this.data.add(mapper.apply(datum)); // } this.data = new Object[data.size()]; int i = 0; for (T datum : data) { this.data[i++] = mapper.apply(datum); } } /** * Set boolean tensor data. * * @param data tensor data in java array. It's length must match input * shape given in constructor. * @param isBinaryData whether it's in binary format. */ public void setData(boolean[] data, boolean isBinaryData) { Preconditions.checkArgument( this.dataType == DataType.BOOL, "Could not set boolean[] as data of type: %s", this.dataType); if (isBinaryData) { // setBinaryDataImpl(Booleans.asList(data), (buf, b) -> buf.put(b ? // (byte)1 : (byte)0)); this.binaryData = BinaryProtocol.toBytes(this.dataType, data); this.updateBinaryDataSize(); } else { setJSONDataImpl(Booleans.asList(data)); } } /** * Set INT8/UINT8 tensor data. For unsigned type, top bit should be stored in * the sign bit. Note: guava currently had nothing like UnsignedShort. * * @param data tensor data in java array. It's length must match input * shape given in constructor. * @param isBinaryData whether it's in binary format. */ public void setData(byte[] data, boolean isBinaryData) { Preconditions.checkArgument( this.dataType == DataType.INT8 || this.dataType == DataType.UINT8, "Could not set boolean[] as data of type: %s", this.dataType); if (isBinaryData) { // setBinaryDataImpl(Bytes.asList(data), ByteBuffer::put); this.binaryData = BinaryProtocol.toBytes(this.dataType, data); this.updateBinaryDataSize(); } else if (this.dataType.singed) { setJSONDataImpl(Bytes.asList(data)); } else { setJSONDataImpl( Bytes.asList(data), bits -> UnsignedInteger.fromIntBits(bits).longValue()); } } /** * Set INT16/UINT16 tensor data. For unsigned type, top bit should be stored * in the sign bit. You may turn to * {@link com.google.common.primitives.Shorts} for help. *

* * @param data * @param isBinaryData */ public void setData(short[] data, boolean isBinaryData) { Preconditions.checkArgument( this.dataType == DataType.INT16 || this.dataType == DataType.UINT16, "Could not set boolean[] as data of type: %s", this.dataType); if (isBinaryData) { // setBinaryDataImpl(Shorts.asList(data), ByteBuffer::putShort); this.binaryData = BinaryProtocol.toBytes(this.dataType, data); this.updateBinaryDataSize(); } else if (this.dataType.singed) { setJSONDataImpl(Shorts.asList(data)); } else { setJSONDataImpl( Shorts.asList(data), bits -> UnsignedInteger.fromIntBits(bits).longValue()); } } /** * Set INT32/UINT32 tensor data. For unsigned type, top bit should be stored * in the sign bit. You may turn to * {@link com.google.common.primitives.UnsignedInteger} and {@link * com.google.common.primitives.UnsignedInts} for help. * * @param data tensor data in java array. It's length must match input * shape given in constructor. * @param isBinaryData whether it's in binary format. */ public void setData(int[] data, boolean isBinaryData) { Preconditions.checkArgument( this.dataType == DataType.INT32 || this.dataType == DataType.UINT32, "Could not set boolean[] as data of type: %s", this.dataType); if (isBinaryData) { // setBinaryDataImpl(Ints.asList(data), ByteBuffer::putInt); this.binaryData = BinaryProtocol.toBytes(this.dataType, data); this.updateBinaryDataSize(); } else { setJSONDataImpl(Ints.asList(data)); if (this.dataType.singed) { setJSONDataImpl(Ints.asList(data)); } else { // uint32 setJSONDataImpl( Ints.asList(data), bits -> UnsignedInteger.fromIntBits(bits).longValue()); } } } /** * Set INT64/UINT64 tensor data. For UINT64, top bit should be stored in the * sign bit. You may turn to {@link com.google.common.primitives.UnsignedLong} * and {@link com.google.common.primitives.UnsignedLongs} for help. * * @param data tensor data in java array. It's length must match input * shape given in constructor. * @param isBinaryData whether it's in binary format. */ public void setData(long[] data, boolean isBinaryData) { this.checkShape(data.length); Preconditions.checkArgument( this.dataType == DataType.INT64 || this.dataType == DataType.UINT64, "Could not set long[] as data of type: %s", this.dataType); if (isBinaryData) { this.binaryData = BinaryProtocol.toBytes(this.dataType, data); this.updateBinaryDataSize(); } else if (this.dataType.singed) { setJSONDataImpl(Longs.asList(data)); } else { // TODO(xiafei.qiuxf): This is a little bit inefficient, maybe we can // write a custom json // serializer/deserializer for unsigned types. setJSONDataImpl( Longs.asList(data), bits -> UnsignedLong.fromLongBits(bits).bigIntegerValue()); } } /** * Set FP32 tensor data. * * @param data tensor data in java array. It's length must match input * shape given in constructor. * @param isBinaryData whether it's in binary format. */ public void setData(float[] data, boolean isBinaryData) { this.checkShape(data.length); Preconditions.checkArgument( this.dataType == DataType.FP32, "Could not set float[] as data of type: %s", this.dataType); if (isBinaryData) { this.binaryData = BinaryProtocol.toBytes(this.dataType, data); this.updateBinaryDataSize(); } else { setJSONDataImpl(Floats.asList(data)); } } /** * Set FP64 tensor data. * * @param data tensor data in java array. It's length must match input * shape given in constructor. * @param isBinaryData whether it's in binary format. */ public void setData(double[] data, boolean isBinaryData) { this.checkShape(data.length); Preconditions.checkArgument( this.dataType == DataType.FP64, "Could not set double[] as data of type: %s", this.dataType); if (isBinaryData) { this.binaryData = BinaryProtocol.toBytes(this.dataType, data); this.updateBinaryDataSize(); } else { setJSONDataImpl(Doubles.asList(data)); } } /** * Set boolean tensor data. * * @param data tensor data in java array. It's length must match input * shape given in constructor. * @param isBinaryData whether it's in binary format. */ public void setData(String[] data, boolean isBinaryData) { this.checkShape(data.length); Preconditions.checkArgument( this.dataType == DataType.BYTES, "Could not set String[] as data of type: %s", this.dataType); if (isBinaryData) { this.binaryData = BinaryProtocol.toBytes(this.dataType, data); this.updateBinaryDataSize(); } else { // this.data = new JSONArray(data.length); // this.data.addAll(Arrays.asList(data)); this.data = new Object[data.length]; System.arraycopy(data, 0, this.data, 0, data.length); } } private void updateBinaryDataSize() { this.parameters.put( Parameters.KEY_BINARY_DATA_SIZE, this.binaryData.length); } public String getName() { return name; } IOTensor getTensor() { Preconditions.checkArgument( this.binaryData != null || this.data != null, ".setData method not call on InferInput %s", this.name); IOTensor tensor = new IOTensor(); tensor.setName(this.name); tensor.setDatatype(this.dataType); tensor.setShape(this.shape); tensor.setParameters(this.parameters); tensor.setData(this.data); return tensor; } /** * Get binary representation of tensor data of this inference input. * * @return null if this inference input is in JSON format. */ byte[] getBinaryData() { return binaryData; } Object[] getJSONData() { return this.data; } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/InferRequestedOutput.java000066400000000000000000000060671527052371200310760ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client; import triton.client.pojo.IOTensor; import triton.client.pojo.Parameters; /** * @author xiafei.qiuxf * @date 2021/4/13 */ public class InferRequestedOutput { /** * The name of output tensor to associate with this object. */ private final String name; /** * Indicates whether to return result data for the output in * binary format or explicit tensor within JSON. The default * value is True, which means the data will be delivered as * binary data in the HTTP body after JSON object. This field * will be unset if shared memory is set for the output. */ private final boolean isBinary; /** * The number of classifications to be requested. The default * value is 0 which means the classification results are not * requested. */ private final int classCount; private final Parameters parameters = new Parameters(); public InferRequestedOutput(String name, boolean isBinary, int classCount) { this.name = name; this.isBinary = isBinary; this.classCount = classCount; if (this.classCount != 0) { this.parameters.put("classification", classCount); } this.parameters.put("binary_data", this.isBinary); } public InferRequestedOutput(String name) { this(name, true, 0); } public InferRequestedOutput(String name, boolean isBinary) { this(name, isBinary, 0); } public IOTensor getTensor() { IOTensor tensor = new IOTensor(); tensor.setName(this.name); tensor.setParameters(this.parameters); return tensor; } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/InferResult.java000066400000000000000000000270571527052371200271740ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Array; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; import org.apache.commons.io.IOUtils; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import triton.client.pojo.DataType; import triton.client.pojo.IOTensor; import triton.client.pojo.InferenceResponse; import triton.client.pojo.Parameters; import triton.client.pojo.ResponseError; /** * An object of InferResult class holds the response of an inference request and * provide methods to retrieve inference results. */ public class InferResult { static class Index { int start; int length; public Index(int start, int length) { this.start = start; this.length = length; } } private final InferenceResponse response; private final Map nameToBinaryIdx; private final byte[] binaryData; public InferResult(HttpResponse resp) throws IOException, InferenceException { HttpEntity entity = resp.getEntity(); Preconditions.checkState( entity != null, "Get null entity from HTTP response."); InputStream stream = entity.getContent(); int httpCode = resp.getStatusLine().getStatusCode(); if (httpCode != HttpStatus.SC_OK) { byte[] bodyBytes = IOUtils.toByteArray(stream); if (bodyBytes.length > 0) { String bodyJson = new String(bodyBytes, Charsets.UTF_8); try { ResponseError err = Util.fromJson(bodyJson, ResponseError.class); throw new InferenceException(err.getError()); } catch (JsonProcessingException e) { throw new InferenceException("Malformed error response: " + bodyJson); } } else { throw new InferenceException("Inference return status: " + httpCode); } } this.nameToBinaryIdx = new HashMap<>(); Header contentLenHeader = resp.getFirstHeader("Inference-Header-Content-Length"); if (contentLenHeader != null) { // Construct response object from json part. int jsonLen = Integer.parseInt(contentLenHeader.getValue()); byte[] bodyBytes = new byte[jsonLen]; int readLen = stream.read(bodyBytes); Preconditions.checkState( readLen == jsonLen, "Expect content length: %d, but got %d.", jsonLen, readLen); String bodyJson = new String(bodyBytes, Charsets.UTF_8); this.response = Util.fromJson(bodyJson, InferenceResponse.class); // Construct name to binary index mapping. int startPos = 0; for (IOTensor output : this.response.getOutputs()) { Parameters param = output.getParameters(); if (param == null) { continue; } Integer size = param.getInt(Parameters.KEY_BINARY_DATA_SIZE); if (size == null) { continue; } this.nameToBinaryIdx.put(output.getName(), new Index(startPos, size)); startPos += size; } // Read and check binary data. this.binaryData = IOUtils.toByteArray(stream); Preconditions.checkState(this.binaryData.length == startPos); } else { String bodyJson = new String(IOUtils.toByteArray(stream), Charsets.UTF_8); this.response = Util.fromJson(bodyJson, InferenceResponse.class); this.binaryData = null; } } @VisibleForTesting InferResult( InferenceResponse response, Map nameToBinaryIdx, byte[] binaryData) { this.response = response; this.nameToBinaryIdx = nameToBinaryIdx; this.binaryData = binaryData; } @VisibleForTesting public InferenceResponse getResponse() { return response; } @VisibleForTesting public Map getNameToBinaryIdx() { return nameToBinaryIdx; } @VisibleForTesting public byte[] getBinaryData() { return binaryData; } public List getOutputs() { List outputs = this.response.getOutputs(); List ret = new ArrayList<>(outputs.size()); for (IOTensor out : outputs) { ret.add(out.getName()); } return ret; } /** * Get boolean tensor named as by parameter output. The tensor must be of * DataType.BOOL. * * @param output name of output tensor. * @return null if output not found or the tensor in boolean array. */ public boolean[] getOutputAsBool(String output) { IOTensor out = this.response.getOutputByName(output); if (out == null) { return null; } Preconditions.checkArgument( out.getDatatype() == DataType.BOOL, "Could not get boolean[] from data of type %s on output %s.", out.getDatatype(), out.getName()); return (boolean[]) getOutputImpl(out, boolean.class, buf -> buf.get() != 0); } /** * Get boolean tensor named as by parameter output. The tensor must be of * DataType.INT8 or DataType.UINT8. * * @param output name of output tensor. * @return null if output not found or the tensor in byte array. */ public byte[] getOutputAsByte(String output) { IOTensor out = this.response.getOutputByName(output); if (out == null) { return null; } Preconditions.checkArgument( out.getDatatype() == DataType.INT8 || out.getDatatype() == DataType.UINT8, "Could not get byte[] from data of type %s on output %s.", out.getDatatype(), out.getName()); return (byte[]) getOutputImpl(out, byte.class, ByteBuffer::get); } /** * Get boolean tensor named as by parameter output. The tensor must be of * DataType.INT16 or DataType.UINT16. * * @param output name of output tensor. * @return null if output not found or the tensor in short array. */ public short[] getOutputAsShort(String output) { IOTensor out = this.response.getOutputByName(output); if (out == null) { return null; } Preconditions.checkArgument( out.getDatatype() == DataType.INT16 || out.getDatatype() == DataType.UINT16, "Could not get short[] from data of type %s on output %s.", out.getDatatype(), out.getName()); return (short[]) getOutputImpl(out, short.class, ByteBuffer::getShort); } /** * Get boolean tensor named as by parameter output. The tensor must be of * DataType.INT32 or DataType.UINT32. * * @param output name of output tensor. * @return null if output not found or the tensor in int array. */ public int[] getOutputAsInt(String output) { IOTensor out = this.response.getOutputByName(output); if (out == null) { return null; } Preconditions.checkArgument( out.getDatatype() == DataType.INT32 || out.getDatatype() == DataType.UINT32, "Could not get int[] from data of type %s on output %s.", out.getDatatype(), out.getName()); return (int[]) getOutputImpl(out, int.class, ByteBuffer::getInt); } /** * Get boolean tensor named as by parameter output. The tensor must be of * DataType.INT64 or DataType.UINT64. * * @param output name of output tensor. * @return null if output not found or the tensor in long array. */ public long[] getOutputAsLong(String output) { IOTensor out = this.response.getOutputByName(output); if (out == null) { return null; } Preconditions.checkArgument( out.getDatatype() == DataType.INT64 || out.getDatatype() == DataType.UINT64, "Could not get long[] from data of type %s on output %s.", out.getDatatype(), out.getName()); return (long[]) getOutputImpl(out, long.class, ByteBuffer::getLong); } /** * Get boolean tensor named as by parameter output. The tensor must be of * DataType.FP32. * * @param output name of output tensor. * @return null if output not found or the tensor in float array. */ public float[] getOutputAsFloat(String output) { IOTensor out = this.response.getOutputByName(output); if (out == null) { return null; } Preconditions.checkArgument( out.getDatatype() == DataType.FP32, "Could not get float[] from data of type %s on output %s.", out.getDatatype(), out.getName()); return (float[]) getOutputImpl(out, float.class, ByteBuffer::getFloat); } /** * Get boolean tensor named as by parameter output. The tensor must be of * DataType.FP64. * * @param output name of output tensor. * @return null if output not found or the tensor in double array. */ public double[] getOutputAsDouble(String output) { IOTensor out = this.response.getOutputByName(output); if (out == null) { return null; } Preconditions.checkArgument( out.getDatatype() == DataType.FP64, "Could not get double[] from data of type %s on output %s.", out.getDatatype(), out.getName()); return (double[]) getOutputImpl(out, double.class, ByteBuffer::getDouble); } private Object getOutputImpl(IOTensor out, Class clazz, Function getter) { Index idx = this.nameToBinaryIdx.get(out.getName()); if (idx != null) { // Output in binary format. Preconditions.checkState(this.binaryData != null); long numElem = Util.elemNumFromShape(out.getShape()); Object array = Array.newInstance(clazz, (int) numElem); ByteBuffer buf = ByteBuffer.wrap(this.binaryData, idx.start, idx.length); buf.order(ByteOrder.LITTLE_ENDIAN); for (int i = 0; i < numElem; i++) { Array.set(array, i, getter.apply(buf)); } return array; } else { // Output in json format. Object[] data = out.getData(); Object array = Array.newInstance(clazz, data.length); for (int i = 0; i < data.length; i++) { Array.set(array, i, Util.numericCast(data[i], clazz)); } return array; } } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/InferenceException.java000066400000000000000000000036311527052371200304770ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client; import triton.client.pojo.ResponseError; /** * Universal exceptions of triton client. */ public class InferenceException extends Exception { public InferenceException(ResponseError err) { super(err.getError()); } public InferenceException(String message) { super(message); } public InferenceException(Throwable cause) { super(cause); } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/InferenceServerClient.java000066400000000000000000000356231527052371200311540ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client; /** * @author xiafei.qiuxf * @date 2021/4/13 */ import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClients; import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; import org.apache.http.impl.nio.reactor.IOReactorConfig; import org.apache.http.nio.entity.NByteArrayEntity; import org.apache.http.nio.reactor.ConnectingIOReactor; import triton.client.endpoint.AbstractEndpoint; import triton.client.endpoint.FixedEndpoint; import triton.client.pojo.IOTensor; import triton.client.pojo.Parameters; /** * An InferenceServerClient object is used to perform any kind of communication * with the InferenceServer using http protocol. None of the methods are thread * safe. The object is intended to be used by a single thread and simultaneously * calling different methods with different threads is not supported and will * cause undefined behavior. */ public class InferenceServerClient implements AutoCloseable { private static final Log LOG = LogFactory.getLog(InferenceServerClient.class); private static class HttpConfig { private int ioThreadNum; private int readTimeout; private int connectTimeout; private int requestTimeout; private int maxConnectionCount; private int maxConnectionPerRoute; private boolean keepAlive; public HttpConfig() { this.ioThreadNum = 10; this.readTimeout = 5000; this.connectTimeout = 5000; this.maxConnectionCount = 1000; this.maxConnectionPerRoute = 1000; this.requestTimeout = 0; this.keepAlive = true; } public HttpConfig( int ioThreadNum, int readTimeout, int connectTimeout, int maxConnectionCount, int maxConnectionPerRoute) { super(); this.ioThreadNum = ioThreadNum; this.readTimeout = readTimeout; this.connectTimeout = connectTimeout; this.maxConnectionCount = maxConnectionCount; this.maxConnectionPerRoute = maxConnectionPerRoute; this.keepAlive = true; } public HttpConfig( int ioThreadNum, int readTimeout, int connectTimeout, int maxConnectionCount, int maxConnectionPerRoute, int requestTimeout) { this( ioThreadNum, readTimeout, connectTimeout, maxConnectionCount, maxConnectionPerRoute); this.requestTimeout = requestTimeout; } public int getIoThreadNum() { return ioThreadNum; } public void setIoThreadNum(int ioThreadNum) { this.ioThreadNum = ioThreadNum; } public int getReadTimeout() { return readTimeout; } public void setReadTimeout(int readTimeout) { this.readTimeout = readTimeout; } public int getConnectTimeout() { return connectTimeout; } public void setConnectTimeout(int connectTimeout) { this.connectTimeout = connectTimeout; } public int getMaxConnectionCount() { return maxConnectionCount; } public void setMaxConnectionCount(int maxConnectionCount) { this.maxConnectionCount = maxConnectionCount; } public int getMaxConnectionPerRoute() { return maxConnectionPerRoute; } public void setMaxConnectionPerRoute(int maxConnectionPerRoute) { this.maxConnectionPerRoute = maxConnectionPerRoute; } public int getRequestTimeout() { return requestTimeout; } public void setRequestTimeout(int requestTimeout) { this.requestTimeout = requestTimeout; } public boolean isKeepAlive() { return keepAlive; } public void setKeepAlive(boolean keepAlive) { this.keepAlive = keepAlive; } } /** * Inference server url. */ private final AbstractEndpoint endpoint; private final int requestTimeout; private final CloseableHttpAsyncClient httpClient; private int retryCnt = 3; /** * Create an InferenceServerClient object. * * @param endpoint The endpoint to connect with. * @param connectionTimeout The timeout value for the connection. Default * value is 5 sec. * @param networkTimeout The timeout value for the network. Default value * is 5 sec. */ public InferenceServerClient( AbstractEndpoint endpoint, int connectionTimeout, int networkTimeout) throws IOException { this.endpoint = endpoint; this.requestTimeout = networkTimeout; HttpConfig httpConfig = new HttpConfig(); httpConfig.setConnectTimeout(connectionTimeout); httpConfig.setReadTimeout(networkTimeout); this.httpClient = createHttpClient(httpConfig); } public InferenceServerClient( String ipPort, int connectionTimeout, int networkTimeout) throws IOException { this(new FixedEndpoint(ipPort), connectionTimeout, networkTimeout); } public InferenceServerClient(AbstractEndpoint endpoint, HttpConfig httpConfig) throws IOException { this.endpoint = endpoint; this.requestTimeout = httpConfig.getRequestTimeout(); this.httpClient = createHttpClient(httpConfig); } private static CloseableHttpAsyncClient createHttpClient( HttpConfig httpConfig) throws IOException { ConnectingIOReactor ioReactor = new DefaultConnectingIOReactor(); PoolingNHttpClientConnectionManager cm = new PoolingNHttpClientConnectionManager(ioReactor); cm.setMaxTotal(httpConfig.getMaxConnectionCount()); cm.setDefaultMaxPerRoute(httpConfig.getMaxConnectionPerRoute()); IOReactorConfig config = IOReactorConfig.custom() .setTcpNoDelay(true) .setSoTimeout(httpConfig.getReadTimeout()) .setSoReuseAddress(true) .setConnectTimeout(httpConfig.getConnectTimeout()) .setIoThreadCount(httpConfig.getIoThreadNum()) .setSoKeepAlive(httpConfig.isKeepAlive()) .build(); final RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout(httpConfig.getConnectTimeout()) .setSocketTimeout(httpConfig.getReadTimeout()) .build(); CloseableHttpAsyncClient httpClient = HttpAsyncClients.custom() .setConnectionManager(cm) .setDefaultIOReactorConfig(config) .setDefaultRequestConfig(requestConfig) .build(); httpClient.start(); return httpClient; } public void setRetryCnt(int retryCnt) { Preconditions.checkArgument( retryCnt >= 0, "Invalid retryCount: %s", retryCnt); this.retryCnt = retryCnt; } public InferResult infer(InferArguments arg) throws InferenceException { Preconditions.checkArgument(arg != null, "arguments should not be null."); Map inferReq = new HashMap<>(); Parameters param = new Parameters(); if (!Util.isEmpty(arg.requestId)) { inferReq.put("id", arg.requestId); } if (arg.sequenceId != 0) { param.put("sequence_id", arg.sequenceId); param.put("sequence_start", arg.sequenceStart); param.put("sequence_end", arg.sequenceEnd); } if (arg.priority != 0) { param.putUnsignedLong("priority", arg.priority); } if (arg.timeout > 0) { param.put("timeout", arg.timeout); } List inputs = Lists.transform(arg.inputs, InferInput::getTensor); inferReq.put("inputs", inputs); if (!Util.isEmpty(arg.outputs)) { List outputs = Lists.transform(arg.outputs, InferRequestedOutput::getTensor); inferReq.put("outputs", outputs); } else { // Looks strange, just do as in Triton's python SDK. param.put("binary_data_output", true); } if (!param.isEmpty()) { inferReq.put("parameters", param); } HttpPost post; try { post = createHttpPost(arg, inferReq); } catch (Exception e) { throw new InferenceException(e); } for (int i = 0; i <= this.retryCnt; i++) { try { Future future = this.httpClient.execute(post, null); HttpResponse resp = this.requestTimeout > 0 ? future.get(this.requestTimeout, TimeUnit.MILLISECONDS) : future.get(); return new InferResult(resp); } catch (Exception e) { if (i == this.retryCnt) { LOG.error( String.format( "Inference failed on %dth retry, url: %s", i, post.getURI().toString()), e); throw new InferenceException(e); } else if (LOG.isDebugEnabled()) { LOG.debug( String.format( "Inference failed on %dth retry, url: %s", i, post.getURI().toString()), e); } } } // Should never reach here. return null; } private HttpPost createHttpPost( InferArguments arg, Map inferReq) throws Exception { // Create post body in binary format: // ... ByteArrayOutputStream bodyBytes = new ByteArrayOutputStream(); byte[] jsonBytes = Util.toJson(inferReq).getBytes(StandardCharsets.UTF_8); boolean hasBinaryInput = false; bodyBytes.write(jsonBytes); for (InferInput input : arg.inputs) { byte[] binInput = input.getBinaryData(); if (binInput != null) { bodyBytes.write(binInput); hasBinaryInput = true; } } if (hasBinaryInput) { arg.headers.put( "Inference-Header-Content-Length", String.valueOf(jsonBytes.length)); } // Create target URI. URIBuilder ub = new URIBuilder(this.getUrl()); String safeModelName = URLEncoder.encode(arg.modelName, Charsets.UTF_8.toString()); String requestUri = Util.isEmpty(arg.modelVersion) ? String.format("/v2/models/%s/infer", safeModelName) : String.format( "/v2/models/%s/versions/%s/infer", safeModelName, arg.modelVersion); String addPath = ub.getPath() == null ? requestUri : ub.getPath() + requestUri; ub.setPath(addPath); arg.queryParams.forEach(ub::addParameter); // Crete HttpPost, uri, body and headers. HttpPost post = new HttpPost(ub.build()); arg.headers.forEach(post::setHeader); post.setEntity(new NByteArrayEntity(bodyBytes.toByteArray())); return post; } private String getUrl() throws Exception { return "http://" + this.endpoint.getEndpoint(); } public InferResult infer( String modelName, List inputs, List outputs) throws InferenceException { return this.infer(new InferArguments(modelName, inputs, outputs)); } @Override public void close() throws Exception { this.httpClient.close(); } public static class InferArguments { final String modelName; final List inputs; final List outputs; String modelVersion = ""; String requestId = ""; int sequenceId = 0; boolean sequenceStart = false; boolean sequenceEnd = false; long priority = 0; int timeout = -1; Map headers = new HashMap<>(); Map queryParams = new HashMap<>(); public InferArguments( String modelName, List inputs, List outputs) { this.modelName = modelName; this.inputs = inputs; this.outputs = outputs; } public InferArguments(String modelName, List inputs) { this.modelName = modelName; this.inputs = inputs; this.outputs = null; } public InferArguments(String modelName, InferInput... inputs) { this.modelName = modelName; this.inputs = new ArrayList<>(Arrays.asList(inputs)); this.outputs = null; } public InferArguments setModelVersion(String modelVersion) { this.modelVersion = modelVersion; return this; } public InferArguments setRequestId(String requestId) { this.requestId = requestId; return this; } public InferArguments setSequenceId(int sequenceId) { this.sequenceId = sequenceId; return this; } public InferArguments setSequenceStart(boolean sequenceStart) { this.sequenceStart = sequenceStart; return this; } public InferArguments setSequenceEnd(boolean sequenceEnd) { this.sequenceEnd = sequenceEnd; return this; } public InferArguments setPriority(long priority) { this.priority = priority; return this; } public InferArguments setTimeout(int timeout) { this.timeout = timeout; return this; } public InferArguments setHeader(String key, String value) { this.headers.put(key, value); return this; } public InferArguments addQueryParam(String key, String value) { this.queryParams.put(key, value); return this; } } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/Util.java000066400000000000000000000112771527052371200256440ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Collection; public class Util { /** * Check whether a string object is null or empty. * * @param s the string object. * @return true if it's null or empty */ public static boolean isEmpty(String s) { return s == null || s.isEmpty(); } /** * Check whether a collection object is null or empty. * * @param c the collection object. * @return true if it's null or empty */ public static boolean isEmpty(Collection c) { return c == null || c.isEmpty(); } /** * Calculate element number from tensor shape. * * @param shape tensor shape * @return element number */ public static long elemNumFromShape(long[] shape) { long ret = 1; for (long n : shape) { ret *= n; } return ret; } /** * Convert int to bytes in little endian. * * @param a * @return */ public static byte[] intToBytes(int a) { byte[] ret = new byte[4]; ret[0] = (byte) (a & 0xFF); ret[1] = (byte) ((a >> 8) & 0xFF); ret[2] = (byte) ((a >> 16) & 0xFF); ret[3] = (byte) ((a >> 24) & 0xFF); return ret; } private static final ObjectMapper jsonMapper = new ObjectMapper(); /** * Serialize object to JSON string. * * @param obj * @return JSON string. */ public static String toJson(Object obj) throws JsonProcessingException { return jsonMapper.writeValueAsString(obj); } /** * Parse JSON string to object. * * @param text JSON string. * @param clazz Class of target object. * @param * @return Parsed object. */ public static T fromJson(String text, Class clazz) throws JsonProcessingException { return jsonMapper.readValue(text, clazz); } public static Object numericCast(Object input, Class clazz) { if (clazz == boolean.class || clazz == Boolean.class) { if (input.getClass() != Boolean.class) { throw new UnsupportedOperationException(String.format( "Casting %s to %s.", input.getClass().getCanonicalName(), clazz.getCanonicalName())); } return input; } if (!Number.class.isAssignableFrom(input.getClass())) { throw new UnsupportedOperationException(String.format( "Input should be boolean or numeric types, %s is not supported", input.getClass().getCanonicalName())); } Number num = (Number) input; if (clazz == byte.class || clazz == Byte.class) { return num.byteValue(); } if (clazz == short.class || clazz == Short.class) { return num.shortValue(); } if (clazz == int.class || clazz == Integer.class) { return num.intValue(); } if (clazz == long.class || clazz == Long.class) { return num.longValue(); } if (clazz == float.class || clazz == Float.class) { return num.floatValue(); } if (clazz == double.class || clazz == Double.class) { return num.doubleValue(); } throw new UnsupportedOperationException(String.format( "Unsupported target type: %s.", clazz.getCanonicalName())); } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/endpoint/000077500000000000000000000000001527052371200256745ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/endpoint/AbstractEndpoint.java000066400000000000000000000053271527052371200320120ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.endpoint; import com.google.common.base.Preconditions; import java.util.Objects; import triton.client.InferenceServerClient; import triton.client.Util; /** * Endpoint is an abstraction that allow different kinds of strategy to provide * ip and port for * {@link InferenceServerClient} to send requests. */ public abstract class AbstractEndpoint { private static final int RETRY_COUNT = 10; private String lastResult = ""; abstract String getEndpointImpl() throws Exception; abstract int getEndpointNum() throws Exception; /** * Get string in ip:port[/path] format. * * @return * @throws Exception */ public String getEndpoint() throws Exception { for (int i = 0; i < RETRY_COUNT; i++) { String url = this.getEndpointImpl(); Preconditions.checkState( !Util.isEmpty(url), "getEndpointImpl should not return null or empty string!"); if (!Objects.equals(this.lastResult, url) || this.getEndpointNum() < 2) { this.lastResult = url; return url; } } throw new RuntimeException(String.format( "Failed to get endpoint address after trying %d times.", RETRY_COUNT)); } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/endpoint/FixedEndpoint.java000066400000000000000000000045351527052371200313060ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.endpoint; import com.google.common.base.Preconditions; import triton.client.Util; /** * Endpoint that connect to single address. */ public class FixedEndpoint extends AbstractEndpoint { private final String addr; /** * Create a endpoint connecting to a fixed address. * * @param endpoint Endpoint in host:port[/path] format without schema part. */ public FixedEndpoint(String endpoint) { Preconditions.checkArgument( !Util.isEmpty(endpoint), "endpoint should not be null or empty."); Preconditions.checkArgument( !endpoint.contains("://"), "endpoint should be in host:port[/path] format without scheme."); this.addr = endpoint; } @Override String getEndpointImpl() { return this.addr; } @Override int getEndpointNum() { return 1; } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/examples/000077500000000000000000000000001527052371200256725ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/examples/MemoryGrowthTest.java000066400000000000000000000062101527052371200320370ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.examples; import com.google.common.collect.Lists; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; import triton.client.InferInput; import triton.client.InferRequestedOutput; import triton.client.InferResult; import triton.client.InferenceServerClient; import triton.client.pojo.DataType; public class MemoryGrowthTest { public static void main(String[] args) throws Exception { // Number of inferences to run. int repetitions = 100; for (int i = 0; i < repetitions; i++) { // Create the data for the input tensor. boolean isBinary = true; InferInput input0 = new InferInput("INPUT0", new long[] {1L, 16}, DataType.INT32); List lst_0 = IntStream.rangeClosed(1, 16).boxed().collect(Collectors.toList()); int[] input0_data = lst_0.stream().mapToInt(j -> j).toArray(); input0.setData(input0_data, isBinary); List inputs = Lists.newArrayList(input0); List outputs = Lists.newArrayList(new InferRequestedOutput("OUTPUT0", isBinary)); InferenceServerClient client = new InferenceServerClient("0.0.0.0:8000", 5000, 5000); InferResult result = client.infer("custom_identity_int32", inputs, outputs); // Get the output arrays from the results and verify int[] output0 = result.getOutputAsInt("OUTPUT0"); if (!Arrays.equals(input0_data, output0)) { System.out.println("incorrect output"); System.exit(1); } } System.exit(0); } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/examples/SimpleInferClient.java000066400000000000000000000077461527052371200321270ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.examples; import com.google.common.collect.Lists; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; import triton.client.InferInput; import triton.client.InferRequestedOutput; import triton.client.InferResult; import triton.client.InferenceServerClient; import triton.client.pojo.DataType; /** * @author xiafei.qiuxf * @date 2021/7/28 */ public class SimpleInferClient { public static void main(String[] args) throws Exception { // Initialize the data boolean isBinary0 = false; InferInput input0 = new InferInput("INPUT0", new long[] {1L, 16}, DataType.INT32); List lst_0 = IntStream.rangeClosed(1, 16).boxed().collect(Collectors.toList()); int[] input0_data = lst_0.stream().mapToInt(i -> i).toArray(); input0.setData(input0_data, isBinary0); boolean isBinary1 = true; InferInput input1 = new InferInput("INPUT1", new long[] {1L, 16}, DataType.INT32); List lst_1 = IntStream.rangeClosed(1, 16).boxed().collect(Collectors.toList()); int[] input1_data = lst_1.stream().mapToInt(i -> i).toArray(); input1.setData(input1_data, isBinary1); List inputs = Lists.newArrayList(input0, input1); List outputs = Lists.newArrayList( new InferRequestedOutput("OUTPUT0", isBinary0), new InferRequestedOutput("OUTPUT1", isBinary1)); InferenceServerClient client = new InferenceServerClient("0.0.0.0:8000", 5000, 5000); InferResult result = client.infer("simple", inputs, outputs); // Get the output arrays from the results int[] op0 = result.getOutputAsInt("OUTPUT0"); int[] op1 = result.getOutputAsInt("OUTPUT1"); // Validate outputs for (int i = 0; i < op0.length; i++) { System.out.println( Integer.toString(lst_0.get(i)) + " + " + Integer.toString(lst_1.get(i)) + " = " + Integer.toString(op0[i])); System.out.println( Integer.toString(lst_0.get(i)) + " - " + Integer.toString(lst_1.get(i)) + " = " + Integer.toString(op1[i])); if (op0[i] != (lst_0.get(i) + lst_1.get(i))) { System.out.println("OUTPUT0 contains incorrect sum"); System.exit(1); } if (op1[i] != (lst_0.get(i) - lst_1.get(i))) { System.out.println("OUTPUT1 contains incorrect difference"); System.exit(1); } } System.exit(0); } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/examples/SimpleInferPerf.java000066400000000000000000000127431527052371200315760ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.examples; import com.google.common.collect.Lists; import com.google.common.util.concurrent.AtomicDouble; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; import triton.client.InferInput; import triton.client.InferRequestedOutput; import triton.client.InferResult; import triton.client.InferenceServerClient; import triton.client.InferenceServerClient.InferArguments; import triton.client.endpoint.FixedEndpoint; import triton.client.pojo.DataType; /** * Do inference via a stand-alone (which is not hosted inside EAS) triton * server. * * @author xiafei.qiuxf * @date 2021/5/7 */ public class SimpleInferPerf { public static void main(String[] args) throws Exception { // Initialize the data boolean isBinary0 = false; InferInput input0 = new InferInput("INPUT0", new long[] {1L, 16}, DataType.INT32); List lst_0 = IntStream.rangeClosed(1, 16).boxed().collect(Collectors.toList()); int[] input0_data = lst_0.stream().mapToInt(i -> i).toArray(); input0.setData(input0_data, isBinary0); boolean isBinary1 = true; InferInput input1 = new InferInput("INPUT1", new long[] {1L, 16}, DataType.INT32); List lst_1 = IntStream.rangeClosed(1, 16).boxed().collect(Collectors.toList()); int[] input1_data = lst_1.stream().mapToInt(i -> i).toArray(); input1.setData(input1_data, isBinary1); List inputs = Lists.newArrayList(input0, input1); List outputs = Lists.newArrayList( new InferRequestedOutput("OUTPUT0", isBinary0), new InferRequestedOutput("OUTPUT1", isBinary1)); final String model_name = "simple"; final int nThreads = 8; System.out.printf("Testing %s with %d threads.%n", model_name, nThreads); List threads = Lists.newArrayList(); AtomicDouble totalQps = new AtomicDouble(0); AtomicDouble avgLatency = new AtomicDouble(0); for (int t = 0; t < nThreads; t++) { Thread thread = new Thread(() -> { long tid = Thread.currentThread().getId(); // For a stand-alone triton server, FixedEndpoint is used to connected // to it. FixedEndpoint endpoint = new FixedEndpoint("0.0.0.0:8000"); try ( InferenceServerClient client = new InferenceServerClient(endpoint, 5000, 5000)) { InferArguments inferArg = new InferArguments(model_name, inputs, outputs); final int N = 1000; final int GAP = 100; long start = System.currentTimeMillis(); long lastGapStart = start; for (int i = 0; i < N; i++) { InferResult result = client.infer(inferArg); if ((i + 1) % GAP == 0) { long now = System.currentTimeMillis(); long gapElapsedMs = now - lastGapStart; double latency = 1.0 * gapElapsedMs / GAP; System.out.printf( "[%d][GAP] Requests: %d, avg latency(ms): %.2f%n", tid, i + 1, latency); lastGapStart = now; } } long totalMs = System.currentTimeMillis() - start; double latency = 1.0 * totalMs / N; double qps = 1000.0 * N / totalMs; System.out.printf( "[%d][TOTAL] avg latency(ms): %.2f, qps: %.2f%n", tid, latency, qps); totalQps.addAndGet(qps); avgLatency.addAndGet(latency); } catch (Exception e) { e.printStackTrace(); } }); thread.start(); threads.add(thread); } for (Thread thread : threads) { thread.join(); } System.out.println("=================================="); System.out.printf("[ALL] QPS: %.2f\n", totalQps.get()); System.out.printf("[ALL] Latency(ms): %.2f\n", avgLatency.get() / nThreads); System.out.println("=================================="); } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/pojo/000077500000000000000000000000001527052371200250235ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/pojo/DataType.java000066400000000000000000000040121527052371200273760ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.pojo; /** * Data types from */ public enum DataType { BOOL(1, false), INT8(1, true), INT16(2, true), INT32(4, true), INT64(8, true), UINT8(1, false), UINT16(2, false), UINT32(4, false), UINT64(8, false), FP16(2, false), FP32(4, false), FP64(8, false), BYTES(-1, false); public final int numByte; public final boolean singed; DataType(int numByte, boolean singed) { this.numByte = numByte; this.singed = singed; } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/pojo/IOTensor.java000066400000000000000000000055371527052371200274020ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.pojo; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; /** * This class represent * request_input * and * response_output * in kfserving's v2 rest schema. The two objects share the same definitions. */ @JsonInclude(Include.NON_NULL) public class IOTensor { private String name; private long[] shape; private DataType datatype; private Parameters parameters; private Object[] data; public IOTensor() {} public String getName() { return name; } public void setName(String name) { this.name = name; } public long[] getShape() { return shape; } public void setShape(long[] shape) { this.shape = shape; } public DataType getDatatype() { return datatype; } public void setDatatype(DataType datatype) { this.datatype = datatype; } public Parameters getParameters() { return parameters; } public void setParameters(Parameters parameters) { this.parameters = parameters; } public Object[] getData() { return data; } public void setData(Object[] data) { this.data = data; } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/pojo/InferenceResponse.java000066400000000000000000000061371527052371200313120ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.pojo; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; /** * JSON object for * inference_response * object in kfserving's v2 rest schema. */ @JsonInclude(Include.NON_NULL) public class InferenceResponse { @JsonProperty("model_name") private String modelName; @JsonProperty("model_version") private String modelVersion; private String id; private Parameters parameters; private List outputs; public InferenceResponse() {} public void setModelName(String modelName) { this.modelName = modelName; } public void setModelVersion(String modelVersion) { this.modelVersion = modelVersion; } public void setId(String id) { this.id = id; } public void setParameters(Parameters parameters) { this.parameters = parameters; } public void setOutputs(List outputs) { this.outputs = outputs; } public String getModelName() { return modelName; } public String getModelVersion() { return modelVersion; } public String getId() { return id; } public Parameters getParameters() { return parameters; } public List getOutputs() { return outputs; } public IOTensor getOutputByName(String name) { for (IOTensor output : this.outputs) { if (output.getName().equals(name)) { return output; } } return null; } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/pojo/Parameters.java000066400000000000000000000203141527052371200277710ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.pojo; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.IOException; import java.math.BigInteger; import java.util.HashMap; import java.util.Map; import triton.client.pojo.Parameters.ParamDeserializer; import triton.client.pojo.Parameters.ParamSerializer; /** * This class represent * parameters * object in kfserving's v2 rest schema is just a JSON string, and offer some * util methods. When serializing/deserializing, Parameters should act just like * a map. */ @JsonSerialize(using = ParamSerializer.class) @JsonDeserialize(using = ParamDeserializer.class) public class Parameters { public final static String KEY_BINARY_DATA_SIZE = "binary_data_size"; private Map params; public Parameters() { this.params = new HashMap<>(); } public Parameters(Map params) { this.params = params; } /** * Add or over-write parameter key-values. * * @param key name of new parameter. * @param value value of new parameter. * @return The original value if key exists. */ public Object put(String key, Object value) { return this.params.put(key, value); } /** * Add or over-write long parameter as unsigned. * As Java doesn't have a native unsigned long type * we convert to BigInteger if the value < 0 * * @param key name of new parameter. * @param value value of new parameter. * @return The original value if key exists. */ public Object putUnsignedLong(String key, long value) { Object unsigned_value = value; if (value < 0) { unsigned_value = new BigInteger(Long.toUnsignedString(value)); } return this.params.put(key, unsigned_value); } /** * Remove key from parameters. * * @param key name of new parameter. * @return The original value if key exists. */ public Object remove(String key) { return this.params.remove(key); } /** * Check if parameters are empty. * * @return true if empty. */ public boolean isEmpty() { return this.params.isEmpty(); } /** * Get an parameter value as bool. Some conversions are done under the hood. * * @param name name of the parameter. * @return null if named parameter not exists, otherwise the parameter value * in bool format. * @throws IllegalArgumentException if conversions failed. */ public Boolean getBool(final String name) { Object o = this.params.get(name); if (o == null) { return null; } else if (o instanceof Boolean) { return (Boolean) o; } else if (o instanceof String) { return Boolean.valueOf(((String) o)); } else { throw new IllegalArgumentException(String.format( "Could not convert type %s to Boolean", o.getClass().getCanonicalName())); } } /** * Get an parameter value as integer. Some conversions are done under the * hood. * * @param name name of the parameter. * @return null if named parameter not exists, otherwise the parameter value * in integer format. * @throws IllegalArgumentException if conversions failed. */ public Integer getInt(final String name) { Object o = this.params.get(name); if (o == null) { return null; } else if (o instanceof Number) { return ((Number) o).intValue(); } else if (o instanceof String) { return Integer.valueOf(((String) o)); } else { throw new IllegalArgumentException(String.format( "Could not convert type %s to Integer", o.getClass().getCanonicalName())); } } /** * Get an parameter value as float. Some conversions are done under the hood. * * @param name name of the parameter. * @return null if named parameter not exists, otherwise the parameter value * in float format. * @throws IllegalArgumentException if conversions failed. */ public Float getFloat(final String name) { Object o = this.params.get(name); if (o == null) { return null; } else if (o instanceof Number) { return ((Number) o).floatValue(); } else if (o instanceof String) { return Float.valueOf(((String) o)); } else { throw new IllegalArgumentException(String.format( "Could not convert type %s to Float", o.getClass().getCanonicalName())); } } /** * Get an parameter value as double. Some conversions are done under the hood. * * @param name name of the parameter. * @return null if named parameter not exists, otherwise the parameter value * in double format. * @throws IllegalArgumentException if conversions failed. */ public Double getDouble(final String name) { Object o = this.params.get(name); if (o == null) { return null; } else if (o instanceof Number) { return ((Number) o).doubleValue(); } else if (o instanceof String) { return Double.valueOf(((String) o)); } else { throw new IllegalArgumentException(String.format( "Could not convert type %s to Double", o.getClass().getCanonicalName())); } } /** * Get an parameter value as string. * * @param name name of the parameter. * @return null if named parameter not exists, otherwise the parameter value * in string format. */ public String getString(final String name) { Object o = this.params.get(name); if (o == null) { return null; } else { return String.valueOf(o); } } static class ParamSerializer extends JsonSerializer { @Override public void serialize( Parameters parameters, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { jsonGenerator.writeObject(parameters.params); } } static class ParamDeserializer extends JsonDeserializer { @Override public Parameters deserialize(JsonParser p, DeserializationContext ctx) throws IOException { ObjectMapper mapper = new ObjectMapper(); final Map obj = mapper.readValue( p, new com.fasterxml.jackson.core.type .TypeReference>() {}); return new Parameters(obj); } } } kytensor-2.49.0.7/client/src/java/src/main/java/triton/client/pojo/ResponseError.java000066400000000000000000000034701527052371200305020ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package triton.client.pojo; /** * This is a JSON object representing error response body of triton server. */ public class ResponseError { private String error; public String getError() { return error; } public void setError(String error) { this.error = error; } } kytensor-2.49.0.7/client/src/python/000077500000000000000000000000001527052371200172235ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/CMakeLists.txt000066400000000000000000000067701527052371200217750ustar00rootroot00000000000000# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(python-clients LANGUAGES C CXX) # # Options # set(TRITON_VERSION "0.0.0" CACHE STRING "Version for the clients") option(TRITON_ENABLE_PYTHON_HTTP "Enable Python HTTP client libraries" OFF) option(TRITON_ENABLE_PYTHON_GRPC "Enable Python GRPC client libraries" OFF) option(TRITON_ENABLE_EXAMPLES "Include examples in build" OFF) option(TRITON_ENABLE_TESTS "Include tests in build" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in libraries" OFF) # Package Perf Analyzer with the Python Client -- Intended to be used by PA repo option(TRITON_PACKAGE_PERF_ANALYZER "Include Perf Analyzer in pip wheel" OFF) set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) if(TRITON_ENABLE_PYTHON_GRPC) set(TRITON_COMMON_ENABLE_PROTOBUF ON) set(TRITON_COMMON_ENABLE_GRPC ON) endif() # TRITON_ENABLE_PYTHON_GRPC FetchContent_MakeAvailable(repo-common) # # CUDA # if(${TRITON_ENABLE_GPU}) find_package(CUDAToolkit REQUIRED) endif() # TRITON_ENABLE_GPU # # Protobuf # if(TRITON_ENABLE_PYTHON_GRPC) set(protobuf_MODULE_COMPATIBLE TRUE CACHE BOOL "protobuf_MODULE_COMPATIBLE" FORCE) find_package(Protobuf REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") include_directories(${Protobuf_INCLUDE_DIRS}) endif() # TRITON_ENABLE_PYTHON_GRPC # # GRPC # if(TRITON_ENABLE_PYTHON_GRPC) find_package(gRPC CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") include_directories($) endif() # TRITON_ENABLE_PYTHON_GRPC if(TRITON_ENABLE_PYTHON_HTTP OR TRITON_ENABLE_PYTHON_GRPC) add_subdirectory(library) if(TRITON_ENABLE_EXAMPLES) add_subdirectory(examples) endif() # TRITON_ENABLE_EXAMPLES endif() # TRITON_ENABLE_PYTHON_HTTP OR TRITON_ENABLE_PYTHON_GRPC kytensor-2.49.0.7/client/src/python/examples/000077500000000000000000000000001527052371200210415ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/examples/CMakeLists.txt000066400000000000000000000066741527052371200236160ustar00rootroot00000000000000# Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) if(${TRITON_ENABLE_PYTHON_HTTP}) install( PROGRAMS simple_http_aio_infer_client.py simple_http_health_metadata.py simple_http_infer_client.py simple_http_string_infer_client.py simple_http_async_infer_client.py simple_http_sequence_sync_infer_client.py simple_http_shm_client.py simple_http_shm_string_client.py simple_http_model_control.py DESTINATION python ) if(${TRITON_ENABLE_GPU}) install( PROGRAMS simple_http_cudashm_client.py DESTINATION python ) endif() # TRITON_ENABLE_GPU endif() # TRITON_ENABLE_PYTHON_HTTP if(${TRITON_ENABLE_PYTHON_GRPC}) install( PROGRAMS grpc_client.py grpc_explicit_byte_content_client.py grpc_explicit_int_content_client.py grpc_explicit_int8_content_client.py grpc_image_client.py simple_grpc_aio_infer_client.py simple_grpc_aio_sequence_stream_infer_client.py simple_grpc_custom_repeat.py simple_grpc_health_metadata.py simple_grpc_async_infer_client.py simple_grpc_infer_client.py simple_grpc_sequence_stream_infer_client.py simple_grpc_sequence_sync_infer_client.py simple_grpc_string_infer_client.py simple_grpc_shm_client.py simple_grpc_shm_string_client.py simple_grpc_model_control.py simple_grpc_keepalive_client.py simple_grpc_custom_args_client.py DESTINATION python ) if(${TRITON_ENABLE_GPU}) install( PROGRAMS simple_grpc_cudashm_client.py DESTINATION python ) endif() # TRITON_ENABLE_GPU endif() # TRITON_ENABLE_PYTHON_GRPC if(${TRITON_ENABLE_PYTHON_HTTP} OR ${TRITON_ENABLE_PYTHON_GRPC}) install( PROGRAMS ensemble_image_client.py image_client.py reuse_infer_objects_client.py memory_growth_test.py DESTINATION python ) endif() # TRITON_ENABLE_PYTHON_HTTP || TRITON_ENABLE_PYTHON_GRPC kytensor-2.49.0.7/client/src/python/examples/ensemble_image_client.py000077500000000000000000000226211527052371200257130ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os import sys from builtins import range import numpy as np import tritonclient.grpc as grpcclient import tritonclient.grpc.model_config_pb2 as model_config import tritonclient.http as httpclient from tritonclient.utils import InferenceServerException FLAGS = None def parse_model_grpc(model_metadata, model_config): """ Check the configuration of a model to make sure it meets the requirements for an image classification network (as expected by this client) """ if len(model_metadata.inputs) != 1: raise Exception("expecting 1 input, got {}".format(len(model_metadata.inputs))) if len(model_config.input) != 1: raise Exception( "expecting 1 input in model configuration, got {}".format( len(model_config.input) ) ) input_metadata = model_metadata.inputs[0] output_metadata = model_metadata.outputs return (input_metadata.name, output_metadata, model_config.max_batch_size) def parse_model_http(model_metadata, model_config): """ Check the configuration of a model to make sure it meets the requirements for an image classification network (as expected by this client) """ if len(model_metadata["inputs"]) != 1: raise Exception( "expecting 1 input, got {}".format(len(model_metadata["inputs"])) ) if len(model_config["input"]) != 1: raise Exception( "expecting 1 input in model configuration, got {}".format( len(model_config["input"]) ) ) input_metadata = model_metadata["inputs"][0] output_metadata = model_metadata["outputs"] return (input_metadata["name"], output_metadata, model_config["max_batch_size"]) def postprocess(results, output_names, filenames, batch_size): """ Post-process results to show classifications. """ output_dict = {} for output_name in output_names: output_dict[output_name] = results.as_numpy(output_name) if len(output_dict[output_name]) != batch_size: raise Exception( "expected {} results for output {}, got {}".format( batch_size, output_name, len(output_dict[output_name]) ) ) for n, f in enumerate(filenames): print('\n"{}":'.format(f)) for output_name in output_names: print(" [{}]:".format(output_name)) for result in output_dict[output_name][n]: if output_dict[output_name][n].dtype.type == np.object_: cls = "".join(chr(x) for x in result).split(":") else: cls = result.split(":") print(" {} ({}) = {}".format(cls[0], cls[1], cls[2])) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-m", "--model-name", type=str, required=False, default="preprocess_inception_ensemble", help="Name of model. Default is preprocess_inception_ensemble.", ) parser.add_argument( "-c", "--classes", type=int, required=False, default=1, help="Number of class results to report. Default is 1.", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) parser.add_argument( "-i", "--protocol", type=str, required=False, default="HTTP", help="Protocol (HTTP/gRPC) used to " + "communicate with inference service. Default is HTTP.", ) parser.add_argument( "image_filename", type=str, nargs="?", default=None, help="Input image / Input folder.", ) FLAGS = parser.parse_args() protocol = FLAGS.protocol.lower() try: if protocol == "grpc": # Create gRPC client for communicating with the server triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) else: # Create HTTP client for communicating with the server triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("client creation failed: " + str(e)) sys.exit(1) model_name = FLAGS.model_name # Make sure the model matches our requirements, and get some # properties of the model that we need for preprocessing try: model_metadata = triton_client.get_model_metadata(model_name=model_name) except InferenceServerException as e: print("failed to retrieve the metadata: " + str(e)) sys.exit(1) try: model_config = triton_client.get_model_config(model_name=model_name) except InferenceServerException as e: print("failed to retrieve the config: " + str(e)) sys.exit(1) if FLAGS.protocol.lower() == "grpc": input_name, output_metadata, batch_size = parse_model_grpc( model_metadata, model_config.config ) else: input_name, output_metadata, batch_size = parse_model_http( model_metadata, model_config ) filenames = [] if os.path.isdir(FLAGS.image_filename): filenames = [ os.path.join(FLAGS.image_filename, f) for f in os.listdir(FLAGS.image_filename) if os.path.isfile(os.path.join(FLAGS.image_filename, f)) ] else: filenames = [ FLAGS.image_filename, ] filenames.sort() # Set batch size to the smaller value of image size and max batch size if len(filenames) <= batch_size: batch_size = len(filenames) else: print( "The number of images exceeds maximum batch size," "only the first {} images, sorted by name alphabetically," " will be processed".format(batch_size) ) # Preprocess the images into input data according to model # requirements image_data = [] for idx in range(batch_size): with open(filenames[idx], "rb") as fd: image_data.append(np.array([fd.read()], dtype=bytes)) # Send requests of batch_size images. input_filenames = [] repeated_image_data = [] for idx in range(batch_size): input_filenames.append(filenames[idx]) repeated_image_data.append(image_data[idx]) batched_image_data = np.stack(repeated_image_data, axis=0) # Set the input data inputs = [] if FLAGS.protocol.lower() == "grpc": inputs.append( grpcclient.InferInput(input_name, batched_image_data.shape, "BYTES") ) inputs[0].set_data_from_numpy(batched_image_data) else: inputs.append( httpclient.InferInput(input_name, batched_image_data.shape, "BYTES") ) inputs[0].set_data_from_numpy(batched_image_data, binary_data=True) output_names = [ output.name if FLAGS.protocol.lower() == "grpc" else output["name"] for output in output_metadata ] outputs = [] for output_name in output_names: if FLAGS.protocol.lower() == "grpc": outputs.append( grpcclient.InferRequestedOutput(output_name, class_count=FLAGS.classes) ) else: outputs.append( httpclient.InferRequestedOutput( output_name, binary_data=True, class_count=FLAGS.classes ) ) # Send request result = triton_client.infer(model_name, inputs, outputs=outputs) postprocess(result, output_names, input_filenames, batch_size) print("PASS") kytensor-2.49.0.7/client/src/python/examples/grpc_client.py000077500000000000000000000101151527052371200237050ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import grpc from tritonclient.grpc import service_pb2, service_pb2_grpc FLAGS = None if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() model_name = "inception_graphdef" model_version = "" batch_size = 1 # Create gRPC stub for communicating with the server channel = grpc.insecure_channel(FLAGS.url) grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub(channel) # Health try: request = service_pb2.ServerLiveRequest() response = grpc_stub.ServerLive(request) print("server {}".format(response)) except Exception as ex: print(ex) request = service_pb2.ServerReadyRequest() response = grpc_stub.ServerReady(request) print("server {}".format(response)) request = service_pb2.ModelReadyRequest(name=model_name, version=model_version) response = grpc_stub.ModelReady(request) print("model {}".format(response)) # Metadata request = service_pb2.ServerMetadataRequest() response = grpc_stub.ServerMetadata(request) print("server metadata:\n{}".format(response)) request = service_pb2.ModelMetadataRequest(name=model_name, version=model_version) response = grpc_stub.ModelMetadata(request) print("model metadata:\n{}".format(response)) # Configuration request = service_pb2.ModelConfigRequest(name=model_name, version=model_version) response = grpc_stub.ModelConfig(request) print("model config:\n{}".format(response)) # Infer request = service_pb2.ModelInferRequest() request.model_name = model_name request.model_version = model_version request.id = "my request id" input = service_pb2.ModelInferRequest().InferInputTensor() input.name = "input" input.datatype = "FP32" input.shape.extend([1, 299, 299, 3]) request.inputs.extend([input]) output = service_pb2.ModelInferRequest().InferRequestedOutputTensor() output.name = "InceptionV3/Predictions/Softmax" request.outputs.extend([output]) request.raw_input_contents.extend([bytes(1072812 * "a", "utf-8")]) response = grpc_stub.ModelInfer(request) print("model infer:\n{}".format(response)) print("PASS") kytensor-2.49.0.7/client/src/python/examples/grpc_explicit_byte_content_client.py000077500000000000000000000113061527052371200303660ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import grpc import numpy as np from tritonclient import utils from tritonclient.grpc import service_pb2, service_pb2_grpc if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple_string" model_version = "" batch_size = 1 # Create gRPC stub for communicating with the server channel = grpc.insecure_channel(FLAGS.url) grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub(channel) # Generate the request request = service_pb2.ModelInferRequest() request.model_name = model_name request.model_version = model_version # Populate the inputs in inference request input0 = service_pb2.ModelInferRequest().InferInputTensor() input0.name = "INPUT0" input0.datatype = "BYTES" input0.shape.extend([1, 16]) for i in range(16): input0.contents.bytes_contents.append(("{}".format(i)).encode("utf-8")) input1 = service_pb2.ModelInferRequest().InferInputTensor() input1.name = "INPUT1" input1.datatype = "BYTES" input1.shape.extend([1, 16]) for i in range(16): input1.contents.bytes_contents.append("1".encode("utf-8")) request.inputs.extend([input0, input1]) # Populate the outputs in the inference request output0 = service_pb2.ModelInferRequest().InferRequestedOutputTensor() output0.name = "OUTPUT0" output1 = service_pb2.ModelInferRequest().InferRequestedOutputTensor() output1.name = "OUTPUT1" request.outputs.extend([output0, output1]) response = grpc_stub.ModelInfer(request) # Deserialize the output raw tensor to numpy array for proper comparison output_results = [] index = 0 for output in response.outputs: shape = [] for value in output.shape: shape.append(value) output_results.append( utils.deserialize_bytes_tensor(response.raw_output_contents[index]) ) output_results[-1] = np.resize(output_results[-1], shape) index += 1 if len(output_results) != 2: print("expected two output results") sys.exit(1) for i in range(16): print("{} + 1 = {}".format(i, output_results[0][0][i])) print("{} - 1 = {}".format(i, output_results[1][0][i])) if (i + 1) != int(output_results[0][0][i]): print("explicit string infer error: incorrect sum") sys.exit(1) if (i - 1) != int(output_results[1][0][i]): print("explicit string infer error: incorrect difference") sys.exit(1) print("PASS: explicit string") kytensor-2.49.0.7/client/src/python/examples/grpc_explicit_int8_content_client.py000077500000000000000000000122271527052371200303100ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import grpc import numpy as np from tritonclient.grpc import service_pb2, service_pb2_grpc if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple_int8" model_version = "" batch_size = 1 # Create gRPC stub for communicating with the server channel = grpc.insecure_channel(FLAGS.url) grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub(channel) # Generate the request request = service_pb2.ModelInferRequest() request.model_name = model_name request.model_version = model_version # Input data input0_data = [i for i in range(16)] input1_data = [1 for i in range(16)] # Populate the inputs in inference request input0 = service_pb2.ModelInferRequest().InferInputTensor() input0.name = "INPUT0" input0.datatype = "INT8" input0.shape.extend([1, 16]) input0.contents.int_contents[:] = input0_data input1 = service_pb2.ModelInferRequest().InferInputTensor() input1.name = "INPUT1" input1.datatype = "INT8" input1.shape.extend([1, 16]) input1.contents.int_contents[:] = input1_data request.inputs.extend([input0, input1]) # Populate the outputs in the inference request output0 = service_pb2.ModelInferRequest().InferRequestedOutputTensor() output0.name = "OUTPUT0" output1 = service_pb2.ModelInferRequest().InferRequestedOutputTensor() output1.name = "OUTPUT1" request.outputs.extend([output0, output1]) response = grpc_stub.ModelInfer(request) output_results = [] index = 0 for output in response.outputs: shape = [] for value in output.shape: shape.append(value) output_results.append( np.frombuffer(response.raw_output_contents[index], dtype=np.int8) ) output_results[-1] = np.resize(output_results[-1], shape) index += 1 if len(output_results) != 2: print("expected two output results") sys.exit(1) for i in range(16): print( str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(output_results[0][0][i]) ) print( str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(output_results[1][0][i]) ) if (input0_data[i] + input1_data[i]) != output_results[0][0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[i] - input1_data[i]) != output_results[1][0][i]: print("sync infer error: incorrect difference") sys.exit(1) # Server should catch wrong model version specification request.model_version = "wrong_specification" try: response = grpc_stub.ModelInfer(request) except Exception as e: if ( "failed to get model version from specified version string 'wrong_specification'" in e.__str__() ): print("PASS: explicit int8") kytensor-2.49.0.7/client/src/python/examples/grpc_explicit_int_content_client.py000077500000000000000000000124171527052371200302210ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import grpc import numpy as np from tritonclient.grpc import service_pb2, service_pb2_grpc if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple" model_version = "" batch_size = 1 # Create gRPC stub for communicating with the server channel = grpc.insecure_channel(FLAGS.url) grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub(channel) # Generate the request request = service_pb2.ModelInferRequest() request.model_name = model_name request.model_version = model_version # Input data input0_data = [i for i in range(16)] input1_data = [1 for i in range(16)] # Populate the inputs in inference request input0 = service_pb2.ModelInferRequest().InferInputTensor() input0.name = "INPUT0" input0.datatype = "INT32" input0.shape.extend([1, 16]) input0.contents.int_contents[:] = input0_data input1 = service_pb2.ModelInferRequest().InferInputTensor() input1.name = "INPUT1" input1.datatype = "INT32" input1.shape.extend([1, 16]) input1.contents.int_contents[:] = input1_data request.inputs.extend([input0, input1]) # Populate the outputs in the inference request output0 = service_pb2.ModelInferRequest().InferRequestedOutputTensor() output0.name = "OUTPUT0" output1 = service_pb2.ModelInferRequest().InferRequestedOutputTensor() output1.name = "OUTPUT1" request.outputs.extend([output0, output1]) response = grpc_stub.ModelInfer(request) output_results = [] index = 0 for output in response.outputs: shape = [] for value in output.shape: shape.append(value) output_results.append( np.frombuffer(response.raw_output_contents[index], dtype=np.int32) ) output_results[-1] = np.resize(output_results[-1], shape) index += 1 if len(output_results) != 2: print("expected two output results") sys.exit(1) for i in range(16): print( str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(output_results[0][0][i]) ) print( str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(output_results[1][0][i]) ) if (input0_data[i] + input1_data[i]) != output_results[0][0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[i] - input1_data[i]) != output_results[1][0][i]: print("sync infer error: incorrect difference") sys.exit(1) # Populating additional content field should generate an error request.raw_input_contents.extend([np.array(input0_data[0:8]).tobytes()]) request.inputs[0].contents.int_contents[:] = input0_data[8:] try: response = grpc_stub.ModelInfer(request) except Exception as e: if ( "contents field must not be specified when using " "raw_input_contents for 'INPUT0' for model 'simple'" in e.__str__() ): print("PASS: explicit int") kytensor-2.49.0.7/client/src/python/examples/grpc_image_client.py000077500000000000000000000373171527052371200250640ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os import struct import sys import grpc import numpy as np import tritonclient.grpc.model_config_pb2 as mc from PIL import Image from tritonclient.grpc import service_pb2, service_pb2_grpc FLAGS = None def model_dtype_to_np(model_dtype): if model_dtype == "BOOL": return bool elif model_dtype == "INT8": return np.int8 elif model_dtype == "INT16": return np.int16 elif model_dtype == "INT32": return np.int32 elif model_dtype == "INT64": return np.int64 elif model_dtype == "UINT8": return np.uint8 elif model_dtype == "UINT16": return np.uint16 elif model_dtype == "FP16": return np.float16 elif model_dtype == "FP32": return np.float32 elif model_dtype == "FP64": return np.float64 elif model_dtype == "BYTES": return np.dtype(object) return None def deserialize_bytes_tensor(encoded_tensor): strs = list() offset = 0 val_buf = encoded_tensor while offset < len(val_buf): l = struct.unpack_from(" 0 non_one_cnt = 0 for dim in output_metadata.shape: if output_batch_dim: output_batch_dim = False elif dim > 1: non_one_cnt += 1 if non_one_cnt > 1: raise Exception("expecting model output to be a vector") # Model input must have 3 dims, either CHW or HWC (not counting # the batch dimension), either CHW or HWC input_batch_dim = model_config.max_batch_size > 0 expected_input_dims = 3 + (1 if input_batch_dim else 0) if len(input_metadata.shape) != expected_input_dims: raise Exception( "expecting input to have {} dimensions, model '{}' input has {}".format( expected_input_dims, model_metadata.name, len(input_metadata.shape) ) ) if (input_config.format != mc.ModelInput.FORMAT_NCHW) and ( input_config.format != mc.ModelInput.FORMAT_NHWC ): raise Exception( "unexpected input format " + mc.ModelInput.Format.Name(input_config.format) + ", expecting " + mc.ModelInput.Format.Name(mc.ModelInput.FORMAT_NCHW) + " or " + mc.ModelInput.Format.Name(mc.ModelInput.FORMAT_NHWC) ) if input_config.format == mc.ModelInput.FORMAT_NHWC: h = input_metadata.shape[1 if input_batch_dim else 0] w = input_metadata.shape[2 if input_batch_dim else 1] c = input_metadata.shape[3 if input_batch_dim else 2] else: c = input_metadata.shape[1 if input_batch_dim else 0] h = input_metadata.shape[2 if input_batch_dim else 1] w = input_metadata.shape[3 if input_batch_dim else 2] return ( model_config.max_batch_size, input_metadata.name, output_metadata.name, c, h, w, input_config.format, input_metadata.datatype, ) def preprocess(img, format, dtype, c, h, w, scaling): """ Pre-process an image to meet the size, type and format requirements specified by the parameters. """ # np.set_printoptions(threshold='nan') if c == 1: sample_img = img.convert("L") else: sample_img = img.convert("RGB") resized_img = sample_img.resize((w, h), Image.BILINEAR) resized = np.array(resized_img) if resized.ndim == 2: resized = resized[:, :, np.newaxis] npdtype = model_dtype_to_np(dtype) typed = resized.astype(npdtype) if scaling == "INCEPTION": scaled = (typed / 127.5) - 1 elif scaling == "VGG": if c == 1: scaled = typed - np.asarray((128,), dtype=npdtype) else: scaled = typed - np.asarray((123, 117, 104), dtype=npdtype) else: scaled = typed # Swap to CHW if necessary if format == mc.ModelInput.FORMAT_NCHW: ordered = np.transpose(scaled, (2, 0, 1)) else: ordered = scaled # Channels are in RGB order. Currently model configuration data # doesn't provide any information as to other channel orderings # (like BGR) so we just assume RGB. return ordered def postprocess(response, filenames, batch_size, supports_batching): """ Post-process response to show classifications. """ if len(response.outputs) != 1: raise Exception("expected 1 output, got {}".format(len(response.outputs))) if len(response.raw_output_contents) != 1: raise Exception( "expected 1 output content, got {}".format( len(response.raw_output_contents) ) ) batched_result = deserialize_bytes_tensor(response.raw_output_contents[0]) contents = np.reshape(batched_result, response.outputs[0].shape) if supports_batching and len(contents) != batch_size: raise Exception("expected {} results, got {}".format(batch_size, len(contents))) if supports_batching and len(filenames) != batch_size: raise Exception( "expected {} filenames, got {}".format(batch_size, len(filenames)) ) if not supports_batching: contents = [contents] for index, results in enumerate(contents): print("Image '{}':".format(filenames[index])) for result in results: cls = "".join(chr(x) for x in result).split(":") print(" {} ({}) = {}".format(cls[0], cls[1], cls[2])) def requestGenerator( input_name, output_name, c, h, w, format, dtype, FLAGS, result_filenames, supports_batching, ): request = service_pb2.ModelInferRequest() request.model_name = FLAGS.model_name request.model_version = FLAGS.model_version filenames = [] if os.path.isdir(FLAGS.image_filename): filenames = [ os.path.join(FLAGS.image_filename, f) for f in os.listdir(FLAGS.image_filename) if os.path.isfile(os.path.join(FLAGS.image_filename, f)) ] else: filenames = [ FLAGS.image_filename, ] filenames.sort() output = service_pb2.ModelInferRequest().InferRequestedOutputTensor() output.name = output_name output.parameters["classification"].int64_param = FLAGS.classes request.outputs.extend([output]) input = service_pb2.ModelInferRequest().InferInputTensor() input.name = input_name input.datatype = dtype if format == mc.ModelInput.FORMAT_NHWC: input.shape.extend( [FLAGS.batch_size, h, w, c] if supports_batching else [h, w, c] ) else: input.shape.extend( [FLAGS.batch_size, c, h, w] if supports_batching else [c, h, w] ) # Preprocess image into input data according to model requirements # Preprocess the images into input data according to model # requirements image_data = [] for filename in filenames: img = Image.open(filename) image_data.append(preprocess(img, format, dtype, c, h, w, FLAGS.scaling)) # Send requests of FLAGS.batch_size images. If the number of # images isn't an exact multiple of FLAGS.batch_size then just # start over with the first images until the batch is filled. image_idx = 0 last_request = False while not last_request: input_bytes = None input_filenames = [] request.ClearField("inputs") request.ClearField("raw_input_contents") for idx in range(FLAGS.batch_size): input_filenames.append(filenames[image_idx]) if input_bytes is None: input_bytes = image_data[image_idx].tobytes() else: input_bytes += image_data[image_idx].tobytes() image_idx = (image_idx + 1) % len(image_data) if image_idx == 0: last_request = True request.inputs.extend([input]) result_filenames.append(input_filenames) request.raw_input_contents.extend([input_bytes]) yield request if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-a", "--async", dest="async_set", action="store_true", required=False, default=False, help="Use asynchronous inference API", ) parser.add_argument( "--streaming", action="store_true", required=False, default=False, help="Use streaming inference API", ) parser.add_argument( "-m", "--model-name", type=str, required=True, help="Name of model" ) parser.add_argument( "-x", "--model-version", type=str, required=False, default="", help="Version of model. Default is to use latest version.", ) parser.add_argument( "-b", "--batch-size", type=int, required=False, default=1, help="Batch size. Default is 1.", ) parser.add_argument( "-c", "--classes", type=int, required=False, default=1, help="Number of class results to report. Default is 1.", ) parser.add_argument( "-s", "--scaling", type=str, choices=["NONE", "INCEPTION", "VGG"], required=False, default="NONE", help="Type of scaling to apply to image pixels. Default is NONE.", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) parser.add_argument( "image_filename", type=str, nargs="?", default=None, help="Input image / Input folder.", ) FLAGS = parser.parse_args() # Create gRPC stub for communicating with the server channel = grpc.insecure_channel(FLAGS.url) grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub(channel) # Make sure the model matches our requirements, and get some # properties of the model that we need for preprocessing metadata_request = service_pb2.ModelMetadataRequest( name=FLAGS.model_name, version=FLAGS.model_version ) metadata_response = grpc_stub.ModelMetadata(metadata_request) config_request = service_pb2.ModelConfigRequest( name=FLAGS.model_name, version=FLAGS.model_version ) config_response = grpc_stub.ModelConfig(config_request) max_batch_size, input_name, output_name, c, h, w, format, dtype = parse_model( metadata_response, config_response.config ) supports_batching = max_batch_size > 0 if not supports_batching and FLAGS.batch_size != 1: raise Exception("This model doesn't support batching.") # Send requests of FLAGS.batch_size images. If the number of # images isn't an exact multiple of FLAGS.batch_size then just # start over with the first images until the batch is filled. requests = [] responses = [] result_filenames = [] # Send request if FLAGS.streaming: for response in grpc_stub.ModelStreamInfer( requestGenerator( input_name, output_name, c, h, w, format, dtype, FLAGS, result_filenames, supports_batching, ) ): responses.append(response) else: for request in requestGenerator( input_name, output_name, c, h, w, format, dtype, FLAGS, result_filenames, supports_batching, ): if not FLAGS.async_set: responses.append(grpc_stub.ModelInfer(request)) else: requests.append(grpc_stub.ModelInfer.future(request)) # For async, retrieve results according to the send order if FLAGS.async_set: for request in requests: responses.append(request.result()) error_found = False idx = 0 for response in responses: if FLAGS.streaming: if response.error_message != "": error_found = True print(response.error_message) else: postprocess( response.infer_response, result_filenames[idx], FLAGS.batch_size, supports_batching, ) else: postprocess( response, result_filenames[idx], FLAGS.batch_size, supports_batching ) idx += 1 if error_found: sys.exit(1) print("PASS") kytensor-2.49.0.7/client/src/python/examples/image_client.py000077500000000000000000000433131527052371200240420ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os import sys from functools import partial import numpy as np import tritonclient.grpc as grpcclient import tritonclient.grpc.model_config_pb2 as mc import tritonclient.http as httpclient from PIL import Image from tritonclient.utils import InferenceServerException, triton_to_np_dtype if sys.version_info >= (3, 0): import queue else: import Queue as queue class UserData: def __init__(self): self._completed_requests = queue.Queue() # Callback function used for async_stream_infer() def completion_callback(user_data, result, error): # passing error raise and handling out user_data._completed_requests.put((result, error)) FLAGS = None def parse_model(model_metadata, model_config): """ Check the configuration of a model to make sure it meets the requirements for an image classification network (as expected by this client) """ if len(model_metadata.inputs) != 1: raise Exception("expecting 1 input, got {}".format(len(model_metadata.inputs))) if len(model_metadata.outputs) != 1: raise Exception( "expecting 1 output, got {}".format(len(model_metadata.outputs)) ) if len(model_config.input) != 1: raise Exception( "expecting 1 input in model configuration, got {}".format( len(model_config.input) ) ) input_metadata = model_metadata.inputs[0] input_config = model_config.input[0] output_metadata = model_metadata.outputs[0] if output_metadata.datatype != "FP32": raise Exception( "expecting output datatype to be FP32, model '" + model_metadata.name + "' output type is " + output_metadata.datatype ) # Output is expected to be a vector. But allow any number of # dimensions as long as all but 1 is size 1 (e.g. { 10 }, { 1, 10 # }, { 10, 1, 1 } are all ok). Ignore the batch dimension if there # is one. output_batch_dim = model_config.max_batch_size > 0 non_one_cnt = 0 for dim in output_metadata.shape: if output_batch_dim: output_batch_dim = False elif dim > 1: non_one_cnt += 1 if non_one_cnt > 1: raise Exception("expecting model output to be a vector") # Model input must have 3 dims, either CHW or HWC (not counting # the batch dimension), either CHW or HWC input_batch_dim = model_config.max_batch_size > 0 expected_input_dims = 3 + (1 if input_batch_dim else 0) if len(input_metadata.shape) != expected_input_dims: raise Exception( "expecting input to have {} dimensions, model '{}' input has {}".format( expected_input_dims, model_metadata.name, len(input_metadata.shape) ) ) if type(input_config.format) == str: FORMAT_ENUM_TO_INT = dict(mc.ModelInput.Format.items()) input_config.format = FORMAT_ENUM_TO_INT[input_config.format] if (input_config.format != mc.ModelInput.FORMAT_NCHW) and ( input_config.format != mc.ModelInput.FORMAT_NHWC ): raise Exception( "unexpected input format " + mc.ModelInput.Format.Name(input_config.format) + ", expecting " + mc.ModelInput.Format.Name(mc.ModelInput.FORMAT_NCHW) + " or " + mc.ModelInput.Format.Name(mc.ModelInput.FORMAT_NHWC) ) if input_config.format == mc.ModelInput.FORMAT_NHWC: h = input_metadata.shape[1 if input_batch_dim else 0] w = input_metadata.shape[2 if input_batch_dim else 1] c = input_metadata.shape[3 if input_batch_dim else 2] else: c = input_metadata.shape[1 if input_batch_dim else 0] h = input_metadata.shape[2 if input_batch_dim else 1] w = input_metadata.shape[3 if input_batch_dim else 2] return ( model_config.max_batch_size, input_metadata.name, output_metadata.name, c, h, w, input_config.format, input_metadata.datatype, ) def preprocess(img, format, dtype, c, h, w, scaling, protocol): """ Pre-process an image to meet the size, type and format requirements specified by the parameters. """ # np.set_printoptions(threshold='nan') if c == 1: sample_img = img.convert("L") else: sample_img = img.convert("RGB") resized_img = sample_img.resize((w, h), Image.BILINEAR) resized = np.array(resized_img) if resized.ndim == 2: resized = resized[:, :, np.newaxis] npdtype = triton_to_np_dtype(dtype) typed = resized.astype(npdtype) if scaling == "INCEPTION": scaled = (typed / 127.5) - 1 elif scaling == "VGG": if c == 1: scaled = typed - np.asarray((128,), dtype=npdtype) else: scaled = typed - np.asarray((123, 117, 104), dtype=npdtype) else: scaled = typed # Swap to CHW if necessary if format == mc.ModelInput.FORMAT_NCHW: ordered = np.transpose(scaled, (2, 0, 1)) else: ordered = scaled # Channels are in RGB order. Currently model configuration data # doesn't provide any information as to other channel orderings # (like BGR) so we just assume RGB. return ordered def postprocess(results, output_name, batch_size, supports_batching): """ Post-process results to show classifications. """ output_array = results.as_numpy(output_name) if supports_batching and len(output_array) != batch_size: raise Exception( "expected {} results, got {}".format(batch_size, len(output_array)) ) # Include special handling for non-batching models for results in output_array: if not supports_batching: results = [results] for result in results: if output_array.dtype.type == np.object_: cls = "".join(chr(x) for x in result).split(":") else: cls = result.split(":") print(" {} ({}) = {}".format(cls[0], cls[1], cls[2])) def requestGenerator(batched_image_data, input_name, output_name, dtype, FLAGS): protocol = FLAGS.protocol.lower() if protocol == "grpc": client = grpcclient else: client = httpclient # Set the input data inputs = [client.InferInput(input_name, batched_image_data.shape, dtype)] inputs[0].set_data_from_numpy(batched_image_data) outputs = [client.InferRequestedOutput(output_name, class_count=FLAGS.classes)] yield inputs, outputs, FLAGS.model_name, FLAGS.model_version def convert_http_metadata_config(_metadata, _config): # NOTE: attrdict broken in python 3.10 and not maintained. # https://github.com/wallento/wavedrompy/issues/32#issuecomment-1306701776 try: from attrdict import AttrDict except ImportError: # Monkey patch collections import collections import collections.abc for type_name in collections.abc.__all__: setattr(collections, type_name, getattr(collections.abc, type_name)) from attrdict import AttrDict return AttrDict(_metadata), AttrDict(_config) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-a", "--async", dest="async_set", action="store_true", required=False, default=False, help="Use asynchronous inference API", ) parser.add_argument( "--streaming", action="store_true", required=False, default=False, help="Use streaming inference API. " + "The flag is only available with gRPC protocol.", ) parser.add_argument( "-m", "--model-name", type=str, required=True, help="Name of model" ) parser.add_argument( "-x", "--model-version", type=str, required=False, default="", help="Version of model. Default is to use latest version.", ) parser.add_argument( "-b", "--batch-size", type=int, required=False, default=1, help="Batch size. Default is 1.", ) parser.add_argument( "-c", "--classes", type=int, required=False, default=1, help="Number of class results to report. Default is 1.", ) parser.add_argument( "-s", "--scaling", type=str, choices=["NONE", "INCEPTION", "VGG"], required=False, default="NONE", help="Type of scaling to apply to image pixels. Default is NONE.", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) parser.add_argument( "-i", "--protocol", type=str, required=False, default="HTTP", help="Protocol (HTTP/gRPC) used to communicate with " + "the inference service. Default is HTTP.", ) parser.add_argument( "image_filename", type=str, nargs="?", default=None, help="Input image / Input folder.", ) FLAGS = parser.parse_args() if FLAGS.streaming and FLAGS.protocol.lower() != "grpc": raise Exception("Streaming is only allowed with gRPC protocol") try: if FLAGS.protocol.lower() == "grpc": # Create gRPC client for communicating with the server triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) else: # Specify large enough concurrency to handle the # the number of requests. concurrency = 20 if FLAGS.async_set else 1 triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose, concurrency=concurrency ) except Exception as e: print("client creation failed: " + str(e)) sys.exit(1) # Make sure the model matches our requirements, and get some # properties of the model that we need for preprocessing try: model_metadata = triton_client.get_model_metadata( model_name=FLAGS.model_name, model_version=FLAGS.model_version ) except InferenceServerException as e: print("failed to retrieve the metadata: " + str(e)) sys.exit(1) try: model_config = triton_client.get_model_config( model_name=FLAGS.model_name, model_version=FLAGS.model_version ) except InferenceServerException as e: print("failed to retrieve the config: " + str(e)) sys.exit(1) if FLAGS.protocol.lower() == "grpc": model_config = model_config.config else: model_metadata, model_config = convert_http_metadata_config( model_metadata, model_config ) max_batch_size, input_name, output_name, c, h, w, format, dtype = parse_model( model_metadata, model_config ) supports_batching = max_batch_size > 0 if not supports_batching and FLAGS.batch_size != 1: print("ERROR: This model doesn't support batching.") sys.exit(1) filenames = [] if os.path.isdir(FLAGS.image_filename): filenames = [ os.path.join(FLAGS.image_filename, f) for f in os.listdir(FLAGS.image_filename) if os.path.isfile(os.path.join(FLAGS.image_filename, f)) ] else: filenames = [ FLAGS.image_filename, ] filenames.sort() # Preprocess the images into input data according to model # requirements image_data = [] for filename in filenames: img = Image.open(filename) image_data.append( preprocess( img, format, dtype, c, h, w, FLAGS.scaling, FLAGS.protocol.lower() ) ) # Send requests of FLAGS.batch_size images. If the number of # images isn't an exact multiple of FLAGS.batch_size then just # start over with the first images until the batch is filled. requests = [] responses = [] result_filenames = [] request_ids = [] image_idx = 0 last_request = False user_data = UserData() # Holds the handles to the ongoing HTTP async requests. async_requests = [] sent_count = 0 if FLAGS.streaming: triton_client.start_stream(partial(completion_callback, user_data)) while not last_request: input_filenames = [] repeated_image_data = [] for idx in range(FLAGS.batch_size): input_filenames.append(filenames[image_idx]) repeated_image_data.append(image_data[image_idx]) image_idx = (image_idx + 1) % len(image_data) if image_idx == 0: last_request = True if supports_batching: batched_image_data = np.stack(repeated_image_data, axis=0) else: batched_image_data = repeated_image_data[0] # Send request try: for inputs, outputs, model_name, model_version in requestGenerator( batched_image_data, input_name, output_name, dtype, FLAGS ): sent_count += 1 if FLAGS.streaming: triton_client.async_stream_infer( FLAGS.model_name, inputs, request_id=str(sent_count), model_version=FLAGS.model_version, outputs=outputs, ) elif FLAGS.async_set: if FLAGS.protocol.lower() == "grpc": triton_client.async_infer( FLAGS.model_name, inputs, partial(completion_callback, user_data), request_id=str(sent_count), model_version=FLAGS.model_version, outputs=outputs, ) else: async_requests.append( triton_client.async_infer( FLAGS.model_name, inputs, request_id=str(sent_count), model_version=FLAGS.model_version, outputs=outputs, ) ) else: responses.append( triton_client.infer( FLAGS.model_name, inputs, request_id=str(sent_count), model_version=FLAGS.model_version, outputs=outputs, ) ) except InferenceServerException as e: print("inference failed: " + str(e)) if FLAGS.streaming: triton_client.stop_stream() sys.exit(1) if FLAGS.streaming: triton_client.stop_stream() if FLAGS.protocol.lower() == "grpc": if FLAGS.streaming or FLAGS.async_set: processed_count = 0 while processed_count < sent_count: (results, error) = user_data._completed_requests.get() processed_count += 1 if error is not None: print("inference failed: " + str(error)) sys.exit(1) responses.append(results) else: if FLAGS.async_set: # Collect results from the ongoing async requests # for HTTP Async requests. for async_request in async_requests: responses.append(async_request.get_result()) for response in responses: if FLAGS.protocol.lower() == "grpc": this_id = response.get_response().id else: this_id = response.get_response()["id"] print("Request {}, batch size {}".format(this_id, FLAGS.batch_size)) postprocess(response, output_name, FLAGS.batch_size, supports_batching) print("PASS") kytensor-2.49.0.7/client/src/python/examples/memory_growth_test.py000077500000000000000000000077411527052371200253700ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys from functools import partial import numpy as np if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-i", "--protocol", type=str, required=False, default="HTTP", help="Protocol (HTTP/gRPC) used to communicate with " + "the inference service. Default is HTTP.", ) parser.add_argument( "-r", "--repetitions", type=int, required=False, default=100, help="Number of inferences to run. Default is 100.", ) FLAGS = parser.parse_args() model_name = "custom_identity_int32" # Create the data for the input tensor. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) if FLAGS.protocol.lower() == "grpc": import tritonclient.grpc as grpcclient create_client = partial( grpcclient.InferenceServerClient, url="localhost:8001", verbose=FLAGS.verbose, ) create_input = partial(grpcclient.InferInput) create_output = partial(grpcclient.InferRequestedOutput) else: import tritonclient.http as httpclient create_client = partial( httpclient.InferenceServerClient, url="localhost:8000", verbose=FLAGS.verbose, ) create_input = partial(httpclient.InferInput) create_output = partial(httpclient.InferRequestedOutput) for i in range(FLAGS.repetitions): triton_client = create_client() # Infer inputs = [] outputs = [] inputs.append(create_input("INPUT0", [1, 16], "INT32")) inputs[0].set_data_from_numpy(input0_data) outputs.append(create_output("OUTPUT0")) results = triton_client.infer( model_name=model_name, inputs=inputs, outputs=outputs ) # Get the output arrays from the results and verify output0_data = results.as_numpy("OUTPUT0") if ( (output0_data.dtype != input0_data.dtype) or (output0_data.shape != input0_data.shape) or not (np.array_equal(output0_data, input0_data)) ): print("incorrect output") sys.exit(1) kytensor-2.49.0.7/client/src/python/examples/reuse_infer_objects_client.py000077500000000000000000000234471527052371200270050ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys from builtins import range import numpy as np import tritonclient.grpc as grpcclient import tritonclient.http as httpclient import tritonclient.utils as utils import tritonclient.utils.shared_memory as shm FLAGS = None def infer_and_validata(use_shared_memory, orig_input0_data, orig_input1_data): if use_shared_memory: input0_data = orig_input0_data input1_data = orig_input1_data byte_size = input0_data.size * input0_data.itemsize inputs[0].set_shared_memory("input0_data", byte_size) inputs[1].set_shared_memory("input1_data", byte_size) outputs[0].set_shared_memory("output0_data", byte_size) outputs[1].set_shared_memory("output1_data", byte_size) else: input0_data = orig_input0_data input1_data = orig_input1_data * 2 inputs[0].set_data_from_numpy(np.expand_dims(input0_data, axis=0)) inputs[1].set_data_from_numpy(np.expand_dims(input1_data, axis=0)) outputs[0].unset_shared_memory() outputs[1].unset_shared_memory() results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Read results from the shared memory. output0 = results.get_output("OUTPUT0") if output0 is not None: if use_shared_memory: if protocol == "grpc": output0_data = shm.get_contents_as_numpy( shm_op0_handle, utils.triton_to_np_dtype(output0.datatype), output0.shape, ) else: output0_data = shm.get_contents_as_numpy( shm_op0_handle, utils.triton_to_np_dtype(output0["datatype"]), output0["shape"], ) else: output0_data = results.as_numpy("OUTPUT0") else: print("OUTPUT0 is missing in the response.") sys.exit(1) output1 = results.get_output("OUTPUT1") if output1 is not None: if use_shared_memory: if protocol == "grpc": output1_data = shm.get_contents_as_numpy( shm_op1_handle, utils.triton_to_np_dtype(output1.datatype), output1.shape, ) else: output1_data = shm.get_contents_as_numpy( shm_op1_handle, utils.triton_to_np_dtype(output1["datatype"]), output1["shape"], ) else: output1_data = results.as_numpy("OUTPUT1") else: print("OUTPUT1 is missing in the response.") sys.exit(1) if use_shared_memory: print("\n\n======== SHARED_MEMORY ========\n") else: print("\n\n======== NO_SHARED_MEMORY ========\n") for i in range(16): print( str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(output1_data[0][i]) ) if (input0_data[i] + input1_data[i]) != output0_data[0][i]: print("shm infer error: incorrect sum") sys.exit(1) if (input0_data[i] - input1_data[i]) != output1_data[0][i]: print("shm infer error: incorrect difference") sys.exit(1) print("\n======== END ========\n\n") # Tests whether the same InferInput and InferRequestedOutput objects can be # successfully used repeatedly for different inferences using/not-using # shared memory. if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-i", "--protocol", type=str, required=False, default="HTTP", help="Protocol (HTTP/gRPC) used to communicate with " + "the inference service. Default is HTTP.", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() protocol = FLAGS.protocol.lower() try: if protocol == "grpc": # Create gRPC client for communicating with the server triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) else: # Create HTTP client for communicating with the server triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("client creation failed: " + str(e)) sys.exit(1) # To make sure no shared memory regions are registered with the # server. triton_client.unregister_system_shared_memory() triton_client.unregister_cuda_shared_memory() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple" model_version = "" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input1_data = np.ones(shape=16, dtype=np.int32) input_byte_size = input0_data.size * input0_data.itemsize output_byte_size = input_byte_size # Create Output0 and Output1 in Shared Memory and store shared memory handles shm_op0_handle = shm.create_shared_memory_region( "output0_data", "/output0_simple", output_byte_size ) shm_op1_handle = shm.create_shared_memory_region( "output1_data", "/output1_simple", output_byte_size ) # Register Output0 and Output1 shared memory with Triton Server triton_client.register_system_shared_memory( "output0_data", "/output0_simple", output_byte_size ) triton_client.register_system_shared_memory( "output1_data", "/output1_simple", output_byte_size ) # Create Input0 and Input1 in Shared Memory and store shared memory handles shm_ip0_handle = shm.create_shared_memory_region( "input0_data", "/input0_simple", input_byte_size ) shm_ip1_handle = shm.create_shared_memory_region( "input1_data", "/input1_simple", input_byte_size ) # Put input data values into shared memory shm.set_shared_memory_region(shm_ip0_handle, [input0_data]) shm.set_shared_memory_region(shm_ip1_handle, [input1_data]) # Register Input0 and Input1 shared memory with Triton Server triton_client.register_system_shared_memory( "input0_data", "/input0_simple", input_byte_size ) triton_client.register_system_shared_memory( "input1_data", "/input1_simple", input_byte_size ) # Set the parameters to use data from shared memory inputs = [] if protocol == "grpc": inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) else: inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) outputs = [] if protocol == "grpc": outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) else: outputs.append(httpclient.InferRequestedOutput("OUTPUT0")) outputs.append(httpclient.InferRequestedOutput("OUTPUT1")) # Use shared memory infer_and_validata(True, input0_data, input1_data) # Don't use shared memory infer_and_validata(False, input0_data, input1_data) # Use shared memory infer_and_validata(True, input0_data, input1_data) triton_client.unregister_system_shared_memory() shm.destroy_shared_memory_region(shm_ip0_handle) shm.destroy_shared_memory_region(shm_ip1_handle) shm.destroy_shared_memory_region(shm_op0_handle) shm.destroy_shared_memory_region(shm_op1_handle) kytensor-2.49.0.7/client/src/python/examples/simple_grpc_aio_infer_client.py000077500000000000000000000155621527052371200273040ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import asyncio import sys import numpy as np import tritonclient.grpc.aio as grpcclient def get_triton_client(FLAGS): try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose, ssl=FLAGS.ssl, root_certificates=FLAGS.root_certificates, private_key=FLAGS.private_key, certificate_chain=FLAGS.certificate_chain, ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit() return triton_client def get_inputs_and_outputs(): # Infer inputs = [] outputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) input1_data = np.ones(shape=(1, 16), dtype=np.int32) # Initialize the data inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) return inputs, outputs, input0_data, input1_data def get_output(input0_data, input1_data, results): # Get the output arrays from the results output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") # Validate the output for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) async def main(FLAGS): # Initialize triton_client = get_triton_client(FLAGS) model_name = "simple" inputs, outputs, input0_data, input1_data = get_inputs_and_outputs() # Test with outputs results = await triton_client.infer( model_name=model_name, inputs=inputs, outputs=outputs, client_timeout=FLAGS.client_timeout, headers={"test": "1"}, compression_algorithm=FLAGS.grpc_compression_algorithm, ) statistics = await triton_client.get_inference_statistics(model_name=model_name) print(statistics) if len(statistics.model_stats) != 1: print("FAILED: Inference Statistics") sys.exit(1) get_output(input0_data, input1_data, results) # Test with no outputs, in parallel result_1, result_2 = await asyncio.gather( triton_client.infer( model_name=model_name, inputs=inputs, outputs=None, compression_algorithm=FLAGS.grpc_compression_algorithm, ), triton_client.infer( model_name=model_name, inputs=inputs, outputs=None, compression_algorithm=FLAGS.grpc_compression_algorithm, ), ) get_output(input0_data, input1_data, result_1) get_output(input0_data, input1_data, result_2) print("PASS: grpc aio infer") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) parser.add_argument( "-s", "--ssl", action="store_true", required=False, default=False, help="Enable SSL encrypted channel to the server", ) parser.add_argument( "-t", "--client-timeout", type=float, required=False, default=None, help="Client timeout in seconds. Default is None.", ) parser.add_argument( "-r", "--root-certificates", type=str, required=False, default=None, help="File holding PEM-encoded root certificates. Default is None.", ) parser.add_argument( "-p", "--private-key", type=str, required=False, default=None, help="File holding PEM-encoded private key. Default is None.", ) parser.add_argument( "-x", "--certificate-chain", type=str, required=False, default=None, help="File holding PEM-encoded certificate chain. Default is None.", ) parser.add_argument( "-C", "--grpc-compression-algorithm", type=str, required=False, default=None, help="The compression algorithm to be used when sending request to server. Default is None.", ) FLAGS = parser.parse_args() asyncio.run(main(FLAGS)) kytensor-2.49.0.7/client/src/python/examples/simple_grpc_aio_sequence_stream_infer_client.py000077500000000000000000000227631527052371200325500ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import asyncio import queue import sys import uuid import numpy as np import tritonclient.grpc.aio as grpcclient from tritonclient.utils import InferenceServerException class UserData: def __init__(self): self._completed_requests = queue.Queue() async def async_stream_yield( values, batch_size, sequence_id, model_name, model_version ): count = 1 for value in values: # Create the tensor for INPUT value_data = np.full(shape=[batch_size, 1], fill_value=value, dtype=np.int32) inputs = [] inputs.append(grpcclient.InferInput("INPUT", value_data.shape, "INT32")) # Initialize the data inputs[0].set_data_from_numpy(value_data) outputs = [] outputs.append(grpcclient.InferRequestedOutput("OUTPUT")) # Issue the asynchronous sequence inference. yield { "model_name": model_name, "inputs": inputs, "outputs": outputs, "request_id": "{}_{}".format(sequence_id, count), "sequence_id": sequence_id, "sequence_start": (count == 1), "sequence_end": (count == len(values)), } count = count + 1 async def main(FLAGS): # We use custom "sequence" models which take 1 input # value. The output is the accumulated value of the inputs. See # src/custom/sequence. int_sequence_model_name = ( "simple_dyna_sequence" if FLAGS.dyna else "simple_sequence" ) string_sequence_model_name = ( "simple_string_dyna_sequence" if FLAGS.dyna else "simple_sequence" ) model_version = "" batch_size = 1 values = [11, 7, 5, 3, 2, 0, 1] # Will use two sequences and send them asynchronously. Note the # sequence IDs should be non-zero because zero is reserved for # non-sequence requests. int_sequence_id0 = 1000 + FLAGS.offset * 2 int_sequence_id1 = 1001 + FLAGS.offset * 2 # For string sequence IDs, the dyna backend requires that the # sequence id be decodable into an integer, otherwise we'll use # a UUID4 sequence id and a model that doesn't require corrid # control. string_sequence_id0 = str(1002 + FLAGS.offset) if FLAGS.dyna else str(uuid.uuid4()) int_result0_list = [] int_result1_list = [] string_result0_list = [] async with grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) as triton_client: # Request iterator that yields the next request async def async_request_iterator(): async for request in async_stream_yield( [0] + values, batch_size, int_sequence_id0, int_sequence_model_name, model_version, ): yield request async for request in async_stream_yield( [100] + [-1 * val for val in values], batch_size, int_sequence_id1, int_sequence_model_name, model_version, ): yield request async for request in async_stream_yield( [20] + [-1 * val for val in values], batch_size, string_sequence_id0, string_sequence_model_name, model_version, ): yield request try: # Start streaming response_iterator = triton_client.stream_infer( inputs_iterator=async_request_iterator(), stream_timeout=FLAGS.stream_timeout, ) # Read response from the stream user_data = UserData() async for response in response_iterator: result, error = response if error: user_data._completed_requests.put(error) else: user_data._completed_requests.put(result) except InferenceServerException as error: print(error) sys.exit(1) # Retrieve results recv_count = 0 while recv_count < (3 * (len(values) + 1)): data_item = user_data._completed_requests.get() if type(data_item) == InferenceServerException: print(data_item) sys.exit(1) else: try: this_id = data_item.get_response().id.split("_")[0] if int(this_id) == int_sequence_id0: int_result0_list.append(data_item.as_numpy("OUTPUT")) elif int(this_id) == int_sequence_id1: int_result1_list.append(data_item.as_numpy("OUTPUT")) elif this_id == string_sequence_id0: string_result0_list.append(data_item.as_numpy("OUTPUT")) else: print( "unexpected sequence id returned by the server: {}".format( this_id ) ) sys.exit(1) except ValueError: string_result0_list.append(data_item.as_numpy("OUTPUT")) recv_count = recv_count + 1 # Check results for i in range(len(int_result0_list)): int_seq0_expected = 1 if (i == 0) else values[i - 1] int_seq1_expected = 101 if (i == 0) else values[i - 1] * -1 # For string sequence ID we are testing two different backends if i == 0 and FLAGS.dyna: string_seq0_expected = 20 elif i == 0 and not FLAGS.dyna: string_seq0_expected = 21 elif i != 0 and FLAGS.dyna: string_seq0_expected = values[i - 1] * -1 + int( string_result0_list[i - 1][0][0] ) else: string_seq0_expected = values[i - 1] * -1 # The dyna_sequence custom backend adds the correlation ID # to the last request in a sequence. if FLAGS.dyna and (i != 0) and (values[i - 1] == 1): int_seq0_expected += int_sequence_id0 int_seq1_expected += int_sequence_id1 string_seq0_expected += int(string_sequence_id0) print( "[" + str(i) + "] " + str(int_result0_list[i][0][0]) + " : " + str(int_result1_list[i][0][0]) + " : " + str(string_result0_list[i][0][0]) ) if ( (int_seq0_expected != int_result0_list[i][0][0]) or (int_seq1_expected != int_result1_list[i][0][0]) or (string_seq0_expected != string_result0_list[i][0][0]) ): print( "[ expected ] " + str(int_seq0_expected) + " : " + str(int_seq1_expected) + " : " + str(string_seq0_expected) ) sys.exit(1) print("PASS: grpc aio sequence stream") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL and it gRPC port. Default is localhost:8001.", ) parser.add_argument( "-t", "--stream-timeout", type=float, required=False, default=None, help="Stream timeout in seconds. Default is None.", ) parser.add_argument( "-d", "--dyna", action="store_true", required=False, default=False, help="Assume dynamic sequence model", ) parser.add_argument( "-o", "--offset", type=int, required=False, default=0, help="Add offset to sequence ID used", ) FLAGS = parser.parse_args() asyncio.run(main(FLAGS)) kytensor-2.49.0.7/client/src/python/examples/simple_grpc_async_infer_client.py000077500000000000000000000127661527052371200276540ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import time from functools import partial import numpy as np import tritonclient.grpc as grpcclient from tritonclient.utils import InferenceServerException if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) parser.add_argument( "-t", "--client-timeout", type=float, required=False, default=None, help="Client timeout in seconds. Default is None.", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit() model_name = "simple" # Infer inputs = [] outputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) input1_data = np.ones(shape=(1, 16), dtype=np.int32) # Initialize the data inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) # Define the callback function. Note the last two parameters should be # result and error. InferenceServerClient would povide the results of an # inference as grpcclient.InferResult in result. For successful # inference, error will be None, otherwise it will be an object of # tritonclientutils.InferenceServerException holding the error details def callback(user_data, result, error): if error: user_data.append(error) else: user_data.append(result) # list to hold the results of inference. user_data = [] # Inference call triton_client.async_infer( model_name=model_name, inputs=inputs, callback=partial(callback, user_data), outputs=outputs, client_timeout=FLAGS.client_timeout, ) # Wait until the results are available in user_data time_out = 10 while (len(user_data) == 0) and time_out > 0: time_out = time_out - 1 time.sleep(1) # Display and validate the available results if len(user_data) == 1: # Check for the errors if type(user_data[0]) == InferenceServerException: print(user_data[0]) sys.exit(1) # Validate the values by matching with already computed expected # values. output0_data = user_data[0].as_numpy("OUTPUT0") output1_data = user_data[0].as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) print("PASS: Async infer") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_cudashm_client.py000077500000000000000000000162231527052371200267700ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys from builtins import range from ctypes import * import numpy as np import tritonclient.grpc as grpcclient import tritonclient.utils.cuda_shared_memory as cudashm from tritonclient import utils FLAGS = None if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) # To make sure no shared memory regions are registered with the # server. triton_client.unregister_system_shared_memory() triton_client.unregister_cuda_shared_memory() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple" model_version = "" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input1_data = np.ones(shape=16, dtype=np.int32) input_byte_size = input0_data.size * input0_data.itemsize output_byte_size = input_byte_size # Create Output0 and Output1 in Shared Memory and store shared memory handles shm_op0_handle = cudashm.create_shared_memory_region( "output0_data", output_byte_size, 0 ) shm_op1_handle = cudashm.create_shared_memory_region( "output1_data", output_byte_size, 0 ) # Register Output0 and Output1 shared memory with Triton Server triton_client.register_cuda_shared_memory( "output0_data", cudashm.get_raw_handle(shm_op0_handle), 0, output_byte_size ) triton_client.register_cuda_shared_memory( "output1_data", cudashm.get_raw_handle(shm_op1_handle), 0, output_byte_size ) # Create Input0 and Input1 in Shared Memory and store shared memory handles shm_ip0_handle = cudashm.create_shared_memory_region( "input0_data", input_byte_size, 0 ) shm_ip1_handle = cudashm.create_shared_memory_region( "input1_data", input_byte_size, 0 ) # Put input data values into shared memory cudashm.set_shared_memory_region(shm_ip0_handle, [input0_data]) cudashm.set_shared_memory_region(shm_ip1_handle, [input1_data]) # Register Input0 and Input1 shared memory with Triton Server triton_client.register_cuda_shared_memory( "input0_data", cudashm.get_raw_handle(shm_ip0_handle), 0, input_byte_size ) triton_client.register_cuda_shared_memory( "input1_data", cudashm.get_raw_handle(shm_ip1_handle), 0, input_byte_size ) # Set the parameters to use data from shared memory inputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs[-1].set_shared_memory("input0_data", input_byte_size) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) inputs[-1].set_shared_memory("input1_data", input_byte_size) outputs = [] outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs[-1].set_shared_memory("output0_data", output_byte_size) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) outputs[-1].set_shared_memory("output1_data", output_byte_size) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Read results from the shared memory. output0 = results.get_output("OUTPUT0") if output0 is not None: output0_data = cudashm.get_contents_as_numpy( shm_op0_handle, utils.triton_to_np_dtype(output0.datatype), output0.shape ) else: print("OUTPUT0 is missing in the response.") sys.exit(1) output1 = results.get_output("OUTPUT1") if output1 is not None: output1_data = cudashm.get_contents_as_numpy( shm_op1_handle, utils.triton_to_np_dtype(output1.datatype), output1.shape ) else: print("OUTPUT1 is missing in the response.") sys.exit(1) for i in range(16): print( str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(output1_data[0][i]) ) if (input0_data[i] + input1_data[i]) != output0_data[0][i]: print("cudashm infer error: incorrect sum") sys.exit(1) if (input0_data[i] - input1_data[i]) != output1_data[0][i]: print("cudashm infer error: incorrect difference") sys.exit(1) print(triton_client.get_cuda_shared_memory_status()) triton_client.unregister_cuda_shared_memory() assert len(cudashm.allocated_shared_memory_regions()) == 4 cudashm.destroy_shared_memory_region(shm_ip0_handle) cudashm.destroy_shared_memory_region(shm_ip1_handle) cudashm.destroy_shared_memory_region(shm_op0_handle) cudashm.destroy_shared_memory_region(shm_op1_handle) assert len(cudashm.allocated_shared_memory_regions()) == 0 print("PASS: cudashm") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_custom_args_client.py000077500000000000000000000115431527052371200276720ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonclient.grpc as grpcclient if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() try: # Example of setting custom GRPC ChannelArguments. Users can set any # channel_args here at your own risk, Triton will assume they are # correct and complete. channel_args = [ # Define custom max message sizes: 1MB here is an arbitrary example. ("grpc.max_send_message_length", 1024 * 1024), ("grpc.max_receive_message_length", 1024 * 1024), # Example of setting KeepAlive options through generic channel_args ("grpc.keepalive_time_ms", 2**31 - 1), ("grpc.keepalive_timeout_ms", 20000), ("grpc.keepalive_permit_without_calls", False), ("grpc.http2.max_pings_without_data", 2), # Example arg requested for the feature ("grpc.dns_enable_srv_queries", 1), ] triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose, channel_args=channel_args ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit() model_name = "simple" # Infer inputs = [] outputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) input1_data = np.ones(shape=(1, 16), dtype=np.int32) # Initialize the data inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) # Test with outputs results = triton_client.infer( model_name=model_name, inputs=inputs, outputs=outputs, headers={"test": "1"} ) # Get the output arrays from the results output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) print("PASS: CustomArgs") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_custom_repeat.py000077500000000000000000000130051527052371200266530ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import queue import sys from functools import partial import numpy as np import tritonclient.grpc as grpcclient from tritonclient.utils import InferenceServerException FLAGS = None class UserData: def __init__(self): self._completed_requests = queue.Queue() # Define the callback function. Note the last two parameters should be # result and error. InferenceServerClient would povide the results of an # inference as grpcclient.InferResult in result. For successful # inference, error will be None, otherwise it will be an object of # tritonclientutils.InferenceServerException holding the error details def callback(user_data, result, error): if error: user_data._completed_requests.put(error) else: user_data._completed_requests.put(result) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL and it gRPC port. Default is localhost:8001.", ) FLAGS = parser.parse_args() # We use the custom "repeat_int32" model which takes 3 inputs and # 1 output. For a single request the model will generate 'repeat_count' # responses. See is src/backends/backend/examples/repeat.cc. model_name = "repeat_int32" model_version = "" repeat_count = 10 data_offset = 100 delay_time = 1000 wait_time = 1000 # Generate the data input_data = np.arange( start=data_offset, stop=data_offset + repeat_count, dtype=np.int32 ) delay_data = (np.ones([repeat_count], dtype=np.uint32)) * delay_time wait_data = np.array([wait_time], dtype=np.uint32) # Initialize the data. inputs = [] inputs.append(grpcclient.InferInput("IN", [repeat_count], "INT32")) inputs[-1].set_data_from_numpy(input_data) inputs.append(grpcclient.InferInput("DELAY", [repeat_count], "UINT32")) inputs[-1].set_data_from_numpy(delay_data) inputs.append(grpcclient.InferInput("WAIT", [1], "UINT32")) inputs[-1].set_data_from_numpy(wait_data) outputs = [] outputs.append(grpcclient.InferRequestedOutput("OUT")) result_list = [] user_data = UserData() # It is advisable to use client object within with..as clause # when sending streaming requests. This ensures the client # is closed when the block inside with exits. with grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) as triton_client: try: # Establish stream triton_client.start_stream(callback=partial(callback, user_data)) # Send a single inference request triton_client.async_stream_infer( model_name=model_name, inputs=inputs, outputs=outputs ) except InferenceServerException as error: print(error) sys.exit(1) # Retrieve results... recv_count = 0 while recv_count < repeat_count: data_item = user_data._completed_requests.get() if type(data_item) == InferenceServerException: print(data_item) sys.exit(1) else: result_list.append(data_item.as_numpy("OUT")) recv_count = recv_count + 1 expected_data = data_offset for i in range(len(result_list)): if len(result_list[i]) != 1: print( "unexpected number of elements in the output, expected 1, got {}".format( len(result_list[i]) ) ) sys.exit(1) print("{} : {}".format(result_list[i][0], expected_data)) if result_list[i][0] != expected_data: print("mismatch in the results") sys.exit(1) expected_data += 1 print("PASS: Decoupled API") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_health_metadata.py000077500000000000000000000074071527052371200271170ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import tritonclient.grpc as grpcclient from tritonclient.utils import InferenceServerException if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit() model_name = "simple" # Health if not triton_client.is_server_live(headers={"test": "1", "dummy": "2"}): print("FAILED : is_server_live") sys.exit(1) if not triton_client.is_server_ready(): print("FAILED : is_server_ready") sys.exit(1) if not triton_client.is_model_ready(model_name): print("FAILED : is_model_ready") sys.exit(1) # Metadata metadata = triton_client.get_server_metadata() if not (metadata.name == "triton"): print("FAILED : get_server_metadata") sys.exit(1) print(metadata) metadata = triton_client.get_model_metadata( model_name, headers={"test": "1", "dummy": "2"} ) if not (metadata.name == model_name): print("FAILED : get_model_metadata") sys.exit(1) print(metadata) # Passing incorrect model name try: metadata = triton_client.get_model_metadata("wrong_model_name") except InferenceServerException as ex: if "Request for unknown model" not in ex.message(): print("FAILED : get_model_metadata wrong_model_name") print("Got: {}".format(ex.message())) sys.exit(1) else: print("FAILED : get_model_metadata wrong_model_name") sys.exit(1) # Configuration config = triton_client.get_model_config(model_name) if not (config.config.name == model_name): print("FAILED: get_model_config") sys.exit(1) kytensor-2.49.0.7/client/src/python/examples/simple_grpc_infer_client.py000077500000000000000000000154611527052371200264520ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonclient.grpc as grpcclient if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) parser.add_argument( "-s", "--ssl", action="store_true", required=False, default=False, help="Enable SSL encrypted channel to the server", ) parser.add_argument( "-t", "--client-timeout", type=float, required=False, default=None, help="Client timeout in seconds. Default is None.", ) parser.add_argument( "-r", "--root-certificates", type=str, required=False, default=None, help="File holding PEM-encoded root certificates. Default is None.", ) parser.add_argument( "-p", "--private-key", type=str, required=False, default=None, help="File holding PEM-encoded private key. Default is None.", ) parser.add_argument( "-x", "--certificate-chain", type=str, required=False, default=None, help="File holding PEM-encoded certificate chain. Default is None.", ) parser.add_argument( "-C", "--grpc-compression-algorithm", type=str, required=False, default=None, help="The compression algorithm to be used when sending request to server. Default is None.", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose, ssl=FLAGS.ssl, root_certificates=FLAGS.root_certificates, private_key=FLAGS.private_key, certificate_chain=FLAGS.certificate_chain, ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit() model_name = "simple" # Infer inputs = [] outputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) input1_data = np.ones(shape=(1, 16), dtype=np.int32) # Initialize the data inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) # Test with outputs results = triton_client.infer( model_name=model_name, inputs=inputs, outputs=outputs, client_timeout=FLAGS.client_timeout, headers={"test": "1"}, compression_algorithm=FLAGS.grpc_compression_algorithm, ) statistics = triton_client.get_inference_statistics(model_name=model_name) print(statistics) if len(statistics.model_stats) != 1: print("FAILED: Inference Statistics") sys.exit(1) # Get the output arrays from the results output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) # Test with no outputs results = triton_client.infer( model_name=model_name, inputs=inputs, outputs=None, compression_algorithm=FLAGS.grpc_compression_algorithm, ) # Get the output arrays from the results output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) print("PASS: infer") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_keepalive_client.py000077500000000000000000000135241527052371200273120ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonclient.grpc as grpcclient if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) # GRPC KeepAlive: https://github.com/grpc/grpc/blob/master/doc/keepalive.md parser.add_argument( "--grpc-keepalive-time", type=int, required=False, default=2**31 - 1, help="The period (in milliseconds) after which a keepalive ping is sent on " "the transport. Default is 2**31-1 (INT_MAX: disabled).", ) parser.add_argument( "--grpc-keepalive-timeout", type=int, required=False, default=20000, help="The period (in milliseconds) the sender of the keepalive ping waits " "for an acknowledgement. If it does not receive an acknowledgment " "within this time, it will close the connection. " "Default is 20000 (20 seconds).", ) parser.add_argument( "--grpc-keepalive-permit-without-calls", action="store_true", required=False, default=False, help="Allows keepalive pings to be sent even if there are no calls in " "flight. Default is False.", ) parser.add_argument( "--grpc-http2-max-pings-without-data", type=int, required=False, default=2, help="The maximum number of pings that can be sent when there is no " "data/header frame to be sent. gRPC Core will not continue sending " "pings if we run over the limit. Setting it to 0 allows sending pings " "without such a restriction. Default is 2.", ) FLAGS = parser.parse_args() try: keepalive_options = grpcclient.KeepAliveOptions( keepalive_time_ms=FLAGS.grpc_keepalive_time, keepalive_timeout_ms=FLAGS.grpc_keepalive_timeout, keepalive_permit_without_calls=FLAGS.grpc_keepalive_permit_without_calls, http2_max_pings_without_data=FLAGS.grpc_http2_max_pings_without_data, ) triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose, keepalive_options=keepalive_options ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit() model_name = "simple" # Infer inputs = [] outputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) input1_data = np.ones(shape=(1, 16), dtype=np.int32) # Initialize the data inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) # Test with outputs results = triton_client.infer( model_name=model_name, inputs=inputs, outputs=outputs, headers={"test": "1"} ) # Get the output arrays from the results output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) print("PASS: KeepAlive") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_model_control.py000077500000000000000000000102001527052371200266330ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import tritonclient.grpc as grpcclient from tritonclient.utils import InferenceServerException if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit(1) model_name = "simple" # There are eight models in the repository directory if len(triton_client.get_model_repository_index().models) != 8: print("FAILED : Repository Index") sys.exit(1) triton_client.load_model(model_name) if not triton_client.is_model_ready(model_name): print("FAILED : Load Model") sys.exit(1) # Request to load the model with override config in original name # Send the config with wrong format try: config = '"parameters": {"config": {{"max_batch_size": "16"}}}' triton_client.load_model(model_name, config=config) except InferenceServerException as e: if "failed to load" not in e.message(): sys.exit(1) else: print("Expect error occurs for invalid override config.") sys.exit(1) # Send the config with the correct format config = '{"max_batch_size":"16"}' triton_client.load_model(model_name, config=config) # Check that the model with original name is changed. # The value of max_batch_size should be changed from "8" to "16". updated_model_config = triton_client.get_model_config(model_name) max_batch_size = updated_model_config.config.max_batch_size if max_batch_size != 16: print("Expect max_batch_size = 16, got: {}".format(max_batch_size)) sys.exit(1) triton_client.unload_model(model_name) if triton_client.is_model_ready(model_name): print("FAILED : Unload Model") sys.exit(1) # Trying to load wrong model name should emit exception try: triton_client.load_model("wrong_model_name") except InferenceServerException as e: if "failed to load" in e.message(): print("PASS: model control") sys.exit(0) print("FAILED") sys.exit(1) kytensor-2.49.0.7/client/src/python/examples/simple_grpc_sequence_stream_infer_client.py000077500000000000000000000232401527052371200317070ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import queue import sys import uuid from functools import partial import numpy as np import tritonclient.grpc as grpcclient from tritonclient.utils import InferenceServerException FLAGS = None class UserData: def __init__(self): self._completed_requests = queue.Queue() # Define the callback function. Note the last two parameters should be # result and error. InferenceServerClient would povide the results of an # inference as grpcclient.InferResult in result. For successful # inference, error will be None, otherwise it will be an object of # tritonclientutils.InferenceServerException holding the error details def callback(user_data, result, error): if error: user_data._completed_requests.put(error) else: user_data._completed_requests.put(result) def async_stream_send( triton_client, values, batch_size, sequence_id, model_name, model_version ): count = 1 for value in values: # Create the tensor for INPUT value_data = np.full(shape=[batch_size, 1], fill_value=value, dtype=np.int32) inputs = [] inputs.append(grpcclient.InferInput("INPUT", value_data.shape, "INT32")) # Initialize the data inputs[0].set_data_from_numpy(value_data) outputs = [] outputs.append(grpcclient.InferRequestedOutput("OUTPUT")) # Issue the asynchronous sequence inference. triton_client.async_stream_infer( model_name=model_name, inputs=inputs, outputs=outputs, request_id="{}_{}".format(sequence_id, count), sequence_id=sequence_id, sequence_start=(count == 1), sequence_end=(count == len(values)), ) count = count + 1 if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL and it gRPC port. Default is localhost:8001.", ) parser.add_argument( "-t", "--stream-timeout", type=float, required=False, default=None, help="Stream timeout in seconds. Default is None.", ) parser.add_argument( "-d", "--dyna", action="store_true", required=False, default=False, help="Assume dynamic sequence model", ) parser.add_argument( "-o", "--offset", type=int, required=False, default=0, help="Add offset to sequence ID used", ) FLAGS = parser.parse_args() # We use custom "sequence" models which take 1 input # value. The output is the accumulated value of the inputs. See # src/custom/sequence. int_sequence_model_name = ( "simple_dyna_sequence" if FLAGS.dyna else "simple_sequence" ) string_sequence_model_name = ( "simple_string_dyna_sequence" if FLAGS.dyna else "simple_sequence" ) model_version = "" batch_size = 1 values = [11, 7, 5, 3, 2, 0, 1] # Will use two sequences and send them asynchronously. Note the # sequence IDs should be non-zero because zero is reserved for # non-sequence requests. int_sequence_id0 = 1000 + FLAGS.offset * 2 int_sequence_id1 = 1001 + FLAGS.offset * 2 # For string sequence IDs, the dyna backend requires that the # sequence id be decodable into an integer, otherwise we'll use # a UUID4 sequence id and a model that doesn't require corrid # control. string_sequence_id0 = str(1002 + FLAGS.offset) if FLAGS.dyna else str(uuid.uuid4()) int_result0_list = [] int_result1_list = [] string_result0_list = [] user_data = UserData() # It is advisable to use client object within with..as clause # when sending streaming requests. This ensures the client # is closed when the block inside with exits. with grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) as triton_client: try: # Establish stream triton_client.start_stream( callback=partial(callback, user_data), stream_timeout=FLAGS.stream_timeout, ) # Now send the inference sequences... async_stream_send( triton_client, [0] + values, batch_size, int_sequence_id0, int_sequence_model_name, model_version, ) async_stream_send( triton_client, [100] + [-1 * val for val in values], batch_size, int_sequence_id1, int_sequence_model_name, model_version, ) async_stream_send( triton_client, [20] + [-1 * val for val in values], batch_size, string_sequence_id0, string_sequence_model_name, model_version, ) except InferenceServerException as error: print(error) sys.exit(1) # Retrieve results... recv_count = 0 while recv_count < (3 * (len(values) + 1)): data_item = user_data._completed_requests.get() if type(data_item) == InferenceServerException: print(data_item) sys.exit(1) else: try: this_id = data_item.get_response().id.split("_")[0] if int(this_id) == int_sequence_id0: int_result0_list.append(data_item.as_numpy("OUTPUT")) elif int(this_id) == int_sequence_id1: int_result1_list.append(data_item.as_numpy("OUTPUT")) elif this_id == string_sequence_id0: string_result0_list.append(data_item.as_numpy("OUTPUT")) else: print( "unexpected sequence id returned by the server: {}".format( this_id ) ) sys.exit(1) except ValueError: string_result0_list.append(data_item.as_numpy("OUTPUT")) recv_count = recv_count + 1 for i in range(len(int_result0_list)): int_seq0_expected = 1 if (i == 0) else values[i - 1] int_seq1_expected = 101 if (i == 0) else values[i - 1] * -1 # For string sequence ID we are testing two different backends if i == 0 and FLAGS.dyna: string_seq0_expected = 20 elif i == 0 and not FLAGS.dyna: string_seq0_expected = 21 elif i != 0 and FLAGS.dyna: string_seq0_expected = values[i - 1] * -1 + int( string_result0_list[i - 1][0][0] ) else: string_seq0_expected = values[i - 1] * -1 # The dyna_sequence custom backend adds the correlation ID # to the last request in a sequence. if FLAGS.dyna and (i != 0) and (values[i - 1] == 1): int_seq0_expected += int_sequence_id0 int_seq1_expected += int_sequence_id1 string_seq0_expected += int(string_sequence_id0) print( "[" + str(i) + "] " + str(int_result0_list[i][0][0]) + " : " + str(int_result1_list[i][0][0]) + " : " + str(string_result0_list[i][0][0]) ) if ( (int_seq0_expected != int_result0_list[i][0][0]) or (int_seq1_expected != int_result1_list[i][0][0]) or (string_seq0_expected != string_result0_list[i][0][0]) ): print( "[ expected ] " + str(int_seq0_expected) + " : " + str(int_seq1_expected) + " : " + str(string_seq0_expected) ) sys.exit(1) print("PASS: Sequence") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_sequence_sync_infer_client.py000077500000000000000000000166121527052371200313750ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import queue import sys import uuid import numpy as np import tritonclient.grpc as grpcclient from tritonclient.utils import InferenceServerException FLAGS = None class UserData: def __init__(self): self._completed_requests = queue.Queue() def sync_send( triton_client, result_list, values, batch_size, sequence_id, model_name, model_version, ): count = 1 for value in values: # Create the tensor for INPUT value_data = np.full(shape=[batch_size, 1], fill_value=value, dtype=np.int32) inputs = [] inputs.append(grpcclient.InferInput("INPUT", value_data.shape, "INT32")) # Initialize the data inputs[0].set_data_from_numpy(value_data) outputs = [] outputs.append(grpcclient.InferRequestedOutput("OUTPUT")) # Issue the synchronous sequence inference. result = triton_client.infer( model_name=model_name, inputs=inputs, outputs=outputs, sequence_id=sequence_id, sequence_start=(count == 1), sequence_end=(count == len(values)), ) result_list.append(result.as_numpy("OUTPUT")) count = count + 1 if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL and it gRPC port. Default is localhost:8001.", ) parser.add_argument( "-d", "--dyna", action="store_true", required=False, default=False, help="Assume dynamic sequence model", ) parser.add_argument( "-o", "--offset", type=int, required=False, default=0, help="Add offset to sequence ID used", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit() # We use custom "sequence" models which take 1 input # value. The output is the accumulated value of the inputs. See # src/custom/sequence. int_sequence_model_name = ( "simple_dyna_sequence" if FLAGS.dyna else "simple_sequence" ) string_sequence_model_name = ( "simple_string_dyna_sequence" if FLAGS.dyna else "simple_sequence" ) model_version = "" batch_size = 1 values = [11, 7, 5, 3, 2, 0, 1] # Will use two sequences and send them synchronously. Note the # sequence IDs should be non-zero because zero is reserved for # non-sequence requests. int_sequence_id0 = 1000 + FLAGS.offset * 2 int_sequence_id1 = 1001 + FLAGS.offset * 2 # For string sequence IDs, the dyna backend requires that the # sequence id be decodable into an integer, otherwise we'll use # a UUID4 sequence id and a model that doesn't require corrid # control. string_sequence_id0 = str(1002) if FLAGS.dyna else str(uuid.uuid4()) int_result0_list = [] int_result1_list = [] string_result0_list = [] user_data = UserData() try: sync_send( triton_client, int_result0_list, [0] + values, batch_size, int_sequence_id0, int_sequence_model_name, model_version, ) sync_send( triton_client, int_result1_list, [100] + [-1 * val for val in values], batch_size, int_sequence_id1, int_sequence_model_name, model_version, ) sync_send( triton_client, string_result0_list, [20] + [-1 * val for val in values], batch_size, string_sequence_id0, string_sequence_model_name, model_version, ) except InferenceServerException as error: print(error) sys.exit(1) for i in range(len(int_result0_list)): int_seq0_expected = 1 if (i == 0) else values[i - 1] int_seq1_expected = 101 if (i == 0) else values[i - 1] * -1 # For string sequence ID we are testing two different backends if i == 0 and FLAGS.dyna: string_seq0_expected = 20 elif i == 0 and not FLAGS.dyna: string_seq0_expected = 21 elif i != 0 and FLAGS.dyna: string_seq0_expected = values[i - 1] * -1 + int( string_result0_list[i - 1][0][0] ) else: string_seq0_expected = values[i - 1] * -1 # The dyna_sequence custom backend adds the correlation ID # to the last request in a sequence. if FLAGS.dyna and (i != 0) and (values[i - 1] == 1): int_seq0_expected += int_sequence_id0 int_seq1_expected += int_sequence_id1 string_seq0_expected += int(string_sequence_id0) print( "[" + str(i) + "] " + str(int_result0_list[i][0][0]) + " : " + str(int_result1_list[i][0][0]) + " : " + str(string_result0_list[i][0][0]) ) if ( (int_seq0_expected != int_result0_list[i][0][0]) or (int_seq1_expected != int_result1_list[i][0][0]) or (string_seq0_expected != string_result0_list[i][0][0]) ): print( "[ expected ] " + str(int_seq0_expected) + " : " + str(int_seq1_expected) + " : " + str(string_seq0_expected) ) sys.exit(1) print("PASS: Sequence") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_shm_client.py000077500000000000000000000152041527052371200261310ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys from builtins import range from ctypes import * import numpy as np import tritonclient.grpc as grpcclient import tritonclient.utils.shared_memory as shm from tritonclient import utils FLAGS = None if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) # To make sure no shared memory regions are registered with the # server. triton_client.unregister_system_shared_memory() triton_client.unregister_cuda_shared_memory() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple" model_version = "" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input1_data = np.ones(shape=16, dtype=np.int32) input_byte_size = input0_data.size * input0_data.itemsize output_byte_size = input_byte_size # Create shared memory region for output and store shared memory handle shm_op_handle = shm.create_shared_memory_region( "output_data", "/output_simple", output_byte_size * 2 ) # Register shared memory region for outputs with Triton Server triton_client.register_system_shared_memory( "output_data", "/output_simple", output_byte_size * 2 ) # Create shared memory region for input and store shared memory handle shm_ip_handle = shm.create_shared_memory_region( "input_data", "/input_simple", input_byte_size * 2 ) # Put input data values into shared memory shm.set_shared_memory_region(shm_ip_handle, [input0_data]) shm.set_shared_memory_region(shm_ip_handle, [input1_data], offset=input_byte_size) # Register shared memory region for inputs with Triton Server triton_client.register_system_shared_memory( "input_data", "/input_simple", input_byte_size * 2 ) # Set the parameters to use data from shared memory inputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs[-1].set_shared_memory("input_data", input_byte_size) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) inputs[-1].set_shared_memory("input_data", input_byte_size, offset=input_byte_size) outputs = [] outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs[-1].set_shared_memory("output_data", output_byte_size) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) outputs[-1].set_shared_memory( "output_data", output_byte_size, offset=output_byte_size ) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Read results from the shared memory. output0 = results.get_output("OUTPUT0") if output0 is not None: output0_data = shm.get_contents_as_numpy( shm_op_handle, utils.triton_to_np_dtype(output0.datatype), output0.shape ) else: print("OUTPUT0 is missing in the response.") sys.exit(1) output1 = results.get_output("OUTPUT1") if output1 is not None: output1_data = shm.get_contents_as_numpy( shm_op_handle, utils.triton_to_np_dtype(output1.datatype), output1.shape, offset=output_byte_size, ) else: print("OUTPUT1 is missing in the response.") sys.exit(1) for i in range(16): print( str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(output1_data[0][i]) ) if (input0_data[i] + input1_data[i]) != output0_data[0][i]: print("shm infer error: incorrect sum") sys.exit(1) if (input0_data[i] - input1_data[i]) != output1_data[0][i]: print("shm infer error: incorrect difference") sys.exit(1) print(triton_client.get_system_shared_memory_status()) triton_client.unregister_system_shared_memory() assert len(shm.mapped_shared_memory_regions()) == 2 shm.destroy_shared_memory_region(shm_ip_handle) shm.destroy_shared_memory_region(shm_op_handle) assert len(shm.mapped_shared_memory_regions()) == 0 print("PASS: system shared memory") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_shm_string_client.py000077500000000000000000000177741527052371200275350ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys from builtins import range import numpy as np import tritonclient.grpc as grpcclient import tritonclient.utils.shared_memory as shm from tritonclient import utils FLAGS = None if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) # To make sure no shared memory regions are registered with the # server. triton_client.unregister_system_shared_memory() triton_client.unregister_cuda_shared_memory() # We use a simple model that takes 2 input tensors of 16 strings # each and returns 2 output tensors of 16 strings each. The input # strings must represent integers. One output tensor is the # element-wise sum of the inputs and one output is the element-wise # difference. model_name = "simple_string" model_version = "" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. in0 = np.arange(start=0, stop=16, dtype=np.int32) in0n = np.array([str(x).encode("utf-8") for x in in0.flatten()], dtype=object) input0_data = in0n.reshape(in0.shape) in1 = np.ones(shape=16, dtype=np.int32) in1n = np.array([str(x).encode("utf-8") for x in in1.flatten()], dtype=object) input1_data = in1n.reshape(in1.shape) expected_sum = np.array( [str(x).encode("utf-8") for x in np.add(in0, in1).flatten()], dtype=object ) expected_diff = np.array( [str(x).encode("utf-8") for x in np.subtract(in0, in1).flatten()], dtype=object ) expected_sum_serialized = utils.serialize_byte_tensor(expected_sum) expected_diff_serialized = utils.serialize_byte_tensor(expected_diff) input0_data_serialized = utils.serialize_byte_tensor(input0_data) input1_data_serialized = utils.serialize_byte_tensor(input1_data) input0_byte_size = utils.serialized_byte_size(input0_data_serialized) input1_byte_size = utils.serialized_byte_size(input1_data_serialized) output0_byte_size = utils.serialized_byte_size(expected_sum_serialized) output1_byte_size = utils.serialized_byte_size(expected_diff_serialized) output_byte_size = max(input0_byte_size, input1_byte_size) + 1 # Create Output0 and Output1 in Shared Memory and store shared memory handles shm_op0_handle = shm.create_shared_memory_region( "output0_data", "/output0_simple", output0_byte_size ) shm_op1_handle = shm.create_shared_memory_region( "output1_data", "/output1_simple", output1_byte_size ) # Register Output0 and Output1 shared memory with Triton Server triton_client.register_system_shared_memory( "output0_data", "/output0_simple", output0_byte_size ) triton_client.register_system_shared_memory( "output1_data", "/output1_simple", output1_byte_size ) # Create Input0 and Input1 in Shared Memory and store shared memory handles shm_ip0_handle = shm.create_shared_memory_region( "input0_data", "/input0_simple", input0_byte_size ) shm_ip1_handle = shm.create_shared_memory_region( "input1_data", "/input1_simple", input1_byte_size ) # Put input data values into shared memory shm.set_shared_memory_region(shm_ip0_handle, [input0_data_serialized]) shm.set_shared_memory_region(shm_ip1_handle, [input1_data_serialized]) # Register Input0 and Input1 shared memory with Triton Server triton_client.register_system_shared_memory( "input0_data", "/input0_simple", input0_byte_size ) triton_client.register_system_shared_memory( "input1_data", "/input1_simple", input1_byte_size ) # Set the parameters to use data from shared memory inputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "BYTES")) inputs[-1].set_shared_memory("input0_data", input0_byte_size) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "BYTES")) inputs[-1].set_shared_memory("input1_data", input1_byte_size) outputs = [] outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs[-1].set_shared_memory("output0_data", output0_byte_size) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) outputs[-1].set_shared_memory("output1_data", output1_byte_size) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Read results from the shared memory. output0 = results.get_output("OUTPUT0") print(utils.triton_to_np_dtype(output0.datatype)) if output0 is not None: output0_data = shm.get_contents_as_numpy( shm_op0_handle, utils.triton_to_np_dtype(output0.datatype), output0.shape ) else: print("OUTPUT0 is missing in the response.") sys.exit(1) output1 = results.get_output("OUTPUT1") if output1 is not None: output1_data = shm.get_contents_as_numpy( shm_op1_handle, utils.triton_to_np_dtype(output1.datatype), output1.shape ) else: print("OUTPUT1 is missing in the response.") sys.exit(1) for i in range(16): r0 = output0_data[0][i] r1 = output1_data[0][i] print(str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(r0)) print(str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(r1)) if expected_sum[i] != r0: print("shm infer error: incorrect sum") sys.exit(1) if expected_diff[i] != r1: print("shm infer error: incorrect difference") sys.exit(1) print(triton_client.get_system_shared_memory_status()) triton_client.unregister_system_shared_memory() assert len(shm.mapped_shared_memory_regions()) == 4 shm.destroy_shared_memory_region(shm_ip0_handle) shm.destroy_shared_memory_region(shm_ip1_handle) shm.destroy_shared_memory_region(shm_op0_handle) shm.destroy_shared_memory_region(shm_op1_handle) assert len(shm.mapped_shared_memory_regions()) == 0 print("PASS: system shared memory") kytensor-2.49.0.7/client/src/python/examples/simple_grpc_string_infer_client.py000077500000000000000000000147441527052371200300430ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonclient.grpc as grpcclient # unicode() doesn't exist on python3, for how we use it the # corresponding function is bytes() if sys.version_info.major == 3: unicode = bytes def simple_string_inference(triton_client): model_name = "simple_string" inputs = [] outputs = [] inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "BYTES")) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "BYTES")) # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. in0 = np.arange(start=0, stop=16, dtype=np.int32) in0 = np.expand_dims(in0, axis=0) in1 = np.ones(shape=(1, 16), dtype=np.int32) expected_sum = np.add(in0, in1) expected_diff = np.subtract(in0, in1) # The 'simple_string' model expects 2 BYTES tensors where each # element in those tensors is the utf-8 string representation of # an integer. The BYTES tensors must be represented by a numpy # array with dtype=np.object_. in0n = np.array( [str(x).encode("utf-8") for x in in0.reshape(in0.size)], dtype=np.object_ ) input0_data = in0n.reshape(in0.shape) in1n = np.array( [str(x).encode("utf-8") for x in in1.reshape(in1.size)], dtype=np.object_ ) input1_data = in1n.reshape(in1.shape) # Initialize the data inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Get the output arrays from the results output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) # Convert result from string to int to check result r0 = int(output0_data[0][i]) r1 = int(output1_data[0][i]) if expected_sum[0][i] != r0: print("error: incorrect sum") sys.exit(1) if expected_diff[0][i] != r1: print("error: incorrect difference") sys.exit(1) def identity_inference(triton_client, np_array): model_name = "simple_identity" inputs = [] outputs = [] inputs.append(grpcclient.InferInput("INPUT0", np_array.shape, "BYTES")) inputs[0].set_data_from_numpy(np_array) outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) if np_array.dtype == np.object_: print(results.as_numpy("OUTPUT0")) if not np.array_equal(np_array, results.as_numpy("OUTPUT0")): print(results.as_numpy("OUTPUT0")) print("error: incorrect output") sys.exit(1) else: encoded_results = np.char.encode(results.as_numpy("OUTPUT0").astype(str)) if not np.array_equal(np_array, encoded_results): print(encoded_results) print("error: incorrect output") sys.exit(1) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) FLAGS = parser.parse_args() try: triton_client = grpcclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit() # Example using BYTES input tensors with utf-8 encoded string # elements simple_string_inference(triton_client) # Example using BYTES input tensor with utf-8 encoded string that # has an embedded null character. null_chars_array = np.array( ["he\x00llo".encode("utf-8") for i in range(16)], dtype=np.object_ ) null_char_data = null_chars_array.reshape([1, 16]) identity_inference(triton_client, null_char_data) # Example using BYTES input tensor with 16 elements, where each # element is a 4-byte binary blob with value 0x00010203. Can use # dtype=np.bytes_ in this case. bytes_data = [b"\x00\x01\x02\x03" for i in range(16)] np_bytes_data = np.array(bytes_data, dtype=np.bytes_) np_bytes_data = np_bytes_data.reshape([1, 16]) identity_inference(triton_client, np_bytes_data) print("PASS: string") kytensor-2.49.0.7/client/src/python/examples/simple_http_aio_infer_client.py000077500000000000000000000254561527052371200273330ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import asyncio import ssl import sys import numpy as np import tritonclient.http.aio as httpclient from tritonclient.utils import InferenceServerException async def test_infer( triton_client, model_name, input0_data, input1_data, headers=None, request_compression_algorithm=None, response_compression_algorithm=None, ): inputs = [] outputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) # Initialize the data inputs[0].set_data_from_numpy(input0_data, binary_data=False) inputs[1].set_data_from_numpy(input1_data, binary_data=True) outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=False)) query_params = {"test_1": 1, "test_2": 2} results = await triton_client.infer( model_name, inputs, outputs=outputs, query_params=query_params, headers=headers, request_compression_algorithm=request_compression_algorithm, response_compression_algorithm=response_compression_algorithm, ) return results async def test_infer_no_outputs( triton_client, model_name, input0_data, input1_data, headers=None, request_compression_algorithm=None, response_compression_algorithm=None, ): inputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) # Initialize the data inputs[0].set_data_from_numpy(input0_data, binary_data=False) inputs[1].set_data_from_numpy(input1_data, binary_data=True) query_params = {"test_1": 1, "test_2": 2} results = await triton_client.infer( model_name, inputs, outputs=None, query_params=query_params, headers=headers, request_compression_algorithm=request_compression_algorithm, response_compression_algorithm=response_compression_algorithm, ) return results def validate_result(results, input0_data, input1_data): output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) async def main(FLAGS): # Initialize triton_client try: if FLAGS.ssl: ssl_context = None # default SSL check if FLAGS.insecure: ssl_context = False # Skip SSL certification validation elif ( FLAGS.key_file is not None and FLAGS.cert_file is not None ) or FLAGS.ca_certs is not None: # Custom certification validation ssl_context = ssl.create_default_context() if FLAGS.key_file is not None and FLAGS.cert_file is not None: ssl_context.load_cert_chain( certfile=FLAGS.cert_file, keyfile=FLAGS.key_file ) if FLAGS.ca_certs is not None: ssl_context.load_verify_locations(cafile=FLAGS.ca_certs) triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose, ssl=True, ssl_context=ssl_context ) else: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) model_name = "simple" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) input1_data = np.full(shape=(1, 16), fill_value=-1, dtype=np.int32) if FLAGS.http_headers is not None: headers_dict = {l.split(":")[0]: l.split(":")[1] for l in FLAGS.http_headers} else: headers_dict = None # Infer with requested Outputs results = await test_infer( triton_client, model_name, input0_data, input1_data, headers_dict, FLAGS.request_compression_algorithm, FLAGS.response_compression_algorithm, ) print(results.get_response()) statistics = await triton_client.get_inference_statistics( model_name=model_name, headers=headers_dict ) print(statistics) if len(statistics["model_stats"]) != 1: print("FAILED: Inference Statistics") sys.exit(1) # Validate the results by comparing with precomputed values. validate_result(results, input0_data, input1_data) # Infer without requested Outputs results = await test_infer_no_outputs( triton_client, model_name, input0_data, input1_data, headers_dict, FLAGS.request_compression_algorithm, FLAGS.response_compression_algorithm, ) print(results.get_response()) # Validate the results by comparing with precomputed values. validate_result(results, input0_data, input1_data) # Infer with incorrect model name try: ( await test_infer( triton_client, "wrong_model_name", input0_data, input1_data ) ).get_response() print("expected error message for wrong model name") sys.exit(1) except InferenceServerException as ex: print(ex) if not (ex.message().startswith("Request for unknown model")): print("improper error message for wrong model name") sys.exit(1) # Infer in parallel result_1, result_2 = await asyncio.gather( test_infer( triton_client, model_name, input0_data, input1_data, headers_dict, FLAGS.request_compression_algorithm, FLAGS.response_compression_algorithm, ), test_infer_no_outputs( triton_client, model_name, input0_data, input1_data, headers_dict, FLAGS.request_compression_algorithm, FLAGS.response_compression_algorithm, ), ) # Validate the results by comparing with precomputed values. print(result_1.get_response()) validate_result(result_1, input0_data, input1_data) print(result_2.get_response()) validate_result(result_2, input0_data, input1_data) await triton_client.close() print("PASS: infer") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) parser.add_argument( "-s", "--ssl", action="store_true", required=False, default=False, help="Enable encrypted link to the server using HTTPS", ) parser.add_argument( "--key-file", type=str, required=False, default=None, help="File holding client private key. Default is None.", ) parser.add_argument( "--cert-file", type=str, required=False, default=None, help="File holding client certificate. Default is None.", ) parser.add_argument( "--ca-certs", type=str, required=False, default=None, help="File holding ca certificate. Default is None.", ) parser.add_argument( "--insecure", action="store_true", required=False, default=False, help="Use no peer verification in SSL communications. Use with caution. Default is False.", ) parser.add_argument( "-H", dest="http_headers", metavar="HTTP_HEADER", required=False, action="append", help="HTTP headers to add to inference server requests. " + 'Format is -H"Header:Value".', ) parser.add_argument( "--request-compression-algorithm", type=str, required=False, default=None, help="The compression algorithm to be used when sending request body to server. Default is None.", ) parser.add_argument( "--response-compression-algorithm", type=str, required=False, default=None, help="The compression algorithm to be used when receiving response body from server. Default is None.", ) FLAGS = parser.parse_args() asyncio.run(main(FLAGS)) kytensor-2.49.0.7/client/src/python/examples/simple_http_async_infer_client.py000077500000000000000000000113331527052371200276650ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonclient.http as httpclient if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() request_count = 2 try: # Need to specify large enough concurrency to issue all the # inference requests to the server in parallel. triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose, concurrency=request_count ) except Exception as e: print("context creation failed: " + str(e)) sys.exit() model_name = "simple" # Infer inputs = [] outputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) input1_data = np.ones(shape=(1, 16), dtype=np.int32) # Initialize the data inputs[0].set_data_from_numpy(input0_data, binary_data=True) inputs[1].set_data_from_numpy(input1_data, binary_data=True) outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=True)) async_requests = [] for i in range(request_count): # Asynchronous inference call. async_requests.append( triton_client.async_infer( model_name=model_name, inputs=inputs, outputs=outputs ) ) for async_request in async_requests: # Get the result from the initiated asynchronous inference request. # Note the call will block till the server responds. result = async_request.get_result() print(result.get_response()) # Validate the results by comparing with precomputed values. output0_data = result.as_numpy("OUTPUT0") output1_data = result.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("async infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("async infer error: incorrect difference") sys.exit(1) kytensor-2.49.0.7/client/src/python/examples/simple_http_cudashm_client.py000077500000000000000000000163351527052371200270200ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys from builtins import range import numpy as np import tritonclient.http as httpclient import tritonclient.utils.cuda_shared_memory as cudashm from tritonclient import utils FLAGS = None if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() try: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) # To make sure no shared memory regions are registered with the # server. triton_client.unregister_system_shared_memory() triton_client.unregister_cuda_shared_memory() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple" model_version = "" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input1_data = np.ones(shape=16, dtype=np.int32) input_byte_size = input0_data.size * input0_data.itemsize output_byte_size = input_byte_size # Create Output0 and Output1 in Shared Memory and store shared memory handles shm_op0_handle = cudashm.create_shared_memory_region( "output0_data", output_byte_size, 0 ) shm_op1_handle = cudashm.create_shared_memory_region( "output1_data", output_byte_size, 0 ) # Register Output0 and Output1 shared memory with Triton Server triton_client.register_cuda_shared_memory( "output0_data", cudashm.get_raw_handle(shm_op0_handle), 0, output_byte_size ) triton_client.register_cuda_shared_memory( "output1_data", cudashm.get_raw_handle(shm_op1_handle), 0, output_byte_size ) # Create Input0 and Input1 in Shared Memory and store shared memory handles shm_ip0_handle = cudashm.create_shared_memory_region( "input0_data", input_byte_size, 0 ) shm_ip1_handle = cudashm.create_shared_memory_region( "input1_data", input_byte_size, 0 ) # Put input data values into shared memory cudashm.set_shared_memory_region(shm_ip0_handle, [input0_data]) cudashm.set_shared_memory_region(shm_ip1_handle, [input1_data]) # Register Input0 and Input1 shared memory with Triton Server triton_client.register_cuda_shared_memory( "input0_data", cudashm.get_raw_handle(shm_ip0_handle), 0, input_byte_size ) triton_client.register_cuda_shared_memory( "input1_data", cudashm.get_raw_handle(shm_ip1_handle), 0, input_byte_size ) # Set the parameters to use data from shared memory inputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs[-1].set_shared_memory("input0_data", input_byte_size) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) inputs[-1].set_shared_memory("input1_data", input_byte_size) outputs = [] outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) outputs[-1].set_shared_memory("output0_data", output_byte_size) outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=True)) outputs[-1].set_shared_memory("output1_data", output_byte_size) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Read results from the shared memory. output0 = results.get_output("OUTPUT0") if output0 is not None: output0_data = cudashm.get_contents_as_numpy( shm_op0_handle, utils.triton_to_np_dtype(output0["datatype"]), output0["shape"], ) else: print("OUTPUT0 is missing in the response.") sys.exit(1) output1 = results.get_output("OUTPUT1") if output1 is not None: output1_data = cudashm.get_contents_as_numpy( shm_op1_handle, utils.triton_to_np_dtype(output1["datatype"]), output1["shape"], ) else: print("OUTPUT1 is missing in the response.") sys.exit(1) for i in range(16): print( str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(output1_data[0][i]) ) if (input0_data[i] + input1_data[i]) != output0_data[0][i]: print("cudashm infer error: incorrect sum") sys.exit(1) if (input0_data[i] - input1_data[i]) != output1_data[0][i]: print("cudashm infer error: incorrect difference") sys.exit(1) print(triton_client.get_cuda_shared_memory_status()) triton_client.unregister_cuda_shared_memory() assert len(cudashm.allocated_shared_memory_regions()) == 4 cudashm.destroy_shared_memory_region(shm_ip0_handle) cudashm.destroy_shared_memory_region(shm_ip1_handle) cudashm.destroy_shared_memory_region(shm_op0_handle) cudashm.destroy_shared_memory_region(shm_op1_handle) assert len(cudashm.allocated_shared_memory_regions()) == 0 print("PASS: cuda shared memory") kytensor-2.49.0.7/client/src/python/examples/simple_http_health_metadata.py000077500000000000000000000070641527052371200271420ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import tritonclient.http as httpclient from tritonclient.utils import InferenceServerException if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() try: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit() model_name = "simple" # Health if not triton_client.is_server_live(query_params={"test_1": 1, "test_2": 2}): print("FAILED : is_server_live") sys.exit(1) if not triton_client.is_server_ready(): print("FAILED : is_server_ready") sys.exit(1) if not triton_client.is_model_ready(model_name): print("FAILED : is_model_ready") sys.exit(1) # Metadata metadata = triton_client.get_server_metadata() if not (metadata["name"] == "triton"): print("FAILED : get_server_metadata") sys.exit(1) print(metadata) metadata = triton_client.get_model_metadata( model_name, query_params={"test_1": 1, "test_2": 2} ) if not (metadata["name"] == model_name): print("FAILED : get_model_metadata") sys.exit(1) print(metadata) # Passing incorrect model name try: metadata = triton_client.get_model_metadata("wrong_model_name") except InferenceServerException as ex: if "Request for unknown model" not in ex.message(): print("FAILED : get_model_metadata wrong_model_name") sys.exit(1) else: print("FAILED : get_model_metadata wrong_model_name") sys.exit(1) kytensor-2.49.0.7/client/src/python/examples/simple_http_infer_client.py000077500000000000000000000241411527052371200264710ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import gevent.ssl import numpy as np import tritonclient.http as httpclient from tritonclient.utils import InferenceServerException def test_infer( model_name, input0_data, input1_data, headers=None, request_compression_algorithm=None, response_compression_algorithm=None, ): inputs = [] outputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) # Initialize the data inputs[0].set_data_from_numpy(input0_data, binary_data=False) inputs[1].set_data_from_numpy(input1_data, binary_data=True) outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=False)) query_params = {"test_1": 1, "test_2": 2} results = triton_client.infer( model_name, inputs, outputs=outputs, query_params=query_params, headers=headers, request_compression_algorithm=request_compression_algorithm, response_compression_algorithm=response_compression_algorithm, ) return results def test_infer_no_outputs( model_name, input0_data, input1_data, headers=None, request_compression_algorithm=None, response_compression_algorithm=None, ): inputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) # Initialize the data inputs[0].set_data_from_numpy(input0_data, binary_data=False) inputs[1].set_data_from_numpy(input1_data, binary_data=True) query_params = {"test_1": 1, "test_2": 2} results = triton_client.infer( model_name, inputs, outputs=None, query_params=query_params, headers=headers, request_compression_algorithm=request_compression_algorithm, response_compression_algorithm=response_compression_algorithm, ) return results if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) parser.add_argument( "-s", "--ssl", action="store_true", required=False, default=False, help="Enable encrypted link to the server using HTTPS", ) parser.add_argument( "--key-file", type=str, required=False, default=None, help="File holding client private key. Default is None.", ) parser.add_argument( "--cert-file", type=str, required=False, default=None, help="File holding client certificate. Default is None.", ) parser.add_argument( "--ca-certs", type=str, required=False, default=None, help="File holding ca certificate. Default is None.", ) parser.add_argument( "--insecure", action="store_true", required=False, default=False, help="Use no peer verification in SSL communications. Use with caution. Default is False.", ) parser.add_argument( "-H", dest="http_headers", metavar="HTTP_HEADER", required=False, action="append", help="HTTP headers to add to inference server requests. " + 'Format is -H"Header:Value".', ) parser.add_argument( "--request-compression-algorithm", type=str, required=False, default=None, help="The compression algorithm to be used when sending request body to server. Default is None.", ) parser.add_argument( "--response-compression-algorithm", type=str, required=False, default=None, help="The compression algorithm to be used when receiving response body from server. Default is None.", ) FLAGS = parser.parse_args() try: if FLAGS.ssl: ssl_options = {} if FLAGS.key_file is not None: ssl_options["keyfile"] = FLAGS.key_file if FLAGS.cert_file is not None: ssl_options["certfile"] = FLAGS.cert_file if FLAGS.ca_certs is not None: ssl_options["ca_certs"] = FLAGS.ca_certs ssl_context_factory = None if FLAGS.insecure: ssl_context_factory = gevent.ssl._create_unverified_context triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose, ssl=True, ssl_options=ssl_options, insecure=FLAGS.insecure, ssl_context_factory=ssl_context_factory, ) else: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) model_name = "simple" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input0_data = np.expand_dims(input0_data, axis=0) input1_data = np.full(shape=(1, 16), fill_value=-1, dtype=np.int32) if FLAGS.http_headers is not None: headers_dict = {l.split(":")[0]: l.split(":")[1] for l in FLAGS.http_headers} else: headers_dict = None # Infer with requested Outputs results = test_infer( model_name, input0_data, input1_data, headers_dict, FLAGS.request_compression_algorithm, FLAGS.response_compression_algorithm, ) print(results.get_response()) statistics = triton_client.get_inference_statistics( model_name=model_name, headers=headers_dict ) print(statistics) if len(statistics["model_stats"]) != 1: print("FAILED: Inference Statistics") sys.exit(1) # Validate the results by comparing with precomputed values. output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) # Infer without requested Outputs results = test_infer_no_outputs( model_name, input0_data, input1_data, headers_dict, FLAGS.request_compression_algorithm, FLAGS.response_compression_algorithm, ) print(results.get_response()) # Validate the results by comparing with precomputed values. output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]: print("sync infer error: incorrect sum") sys.exit(1) if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]: print("sync infer error: incorrect difference") sys.exit(1) # Infer with incorrect model name try: _ = test_infer("wrong_model_name", input0_data, input1_data).get_response() print("expected error message for wrong model name") sys.exit(1) except InferenceServerException as ex: print(ex) if not (ex.message().startswith("Request for unknown model")): print("improper error message for wrong model name") sys.exit(1) kytensor-2.49.0.7/client/src/python/examples/simple_http_model_control.py000077500000000000000000000100071527052371200266640ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import tritonclient.http as httpclient from tritonclient.utils import InferenceServerException if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() try: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit(1) model_name = "simple" # There are seven models in the repository directory if len(triton_client.get_model_repository_index()) != 7: sys.exit(1) triton_client.load_model(model_name) if not triton_client.is_model_ready(model_name): sys.exit(1) # Request to load the model with override config in original name # Send the config with wrong format try: config = '"parameters": {"config": {{"max_batch_size": "16"}}}' triton_client.load_model(model_name, config=config) except InferenceServerException as e: if "failed to load" not in e.message(): sys.exit(1) else: print("Expect error occurs for invalid override config.") sys.exit(1) # Send the config with the correct format config = '{"max_batch_size":"16"}' triton_client.load_model(model_name, config=config) # Check that the model with original name is changed. # The value of max_batch_size should be changed from "8" to "16". updated_model_config = triton_client.get_model_config(model_name) if updated_model_config["max_batch_size"] != 16: print( "Expect max_batch_size = 16, got: {}".format( updated_model_config["max_batch_size"] ) ) sys.exit(1) triton_client.unload_model(model_name) if triton_client.is_model_ready(model_name): sys.exit(1) # Trying to load wrong model name should emit exception try: triton_client.load_model("wrong_model_name") except InferenceServerException as e: if "failed to load" in e.message(): print("PASS: model control") else: sys.exit(1) kytensor-2.49.0.7/client/src/python/examples/simple_http_sequence_sync_infer_client.py000077500000000000000000000166121527052371200314210ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import queue import sys import uuid import numpy as np import tritonclient.http as httpclient from tritonclient.utils import InferenceServerException FLAGS = None class UserData: def __init__(self): self._completed_requests = queue.Queue() def sync_send( triton_client, result_list, values, batch_size, sequence_id, model_name, model_version, ): count = 1 for value in values: # Create the tensor for INPUT value_data = np.full(shape=[batch_size, 1], fill_value=value, dtype=np.int32) inputs = [] inputs.append(httpclient.InferInput("INPUT", value_data.shape, "INT32")) # Initialize the data inputs[0].set_data_from_numpy(value_data) outputs = [] outputs.append(httpclient.InferRequestedOutput("OUTPUT")) # Issue the synchronous sequence inference. result = triton_client.infer( model_name=model_name, inputs=inputs, outputs=outputs, sequence_id=sequence_id, sequence_start=(count == 1), sequence_end=(count == len(values)), ) result_list.append(result.as_numpy("OUTPUT")) count = count + 1 if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL and it HTTP port. Default is localhost:8000.", ) parser.add_argument( "-d", "--dyna", action="store_true", required=False, default=False, help="Assume dynamic sequence model", ) parser.add_argument( "-o", "--offset", type=int, required=False, default=0, help="Add offset to sequence ID used", ) FLAGS = parser.parse_args() try: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit() # We use custom "sequence" models which take 1 input # value. The output is the accumulated value of the inputs. See # src/custom/sequence. int_sequence_model_name = ( "simple_dyna_sequence" if FLAGS.dyna else "simple_sequence" ) string_sequence_model_name = ( "simple_string_dyna_sequence" if FLAGS.dyna else "simple_sequence" ) model_version = "" batch_size = 1 values = [11, 7, 5, 3, 2, 0, 1] # Will use two sequences and send them synchronously. Note the # sequence IDs should be non-zero because zero is reserved for # non-sequence requests. int_sequence_id0 = 1000 + FLAGS.offset * 2 int_sequence_id1 = 1001 + FLAGS.offset * 2 # For string sequence IDs, the dyna backend requires that the # sequence id be decodable into an integer, otherwise we'll use # a UUID4 sequence id and a model that doesn't require corrid # control. string_sequence_id0 = str(1002) if FLAGS.dyna else str(uuid.uuid4()) int_result0_list = [] int_result1_list = [] string_result0_list = [] user_data = UserData() try: sync_send( triton_client, int_result0_list, [0] + values, batch_size, int_sequence_id0, int_sequence_model_name, model_version, ) sync_send( triton_client, int_result1_list, [100] + [-1 * val for val in values], batch_size, int_sequence_id1, int_sequence_model_name, model_version, ) sync_send( triton_client, string_result0_list, [20] + [-1 * val for val in values], batch_size, string_sequence_id0, string_sequence_model_name, model_version, ) except InferenceServerException as error: print(error) sys.exit(1) for i in range(len(int_result0_list)): int_seq0_expected = 1 if (i == 0) else values[i - 1] int_seq1_expected = 101 if (i == 0) else values[i - 1] * -1 # For string sequence ID we are testing two different backends if i == 0 and FLAGS.dyna: string_seq0_expected = 20 elif i == 0 and not FLAGS.dyna: string_seq0_expected = 21 elif i != 0 and FLAGS.dyna: string_seq0_expected = values[i - 1] * -1 + int( string_result0_list[i - 1][0][0] ) else: string_seq0_expected = values[i - 1] * -1 # The dyna_sequence custom backend adds the correlation ID # to the last request in a sequence. if FLAGS.dyna and (i != 0) and (values[i - 1] == 1): int_seq0_expected += int_sequence_id0 int_seq1_expected += int_sequence_id1 string_seq0_expected += int(string_sequence_id0) print( "[" + str(i) + "] " + str(int_result0_list[i][0][0]) + " : " + str(int_result1_list[i][0][0]) + " : " + str(string_result0_list[i][0][0]) ) if ( (int_seq0_expected != int_result0_list[i][0][0]) or (int_seq1_expected != int_result1_list[i][0][0]) or (string_seq0_expected != string_result0_list[i][0][0]) ): print( "[ expected ] " + str(int_seq0_expected) + " : " + str(int_seq1_expected) + " : " + str(string_seq0_expected) ) sys.exit(1) print("PASS: Sequence") kytensor-2.49.0.7/client/src/python/examples/simple_http_shm_client.py000077500000000000000000000152701527052371200261600ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys from builtins import range import numpy as np import tritonclient.http as httpclient import tritonclient.utils.shared_memory as shm from tritonclient import utils FLAGS = None if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() try: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) # To make sure no shared memory regions are registered with the # server. triton_client.unregister_system_shared_memory() triton_client.unregister_cuda_shared_memory() # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "simple" model_version = "" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. input0_data = np.arange(start=0, stop=16, dtype=np.int32) input1_data = np.ones(shape=16, dtype=np.int32) input_byte_size = input0_data.size * input0_data.itemsize output_byte_size = input_byte_size # Create shared memory region for output and store shared memory handle shm_op_handle = shm.create_shared_memory_region( "output_data", "/output_simple", output_byte_size * 2 ) # Register shared memory region for outputs with Triton Server triton_client.register_system_shared_memory( "output_data", "/output_simple", output_byte_size * 2 ) # Create shared memory region for input and store shared memory handle shm_ip_handle = shm.create_shared_memory_region( "input_data", "/input_simple", input_byte_size * 2 ) # Put input data values into shared memory shm.set_shared_memory_region(shm_ip_handle, [input0_data]) shm.set_shared_memory_region(shm_ip_handle, [input1_data], offset=input_byte_size) # Register shared memory region for inputs with Triton Server triton_client.register_system_shared_memory( "input_data", "/input_simple", input_byte_size * 2 ) # Set the parameters to use data from shared memory inputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs[-1].set_shared_memory("input_data", input_byte_size) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) inputs[-1].set_shared_memory("input_data", input_byte_size, offset=input_byte_size) outputs = [] outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) outputs[-1].set_shared_memory("output_data", output_byte_size) outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=True)) outputs[-1].set_shared_memory( "output_data", output_byte_size, offset=output_byte_size ) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Read results from the shared memory. output0 = results.get_output("OUTPUT0") if output0 is not None: output0_data = shm.get_contents_as_numpy( shm_op_handle, utils.triton_to_np_dtype(output0["datatype"]), output0["shape"], ) else: print("OUTPUT0 is missing in the response.") sys.exit(1) output1 = results.get_output("OUTPUT1") if output1 is not None: output1_data = shm.get_contents_as_numpy( shm_op_handle, utils.triton_to_np_dtype(output1["datatype"]), output1["shape"], offset=output_byte_size, ) else: print("OUTPUT1 is missing in the response.") sys.exit(1) for i in range(16): print( str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(output1_data[0][i]) ) if (input0_data[i] + input1_data[i]) != output0_data[0][i]: print("shm infer error: incorrect sum") sys.exit(1) if (input0_data[i] - input1_data[i]) != output1_data[0][i]: print("shm infer error: incorrect difference") sys.exit(1) print(triton_client.get_system_shared_memory_status()) triton_client.unregister_system_shared_memory() assert len(shm.mapped_shared_memory_regions()) == 2 shm.destroy_shared_memory_region(shm_ip_handle) shm.destroy_shared_memory_region(shm_op_handle) assert len(shm.mapped_shared_memory_regions()) == 0 print("PASS: system shared memory") kytensor-2.49.0.7/client/src/python/examples/simple_http_shm_string_client.py000077500000000000000000000200361527052371200275420ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys from builtins import range import numpy as np import tritonclient.http as httpclient import tritonclient.utils.shared_memory as shm from tritonclient import utils FLAGS = None if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() try: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) # To make sure no shared memory regions are registered with the # server. triton_client.unregister_system_shared_memory() triton_client.unregister_cuda_shared_memory() # We use a simple model that takes 2 input tensors of 16 strings # each and returns 2 output tensors of 16 strings each. The input # strings must represent integers. One output tensor is the # element-wise sum of the inputs and one output is the element-wise # difference. model_name = "simple_string" model_version = "" # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. in0 = np.arange(start=0, stop=16, dtype=np.int32) in0n = np.array([str(x).encode("utf-8") for x in in0.flatten()], dtype=object) input0_data = in0n.reshape(in0.shape) in1 = np.ones(shape=16, dtype=np.int32) in1n = np.array([str(x).encode("utf-8") for x in in1.flatten()], dtype=object) input1_data = in1n.reshape(in1.shape) expected_sum = np.array( [str(x).encode("utf-8") for x in np.add(in0, in1).flatten()], dtype=object ) expected_diff = np.array( [str(x).encode("utf-8") for x in np.subtract(in0, in1).flatten()], dtype=object ) expected_sum_serialized = utils.serialize_byte_tensor(expected_sum) expected_diff_serialized = utils.serialize_byte_tensor(expected_diff) input0_data_serialized = utils.serialize_byte_tensor(input0_data) input1_data_serialized = utils.serialize_byte_tensor(input1_data) input0_byte_size = utils.serialized_byte_size(input0_data_serialized) input1_byte_size = utils.serialized_byte_size(input1_data_serialized) output0_byte_size = utils.serialized_byte_size(expected_sum_serialized) output1_byte_size = utils.serialized_byte_size(expected_diff_serialized) # Create Output0 and Output1 in Shared Memory and store shared memory handles shm_op0_handle = shm.create_shared_memory_region( "output0_data", "/output0_simple", output0_byte_size ) shm_op1_handle = shm.create_shared_memory_region( "output1_data", "/output1_simple", output1_byte_size ) # Register Output0 and Output1 shared memory with Triton Server triton_client.register_system_shared_memory( "output0_data", "/output0_simple", output0_byte_size ) triton_client.register_system_shared_memory( "output1_data", "/output1_simple", output1_byte_size ) # Create Input0 and Input1 in Shared Memory and store shared memory handles shm_ip0_handle = shm.create_shared_memory_region( "input0_data", "/input0_simple", input0_byte_size ) shm_ip1_handle = shm.create_shared_memory_region( "input1_data", "/input1_simple", input1_byte_size ) # Put input data values into shared memory shm.set_shared_memory_region(shm_ip0_handle, [input0_data_serialized]) shm.set_shared_memory_region(shm_ip1_handle, [input1_data_serialized]) # Register Input0 and Input1 shared memory with Triton Server triton_client.register_system_shared_memory( "input0_data", "/input0_simple", input0_byte_size ) triton_client.register_system_shared_memory( "input1_data", "/input1_simple", input1_byte_size ) # Set the parameters to use data from shared memory inputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "BYTES")) inputs[-1].set_shared_memory("input0_data", input0_byte_size) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "BYTES")) inputs[-1].set_shared_memory("input1_data", input1_byte_size) outputs = [] outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) outputs[-1].set_shared_memory("output0_data", output0_byte_size) outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=True)) outputs[-1].set_shared_memory("output1_data", output1_byte_size) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Read results from the shared memory. output0 = results.get_output("OUTPUT0") print(utils.triton_to_np_dtype(output0["datatype"])) if output0 is not None: output0_data = shm.get_contents_as_numpy( shm_op0_handle, utils.triton_to_np_dtype(output0["datatype"]), output0["shape"], ) else: print("OUTPUT0 is missing in the response.") sys.exit(1) output1 = results.get_output("OUTPUT1") if output1 is not None: output1_data = shm.get_contents_as_numpy( shm_op1_handle, utils.triton_to_np_dtype(output1["datatype"]), output1["shape"], ) else: print("OUTPUT1 is missing in the response.") sys.exit(1) for i in range(16): r0 = output0_data[0][i] r1 = output1_data[0][i] print(str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(r0)) print(str(input0_data[i]) + " - " + str(input1_data[i]) + " = " + str(r1)) if expected_sum[i] != r0: print("shm infer error: incorrect sum") sys.exit(1) if expected_diff[i] != r1: print("shm infer error: incorrect difference") sys.exit(1) print(triton_client.get_system_shared_memory_status()) triton_client.unregister_system_shared_memory() assert len(shm.mapped_shared_memory_regions()) == 4 shm.destroy_shared_memory_region(shm_ip0_handle) shm.destroy_shared_memory_region(shm_ip1_handle) shm.destroy_shared_memory_region(shm_op0_handle) shm.destroy_shared_memory_region(shm_op1_handle) assert len(shm.mapped_shared_memory_regions()) == 0 print("PASS: system shared memory") kytensor-2.49.0.7/client/src/python/examples/simple_http_string_infer_client.py000077500000000000000000000165471527052371200300720ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import sys import numpy as np import tritonclient.http as httpclient # unicode() doesn't exist on python3, for how we use it the # corresponding function is bytes() if sys.version_info.major == 3: unicode = bytes def simple_string_inference(triton_client): model_name = "simple_string" inputs = [] outputs = [] inputs.append(httpclient.InferInput("INPUT0", [1, 16], "BYTES")) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "BYTES")) # Create the data for the two input tensors. Initialize the first # to unique integers and the second to all ones. in0 = np.arange(start=0, stop=16, dtype=np.int32) in0 = np.expand_dims(in0, axis=0) in1 = np.ones(shape=(1, 16), dtype=np.int32) expected_sum = np.add(in0, in1) expected_diff = np.subtract(in0, in1) # The 'simple_string' model expects 2 BYTES tensors where each # element in those tensors is the utf-8 string representation of # an integer. The BYTES tensors must be represented by a numpy # array with dtype=np.object_. in0n = np.array( [str(x).encode("utf-8") for x in in0.reshape(in0.size)], dtype=np.object_ ) input0_data = in0n.reshape(in0.shape) in1n = np.array( [str(x).encode("utf-8") for x in in1.reshape(in1.size)], dtype=np.object_ ) input1_data = in1n.reshape(in1.shape) # Initialize the data inputs[0].set_data_from_numpy(input0_data, binary_data=True) inputs[1].set_data_from_numpy(input1_data, binary_data=False) outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=False)) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Get the output arrays from the results output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") for i in range(16): print( str(input0_data[0][i]) + " + " + str(input1_data[0][i]) + " = " + str(output0_data[0][i]) ) print( str(input0_data[0][i]) + " - " + str(input1_data[0][i]) + " = " + str(output1_data[0][i]) ) # Convert result from string to int to check result r0 = int(output0_data[0][i]) r1 = int(output1_data[0][i]) if expected_sum[0][i] != r0: print("error: incorrect sum") sys.exit(1) if expected_diff[0][i] != r1: print("error: incorrect difference") sys.exit(1) def identity_inference(triton_client, np_array, binary_data): model_name = "simple_identity" inputs = [] outputs = [] inputs.append(httpclient.InferInput("INPUT0", np_array.shape, "BYTES")) inputs[0].set_data_from_numpy(np_array, binary_data=binary_data) outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=binary_data)) results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) if np_array.dtype == np.object_: print(results.as_numpy("OUTPUT0")) if binary_data: if not np.array_equal(np_array, results.as_numpy("OUTPUT0")): print(results.as_numpy("OUTPUT0")) print("error: incorrect output") sys.exit(1) else: expected_array = np.array( [ unicode(str(x), encoding="utf-8") for x in results.as_numpy("OUTPUT0").flatten() ], dtype=np.object_, ) expected_array = expected_array.reshape([1, 16]) if not np.array_equal(np_array, expected_array): print(results.as_numpy("OUTPUT0")) print("error: incorrect output") sys.exit(1) else: encoded_results = np.char.encode(results.as_numpy("OUTPUT0").astype(str)) if not np.array_equal(np_array, encoded_results): print(encoded_results) print("error: incorrect output") sys.exit(1) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "-u", "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) FLAGS = parser.parse_args() try: triton_client = httpclient.InferenceServerClient( url=FLAGS.url, verbose=FLAGS.verbose ) except Exception as e: print("context creation failed: " + str(e)) sys.exit() # Example using BYTES input tensors with utf-8 encoded string # elements simple_string_inference(triton_client) # Example using BYTES input tensor with utf-8 encoded string that # has an embedded null character. null_chars_array = np.array( ["he\x00llo".encode("utf-8") for i in range(16)], dtype=np.object_ ) null_char_data = null_chars_array.reshape([1, 16]) identity_inference(triton_client, null_char_data, True) # Using binary data identity_inference(triton_client, null_char_data, False) # Using JSON data # Example using BYTES input tensor with 16 elements, where each # element is a 4-byte binary blob with value 0x00010203. Can use # dtype=np.bytes_ in this case. bytes_data = [b"\x00\x01\x02\x03" for i in range(16)] np_bytes_data = np.array(bytes_data, dtype=np.bytes_) np_bytes_data = np_bytes_data.reshape([1, 16]) identity_inference(triton_client, np_bytes_data, True) # Using binary data identity_inference(triton_client, np_bytes_data, False) # Using JSON data print("PASS: string") kytensor-2.49.0.7/client/src/python/library/000077500000000000000000000000001527052371200206675ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/CMakeLists.txt000066400000000000000000000114361527052371200234340ustar00rootroot00000000000000# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) add_subdirectory(tritonclient) # Needed for the backwards-compatibility # Remove when moving completely to the # new structure. if(${TRITON_ENABLE_PYTHON_GRPC}) file(COPY tritongrpcclient DESTINATION .) endif() # TRITON_ENABLE_PYTHON_GRPC if(${TRITON_ENABLE_PYTHON_HTTP}) file(COPY tritonhttpclient DESTINATION .) endif() # TRITON_ENABLE_PYTHON_HTTP file(COPY tritonclientutils DESTINATION .) if (NOT WIN32) file(COPY tritonshmutils DESTINATION .) endif() # NOT WIN32 #################################### file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION ${TRITON_VERSION}) configure_file(LICENSE.txt LICENSE.txt COPYONLY) configure_file(setup.py setup.py @ONLY) file(COPY requirements DESTINATION .) set(WHEEL_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION ${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt ${CMAKE_CURRENT_BINARY_DIR}/setup.py ) if(${TRITON_ENABLE_PYTHON_GRPC}) set(WHEEL_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/tritonclient/grpc proto-py-library grpc-service-py-library ${WHEEL_DEPENDS} ) endif() # TRITON_ENABLE_PYTHON_GRPC if(${TRITON_ENABLE_PYTHON_HTTP}) set(WHEEL_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/tritonclient/http ${WHEEL_DEPENDS} ) endif() # TRITON_ENABLE_PYTHON_HTTP # # Generic Wheel file # set(wheel_stamp_file "stamp.whl") add_custom_command( OUTPUT "${wheel_stamp_file}" COMMAND python3 ARGS "${CMAKE_CURRENT_SOURCE_DIR}/build_wheel.py" --dest-dir "${CMAKE_CURRENT_BINARY_DIR}/generic" DEPENDS ${WHEEL_DEPENDS} ) add_custom_target( generic-client-wheel ALL DEPENDS "${wheel_stamp_file}" ) # # Linux specific Wheel file. Compatible with x86, x64 and aarch64 # if (NOT WIN32) # Can generate linux specific wheel file on linux systems only. set(LINUX_WHEEL_DEPENDS cshm ${WHEEL_DEPENDS} ) if (${TRITON_PACKAGE_PERF_ANALYZER}) set(perf_analyzer_arg --perf-analyzer ${CMAKE_INSTALL_PREFIX}/bin/perf_analyzer) endif() set(linux_wheel_stamp_file "linux_stamp.whl") add_custom_command( OUTPUT "${linux_wheel_stamp_file}" COMMAND python3 ARGS "${CMAKE_CURRENT_SOURCE_DIR}/build_wheel.py" --dest-dir "${CMAKE_CURRENT_BINARY_DIR}/linux" --linux ${perf_analyzer_arg} DEPENDS ${LINUX_WHEEL_DEPENDS} ) add_custom_target( linux-client-wheel ALL DEPENDS "${linux_wheel_stamp_file}" ) endif() # NOT WIN32 if(${TRITON_ENABLE_PYTHON_GRPC}) add_dependencies( generic-client-wheel grpc-service-py-library proto-py-library ) if (NOT WIN32) add_dependencies( linux-client-wheel grpc-service-py-library proto-py-library ) endif() # NOT WIN32 file( GLOB generated-py ${CMAKE_CURRENT_BINARY_DIR}/../../core/*_pb2.py ${CMAKE_CURRENT_BINARY_DIR}/../../core/*_grpc.py ) install( FILES ${generated-py} DESTINATION python ) endif() # TRITON_ENABLE_PYTHON_GRPC install( CODE "file(GLOB _Wheel \"${CMAKE_CURRENT_BINARY_DIR}/generic/triton*.whl\")" CODE "file(INSTALL \${_Wheel} DESTINATION \"${CMAKE_INSTALL_PREFIX}/python\")" ) if (NOT WIN32) install( CODE "file(GLOB _Wheel \"${CMAKE_CURRENT_BINARY_DIR}/linux/triton*.whl\")" CODE "file(INSTALL \${_Wheel} DESTINATION \"${CMAKE_INSTALL_PREFIX}/python\")" ) endif() # NOT WIN32 kytensor-2.49.0.7/client/src/python/library/LICENSE.txt000066400000000000000000000027751527052371200225250ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/client/src/python/library/build_wheel.py000077500000000000000000000203171527052371200235320ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os import pathlib import re import shutil import subprocess import sys from tempfile import mkstemp def fail_if(p, msg): if p: print("error: {}".format(msg), file=sys.stderr) sys.exit(1) def mkdir(path): pathlib.Path(path).mkdir(parents=True, exist_ok=True) def touch(path): pathlib.Path(path).touch() def cpdir(src, dest): shutil.copytree(src, dest, symlinks=True, dirs_exist_ok=True) def sed(pattern, replace, source, dest=None): fin = open(source, "r") if dest: fout = open(dest, "w") else: fd, name = mkstemp() fout = open(name, "w") for line in fin: out = re.sub(pattern, replace, line) fout.write(out) fin.close() fout.close() if not dest: shutil.copyfile(name, source) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--dest-dir", type=str, required=True, help="Destination directory." ) parser.add_argument( "--linux", action="store_true", required=False, help="Include linux specific artifacts.", ) parser.add_argument( "--perf-analyzer", type=str, required=False, default=None, help="perf-analyzer path.", ) FLAGS = parser.parse_args() FLAGS.triton_version = None with open("TRITON_VERSION", "r") as vfile: FLAGS.triton_version = vfile.readline().strip() FLAGS.whl_dir = os.path.join(FLAGS.dest_dir, "wheel") print("=== Building in: {}".format(os.getcwd())) print("=== Using builddir: {}".format(FLAGS.whl_dir)) print("Adding package files") mkdir(os.path.join(FLAGS.whl_dir, "tritonclient")) shutil.copy("tritonclient/__init__.py", os.path.join(FLAGS.whl_dir, "tritonclient")) shutil.copy("tritonclient/_client.py", os.path.join(FLAGS.whl_dir, "tritonclient")) shutil.copy("tritonclient/_plugin.py", os.path.join(FLAGS.whl_dir, "tritonclient")) shutil.copy("tritonclient/_request.py", os.path.join(FLAGS.whl_dir, "tritonclient")) shutil.copy("tritonclient/_auth.py", os.path.join(FLAGS.whl_dir, "tritonclient")) # Needed for backwards-compatibility; remove when moving # completely to the new structure. if os.path.isdir("tritonclientutils"): cpdir("tritonclientutils", os.path.join(FLAGS.whl_dir, "tritonclientutils")) if os.path.isdir("tritonhttpclient"): cpdir("tritonhttpclient", os.path.join(FLAGS.whl_dir, "tritonhttpclient")) if os.path.isdir("tritongrpcclient"): cpdir("tritongrpcclient", os.path.join(FLAGS.whl_dir, "tritongrpcclient")) if FLAGS.linux: if os.path.isdir("tritonshmutils"): cpdir("tritonshmutils", os.path.join(FLAGS.whl_dir, "tritonshmutils")) if os.path.isdir("tritonclient/grpc"): cpdir("tritonclient/grpc", os.path.join(FLAGS.whl_dir, "tritonclient/grpc")) shutil.copyfile( "../_deps/repo-common-build/protobuf/model_config_pb2.py", os.path.join(FLAGS.whl_dir, "tritonclient/grpc/model_config_pb2.py"), ) shutil.copyfile( "../_deps/repo-common-build/protobuf/grpc_service_pb2.py", os.path.join(FLAGS.whl_dir, "tritonclient/grpc/service_pb2.py"), ) shutil.copyfile( "../_deps/repo-common-build/protobuf/grpc_service_pb2_grpc.py", os.path.join(FLAGS.whl_dir, "tritonclient/grpc/service_pb2_grpc.py"), ) # Use 'sed' command to fix protoc compiled imports (see # https://github.com/google/protobuf/issues/1491). for fl in ("model_config_pb2.py", "service_pb2.py"): sed( "^import ([^ ]*)_pb2 as ([^ ]*)$", "from tritonclient.grpc import \\1_pb2 as \\2", os.path.join(FLAGS.whl_dir, "tritonclient", "grpc", fl), ) sed( "^import grpc_([^ ]*)_pb2 as ([^ ]*)$", "from tritonclient.grpc import \\1_pb2 as \\2", os.path.join(FLAGS.whl_dir, "tritonclient/grpc/service_pb2_grpc.py"), ) if os.path.isdir("tritonclient/http"): cpdir("tritonclient/http", os.path.join(FLAGS.whl_dir, "tritonclient/http")) mkdir(os.path.join(FLAGS.whl_dir, "tritonclient/utils")) shutil.copyfile( "tritonclient/utils/__init__.py", os.path.join(FLAGS.whl_dir, "tritonclient/utils/__init__.py"), ) shutil.copyfile( "tritonclient/utils/_dlpack.py", os.path.join(FLAGS.whl_dir, "tritonclient/utils/_dlpack.py"), ) shutil.copyfile( "tritonclient/utils/_shared_memory_tensor.py", os.path.join(FLAGS.whl_dir, "tritonclient/utils/_shared_memory_tensor.py"), ) if FLAGS.linux: cpdir( "tritonclient/utils/shared_memory", os.path.join(FLAGS.whl_dir, "tritonclient/utils/shared_memory"), ) shutil.copyfile( "tritonclient/utils/libcshm.so", os.path.join(FLAGS.whl_dir, "tritonclient/utils/shared_memory/libcshm.so"), ) cpdir( "tritonclient/utils/cuda_shared_memory", os.path.join(FLAGS.whl_dir, "tritonclient/utils/cuda_shared_memory"), ) # Copy the pre-compiled perf_analyzer binary if FLAGS.perf_analyzer is not None: # The permission bits need to be copied to along with the executable shutil.copy( FLAGS.perf_analyzer, os.path.join(FLAGS.whl_dir, "perf_analyzer") ) # Create a symbolic link for backwards compatibility if not os.path.exists(os.path.join(FLAGS.whl_dir, "perf_client")): os.symlink("perf_analyzer", os.path.join(FLAGS.whl_dir, "perf_client")) shutil.copyfile("LICENSE.txt", os.path.join(FLAGS.whl_dir, "LICENSE.txt")) shutil.copyfile("setup.py", os.path.join(FLAGS.whl_dir, "setup.py")) cpdir("requirements", os.path.join(FLAGS.whl_dir, "requirements")) os.chdir(FLAGS.whl_dir) print("=== Building wheel") if FLAGS.linux: if os.uname().machine == "aarch64": platform_name = "manylinux2014_aarch64" elif os.uname().machine == "ppc64le": platform_name = "manylinux2014_ppc64le" else: platform_name = "manylinux1_x86_64" args = ["python3", "setup.py", "bdist_wheel", "--plat-name", platform_name] else: args = ["python3", "setup.py", "bdist_wheel"] wenv = os.environ.copy() wenv["VERSION"] = FLAGS.triton_version p = subprocess.Popen(args, env=wenv) p.wait() fail_if(p.returncode != 0, "setup.py failed") cpdir("dist", FLAGS.dest_dir) print("=== Output wheel file is in: {}".format(FLAGS.dest_dir)) touch(os.path.join(FLAGS.dest_dir, "stamp.whl")) kytensor-2.49.0.7/client/src/python/library/requirements/000077500000000000000000000000001527052371200234125ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/requirements/requirements.txt000066400000000000000000000031041527052371200266740ustar00rootroot00000000000000# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. numpy>=1.19.1,<2 python-rapidjson>=0.9.1 urllib3>=2.0.7 kytensor-2.49.0.7/client/src/python/library/requirements/requirements_cuda.txt000066400000000000000000000030231527052371200276700ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cuda-python kytensor-2.49.0.7/client/src/python/library/requirements/requirements_grpc.txt000066400000000000000000000034521527052371200277150ustar00rootroot00000000000000# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # There is memory leak in later Python GRPC (1.43.0 to be specific), # use known working version until the memory leak is resolved in the future # (see https://github.com/grpc/grpc/issues/28513) grpcio>=1.41.0 numpy>=1.19.1,<2 packaging>=14.1 protobuf>=3.5.0,<5 python-rapidjson>=0.9.1 kytensor-2.49.0.7/client/src/python/library/requirements/requirements_http.txt000066400000000000000000000031421527052371200277350ustar00rootroot00000000000000# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. aiohttp>=3.8.1,<4.0.0 geventhttpclient>=1.4.4,<=2.0.2 numpy>=1.19.1,<2 python-rapidjson>=0.9.1 kytensor-2.49.0.7/client/src/python/library/setup.py000077500000000000000000000122041527052371200224030ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os import sys from itertools import chain from setuptools import find_packages, setup if "--plat-name" in sys.argv: PLATFORM_FLAG = sys.argv[sys.argv.index("--plat-name") + 1] else: PLATFORM_FLAG = "any" if "VERSION" not in os.environ: raise Exception("envvar VERSION must be specified") VERSION = os.environ["VERSION"] try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel class bdist_wheel(_bdist_wheel): def finalize_options(self): _bdist_wheel.finalize_options(self) self.root_is_pure = False def get_tag(self): pyver, abi, plat = "py3", "none", PLATFORM_FLAG return pyver, abi, plat except ImportError: bdist_wheel = None this_directory = os.path.abspath(os.path.dirname(__file__)) def req_file(filename, folder="requirements"): with open(os.path.join(folder, filename)) as f: content = f.readlines() return [x.strip() for x in content if not x.startswith("#")] install_requires = req_file("requirements.txt") extras_require = { "grpc": req_file("requirements_grpc.txt"), "http": req_file("requirements_http.txt"), "cuda": req_file("requirements_cuda.txt"), } extras_require["all"] = list(chain(extras_require.values())) platform_package_data = [] if PLATFORM_FLAG != "any": platform_package_data += ["libcshm.so"] data_files = [ ("", ["LICENSE.txt"]), ] if (PLATFORM_FLAG != "any") and ("@TRITON_PACKAGE_PERF_ANALYZER@" == "ON"): data_files += [("bin", ["perf_analyzer", "perf_client"])] setup( name="tritonclient", version=VERSION, author="NVIDIA Inc.", author_email="sw-dl-triton@nvidia.com", description="Python client library and utilities for communicating with Triton Inference Server", long_description="""See [download-using-python-package-installer-pip](https://github.com/triton-inference-server/client/tree/main#download-using-python-package-installer-pip) """ """for package details.\n\nThe [client examples](https://github.com/triton-inference-server/client/tree/main/src/python/examples) demonstrate how to use the """ """package to issue request to [triton inference server](https://github.com/triton-inference-server/server).""", long_description_content_type="text/markdown", license="BSD", url="https://developer.nvidia.com/nvidia-triton-inference-server", keywords=[ "grpc", "http", "triton", "tensorrt", "inference", "server", "service", "client", "nvidia", ], classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Intended Audience :: Information Technology", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Image Recognition", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries", "Topic :: Utilities", "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Environment :: Console", "Natural Language :: English", "Operating System :: OS Independent", ], install_requires=install_requires, extras_require=extras_require, packages=find_packages(), package_data={ "": platform_package_data, }, zip_safe=False, cmdclass={"bdist_wheel": bdist_wheel}, data_files=data_files, ) kytensor-2.49.0.7/client/src/python/library/tests/000077500000000000000000000000001527052371200220315ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tests/__init__.py000077500000000000000000000030361527052371200241470ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/client/src/python/library/tests/test_cuda_shared_memory.py000066400000000000000000000153031527052371200272760ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest import numpy # Torch support read / write DLPack object on GPU import torch import tritonclient.utils as utils import tritonclient.utils.cuda_shared_memory as cudashm class DLPackTest(unittest.TestCase): """ Testing DLPack implementation in CUDA shared memory utilities """ def test_from_gpu(self): # Create GPU tensor via PyTorch and CUDA shared memory region with # enough space gpu_tensor = torch.ones(4, 4).cuda(0) byte_size = 64 shm_handle = cudashm.create_shared_memory_region("cudashm_data", byte_size, 0) # Set data from DLPack specification of PyTorch tensor cudashm.set_shared_memory_region_from_dlpack(shm_handle, [gpu_tensor]) # Make sure the DLPack specification of the shared memory region can # be consumed by PyTorch smt = cudashm.as_shared_memory_tensor(shm_handle, "FP32", [4, 4]) generated_torch_tensor = torch.from_dlpack(smt) self.assertTrue(torch.allclose(gpu_tensor, generated_torch_tensor)) cudashm.destroy_shared_memory_region(shm_handle) def test_from_cpu(self): # Create CPU tensor via numpy and CUDA shared memory region with # enough space cpu_tensor = numpy.ones([4, 4], dtype=numpy.float32) byte_size = 64 shm_handle = cudashm.create_shared_memory_region("cudashm_data", byte_size, 0) # Set data from DLPack specification of Numpy array cudashm.set_shared_memory_region_from_dlpack(shm_handle, [cpu_tensor]) # Make sure the DLPack specification of the shared memory region can # be consumed by PyTorch. # Need to pass to PyTorch first as numpy doesn't consume GPU DLPack smt = cudashm.as_shared_memory_tensor(shm_handle, "FP32", [4, 4]) generated_torch_tensor = torch.from_dlpack(smt) self.assertTrue( numpy.allclose(cpu_tensor, numpy.from_dlpack(generated_torch_tensor.cpu())) ) cudashm.destroy_shared_memory_region(shm_handle) class NumpyTest(unittest.TestCase): """ Testing Numpy implementation in CUDA shared memory utilities """ def test_from_numpy(self): # Retrieve data from shared memory region via DLPack as it has been # verified in 'DLPackTest' # Create CPU tensor via numpy and CUDA shared memory region with # enough space cpu_tensor = numpy.ones([4, 4], dtype=numpy.float32) byte_size = 64 shm_handle = cudashm.create_shared_memory_region("cudashm_data", byte_size, 0) # Set data from Numpy array cudashm.set_shared_memory_region(shm_handle, [cpu_tensor]) # Make sure the DLPack specification of the shared memory region can # be consumed by PyTorch. # Need to pass to PyTorch first as numpy doesn't consume GPU DLPack smt = cudashm.as_shared_memory_tensor(shm_handle, "FP32", [4, 4]) generated_torch_tensor = torch.from_dlpack(smt) self.assertTrue( numpy.allclose(cpu_tensor, numpy.from_dlpack(generated_torch_tensor.cpu())) ) cudashm.destroy_shared_memory_region(shm_handle) def test_to_numpy(self): # Retrieve data from shared memory region via DLPack as it has been # verified in 'DLPackTest' # Create CPU tensor via numpy and CUDA shared memory region with # enough space cpu_tensor = numpy.ones([4, 4], dtype=numpy.float32) byte_size = 64 shm_handle = cudashm.create_shared_memory_region("cudashm_data", byte_size, 0) # Set data from Numpy array cudashm.set_shared_memory_region(shm_handle, [cpu_tensor]) # Make sure the DLPack specification of the shared memory region can # be consumed by PyTorch. # Need to pass to PyTorch first as numpy doesn't consume GPU DLPack generated_tensor = cudashm.get_contents_as_numpy( shm_handle, numpy.float32, [4, 4] ) self.assertTrue(numpy.allclose(cpu_tensor, generated_tensor)) cudashm.destroy_shared_memory_region(shm_handle) def test_numpy_bytes(self): int_tensor = numpy.arange(start=0, stop=16, dtype=numpy.int32) bytes_tensor = numpy.array( [str(x).encode("utf-8") for x in int_tensor.flatten()], dtype=object ) bytes_tensor = bytes_tensor.reshape(int_tensor.shape) bytes_tensor_serialized = utils.serialize_byte_tensor(bytes_tensor) byte_size = utils.serialized_byte_size(bytes_tensor_serialized) shm_handle = cudashm.create_shared_memory_region("cudashm_data", byte_size, 0) # Set data from Numpy array cudashm.set_shared_memory_region(shm_handle, [bytes_tensor_serialized]) # Make sure the DLPack specification of the shared memory region can # be consumed by PyTorch. # Need to pass to PyTorch first as numpy doesn't consume GPU DLPack generated_tensor = cudashm.get_contents_as_numpy( shm_handle, numpy.object_, [ 16, ], ) self.assertTrue(numpy.array_equal(bytes_tensor, generated_tensor)) cudashm.destroy_shared_memory_region(shm_handle) if __name__ == "__main__": unittest.main() kytensor-2.49.0.7/client/src/python/library/tests/test_inference_server_client.py000077500000000000000000000106251527052371200303330ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. try: from geventhttpclient.response import HTTPSocketResponse except ModuleNotFoundError as error: raise RuntimeError( "The installation does not include http support. Specify 'http' or 'all' while installing the tritonclient package to include the support" ) from error import unittest from unittest.mock import MagicMock, patch import rapidjson from tritonclient.http import * from tritonclient.http._client import _raise_if_error from tritonclient.utils import * json_error_response = """{ "error":"foo", "status_code":"404" }""" class TestInferenceServerClient(unittest.TestCase): """ Testing the various methods in InferenceServerClient to ensure proper functionality """ @patch( "geventhttpclient.response.HTTPSocketResponse._read_headers", MagicMock(return_value=None), ) def setUp(self) -> None: self.response = HTTPSocketResponse("dummy_sock") @patch( "tritonclient.http.InferenceServerClient._get", MagicMock(return_value={"status_code": 200}), ) def test_get_method_success(self): """ Verify that a 200 return code on a get call is a success and does not throw an error """ url = "dummy_url" self.client = InferenceServerClient(url) self.assertEqual(self.client._get(url, None, None)["status_code"], 200) @patch( "geventhttpclient.response.HTTPSocketResponse.get_code", MagicMock(return_value=400), ) @patch( "geventhttpclient.response.HTTPSocketResponse.read", MagicMock(return_value=json_error_response), ) def test_get_method_failure(self): """ Verify that non 200 return codes are raising an exception in the client """ with self.assertRaises(InferenceServerException): _raise_if_error(self.response) @patch( "tritonclient.http.InferenceServerClient._post", MagicMock(return_value={"status_code": 200}), ) def test_post_method_success(self): """ Verify that a 200 return code on a post call is a success and does not throw an error """ url = "dummy_url" body = "dummy_body" self.client = InferenceServerClient(url) self.assertEqual(self.client._post(url, body, None, None)["status_code"], 200) @patch( "geventhttpclient.response.HTTPSocketResponse.get_code", MagicMock(return_value=404), ) @patch( "geventhttpclient.response.HTTPSocketResponse.read", MagicMock(return_value="error_string"), ) def test_error_plain_text(self): """ Verify that a non JSON response returns an InferenceServerException and not a JSONDecodeError """ with self.assertRaises(InferenceServerException): _raise_if_error(self.response) kytensor-2.49.0.7/client/src/python/library/tritonclient/000077500000000000000000000000001527052371200234055ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/CMakeLists.txt000066400000000000000000000037031527052371200261500ustar00rootroot00000000000000# Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) if(${TRITON_ENABLE_PYTHON_HTTP}) file(COPY http DESTINATION .) endif() # TRITON_ENABLE_PYTHON_HTTP if(${TRITON_ENABLE_PYTHON_GRPC}) file(COPY grpc DESTINATION .) endif() # TRITON_ENABLE_PYTHON_GRPC file(COPY _client.py DESTINATION .) file(COPY _plugin.py DESTINATION .) file(COPY __init__.py DESTINATION .) file(COPY _request.py DESTINATION .) file(COPY _auth.py DESTINATION .) add_subdirectory(utils) kytensor-2.49.0.7/client/src/python/library/tritonclient/__init__.py000077500000000000000000000030361527052371200255230ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/client/src/python/library/tritonclient/_auth.py000077500000000000000000000040571527052371200250700ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import base64 from ._plugin import InferenceServerClientPlugin class BasicAuth(InferenceServerClientPlugin): """Basic Authentincation Plugin.""" def __init__(self, username, password): username = username.encode("ascii") password = password.encode("ascii") self._auth_string = ( "Basic " + base64.b64encode(b":".join((username, password))).decode("ascii").strip() ) def __call__(self, request): request.headers["authorization"] = self._auth_string kytensor-2.49.0.7/client/src/python/library/tritonclient/_client.py000077500000000000000000000056611527052371200254070ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from tritonclient.utils import raise_error class InferenceServerClientBase: def __init__(self): self._plugin = None def _call_plugin(self, request): """Called by the subclasses before sending a request to the network. """ if self._plugin != None: self._plugin(request) def register_plugin(self, plugin): """Register a Client Plugin. Parameters ---------- plugin : InferenceServerClientPlugin A client plugin Raises ------ InferenceServerException If a plugin is already registered. """ if self._plugin is None: self._plugin = plugin else: raise_error( "A plugin is already registered. Please " "unregister the previous plugin first before" " registering a new plugin." ) def plugin(self): """Retrieve the registered plugin if any. Returns ------ InferenceServerClientPlugin or None """ return self._plugin def unregister_plugin(self): """Unregister a plugin. Raises ------ InferenceServerException If no plugin has been registered. """ if self._plugin is None: raise_error("No plugin has been registered.") self._plugin = None kytensor-2.49.0.7/client/src/python/library/tritonclient/_plugin.py000077500000000000000000000040711527052371200254210ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from abc import ABC, abstractmethod class InferenceServerClientPlugin(ABC): """Every Triton Client Plugin should extend this class. Each plugin needs to implement the :py:meth:`__call__` method. """ @abstractmethod def __call__(self, request): """This method will be called when any of the client functions are invoked. Note that the request object must be modified in-place. Parameters ---------- request : Request The request object. """ pass kytensor-2.49.0.7/client/src/python/library/tritonclient/_request.py000077500000000000000000000034261527052371200256160ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. class Request: """A request object. Parameters ---------- headers : dict A dictionary containing the request headers. """ def __init__(self, headers): self.headers = headers if headers is not None else {} kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/000077500000000000000000000000001527052371200243405ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/__init__.py000077500000000000000000000060641527052371200264620ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. try: import grpc from tritonclient.grpc import model_config_pb2, service_pb2, service_pb2_grpc from tritonclient.utils import * from .._plugin import InferenceServerClientPlugin from .._request import Request from ._client import MAX_GRPC_MESSAGE_SIZE, InferenceServerClient, KeepAliveOptions from ._infer_input import InferInput from ._infer_result import InferResult from ._requested_output import InferRequestedOutput from ._utils import raise_error, raise_error_grpc except ModuleNotFoundError as error: raise RuntimeError( "The installation does not include grpc support. " "Specify 'grpc' or 'all' while installing the tritonclient " "package to include the support" ) from error import warnings from packaging import version # Check grpc version and issue warnings if grpc version is known to have # memory leakage issue. if version.parse(grpc.__version__) >= version.parse("1.43.0") and version.parse( grpc.__version__ ) < version.parse("1.51.1"): warnings.warn( f"Imported version of grpc is {grpc.__version__}. There is a memory " "leak in certain Python GRPC versions (1.43.0 to be specific). Please " "use versions <1.43.0 or >=1.51.1 to avoid leaks " "(see https://github.com/grpc/grpc/issues/28513)." ) __all__ = [ "InferenceServerClientPlugin", "Request", "InferenceServerClient", "InferInput", "InferRequestedOutput", "InferResult", "KeepAliveOptions", "InferenceServerException", ] kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/_client.py000077500000000000000000002302651527052371200263420ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import base64 import struct import grpc import rapidjson as json from google.protobuf.json_format import MessageToJson from tritonclient.grpc import service_pb2, service_pb2_grpc from .._client import InferenceServerClientBase from .._request import Request from ._infer_result import InferResult from ._infer_stream import _InferStream, _RequestIterator from ._utils import ( _get_inference_request, _grpc_compression_type, get_cancelled_error, get_error_grpc, raise_error, raise_error_grpc, ) # Should be kept consistent with the value specified in # src/core/constants.h, which specifies MAX_GRPC_MESSAGE_SIZE # as INT32_MAX. INT32_MAX = 2 ** (struct.Struct("i").size * 8 - 1) - 1 MAX_GRPC_MESSAGE_SIZE = INT32_MAX class KeepAliveOptions: """A KeepAliveOptions object is used to encapsulate GRPC KeepAlive related parameters for initiating an InferenceServerclient object. See the https://github.com/grpc/grpc/blob/master/doc/keepalive.md documentation for more information. Parameters ---------- keepalive_time_ms: int The period (in milliseconds) after which a keepalive ping is sent on the transport. Default is INT32_MAX. keepalive_timeout_ms: int The period (in milliseconds) the sender of the keepalive ping waits for an acknowledgement. If it does not receive an acknowledgment within this time, it will close the connection. Default is 20000 (20 seconds). keepalive_permit_without_calls: bool Allows keepalive pings to be sent even if there are no calls in flight. Default is False. http2_max_pings_without_data: int The maximum number of pings that can be sent when there is no data/header frame to be sent. gRPC Core will not continue sending pings if we run over the limit. Setting it to 0 allows sending pings without such a restriction. Default is 2. """ def __init__( self, keepalive_time_ms=INT32_MAX, keepalive_timeout_ms=20000, keepalive_permit_without_calls=False, http2_max_pings_without_data=2, ): self.keepalive_time_ms = keepalive_time_ms self.keepalive_timeout_ms = keepalive_timeout_ms self.keepalive_permit_without_calls = keepalive_permit_without_calls self.http2_max_pings_without_data = http2_max_pings_without_data class CallContext: """This is a wrapper over grpc future call which can be used to issue cancellation on an ongoing RPC call. Parameters ---------- grpc_future : gRPC.Future The future tracking gRPC call. """ def __init__(self, grpc_future): self.__grpc_future = grpc_future def cancel(self): """Issues cancellation on the underlying request.""" self.__grpc_future.cancel() class InferenceServerClient(InferenceServerClientBase): """An InferenceServerClient object is used to perform any kind of communication with the InferenceServer using gRPC protocol. Most of the methods are thread-safe except start_stream, stop_stream and async_stream_infer. Accessing a client stream with different threads will cause undefined behavior. Parameters ---------- url : str The inference server URL, e.g. 'localhost:8001'. verbose : bool If True generate verbose output. Default value is False. ssl : bool If True use SSL encrypted secure channel. Default is False. root_certificates : str File holding the PEM-encoded root certificates as a byte string, or None to retrieve them from a default location chosen by gRPC runtime. The option is ignored if `ssl` is False. Default is None. private_key : str File holding the PEM-encoded private key as a byte string, or None if no private key should be used. The option is ignored if `ssl` is False. Default is None. certificate_chain : str File holding PEM-encoded certificate chain as a byte string to use or None if no certificate chain should be used. The option is ignored if `ssl` is False. Default is None. creds: grpc.ChannelCredentials A grpc.ChannelCredentials object to use for the connection. The ssl, root_certificates, private_key and certificate_chain options will be ignored when using this option. Default is None. keepalive_options: KeepAliveOptions Object encapsulating various GRPC KeepAlive options. See the class definition for more information. Default is None. channel_args: List[Tuple] List of Tuple pairs ("key", value) to be passed directly to the GRPC channel as the channel_arguments. If this argument is provided, it is expected the channel arguments are correct and complete, and the keepalive_options parameter will be ignored since the corresponding keepalive channel arguments can be set directly in this parameter. See https://grpc.github.io/grpc/python/glossary.html#term-channel_arguments for more details. Default is None. Raises ------ Exception If unable to create a client. """ def __init__( self, url, verbose=False, ssl=False, root_certificates=None, private_key=None, certificate_chain=None, creds=None, keepalive_options=None, channel_args=None, ): super().__init__() # Explicitly check "is not None" here to support passing an empty # list to specify setting no channel arguments. if channel_args is not None: channel_opt = channel_args else: # Use GRPC KeepAlive client defaults if unspecified if not keepalive_options: keepalive_options = KeepAliveOptions() # To specify custom channel_opt, see the channel_args parameter. channel_opt = [ ("grpc.max_send_message_length", MAX_GRPC_MESSAGE_SIZE), ("grpc.max_receive_message_length", MAX_GRPC_MESSAGE_SIZE), ("grpc.keepalive_time_ms", keepalive_options.keepalive_time_ms), ("grpc.keepalive_timeout_ms", keepalive_options.keepalive_timeout_ms), ( "grpc.keepalive_permit_without_calls", keepalive_options.keepalive_permit_without_calls, ), ( "grpc.http2.max_pings_without_data", keepalive_options.http2_max_pings_without_data, ), ] if creds: self._channel = grpc.secure_channel(url, creds, options=channel_opt) elif ssl: rc_bytes = pk_bytes = cc_bytes = None if root_certificates is not None: with open(root_certificates, "rb") as rc_fs: rc_bytes = rc_fs.read() if private_key is not None: with open(private_key, "rb") as pk_fs: pk_bytes = pk_fs.read() if certificate_chain is not None: with open(certificate_chain, "rb") as cc_fs: cc_bytes = cc_fs.read() creds = grpc.ssl_channel_credentials( root_certificates=rc_bytes, private_key=pk_bytes, certificate_chain=cc_bytes, ) self._channel = grpc.secure_channel(url, creds, options=channel_opt) else: self._channel = grpc.insecure_channel(url, options=channel_opt) self._client_stub = service_pb2_grpc.GRPCInferenceServiceStub(self._channel) self._verbose = verbose self._stream = None def _get_metadata(self, headers): request = Request(headers) self._call_plugin(request) request_metadata = ( request.headers.items() if request.headers is not None else () ) return request_metadata def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __del__(self): self.close() def close(self): """Close the client. Any future calls to server will result in an Error. """ self.stop_stream() self._channel.close() def is_server_live(self, headers=None, client_timeout=None): """Contact the inference server and get liveness. Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- bool True if server is live, False if server is not live. Raises ------ InferenceServerException If unable to get liveness or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.ServerLiveRequest() if self._verbose: print("is_server_live, metadata {}\n{}".format(metadata, request)) response = self._client_stub.ServerLive( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return response.live except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def is_server_ready(self, headers=None, client_timeout=None): """Contact the inference server and get readiness. Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- bool True if server is ready, False if server is not ready. Raises ------ InferenceServerException If unable to get readiness or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.ServerReadyRequest() if self._verbose: print("is_server_ready, metadata {}\n{}".format(metadata, request)) response = self._client_stub.ServerReady( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return response.ready except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def is_model_ready( self, model_name, model_version="", headers=None, client_timeout=None ): """Contact the inference server and get the readiness of specified model. Parameters ---------- model_name: str The name of the model to check for readiness. model_version: str The version of the model to check for readiness. The default value is an empty string which means then the server will choose a version based on the model and internal policy. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- bool True if the model is ready, False if not ready. Raises ------ InferenceServerException If unable to get model readiness or has timed out. """ metadata = self._get_metadata(headers) try: if type(model_version) != str: raise_error("model version must be a string") request = service_pb2.ModelReadyRequest( name=model_name, version=model_version ) if self._verbose: print("is_model_ready, metadata {}\n{}".format(metadata, request)) response = self._client_stub.ModelReady( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return response.ready except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_server_metadata(self, headers=None, as_json=False, client_timeout=None): """Contact the inference server and get its metadata. Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns server metadata as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or ServerMetadataResponse message holding the metadata. Raises ------ InferenceServerException If unable to get server metadata or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.ServerMetadataRequest() if self._verbose: print("get_server_metadata, metadata {}\n{}".format(metadata, request)) response = self._client_stub.ServerMetadata( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_model_metadata( self, model_name, model_version="", headers=None, as_json=False, client_timeout=None, ): """Contact the inference server and get the metadata for specified model. Parameters ---------- model_name: str The name of the model model_version: str The version of the model to get metadata. The default value is an empty string which means then the server will choose a version based on the model and internal policy. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns model metadata as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or ModelMetadataResponse message holding the metadata. Raises ------ InferenceServerException If unable to get model metadata or has timed out. """ metadata = self._get_metadata(headers) try: if type(model_version) != str: raise_error("model version must be a string") request = service_pb2.ModelMetadataRequest( name=model_name, version=model_version ) if self._verbose: print("get_model_metadata, metadata {}\n{}".format(metadata, request)) response = self._client_stub.ModelMetadata( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_model_config( self, model_name, model_version="", headers=None, as_json=False, client_timeout=None, ): """Contact the inference server and get the configuration for specified model. Parameters ---------- model_name: str The name of the model model_version: str The version of the model to get configuration. The default value is an empty string which means then the server will choose a version based on the model and internal policy. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns configuration as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or ModelConfigResponse message holding the metadata. Raises ------ InferenceServerException If unable to get model configuration or has timed out. """ metadata = self._get_metadata(headers) try: if type(model_version) != str: raise_error("model version must be a string") request = service_pb2.ModelConfigRequest( name=model_name, version=model_version ) if self._verbose: print("get_model_config, metadata {}\n{}".format(metadata, request)) response = self._client_stub.ModelConfig( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_model_repository_index( self, headers=None, as_json=False, client_timeout=None ): """Get the index of model repository contents Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns model repository index as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or RepositoryIndexResponse message holding the model repository index. """ metadata = self._get_metadata(headers) try: request = service_pb2.RepositoryIndexRequest() if self._verbose: print( "get_model_repository_index, metadata {}\n{}".format( metadata, request ) ) response = self._client_stub.RepositoryIndex( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def load_model( self, model_name, headers=None, config=None, files=None, client_timeout=None, ): """Request the inference server to load or reload specified model. Parameters ---------- model_name : str The name of the model to be loaded. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. config: str Optional JSON representation of a model config provided for the load request, if provided, this config will be used for loading the model. files: dict Optional dictionary specifying file path (with "file:" prefix) in the override model directory to the file content as bytes. The files will form the model directory that the model will be loaded from. If specified, 'config' must be provided to be the model configuration of the override model directory. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Raises ------ InferenceServerException If unable to load the model or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.RepositoryModelLoadRequest(model_name=model_name) if config is not None: request.parameters["config"].string_param = config if self._verbose: # Don't print file content which can be large print( "load_model, metadata {}\noverride files omitted:\n{}".format( metadata, request ) ) if files is not None: for path, content in files.items(): request.parameters[path].bytes_param = content self._client_stub.RepositoryModelLoad( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print("Loaded model '{}'".format(model_name)) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def unload_model( self, model_name, headers=None, unload_dependents=False, client_timeout=None, ): """Request the inference server to unload specified model. Parameters ---------- model_name : str The name of the model to be unloaded. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. unload_dependents : bool Whether the dependents of the model should also be unloaded. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Raises ------ InferenceServerException If unable to unload the model or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.RepositoryModelUnloadRequest(model_name=model_name) request.parameters["unload_dependents"].bool_param = unload_dependents if self._verbose: print("unload_model, metadata {}\n{}".format(metadata, request)) self._client_stub.RepositoryModelUnload( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print("Unloaded model '{}'".format(model_name)) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_inference_statistics( self, model_name="", model_version="", headers=None, as_json=False, client_timeout=None, ): """Get the inference statistics for the specified model name and version. Parameters ---------- model_name : str The name of the model to get statistics. The default value is an empty string, which means statistics of all models will be returned. model_version: str The version of the model to get inference statistics. The default value is an empty string which means then the server will return the statistics of all available model versions. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns inference statistics as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Raises ------ InferenceServerException If unable to get the model inference statistics or has timed out. """ metadata = self._get_metadata(headers) try: if type(model_version) != str: raise_error("model version must be a string") request = service_pb2.ModelStatisticsRequest( name=model_name, version=model_version ) if self._verbose: print( "get_inference_statistics, metadata {}\n{}".format( metadata, request ) ) response = self._client_stub.ModelStatistics( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def update_trace_settings( self, model_name=None, settings={}, headers=None, as_json=False, client_timeout=None, ): """Update the trace settings for the specified model name, or global trace settings if model name is not given. Returns the trace settings after the update. Parameters ---------- model_name : str The name of the model to update trace settings. Specifying None or empty string will update the global trace settings. The default value is None. settings: dict The new trace setting values. Only the settings listed will be updated. If a trace setting is listed in the dictionary with a value of 'None', that setting will be cleared. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns trace settings as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or TraceSettingResponse message holding the updated trace settings. Raises ------ InferenceServerException If unable to update the trace settings or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.TraceSettingRequest() if (model_name is not None) and (model_name != ""): request.model_name = model_name for key, value in settings.items(): if value is None: request.settings[key] else: request.settings[key].value.extend( value if isinstance(value, list) else [value] ) if self._verbose: print( "update_trace_settings, metadata {}\n{}".format(metadata, request) ) response = self._client_stub.TraceSetting( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_trace_settings( self, model_name=None, headers=None, as_json=False, client_timeout=None ): """Get the trace settings for the specified model name, or global trace settings if model name is not given Parameters ---------- model_name : str The name of the model to get trace settings. Specifying None or empty string will return the global trace settings. The default value is None. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns trace settings as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or TraceSettingResponse message holding the trace settings. Raises ------ InferenceServerException If unable to get the trace settings or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.TraceSettingRequest() if (model_name is not None) and (model_name != ""): request.model_name = model_name if self._verbose: print("get_trace_settings, metadata {}\n{}".format(metadata, request)) response = self._client_stub.TraceSetting( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def update_log_settings( self, settings, headers=None, as_json=False, client_timeout=None ): """Update the global log settings. Returns the log settings after the update. Parameters ---------- settings: dict The new log setting values. Only the settings listed will be updated. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns trace settings as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or LogSettingsResponse message holding the updated log settings. Raises ------ InferenceServerException If unable to update the log settings or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.LogSettingsRequest() for key, value in settings.items(): if value is None: request.settings[key] else: if key == "log_file" or key == "log_format": request.settings[key].string_param = value elif key == "log_verbose_level": request.settings[key].uint32_param = value else: request.settings[key].bool_param = value if self._verbose: print("update_log_settings, metadata {}\n{}".format(metadata, request)) response = self._client_stub.LogSettings( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_log_settings(self, headers=None, as_json=False, client_timeout=None): """Get the global log settings. Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns log settings as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or LogSettingsResponse message holding the log settings. Raises ------ InferenceServerException If unable to get the log settings or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.LogSettingsRequest() if self._verbose: print("get_log_settings, metadata {}\n{}".format(metadata, request)) response = self._client_stub.LogSettings( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_system_shared_memory_status( self, region_name="", headers=None, as_json=False, client_timeout=None ): """Request system shared memory status from the server. Parameters ---------- region_name : str The name of the region to query status. The default value is an empty string, which means that the status of all active system shared memory will be returned. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns system shared memory status as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or SystemSharedMemoryStatusResponse message holding the system shared memory status. Raises ------ InferenceServerException If unable to get the status of specified shared memory or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.SystemSharedMemoryStatusRequest(name=region_name) if self._verbose: print( "get_system_shared_memory_status, metadata {}\n{}".format( metadata, request ) ) response = self._client_stub.SystemSharedMemoryStatus( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def register_system_shared_memory( self, name, key, byte_size, offset=0, headers=None, client_timeout=None ): """Request the server to register a system shared memory with the following specification. Parameters ---------- name : str The name of the region to register. key : str The key of the underlying memory object that contains the system shared memory region. byte_size : int The size of the system shared memory region, in bytes. offset : int Offset, in bytes, within the underlying memory object to the start of the system shared memory region. The default value is zero. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Raises ------ InferenceServerException If unable to register the specified system shared memory or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.SystemSharedMemoryRegisterRequest( name=name, key=key, offset=offset, byte_size=byte_size ) if self._verbose: print( "register_system_shared_memory, metadata {}\n{}".format( metadata, request ) ) self._client_stub.SystemSharedMemoryRegister( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print("Registered system shared memory with name '{}'".format(name)) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def unregister_system_shared_memory( self, name="", headers=None, client_timeout=None ): """Request the server to unregister a system shared memory with the specified name. Parameters ---------- name : str The name of the region to unregister. The default value is empty string which means all the system shared memory regions will be unregistered. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Raises ------ InferenceServerException If unable to unregister the specified system shared memory region or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.SystemSharedMemoryUnregisterRequest(name=name) if self._verbose: print( "unregister_system_shared_memory, metadata {}\n{}".format( metadata, request ) ) self._client_stub.SystemSharedMemoryUnregister( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: if name != "": print( "Unregistered system shared memory with name '{}'".format(name) ) else: print("Unregistered all system shared memory regions") except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def get_cuda_shared_memory_status( self, region_name="", headers=None, as_json=False, client_timeout=None ): """Request cuda shared memory status from the server. Parameters ---------- region_name : str The name of the region to query status. The default value is an empty string, which means that the status of all active cuda shared memory will be returned. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. as_json : bool If True then returns cuda shared memory status as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Returns ------- dict or protobuf message The JSON dict or CudaSharedMemoryStatusResponse message holding the cuda shared memory status. Raises ------ InferenceServerException If unable to get the status of specified shared memory or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.CudaSharedMemoryStatusRequest(name=region_name) if self._verbose: print( "get_cuda_shared_memory_status, metadata {}\n{}".format( metadata, request ) ) response = self._client_stub.CudaSharedMemoryStatus( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) if as_json: return json.loads( MessageToJson(response, preserving_proto_field_name=True) ) else: return response except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def register_cuda_shared_memory( self, name, raw_handle, device_id, byte_size, headers=None, client_timeout=None, ): """Request the server to register a system shared memory with the following specification. Parameters ---------- name : str The name of the region to register. raw_handle : bytes The raw serialized cudaIPC handle in base64 encoding. device_id : int The GPU device ID on which the cudaIPC handle was created. byte_size : int The size of the cuda shared memory region, in bytes. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Raises ------ InferenceServerException If unable to register the specified cuda shared memory or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.CudaSharedMemoryRegisterRequest( name=name, raw_handle=base64.b64decode(raw_handle), device_id=device_id, byte_size=byte_size, ) if self._verbose: print( "register_cuda_shared_memory, metadata {}\n{}".format( metadata, request ) ) self._client_stub.CudaSharedMemoryRegister( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print("Registered cuda shared memory with name '{}'".format(name)) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def unregister_cuda_shared_memory(self, name="", headers=None, client_timeout=None): """Request the server to unregister a cuda shared memory with the specified name. Parameters ---------- name : str The name of the region to unregister. The default value is empty string which means all the cuda shared memory regions will be unregistered. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. Raises ------ InferenceServerException If unable to unregister the specified cuda shared memory region or has timed out. """ metadata = self._get_metadata(headers) try: request = service_pb2.CudaSharedMemoryUnregisterRequest(name=name) if self._verbose: print( "unregister_cuda_shared_memory, metadata {}\n{}".format( metadata, request ) ) self._client_stub.CudaSharedMemoryUnregister( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: if name != "": print("Unregistered cuda shared memory with name '{}'".format(name)) else: print("Unregistered all cuda shared memory regions") except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def infer( self, model_name, inputs, model_version="", outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, priority=0, timeout=None, client_timeout=None, headers=None, compression_algorithm=None, parameters=None, ): """Run synchronous inference using the supplied 'inputs' requesting the outputs specified by 'outputs'. Parameters ---------- model_name: str The name of the model to run inference. inputs : list A list of :py:class:`InferInput` objects, each describing data for a input tensor required by the model. model_version : str The version of the model to run inference. The default value is an empty string which means then the server will choose a version based on the model and internal policy. outputs : list A list of :py:class:`InferRequestedOutput` objects, each describing how the output data must be returned. If not specified all outputs produced by the model will be returned using default settings. request_id : str Optional identifier for the request. If specified will be returned in the response. Default value is an empty string which means no request_id will be used. sequence_id : int The unique identifier for the sequence being represented by the object. Default value is 0 which means that the request does not belong to a sequence. sequence_start : bool Indicates whether the request being added marks the start of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. sequence_end : bool Indicates whether the request being added marks the end of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. priority : int Indicates the priority of the request. Priority value zero indicates that the default priority level should be used (i.e. same behavior as not specifying the priority parameter). Lower value priorities indicate higher priority levels. Thus the highest priority level is indicated by setting the parameter to 1, the next highest is 2, etc. If not provided, the server will handle the request using default setting for the model. timeout : int The timeout value for the request, in microseconds. If the request cannot be completed within the time the server can take a model-specific action such as terminating the request. If not provided, the server will handle the request using default setting for the model. This option is only respected by the model that is configured with dynamic batching. See here for more details: https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher client_timeout : float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. headers : dict Optional dictionary specifying additional HTTP headers to include in the request. compression_algorithm : str Optional grpc compression algorithm to be used on client side. Currently supports "deflate", "gzip" and None. By default, no compression is used. parameters : dict Optional custom parameters to be included in the inference request. Returns ------- InferResult The object holding the result of the inference. Raises ------ InferenceServerException If server fails to perform inference. """ metadata = self._get_metadata(headers) if type(model_version) != str: raise_error("model version must be a string") request = _get_inference_request( model_name=model_name, inputs=inputs, model_version=model_version, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, parameters=parameters, ) if self._verbose: print("infer, metadata {}\n{}".format(metadata, request)) try: response = self._client_stub.ModelInfer( request=request, metadata=metadata, timeout=client_timeout, compression=_grpc_compression_type(compression_algorithm), ) if self._verbose: print(response) result = InferResult(response) return result except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def async_infer( self, model_name, inputs, callback, model_version="", outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, priority=0, timeout=None, client_timeout=None, headers=None, compression_algorithm=None, parameters=None, ): """Run asynchronous inference using the supplied 'inputs' requesting the outputs specified by 'outputs'. Parameters ---------- model_name: str The name of the model to run inference. inputs : list A list of :py:class:`InferInput` objects, each describing data for a input tensor required by the model. callback : function Python function that is invoked once the request is completed. The function must reserve the last two arguments (result, error) to hold :py:class:`InferResult` and :py:class:`InferenceServerException` objects respectively which will be provided to the function when executing the callback. The ownership of these objects will be given to the user. The 'error' would be None for a successful inference. model_version: str The version of the model to run inference. The default value is an empty string which means then the server will choose a version based on the model and internal policy. outputs : list A list of :py:class:`InferRequestedOutput` objects, each describing how the output data must be returned. If not specified all outputs produced by the model will be returned using default settings. request_id : str Optional identifier for the request. If specified will be returned in the response. Default value is an empty string which means no request_id will be used. sequence_id : int The unique identifier for the sequence being represented by the object. Default value is 0 which means that the request does not belong to a sequence. sequence_start: bool Indicates whether the request being added marks the start of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. sequence_end: bool Indicates whether the request being added marks the end of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. priority : int Indicates the priority of the request. Priority value zero indicates that the default priority level should be used (i.e. same behavior as not specifying the priority parameter). Lower value priorities indicate higher priority levels. Thus the highest priority level is indicated by setting the parameter to 1, the next highest is 2, etc. If not provided, the server will handle the request using default setting for the model. timeout : int The timeout value for the request, in microseconds. If the request cannot be completed within the time the server can take a model-specific action such as terminating the request. If not provided, the server will handle the request using default setting for the model. This option is only respected by the model that is configured with dynamic batching. See here for more details: https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and provide error with message "Deadline Exceeded" in the callback when the specified time elapses. The default value is None which means client will wait for the response from the server. client_timeout: float The maximum end-to-end time, in seconds, the request is allowed to take. The client will abort request and raise InferenceServerExeption with message "Deadline Exceeded" when the specified time elapses. The default value is None which means client will wait for the response from the server. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. compression_algorithm : str Optional grpc compression algorithm to be used on client side. Currently supports "deflate", "gzip" and None. By default, no compression is used. parameters : dict Optional custom parameters to be included in the inference request. Returns ------- CallContext A representation of a computation in another control flow. Computations represented by a Future may be yet to be begun, ongoing, or have already completed. Note ---- This object can be used to cancel the inference request like below: >>> future = async_infer(...) >>> ret = future.cancel() Raises ------ InferenceServerException If server fails to issue inference. """ def wrapped_callback(call_future): error = result = None try: response = call_future.result() if self._verbose: print(response) result = InferResult(response) except grpc.RpcError as rpc_error: error = get_error_grpc(rpc_error) except grpc.FutureCancelledError as err: error = get_cancelled_error() callback(result=result, error=error) metadata = self._get_metadata(headers) if type(model_version) != str: raise_error("model version must be a string") request = _get_inference_request( model_name=model_name, inputs=inputs, model_version=model_version, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, parameters=parameters, ) if self._verbose: print("async_infer, metadata {}\n{}".format(metadata, request)) try: self._call_future = self._client_stub.ModelInfer.future( request=request, metadata=metadata, timeout=client_timeout, compression=_grpc_compression_type(compression_algorithm), ) if self._verbose: verbose_message = "Sent request" if request_id != "": verbose_message = verbose_message + " '{}'".format(request_id) print(verbose_message) self._call_future.add_done_callback(wrapped_callback) return CallContext(self._call_future) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def start_stream( self, callback, stream_timeout=None, headers=None, compression_algorithm=None ): """Starts a grpc bi-directional stream to send streaming inferences. Note: When using stream, user must ensure the InferenceServerClient.close() gets called at exit. Parameters ---------- callback : function Python function that is invoked upon receiving response from the underlying stream. The function must reserve the last two arguments (result, error) to hold :py:class:`InferResult` and :py:class:`InferenceServerException` objects respectively which will be provided to the function when executing the callback. The ownership of these objects will be given to the user. The 'error' would be None for a successful inference. stream_timeout : float Optional stream timeout (in seconds). The stream will be closed once the specified timeout expires. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. compression_algorithm : str Optional grpc compression algorithm to be used on client side. Currently supports "deflate", "gzip" and None. By default, no compression is used. Raises ------ InferenceServerException If unable to start a stream or a stream was already running for this client or has timed out. """ if self._stream is not None: raise_error( "cannot start another stream with one already running. " "'InferenceServerClient' supports only a single active " "stream at a given time." ) metadata = self._get_metadata(headers) self._stream = _InferStream(callback, self._verbose) try: response_iterator = self._client_stub.ModelStreamInfer( _RequestIterator(self._stream), metadata=metadata, timeout=stream_timeout, compression=_grpc_compression_type(compression_algorithm), ) self._stream._init_handler(response_iterator) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def stop_stream(self, cancel_requests=False): """Stops a stream if one available. Parameters ---------- cancel_requests : bool If set True, then client cancels all the pending requests and closes the stream. If set False, the call blocks till all the pending requests on the stream are processed. """ if self._stream is not None: self._stream.close(cancel_requests) self._stream = None def async_stream_infer( self, model_name, inputs, model_version="", outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, enable_empty_final_response=False, priority=0, timeout=None, parameters=None, ): """Runs an asynchronous inference over gRPC bi-directional streaming API. A stream must be established with a call to start_stream() before calling this function. All the results will be provided to the callback function associated with the stream. Parameters ---------- model_name: str The name of the model to run inference. inputs : list A list of :py:class:`InferInput` objects, each describing data for a input tensor required by the model. model_version: str The version of the model to run inference. The default value is an empty string which means then the server will choose a version based on the model and internal policy. outputs : list A list of :py:class:`InferRequestedOutput` objects, each describing how the output data must be returned. If not specified all outputs produced by the model will be returned using default settings. request_id : str Optional identifier for the request. If specified will be returned in the response. Default value is an empty string which means no request_id will be used. sequence_id : int or str The unique identifier for the sequence being represented by the object. A value of 0 or "" means that the request does not belong to a sequence. Default is 0. sequence_start: bool Indicates whether the request being added marks the start of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0 or "". sequence_end: bool Indicates whether the request being added marks the end of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0 or "". enable_empty_final_response: bool Indicates whether "empty" responses should be generated and sent back to the client from the server during streaming inference when they contain the TRITONSERVER_RESPONSE_COMPLETE_FINAL flag. This strictly relates to the case of models/backends that send flags-only responses (use TRITONBACKEND_ResponseFactorySendFlags(TRITONSERVER_RESPONSE_COMPLETE_FINAL) or InferenceResponseSender.send(flags=TRITONSERVER_RESPONSE_COMPLETE_FINAL)) Currently, this only occurs for decoupled models, and can be used to communicate to the client when a request has received its final response from the model. If the backend sends the final flag along with a non-empty response, this arg is not needed. Default value is False. priority : int Indicates the priority of the request. Priority value zero indicates that the default priority level should be used (i.e. same behavior as not specifying the priority parameter). Lower value priorities indicate higher priority levels. Thus the highest priority level is indicated by setting the parameter to 1, the next highest is 2, etc. If not provided, the server will handle the request using default setting for the model. timeout : int The timeout value for the request, in microseconds. If the request cannot be completed within the time the server can take a model-specific action such as terminating the request. If not provided, the server will handle the request using default setting for the model. This does not stop the grpc stream itself and is only respected by the model that is configured with dynamic batching. See here for more details: https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher parameters : dict Optional custom parameters to be included in the inference request. Raises ------ InferenceServerException If server fails to issue inference. """ if self._stream is None: raise_error( "stream not available, use start_stream() to make one available." ) if type(model_version) != str: raise_error("model version must be a string") request = _get_inference_request( model_name=model_name, inputs=inputs, model_version=model_version, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, parameters=parameters, ) # Unique to streaming inference as it only pertains to decoupled models # Only attach the parameter if True, no need to send/parse when False. if enable_empty_final_response: request.parameters["triton_enable_empty_final_response"].bool_param = True if self._verbose: print("async_stream_infer\n{}".format(request)) # Enqueues the request to the stream self._stream._enqueue_request(request) if self._verbose: print("enqueued request {} to stream...".format(request_id)) kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/_infer_input.py000077500000000000000000000166371527052371200274130ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import numpy as np from tritonclient.grpc import service_pb2 from tritonclient.utils import * from ._utils import raise_error class InferInput: """An object of InferInput class is used to describe input tensor for an inference request. Parameters ---------- name : str The name of input whose data will be described by this object shape : list The shape of the associated input. datatype : str The datatype of the associated input. """ def __init__(self, name, shape, datatype): self._input = service_pb2.ModelInferRequest().InferInputTensor() self._input.name = name self._input.ClearField("shape") self._input.shape.extend(shape) self._input.datatype = datatype self._raw_content = None def name(self): """Get the name of input associated with this object. Returns ------- str The name of input """ return self._input.name def datatype(self): """Get the datatype of input associated with this object. Returns ------- str The datatype of input """ return self._input.datatype def shape(self): """Get the shape of input associated with this object. Returns ------- list The shape of input """ return self._input.shape def set_shape(self, shape): """Set the shape of input. Parameters ---------- shape : list The shape of the associated input. Returns ------- InferInput The updated input """ self._input.ClearField("shape") self._input.shape.extend(shape) return self def set_data_from_numpy(self, input_tensor): """Set the tensor data from the specified numpy array for input associated with this object. Parameters ---------- input_tensor : numpy array The tensor data in numpy array format Returns ------- InferInput The updated input Raises ------ InferenceServerException If failed to set data for the tensor. """ if not isinstance(input_tensor, (np.ndarray,)): raise_error("input_tensor must be a numpy array") # DLIS-3986: Special handling for bfloat16 until Numpy officially supports it if self._input.datatype == "BF16": if input_tensor.dtype != triton_to_np_dtype(self._input.datatype): raise_error( "got unexpected datatype {} from numpy array, expected {} for BF16 type".format( input_tensor.dtype, triton_to_np_dtype(self._input.datatype) ) ) else: dtype = np_to_triton_dtype(input_tensor.dtype) if self._input.datatype != dtype: raise_error( "got unexpected datatype {} from numpy array, expected {}".format( dtype, self._input.datatype ) ) valid_shape = True if len(self._input.shape) != len(input_tensor.shape): valid_shape = False for i in range(len(self._input.shape)): if self._input.shape[i] != input_tensor.shape[i]: valid_shape = False if not valid_shape: raise_error( "got unexpected numpy array shape [{}], expected [{}]".format( str(input_tensor.shape)[1:-1], str(self._input.shape)[1:-1] ) ) self._input.parameters.pop("shared_memory_region", None) self._input.parameters.pop("shared_memory_byte_size", None) self._input.parameters.pop("shared_memory_offset", None) if self._input.datatype == "BYTES": serialized_output = serialize_byte_tensor(input_tensor) if serialized_output.size > 0: self._raw_content = serialized_output.item() else: self._raw_content = b"" elif self._input.datatype == "BF16": serialized_output = serialize_bf16_tensor(input_tensor) if serialized_output.size > 0: self._raw_content = serialized_output.item() else: self._raw_content = b"" else: self._raw_content = input_tensor.tobytes() return self def set_shared_memory(self, region_name, byte_size, offset=0): """Set the tensor data from the specified shared memory region. Parameters ---------- region_name : str The name of the shared memory region holding tensor data. byte_size : int The size of the shared memory region holding tensor data. offset : int The offset, in bytes, into the region where the data for the tensor starts. The default value is 0. Returns ------- InferInput The updated input """ self._input.ClearField("contents") self._raw_content = None self._input.parameters["shared_memory_region"].string_param = region_name self._input.parameters["shared_memory_byte_size"].int64_param = byte_size if offset != 0: self._input.parameters["shared_memory_offset"].int64_param = offset return self def _get_tensor(self): """Retrieve the underlying InferInputTensor message. Returns ------- protobuf message The underlying InferInputTensor protobuf message. """ return self._input def _get_content(self): """Retrieve the contents for this tensor in raw bytes. Returns ------- bytes The associated contents for this tensor in raw bytes. """ return self._raw_content kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/_infer_result.py000077500000000000000000000142061527052371200275600ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import rapidjson as json from google.protobuf.json_format import MessageToJson from tritonclient.utils import * class InferResult: """An object of :py:class:`InferResult` class holds the response of an inference request and provide methods to retrieve inference results. Parameters ---------- result : protobuf message The ModelInferResponse returned by the server """ def __init__(self, result): self._result = result def as_numpy(self, name): """Get the tensor data for output associated with this object in numpy format Parameters ---------- name : str The name of the output tensor whose result is to be retrieved. Returns ------- numpy array The numpy array containing the response data for the tensor or None if the data for specified tensor name is not found. """ index = 0 for output in self._result.outputs: if output.name == name: shape = [] for value in output.shape: shape.append(value) datatype = output.datatype if index < len(self._result.raw_output_contents): if datatype == "BYTES": # String results contain a 4-byte string length # followed by the actual string characters. Hence, # need to decode the raw bytes to convert into # array elements. np_array = deserialize_bytes_tensor( self._result.raw_output_contents[index] ) elif datatype == "BF16": np_array = deserialize_bf16_tensor( self._result.raw_output_contents[index] ) else: np_array = np.frombuffer( self._result.raw_output_contents[index], dtype=triton_to_np_dtype(datatype), ) elif len(output.contents.bytes_contents) != 0: np_array = np.array(output.contents.bytes_contents, copy=False) else: np_array = np.empty(0) np_array = np_array.reshape(shape) return np_array else: index += 1 return None def get_output(self, name, as_json=False): """Retrieves the InferOutputTensor corresponding to the named output. Parameters ---------- name : str The name of the tensor for which Output is to be retrieved. as_json : bool If True then returns response as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. Returns ------- protobuf message or dict If a InferOutputTensor with specified name is present in ModelInferResponse then returns it as a protobuf message or dict, otherwise returns None. """ for output in self._result.outputs: if output.name == name: if as_json: MessageToJson(output, preserving_proto_field_name=True) else: return output return None def get_response(self, as_json=False): """Retrieves the complete ModelInferResponse as a json dict object or protobuf message Parameters ---------- as_json : bool If True then returns response as a json dict, otherwise as a protobuf message. Default value is False. The returned json is generated from the protobuf message using MessageToJson and as a result int64 values are represented as string. It is the caller's responsibility to convert these strings back to int64 values as necessary. Returns ------- protobuf message or dict The underlying ModelInferResponse as a protobuf message or dict. """ if as_json: return json.loads( MessageToJson(self._result, preserving_proto_field_name=True) ) else: return self._result kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/_infer_stream.py000077500000000000000000000157311527052371200275410ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import queue import threading import grpc from tritonclient.utils import * from ._infer_result import InferResult from ._utils import get_cancelled_error, get_error_grpc, raise_error class _InferStream: """Supports sending inference requests and receiving corresponding requests on a gRPC bi-directional stream. Parameters ---------- callback : function Python function that is invoked upon receiving response from the underlying stream. The function must reserve the last two arguments (result, error) to hold :py:class:`InferResult` and :py:class:`InferenceServerException` objects respectively which will be provided to the function when executing the callback. The ownership of these objects will be given to the user. The 'error' would be None for a successful inference. verbose : bool Enables verbose mode if set True. """ def __init__(self, callback, verbose): self._callback = callback self._verbose = verbose self._request_queue = queue.Queue() self._handler = None self._cancelled = False self._active = True self._response_iterator = None def __del__(self): self.close(cancel_requests=True) def close(self, cancel_requests=False): """Gracefully close underlying gRPC streams. If cancel_requests is set True, then client cancels all the pending requests and closes the stream. If set False, the call blocks till all the pending requests on the stream are processed. """ if cancel_requests and self._response_iterator: self._response_iterator.cancel() self._cancelled = True if self._handler is not None: if not self._cancelled: self._request_queue.put(None) if self._handler.is_alive(): self._handler.join() if self._verbose: print("stream stopped...") self._handler = None def _init_handler(self, response_iterator): """Initializes the handler to process the response from stream and execute the callbacks. Parameters ---------- response_iterator : iterator The iterator over the gRPC response stream. """ self._response_iterator = response_iterator if self._handler is not None: raise_error("Attempted to initialize already initialized InferStream") # Create a new thread to handle the gRPC response stream self._handler = threading.Thread(target=self._process_response) self._handler.start() if self._verbose: print("stream started...") def _enqueue_request(self, request): """Enqueues the specified request object to be provided in gRPC request stream. Parameters ---------- request : ModelInferRequest The protobuf message holding the ModelInferRequest """ if self._active: self._request_queue.put(request) else: raise_error( "The stream is no longer in valid state, the error detail " "is reported through provided callback. A new stream should " "be started after stopping the current stream." ) def _get_request(self): """Returns the request details in the order they were added. The call to this function will block until the requests are available in the queue. InferStream._enqueue_request adds the request to the queue. Returns ------- protobuf message The ModelInferRequest protobuf message. """ request = self._request_queue.get() return request def _process_response(self): """Worker thread function to iterate through the response stream and executes the provided callbacks. """ try: for response in self._response_iterator: if self._verbose: print(response) result = error = None if response.error_message != "": error = InferenceServerException(msg=response.error_message) else: result = InferResult(response.infer_response) self._callback(result=result, error=error) except grpc.RpcError as rpc_error: # On GRPC error, refresh the active state to indicate if the stream # can still be used. The stream won't be closed here as the thread # executing this function is managed by stream and may cause # circular wait self._active = self._response_iterator.is_active() if rpc_error.code() == grpc.StatusCode.CANCELLED: error = get_cancelled_error(rpc_error.details()) else: error = get_error_grpc(rpc_error) self._callback(result=None, error=error) class _RequestIterator: """An iterator class to provide data to gRPC request stream. Parameters ---------- stream : InferStream The InferStream that holds the context to an active stream. """ def __init__(self, stream): self._stream = stream def __iter__(self): return self def __next__(self): request = self._stream._get_request() if request is None: raise StopIteration return request kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/_requested_output.py000077500000000000000000000105061527052371200304770ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from tritonclient.grpc import service_pb2 from ._utils import raise_error class InferRequestedOutput: """An object of :py:class:`InferRequestedOutput` class is used to describe a requested output tensor for an inference request. Parameters ---------- name : str The name of output tensor to associate with this object class_count : int The number of classifications to be requested. The default value is 0 which means the classification results are not requested. """ def __init__(self, name, class_count=0): self._output = service_pb2.ModelInferRequest().InferRequestedOutputTensor() self._output.name = name if class_count != 0: self._output.parameters["classification"].int64_param = class_count def name(self): """Get the name of output associated with this object. Returns ------- str The name of output """ return self._output.name def set_shared_memory(self, region_name, byte_size, offset=0): """Marks the output to return the inference result in specified shared memory region. Parameters ---------- region_name : str The name of the shared memory region to hold tensor data. byte_size : int The size of the shared memory region to hold tensor data. offset : int The offset, in bytes, into the region where the data for the tensor starts. The default value is 0. Raises ------ InferenceServerException If failed to set shared memory for the tensor. """ if "classification" in self._output.parameters: raise_error("shared memory can't be set on classification output") self._output.parameters["shared_memory_region"].string_param = region_name self._output.parameters["shared_memory_byte_size"].int64_param = byte_size if offset != 0: self._output.parameters["shared_memory_offset"].int64_param = offset def unset_shared_memory(self): """Clears the shared memory option set by the last call to :py:meth:`InferRequestedOutput.set_shared_memory()`. After call to this function requested output will no longer be returned in a shared memory region. """ self._output.parameters.pop("shared_memory_region", None) self._output.parameters.pop("shared_memory_byte_size", None) self._output.parameters.pop("shared_memory_offset", None) def _get_tensor(self): """Retrieve the underlying InferRequestedOutputTensor message. Returns ------- protobuf message The underlying InferRequestedOutputTensor protobuf message. """ return self._output kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/_utils.py000077500000000000000000000126341527052371200262220ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import grpc from tritonclient.grpc import service_pb2 from tritonclient.utils import * def get_error_grpc(rpc_error): """Convert a gRPC error to an :py:class:`InferenceServerException`. Parameters ---------- rpc_error : grpc.RpcError The gRPC error Returns ------- InferenceServerException """ return InferenceServerException( msg=rpc_error.details(), status=str(rpc_error.code()), debug_details=rpc_error.debug_error_string(), ) def get_cancelled_error(msg=None): """Get :py:class:`InferenceServerException` object for a cancelled RPC. Returns ------- InferenceServerException """ if not msg: msg = "Locally cancelled by application!" return InferenceServerException(msg=msg, status="StatusCode.CANCELLED") def raise_error_grpc(rpc_error): """Raise an :py:class:`InferenceServerException` from a gRPC error. Parameters ---------- rpc_error : grpc.RpcError The gRPC error Raises ------- InferenceServerException """ raise get_error_grpc(rpc_error) from None def _get_inference_request( model_name, inputs, model_version, request_id, outputs, sequence_id, sequence_start, sequence_end, priority, timeout, parameters, ): request = service_pb2.ModelInferRequest() request.model_name = model_name request.model_version = model_version if request_id != "": request.id = request_id for infer_input in inputs: request.inputs.extend([infer_input._get_tensor()]) if infer_input._get_content() is not None: request.raw_input_contents.extend([infer_input._get_content()]) if outputs is not None: for infer_output in outputs: request.outputs.extend([infer_output._get_tensor()]) if sequence_id != 0 and sequence_id != "": if isinstance(sequence_id, str): request.parameters["sequence_id"].string_param = sequence_id else: request.parameters["sequence_id"].int64_param = sequence_id request.parameters["sequence_start"].bool_param = sequence_start request.parameters["sequence_end"].bool_param = sequence_end if priority != 0: request.parameters["priority"].uint64_param = priority if timeout is not None: request.parameters["timeout"].int64_param = timeout if parameters: for key, value in parameters.items(): if ( key == "sequence_id" or key == "sequence_start" or key == "sequence_end" or key == "priority" or key == "binary_data_output" ): raise_error( f'Parameter "{key}" is a reserved parameter and cannot be specified.' ) else: if isinstance(value, str): request.parameters[key].string_param = value elif isinstance(value, bool): request.parameters[key].bool_param = value elif isinstance(value, int): request.parameters[key].int64_param = value elif isinstance(value, float): request.parameters[key].double_param = value else: raise_error( f'The parameter datatype "{type(value)}" for key "{key}" is not supported.' ) return request def _grpc_compression_type(algorithm_str): if algorithm_str is None: return grpc.Compression.NoCompression elif algorithm_str.lower() == "deflate": return grpc.Compression.Deflate elif algorithm_str.lower() == "gzip": return grpc.Compression.Gzip print( "The provided client-side compression algorithm is not supported... " "using no compression" ) return grpc.Compression.NoCompression kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/aio/000077500000000000000000000000001527052371200251105ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/aio/__init__.py000077500000000000000000000772531527052371200272420ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import base64 import sys import grpc import rapidjson as json from google.protobuf.json_format import MessageToJson from tritonclient.grpc import * from tritonclient.grpc import model_config_pb2, service_pb2, service_pb2_grpc from ... import _auth as auth from ..._plugin import InferenceServerClientPlugin from ..._request import Request from .._client import MAX_GRPC_MESSAGE_SIZE, InferenceServerClientBase, KeepAliveOptions from .._utils import ( _get_inference_request, _grpc_compression_type, raise_error, raise_error_grpc, ) class InferenceServerClient(InferenceServerClientBase): """This feature is currently in beta and may be subject to change. An analogy of the :py:class:`tritonclient.grpc.InferenceServerClient` to enable calling via asyncio syntax. The object is intended to be used by a single thread and simultaneously calling methods with different threads is not supported and can cause undefined behavior. """ def __init__( self, url, verbose=False, ssl=False, root_certificates=None, private_key=None, certificate_chain=None, creds=None, keepalive_options=None, channel_args=None, ): super().__init__() # Explicitly check "is not None" here to support passing an empty # list to specify setting no channel arguments. if channel_args is not None: channel_opt = channel_args else: # Use GRPC KeepAlive client defaults if unspecified if not keepalive_options: keepalive_options = KeepAliveOptions() # To specify custom channel_opt, see the channel_args parameter. channel_opt = [ ("grpc.max_send_message_length", MAX_GRPC_MESSAGE_SIZE), ("grpc.max_receive_message_length", MAX_GRPC_MESSAGE_SIZE), ("grpc.keepalive_time_ms", keepalive_options.keepalive_time_ms), ("grpc.keepalive_timeout_ms", keepalive_options.keepalive_timeout_ms), ( "grpc.keepalive_permit_without_calls", keepalive_options.keepalive_permit_without_calls, ), ( "grpc.http2.max_pings_without_data", keepalive_options.http2_max_pings_without_data, ), ] if creds: self._channel = grpc.aio.secure_channel(url, creds, options=channel_opt) elif ssl: rc_bytes = pk_bytes = cc_bytes = None if root_certificates is not None: with open(root_certificates, "rb") as rc_fs: rc_bytes = rc_fs.read() if private_key is not None: with open(private_key, "rb") as pk_fs: pk_bytes = pk_fs.read() if certificate_chain is not None: with open(certificate_chain, "rb") as cc_fs: cc_bytes = cc_fs.read() creds = grpc.ssl_channel_credentials( root_certificates=rc_bytes, private_key=pk_bytes, certificate_chain=cc_bytes, ) self._channel = grpc.aio.secure_channel(url, creds, options=channel_opt) else: self._channel = grpc.aio.insecure_channel(url, options=channel_opt) self._client_stub = service_pb2_grpc.GRPCInferenceServiceStub(self._channel) self._verbose = verbose def _return_response(self, response, as_json): if as_json: return json.loads(MessageToJson(response, preserving_proto_field_name=True)) else: return response async def __aenter__(self): return self async def __aexit__(self, type, value, traceback): await self.close() async def close(self): """Close the client. Any future calls to server will result in an Error. """ await self._channel.close() def _get_metadata(self, headers): request = Request(headers) self._call_plugin(request) request_metadata = ( request.headers.items() if request.headers is not None else () ) return request_metadata async def is_server_live(self, headers=None, client_timeout=None): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.is_server_live`""" metadata = self._get_metadata(headers) try: request = service_pb2.ServerLiveRequest() if self._verbose: print("is_server_live, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.ServerLive( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return response.live except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def is_server_ready(self, headers=None, client_timeout=None): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.is_server_ready`""" metadata = self._get_metadata(headers) try: request = service_pb2.ServerReadyRequest() if self._verbose: print("is_server_ready, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.ServerReady( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return response.ready except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def is_model_ready( self, model_name, model_version="", headers=None, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.is_model_ready`""" metadata = self._get_metadata(headers) try: if type(model_version) != str: raise_error("model version must be a string") request = service_pb2.ModelReadyRequest( name=model_name, version=model_version ) if self._verbose: print("is_model_ready, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.ModelReady( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return response.ready except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_server_metadata( self, headers=None, as_json=False, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.get_server_metadata`""" metadata = self._get_metadata(headers) try: request = service_pb2.ServerMetadataRequest() if self._verbose: print("get_server_metadata, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.ServerMetadata( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_model_metadata( self, model_name, model_version="", headers=None, as_json=False, client_timeout=None, ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.get_model_metadata`""" metadata = self._get_metadata(headers) try: if type(model_version) != str: raise_error("model version must be a string") request = service_pb2.ModelMetadataRequest( name=model_name, version=model_version ) if self._verbose: print("get_model_metadata, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.ModelMetadata( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_model_config( self, model_name, model_version="", headers=None, as_json=False, client_timeout=None, ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.get_model_config`""" metadata = self._get_metadata(headers) try: if type(model_version) != str: raise_error("model version must be a string") request = service_pb2.ModelConfigRequest( name=model_name, version=model_version ) if self._verbose: print("get_model_config, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.ModelConfig( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_model_repository_index( self, headers=None, as_json=False, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.get_model_repository_index`""" metadata = self._get_metadata(headers) try: request = service_pb2.RepositoryIndexRequest() if self._verbose: print( "get_model_repository_index, metadata {}\n{}".format( metadata, request ) ) response = await self._client_stub.RepositoryIndex( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def load_model( self, model_name, headers=None, config=None, files=None, client_timeout=None, ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.load_model`""" metadata = self._get_metadata(headers) try: request = service_pb2.RepositoryModelLoadRequest(model_name=model_name) if config is not None: request.parameters["config"].string_param = config if self._verbose: # Don't print file content which can be large print( "load_model, metadata {}\noverride files omitted:\n{}".format( metadata, request ) ) if files is not None: for path, content in files.items(): request.parameters[path].bytes_param = content await self._client_stub.RepositoryModelLoad( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print("Loaded model '{}'".format(model_name)) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def unload_model( self, model_name, headers=None, unload_dependents=False, client_timeout=None, ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.unload_model`""" metadata = self._get_metadata(headers) try: request = service_pb2.RepositoryModelUnloadRequest(model_name=model_name) request.parameters["unload_dependents"].bool_param = unload_dependents if self._verbose: print("unload_model, metadata {}\n{}".format(metadata, request)) await self._client_stub.RepositoryModelUnload( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print("Unloaded model '{}'".format(model_name)) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_inference_statistics( self, model_name="", model_version="", headers=None, as_json=False, client_timeout=None, ): """Refer to ::py:meth:`tritonclient.grpc.InferenceServerClient.get_inference_statistics`""" metadata = self._get_metadata(headers) try: if type(model_version) != str: raise_error("model version must be a string") request = service_pb2.ModelStatisticsRequest( name=model_name, version=model_version ) if self._verbose: print( "get_inference_statistics, metadata {}\n{}".format( metadata, request ) ) response = await self._client_stub.ModelStatistics( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def update_trace_settings( self, model_name=None, settings={}, headers=None, as_json=False, client_timeout=None, ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.update_trace_settings`""" metadata = self._get_metadata(headers) try: request = service_pb2.TraceSettingRequest() if (model_name is not None) and (model_name != ""): request.model_name = model_name for key, value in settings.items(): if value is None: request.settings[key] else: request.settings[key].value.extend( value if isinstance(value, list) else [value] ) if self._verbose: print( "update_trace_settings, metadata {}\n{}".format(metadata, request) ) response = await self._client_stub.TraceSetting( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_trace_settings( self, model_name=None, headers=None, as_json=False, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.get_trace_settings`""" metadata = self._get_metadata(headers) try: request = service_pb2.TraceSettingRequest() if (model_name is not None) and (model_name != ""): request.model_name = model_name if self._verbose: print("get_trace_settings, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.TraceSetting( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def update_log_settings( self, settings, headers=None, as_json=False, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.update_log_settings`""" metadata = self._get_metadata(headers) try: request = service_pb2.LogSettingsRequest() for key, value in settings.items(): if value is None: request.settings[key] else: if key == "log_file" or key == "log_format": request.settings[key].string_param = value elif key == "log_verbose_level": request.settings[key].uint32_param = value else: request.settings[key].bool_param = value if self._verbose: print("update_log_settings, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.LogSettings( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_log_settings(self, headers=None, as_json=False, client_timeout=None): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.get_log_settings`""" metadata = self._get_metadata(headers) try: request = service_pb2.LogSettingsRequest() if self._verbose: print("get_log_settings, metadata {}\n{}".format(metadata, request)) response = await self._client_stub.LogSettings( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_system_shared_memory_status( self, region_name="", headers=None, as_json=False, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.get_system_shared_memory_status`""" metadata = self._get_metadata(headers) try: request = service_pb2.SystemSharedMemoryStatusRequest(name=region_name) if self._verbose: print( "get_system_shared_memory_status, metadata {}\n{}".format( metadata, request ) ) response = await self._client_stub.SystemSharedMemoryStatus( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def register_system_shared_memory( self, name, key, byte_size, offset=0, headers=None, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.register_system_shared_memory`""" metadata = self._get_metadata(headers) try: request = service_pb2.SystemSharedMemoryRegisterRequest( name=name, key=key, offset=offset, byte_size=byte_size ) if self._verbose: print( "register_system_shared_memory, metadata {}\n{}".format( metadata, request ) ) await self._client_stub.SystemSharedMemoryRegister( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print("Registered system shared memory with name '{}'".format(name)) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def unregister_system_shared_memory( self, name="", headers=None, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.unregister_system_shared_memory`""" metadata = self._get_metadata(headers) try: request = service_pb2.SystemSharedMemoryUnregisterRequest(name=name) if self._verbose: print( "unregister_system_shared_memory, metadata {}\n{}".format( metadata, request ) ) await self._client_stub.SystemSharedMemoryUnregister( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: if name != "": print( "Unregistered system shared memory with name '{}'".format(name) ) else: print("Unregistered all system shared memory regions") except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def get_cuda_shared_memory_status( self, region_name="", headers=None, as_json=False, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.get_cuda_shared_memory_status`""" metadata = self._get_metadata(headers) try: request = service_pb2.CudaSharedMemoryStatusRequest(name=region_name) if self._verbose: print( "get_cuda_shared_memory_status, metadata {}\n{}".format( metadata, request ) ) response = await self._client_stub.CudaSharedMemoryStatus( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print(response) return self._return_response(response, as_json) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def register_cuda_shared_memory( self, name, raw_handle, device_id, byte_size, headers=None, client_timeout=None, ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.register_cuda_shared_memory`""" metadata = self._get_metadata(headers) try: request = service_pb2.CudaSharedMemoryRegisterRequest( name=name, raw_handle=base64.b64decode(raw_handle), device_id=device_id, byte_size=byte_size, ) if self._verbose: print( "register_cuda_shared_memory, metadata {}\n{}".format( metadata, request ) ) await self._client_stub.CudaSharedMemoryRegister( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: print("Registered cuda shared memory with name '{}'".format(name)) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def unregister_cuda_shared_memory( self, name="", headers=None, client_timeout=None ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.unregister_cuda_shared_memory`""" metadata = self._get_metadata(headers) try: request = service_pb2.CudaSharedMemoryUnregisterRequest(name=name) if self._verbose: print( "unregister_cuda_shared_memory, metadata {}\n{}".format( metadata, request ) ) await self._client_stub.CudaSharedMemoryUnregister( request=request, metadata=metadata, timeout=client_timeout ) if self._verbose: if name != "": print("Unregistered cuda shared memory with name '{}'".format(name)) else: print("Unregistered all cuda shared memory regions") except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) async def infer( self, model_name, inputs, model_version="", outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, priority=0, timeout=None, client_timeout=None, headers=None, compression_algorithm=None, parameters=None, ): """Refer to :py:meth:`tritonclient.grpc.InferenceServerClient.infer`""" metadata = self._get_metadata(headers) if type(model_version) != str: raise_error("model version must be a string") request = _get_inference_request( model_name=model_name, inputs=inputs, model_version=model_version, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, parameters=parameters, ) if self._verbose: print("infer, metadata {}\n{}".format(metadata, request)) try: response = await self._client_stub.ModelInfer( request=request, metadata=metadata, timeout=client_timeout, compression=_grpc_compression_type(compression_algorithm), ) if self._verbose: print(response) result = InferResult(response) return result except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) def stream_infer( self, inputs_iterator, stream_timeout=None, headers=None, compression_algorithm=None, ): """Runs an asynchronous inference over gRPC bi-directional streaming API. Parameters ---------- inputs_iterator : asynchronous iterator Async iterator that yields a dict(s) consists of the input parameters to the :py:meth:`tritonclient.grpc.InferenceServerClient.async_stream_infer` function defined in :py:class:`tritonclient.grpc.InferenceServerClient`. stream_timeout : float Optional stream timeout. The stream will be closed once the specified timeout expires. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. compression_algorithm : str Optional grpc compression algorithm to be used on client side. Currently supports "deflate", "gzip" and None. By default, no compression is used. Returns ------- asynchronous iterator Yield tuple holding (:py:class:`tritonclient.grpc.InferResult`, :py:class:`tritonclient.grpc.InferenceServerException`) objects. Note ---- This object can be used to cancel the inference request like below: >>> it = stream_infer(...) >>> ret = it.cancel() Raises ------ :py:class:`tritonclient.grpc.InferenceServerException` If inputs_iterator does not yield the correct input. """ metadata = self._get_metadata(headers) async def _request_iterator(inputs_iterator): # Internal iterator for converting into grpc request async for inputs in inputs_iterator: if type(inputs) != dict: raise_error("inputs_iterator is not yielding a dict") if "model_name" not in inputs or "inputs" not in inputs: raise_error( "model_name and/or inputs is missing from inputs_iterator's yielded dict" ) if "model_version" not in inputs: inputs["model_version"] = "" if type(inputs["model_version"]) != str: raise_error("model_version must be a string") if "outputs" not in inputs: inputs["outputs"] = None if "request_id" not in inputs: inputs["request_id"] = "" if "sequence_id" not in inputs: inputs["sequence_id"] = 0 if "sequence_start" not in inputs: inputs["sequence_start"] = False if "sequence_end" not in inputs: inputs["sequence_end"] = False if "priority" not in inputs: inputs["priority"] = 0 if "timeout" not in inputs: inputs["timeout"] = None if "parameters" not in inputs: inputs["parameters"] = None yield _get_inference_request( model_name=inputs["model_name"], inputs=inputs["inputs"], model_version=inputs["model_version"], request_id=inputs["request_id"], outputs=inputs["outputs"], sequence_id=inputs["sequence_id"], sequence_start=inputs["sequence_start"], sequence_end=inputs["sequence_end"], priority=inputs["priority"], timeout=inputs["timeout"], parameters=inputs["parameters"], ) class _ResponseIterator: def __init__(self, grpc_call, verbose): self._grpc_call = grpc_call self._verbose = verbose def __aiter__(self): return self async def __anext__(self): response = await self._grpc_call.__aiter__().__anext__() if self._verbose: print(response) result = error = None if response.error_message != "": error = InferenceServerException(msg=response.error_message) else: result = InferResult(response.infer_response) return result, error def cancel(self): return self._grpc_call.cancel() try: grpc_call = self._client_stub.ModelStreamInfer( _request_iterator(inputs_iterator), metadata=metadata, timeout=stream_timeout, compression=_grpc_compression_type(compression_algorithm), ) return _ResponseIterator(grpc_call, self._verbose) except grpc.RpcError as rpc_error: raise_error_grpc(rpc_error) kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/aio/auth/000077500000000000000000000000001527052371200260515ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/aio/auth/__init__.py000077500000000000000000000030751527052371200301720ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from ...auth import BasicAuth kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/auth/000077500000000000000000000000001527052371200253015ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/grpc/auth/__init__.py000077500000000000000000000031271527052371200274200ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from ..._auth import BasicAuth __all__ = ["BasicAuth"] kytensor-2.49.0.7/client/src/python/library/tritonclient/http/000077500000000000000000000000001527052371200243645ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/http/__init__.py000077500000000000000000000045401527052371200265030ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. try: from tritonclient.utils import * from .._plugin import InferenceServerClientPlugin from .._request import Request from ._client import InferAsyncRequest, InferenceServerClient from ._infer_input import InferInput from ._infer_result import InferResult from ._requested_output import InferRequestedOutput from ._utils import InferenceServerException except ModuleNotFoundError as error: raise RuntimeError( "The installation does not include http support. Specify 'http' or 'all' while installing the tritonclient package to include the support" ) from error __all__ = [ "InferenceServerClientPlugin", "Request", "InferenceServerClient", "InferInput", "InferRequestedOutput", "InferResult", "InferAsyncRequest", "InferenceServerException", ] kytensor-2.49.0.7/client/src/python/library/tritonclient/http/_client.py000077500000000000000000001624711527052371200263710ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import base64 import gzip import zlib from urllib.parse import quote import gevent import gevent.pool import rapidjson as json from geventhttpclient import HTTPClient from geventhttpclient.url import URL from tritonclient.utils import raise_error from .._client import InferenceServerClientBase from .._request import Request from ._infer_result import InferResult from ._utils import _get_inference_request, _get_query_string, _raise_if_error class InferAsyncRequest: """An object of InferAsyncRequest class is used to describe a handle to an ongoing asynchronous inference request. Parameters ---------- greenlet : gevent.Greenlet The greenlet object which will provide the results. For further details about greenlets refer http://www.gevent.org/api/gevent.greenlet.html. verbose : bool If True generate verbose output. Default value is False. """ def __init__(self, greenlet, verbose=False): self._greenlet = greenlet self._verbose = verbose def get_result(self, block=True, timeout=None): """Get the results of the associated asynchronous inference. Parameters ---------- block : bool If block is True, the function will wait till the corresponding response is received from the server. Default value is True. timeout : int The maximum wait time for the function. This setting is ignored if the block is set False. Default is None, which means the function will block indefinitely till the corresponding response is received. Returns ------- InferResult The object holding the result of the async inference. Raises ------ InferenceServerException If server fails to perform inference or failed to respond within specified timeout. """ try: response = self._greenlet.get(block=block, timeout=timeout) except gevent.Timeout as e: raise_error("failed to obtain inference response") _raise_if_error(response) return InferResult(response, self._verbose) class InferenceServerClient(InferenceServerClientBase): """An InferenceServerClient object is used to perform any kind of communication with the InferenceServer using http protocol. None of the methods are thread safe. The object is intended to be used by a single thread and simultaneously calling different methods with different threads is not supported and will cause undefined behavior. Parameters ---------- url : str The inference server name, port and optional base path in the following format: host:port/, e.g. 'localhost:8000'. verbose : bool If True generate verbose output. Default value is False. concurrency : int The number of connections to create for this client. Default value is 1. connection_timeout : float The timeout value for the connection. Default value is 60.0 sec. network_timeout : float The timeout value for the network. Default value is 60.0 sec max_greenlets : int Determines the maximum allowed number of worker greenlets for handling asynchronous inference requests. Default value is None, which means there will be no restriction on the number of greenlets created. ssl : bool If True, channels the requests to encrypted https scheme. Some improper settings may cause connection to prematurely terminate with an unsuccessful handshake. See `ssl_context_factory` option for using secure default settings. Default value for this option is False. ssl_options : dict Any options supported by `ssl.wrap_socket` specified as dictionary. The argument is ignored if 'ssl' is specified False. ssl_context_factory : SSLContext callable It must be a callbable that returns a SSLContext. Set to `gevent.ssl.create_default_context` to use contexts with secure default settings. This should most likely resolve connection issues in a secure way. The default value for this option is None which directly wraps the socket with the options provided via `ssl_options`. The argument is ignored if 'ssl' is specified False. insecure : bool If True, then does not match the host name with the certificate. Default value is False. The argument is ignored if 'ssl' is specified False. Raises ------ Exception If unable to create a client. """ def __init__( self, url, verbose=False, concurrency=1, connection_timeout=60.0, network_timeout=60.0, max_greenlets=None, ssl=False, ssl_options=None, ssl_context_factory=None, insecure=False, ): super().__init__() if url.startswith("http://") or url.startswith("https://"): raise_error("url should not include the scheme") scheme = "https://" if ssl else "http://" self._parsed_url = URL(scheme + url) self._base_uri = self._parsed_url.request_uri.rstrip("/") self._client_stub = HTTPClient.from_url( self._parsed_url, concurrency=concurrency, connection_timeout=connection_timeout, network_timeout=network_timeout, ssl_options=ssl_options, ssl_context_factory=ssl_context_factory, insecure=insecure, ) self._pool = gevent.pool.Pool(max_greenlets) self._verbose = verbose self._closed = False def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __del__(self): self.close() def close(self): """Close the client. Any future calls to server will result in an Error. """ if not self._closed: self._pool.join() self._client_stub.close() self._closed = True def _get(self, request_uri, headers, query_params): """Issues the GET request to the server Parameters ---------- request_uri: str The request URI to be used in GET request. headers: dict Additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- geventhttpclient.response.HTTPSocketPoolResponse The response from server. """ request = Request(headers) self._call_plugin(request) # Update the headers based on plugin invocation headers = request.headers self._validate_headers(headers) if self._base_uri is not None: request_uri = self._base_uri + "/" + request_uri if query_params is not None: request_uri = request_uri + "?" + _get_query_string(query_params) if self._verbose: print("GET {}, headers {}".format(request_uri, headers)) if headers is not None: response = self._client_stub.get(request_uri, headers=headers) else: response = self._client_stub.get(request_uri) if self._verbose: print(response) return response def _post(self, request_uri, request_body, headers, query_params): """Issues the POST request to the server Parameters ---------- request_uri: str The request URI to be used in POST request. request_body: str The body of the request headers: dict Additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- geventhttpclient.response.HTTPSocketPoolResponse The response from server. """ request = Request(headers) self._call_plugin(request) # Update the headers based on plugin invocation headers = request.headers self._validate_headers(headers) if self._base_uri is not None: request_uri = self._base_uri + "/" + request_uri if query_params is not None: request_uri = request_uri + "?" + _get_query_string(query_params) if self._verbose: print("POST {}, headers {}\n{}".format(request_uri, headers, request_body)) if headers is not None: response = self._client_stub.post( request_uri=request_uri, body=request_body, headers=headers ) else: response = self._client_stub.post( request_uri=request_uri, body=request_body ) if self._verbose: print(response) return response def _validate_headers(self, headers): """Checks for any unsupported HTTP headers before processing a request. Parameters ---------- headers: dict HTTP headers to validate before processing the request. Raises ------ InferenceServerException If an unsupported HTTP header is included in a request. """ if not headers: return # HTTP headers are case-insensitive, so force lowercase for comparison headers_lowercase = {k.lower(): v for k, v in headers.items()} # The python client lirary (and geventhttpclient) do not encode request # data based on "Transfer-Encoding" header, so reject this header if # included. Other libraries may do this encoding under the hood. # The python client library does expose special arguments to support # some "Content-Encoding" headers. if "transfer-encoding" in headers_lowercase: raise_error( "Unsupported HTTP header: 'Transfer-Encoding' is not " "supported in the Python client library. Use raw HTTP " "request libraries or the C++ client instead for this " "header." ) def is_server_live(self, headers=None, query_params=None): """Contact the inference server and get liveness. Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- bool True if server is live, False if server is not live. Raises ------ Exception If unable to get liveness. """ request_uri = "v2/health/live" response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) return response.status_code == 200 def is_server_ready(self, headers=None, query_params=None): """Contact the inference server and get readiness. Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- bool True if server is ready, False if server is not ready. Raises ------ Exception If unable to get readiness. """ request_uri = "v2/health/ready" response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) return response.status_code == 200 def is_model_ready( self, model_name, model_version="", headers=None, query_params=None ): """Contact the inference server and get the readiness of specified model. Parameters ---------- model_name: str The name of the model to check for readiness. model_version: str The version of the model to check for readiness. The default value is an empty string which means then the server will choose a version based on the model and internal policy. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- bool True if the model is ready, False if not ready. Raises ------ Exception If unable to get model readiness. """ if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}/ready".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/ready".format(quote(model_name)) response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) return response.status_code == 200 def get_server_metadata(self, headers=None, query_params=None): """Contact the inference server and get its metadata. Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- dict The JSON dict holding the metadata. Raises ------ InferenceServerException If unable to get server metadata. """ request_uri = "v2" response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def get_model_metadata( self, model_name, model_version="", headers=None, query_params=None ): """Contact the inference server and get the metadata for specified model. Parameters ---------- model_name: str The name of the model model_version: str The version of the model to get metadata. The default value is an empty string which means then the server will choose a version based on the model and internal policy. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding the metadata. Raises ------ InferenceServerException If unable to get model metadata. """ if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}".format(quote(model_name)) response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def get_model_config( self, model_name, model_version="", headers=None, query_params=None ): """Contact the inference server and get the configuration for specified model. Parameters ---------- model_name: str The name of the model model_version: str The version of the model to get configuration. The default value is an empty string which means then the server will choose a version based on the model and internal policy. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding the model config. Raises ------ InferenceServerException If unable to get model configuration. """ if model_version != "": request_uri = "v2/models/{}/versions/{}/config".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/config".format(quote(model_name)) response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def get_model_repository_index(self, headers=None, query_params=None): """Get the index of model repository contents Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding the model repository index. Raises ------ InferenceServerException If unable to get the repository index. """ request_uri = "v2/repository/index" response = self._post( request_uri=request_uri, request_body="", headers=headers, query_params=query_params, ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def load_model( self, model_name, headers=None, query_params=None, config=None, files=None ): """Request the inference server to load or reload specified model. Parameters ---------- model_name : str The name of the model to be loaded. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. config: str Optional JSON representation of a model config provided for the load request, if provided, this config will be used for loading the model. files: dict Optional dictionary specifying file path (with "file:" prefix) in the override model directory to the file content as bytes. The files will form the model directory that the model will be loaded from. If specified, 'config' must be provided to be the model configuration of the override model directory. Raises ------ InferenceServerException If unable to load the model. """ request_uri = "v2/repository/models/{}/load".format(quote(model_name)) load_request = {} if config is not None: if "parameters" not in load_request: load_request["parameters"] = {} load_request["parameters"]["config"] = config if files is not None: for path, content in files.items(): if "parameters" not in load_request: load_request["parameters"] = {} load_request["parameters"][path] = base64.b64encode(content) response = self._post( request_uri=request_uri, request_body=json.dumps(load_request), headers=headers, query_params=query_params, ) _raise_if_error(response) if self._verbose: print("Loaded model '{}'".format(model_name)) def unload_model( self, model_name, headers=None, query_params=None, unload_dependents=False ): """Request the inference server to unload specified model. Parameters ---------- model_name : str The name of the model to be unloaded. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction unload_dependents : bool Whether the dependents of the model should also be unloaded. Raises ------ InferenceServerException If unable to unload the model. """ request_uri = "v2/repository/models/{}/unload".format(quote(model_name)) unload_request = {"parameters": {"unload_dependents": unload_dependents}} response = self._post( request_uri=request_uri, request_body=json.dumps(unload_request), headers=headers, query_params=query_params, ) _raise_if_error(response) if self._verbose: print("Loaded model '{}'".format(model_name)) def get_inference_statistics( self, model_name="", model_version="", headers=None, query_params=None ): """Get the inference statistics for the specified model name and version. Parameters ---------- model_name : str The name of the model to get statistics. The default value is an empty string, which means statistics of all models will be returned. model_version: str The version of the model to get inference statistics. The default value is an empty string which means then the server will return the statistics of all available model versions. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding the model inference statistics. Raises ------ InferenceServerException If unable to get the model inference statistics. """ if model_name != "": if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}/stats".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/stats".format(quote(model_name)) else: request_uri = "v2/models/stats" response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def update_trace_settings( self, model_name=None, settings={}, headers=None, query_params=None ): """Update the trace settings for the specified model name, or global trace settings if model name is not given. Returns the trace settings after the update. Parameters ---------- model_name : str The name of the model to update trace settings. Specifying None or empty string will update the global trace settings. The default value is None. settings: dict The new trace setting values. Only the settings listed will be updated. If a trace setting is listed in the dictionary with a value of 'None', that setting will be cleared. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding the updated trace settings. Raises ------ InferenceServerException If unable to update the trace settings. """ if (model_name is not None) and (model_name != ""): request_uri = "v2/models/{}/trace/setting".format(quote(model_name)) else: request_uri = "v2/trace/setting" response = self._post( request_uri=request_uri, request_body=json.dumps(settings), headers=headers, query_params=query_params, ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def get_trace_settings(self, model_name=None, headers=None, query_params=None): """Get the trace settings for the specified model name, or global trace settings if model name is not given Parameters ---------- model_name : str The name of the model to get trace settings. Specifying None or empty string will return the global trace settings. The default value is None. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding the trace settings. Raises ------ InferenceServerException If unable to get the trace settings. """ if (model_name is not None) and (model_name != ""): request_uri = "v2/models/{}/trace/setting".format(quote(model_name)) else: request_uri = "v2/trace/setting" response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def update_log_settings(self, settings, headers=None, query_params=None): """Update the global log settings of the Triton server. Parameters ---------- settings: dict The new log setting values. Only the settings listed will be updated. headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding the updated log settings. Raises ------ InferenceServerException If unable to update the log settings. """ request_uri = "v2/logging" response = self._post( request_uri=request_uri, request_body=json.dumps(settings), headers=headers, query_params=query_params, ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def get_log_settings(self, headers=None, query_params=None): """Get the global log settings for the Triton server Parameters ---------- headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding the log settings. Raises ------ InferenceServerException If unable to get the log settings. """ request_uri = "v2/logging" response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def get_system_shared_memory_status( self, region_name="", headers=None, query_params=None ): """Request system shared memory status from the server. Parameters ---------- region_name : str The name of the region to query status. The default value is an empty string, which means that the status of all active system shared memory will be returned. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding system shared memory status. Raises ------ InferenceServerException If unable to get the status of specified shared memory. """ if region_name != "": request_uri = "v2/systemsharedmemory/region/{}/status".format( quote(region_name) ) else: request_uri = "v2/systemsharedmemory/status" response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def register_system_shared_memory( self, name, key, byte_size, offset=0, headers=None, query_params=None ): """Request the server to register a system shared memory with the following specification. Parameters ---------- name : str The name of the region to register. key : str The key of the underlying memory object that contains the system shared memory region. byte_size : int The size of the system shared memory region, in bytes. offset : int Offset, in bytes, within the underlying memory object to the start of the system shared memory region. The default value is zero. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Raises ------ InferenceServerException If unable to register the specified system shared memory. """ request_uri = "v2/systemsharedmemory/region/{}/register".format(quote(name)) register_request = {"key": key, "offset": offset, "byte_size": byte_size} request_body = json.dumps(register_request) response = self._post( request_uri=request_uri, request_body=request_body, headers=headers, query_params=query_params, ) _raise_if_error(response) if self._verbose: print("Registered system shared memory with name '{}'".format(name)) def unregister_system_shared_memory(self, name="", headers=None, query_params=None): """Request the server to unregister a system shared memory with the specified name. Parameters ---------- name : str The name of the region to unregister. The default value is empty string which means all the system shared memory regions will be unregistered. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Raises ------ InferenceServerException If unable to unregister the specified system shared memory region. """ if name != "": request_uri = "v2/systemsharedmemory/region/{}/unregister".format( quote(name) ) else: request_uri = "v2/systemsharedmemory/unregister" response = self._post( request_uri=request_uri, request_body="", headers=headers, query_params=query_params, ) _raise_if_error(response) if self._verbose: if name != "": print("Unregistered system shared memory with name '{}'".format(name)) else: print("Unregistered all system shared memory regions") def get_cuda_shared_memory_status( self, region_name="", headers=None, query_params=None ): """Request cuda shared memory status from the server. Parameters ---------- region_name : str The name of the region to query status. The default value is an empty string, which means that the status of all active cuda shared memory will be returned. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Returns ------- dict The JSON dict holding cuda shared memory status. Raises ------ InferenceServerException If unable to get the status of specified shared memory. """ if region_name != "": request_uri = "v2/cudasharedmemory/region/{}/status".format( quote(region_name) ) else: request_uri = "v2/cudasharedmemory/status" response = self._get( request_uri=request_uri, headers=headers, query_params=query_params ) _raise_if_error(response) content = response.read() if self._verbose: print(content) return json.loads(content) def register_cuda_shared_memory( self, name, raw_handle, device_id, byte_size, headers=None, query_params=None ): """Request the server to register a system shared memory with the following specification. Parameters ---------- name : str The name of the region to register. raw_handle : bytes The raw serialized cudaIPC handle in base64 encoding. device_id : int The GPU device ID on which the cudaIPC handle was created. byte_size : int The size of the cuda shared memory region, in bytes. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Raises ------ InferenceServerException If unable to register the specified cuda shared memory. """ request_uri = "v2/cudasharedmemory/region/{}/register".format(quote(name)) register_request = { "raw_handle": {"b64": raw_handle}, "device_id": device_id, "byte_size": byte_size, } request_body = json.dumps(register_request) response = self._post( request_uri=request_uri, request_body=request_body, headers=headers, query_params=query_params, ) _raise_if_error(response) if self._verbose: print("Registered cuda shared memory with name '{}'".format(name)) def unregister_cuda_shared_memory(self, name="", headers=None, query_params=None): """Request the server to unregister a cuda shared memory with the specified name. Parameters ---------- name : str The name of the region to unregister. The default value is empty string which means all the cuda shared memory regions will be unregistered. headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction Raises ------ InferenceServerException If unable to unregister the specified cuda shared memory region. """ if name != "": request_uri = "v2/cudasharedmemory/region/{}/unregister".format(quote(name)) else: request_uri = "v2/cudasharedmemory/unregister" response = self._post( request_uri=request_uri, request_body="", headers=headers, query_params=query_params, ) _raise_if_error(response) if self._verbose: if name != "": print("Unregistered cuda shared memory with name '{}'".format(name)) else: print("Unregistered all cuda shared memory regions") @staticmethod def generate_request_body( inputs, outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, priority=0, timeout=None, parameters=None, ): """Generate a request body for inference using the supplied 'inputs' requesting the outputs specified by 'outputs'. Parameters ---------- inputs : list A list of :py:class:`InferInput` objects, each describing data for a input tensor required by the model. outputs : list A list of :py:class:`InferRequestedOutput` objects, each describing how the output data must be returned. If not specified all outputs produced by the model will be returned using default settings. request_id: str Optional identifier for the request. If specified will be returned in the response. Default value is an empty string which means no request_id will be used. sequence_id : int or str The unique identifier for the sequence being represented by the object. A value of 0 or "" means that the request does not belong to a sequence. Default is 0. sequence_start: bool Indicates whether the request being added marks the start of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. sequence_end: bool Indicates whether the request being added marks the end of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. priority : int Indicates the priority of the request. Priority value zero indicates that the default priority level should be used (i.e. same behavior as not specifying the priority parameter). Lower value priorities indicate higher priority levels. Thus the highest priority level is indicated by setting the parameter to 1, the next highest is 2, etc. If not provided, the server will handle the request using default setting for the model. timeout : int The timeout value for the request, in microseconds. If the request cannot be completed within the time the server can take a model-specific action such as terminating the request. If not provided, the server will handle the request using default setting for the model. This option is only respected by the model that is configured with dynamic batching. See here for more details: https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher parameters: dict Optional fields to be included in the 'parameters' fields. Returns ------- Bytes The request body of the inference. Int The byte size of the inference request header in the request body. Returns None if the whole request body constitutes the request header. Raises ------ InferenceServerException If server fails to perform inference. """ return _get_inference_request( inputs=inputs, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, custom_parameters=parameters, ) @staticmethod def parse_response_body( response_body, verbose=False, header_length=None, content_encoding=None ): """Generate a :py:class:`InferResult` object from the given 'response_body' Parameters ---------- response_body : bytes The inference response from the server verbose : bool If True generate verbose output. Default value is False. header_length : int The length of the inference header if the header does not occupy the whole response body. Default value is None. content_encoding : string The encoding of the response body if it is compressed. Default value is None. Returns ------- InferResult The InferResult object generated from the response body """ return InferResult.from_response_body( response_body, verbose, header_length, content_encoding ) def infer( self, model_name, inputs, model_version="", outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, priority=0, timeout=None, headers=None, query_params=None, request_compression_algorithm=None, response_compression_algorithm=None, parameters=None, ): """Run synchronous inference using the supplied 'inputs' requesting the outputs specified by 'outputs'. Parameters ---------- model_name: str The name of the model to run inference. inputs : list A list of :py:class:`InferInput` objects, each describing data for a input tensor required by the model. model_version: str The version of the model to run inference. The default value is an empty string which means then the server will choose a version based on the model and internal policy. outputs : list A list of :py:class:`InferRequestedOutput` objects, each describing how the output data must be returned. If not specified all outputs produced by the model will be returned using default settings. request_id: str Optional identifier for the request. If specified will be returned in the response. Default value is an empty string which means no request_id will be used. sequence_id : int The unique identifier for the sequence being represented by the object. Default value is 0 which means that the request does not belong to a sequence. sequence_start: bool Indicates whether the request being added marks the start of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. sequence_end: bool Indicates whether the request being added marks the end of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. priority : int Indicates the priority of the request. Priority value zero indicates that the default priority level should be used (i.e. same behavior as not specifying the priority parameter). Lower value priorities indicate higher priority levels. Thus the highest priority level is indicated by setting the parameter to 1, the next highest is 2, etc. If not provided, the server will handle the request using default setting for the model. timeout : int The timeout value for the request, in microseconds. If the request cannot be completed within the time the server can take a model-specific action such as terminating the request. If not provided, the server will handle the request using default setting for the model. This option is only respected by the model that is configured with dynamic batching. See here for more details: https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher headers: dict Optional dictionary specifying additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. request_compression_algorithm : str Optional HTTP compression algorithm to use for the request body on client side. Currently supports "deflate", "gzip" and None. By default, no compression is used. response_compression_algorithm : str Optional HTTP compression algorithm to request for the response body. Note that the response may not be compressed if the server does not support the specified algorithm. Currently supports "deflate", "gzip" and None. By default, no compression is requested. parameters: dict Optional fields to be included in the 'parameters' fields. Returns ------- InferResult The object holding the result of the inference. Raises ------ InferenceServerException If server fails to perform inference. """ request_body, json_size = _get_inference_request( inputs=inputs, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, custom_parameters=parameters, ) if request_compression_algorithm == "gzip": if headers is None: headers = {} headers["Content-Encoding"] = "gzip" request_body = gzip.compress(request_body) elif request_compression_algorithm == "deflate": if headers is None: headers = {} headers["Content-Encoding"] = "deflate" # "Content-Encoding: deflate" actually means compressing in zlib structure # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding request_body = zlib.compress(request_body) if response_compression_algorithm == "gzip": if headers is None: headers = {} headers["Accept-Encoding"] = "gzip" elif response_compression_algorithm == "deflate": if headers is None: headers = {} headers["Accept-Encoding"] = "deflate" if json_size is not None: if headers is None: headers = {} headers["Inference-Header-Content-Length"] = json_size if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}/infer".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/infer".format(quote(model_name)) response = self._post( request_uri=request_uri, request_body=request_body, headers=headers, query_params=query_params, ) _raise_if_error(response) return InferResult(response, self._verbose) def async_infer( self, model_name, inputs, model_version="", outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, priority=0, timeout=None, headers=None, query_params=None, request_compression_algorithm=None, response_compression_algorithm=None, parameters=None, ): """Run asynchronous inference using the supplied 'inputs' requesting the outputs specified by 'outputs'. Even though this call is non-blocking, however, the actual number of concurrent requests to the server will be limited by the 'concurrency' parameter specified while creating this client. In other words, if the inflight async_infer exceeds the specified 'concurrency', the delivery of the exceeding request(s) to server will be blocked till the slot is made available by retrieving the results of previously issued requests. Parameters ---------- model_name: str The name of the model to run inference. inputs : list A list of :py:class:`InferInput` objects, each describing data for a input tensor required by the model. model_version: str The version of the model to run inference. The default value is an empty string which means then the server will choose a version based on the model and internal policy. outputs : list A list of :py:class:`InferRequestedOutput` objects, each describing how the output data must be returned. If not specified all outputs produced by the model will be returned using default settings. request_id: str Optional identifier for the request. If specified will be returned in the response. Default value is 'None' which means no request_id will be used. sequence_id : int The unique identifier for the sequence being represented by the object. Default value is 0 which means that the request does not belong to a sequence. sequence_start: bool Indicates whether the request being added marks the start of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. sequence_end: bool Indicates whether the request being added marks the end of the sequence. Default value is False. This argument is ignored if 'sequence_id' is 0. priority : int Indicates the priority of the request. Priority value zero indicates that the default priority level should be used (i.e. same behavior as not specifying the priority parameter). Lower value priorities indicate higher priority levels. Thus the highest priority level is indicated by setting the parameter to 1, the next highest is 2, etc. If not provided, the server will handle the request using default setting for the model. timeout : int The timeout value for the request, in microseconds. If the request cannot be completed within the time the server can take a model-specific action such as terminating the request. If not provided, the server will handle the request using default setting for the model. This option is only respected by the model that is configured with dynamic batching. See here for more details: https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher headers: dict Optional dictionary specifying additional HTTP headers to include in the request query_params: dict Optional url query parameters to use in network transaction. request_compression_algorithm : str Optional HTTP compression algorithm to use for the request body on client side. Currently supports "deflate", "gzip" and None. By default, no compression is used. response_compression_algorithm : str Optional HTTP compression algorithm to request for the response body. Note that the response may not be compressed if the server does not support the specified algorithm. Currently supports "deflate", "gzip" and None. By default, no compression is requested. parameters : dict Optional custom parameters to be included in the inference request. Returns ------- InferAsyncRequest The handle to the asynchronous inference request. Raises ------ InferenceServerException If server fails to issue inference. """ def wrapped_post(request_uri, request_body, headers, query_params): return self._post(request_uri, request_body, headers, query_params) request_body, json_size = _get_inference_request( inputs=inputs, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, custom_parameters=parameters, ) if request_compression_algorithm == "gzip": if headers is None: headers = {} headers["Content-Encoding"] = "gzip" request_body = gzip.compress(request_body) elif request_compression_algorithm == "deflate": if headers is None: headers = {} headers["Content-Encoding"] = "deflate" # "Content-Encoding: deflate" actually means compressing in zlib structure # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding request_body = zlib.compress(request_body) if response_compression_algorithm == "gzip": if headers is None: headers = {} headers["Accept-Encoding"] = "gzip" elif response_compression_algorithm == "deflate": if headers is None: headers = {} headers["Accept-Encoding"] = "deflate" if json_size is not None: if headers is None: headers = {} headers["Inference-Header-Content-Length"] = json_size if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}/infer".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/infer".format(quote(model_name)) g = self._pool.apply_async( wrapped_post, (request_uri, request_body, headers, query_params) ) # Schedule the greenlet to run in this loop iteration g.start() # Relinquish control to greenlet loop. Using non-zero # value to ensure the control is transferred to the # event loop. gevent.sleep(0.01) if self._verbose: verbose_message = "Sent request" if request_id != "": verbose_message = verbose_message + " '{}'".format(request_id) print(verbose_message) return InferAsyncRequest(g, self._verbose) kytensor-2.49.0.7/client/src/python/library/tritonclient/http/_infer_input.py000077500000000000000000000231151527052371200274240ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import numpy as np from tritonclient.utils import ( np_to_triton_dtype, raise_error, serialize_bf16_tensor, serialize_byte_tensor, triton_to_np_dtype, ) class InferInput: """An object of InferInput class is used to describe input tensor for an inference request. Parameters ---------- name : str The name of input whose data will be described by this object shape : list The shape of the associated input. datatype : str The datatype of the associated input. """ def __init__(self, name, shape, datatype): self._name = name self._shape = shape self._datatype = datatype self._parameters = {} self._data = None self._raw_data = None def name(self): """Get the name of input associated with this object. Returns ------- str The name of input """ return self._name def datatype(self): """Get the datatype of input associated with this object. Returns ------- str The datatype of input """ return self._datatype def shape(self): """Get the shape of input associated with this object. Returns ------- list The shape of input """ return self._shape def set_shape(self, shape): """Set the shape of input. Parameters ---------- shape : list The shape of the associated input. Returns ------- InferInput The updated input """ self._shape = shape return self def set_data_from_numpy(self, input_tensor, binary_data=True): """Set the tensor data from the specified numpy array for input associated with this object. Parameters ---------- input_tensor : numpy array The tensor data in numpy array format binary_data : bool Indicates whether to set data for the input in binary format or explicit tensor within JSON. The default value is True, which means the data will be delivered as binary data in the HTTP body after the JSON object. Returns ------- InferInput The updated input Raises ------ InferenceServerException If failed to set data for the tensor. """ if not isinstance(input_tensor, (np.ndarray,)): raise_error("input_tensor must be a numpy array") # DLIS-3986: Special handling for bfloat16 until Numpy officially supports it if self._datatype == "BF16": if input_tensor.dtype != triton_to_np_dtype(self._datatype): raise_error( "got unexpected datatype {} from numpy array, expected {} for BF16 type".format( input_tensor.dtype, triton_to_np_dtype(self._datatype) ) ) else: dtype = np_to_triton_dtype(input_tensor.dtype) if self._datatype != dtype: raise_error( "got unexpected datatype {} from numpy array, expected {}".format( dtype, self._datatype ) ) valid_shape = True if len(self._shape) != len(input_tensor.shape): valid_shape = False else: for i in range(len(self._shape)): if self._shape[i] != input_tensor.shape[i]: valid_shape = False if not valid_shape: raise_error( "got unexpected numpy array shape [{}], expected [{}]".format( str(input_tensor.shape)[1:-1], str(self._shape)[1:-1] ) ) self._parameters.pop("shared_memory_region", None) self._parameters.pop("shared_memory_byte_size", None) self._parameters.pop("shared_memory_offset", None) if not binary_data: self._parameters.pop("binary_data_size", None) self._raw_data = None if self._datatype == "BF16": raise_error( "BF16 inputs must be sent as binary data over HTTP. Please set binary_data=True" ) if self._datatype == "BYTES": self._data = [] try: if input_tensor.size > 0: for obj in np.nditer( input_tensor, flags=["refs_ok"], order="C" ): # We need to convert the object to string using utf-8, # if we want to use the binary_data=False. JSON requires # the input to be a UTF-8 string. if input_tensor.dtype == np.object_: if type(obj.item()) == bytes: self._data.append(str(obj.item(), encoding="utf-8")) else: self._data.append(str(obj.item())) else: self._data.append(str(obj.item(), encoding="utf-8")) except UnicodeDecodeError: raise_error( f'Failed to encode "{obj.item()}" using UTF-8. Please use binary_data=True, if' " you want to pass a byte array." ) else: self._data = [val.item() for val in input_tensor.flatten()] else: self._data = None if self._datatype == "BYTES": serialized_output = serialize_byte_tensor(input_tensor) if serialized_output.size > 0: self._raw_data = serialized_output.item() else: self._raw_data = b"" elif self._datatype == "BF16": serialized_output = serialize_bf16_tensor(input_tensor) if serialized_output.size > 0: self._raw_data = serialized_output.item() else: self._raw_data = b"" else: self._raw_data = input_tensor.tobytes() self._parameters["binary_data_size"] = len(self._raw_data) return self def set_shared_memory(self, region_name, byte_size, offset=0): """Set the tensor data from the specified shared memory region. Parameters ---------- region_name : str The name of the shared memory region holding tensor data. byte_size : int The size of the shared memory region holding tensor data. offset : int The offset, in bytes, into the region where the data for the tensor starts. The default value is 0. Returns ------- InferInput The updated input """ self._data = None self._raw_data = None self._parameters.pop("binary_data_size", None) self._parameters["shared_memory_region"] = region_name self._parameters["shared_memory_byte_size"] = byte_size if offset != 0: self._parameters["shared_memory_offset"] = offset return self def _get_binary_data(self): """Returns the raw binary data if available Returns ------- bytes The raw data for the input tensor """ return self._raw_data def _get_tensor(self): """Retrieve the underlying input as json dict. Returns ------- dict The underlying tensor specification as dict """ tensor = {"name": self._name, "shape": self._shape, "datatype": self._datatype} if self._parameters: tensor["parameters"] = self._parameters if ( self._parameters.get("shared_memory_region") is None and self._raw_data is None ): if self._data is not None: tensor["data"] = self._data return tensor kytensor-2.49.0.7/client/src/python/library/tritonclient/http/_infer_result.py000077500000000000000000000232631527052371200276070ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import gzip import zlib import numpy as np import rapidjson as json from tritonclient.utils import ( deserialize_bf16_tensor, deserialize_bytes_tensor, raise_error, triton_to_np_dtype, ) class InferResult: """An object of :py:class:`InferResult` class holds the response of an inference request and provide methods to retrieve inference results. Parameters ---------- response : geventhttpclient.response.HTTPSocketPoolResponse The inference response from the server verbose : bool If True generate verbose output. Default value is False. """ def __init__(self, response, verbose): header_length = response.get("Inference-Header-Content-Length") # Internal class that simulate the interface of 'response' class DecompressedResponse: def __init__(self, decompressed_data): self.decompressed_data_ = decompressed_data self.offset_ = 0 def read(self, length=-1): if length == -1: return self.decompressed_data_[self.offset_ :] else: prev_offset = self.offset_ self.offset_ += length return self.decompressed_data_[prev_offset : self.offset_] content_encoding = response.get("Content-Encoding") if content_encoding is not None: if content_encoding == "gzip": response = DecompressedResponse(gzip.decompress(response.read())) elif content_encoding == "deflate": response = DecompressedResponse(zlib.decompress(response.read())) if header_length is None: content = response.read() if verbose: print(content) try: self._result = json.loads(content) except UnicodeDecodeError as e: raise_error( f"Failed to encode using UTF-8. Please use binary_data=True, if" f" you want to pass a byte array. UnicodeError: {e}" ) else: header_length = int(header_length) content = response.read(length=header_length) if verbose: print(content) self._result = json.loads(content) # Maps the output name to the index in buffer for quick retrieval self._output_name_to_buffer_map = {} # Read the remaining data off the response body. self._buffer = response.read() buffer_index = 0 for output in self._result["outputs"]: parameters = output.get("parameters") if parameters is not None: this_data_size = parameters.get("binary_data_size") if this_data_size is not None: self._output_name_to_buffer_map[output["name"]] = buffer_index buffer_index = buffer_index + this_data_size @classmethod def from_response_body( cls, response_body, verbose=False, header_length=None, content_encoding=None ): """A class method to construct :py:class:`InferResult` object from a given 'response_body'. Parameters ---------- response_body : bytes The inference response from the server verbose : bool If True generate verbose output. Default value is False. header_length : int The length of the inference header if the header does not occupy the whole response body. Default value is None. content_encoding : string The encoding of the response body if it is compressed. Default value is None. Returns ------- InferResult The InferResult object generated from the response body """ # Internal class that simulate the interface of 'response' class Response: def __init__(self, response_body, header_length, content_encoding): self.response_body_ = response_body self.offset_ = 0 self.parameters_ = { "Inference-Header-Content-Length": header_length, "Content-Encoding": content_encoding, } def get(self, key): return self.parameters_.get(key) def read(self, length=-1): if length == -1: return self.response_body_[self.offset_ :] else: prev_offset = self.offset_ self.offset_ += length return self.response_body_[prev_offset : self.offset_] return cls(Response(response_body, header_length, content_encoding), verbose) def as_numpy(self, name): """Get the tensor data for output associated with this object in numpy format Parameters ---------- name : str The name of the output tensor whose result is to be retrieved. Returns ------- numpy array The numpy array containing the response data for the tensor or None if the data for specified tensor name is not found. """ if self._result.get("outputs") is not None: for output in self._result["outputs"]: if output["name"] == name: datatype = output["datatype"] has_binary_data = False parameters = output.get("parameters") if parameters is not None: this_data_size = parameters.get("binary_data_size") if this_data_size is not None: has_binary_data = True if this_data_size != 0: start_index = self._output_name_to_buffer_map[name] end_index = start_index + this_data_size if datatype == "BYTES": # String results contain a 4-byte string length # followed by the actual string characters. Hence, # need to decode the raw bytes to convert into # array elements. np_array = deserialize_bytes_tensor( self._buffer[start_index:end_index] ) elif datatype == "BF16": np_array = deserialize_bf16_tensor( self._buffer[start_index:end_index] ) else: np_array = np.frombuffer( self._buffer[start_index:end_index], dtype=triton_to_np_dtype(datatype), ) else: np_array = np.empty(0) if not has_binary_data: np_array = np.array( output["data"], dtype=triton_to_np_dtype(datatype) ) np_array = np_array.reshape(output["shape"]) return np_array return None def get_output(self, name): """Retrieves the output tensor corresponding to the named output. Parameters ---------- name : str The name of the tensor for which Output is to be retrieved. Returns ------- Dict If an output tensor with specified name is present in the infer response then returns it as a json dict, otherwise returns None. """ for output in self._result["outputs"]: if output["name"] == name: return output return None def get_response(self): """Retrieves the complete response Returns ------- dict The underlying response dict. """ return self._result kytensor-2.49.0.7/client/src/python/library/tritonclient/http/_requested_output.py000077500000000000000000000112061527052371200305210ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from tritonclient.utils import raise_error class InferRequestedOutput: """An object of :py:class:`InferRequestedOutput` class is used to describe a requested output tensor for an inference request. Parameters ---------- name : str The name of output tensor to associate with this object. binary_data : bool Indicates whether to return result data for the output in binary format or explicit tensor within JSON. The default value is True, which means the data will be delivered as binary data in the HTTP body after JSON object. This field will be unset if shared memory is set for the output. class_count : int The number of classifications to be requested. The default value is 0 which means the classification results are not requested. """ def __init__(self, name, binary_data=True, class_count=0): self._name = name self._parameters = {} if class_count != 0: self._parameters["classification"] = class_count self._binary = binary_data self._parameters["binary_data"] = binary_data def name(self): """Get the name of output associated with this object. Returns ------- str The name of output """ return self._name def set_shared_memory(self, region_name, byte_size, offset=0): """Marks the output to return the inference result in specified shared memory region. Parameters ---------- region_name : str The name of the shared memory region to hold tensor data. byte_size : int The size of the shared memory region to hold tensor data. offset : int The offset, in bytes, into the region where the data for the tensor starts. The default value is 0. """ if "classification" in self._parameters: raise_error("shared memory can't be set on classification output") if self._binary: self._parameters["binary_data"] = False self._parameters["shared_memory_region"] = region_name self._parameters["shared_memory_byte_size"] = byte_size if offset != 0: self._parameters["shared_memory_offset"] = offset def unset_shared_memory(self): """Clears the shared memory option set by the last call to :py:meth:`InferRequestedOutput.set_shared_memory()`. After call to this function requested output will no longer be returned in a shared memory region. """ self._parameters["binary_data"] = self._binary self._parameters.pop("shared_memory_region", None) self._parameters.pop("shared_memory_byte_size", None) self._parameters.pop("shared_memory_offset", None) def _get_tensor(self): """Retrieve the underlying input as json dict. Returns ------- dict The underlying tensor as a dict """ tensor = {"name": self._name} if self._parameters: tensor["parameters"] = self._parameters return tensor kytensor-2.49.0.7/client/src/python/library/tritonclient/http/_utils.py000077500000000000000000000123261527052371200262440ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import struct from urllib.parse import quote_plus import rapidjson as json from tritonclient.utils import InferenceServerException, raise_error def _get_error(response): """ Returns the :py:class:`InferenceServerException` object if response indicates the error. If no error then return None """ if response.status_code != 200: body = None try: body = response.read().decode("utf-8") error_response = ( json.loads(body) if len(body) else {"error": "client received an empty response from the server."} ) return InferenceServerException( msg=error_response["error"], status=str(response.status_code) ) except Exception as e: return InferenceServerException( msg=f"an exception occurred in the client while decoding the response: {e}", status=str(response.status_code), debug_details=body, ) else: return None def _raise_if_error(response): """ Raise :py:class:`InferenceServerException` if received non-Success response from the server """ error = _get_error(response) if error is not None: raise error def _get_query_string(query_params): params = [] for key, value in query_params.items(): if isinstance(value, list): for item in value: params.append("%s=%s" % (quote_plus(key), quote_plus(str(item)))) else: params.append("%s=%s" % (quote_plus(key), quote_plus(str(value)))) if params: return "&".join(params) return "" def _get_inference_request( inputs, request_id, outputs, sequence_id, sequence_start, sequence_end, priority, timeout, custom_parameters, ): infer_request = {} parameters = {} if request_id != "": infer_request["id"] = request_id if sequence_id != 0 and sequence_id != "": parameters["sequence_id"] = sequence_id parameters["sequence_start"] = sequence_start parameters["sequence_end"] = sequence_end if priority != 0: parameters["priority"] = priority if timeout is not None: parameters["timeout"] = timeout infer_request["inputs"] = [this_input._get_tensor() for this_input in inputs] if outputs: infer_request["outputs"] = [ this_output._get_tensor() for this_output in outputs ] else: # no outputs specified so set 'binary_data_output' True in the # request so that all outputs are returned in binary format parameters["binary_data_output"] = True if custom_parameters: for key, value in custom_parameters.items(): if ( key == "sequence_id" or key == "sequence_start" or key == "sequence_end" or key == "priority" or key == "binary_data_output" ): raise_error( f'Parameter "{key}" is a reserved parameter and cannot be specified.' ) else: parameters[key] = value if parameters: infer_request["parameters"] = parameters request_json = json.dumps(infer_request) json_size = len(request_json) request_body = [request_json.encode()] for input_tensor in inputs: raw_data = input_tensor._get_binary_data() if raw_data is not None: request_body.append(raw_data) if len(request_body) == 1: # The request body constitutes the whole request return request_body[0], None else: return b"".join(request_body), json_size kytensor-2.49.0.7/client/src/python/library/tritonclient/http/aio/000077500000000000000000000000001527052371200251345ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/http/aio/__init__.py000077500000000000000000000656731527052371200272710ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. try: import aiohttp except ModuleNotFoundError as error: raise RuntimeError( "The installation does not include http support. Specify 'http' or 'all' while installing the tritonclient package to include the support" ) from error import base64 import gzip import zlib from urllib.parse import quote import rapidjson as json # In case user try to import dependency from here from tritonclient.http import * from tritonclient.http import InferInput, InferRequestedOutput from ..._client import InferenceServerClientBase from ..._plugin import InferenceServerClientPlugin from ..._request import Request from .._infer_result import InferResult from .._utils import _get_inference_request, _get_query_string async def _get_error(response): """ Returns the :py:class:`InferenceServerException` object if response indicates the error. If no error then return None """ if response.status != 200: body = None try: result = await response.read() body = result.decode("utf-8") error_response = ( json.loads(body) if len(body) else {"error": "client received an empty response from the server."} ) return InferenceServerException( msg=error_response["error"], status=str(response.status) ) except Exception as e: return InferenceServerException( msg=f"an exception occurred in the client while decoding the response: {e}", status=str(response.status), debug_details=body, ) else: return None async def _raise_if_error(response): """ Raise :py:class:`InferenceServerException` if received non-Success response from the server """ error = await _get_error(response) if error is not None: raise error class InferenceServerClient(InferenceServerClientBase): """This feature is currently in beta and may be subject to change. An analogy of the :py:class:`tritonclient.http.InferenceServerClient` to enable calling via asyncio syntax. The object is intended to be used by a single thread and simultaneously calling methods with different threads is not supported and can cause undefined behavior. """ def __init__( self, url, verbose=False, conn_limit=100, conn_timeout=60.0, ssl=False, ssl_context=None, ): super().__init__() if url.startswith("http://") or url.startswith("https://"): raise_error("url should not include the scheme") scheme = "https://" if ssl else "http://" self._url = scheme + (url if url[-1] != "/" else url[:-1]) self._conn = aiohttp.TCPConnector(ssl=ssl_context, limit=conn_limit) self._stub = aiohttp.ClientSession( connector=self._conn, timeout=aiohttp.ClientTimeout(total=conn_timeout), auto_decompress=False, ) self._verbose = verbose async def __aenter__(self): return self async def __aexit__(self, type, value, traceback): await self.close() async def close(self): """Close the client. Any future calls to server will result in an Error. """ await self._stub.close() await self._conn.close() async def _get(self, request_uri, headers, query_params): """Issues the GET request to the server Parameters ---------- request_uri: str The request URI to be used in GET request. headers: dict Additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- aiohttp.ClientResponse The response from server. """ request = Request(headers) self._call_plugin(request) # Update the headers based on plugin invocation headers = request.headers self._validate_headers(headers) req_url = self._url + "/" + request_uri if query_params is not None: req_url = req_url + "?" + _get_query_string(query_params) if self._verbose: print("GET {}, headers {}".format(req_url, headers)) res = await self._stub.get(url=req_url, headers=self._fix_header(headers)) if self._verbose: print(res) return res async def _post(self, request_uri, request_body, headers, query_params): """Issues the POST request to the server Parameters ---------- request_uri: str The request URI to be used in POST request. request_body: str The body of the request headers: dict Additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- aiohttp.ClientResponse The response from server. """ request = Request(headers) self._call_plugin(request) # Update the headers based on plugin invocation headers = request.headers self._validate_headers(headers) req_url = self._url + "/" + request_uri if query_params is not None: req_url = req_url + "?" + _get_query_string(query_params) if self._verbose: print("POST {}, headers {}\n{}".format(req_url, headers, request_body)) if isinstance(request_body, str): request_body = request_body.encode() res = await self._stub.post( url=req_url, data=request_body, headers=self._fix_header(headers) ) if self._verbose: print(res) return res def _validate_headers(self, headers): """Checks for any unsupported HTTP headers before processing a request. Parameters ---------- headers: dict HTTP headers to validate before processing the request. Raises ------ InferenceServerException If an unsupported HTTP header is included in a request. """ if not headers: return # HTTP headers are case-insensitive, so force lowercase for comparison headers_lowercase = {k.lower(): v for k, v in headers.items()} # The python client lirary (and geventhttpclient) do not encode request # data based on "Transfer-Encoding" header, so reject this header if # included. Other libraries may do this encoding under the hood. # The python client library does expose special arguments to support # some "Content-Encoding" headers. if "transfer-encoding" in headers_lowercase: raise_error( "Unsupported HTTP header: 'Transfer-Encoding' is not " "supported in the Python client library. Use raw HTTP " "request libraries or the C++ client instead for this " "header." ) def _fix_header(self, headers): """Returns a header that is valid for aiohttp. Parameters ---------- headers: dict (or None) HTTP headers to fix before processing the request. """ if headers is None: return None # Convert the headers to only consist of str type fix_header = {} for key, value in headers.items(): fix_header[str(key)] = str(value) return fix_header async def is_server_live(self, headers=None, query_params=None): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.is_server_live`""" request_uri = "v2/health/live" response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) return response.status == 200 async def is_server_ready(self, headers=None, query_params=None): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.is_server_ready`""" request_uri = "v2/health/ready" response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) return response.status == 200 async def is_model_ready( self, model_name, model_version="", headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.is_model_ready`""" if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}/ready".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/ready".format(quote(model_name)) response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) return response.status == 200 async def get_server_metadata(self, headers=None, query_params=None): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_server_metadata`""" request_uri = "v2" response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def get_model_metadata( self, model_name, model_version="", headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_model_metadata`""" if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}".format(quote(model_name)) response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def get_model_config( self, model_name, model_version="", headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_model_config`""" if model_version != "": request_uri = "v2/models/{}/versions/{}/config".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/config".format(quote(model_name)) response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def get_model_repository_index(self, headers=None, query_params=None): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_model_repository_index`""" request_uri = "v2/repository/index" response = await self._post( request_uri=request_uri, request_body="", headers=headers, query_params=query_params, ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def load_model( self, model_name, headers=None, query_params=None, config=None, files=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.load_model`""" request_uri = "v2/repository/models/{}/load".format(quote(model_name)) load_request = {} if config is not None: if "parameters" not in load_request: load_request["parameters"] = {} load_request["parameters"]["config"] = config if files is not None: for path, content in files.items(): if "parameters" not in load_request: load_request["parameters"] = {} load_request["parameters"][path] = base64.b64encode(content) response = await self._post( request_uri=request_uri, request_body=json.dumps(load_request), headers=headers, query_params=query_params, ) await _raise_if_error(response) if self._verbose: print("Loaded model '{}'".format(model_name)) async def unload_model( self, model_name, headers=None, query_params=None, unload_dependents=False ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.unload_model`""" request_uri = "v2/repository/models/{}/unload".format(quote(model_name)) unload_request = {"parameters": {"unload_dependents": unload_dependents}} response = await self._post( request_uri=request_uri, request_body=json.dumps(unload_request), headers=headers, query_params=query_params, ) await _raise_if_error(response) if self._verbose: print("Loaded model '{}'".format(model_name)) async def get_inference_statistics( self, model_name="", model_version="", headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_inference_statistics`""" if model_name != "": if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}/stats".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/stats".format(quote(model_name)) else: request_uri = "v2/models/stats" response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def update_trace_settings( self, model_name=None, settings={}, headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.update_trace_settings`""" if (model_name is not None) and (model_name != ""): request_uri = "v2/models/{}/trace/setting".format(quote(model_name)) else: request_uri = "v2/trace/setting" response = await self._post( request_uri=request_uri, request_body=json.dumps(settings), headers=headers, query_params=query_params, ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def get_trace_settings( self, model_name=None, headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_trace_settings`""" if (model_name is not None) and (model_name != ""): request_uri = "v2/models/{}/trace/setting".format(quote(model_name)) else: request_uri = "v2/trace/setting" response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def update_log_settings(self, settings, headers=None, query_params=None): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.update_log_settings`""" request_uri = "v2/logging" response = await self._post( request_uri=request_uri, request_body=json.dumps(settings), headers=headers, query_params=query_params, ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def get_log_settings(self, headers=None, query_params=None): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_log_settings`""" request_uri = "v2/logging" response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def get_system_shared_memory_status( self, region_name="", headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_system_shared_memory_status`""" if region_name != "": request_uri = "v2/systemsharedmemory/region/{}/status".format( quote(region_name) ) else: request_uri = "v2/systemsharedmemory/status" response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def register_system_shared_memory( self, name, key, byte_size, offset=0, headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.register_system_shared_memory`""" request_uri = "v2/systemsharedmemory/region/{}/register".format(quote(name)) register_request = {"key": key, "offset": offset, "byte_size": byte_size} request_body = json.dumps(register_request) response = await self._post( request_uri=request_uri, request_body=request_body, headers=headers, query_params=query_params, ) await _raise_if_error(response) if self._verbose: print("Registered system shared memory with name '{}'".format(name)) async def unregister_system_shared_memory( self, name="", headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.unregister_system_shared_memory`""" if name != "": request_uri = "v2/systemsharedmemory/region/{}/unregister".format( quote(name) ) else: request_uri = "v2/systemsharedmemory/unregister" response = await self._post( request_uri=request_uri, request_body="", headers=headers, query_params=query_params, ) await _raise_if_error(response) if self._verbose: if name != "": print("Unregistered system shared memory with name '{}'".format(name)) else: print("Unregistered all system shared memory regions") async def get_cuda_shared_memory_status( self, region_name="", headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.get_cuda_shared_memory_status`""" if region_name != "": request_uri = "v2/cudasharedmemory/region/{}/status".format( quote(region_name) ) else: request_uri = "v2/cudasharedmemory/status" response = await self._get( request_uri=request_uri, headers=headers, query_params=query_params ) await _raise_if_error(response) content = await response.read() if self._verbose: print(content) return json.loads(content) async def register_cuda_shared_memory( self, name, raw_handle, device_id, byte_size, headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.register_cuda_shared_memory`""" request_uri = "v2/cudasharedmemory/region/{}/register".format(quote(name)) register_request = { "raw_handle": {"b64": raw_handle}, "device_id": device_id, "byte_size": byte_size, } request_body = json.dumps(register_request) response = await self._post( request_uri=request_uri, request_body=request_body, headers=headers, query_params=query_params, ) await _raise_if_error(response) if self._verbose: print("Registered cuda shared memory with name '{}'".format(name)) async def unregister_cuda_shared_memory( self, name="", headers=None, query_params=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.unregister_cuda_shared_memory`""" if name != "": request_uri = "v2/cudasharedmemory/region/{}/unregister".format(quote(name)) else: request_uri = "v2/cudasharedmemory/unregister" response = await self._post( request_uri=request_uri, request_body="", headers=headers, query_params=query_params, ) await _raise_if_error(response) if self._verbose: if name != "": print("Unregistered cuda shared memory with name '{}'".format(name)) else: print("Unregistered all cuda shared memory regions") @staticmethod def generate_request_body( inputs, outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, priority=0, timeout=None, parameters=None, ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.generate_request_body`""" return _get_inference_request( inputs=inputs, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, custom_parameters=parameters, ) @staticmethod def parse_response_body( response_body, verbose=False, header_length=None, content_encoding=None ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.parse_response_body`""" return InferResult.from_response_body( response_body, verbose, header_length, content_encoding ) async def infer( self, model_name, inputs, model_version="", outputs=None, request_id="", sequence_id=0, sequence_start=False, sequence_end=False, priority=0, timeout=None, headers=None, query_params=None, request_compression_algorithm=None, response_compression_algorithm=None, parameters=None, ): """Refer to :py:meth:`tritonclient.http.InferenceServerClient.infer`""" request_body, json_size = _get_inference_request( inputs=inputs, request_id=request_id, outputs=outputs, sequence_id=sequence_id, sequence_start=sequence_start, sequence_end=sequence_end, priority=priority, timeout=timeout, custom_parameters=parameters, ) if request_compression_algorithm == "gzip": if headers is None: headers = {} headers["Content-Encoding"] = "gzip" request_body = gzip.compress(request_body) elif request_compression_algorithm == "deflate": if headers is None: headers = {} headers["Content-Encoding"] = "deflate" # "Content-Encoding: deflate" actually means compressing in zlib structure # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding request_body = zlib.compress(request_body) if response_compression_algorithm == "gzip": if headers is None: headers = {} headers["Accept-Encoding"] = "gzip" elif response_compression_algorithm == "deflate": if headers is None: headers = {} headers["Accept-Encoding"] = "deflate" if json_size is not None: if headers is None: headers = {} headers["Inference-Header-Content-Length"] = json_size if type(model_version) != str: raise_error("model version must be a string") if model_version != "": request_uri = "v2/models/{}/versions/{}/infer".format( quote(model_name), model_version ) else: request_uri = "v2/models/{}/infer".format(quote(model_name)) response = await self._post( request_uri=request_uri, request_body=request_body, headers=headers, query_params=query_params, ) await _raise_if_error(response) content_encoding = response.headers.get("Content-Encoding", None) header_length = response.headers.get("Inference-Header-Content-Length", None) response_body = await response.read() return InferResult.from_response_body( response_body, self._verbose, header_length, content_encoding ) kytensor-2.49.0.7/client/src/python/library/tritonclient/http/aio/auth/000077500000000000000000000000001527052371200260755ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/http/aio/auth/__init__.py000077500000000000000000000030771527052371200302200ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from ...._auth import BasicAuth kytensor-2.49.0.7/client/src/python/library/tritonclient/http/auth/000077500000000000000000000000001527052371200253255ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/http/auth/__init__.py000077500000000000000000000031271527052371200274440ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from ..._auth import BasicAuth __all__ = ["BasicAuth"] kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/000077500000000000000000000000001527052371200245455ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/CMakeLists.txt000066400000000000000000000045071527052371200273130ustar00rootroot00000000000000# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. configure_file(__init__.py __init__.py COPYONLY) configure_file(_dlpack.py _dlpack.py COPYONLY) configure_file(_shared_memory_tensor.py _shared_memory_tensor.py COPYONLY) if(NOT WIN32) file(COPY shared_memory DESTINATION .) # # libcshm.so # add_library(cshm SHARED shared_memory/shared_memory.cc) if(${TRITON_ENABLE_GPU}) target_compile_definitions(cshm PUBLIC TRITON_ENABLE_GPU=1) target_link_libraries(cshm PUBLIC CUDA::cudart) endif() # TRITON_ENABLE_GPU target_link_libraries(cshm PRIVATE rt) endif() # WIN32 if(NOT WIN32) configure_file(shared_memory/__init__.py shared_memory/__init__.py COPYONLY) configure_file(cuda_shared_memory/__init__.py cuda_shared_memory/__init__.py COPYONLY) configure_file(cuda_shared_memory/_utils.py cuda_shared_memory/_utils.py COPYONLY) endif() # NOT WIN32 kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/__init__.py000077500000000000000000000242731527052371200266710ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import struct import numpy as np from ._shared_memory_tensor import SharedMemoryTensor def raise_error(msg): """ Raise error with the provided message """ raise InferenceServerException(msg=msg) from None def serialized_byte_size(tensor_value): """ Get the underlying number of bytes for a numpy ndarray. Parameters ---------- tensor_value : numpy.ndarray Numpy array to calculate the number of bytes for. Returns ------- int Number of bytes present in this tensor """ if tensor_value.dtype != np.object_: raise_error("The tensor_value dtype must be np.object_") if tensor_value.size > 0: total_bytes = 0 # 'C' order is row-major. for obj in np.nditer(tensor_value, flags=["refs_ok"], order="C"): total_bytes += len(obj.item()) return total_bytes else: return 0 class InferenceServerException(Exception): """Exception indicating non-Success status. Parameters ---------- msg : str A brief description of error status : str The error code debug_details : str The additional details on the error """ def __init__(self, msg, status=None, debug_details=None): self._msg = msg self._status = status self._debug_details = debug_details def __str__(self): msg = super().__str__() if self._msg is None else self._msg if self._status is not None: msg = "[" + self._status + "] " + msg return msg def message(self): """Get the exception message. Returns ------- str The message associated with this exception, or None if no message. """ return self._msg def status(self): """Get the status of the exception. Returns ------- str Returns the status of the exception """ return self._status def debug_details(self): """Get the detailed information about the exception for debugging purposes Returns ------- str Returns the exception details """ return self._debug_details def np_to_triton_dtype(np_dtype): if np_dtype == bool: return "BOOL" elif np_dtype == np.int8: return "INT8" elif np_dtype == np.int16: return "INT16" elif np_dtype == np.int32: return "INT32" elif np_dtype == np.int64: return "INT64" elif np_dtype == np.uint8: return "UINT8" elif np_dtype == np.uint16: return "UINT16" elif np_dtype == np.uint32: return "UINT32" elif np_dtype == np.uint64: return "UINT64" elif np_dtype == np.float16: return "FP16" elif np_dtype == np.float32: return "FP32" elif np_dtype == np.float64: return "FP64" elif np_dtype == np.object_ or np_dtype.type == np.bytes_: return "BYTES" return None def triton_to_np_dtype(dtype): if dtype == "BOOL": return bool elif dtype == "INT8": return np.int8 elif dtype == "INT16": return np.int16 elif dtype == "INT32": return np.int32 elif dtype == "INT64": return np.int64 elif dtype == "UINT8": return np.uint8 elif dtype == "UINT16": return np.uint16 elif dtype == "UINT32": return np.uint32 elif dtype == "UINT64": return np.uint64 elif dtype == "FP16": return np.float16 elif dtype == "FP32" or dtype == "BF16": return np.float32 elif dtype == "FP64": return np.float64 elif dtype == "BYTES": return np.object_ return None def serialize_byte_tensor(input_tensor): """ Serializes a bytes tensor into a flat numpy array of length prepended bytes. The numpy array should use dtype of np.object. For np.bytes, numpy will remove trailing zeros at the end of byte sequence and because of this it should be avoided. Parameters ---------- input_tensor : np.array The bytes tensor to serialize. Returns ------- serialized_bytes_tensor : np.array The 1-D numpy array of type uint8 containing the serialized bytes in row-major form. Raises ------ InferenceServerException If unable to serialize the given tensor. """ if input_tensor.size == 0: return np.empty([0], dtype=np.object_) # If the input is a tensor of string/bytes objects, then must flatten those into # a 1-dimensional array containing the 4-byte byte size followed by the # actual element bytes. All elements are concatenated together in row-major # order. if (input_tensor.dtype != np.object_) and (input_tensor.dtype.type != np.bytes_): raise_error("cannot serialize bytes tensor: invalid datatype") flattened_ls = [] # 'C' order is row-major. for obj in np.nditer(input_tensor, flags=["refs_ok"], order="C"): # If directly passing bytes to BYTES type, # don't convert it to str as Python will encode the # bytes which may distort the meaning if input_tensor.dtype == np.object_: if type(obj.item()) == bytes: s = obj.item() else: s = str(obj.item()).encode("utf-8") else: s = obj.item() flattened_ls.append(struct.pack(" None: # Convert the Python object to ctypes objects expected by # DLPack self._shape = (ctypes.c_int64 * len(shape))(*shape) # No strides: compact and row-major self._strides = ctypes.POINTER(ctypes.c_int64)() def as_manager_ctx(self) -> ctypes.c_void_p: py_obj = ctypes.py_object(self) py_obj_ptr = ctypes.pointer(py_obj) ctypes.pythonapi.Py_IncRef(py_obj) ctypes.pythonapi.Py_IncRef(ctypes.py_object(py_obj_ptr)) return ctypes.cast(py_obj_ptr, ctypes.c_void_p) @ctypes.CFUNCTYPE(None, ctypes.c_void_p) def managed_tensor_deleter(handle: ctypes.c_void_p) -> None: dl_managed_tensor = DLManagedTensor.from_address(handle) py_obj_ptr = ctypes.cast( dl_managed_tensor.manager_ctx, ctypes.POINTER(ctypes.py_object) ) py_obj = py_obj_ptr.contents ctypes.pythonapi.Py_DecRef(py_obj) ctypes.pythonapi.Py_DecRef(ctypes.py_object(py_obj_ptr)) ctypes.pythonapi.PyMem_RawFree(handle) @ctypes.CFUNCTYPE(None, ctypes.c_void_p) def pycapsule_deleter(handle: ctypes.c_void_p) -> None: pycapsule: ctypes.py_object = ctypes.cast(handle, ctypes.py_object) if ctypes.pythonapi.PyCapsule_IsValid(pycapsule, c_str_dltensor): dl_managed_tensor = ctypes.pythonapi.PyCapsule_GetPointer( pycapsule, c_str_dltensor ) managed_tensor_deleter(dl_managed_tensor) ctypes.pythonapi.PyCapsule_SetDestructor(pycapsule, None) def triton_to_dlpack_dtype(dtype): if dtype == "BOOL": type_code = DLDataTypeCode.kDLBool bits = 1 elif dtype == "INT8": type_code = DLDataTypeCode.kDLInt bits = 8 elif dtype == "INT16": type_code = DLDataTypeCode.kDLInt bits = 16 elif dtype == "INT32": type_code = DLDataTypeCode.kDLInt bits = 32 elif dtype == "INT64": type_code = DLDataTypeCode.kDLInt bits = 64 elif dtype == "UINT8": type_code = DLDataTypeCode.kDLUInt bits = 8 elif dtype == "UINT16": type_code = DLDataTypeCode.kDLUInt bits = 16 elif dtype == "UINT32": type_code = DLDataTypeCode.kDLUInt bits = 32 elif dtype == "UINT64": type_code = DLDataTypeCode.kDLUInt bits = 64 elif dtype == "FP16": type_code = DLDataTypeCode.kDLFloat bits = 16 elif dtype == "FP32": type_code = DLDataTypeCode.kDLFloat bits = 32 elif dtype == "FP64": type_code = DLDataTypeCode.kDLFloat bits = 64 elif dtype == "BF16": type_code = DLDataTypeCode.kDLBfloat bits = 16 elif dtype == "BYTES": _raise_error("DLPack currently doesn't suppose BYTES type") else: _raise_error( "Can not covert unknown data type '{}' to DLPack data type".format(dtype) ) return DLDataType(type_code, bits, 1) def is_contiguous_data( ndim: ctypes.c_int, shape: ctypes.POINTER(ctypes.c_int64), stride: ctypes.POINTER(ctypes.c_int64), ): # If 'stride' doesn't capture valid value if (stride is None) or (not bool(stride)): return True calculated_stride = 1 # iterate stride in reverse order [ndim-1, -1) for i in reversed(range(ndim)): if stride[i] != calculated_stride: return False calculated_stride *= shape[i] return True def get_byte_size( dtype: DLDataType, ndim: ctypes.c_int, shape: ctypes.POINTER(ctypes.c_int64) ): element_byte_size = dtype.bits * dtype.lanes // 8 # Assume 8 bits in a byte for i in range(ndim): element_byte_size *= shape[i] return element_byte_size def get_dlpack_capsule(dlpack_obj, stream=None): # Extract PyCapsule of the DLPack object if hasattr(dlpack_obj, "__dlpack__"): if not hasattr(dlpack_obj, "__dlpack_device__"): _raise_error( "DLPack expects '__dlpack_device__' if '__dlpack__' has been defined" ) device = dlpack_obj.__dlpack_device__() # Have to condition on the device type as, using numpy as example, # some DLPack implementation doesn't accept 'stream' as arguments if device != DLDeviceType.kDLCUDA: return dlpack_obj.__dlpack__() else: return dlpack_obj.__dlpack__(stream) else: # Old interface where PyCapsule object is passed directly return dlpack_obj def get_dlpack_device(dlpack_obj): if hasattr(dlpack_obj, "__dlpack_device__"): return dlpack_obj.__dlpack_device__() return None def get_managed_tensor(dlcapsule): ptr = ctypes.pythonapi.PyCapsule_GetPointer(dlcapsule, c_str_dltensor) return DLManagedTensor.from_address(ptr) kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/_shared_memory_tensor.py000066400000000000000000000071401527052371200315100ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import collections import ctypes from typing import Any from . import _dlpack class SharedMemoryTensor: """An object of SharedMemoryTensor class is a view of the shared memory region that follows DLPack specification. This object should be considered invalidated if there is modification on the corresponding shared memory region. https://dmlc.github.io/dlpack/latest/python_spec.html """ def __init__( self, dtype: str, shape: collections.abc.Iterable, shm_addr: Any, offset: int, byte_size: int, device_id: int, ) -> None: self._dtype = dtype self._shape = shape self._shm_addr = shm_addr self._offset = offset self._byte_size = byte_size self._device_id = device_id if device_id != -1: self._dl_device = (_dlpack.DLDeviceType.kDLCUDA, device_id) else: self._dl_device = (_dlpack.DLDeviceType.kDLCPU, 0) def __dlpack__(self, stream=None): context = _dlpack.DataViewContext(self._shape) size = ctypes.c_size_t(ctypes.sizeof(_dlpack.DLManagedTensor)) dl_managed_tensor = _dlpack.DLManagedTensor.from_address( ctypes.pythonapi.PyMem_RawMalloc(size) ) dl_managed_tensor.dl_tensor.data = self._shm_addr dl_managed_tensor.dl_tensor.device = self._dl_device dl_managed_tensor.dl_tensor.dtype = _dlpack.triton_to_dlpack_dtype(self._dtype) dl_managed_tensor.dl_tensor.ndim = len(self._shape) dl_managed_tensor.dl_tensor.shape = context._shape dl_managed_tensor.dl_tensor.strides = context._strides dl_managed_tensor.dl_tensor.byte_offset = self._offset dl_managed_tensor.manager_ctx = context.as_manager_ctx() dl_managed_tensor.deleter = _dlpack.managed_tensor_deleter pycapsule = ctypes.pythonapi.PyCapsule_New( ctypes.byref(dl_managed_tensor), _dlpack.c_str_dltensor, _dlpack.pycapsule_deleter, ) return pycapsule def __dlpack_device__(self): return self._dl_device kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/cuda_shared_memory/000077500000000000000000000000001527052371200303775ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/cuda_shared_memory/__init__.py000077500000000000000000000370601527052371200325210ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Check for dependency before other import so other imports can assume # the module is available (drop "try ... except .."") try: from cuda import cudart except ModuleNotFoundError as error: raise RuntimeError( "CUDA shared memory utilities require Python package 'cuda-python'" "to be installed. Specify 'cuda' or 'all' while installing the tritonclient " "package to include the support." ) from error import base64 import ctypes import os import struct import numpy as np import pkg_resources from .. import _dlpack from .._shared_memory_tensor import SharedMemoryTensor from ._utils import ( CudaSharedMemoryException, CudaSharedMemoryRegion, CudaStream, call_cuda_function, maybe_set_device, ) allocated_shm_regions = [] # Internally managed pool of streams for properly synchronizing on DLPack data, # the stream will be created / destroyed according to 'allocated_shm_regions' # and be reused throughout the process. May revisit for stream pool if # asynchronous write on CUDA shared memory region is requested _dlpack_stream = {} # Helper function to retrieve internally managed CUDA stream def _get_or_create_global_cuda_stream(device_id): global _dlpack_stream if device_id not in _dlpack_stream: _dlpack_stream[device_id] = CudaStream(device_id) return _dlpack_stream[device_id]._stream def _support_uva(shm_device_id, ext_device_id): try: support_uva = call_cuda_function( cudart.cudaDeviceGetAttribute, cudart.cudaDeviceAttr.cudaDevAttrUnifiedAddressing, shm_device_id, ) if (support_uva != 0) and (ext_device_id != -1): support_uva = call_cuda_function( cudart.cudaDeviceGetAttribute, cudart.cudaDeviceAttr.cudaDevAttrUnifiedAddressing, ext_device_id, ) if support_uva == 0: raise CudaSharedMemoryException( "device or platform does not support unified virtual addressing" ) except Exception as ex: if not isinstance(ex, CudaSharedMemoryException): raise CudaSharedMemoryException( "unable to check UVA support on device" ) from ex else: raise ex def _is_device_supported(device: _dlpack.DLDevice): return device[0] in [ _dlpack.DLDeviceType.kDLCPU, _dlpack.DLDeviceType.kDLCUDA, _dlpack.DLDeviceType.kDLCUDAHost, ] def create_shared_memory_region(triton_shm_name, byte_size, device_id): """Creates a shared memory region with the specified name and size. Parameters ---------- triton_shm_name : str The unique name of the cuda shared memory region to be created. byte_size : int The size in bytes of the cuda shared memory region to be created. device_id : int The GPU device ID of the cuda shared memory region to be created. Returns ------- cuda_shm_handle : CudaSharedMemoryRegion The handle for the cuda shared memory region. Raises ------ CudaSharedMemoryException If unable to create the cuda shared memory region on the specified device. """ prev_device = None try: prev_device = call_cuda_function(cudart.cudaGetDevice) call_cuda_function(cudart.cudaSetDevice, device_id) device_ptr = call_cuda_function(cudart.cudaMalloc, byte_size) cuda_shm_handle = call_cuda_function(cudart.cudaIpcGetMemHandle, device_ptr) triton_shm_handle = CudaSharedMemoryRegion( triton_shm_name, cuda_shm_handle, device_ptr, byte_size, device_id ) allocated_shm_regions.append(triton_shm_handle) except Exception as ex: if not isinstance(ex, CudaSharedMemoryException): raise CudaSharedMemoryException( "unable to create cuda shared memory handle" ) from ex else: raise ex finally: if prev_device is not None: maybe_set_device(prev_device) return triton_shm_handle def get_raw_handle(cuda_shm_handle): """Returns the underlying raw serialized cudaIPC handle in base64 encoding. Parameters ---------- cuda_shm_handle : CudaSharedMemoryRegion The handle for the cuda shared memory region. Returns ------- bytes The raw serialized cudaIPC handle of underlying cuda shared memory in base64 encoding. """ # 'reserved' in shared memory handle is not well documented but experiment # showed that it is the equivalent handle used in # cudaIpcOpenMemHandle (C API) return base64.b64encode(cuda_shm_handle._cuda_shm_handle.reserved) def set_shared_memory_region(cuda_shm_handle, input_values): """Copy the contents of the numpy array into the cuda shared memory region. Parameters ---------- cuda_shm_handle : CudaSharedMemoryRegion The handle for the cuda shared memory region. input_values : list The list of numpy arrays to be copied into the shared memory region. Raises ------ CudaSharedMemoryException If unable to set values in the cuda shared memory region. """ if not isinstance(input_values, (list, tuple)): raise CudaSharedMemoryException( "input_values must be specified as a numpy array" ) for input_value in input_values: if not isinstance(input_value, (np.ndarray,)): raise CudaSharedMemoryException( "input_values must be specified as a list/tuple of numpy arrays" ) try: _support_uva(cuda_shm_handle._device_id, -1) stream = _get_or_create_global_cuda_stream(cuda_shm_handle._device_id) offset_current = 0 for input_value in input_values: input_value = np.ascontiguousarray(input_value).flatten() # When the input array is in type "BYTES" (np.object_ is # the numpy equivalent), expect the array has been serialized # via 'tritonclient.utils.serialize_byte_tensor'. if input_value.dtype == np.object_: input_value = input_value.item() byte_size = np.dtype(np.byte).itemsize * len(input_value) call_cuda_function( cudart.cudaMemcpyAsync, cuda_shm_handle._base_addr + offset_current, input_value, byte_size, cudart.cudaMemcpyKind.cudaMemcpyDefault, stream, ) else: byte_size = input_value.size * input_value.itemsize call_cuda_function( cudart.cudaMemcpyAsync, cuda_shm_handle._base_addr + offset_current, input_value.ctypes.data, byte_size, cudart.cudaMemcpyKind.cudaMemcpyDefault, stream, ) offset_current += byte_size call_cuda_function(cudart.cudaStreamSynchronize, stream) except Exception as ex: if not isinstance(ex, CudaSharedMemoryException): raise CudaSharedMemoryException( "unable to set values in cuda shared memory" ) from ex else: raise ex return def get_contents_as_numpy(cuda_shm_handle, datatype, shape): """Generates a numpy array using the data stored in the cuda shared memory region specified with the handle. Parameters ---------- cuda_shm_handle : CudaSharedMemoryRegion The handle for the cuda shared memory region. datatype : np.dtype The datatype of the array to be returned. shape : list The list of int describing the shape of the array to be returned. Returns ------- np.array The numpy array generated using contents from the specified shared memory region. """ try: _support_uva(cuda_shm_handle._device_id, -1) stream = _get_or_create_global_cuda_stream(cuda_shm_handle._device_id) # Numpy can only read from host buffer. host_buffer = (ctypes.c_char * cuda_shm_handle._byte_size)() call_cuda_function( cudart.cudaMemcpyAsync, host_buffer, cuda_shm_handle._base_addr, cuda_shm_handle._byte_size, cudart.cudaMemcpyKind.cudaMemcpyDefault, stream, ) # Sync to ensure the host buffer is ready call_cuda_function(cudart.cudaStreamSynchronize, stream) except Exception as ex: if not isinstance(ex, CudaSharedMemoryException): raise CudaSharedMemoryException( "failed to read cuda shared memory results" ) from ex else: raise ex start_pos = 0 # was 'handle->offset_' if (datatype != np.object_) and (datatype != np.bytes_): requested_byte_size = np.prod(shape) * np.dtype(datatype).itemsize cval_len = start_pos + requested_byte_size if cuda_shm_handle._byte_size < cval_len: raise CudaSharedMemoryException( "The size of the shared memory region is insufficient to provide numpy array with requested size" ) if cval_len == 0: result = np.empty(shape, dtype=datatype) else: val_buf = ctypes.cast( host_buffer, ctypes.POINTER(ctypes.c_byte * cval_len) )[0] val = np.frombuffer(val_buf, dtype=datatype, offset=start_pos) # Reshape the result to the appropriate shape. This copy is only # needed as the temporary CPU buffer is cleared later by # _cshm_cuda_shared_memory_release_host_buffer result = np.reshape(np.copy(val), shape) else: str_offset = start_pos val_buf = ctypes.cast( host_buffer, ctypes.POINTER(ctypes.c_byte * cuda_shm_handle._byte_size) )[0] ii = 0 strs = list() while (ii % np.prod(shape) != 0) or (ii == 0): l = struct.unpack_from(" 1: return res[1:] if len(res) > 2 else res[1] return None class CudaSharedMemoryException(Exception): """Exception indicating non-Success status. Parameters ---------- err : c_void_p Pointer to an Error that should be used to initialize the exception. """ def __init__(self, msg): self._msg = msg def __str__(self): msg = super().__str__() if self._msg is None else self._msg return msg class CudaSharedMemoryRegion: def __init__( self, triton_shm_name: str, cuda_shm_handle: cudart.cudaIpcMemHandle_t, base_addr: Any, byte_size: int, device_id: int, ) -> None: self._triton_shm_name = triton_shm_name self._cuda_shm_handle = cuda_shm_handle self._base_addr = base_addr self._byte_size = byte_size self._device_id = device_id # [FXIME] C implementation has below which is not relevant, need to # revisit when applying similar change to system shared memory utils # and check on whether the "handles" can be unified. # handle->offset_ = 0; # handle->shm_key_ = ""; # handle->shm_fd_ = 0; def __del__(self): # __init__() may fail, don't attempt to release # uninitialized resource. if not hasattr(self, "_base_addr"): return prev_device = None try: prev_device = call_cuda_function(cudart.cudaGetDevice) call_cuda_function(cudart.cudaSetDevice, self._device_id) call_cuda_function(cudart.cudaFree, self._base_addr) finally: if prev_device is not None: maybe_set_device(prev_device) class CudaStream: def __init__(self, device_id): prev_device = None try: prev_device = call_cuda_function(cudart.cudaGetDevice) call_cuda_function(cudart.cudaSetDevice, device_id) self._stream = call_cuda_function(cudart.cudaStreamCreate) finally: if prev_device is not None: maybe_set_device(prev_device) def __del__(self): # __init__() may fail, don't attempt to release # uninitialized resource. if not hasattr(self, "_stream") or self._stream is None: return # [FIXME] __del__ is not the best place for releasing resources call_cuda_function(cudart.cudaStreamDestroy, self._stream) self._stream = None def maybe_set_device(device_id): device = call_cuda_function(cuda_driver.cuDeviceGet, device_id) _, active = call_cuda_function(cuda_driver.cuDevicePrimaryCtxGetState, device) if active: call_cuda_function(cudart.cudaSetDevice, device_id) kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/shared_memory/000077500000000000000000000000001527052371200274035ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/shared_memory/__init__.py000077500000000000000000000257151527052371200315310ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os import struct from ctypes import * import numpy as np import pkg_resources class _utf8(object): @classmethod def from_param(cls, value): if value is None: return None elif isinstance(value, bytes): return value else: return value.encode("utf8") _cshm_lib = "cshm" if os.name == "nt" else "libcshm.so" _cshm_path = pkg_resources.resource_filename( "tritonclient.utils.shared_memory", _cshm_lib ) _cshm = cdll.LoadLibrary(_cshm_path) _cshm_shared_memory_region_create = _cshm.SharedMemoryRegionCreate _cshm_shared_memory_region_create.restype = c_int _cshm_shared_memory_region_create.argtypes = [_utf8, _utf8, c_uint64, POINTER(c_void_p)] _cshm_shared_memory_region_set = _cshm.SharedMemoryRegionSet _cshm_shared_memory_region_set.restype = c_int _cshm_shared_memory_region_set.argtypes = [c_void_p, c_uint64, c_uint64, c_void_p] _cshm_get_shared_memory_handle_info = _cshm.GetSharedMemoryHandleInfo _cshm_get_shared_memory_handle_info.restype = c_int _cshm_get_shared_memory_handle_info.argtypes = [ c_void_p, POINTER(c_char_p), POINTER(c_char_p), POINTER(c_int), POINTER(c_uint64), POINTER(c_uint64), ] _cshm_shared_memory_region_destroy = _cshm.SharedMemoryRegionDestroy _cshm_shared_memory_region_destroy.restype = c_int _cshm_shared_memory_region_destroy.argtypes = [c_void_p] mapped_shm_regions = [] def _raise_if_error(errno): """ Raise SharedMemoryException if 'err' is non-success. Otherwise return nothing. """ if errno.value != 0: ex = SharedMemoryException(errno) raise ex return def _raise_error(msg): ex = SharedMemoryException(msg) raise ex def create_shared_memory_region(triton_shm_name, shm_key, byte_size): """Creates a system shared memory region with the specified name and size. Parameters ---------- triton_shm_name : str The unique name of the shared memory region to be created. shm_key : str The unique key of the shared memory object. byte_size : int The size in bytes of the shared memory region to be created. Returns ------- shm_handle : c_void_p The handle for the system shared memory region. Raises ------ SharedMemoryException If unable to create the shared memory region. """ shm_handle = c_void_p() _raise_if_error( c_int( _cshm_shared_memory_region_create( triton_shm_name, shm_key, byte_size, byref(shm_handle) ) ) ) mapped_shm_regions.append(shm_key) return shm_handle def set_shared_memory_region(shm_handle, input_values, offset=0): """Copy the contents of the numpy array into the system shared memory region. Parameters ---------- shm_handle : c_void_p The handle for the system shared memory region. input_values : list The list of numpy arrays to be copied into the shared memory region. offset : int The offset, in bytes, into the region where you want the array copied. The default value is 0. Raises ------ SharedMemoryException If unable to mmap or set values in the system shared memory region. """ if not isinstance(input_values, (list, tuple)): _raise_error("input_values must be specified as a list/tuple of numpy arrays") for input_value in input_values: if not isinstance(input_value, np.ndarray): _raise_error("each element of input_values must be a numpy array") offset_current = offset for input_value in input_values: input_value = np.ascontiguousarray(input_value).flatten() if input_value.dtype == np.object_: input_value = input_value.item() byte_size = np.dtype(np.byte).itemsize * len(input_value) _raise_if_error( c_int( _cshm_shared_memory_region_set( shm_handle, c_uint64(offset_current), c_uint64(byte_size), cast(input_value, c_void_p), ) ) ) else: byte_size = input_value.size * input_value.itemsize _raise_if_error( c_int( _cshm_shared_memory_region_set( shm_handle, c_uint64(offset_current), c_uint64(byte_size), input_value.ctypes.data_as(c_void_p), ) ) ) offset_current += byte_size return def get_contents_as_numpy(shm_handle, datatype, shape, offset=0): """Generates a numpy array using the data stored in the system shared memory region specified with the handle. Parameters ---------- shm_handle : c_void_p The handle for the system shared memory region. datatype : np.dtype The datatype of the array to be returned. shape : list The list of int describing the shape of the array to be returned. offset : int The offset, in bytes, into the region where you want the array extracted. The default value is 0. Returns ------- np.array The numpy array generated using the contents of the specified shared memory region. """ shm_fd = c_int() region_offset = c_uint64() byte_size = c_uint64() shm_addr = c_char_p() shm_key = c_char_p() _raise_if_error( c_int( _cshm_get_shared_memory_handle_info( shm_handle, byref(shm_addr), byref(shm_key), byref(shm_fd), byref(region_offset), byref(byte_size), ) ) ) start_pos = region_offset.value + offset if (datatype != np.object_) and (datatype != np.bytes_): requested_byte_size = np.prod(shape) * np.dtype(datatype).itemsize cval_len = start_pos + requested_byte_size if byte_size.value < cval_len: _raise_error( "The size of the shared memory region is insufficient to provide numpy array with requested size" ) if cval_len == 0: result = np.empty(shape, dtype=datatype) else: val_buf = cast(shm_addr, POINTER(c_byte * cval_len))[0] val = np.frombuffer(val_buf, dtype=datatype, offset=start_pos) # Reshape the result to the appropriate shape. result = np.reshape(val, shape) else: str_offset = start_pos val_buf = cast(shm_addr, POINTER(c_byte * byte_size.value))[0] ii = 0 strs = list() while (ii % np.prod(shape) != 0) or (ii == 0): l = struct.unpack_from(" #include #include #include #include #include #include #include "shared_memory_handle.h" //============================================================================== // SharedMemoryControlContext namespace { void* SharedMemoryHandleCreate( std::string triton_shm_name, void* shm_addr, std::string shm_key, int shm_fd, size_t offset, size_t byte_size) { SharedMemoryHandle* handle = new SharedMemoryHandle(); handle->triton_shm_name_ = triton_shm_name; handle->base_addr_ = shm_addr; handle->shm_key_ = shm_key; handle->shm_fd_ = shm_fd; handle->offset_ = offset; handle->byte_size_ = byte_size; return reinterpret_cast(handle); } int SharedMemoryRegionMap( int shm_fd, size_t offset, size_t byte_size, void** shm_addr) { // map shared memory to process address space *shm_addr = mmap(NULL, byte_size, PROT_WRITE, MAP_SHARED, shm_fd, offset); if (*shm_addr == MAP_FAILED) { return -1; } // close shared memory descriptor, return 0 if success else return -1 return close(shm_fd); } } // namespace int SharedMemoryRegionCreate( const char* triton_shm_name, const char* shm_key, size_t byte_size, void** shm_handle) { // get shared memory region descriptor int shm_fd = shm_open(shm_key, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (shm_fd == -1) { return -2; } // extend shared memory object as by default it's initialized with size 0 int res = ftruncate(shm_fd, byte_size); if (res == -1) { return -3; } // get base address of shared memory region void* shm_addr = nullptr; int err = SharedMemoryRegionMap(shm_fd, 0, byte_size, &shm_addr); if (err == -1) { return -4; } // create a handle for the shared memory region *shm_handle = SharedMemoryHandleCreate( std::string(triton_shm_name), shm_addr, std::string(shm_key), shm_fd, 0, byte_size); return 0; } int SharedMemoryRegionSet( void* shm_handle, size_t offset, size_t byte_size, const void* data) { void* shm_addr = reinterpret_cast(shm_handle)->base_addr_; char* shm_addr_offset = reinterpret_cast(shm_addr); std::memcpy(shm_addr_offset + offset, data, byte_size); return 0; } int GetSharedMemoryHandleInfo( void* shm_handle, char** shm_addr, const char** shm_key, int* shm_fd, size_t* offset, size_t* byte_size) { SharedMemoryHandle* handle = reinterpret_cast(shm_handle); *shm_addr = reinterpret_cast(handle->base_addr_); *shm_key = handle->shm_key_.c_str(); *shm_fd = handle->shm_fd_; *offset = handle->offset_; *byte_size = handle->byte_size_; return 0; } int SharedMemoryRegionDestroy(void* shm_handle) { std::unique_ptr handle( reinterpret_cast(shm_handle)); void* shm_addr = reinterpret_cast(handle->base_addr_); int status = munmap(shm_addr, handle->byte_size_); if (status == -1) { return -6; } int shm_fd = shm_unlink(handle->shm_key_.c_str()); if (shm_fd == -1) { return -5; } return 0; } //============================================================================== kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/shared_memory/shared_memory.h000066400000000000000000000044461527052371200324220ustar00rootroot00000000000000// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #ifdef __cplusplus extern "C" { #endif //============================================================================== // SharedMemoryControlContext int SharedMemoryRegionCreate( const char* triton_shm_name, const char* shm_key, size_t byte_size, void** shm_handle); int SharedMemoryRegionSet( void* shm_handle, size_t offset, size_t byte_size, const void* data); int GetSharedMemoryHandleInfo( void* shm_handle, char** shm_addr, const char** shm_key, int* shm_fd, size_t* offset, size_t* byte_size); int SharedMemoryRegionDestroy(void* shm_handle); //============================================================================== #ifdef __cplusplus } #endif kytensor-2.49.0.7/client/src/python/library/tritonclient/utils/shared_memory/shared_memory_handle.h000066400000000000000000000036171527052371200337340ustar00rootroot00000000000000// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU struct SharedMemoryHandle { std::string triton_shm_name_; std::string shm_key_; #ifdef TRITON_ENABLE_GPU cudaIpcMemHandle_t cuda_shm_handle_; int device_id_; #endif // TRITON_ENABLE_GPU void* base_addr_; int shm_fd_; size_t offset_; size_t byte_size_; }; kytensor-2.49.0.7/client/src/python/library/tritonclientutils/000077500000000000000000000000001527052371200244665ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonclientutils/__init__.py000077500000000000000000000035111527052371200266020ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritonclientutils` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.utils`", DeprecationWarning, ) from tritonclient.utils import * kytensor-2.49.0.7/client/src/python/library/tritongrpcclient/000077500000000000000000000000001527052371200242615ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritongrpcclient/__init__.py000077500000000000000000000035061527052371200264010ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritongrpcclient` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.grpc`", DeprecationWarning, ) from tritonclient.grpc import * kytensor-2.49.0.7/client/src/python/library/tritongrpcclient/grpc_service_pb2.py000077500000000000000000000035361527052371200300630ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritongrpcclient` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.grpc.service_pb2`", DeprecationWarning, ) from tritonclient.grpc.service_pb2 import * kytensor-2.49.0.7/client/src/python/library/tritongrpcclient/grpc_service_pb2_grpc.py000077500000000000000000000035501527052371200310720ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritongrpcclient` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.grpc.service_pb2_grpc`", DeprecationWarning, ) from tritonclient.grpc.service_pb2_grpc import * kytensor-2.49.0.7/client/src/python/library/tritongrpcclient/model_config_pb2.py000077500000000000000000000035501527052371200300310ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritongrpcclient` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.grpc.model_config_pb2`", DeprecationWarning, ) from tritonclient.grpc.model_config_pb2 import * kytensor-2.49.0.7/client/src/python/library/tritonhttpclient/000077500000000000000000000000001527052371200243055ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonhttpclient/__init__.py000077500000000000000000000035061527052371200264250ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritonhttpclient` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.http`", DeprecationWarning, ) from tritonclient.http import * kytensor-2.49.0.7/client/src/python/library/tritonshmutils/000077500000000000000000000000001527052371200237775ustar00rootroot00000000000000kytensor-2.49.0.7/client/src/python/library/tritonshmutils/__init__.py000077500000000000000000000034441527052371200261200ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritonshmutils` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.utils`", DeprecationWarning, ) kytensor-2.49.0.7/client/src/python/library/tritonshmutils/cuda_shared_memory.py000077500000000000000000000035771527052371200302220ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritonshmutils.cuda_shared_memory` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.utils.cuda_shared_memory`", DeprecationWarning, ) from tritonclient.utils.cuda_shared_memory import * kytensor-2.49.0.7/client/src/python/library/tritonshmutils/shared_memory.py000077500000000000000000000035601527052371200272160ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import warnings warnings.simplefilter("always", DeprecationWarning) warnings.warn( "The package `tritonshmutils.shared_memory` is deprecated and will be " "removed in a future version. Please use instead " "`tritonclient.utils.shared_memory`", DeprecationWarning, ) from tritonclient.utils.shared_memory import * kytensor-2.49.0.7/client/unittest_models/000077500000000000000000000000001527052371200203355ustar00rootroot00000000000000kytensor-2.49.0.7/client/unittest_models/custom_identity_int32/000077500000000000000000000000001527052371200245775ustar00rootroot00000000000000kytensor-2.49.0.7/client/unittest_models/custom_identity_int32/config.pbtxt000066400000000000000000000034741527052371200271370ustar00rootroot00000000000000# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "custom_identity_int32" backend: "identity" max_batch_size: 1024 version_policy: { latest { num_versions: 1 }} instance_group [ { kind: KIND_CPU } ] input [ { name: "INPUT0" data_type: TYPE_INT32 dims: [ -1 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_INT32 dims: [ -1 ] } ]kytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/000077500000000000000000000000001527052371200243745ustar00rootroot00000000000000kytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/1/000077500000000000000000000000001527052371200245345ustar00rootroot00000000000000kytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/1/model.onnx000077500000000000000000000005131527052371200265420ustar00rootroot00000000000000triton:  INPUT0_INPUT0"Identity  INPUT1_INPUT1"Identity  _INPUT0 _INPUT1CAST0"Add  _INPUT0 _INPUT1CAST1"Sub ! CAST0OUTPUT0"Cast* to ! CAST1OUTPUT1"Cast* toonnx_int32_int32_int32Z INPUT0  Z INPUT1  b OUTPUT0  b OUTPUT1  Bkytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/2/000077500000000000000000000000001527052371200245355ustar00rootroot00000000000000kytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/2/model.onnx000077500000000000000000000005131527052371200265430ustar00rootroot00000000000000triton:  INPUT0_INPUT0"Identity  INPUT1_INPUT1"Identity  _INPUT0 _INPUT1CAST1"Add  _INPUT0 _INPUT1CAST0"Sub ! CAST0OUTPUT0"Cast* to ! CAST1OUTPUT1"Cast* toonnx_int32_int32_int32Z INPUT0  Z INPUT1  b OUTPUT0  b OUTPUT1  Bkytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/3/000077500000000000000000000000001527052371200245365ustar00rootroot00000000000000kytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/3/model.onnx000077500000000000000000000005131527052371200265440ustar00rootroot00000000000000triton:  INPUT0_INPUT0"Identity  INPUT1_INPUT1"Identity  _INPUT0 _INPUT1CAST1"Add  _INPUT0 _INPUT1CAST0"Sub ! CAST0OUTPUT0"Cast* to ! CAST1OUTPUT1"Cast* toonnx_int32_int32_int32Z INPUT0  Z INPUT1  b OUTPUT0  b OUTPUT1  Bkytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/config.pbtxt000077500000000000000000000007621527052371200267340ustar00rootroot00000000000000 name: "onnx_int32_int32_int32" platform: "onnxruntime_onnx" max_batch_size: 8 version_policy: { all { }} input [ { name: "INPUT0" data_type: TYPE_INT32 dims: [ 16 ] } ] input [ { name: "INPUT1" data_type: TYPE_INT32 dims: [ 16 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_INT32 dims: [ 16 ] label_filename: "output0_labels.txt" } ] output [ { name: "OUTPUT1" data_type: TYPE_INT32 dims: [ 16 ] } ]kytensor-2.49.0.7/client/unittest_models/onnx_int32_int32_int32/output0_labels.txt000077500000000000000000000001661527052371200301050ustar00rootroot00000000000000label0 label1 label2 label3 label4 label5 label6 label7 label8 label9 label10 label11 label12 label13 label14 label15 kytensor-2.49.0.7/common/000077500000000000000000000000001527052371200151255ustar00rootroot00000000000000kytensor-2.49.0.7/common/.clang-format000066400000000000000000000014261527052371200175030ustar00rootroot00000000000000--- BasedOnStyle: Google IndentWidth: 2 ColumnLimit: 80 ContinuationIndentWidth: 4 UseTab: Never MaxEmptyLinesToKeep: 2 SortIncludes: true CompactNamespaces: true ReflowComments: true DerivePointerAlignment: false PointerAlignment: Left AllowShortIfStatementsOnASingleLine: false AllowShortBlocksOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AlwaysBreakAfterReturnType: TopLevelDefinitions AlignAfterOpenBracket: AlwaysBreak BreakBeforeBraces: Custom BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: true AfterNamespace: false AfterStruct: false AfterUnion: false BeforeCatch: true BinPackArguments: true BinPackParameters: true ConstructorInitializerAllOnOneLineOrOnePerLine: false IndentCaseLabels: true kytensor-2.49.0.7/common/.github/000077500000000000000000000000001527052371200164655ustar00rootroot00000000000000kytensor-2.49.0.7/common/.github/workflows/000077500000000000000000000000001527052371200205225ustar00rootroot00000000000000kytensor-2.49.0.7/common/.github/workflows/pre-commit.yml000066400000000000000000000033201527052371200233170ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: pre-commit on: pull_request: jobs: pre-commit: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - uses: pre-commit/action@v3.0.0 kytensor-2.49.0.7/common/.pre-commit-config.yaml000066400000000000000000000056111527052371200214110ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black rev: 23.1.0 hooks: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] types_or: [python, cython] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.5 hooks: - id: clang-format types_or: [c, c++, cuda, proto, textproto, java] args: ["-fallback-style=none", "-style=file", "-i"] - repo: https://github.com/codespell-project/codespell rev: v2.2.4 hooks: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$) # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-json - id: check-toml - id: check-yaml - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace kytensor-2.49.0.7/common/CMakeLists.txt000066400000000000000000000157041527052371200176740ustar00rootroot00000000000000# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritoncommon LANGUAGES C CXX) # # C++ standard # set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard whose features are requested to build this target.") # # Options # # Some components are expensive to build and have extensive # dependencies, so those parts of the build must be enabled # explicitly. option(TRITON_COMMON_ENABLE_PROTOBUF "Build protobuf artifacts" OFF) option(TRITON_COMMON_ENABLE_PROTOBUF_PYTHON "Build protobuf artifacts for python" ON) option(TRITON_COMMON_ENABLE_GRPC "Build grpc artifacts" OFF) option(TRITON_COMMON_ENABLE_JSON "Build json-related libs" ON) if(TRITON_COMMON_ENABLE_JSON) find_package(RapidJSON CONFIG REQUIRED) message(STATUS "RapidJSON found. Headers: ${RAPIDJSON_INCLUDE_DIRS}") endif() set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message("Using MSVC as compiler, default target on Windows 10. " "If the target system is not Windows 10, please update _WIN32_WINNT " "to corresponding value.") endif() add_library(common-compile-settings INTERFACE) target_compile_features(common-compile-settings INTERFACE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options(common-compile-settings INTERFACE $<$,$,$>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/W0 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) # # SyncQueue # add_library( triton-common-sync-queue INTERFACE ) add_library( TritonCommon::triton-common-sync-queue ALIAS triton-common-sync-queue ) target_include_directories( triton-common-sync-queue INTERFACE $ $ ) # # JSON utilities # if(TRITON_COMMON_ENABLE_JSON) add_library( triton-common-json INTERFACE ) add_library( TritonCommon::triton-common-json ALIAS triton-common-json ) target_include_directories( triton-common-json INTERFACE $ $ $ $ ) endif() add_subdirectory(src) # # Protobuf and GRPC artifacts # if(${TRITON_COMMON_ENABLE_PROTOBUF} OR ${TRITON_COMMON_ENABLE_GRPC}) add_subdirectory(protobuf) set(protobuf_MODULE_COMPATIBLE TRUE CACHE BOOL "protobuf_MODULE_COMPATIBLE" FORCE) find_package(Protobuf REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") # # Model Config (depends on protobuf & generated .pb.h file) # add_library( triton-common-model-config src/model_config.cc ) add_library( TritonCommon::triton-common-model-config ALIAS triton-common-model-config ) target_include_directories( triton-common-model-config PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ${Protobuf_INCLUDE_DIRS} ) target_link_libraries( triton-common-model-config PRIVATE common-compile-settings protobuf::libprotobuf proto-library ) set_target_properties( triton-common-model-config PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE POSITION_INDEPENDENT_CODE ON OUTPUT_NAME tritoncommonmodelconfig ) endif() # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonCommon) install( TARGETS triton-common-async-work-queue triton-common-error triton-common-logging triton-common-sync-queue triton-common-table-printer triton-common-thread-pool common-compile-settings EXPORT triton-common-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) if(TRITON_COMMON_ENABLE_JSON) install( TARGETS triton-common-json EXPORT triton-common-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) endif() if(${TRITON_COMMON_ENABLE_GRPC} OR ${TRITON_COMMON_ENABLE_PROTOBUF}) install( TARGETS proto-library triton-common-model-config # proto-py-library EXPORT triton-common-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) endif() if(${TRITON_COMMON_ENABLE_GRPC}) install( TARGETS grpc-health-library grpc-service-library # grpc-service-py-library EXPORT triton-common-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) endif() install( DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) install( EXPORT triton-common-targets FILE TritonCommonTargets.cmake NAMESPACE TritonCommon:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonCommonConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonCommonConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonCommonConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-common-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonCommonTargets.cmake NAMESPACE TritonCommon:: ) export(PACKAGE TritonCommon) kytensor-2.49.0.7/common/LICENSE000066400000000000000000000027151527052371200161370ustar00rootroot00000000000000Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/common/README.md000066400000000000000000000041301527052371200164020ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Triton Inference Server Common Common source, scripts and utilities shared across all Triton repositories. This repo is not typically built directly but is instead included in the build of other repos. To build directly first install the required dependencies. ``` $ apt-get install rapidjson-dev ``` Use cmake 3.17 or later to build and install in a local directory. ``` $ mkdir build $ cd build $ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install .. $ make install ``` kytensor-2.49.0.7/common/cmake/000077500000000000000000000000001527052371200162055ustar00rootroot00000000000000kytensor-2.49.0.7/common/cmake/TritonCommonConfig.cmake.in000066400000000000000000000042131527052371200233720ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @PACKAGE_INIT@ set_and_check(TRITONCOMMON_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}") list(APPEND CMAKE_MODULE_PATH ${TRITONCOMMON_CMAKE_DIR}) include(CMakeFindDependencyMacro) find_dependency(Threads) if(NOT TARGET TritonCommon::triton-common-json) include("${TRITONCOMMON_CMAKE_DIR}/TritonCommonTargets.cmake") endif() check_required_components(triton-common-json triton-common-sync-queue triton-common-async-work-queue triton-common-thread-pool ) set(TRITONCOMMON_LIBRARIES TritonCommon::triton-common-json TritonCommon::triton-common-sync-queue TritonCommon::triton-common-async-work-queue TritonCommon::triton-common-thread-pool ) kytensor-2.49.0.7/common/include/000077500000000000000000000000001527052371200165505ustar00rootroot00000000000000kytensor-2.49.0.7/common/include/triton/000077500000000000000000000000001527052371200200675ustar00rootroot00000000000000kytensor-2.49.0.7/common/include/triton/common/000077500000000000000000000000001527052371200213575ustar00rootroot00000000000000kytensor-2.49.0.7/common/include/triton/common/async_work_queue.h000066400000000000000000000047141527052371200251210ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "error.h" #include "thread_pool.h" namespace triton { namespace common { // Manager for asynchronous worker threads. Use to accelerate copies and // other such operations by running them in parallel. // Call Initialize to start the worker threads (once) and AddTask to tasks to // the queue. class AsyncWorkQueue { public: // Start 'worker_count' number of worker threads. static Error Initialize(size_t worker_count); // Get the number of worker threads. static size_t WorkerCount(); // Add a 'task' to the queue. The function will take ownership of 'task'. // Therefore std::move should be used when calling AddTask. static Error AddTask(std::function&& task); protected: static void Reset(); private: AsyncWorkQueue() = default; ~AsyncWorkQueue(); static AsyncWorkQueue* GetSingleton(); std::unique_ptr thread_pool_; }; }} // namespace triton::common kytensor-2.49.0.7/common/include/triton/common/error.h000066400000000000000000000053271527052371200226700ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include namespace triton { namespace common { // // Error // // Error returned by utilities from common repo. // class Error { public: enum class Code : uint8_t { SUCCESS, UNKNOWN, INTERNAL, NOT_FOUND, INVALID_ARG, UNAVAILABLE, UNSUPPORTED, ALREADY_EXISTS, CANCELLED }; explicit Error(Code code = Code::SUCCESS) : code_(code) {} explicit Error(Code code, const std::string& msg) : code_(code), msg_(msg) {} // Convenience "success" value. Can be used as Error::Success to // indicate no error. static const Error Success; // Return the code for this status. Code ErrorCode() const { return code_; } // Return the message for this status. const std::string& Message() const { return msg_; } // Return true if this status indicates "ok"/"success", false if // status indicates some kind of failure. bool IsOk() const { return code_ == Code::SUCCESS; } // Return the status as a string. std::string AsString() const; // Return the constant string name for a code. static const char* CodeString(const Code code); protected: Code code_; std::string msg_; }; }} // namespace triton::common kytensor-2.49.0.7/common/include/triton/common/logging.h000066400000000000000000000307051527052371200231630ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include "table_printer.h" #ifdef _WIN32 // exclude winsock apis #define WIN32_LEAN_AND_MEAN // suppress the min and max definitions in Windef.h. #define NOMINMAX #include #else #include #include #include #include #endif namespace triton { namespace common { // Global logger for messages. Controls how log messages are reported. class Logger { public: // Log Formats. enum class Format { kDEFAULT, kISO8601 }; // Log levels. enum class Level : uint8_t { kERROR = 0, kWARNING = 1, kINFO = 2, kEND }; inline static const std::array(Level::kEND)> LEVEL_NAMES{"E", "W", "I"}; Logger(); // Is a log level enabled. bool IsEnabled(Level level) const { return enables_[static_cast(level)]; } // Set enable for a log Level. void SetEnabled(Level level, bool enable) { enables_[static_cast(level)] = enable; } // Get the current verbose logging level. uint32_t VerboseLevel() const { return vlevel_; } // Set the current verbose logging level. void SetVerboseLevel(uint32_t vlevel) { vlevel_ = vlevel; } // Whether to escape log messages // using JSON string escaping rules. // Default is true but can be disabled by setting // the following environment variable to '0'. // If the variable is unset or set to any value !='0' // log messages will be escaped // // TRITON_SERVER_ESCAPE_LOG_MESSAGES=0 bool EscapeLogMessages() const { return escape_log_messages_; }; // Get the logging format. Format LogFormat() { return format_; } // Get the logging format as a string. std::string LogFormatString() { switch (format_) { case Format::kISO8601: return "ISO8601"; case Format::kDEFAULT: return "default"; default: return "Invalid format"; } } // Set the logging format. void SetLogFormat(Format format) { format_ = format; } // Get the log output file name. const std::string& LogFile() { return filename_; } // Set the log output file. Returns an empty string upon // success, else returns an error string. const std::string SetLogFile(const std::string& filename) { const std::lock_guard lock(mutex_); file_stream_.close(); std::string revert_name(filename_); filename_ = filename; if (!filename_.empty()) { file_stream_.open(filename_, std::ios::app); if (file_stream_.fail()) { std::stringstream error; error << __FILE__ << " " << __LINE__ << ": Failed to open log file: " << std::strerror(errno) << std::endl; filename_ = revert_name; file_stream_.open(filename_, std::ios::app); return error.str(); } } // will return an empty string return std::string(); } // Log a message. void Log(const std::string& msg); // Flush the log. void Flush(); private: inline static const char* ESCAPE_ENVIRONMENT_VARIABLE = "TRITON_SERVER_ESCAPE_LOG_MESSAGES"; bool escape_log_messages_; std::array(Level::kEND)> enables_; uint32_t vlevel_; Format format_; std::mutex mutex_; std::string filename_; std::ofstream file_stream_; }; extern Logger gLogger_; // A log message. class LogMessage { public: LogMessage( const char* file, int line, Logger::Level level, const char* heading = nullptr, bool escape_log_messages = gLogger_.EscapeLogMessages()) : path_(file), line_(line), level_(level), pid_(GetProcessId()), heading_(heading), escape_log_messages_(escape_log_messages) { SetTimestamp(); size_t path_start = path_.rfind('/'); if (path_start != std::string::npos) { path_ = path_.substr(path_start + 1, std::string::npos); } } ~LogMessage(); std::stringstream& stream() { return message_; } private: std::string path_; const int line_; const Logger::Level level_; const uint32_t pid_; void LogPreamble(std::stringstream& stream); void LogTimestamp(std::stringstream& stream); #ifdef _WIN32 SYSTEMTIME timestamp_; void SetTimestamp() { GetSystemTime(×tamp_); } static uint32_t GetProcessId() { return static_cast(GetCurrentProcessId()); }; #else struct timeval timestamp_; void SetTimestamp() { gettimeofday(×tamp_, NULL); } static uint32_t GetProcessId() { return static_cast(getpid()); }; #endif std::stringstream message_; const char* heading_; bool escape_log_messages_; }; #define LOG_ENABLE_INFO(E) \ triton::common::gLogger_.SetEnabled(triton::common::Logger::Level::kINFO, (E)) #define LOG_ENABLE_WARNING(E) \ triton::common::gLogger_.SetEnabled( \ triton::common::Logger::Level::kWARNING, (E)) #define LOG_ENABLE_ERROR(E) \ triton::common::gLogger_.SetEnabled( \ triton::common::Logger::Level::kERROR, (E)) #define LOG_SET_VERBOSE(L) \ triton::common::gLogger_.SetVerboseLevel( \ static_cast(std::max(0, (L)))) #define LOG_SET_OUT_FILE(FN) triton::common::gLogger_.SetLogFile((FN)) #define LOG_SET_FORMAT(F) triton::common::gLogger_.SetLogFormat((F)) #define LOG_VERBOSE_LEVEL triton::common::gLogger_.VerboseLevel() #define LOG_FORMAT triton::common::gLogger_.LogFormat() #define LOG_FORMAT_STRING triton::common::gLogger_.LogFormatString() #define LOG_FILE triton::common::gLogger_.LogFile() #ifdef TRITON_ENABLE_LOGGING #define LOG_INFO_IS_ON \ triton::common::gLogger_.IsEnabled(triton::common::Logger::Level::kINFO) #define LOG_WARNING_IS_ON \ triton::common::gLogger_.IsEnabled(triton::common::Logger::Level::kWARNING) #define LOG_ERROR_IS_ON \ triton::common::gLogger_.IsEnabled(triton::common::Logger::Level::kERROR) #define LOG_VERBOSE_IS_ON(L) (triton::common::gLogger_.VerboseLevel() >= (L)) #else // If logging is disabled, define macro to be false to avoid further evaluation #define LOG_INFO_IS_ON false #define LOG_WARNING_IS_ON false #define LOG_ERROR_IS_ON false #define LOG_VERBOSE_IS_ON(L) false #endif // TRITON_ENABLE_LOGGING // Macros that use explicitly given filename and line number. #define LOG_INFO_FL(FN, LN) \ if (LOG_INFO_IS_ON) \ triton::common::LogMessage( \ (char*)(FN), LN, triton::common::Logger::Level::kINFO) \ .stream() #define LOG_WARNING_FL(FN, LN) \ if (LOG_WARNING_IS_ON) \ triton::common::LogMessage( \ (char*)(FN), LN, triton::common::Logger::Level::kWARNING) \ .stream() #define LOG_ERROR_FL(FN, LN) \ if (LOG_ERROR_IS_ON) \ triton::common::LogMessage( \ (char*)(FN), LN, triton::common::Logger::Level::kERROR) \ .stream() #define LOG_VERBOSE_FL(L, FN, LN) \ if (LOG_VERBOSE_IS_ON(L)) \ triton::common::LogMessage( \ (char*)(FN), LN, triton::common::Logger::Level::kINFO) \ .stream() // Macros that use current filename and line number. #define LOG_INFO LOG_INFO_FL(__FILE__, __LINE__) #define LOG_WARNING LOG_WARNING_FL(__FILE__, __LINE__) #define LOG_ERROR LOG_ERROR_FL(__FILE__, __LINE__) #define LOG_VERBOSE(L) LOG_VERBOSE_FL(L, __FILE__, __LINE__) // Macros for use with triton::common::table_printer objects // // Data is assumed to be server / backend generated // and not for use with client input. // // Tables are printed without escaping #define LOG_TABLE_VERBOSE(L, TABLE) \ \ do { \ if (LOG_VERBOSE_IS_ON(L)) \ triton::common::LogMessage( \ __FILE__, __LINE__, triton::common::Logger::Level::kINFO, nullptr, \ false) \ .stream() \ << TABLE.PrintTable(); \ } while (false) #define LOG_TABLE_INFO(TABLE) \ do { \ if (LOG_INFO_IS_ON) \ triton::common::LogMessage( \ __FILE__, __LINE__, triton::common::Logger::Level::kINFO, nullptr, \ false) \ .stream() \ << TABLE.PrintTable(); \ } while (false) // Macros for use with protobuf messages // // Data is serialized via DebugString() // // Data is printed without further escaping #define LOG_PROTOBUF_VERBOSE(L, HEADING, PB_MESSAGE) \ do { \ if (LOG_VERBOSE_IS_ON(L)) \ triton::common::LogMessage( \ __FILE__, __LINE__, triton::common::Logger::Level::kINFO, HEADING, \ false) \ .stream() \ << PB_MESSAGE.DebugString(); \ } while (false) // Macros for logging errors #define LOG_STATUS_ERROR(X, MSG) \ do { \ const Status& status__ = (X); \ if (!status__.IsOk()) { \ LOG_ERROR << (MSG) << ": " << status__.AsString(); \ } \ } while (false) #define LOG_TRITONSERVER_ERROR(X, MSG) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_ERROR << (MSG) << ": " << TRITONSERVER_ErrorCodeString(err__) \ << " - " << TRITONSERVER_ErrorMessage(err__); \ TRITONSERVER_ErrorDelete(err__); \ } \ } while (false) #define LOG_FLUSH triton::common::gLogger_.Flush() }} // namespace triton::common kytensor-2.49.0.7/common/include/triton/common/model_config.h000066400000000000000000000243221527052371200241600ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "model_config.pb.h" namespace triton { namespace common { /// The type for a repeated dims field (used for shape). using DimsList = ::google::protobuf::RepeatedField<::google::protobuf::int64>; /// The type for the metric_tags map. using MetricTagsMap = ::google::protobuf::Map; // Map from a host policy name to map of cmdline // settings for the host policy. using HostPolicyCmdlineConfig = std::map; using HostPolicyCmdlineConfigMap = std::unordered_map; // Map from backend name to list of setting=value pairs of cmdline // settings for the backend. using BackendCmdlineConfig = std::vector>; using BackendCmdlineConfigMap = std::unordered_map; /// The value for a dimension in a shape that indicates that that /// dimension can take on any size. constexpr int WILDCARD_DIM = -1; constexpr int SCHEDULER_DEFAULT_NICE = 5; /// Enumeration for the different platform types. enum Platform { PLATFORM_UNKNOWN = 0, PLATFORM_TENSORRT_PLAN = 1, PLATFORM_TENSORFLOW_GRAPHDEF = 2, PLATFORM_TENSORFLOW_SAVEDMODEL = 3, PLATFORM_ENSEMBLE = 4, PLATFORM_ONNXRUNTIME_ONNX = 5, PLATFORM_PYTORCH_LIBTORCH = 6 }; /// Get the number of elements in a shape. /// \param dims The shape. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const DimsList& dims); /// Get the number of elements in a shape. /// \param dims The shape. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const std::vector& dims); /// Get the number of elements in the shape of a model input. /// \param mio The model input. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const inference::ModelInput& mio); /// Get the number of elements in the shape of a model output. /// \param mio The model output. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const inference::ModelOutput& mio); /// Are values of a datatype fixed-size, or variable-sized. /// \param dtype The data-type. /// \return True if datatype values are fixed-sized, false if /// variable-sized. bool IsFixedSizeDataType(const inference::DataType dtype); /// Get the size of objects of a given datatype in bytes. /// \param dtype The data-type. /// \return The size, in bytes, of objects of the datatype, or 0 if /// size cannot be determine (for example, values of type TYPE_STRING /// have variable length and so size cannot be determine just from the /// type). size_t GetDataTypeByteSize(const inference::DataType dtype); /// Get the size, in bytes, of a tensor based on datatype and /// shape. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize(const inference::DataType& dtype, const DimsList& dims); /// Get the size, in bytes, of a tensor based on datatype and /// shape. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize( const inference::DataType& dtype, const std::vector& dims); /// Get the size, in bytes, of a tensor based on batch-size, datatype /// and shape. A tensor that has empty shape [] and non-zero /// batch-size is sized as a tensor with shape [ batch-size ]. /// \param batch_size The batch-size. May be 0 to indicate no /// batching. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize( const int batch_size, const inference::DataType& dtype, const DimsList& dims); /// Get the size, in bytes, of a tensor based on batch-size, datatype /// and shape. A tensor that has empty shape [] and non-zero /// batch-size is sized as a tensor with shape [ batch-size ]. /// \param batch_size The batch-size. May be 0 to indicate no /// batching. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize( const int batch_size, const inference::DataType& dtype, const std::vector& dims); /// Get the size, in bytes, of a tensor based on ModelInput. /// \param mio The ModelInput protobuf. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize(const inference::ModelInput& mio); /// Get the size, in bytes, of a tensor based on ModelOutput. /// \param mio The ModelOutput protobuf. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize(const inference::ModelOutput& mio); /// Get the CPU thread nice level associate with a model /// configuration's priority. /// \param config The model configuration. /// \return The nice level. int GetCpuNiceLevel(const inference::ModelConfig& config); /// Compare two model configuration shapes for equality. Wildcard /// dimensions (that is, dimensions with size WILDCARD_DIM) are /// compared literally so that to be equal the two shapes must both /// specify WILDCARD_DIM in the same dimensions. /// \params dims0 The first shape. /// \params dims1 The second shape. /// \return True if the shapes are equal, false if not equal. bool CompareDims(const DimsList& dims0, const DimsList& dims1); /// Compare two model configuration shapes for equality. Wildcard /// dimensions (that is, dimensions with size WILDCARD_DIM) are /// compared literally so that to be equal the two shapes must both /// specify WILDCARD_DIM in the same dimensions. /// \params dims0 The first shape. /// \params dims1 The second shape. /// \return True if the shapes are equal, false if not equal. bool CompareDims( const std::vector& dims0, const std::vector& dims1); /// Compare two model configuration shapes for equality. Wildcard /// dimensions (that is, dimensions with size WILDCARD_DIM) are /// allowed to match with any value. So, a dimension in one shape /// specified as WILDCARD_DIM will always match the same dimension in /// the other shape. /// \params dims0 The first shape. /// \params dims1 The second shape. /// \return True if the shapes are equal, false if not equal. bool CompareDimsWithWildcard(const DimsList& dims0, const DimsList& dims1); /// Compare two model configuration shapes for equality. Wildcard /// dimensions (that is, dimensions with size WILDCARD_DIM) are /// allowed to match with any value. So, a dimension in one shape /// specified as WILDCARD_DIM will always match the same dimension in /// the other shape. /// \params dims0 The first shape. /// \params dims1 The second shape. /// \return True if the shapes are equal, false if not equal. bool CompareDimsWithWildcard( const DimsList& dims0, const std::vector& dims1); /// Convert a DimsList to string representation. /// \param dims The DimsList to be converted. /// \return String representation of the DimsList in pattern /// "[d0,d1,...,dn]" std::string DimsListToString(const DimsList& dims); /// Convert a vector representing a shape to string representation. /// \param dims The vector of dimensions to be converted. /// \return String representation of the vector in pattern /// "[d0,d1,...,dn]" std::string DimsListToString( const std::vector& dims, const int start_idx = 0); /// Get the server protocol string representation of a datatype. /// \param dtype The data type. /// \return The string representation. const char* DataTypeToProtocolString(const inference::DataType dtype); /// Get the datatype corresponding to a server protocol string /// representation of a datatype. /// \param dtype string representation. /// \return The data type. inference::DataType ProtocolStringToDataType(const std::string& dtype); /// Get the datatype corresponding to a server protocol string /// representation of a datatype. /// \param dtype Pointer to string. /// \param len Length of the string. /// \return The data type. inference::DataType ProtocolStringToDataType(const char* dtype, size_t len); }} // namespace triton::common kytensor-2.49.0.7/common/include/triton/common/nvtx.h000066400000000000000000000101331527052371200225250ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef TRITON_ENABLE_NVTX #include namespace triton { namespace common { namespace detail { class NvtxTritonDomain { public: static nvtxDomainHandle_t& GetDomain() { static NvtxTritonDomain inst; return inst.triton_nvtx_domain_; } private: NvtxTritonDomain() { triton_nvtx_domain_ = nvtxDomainCreateA("Triton"); } ~NvtxTritonDomain() { nvtxDomainDestroy(triton_nvtx_domain_); } nvtxDomainHandle_t triton_nvtx_domain_; }; } // namespace detail // Updates a server stat with duration measured by a C++ scope. class NvtxRange { public: explicit NvtxRange(const char* label, uint32_t rgb = kNvGreen) { auto attr = GetAttributes(label, rgb); nvtxDomainRangePushEx(detail::NvtxTritonDomain::GetDomain(), &attr); } explicit NvtxRange(const std::string& label, uint32_t rgb = kNvGreen) : NvtxRange(label.c_str(), rgb) { } ~NvtxRange() { nvtxDomainRangePop(detail::NvtxTritonDomain::GetDomain()); } static constexpr uint32_t kNvGreen = 0x76b900; static constexpr uint32_t kRed = 0xc1121f; static constexpr uint32_t kGreen = 0x588157; static constexpr uint32_t kBlue = 0x023047; static constexpr uint32_t kYellow = 0xffb703; static constexpr uint32_t kOrange = 0xfb8500; private: nvtxEventAttributes_t GetAttributes(const char* label, uint32_t rgb) { nvtxEventAttributes_t attr; attr.version = NVTX_VERSION; attr.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; attr.colorType = NVTX_COLOR_ARGB; attr.color = rgb | 0xff000000; attr.messageType = NVTX_MESSAGE_TYPE_ASCII; attr.message.ascii = label; return attr; } }; }} // namespace triton::common #endif // TRITON_ENABLE_NVTX // // Macros to access NVTX functionality. // For `NVTX_RANGE` macro please refer to the usage below. // #ifdef TRITON_ENABLE_NVTX #define NVTX_INITIALIZE nvtxInitialize(nullptr) #define NVTX_RANGE1(V, L) triton::common::NvtxRange V(L) #define NVTX_RANGE2(V, L, RGB) triton::common::NvtxRange V(L, RGB) #define NVTX_MARKER(L) nvtxMarkA(L) #else #define NVTX_INITIALIZE #define NVTX_RANGE1(V, L) #define NVTX_RANGE2(V, L, RGB) #define NVTX_MARKER(L) #endif // TRITON_ENABLE_NVTX // "Overload" for `NVTX_RANGE` macro. // Usage: // NVTX_RANGE(nvtx1, "My message") -> Records NVTX marker with kNvGreen color. // NVTX_RANGE(nvtx1, "My message", NvtxRange::kRed) -> Records NVTX marker with // kRed color. #define GET_NVTX_MACRO(_1, _2, _3, NAME, ...) NAME #define NVTX_RANGE(...) \ GET_NVTX_MACRO(__VA_ARGS__, NVTX_RANGE2, NVTX_RANGE1)(__VA_ARGS__) kytensor-2.49.0.7/common/include/triton/common/sync_queue.h000066400000000000000000000050051527052371200237100ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include namespace triton { namespace common { // // C++11 doesn't have a sync queue so we implement a simple one. // template class SyncQueue { public: SyncQueue() {} bool Empty() { std::lock_guard lk(mu_); return queue_.empty(); } Item Get() { std::unique_lock lk(mu_); if (queue_.empty()) { cv_.wait(lk, [this] { return !queue_.empty(); }); } auto res = std::move(queue_.front()); queue_.pop_front(); return res; } void Put(const Item& value) { { std::lock_guard lk(mu_); queue_.push_back(value); } cv_.notify_all(); } void Put(Item&& value) { { std::lock_guard lk(mu_); queue_.push_back(std::move(value)); } cv_.notify_all(); } private: std::mutex mu_; std::condition_variable cv_; std::deque queue_; }; }} // namespace triton::common kytensor-2.49.0.7/common/include/triton/common/table_printer.h000066400000000000000000000057061527052371200243720ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include namespace triton { namespace common { // // An ASCII table printer. // class TablePrinter { public: // Insert a row at the end of the table void InsertRow(const std::vector& row); // Print the table std::string PrintTable(); // TablePrinter will take the ownership of `headers`. TablePrinter(const std::vector& headers); private: // Update the `shares_` such that all the excess // amount of space not used a column is fairly allocated // to the other columns void FairShare(); // Append a row to `table`. This function handles the cases where a wrapping // occurs. void AddRow(std::stringstream& table, size_t row_index); // Add a row divider void AddRowDivider(std::stringstream& table); // Max row width std::vector max_widths_; // Max row height std::vector max_heights_; // A vector of vectors of vectors containing data items for every column // The record is stored in a vector of string, where each of the vector items // contains a single line from the record. For example, ["Item 1", "Item 2", // "Item 3\n Item 3 line 2"] will be stored as [["Item 1"], ["Item 2"], ["Item // 3", "Item 3 line 2"]] std::vector>> data_; // Fair share of every column std::vector shares_; }; }} // namespace triton::common kytensor-2.49.0.7/common/include/triton/common/thread_pool.h000066400000000000000000000050411527052371200240300ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include namespace triton { namespace common { // Generic fixed-size Thread Pool to execute tasks asynchronously class ThreadPool { public: explicit ThreadPool(std::size_t thread_count); ~ThreadPool(); ThreadPool(const ThreadPool&) = delete; ThreadPool& operator=(const ThreadPool&) = delete; using Task = std::function; // Assigns "task" to the task queue for a worker thread to execute when // available. This will not track the return value of the task. void Enqueue(Task&& task); // Returns the number of threads in thread pool size_t Size() { return workers_.size(); } private: std::queue task_queue_; std::mutex queue_mtx_; std::condition_variable cv_; std::vector workers_; // If true, tells pool to stop accepting work and tells awake worker threads // to exit when no tasks are left on the queue. bool stop_ = false; }; }} // namespace triton::common kytensor-2.49.0.7/common/include/triton/common/triton_json.h000066400000000000000000001244371527052371200241130ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef _WIN32 // Remove GetObject definition from windows.h, which prevents calls to // RapidJSON's GetObject. // https://github.com/Tencent/rapidjson/issues/1448 #undef GetObject #include #else // Disable class-memaccess warning to facilitate compilation with gcc>7 // https://github.com/Tencent/rapidjson/issues/1700 #pragma GCC diagnostic push #if defined(__GNUC__) && __GNUC__ >= 8 #pragma GCC diagnostic ignored "-Wclass-memaccess" #endif #include #pragma GCC diagnostic pop #endif // _WIN32 #include // CrtAllocator (default) for Writer instantiation #include // UTF8 (default) for Writer instantiation #include #include #include #include #include #include #include // This header can be used both within Triton server and externally // (i.e. in source that interacts only via TRITONSERVER or // TRITONBACKEND API). Status is handled differently in these cases so // the following macros must be defined before including this // header. As an example the defines are shown here as returned by the // TRITONSERVER API. // // #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* // #define TRITONJSON_STATUSRETURN(M) // return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) // #define TRITONJSON_STATUSSUCCESS nullptr namespace triton { namespace common { // // A JSON parser/writer. Currently based on rapidjson but the intent // is to provide an abstraction for JSON functions that make it easy // to substitute a different JSON parser. Specifically for rapidjson // the class is also designed to provide safe access and error // reporting to avoid the cases where rapidjson would just abort the // entire application (!). // class TritonJson { public: class Value; enum class ValueType { OBJECT = rapidjson::kObjectType, ARRAY = rapidjson::kArrayType, }; // // Buffer used when writing JSON representation. // class WriteBuffer { public: // Get buffer base address. const char* Base() const { return buffer_.c_str(); } // Get a reference to the buffer itself. Useful to efficiently // move the contents out of the buffer. std::string& MutableContents() { return buffer_; } // Immutable contents. const std::string& Contents() const { return buffer_; } // Interface required by rapidjson::Writer typedef char Ch; void Put(char c) { buffer_.push_back(c); } void Clear() { buffer_.clear(); } void Flush() { return; } size_t Size() const { return buffer_.size(); } private: std::string buffer_; }; // // Utility to serialize input string // as a JSON string value static std::string SerializeString(const std::string& input) { WriteBuffer writebuffer; const unsigned int writeFlags = rapidjson::kWriteNanAndInfFlag; // Provide default template arguments to pass writeFlags rapidjson::Writer< WriteBuffer, rapidjson::UTF8<>, rapidjson::UTF8<>, rapidjson::CrtAllocator, writeFlags> writer(writebuffer); if (RAPIDJSON_UNLIKELY(!writer.String(input.c_str()))) { return "Error Serializing String"; } return writebuffer.Contents(); } // // Value representing the entire document or an element within a // document. // class Value { public: // Empty value. Will become a top-level Document value if // initialized by parsing or a non-top-level value if initialized // any other way. explicit Value() : value_(nullptr), allocator_(nullptr) {} // Construct a top-level JSON document. explicit Value(const ValueType type) : document_(static_cast(type)), value_(nullptr), allocator_(&document_.GetAllocator()) { } // Construct a non-top-level JSON value in a 'document'. explicit Value(TritonJson::Value& document, const ValueType type) { allocator_ = &document.document_.GetAllocator(); value_ = new (allocator_->Malloc(sizeof(rapidjson::Value))) rapidjson::Value(static_cast(type)); } // Move constructor. explicit Value(Value&& other) { *this = std::move(other); } // Move assignment operator. Value& operator=(Value&& other) { document_ = std::move(other.document_); value_ = other.value_; allocator_ = other.allocator_; other.value_ = nullptr; other.allocator_ = nullptr; return *this; } // Parse JSON into document. Can only be called on top-level // document value, otherwise error is returned. TRITONJSON_STATUSTYPE Parse(const char* base, const size_t size) { if (value_ != nullptr) { TRITONJSON_STATUSRETURN( std::string("JSON parsing only available for top-level document")); } const unsigned int parseFlags = rapidjson::kParseNanAndInfFlag; document_.Parse(base, size); if (document_.HasParseError()) { TRITONJSON_STATUSRETURN(std::string( "failed to parse the request JSON buffer: " + std::string(GetParseError_En(document_.GetParseError())) + " at " + std::to_string(document_.GetErrorOffset()))); } allocator_ = &document_.GetAllocator(); return TRITONJSON_STATUSSUCCESS; } // \see Parse(const char* base, const size_t size) TRITONJSON_STATUSTYPE Parse(const std::string& json) { return Parse(json.data(), json.size()); } // Write JSON representation into a 'buffer' in a compact // format. Can only be called for a top-level document value, // otherwise error is returned. TRITONJSON_STATUSTYPE Write(WriteBuffer* buffer) const { if (value_ != nullptr) { TRITONJSON_STATUSRETURN( std::string("JSON writing only available for top-level document")); } const unsigned int writeFlags = rapidjson::kWriteNanAndInfFlag; // Provide default template arguments to pass writeFlags rapidjson::Writer< WriteBuffer, rapidjson::UTF8<>, rapidjson::UTF8<>, rapidjson::CrtAllocator, writeFlags> writer(*buffer); if (!document_.Accept(writer)) { TRITONJSON_STATUSRETURN( std::string("Failed to accept document, invalid JSON.")); } return TRITONJSON_STATUSSUCCESS; } // Write JSON representation into a 'buffer' in an easy-to-read // format. Can only be called for a top-level document value, // otherwise error is returned. TRITONJSON_STATUSTYPE PrettyWrite(WriteBuffer* buffer) const { if (value_ != nullptr) { TRITONJSON_STATUSRETURN( std::string("JSON writing only available for top-level document")); } // Can't pass writeFlags with latest release v1.1.0 of rapidjson-dev. // We would need to build rapidjson from source to capture latest fixes. // See this issue: // https://github.com/Tencent/rapidjson/issues/905#issuecomment-370981353 // PrettyWrite is only used for displaying model configs currently, so // this should not be an issue. rapidjson::PrettyWriter writer(*buffer); if (!document_.Accept(writer)) { TRITONJSON_STATUSRETURN( std::string("Failed to accept document, invalid JSON.")); } return TRITONJSON_STATUSSUCCESS; } // Swap a value with another. TRITONJSON_STATUSTYPE Swap(TritonJson::Value& other) { rapidjson::Value& value = AsMutableValue(); value.Swap(other.AsMutableValue()); return TRITONJSON_STATUSSUCCESS; } // Set/overwrite a boolean in a value. This changes the // type of the value to boolean. TRITONJSON_STATUSTYPE SetBool(const bool value) { rapidjson::Value& v = AsMutableValue(); v.SetBool(value); return TRITONJSON_STATUSSUCCESS; } // Set/overwrite a signed integer in a value. This changes the // type of the value to signed int. TRITONJSON_STATUSTYPE SetInt(const int64_t value) { rapidjson::Value& v = AsMutableValue(); v.SetInt64(value); return TRITONJSON_STATUSSUCCESS; } // Set/overwrite an unsigned integer in a value. This changes the // type of the value to unsigned int. TRITONJSON_STATUSTYPE SetUInt(const uint64_t value) { rapidjson::Value& v = AsMutableValue(); v.SetUint64(value); return TRITONJSON_STATUSSUCCESS; } // Set/overwrite a double in a value. This changes the // type of the value to double. TRITONJSON_STATUSTYPE SetDouble(const double value) { rapidjson::Value& v = AsMutableValue(); v.SetDouble(value); return TRITONJSON_STATUSSUCCESS; } // Set/overwrite a string in a value. This changes the // type of the value to string TRITONJSON_STATUSTYPE SetString(const std::string& value) { rapidjson::Value& v = AsMutableValue(); v.SetString(value.c_str(), value.length(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Set/overwrite a string member with provided name and value in this object TRITONJSON_STATUSTYPE SetStringObject( const char* name, const std::string& value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add/replace JSON member '") + name + "' to non-object"); } auto itr = object.FindMember(name); if (itr == object.MemberEnd()) { AddString(name, value); } else { object.RemoveMember(itr); object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::Value(value.c_str(), value.size(), *allocator_), *allocator_); } return TRITONJSON_STATUSSUCCESS; } // Add an array or object as a new member to this value. 'value' // is moved into this value and so on return 'value' should not be // used. It is assumed that 'name' can be used by reference, it is // the caller's responsibility to make sure the lifetime of 'name' // extends at least as long as the object. TRITONJSON_STATUSTYPE Add(const char* name, TritonJson::Value&& value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } if (value.value_ == nullptr) { rapidjson::Value v2; v2.CopyFrom(value.document_, *allocator_); object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), v2.Move(), *allocator_); } else { object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), value.value_->Move(), *allocator_); } value.Release(); return TRITONJSON_STATUSSUCCESS; } // Add a copy of a string as a new member to this value. It is // assumed that 'name' can be used by reference, it is the // caller's responsibility to make sure the lifetime of 'name' // extends at least as long as the object. TRITONJSON_STATUSTYPE AddString(const char* name, const std::string& value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::Value(value.c_str(), value.size(), *allocator_).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Add a copy of a explicit-length string as a new member to this // value. It is assumed that 'name' can be used by reference, it // is the caller's responsibility to make sure the lifetime of // 'name' extends at least as long as the object. TRITONJSON_STATUSTYPE AddString( const char* name, const char* value, const size_t len) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::Value(value, len, *allocator_).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Add a reference to a string as a new member to this value. It // is assumed that 'name' and 'value' can be used by reference, it // is the caller's responsibility to make sure the lifetime of // 'name' and 'value' extend at least as long as the object. TRITONJSON_STATUSTYPE AddStringRef(const char* name, const char* value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::StringRef(value), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Add a reference to a explicit-length string as a new member to // this value. It is assumed that 'name' and 'value' can be used // by reference, it is the caller's responsibility to make sure // the lifetime of 'name' and 'value' extend at least as long as // the object. TRITONJSON_STATUSTYPE AddStringRef( const char* name, const char* value, const size_t len) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::StringRef(value, len), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Add a boolean new member to this value. It is assumed that // 'name' can be used by reference, it is the caller's // responsibility to make sure the lifetime of 'name' extends at // least as long as the object. TRITONJSON_STATUSTYPE AddBool(const char* name, const bool value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::Value(value).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Add a signed integer as a new member to this value. It is // assumed that 'name' can be used by reference, it is the // caller's responsibility to make sure the lifetime of 'name' // extends at least as long as the object. TRITONJSON_STATUSTYPE AddInt(const char* name, const int64_t value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::Value(value).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Add an unsigned integer as a new member to this value. It is // assumed that 'name' can be used by reference, it is the // caller's responsibility to make sure the lifetime of 'name' // extends at least as long as the object. TRITONJSON_STATUSTYPE AddUInt(const char* name, const uint64_t value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::Value(value).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Add a double as a new member to this value. It is assumed that // 'name' can be used by reference, it is the caller's // responsibility to make sure the lifetime of 'name' extends at // least as long as the object. TRITONJSON_STATUSTYPE AddDouble(const char* name, const double value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to add JSON member '") + name + "' to non-object"); } object.AddMember( rapidjson::Value(rapidjson::StringRef(name)).Move(), rapidjson::Value(value).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Append an array or object to this value, which must be an // array. 'value' is moved into this value and so on return // 'value' should not be used. TRITONJSON_STATUSTYPE Append(TritonJson::Value&& value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } if (value.value_ == nullptr) { rapidjson::Value v2; v2.CopyFrom(value.document_, *allocator_); array.PushBack(v2.Move(), *allocator_); } else { array.PushBack(value.value_->Move(), *allocator_); } value.Release(); return TRITONJSON_STATUSSUCCESS; } // Append a copy of a string to this value, which must be an // array. TRITONJSON_STATUSTYPE AppendString(const std::string& value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } array.PushBack( rapidjson::Value(value.c_str(), value.size(), *allocator_).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Append a copy of an explicit-length string to this value, which // must be an array. TRITONJSON_STATUSTYPE AppendString(const char* value, const size_t len) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } array.PushBack( rapidjson::Value(value, len, *allocator_).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Append a reference to a string to this value, which must be an // array. It is assumed that 'value' can be used by reference, it // is the caller's responsibility to make sure the lifetime of // 'value' extends at least as long as the object. TRITONJSON_STATUSTYPE AppendStringRef(const char* value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } array.PushBack(rapidjson::StringRef(value), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Append a reference to a explicit-length string to this value, // which must be an array. It is assumed that 'value' can be used // by reference, it is the caller's responsibility to make sure // the lifetime of 'value' extends at least as long as the object. TRITONJSON_STATUSTYPE AppendStringRef(const char* value, const size_t len) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } array.PushBack(rapidjson::StringRef(value, len), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Append a boolean to this value, which must be an array. TRITONJSON_STATUSTYPE AppendBool(const bool value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } array.PushBack(rapidjson::Value(value).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Append a signed integer to this value, which must be an array. TRITONJSON_STATUSTYPE AppendInt(const int64_t value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } array.PushBack(rapidjson::Value(value).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Append an unsigned integer to this value, which must be an // array. TRITONJSON_STATUSTYPE AppendUInt(const uint64_t value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } array.PushBack(rapidjson::Value(value).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Append a double to this value, which must be an array. TRITONJSON_STATUSTYPE AppendDouble(const double value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to append JSON member to non-array")); } array.PushBack(rapidjson::Value(value).Move(), *allocator_); return TRITONJSON_STATUSSUCCESS; } // Remove member from this object TRITONJSON_STATUSTYPE Remove(const char* name) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to remove JSON member '") + name + "' to non-object"); } auto itr = object.FindMember(name); if (itr != object.MemberEnd()) { object.RemoveMember(itr); } // else report success return TRITONJSON_STATUSSUCCESS; } // Check if this value is of the specified type. Return appropriate // error if not. TRITONJSON_STATUSTYPE AssertType(TritonJson::ValueType type) const { if (static_cast(type) != AsValue().GetType()) { TRITONJSON_STATUSRETURN(std::string("unexpected type")); } return TRITONJSON_STATUSSUCCESS; } // Get the size of an array. If called on non-array returns zero. size_t ArraySize() const { const rapidjson::Value& array = AsValue(); if (!array.IsArray()) { return 0; } return array.GetArray().Size(); } // Return the specified index contained in this array. TRITONJSON_STATUSTYPE At( const size_t idx, TritonJson::Value* value = nullptr) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } *value = TritonJson::Value(array[idx], allocator_); return TRITONJSON_STATUSSUCCESS; } // Get the names of all members in an object. Error if value is // not an object. TRITONJSON_STATUSTYPE Members(std::vector* names) const { const rapidjson::Value& object = AsValue(); if (!object.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to get members for non-object")); } for (const auto& m : object.GetObject()) { names->push_back(m.name.GetString()); } return TRITONJSON_STATUSSUCCESS; } // Return true if this value is an object and the named member is // contained in this object. bool Find(const char* name) const { const rapidjson::Value& object = AsValue(); return object.IsObject() && object.HasMember(name); } // Return true if this value is an object and the named member is // contained in this object. Return the member in 'value'. bool Find(const char* name, TritonJson::Value* value) { rapidjson::Value& object = AsMutableValue(); if (object.IsObject() && object.HasMember(name)) { if (value != nullptr) { *value = TritonJson::Value(object[name], allocator_); } return true; } return false; } // Whether the object is null value. Note that false will also be returned // if the object is not a JSON value. bool IsNull() const { return ((value_ != nullptr) && value_->IsNull()); } // Return true if the object is an object and it has no members; // false otherwise. bool IsEmpty() const { const rapidjson::Value& object = AsValue(); if (object.IsObject() && object.MemberCount() == 0) { return true; } return false; } // Get value as a string. The string may contain null or other // special characters and so 'len' must be used to determine length. // Error if value is not a string. TRITONJSON_STATUSTYPE AsString(const char** value, size_t* len) const { if ((value_ == nullptr) || !value_->IsString()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-string as string")); } *value = value_->GetString(); *len = value_->GetStringLength(); return TRITONJSON_STATUSSUCCESS; } // Get value as a string. The string may contain null or other // special characters. Error if value is not a string. TRITONJSON_STATUSTYPE AsString(std::string* str) const { if ((value_ == nullptr) || !value_->IsString()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-string as string")); } str->assign(value_->GetString(), value_->GetStringLength()); return TRITONJSON_STATUSSUCCESS; } // Get value as a boolean. Error if value is not a boolean. TRITONJSON_STATUSTYPE AsBool(bool* value) const { if ((value_ == nullptr) || !value_->IsBool()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-boolean as boolean")); } *value = value_->GetBool(); return TRITONJSON_STATUSSUCCESS; } // Get value as a signed integer. Error if value is not a signed // integer. TRITONJSON_STATUSTYPE AsInt(int64_t* value) const { if ((value_ == nullptr) || !value_->IsInt64()) { TRITONJSON_STATUSRETURN(std::string( "attempt to access JSON non-signed-integer as signed-integer")); } *value = value_->GetInt64(); return TRITONJSON_STATUSSUCCESS; } // Get value as an unsigned integer. Error if value is not an // unsigned integer. TRITONJSON_STATUSTYPE AsUInt(uint64_t* value) const { if ((value_ == nullptr) || !value_->IsUint64()) { TRITONJSON_STATUSRETURN(std::string( "attempt to access JSON non-unsigned-integer as unsigned-integer")); } *value = value_->GetUint64(); return TRITONJSON_STATUSSUCCESS; } // Get value as a double. Error if value is not a double. TRITONJSON_STATUSTYPE AsDouble(double* value) const { if ((value_ == nullptr) || !value_->IsNumber()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-number as double")); } *value = value_->GetDouble(); return TRITONJSON_STATUSSUCCESS; } // Get named array member contained in this object. TRITONJSON_STATUSTYPE MemberAsArray( const char* name, TritonJson::Value* value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject() || !object.HasMember(name)) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing object member '") + name + "'"); } auto& v = object[name]; if (!v.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-array as array")); } *value = TritonJson::Value(v, allocator_); return TRITONJSON_STATUSSUCCESS; } // Get named object member contained in this object. TRITONJSON_STATUSTYPE MemberAsObject( const char* name, TritonJson::Value* value) { rapidjson::Value& object = AsMutableValue(); if (!object.IsObject() || !object.HasMember(name)) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing object member '") + name + "'"); } auto& v = object[name]; if (!v.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-object as object")); } *value = TritonJson::Value(v, allocator_); return TRITONJSON_STATUSSUCCESS; } // Get object member as a string. The string may contain null or other // special characters and so 'len' must be used to determine length. // Error if this is not an object or if the member is not a string. TRITONJSON_STATUSTYPE MemberAsString( const char* name, const char** value, size_t* len) const { const rapidjson::Value& object = AsValue(); if (!object.IsObject() || !object.HasMember(name)) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing object member '") + name + "'"); } const auto& v = object[name]; if (!v.IsString()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-string as string")); } *value = v.GetString(); *len = v.GetStringLength(); return TRITONJSON_STATUSSUCCESS; } // Get object member as a string. The string may contain null or // other special characters. Error if this is not an object or if // the member is not a string. TRITONJSON_STATUSTYPE MemberAsString( const char* name, std::string* str) const { const rapidjson::Value& object = AsValue(); if (!object.IsObject() || !object.HasMember(name)) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing object member '") + name + "'"); } const auto& v = object[name]; if (!v.IsString()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-string as string")); } str->assign(v.GetString(), v.GetStringLength()); return TRITONJSON_STATUSSUCCESS; } // Get object member as a boolean. Error if this is not an object // or if the member is not a boolean. TRITONJSON_STATUSTYPE MemberAsBool(const char* name, bool* value) const { const rapidjson::Value& object = AsValue(); if (!object.IsObject() || !object.HasMember(name)) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing object member '") + name + "'"); } const auto& v = object[name]; if (!v.IsBool()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-boolean as boolean")); } *value = v.GetBool(); return TRITONJSON_STATUSSUCCESS; } // Get object member as a signed integer. Error if this is not an object // or if the member is not a signed integer. TRITONJSON_STATUSTYPE MemberAsInt(const char* name, int64_t* value) const { const rapidjson::Value& object = AsValue(); if (!object.IsObject() || !object.HasMember(name)) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing object member '") + name + "'"); } const auto& v = object[name]; if (!v.IsInt64()) { TRITONJSON_STATUSRETURN(std::string( "attempt to access JSON non-signed-integer as signed-integer")); } *value = v.GetInt64(); return TRITONJSON_STATUSSUCCESS; } // Get object member as an unsigned integer. Error if this is not an object // or if the member is not an unsigned integer. TRITONJSON_STATUSTYPE MemberAsUInt(const char* name, uint64_t* value) const { const rapidjson::Value& object = AsValue(); if (!object.IsObject() || !object.HasMember(name)) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing object member '") + name + "'"); } const auto& v = object[name]; if (!v.IsUint64()) { TRITONJSON_STATUSRETURN(std::string( "attempt to access JSON non-unsigned-integer as unsigned-integer")); } *value = v.GetUint64(); return TRITONJSON_STATUSSUCCESS; } // Get object member as a double. Error if this is not an object // or if the member is not a double. TRITONJSON_STATUSTYPE MemberAsDouble(const char* name, double* value) const { const rapidjson::Value& object = AsValue(); if (!object.IsObject() || !object.HasMember(name)) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing object member '") + name + "'"); } const auto& v = object[name]; if (!v.IsNumber()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-number as double")); } *value = v.GetDouble(); return TRITONJSON_STATUSSUCCESS; } bool IsString() { const rapidjson::Value& object = AsValue(); return object.IsString(); } bool IsArray() { const rapidjson::Value& object = AsValue(); return object.IsArray(); } bool IsObject() { const rapidjson::Value& object = AsValue(); return object.IsObject(); } bool IsInt() { const rapidjson::Value& object = AsValue(); return object.IsInt64(); } bool IsBool() { const rapidjson::Value& object = AsValue(); return object.IsBool(); } bool IsNumber() { const rapidjson::Value& object = AsValue(); return object.IsNumber(); } // Get array element at a given index within this array. TRITONJSON_STATUSTYPE IndexAsArray( const size_t idx, TritonJson::Value* value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } auto& v = array[idx]; if (!v.IsArray()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-array as array")); } *value = TritonJson::Value(v, allocator_); return TRITONJSON_STATUSSUCCESS; } // Get object element at a given index within this array. TRITONJSON_STATUSTYPE IndexAsObject( const size_t idx, TritonJson::Value* value) { rapidjson::Value& array = AsMutableValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } auto& v = array[idx]; if (!v.IsObject()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-object as object")); } *value = TritonJson::Value(v, allocator_); return TRITONJSON_STATUSSUCCESS; } // Get array index as a string. The string may contain null or // other special characters and so 'len' must be used to determine // length. Error if this is not an array or if the index element // is not a string. TRITONJSON_STATUSTYPE IndexAsString( const size_t idx, const char** value, size_t* len) const { const rapidjson::Value& array = AsValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } const auto& v = array[idx]; if (!v.IsString()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-string as string")); } *value = v.GetString(); *len = v.GetStringLength(); return TRITONJSON_STATUSSUCCESS; } // Get array index as a string. The string may contain null or // other special characters. Error if this is not an array or if // the index element is not a string. TRITONJSON_STATUSTYPE IndexAsString( const size_t idx, std::string* str) const { const rapidjson::Value& array = AsValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } const auto& v = array[idx]; if (!v.IsString()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-string as string")); } str->assign(v.GetString(), v.GetStringLength()); return TRITONJSON_STATUSSUCCESS; } // Get array index as a boolean. Error if this is not an array or // if the index element is not a boolean. TRITONJSON_STATUSTYPE IndexAsBool(const size_t idx, bool* value) const { const rapidjson::Value& array = AsValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } const auto& v = array[idx]; if (!v.IsBool()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-boolean as boolean")); } *value = v.GetBool(); return TRITONJSON_STATUSSUCCESS; } // Get array index as a signed integer. Error if this is not an array or // if the index element is not a signed integer. TRITONJSON_STATUSTYPE IndexAsInt(const size_t idx, int64_t* value) const { const rapidjson::Value& array = AsValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } const auto& v = array[idx]; if (!v.IsInt64()) { TRITONJSON_STATUSRETURN(std::string( "attempt to access JSON non-signed-integer as signed-integer")); } *value = v.GetInt64(); return TRITONJSON_STATUSSUCCESS; } // Get array index as an unsigned integer. Error if this is not an array or // if the index element is not an unsigned integer. TRITONJSON_STATUSTYPE IndexAsUInt(const size_t idx, uint64_t* value) const { const rapidjson::Value& array = AsValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } const auto& v = array[idx]; if (!v.IsUint64()) { TRITONJSON_STATUSRETURN(std::string( "attempt to access JSON non-unsigned-integer as unsigned-integer")); } *value = v.GetUint64(); return TRITONJSON_STATUSSUCCESS; } // Get array index as a double. Error if this is not an array or // if the index element is not a double. TRITONJSON_STATUSTYPE IndexAsDouble(const size_t idx, double* value) const { const rapidjson::Value& array = AsValue(); if (!array.IsArray() || (idx >= array.GetArray().Size())) { TRITONJSON_STATUSRETURN( std::string("attempt to access non-existing array index '") + std::to_string(idx) + "'"); } const auto& v = array[idx]; if (!v.IsNumber()) { TRITONJSON_STATUSRETURN( std::string("attempt to access JSON non-number as double")); } *value = v.GetDouble(); return TRITONJSON_STATUSSUCCESS; } // Release/clear a value. void Release() { if (value_ != nullptr) { allocator_->Free(value_); } } private: // Construct a non-top-level JSON value that references an // existing element in a document. explicit Value( rapidjson::Value& v, rapidjson::Document::AllocatorType* allocator) : value_(&v), allocator_(allocator) { } // Return a value object that can be used for both a top-level // document as well as an element within a document. const rapidjson::Value& AsValue() const { if (value_ == nullptr) { return document_; } return *value_; } rapidjson::Value& AsMutableValue() { if (value_ == nullptr) { return document_; } return *value_; } // If this object a document or value. Based on this only one or // document_ or value_ is valid. rapidjson::Document document_; rapidjson::Value* value_; rapidjson::Document::AllocatorType* allocator_; }; }; }} // namespace triton::common kytensor-2.49.0.7/common/protobuf/000077500000000000000000000000001527052371200167655ustar00rootroot00000000000000kytensor-2.49.0.7/common/protobuf/CMakeLists.txt000066400000000000000000000136651527052371200215400ustar00rootroot00000000000000# Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) set(protobuf_MODULE_COMPATIBLE TRUE CACHE BOOL "protobuf_MODULE_COMPATIBLE" FORCE) find_package(Protobuf REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") include_directories(${Protobuf_INCLUDE_DIRS}) if(${TRITON_COMMON_ENABLE_GRPC}) find_package(gRPC CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") include_directories($) endif() # # Protobuf # if(${TRITON_COMMON_ENABLE_PROTOBUF}) file(GLOB proto-srcs *.proto) protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${proto-srcs}) if(TRITON_COMMON_ENABLE_PROTOBUF_PYTHON) protobuf_generate_python(PROTO_PY ${proto-srcs}) endif() set(_PROTOBUF_PROTOC $) if(${TRITON_COMMON_ENABLE_GRPC}) set(_GRPC_CPP_PLUGIN_EXECUTABLE $) endif() add_library( proto-library EXCLUDE_FROM_ALL SHARED ${PROTO_SRCS} ${PROTO_HDRS} ) target_include_directories( proto-library PUBLIC $ ) target_link_libraries( proto-library PRIVATE common-compile-settings ) set_target_properties( proto-library PROPERTIES POSITION_INDEPENDENT_CODE ON ) if(TRITON_COMMON_ENABLE_PROTOBUF_PYTHON) add_custom_target(proto-py-library DEPENDS ${PROTO_PY}) endif() install( FILES ${PROTO_HDRS} DESTINATION include ) endif() # # GRPC # if(${TRITON_COMMON_ENABLE_GRPC}) get_filename_component(grpc_service_proto_abspath "grpc_service.proto" ABSOLUTE) get_filename_component(grpc_service_proto_dir "${grpc_service_proto_abspath}" PATH) set(GRPC_SRCS "grpc_service.grpc.pb.cc") set(GRPC_HDRS "grpc_service.grpc.pb.h") if(TRITON_COMMON_ENABLE_PROTOBUF_PYTHON) set(GRPC_PY "grpc_service.grpc.py") endif() add_custom_command( OUTPUT "${GRPC_SRCS}" "${GRPC_HDRS}" COMMAND ${_PROTOBUF_PROTOC} ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" -I "${grpc_service_proto_dir}" --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" "grpc_service.proto" DEPENDS "grpc_service.proto" proto-library ) if(TRITON_COMMON_ENABLE_PROTOBUF_PYTHON) find_package(Python REQUIRED COMPONENTS Interpreter) add_custom_command( OUTPUT "${GRPC_PY}" COMMAND ${Python_EXECUTABLE} ARGS -m grpc_tools.protoc -I "${grpc_service_proto_dir}" --grpc_python_out "${CMAKE_CURRENT_BINARY_DIR}" "grpc_service.proto" DEPENDS "grpc_service.proto" proto-library ) endif() add_library( grpc-service-library EXCLUDE_FROM_ALL OBJECT ${GRPC_SRCS} ${GRPC_HDRS} ) target_include_directories( grpc-service-library PUBLIC $ ) target_link_libraries( grpc-service-library PRIVATE common-compile-settings ) set_target_properties( grpc-service-library PROPERTIES POSITION_INDEPENDENT_CODE ON ) if(TRITON_COMMON_ENABLE_PROTOBUF_PYTHON) add_custom_target(grpc-service-py-library DEPENDS ${GRPC_PY}) endif() install( FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_service.grpc.pb.h DESTINATION include OPTIONAL ) endif() # # GRPC Health Service # if(${TRITON_COMMON_ENABLE_GRPC}) get_filename_component(health_proto_abspath "health.proto" ABSOLUTE) get_filename_component(health_proto_dir "${health_proto_abspath}" PATH) set(HEALTH_SRCS "health.grpc.pb.cc") set(HEALTH_HDRS "health.grpc.pb.h") add_custom_command( OUTPUT "${HEALTH_SRCS}" "${HEALTH_HDRS}" COMMAND ${_PROTOBUF_PROTOC} ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" -I "${health_proto_dir}" --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" "health.proto" DEPENDS "health.proto" proto-library ) add_library( grpc-health-library EXCLUDE_FROM_ALL OBJECT ${HEALTH_SRCS} ${HEALTH_HDRS} ) target_include_directories( grpc-health-library PUBLIC $ ) target_link_libraries( grpc-health-library PRIVATE common-compile-settings ) set_target_properties( grpc-health-library PROPERTIES POSITION_INDEPENDENT_CODE ON ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/health.grpc.pb.h DESTINATION include OPTIONAL ) endif() kytensor-2.49.0.7/common/protobuf/grpc_service.proto000066400000000000000000001406331527052371200225340ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. syntax = "proto3"; package inference; //@@.. cpp:namespace:: inference import "model_config.proto"; //@@ //@@.. cpp:var:: service InferenceService //@@ //@@ Inference Server GRPC endpoints. //@@ service GRPCInferenceService { //@@ .. cpp:var:: rpc ServerLive(ServerLiveRequest) returns //@@ (ServerLiveResponse) //@@ //@@ Check liveness of the inference server. //@@ rpc ServerLive(ServerLiveRequest) returns (ServerLiveResponse) {} //@@ .. cpp:var:: rpc ServerReady(ServerReadyRequest) returns //@@ (ServerReadyResponse) //@@ //@@ Check readiness of the inference server. //@@ rpc ServerReady(ServerReadyRequest) returns (ServerReadyResponse) {} //@@ .. cpp:var:: rpc ModelReady(ModelReadyRequest) returns //@@ (ModelReadyResponse) //@@ //@@ Check readiness of a model in the inference server. //@@ rpc ModelReady(ModelReadyRequest) returns (ModelReadyResponse) {} //@@ .. cpp:var:: rpc ServerMetadata(ServerMetadataRequest) returns //@@ (ServerMetadataResponse) //@@ //@@ Get server metadata. //@@ rpc ServerMetadata(ServerMetadataRequest) returns (ServerMetadataResponse) {} //@@ .. cpp:var:: rpc ModelMetadata(ModelMetadataRequest) returns //@@ (ModelMetadataResponse) //@@ //@@ Get model metadata. //@@ rpc ModelMetadata(ModelMetadataRequest) returns (ModelMetadataResponse) {} //@@ .. cpp:var:: rpc ModelInfer(ModelInferRequest) returns //@@ (ModelInferResponse) //@@ //@@ Perform inference using a specific model. //@@ rpc ModelInfer(ModelInferRequest) returns (ModelInferResponse) {} //@@ .. cpp:var:: rpc ModelStreamInfer(stream ModelInferRequest) returns //@@ (stream ModelStreamInferResponse) //@@ //@@ Perform streaming inference. //@@ rpc ModelStreamInfer(stream ModelInferRequest) returns (stream ModelStreamInferResponse) { } //@@ .. cpp:var:: rpc ModelConfig(ModelConfigRequest) returns //@@ (ModelConfigResponse) //@@ //@@ Get model configuration. //@@ rpc ModelConfig(ModelConfigRequest) returns (ModelConfigResponse) {} //@@ .. cpp:var:: rpc ModelStatistics( //@@ ModelStatisticsRequest) //@@ returns (ModelStatisticsResponse) //@@ //@@ Get the cumulative inference statistics for a model. //@@ rpc ModelStatistics(ModelStatisticsRequest) returns (ModelStatisticsResponse) { } //@@ .. cpp:var:: rpc RepositoryIndex(RepositoryIndexRequest) returns //@@ (RepositoryIndexResponse) //@@ //@@ Get the index of model repository contents. //@@ rpc RepositoryIndex(RepositoryIndexRequest) returns (RepositoryIndexResponse) { } //@@ .. cpp:var:: rpc RepositoryModelLoad(RepositoryModelLoadRequest) returns //@@ (RepositoryModelLoadResponse) //@@ //@@ Load or reload a model from a repository. //@@ rpc RepositoryModelLoad(RepositoryModelLoadRequest) returns (RepositoryModelLoadResponse) { } //@@ .. cpp:var:: rpc RepositoryModelUnload(RepositoryModelUnloadRequest) //@@ returns (RepositoryModelUnloadResponse) //@@ //@@ Unload a model. //@@ rpc RepositoryModelUnload(RepositoryModelUnloadRequest) returns (RepositoryModelUnloadResponse) { } //@@ .. cpp:var:: rpc SystemSharedMemoryStatus( //@@ SystemSharedMemoryStatusRequest) //@@ returns (SystemSharedMemoryStatusRespose) //@@ //@@ Get the status of all registered system-shared-memory regions. //@@ rpc SystemSharedMemoryStatus(SystemSharedMemoryStatusRequest) returns (SystemSharedMemoryStatusResponse) { } //@@ .. cpp:var:: rpc SystemSharedMemoryRegister( //@@ SystemSharedMemoryRegisterRequest) //@@ returns (SystemSharedMemoryRegisterResponse) //@@ //@@ Register a system-shared-memory region. //@@ rpc SystemSharedMemoryRegister(SystemSharedMemoryRegisterRequest) returns (SystemSharedMemoryRegisterResponse) { } //@@ .. cpp:var:: rpc SystemSharedMemoryUnregister( //@@ SystemSharedMemoryUnregisterRequest) //@@ returns (SystemSharedMemoryUnregisterResponse) //@@ //@@ Unregister a system-shared-memory region. //@@ rpc SystemSharedMemoryUnregister(SystemSharedMemoryUnregisterRequest) returns (SystemSharedMemoryUnregisterResponse) { } //@@ .. cpp:var:: rpc CudaSharedMemoryStatus( //@@ CudaSharedMemoryStatusRequest) //@@ returns (CudaSharedMemoryStatusRespose) //@@ //@@ Get the status of all registered CUDA-shared-memory regions. //@@ rpc CudaSharedMemoryStatus(CudaSharedMemoryStatusRequest) returns (CudaSharedMemoryStatusResponse) { } //@@ .. cpp:var:: rpc CudaSharedMemoryRegister( //@@ CudaSharedMemoryRegisterRequest) //@@ returns (CudaSharedMemoryRegisterResponse) //@@ //@@ Register a CUDA-shared-memory region. //@@ rpc CudaSharedMemoryRegister(CudaSharedMemoryRegisterRequest) returns (CudaSharedMemoryRegisterResponse) { } //@@ .. cpp:var:: rpc CudaSharedMemoryUnregister( //@@ CudaSharedMemoryUnregisterRequest) //@@ returns (CudaSharedMemoryUnregisterResponse) //@@ //@@ Unregister a CUDA-shared-memory region. //@@ rpc CudaSharedMemoryUnregister(CudaSharedMemoryUnregisterRequest) returns (CudaSharedMemoryUnregisterResponse) { } //@@ .. cpp:var:: rpc TraceSetting(TraceSettingRequest) //@@ returns (TraceSettingResponse) //@@ //@@ Update and get the trace setting of the Triton server. //@@ rpc TraceSetting(TraceSettingRequest) returns (TraceSettingResponse) {} //@@ .. cpp:var:: rpc LogSettings(LogSettingsRequest) //@@ returns (LogSettingsResponse) //@@ //@@ Update and get the log settings of the Triton server. //@@ rpc LogSettings(LogSettingsRequest) returns (LogSettingsResponse) {} } //@@ //@@.. cpp:var:: message ServerLiveRequest //@@ //@@ Request message for ServerLive. //@@ message ServerLiveRequest {} //@@ //@@.. cpp:var:: message ServerLiveResponse //@@ //@@ Response message for ServerLive. //@@ message ServerLiveResponse { //@@ //@@ .. cpp:var:: bool live //@@ //@@ True if the inference server is live, false it not live. //@@ bool live = 1; } //@@ //@@.. cpp:var:: message ServerReadyRequest //@@ //@@ Request message for ServerReady. //@@ message ServerReadyRequest {} //@@ //@@.. cpp:var:: message ServerReadyResponse //@@ //@@ Response message for ServerReady. //@@ message ServerReadyResponse { //@@ //@@ .. cpp:var:: bool ready //@@ //@@ True if the inference server is ready, false it not ready. //@@ bool ready = 1; } //@@ //@@.. cpp:var:: message ModelReadyRequest //@@ //@@ Request message for ModelReady. //@@ message ModelReadyRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the model to check for readiness. //@@ string name = 1; //@@ .. cpp:var:: string version //@@ //@@ The version of the model to check for readiness. If not given the //@@ server will choose a version based on the model and internal policy. //@@ string version = 2; } //@@ //@@.. cpp:var:: message ModelReadyResponse //@@ //@@ Response message for ModelReady. //@@ message ModelReadyResponse { //@@ //@@ .. cpp:var:: bool ready //@@ //@@ True if the model is ready, false it not ready. //@@ bool ready = 1; } //@@ //@@.. cpp:var:: message ServerMetadataRequest //@@ //@@ Request message for ServerMetadata. //@@ message ServerMetadataRequest {} //@@ //@@.. cpp:var:: message ServerMetadataResponse //@@ //@@ Response message for ServerMetadata. //@@ message ServerMetadataResponse { //@@ //@@ .. cpp:var:: string name //@@ //@@ The server name. //@@ string name = 1; //@@ //@@ .. cpp:var:: string version //@@ //@@ The server version. //@@ string version = 2; //@@ //@@ .. cpp:var:: string extensions (repeated) //@@ //@@ The extensions supported by the server. //@@ repeated string extensions = 3; } //@@ //@@.. cpp:var:: message ModelMetadataRequest //@@ //@@ Request message for ModelMetadata. //@@ message ModelMetadataRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the model. //@@ string name = 1; //@@ .. cpp:var:: string version //@@ //@@ The version of the model to check for readiness. If not //@@ given the server will choose a version based on the //@@ model and internal policy. //@@ string version = 2; } //@@ //@@.. cpp:var:: message ModelMetadataResponse //@@ //@@ Response message for ModelMetadata. //@@ message ModelMetadataResponse { //@@ //@@ .. cpp:var:: message TensorMetadata //@@ //@@ Metadata for a tensor. //@@ message TensorMetadata { //@@ //@@ .. cpp:var:: string name //@@ //@@ The tensor name. //@@ string name = 1; //@@ //@@ .. cpp:var:: string datatype //@@ //@@ The tensor data type. //@@ string datatype = 2; //@@ //@@ .. cpp:var:: int64 shape (repeated) //@@ //@@ The tensor shape. A variable-size dimension is represented //@@ by a -1 value. //@@ repeated int64 shape = 3; } //@@ //@@ .. cpp:var:: string name //@@ //@@ The model name. //@@ string name = 1; //@@ //@@ .. cpp:var:: string versions (repeated) //@@ //@@ The versions of the model. //@@ repeated string versions = 2; //@@ //@@ .. cpp:var:: string platform //@@ //@@ The model's platform. //@@ string platform = 3; //@@ //@@ .. cpp:var:: TensorMetadata inputs (repeated) //@@ //@@ The model's inputs. //@@ repeated TensorMetadata inputs = 4; //@@ //@@ .. cpp:var:: TensorMetadata outputs (repeated) //@@ //@@ The model's outputs. //@@ repeated TensorMetadata outputs = 5; } //@@ //@@.. cpp:var:: message InferParameter //@@ //@@ An inference parameter value. //@@ message InferParameter { //@@ .. cpp:var:: oneof parameter_choice //@@ //@@ The parameter value can be a string, an int64, //@@ an uint64, a double, or a boolean //@@ //@@ Note: double and uint64 are currently //@@ placeholders for future use and //@@ are not supported for custom parameters //@@ oneof parameter_choice { //@@ .. cpp:var:: bool bool_param //@@ //@@ A boolean parameter value. //@@ bool bool_param = 1; //@@ .. cpp:var:: int64 int64_param //@@ //@@ An int64 parameter value. //@@ int64 int64_param = 2; //@@ .. cpp:var:: string string_param //@@ //@@ A string parameter value. //@@ string string_param = 3; //@@ .. cpp:var:: double double_param //@@ //@@ A double parameter value. //@@ double double_param = 4; //@@ .. cpp:var:: uint64 uint64_param //@@ //@@ A uint64 parameter value. //@@ //@@ Not supported for custom parameters //@@ uint64 uint64_param = 5; } } //@@ //@@.. cpp:var:: message InferTensorContents //@@ //@@ The data contained in a tensor represented by the repeated type //@@ that matches the tensor's data type. Protobuf oneof is not used //@@ because oneofs cannot contain repeated fields. //@@ message InferTensorContents { //@@ //@@ .. cpp:var:: bool bool_contents (repeated) //@@ //@@ Representation for BOOL data type. The size must match what is //@@ expected by the tensor's shape. The contents must be the flattened, //@@ one-dimensional, row-major order of the tensor elements. //@@ repeated bool bool_contents = 1; //@@ //@@ .. cpp:var:: int32 int_contents (repeated) //@@ //@@ Representation for INT8, INT16, and INT32 data types. The size //@@ must match what is expected by the tensor's shape. The contents //@@ must be the flattened, one-dimensional, row-major order of the //@@ tensor elements. //@@ repeated int32 int_contents = 2; //@@ //@@ .. cpp:var:: int64 int64_contents (repeated) //@@ //@@ Representation for INT64 data types. The size must match what //@@ is expected by the tensor's shape. The contents must be the //@@ flattened, one-dimensional, row-major order of the tensor elements. //@@ repeated int64 int64_contents = 3; //@@ //@@ .. cpp:var:: uint32 uint_contents (repeated) //@@ //@@ Representation for UINT8, UINT16, and UINT32 data types. The size //@@ must match what is expected by the tensor's shape. The contents //@@ must be the flattened, one-dimensional, row-major order of the //@@ tensor elements. //@@ repeated uint32 uint_contents = 4; //@@ //@@ .. cpp:var:: uint64 uint64_contents (repeated) //@@ //@@ Representation for UINT64 data types. The size must match what //@@ is expected by the tensor's shape. The contents must be the //@@ flattened, one-dimensional, row-major order of the tensor elements. //@@ repeated uint64 uint64_contents = 5; //@@ //@@ .. cpp:var:: float fp32_contents (repeated) //@@ //@@ Representation for FP32 data type. The size must match what is //@@ expected by the tensor's shape. The contents must be the flattened, //@@ one-dimensional, row-major order of the tensor elements. //@@ repeated float fp32_contents = 6; //@@ //@@ .. cpp:var:: double fp64_contents (repeated) //@@ //@@ Representation for FP64 data type. The size must match what is //@@ expected by the tensor's shape. The contents must be the flattened, //@@ one-dimensional, row-major order of the tensor elements. //@@ repeated double fp64_contents = 7; //@@ //@@ .. cpp:var:: bytes bytes_contents (repeated) //@@ //@@ Representation for BYTES data type. The size must match what is //@@ expected by the tensor's shape. The contents must be the flattened, //@@ one-dimensional, row-major order of the tensor elements. //@@ repeated bytes bytes_contents = 8; } //@@ //@@.. cpp:var:: message ModelInferRequest //@@ //@@ Request message for ModelInfer. //@@ message ModelInferRequest { //@@ //@@ .. cpp:var:: message InferInputTensor //@@ //@@ An input tensor for an inference request. //@@ message InferInputTensor { //@@ //@@ .. cpp:var:: string name //@@ //@@ The tensor name. //@@ string name = 1; //@@ //@@ .. cpp:var:: string datatype //@@ //@@ The tensor data type. //@@ string datatype = 2; //@@ //@@ .. cpp:var:: int64 shape (repeated) //@@ //@@ The tensor shape. //@@ repeated int64 shape = 3; //@@ .. cpp:var:: map parameters //@@ //@@ Optional inference input tensor parameters. //@@ map parameters = 4; //@@ .. cpp:var:: InferTensorContents contents //@@ //@@ The tensor contents using a data-type format. This field //@@ must not be specified if tensor contents are being specified //@@ in ModelInferRequest.raw_input_contents. //@@ InferTensorContents contents = 5; } //@@ //@@ .. cpp:var:: message InferRequestedOutputTensor //@@ //@@ An output tensor requested for an inference request. //@@ message InferRequestedOutputTensor { //@@ //@@ .. cpp:var:: string name //@@ //@@ The tensor name. //@@ string name = 1; //@@ .. cpp:var:: map parameters //@@ //@@ Optional requested output tensor parameters. //@@ map parameters = 2; } //@@ .. cpp:var:: string model_name //@@ //@@ The name of the model to use for inferencing. //@@ string model_name = 1; //@@ .. cpp:var:: string model_version //@@ //@@ The version of the model to use for inference. If not //@@ given the latest/most-recent version of the model is used. //@@ string model_version = 2; //@@ .. cpp:var:: string id //@@ //@@ Optional identifier for the request. If specified will be //@@ returned in the response. //@@ string id = 3; //@@ .. cpp:var:: map parameters //@@ //@@ Optional inference parameters. //@@ map parameters = 4; //@@ //@@ .. cpp:var:: InferInputTensor inputs (repeated) //@@ //@@ The input tensors for the inference. //@@ repeated InferInputTensor inputs = 5; //@@ //@@ .. cpp:var:: InferRequestedOutputTensor outputs (repeated) //@@ //@@ The requested output tensors for the inference. Optional, if not //@@ specified all outputs specified in the model config will be //@@ returned. //@@ repeated InferRequestedOutputTensor outputs = 6; //@@ //@@ .. cpp:var:: bytes raw_input_contents //@@ //@@ The data contained in an input tensor can be represented in //@@ "raw" bytes form or in the repeated type that matches the //@@ tensor's data type. Using the "raw" bytes form will //@@ typically allow higher performance due to the way protobuf //@@ allocation and reuse interacts with GRPC. For example, see //@@ https://github.com/grpc/grpc/issues/23231. //@@ //@@ To use the raw representation 'raw_input_contents' must be //@@ initialized with data for each tensor in the same order as //@@ 'inputs'. For each tensor, the size of this content must //@@ match what is expected by the tensor's shape and data //@@ type. The raw data must be the flattened, one-dimensional, //@@ row-major order of the tensor elements without any stride //@@ or padding between the elements. Note that the FP16 and BF16 data //@@ types must be represented as raw content as there is no //@@ specific data type for a 16-bit float type. //@@ //@@ If this field is specified then InferInputTensor::contents //@@ must not be specified for any input tensor. //@@ repeated bytes raw_input_contents = 7; } //@@ //@@.. cpp:var:: message ModelInferResponse //@@ //@@ Response message for ModelInfer. //@@ message ModelInferResponse { //@@ //@@ .. cpp:var:: message InferOutputTensor //@@ //@@ An output tensor returned for an inference request. //@@ message InferOutputTensor { //@@ //@@ .. cpp:var:: string name //@@ //@@ The tensor name. //@@ string name = 1; //@@ //@@ .. cpp:var:: string datatype //@@ //@@ The tensor data type. //@@ string datatype = 2; //@@ //@@ .. cpp:var:: int64 shape (repeated) //@@ //@@ The tensor shape. //@@ repeated int64 shape = 3; //@@ .. cpp:var:: map parameters //@@ //@@ Optional output tensor parameters. //@@ map parameters = 4; //@@ .. cpp:var:: InferTensorContents contents //@@ //@@ The tensor contents using a data-type format. This field //@@ must not be specified if tensor contents are being specified //@@ in ModelInferResponse.raw_output_contents. //@@ InferTensorContents contents = 5; } //@@ .. cpp:var:: string model_name //@@ //@@ The name of the model used for inference. //@@ string model_name = 1; //@@ .. cpp:var:: string model_version //@@ //@@ The version of the model used for inference. //@@ string model_version = 2; //@@ .. cpp:var:: string id //@@ //@@ The id of the inference request if one was specified. //@@ string id = 3; //@@ .. cpp:var:: map parameters //@@ //@@ Optional inference response parameters. //@@ map parameters = 4; //@@ //@@ .. cpp:var:: InferOutputTensor outputs (repeated) //@@ //@@ The output tensors holding inference results. //@@ repeated InferOutputTensor outputs = 5; //@@ //@@ .. cpp:var:: bytes raw_output_contents //@@ //@@ The data contained in an output tensor can be represented in //@@ "raw" bytes form or in the repeated type that matches the //@@ tensor's data type. Using the "raw" bytes form will //@@ typically allow higher performance due to the way protobuf //@@ allocation and reuse interacts with GRPC. For example, see //@@ https://github.com/grpc/grpc/issues/23231. //@@ //@@ To use the raw representation 'raw_output_contents' must be //@@ initialized with data for each tensor in the same order as //@@ 'outputs'. For each tensor, the size of this content must //@@ match what is expected by the tensor's shape and data //@@ type. The raw data must be the flattened, one-dimensional, //@@ row-major order of the tensor elements without any stride //@@ or padding between the elements. Note that the FP16 and BF16 data //@@ types must be represented as raw content as there is no //@@ specific data type for a 16-bit float type. //@@ //@@ If this field is specified then InferOutputTensor::contents //@@ must not be specified for any output tensor. //@@ repeated bytes raw_output_contents = 6; } //@@ //@@.. cpp:var:: message ModelStreamInferResponse //@@ //@@ Response message for ModelStreamInfer. //@@ message ModelStreamInferResponse { //@@ //@@ .. cpp:var:: string error_message //@@ //@@ The message describing the error. The empty message //@@ indicates the inference was successful without errors. //@@ string error_message = 1; //@@ //@@ .. cpp:var:: ModelInferResponse infer_response //@@ //@@ Holds the results of the request. //@@ ModelInferResponse infer_response = 2; } //@@ //@@.. cpp:var:: message ModelConfigRequest //@@ //@@ Request message for ModelConfig. //@@ message ModelConfigRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the model. //@@ string name = 1; //@@ .. cpp:var:: string version //@@ //@@ The version of the model. If not given the model version //@@ is selected automatically based on the version policy. //@@ string version = 2; } //@@ //@@.. cpp:var:: message ModelConfigResponse //@@ //@@ Response message for ModelConfig. //@@ message ModelConfigResponse { //@@ //@@ .. cpp:var:: ModelConfig config //@@ //@@ The model configuration. //@@ ModelConfig config = 1; } //@@ //@@.. cpp:var:: message ModelStatisticsRequest //@@ //@@ Request message for ModelStatistics. //@@ message ModelStatisticsRequest { //@@ .. cpp:var:: string name //@@ //@@ The name of the model. If not given returns statistics for //@@ all models. //@@ string name = 1; //@@ .. cpp:var:: string version //@@ //@@ The version of the model. If not given returns statistics for //@@ all model versions. //@@ string version = 2; } //@@ //@@.. cpp:var:: message StatisticDuration //@@ //@@ Statistic recording a cumulative duration metric. //@@ message StatisticDuration { //@@ .. cpp:var:: uint64 count //@@ //@@ Cumulative number of times this metric occurred. //@@ uint64 count = 1; //@@ .. cpp:var:: uint64 total_time_ns //@@ //@@ Total collected duration of this metric in nanoseconds. //@@ uint64 ns = 2; } //@@ //@@.. cpp:var:: message InferStatistics //@@ //@@ Inference statistics. //@@ message InferStatistics { //@@ .. cpp:var:: StatisticDuration success //@@ //@@ Cumulative count and duration for successful inference //@@ request. The "success" count and cumulative duration includes //@@ cache hits. //@@ StatisticDuration success = 1; //@@ .. cpp:var:: StatisticDuration fail //@@ //@@ Cumulative count and duration for failed inference //@@ request. //@@ StatisticDuration fail = 2; //@@ .. cpp:var:: StatisticDuration queue //@@ //@@ The count and cumulative duration that inference requests wait in //@@ scheduling or other queues. The "queue" count and cumulative //@@ duration includes cache hits. //@@ StatisticDuration queue = 3; //@@ .. cpp:var:: StatisticDuration compute_input //@@ //@@ The count and cumulative duration to prepare input tensor data as //@@ required by the model framework / backend. For example, this duration //@@ should include the time to copy input tensor data to the GPU. //@@ The "compute_input" count and cumulative duration do not account for //@@ requests that were a cache hit. See the "cache_hit" field for more //@@ info. //@@ StatisticDuration compute_input = 4; //@@ .. cpp:var:: StatisticDuration compute_infer //@@ //@@ The count and cumulative duration to execute the model. //@@ The "compute_infer" count and cumulative duration do not account for //@@ requests that were a cache hit. See the "cache_hit" field for more //@@ info. //@@ StatisticDuration compute_infer = 5; //@@ .. cpp:var:: StatisticDuration compute_output //@@ //@@ The count and cumulative duration to extract output tensor data //@@ produced by the model framework / backend. For example, this duration //@@ should include the time to copy output tensor data from the GPU. //@@ The "compute_output" count and cumulative duration do not account for //@@ requests that were a cache hit. See the "cache_hit" field for more //@@ info. //@@ StatisticDuration compute_output = 6; //@@ .. cpp:var:: StatisticDuration cache_hit //@@ //@@ The count of response cache hits and cumulative duration to lookup //@@ and extract output tensor data from the Response Cache on a cache //@@ hit. For example, this duration should include the time to copy //@@ output tensor data from the Response Cache to the response object. //@@ On cache hits, triton does not need to go to the model/backend //@@ for the output tensor data, so the "compute_input", "compute_infer", //@@ and "compute_output" fields are not updated. Assuming the response //@@ cache is enabled for a given model, a cache hit occurs for a //@@ request to that model when the request metadata (model name, //@@ model version, model inputs) hashes to an existing entry in the //@@ cache. On a cache miss, the request hash and response output tensor //@@ data is added to the cache. See response cache docs for more info: //@@ // https://github.com/triton-inference-server/server/blob/main/docs/response_cache.md //@@ StatisticDuration cache_hit = 7; //@@ .. cpp:var:: StatisticDuration cache_miss //@@ //@@ The count of response cache misses and cumulative duration to lookup //@@ and insert output tensor data from the computed response to the // cache. //@@ For example, this duration should include the time to copy //@@ output tensor data from the response object to the Response Cache. //@@ Assuming the response cache is enabled for a given model, a cache //@@ miss occurs for a request to that model when the request metadata //@@ does NOT hash to an existing entry in the cache. See the response //@@ cache docs for more info: //@@ // https://github.com/triton-inference-server/server/blob/main/docs/response_cache.md //@@ StatisticDuration cache_miss = 8; } //@@ //@@.. cpp:var:: message InferResponseStatistics //@@ //@@ Statistics per response. //@@ message InferResponseStatistics { //@@ .. cpp:var:: StatisticDuration compute_infer //@@ //@@ The count and cumulative duration to compute a response. //@@ StatisticDuration compute_infer = 1; //@@ .. cpp:var:: StatisticDuration compute_output //@@ //@@ The count and cumulative duration to extract the output tensors of a //@@ response. //@@ StatisticDuration compute_output = 2; //@@ .. cpp:var:: StatisticDuration success //@@ //@@ The count and cumulative duration for successful responses. //@@ StatisticDuration success = 3; //@@ .. cpp:var:: StatisticDuration fail //@@ //@@ The count and cumulative duration for failed responses. //@@ StatisticDuration fail = 4; //@@ .. cpp:var:: StatisticDuration empty_response //@@ //@@ The count and cumulative duration for empty responses. //@@ StatisticDuration empty_response = 5; //@@ .. cpp:var:: StatisticDuration cancel //@@ //@@ The count and cumulative duration, for cleaning up resources held by //@@ a cancelled request, for cancelled responses. //@@ StatisticDuration cancel = 6; } //@@ //@@.. cpp:var:: message InferBatchStatistics //@@ //@@ Inference batch statistics. //@@ message InferBatchStatistics { //@@ .. cpp:var:: uint64 batch_size //@@ //@@ The size of the batch. //@@ uint64 batch_size = 1; //@@ .. cpp:var:: StatisticDuration compute_input //@@ //@@ The count and cumulative duration to prepare input tensor data as //@@ required by the model framework / backend with the given batch size. //@@ For example, this duration should include the time to copy input //@@ tensor data to the GPU. //@@ StatisticDuration compute_input = 2; //@@ .. cpp:var:: StatisticDuration compute_infer //@@ //@@ The count and cumulative duration to execute the model with the given //@@ batch size. //@@ StatisticDuration compute_infer = 3; //@@ .. cpp:var:: StatisticDuration compute_output //@@ //@@ The count and cumulative duration to extract output tensor data //@@ produced by the model framework / backend with the given batch size. //@@ For example, this duration should include the time to copy output //@@ tensor data from the GPU. //@@ StatisticDuration compute_output = 4; } //@@ //@@.. cpp:var:: message MemoryUsage //@@ //@@ Memory usage. //@@ message MemoryUsage { //@@ .. cpp:var:: string type //@@ //@@ The type of memory, the value can be "CPU", "CPU_PINNED", "GPU". //@@ string type = 1; //@@ .. cpp:var:: int64 id //@@ //@@ The id of the memory, typically used with "type" to identify //@@ a device that hosts the memory. //@@ int64 id = 2; //@@ .. cpp:var:: uint64 byte_size //@@ //@@ The byte size of the memory. //@@ uint64 byte_size = 3; } //@@ //@@.. cpp:var:: message ModelStatistics //@@ //@@ Statistics for a specific model and version. //@@ message ModelStatistics { //@@ .. cpp:var:: string name //@@ //@@ The name of the model. If not given returns statistics for all //@@ string name = 1; //@@ .. cpp:var:: string version //@@ //@@ The version of the model. //@@ string version = 2; //@@ .. cpp:var:: uint64 last_inference //@@ //@@ The timestamp of the last inference request made for this model, //@@ as milliseconds since the epoch. //@@ uint64 last_inference = 3; //@@ .. cpp:var:: uint64 last_inference //@@ //@@ The cumulative count of successful inference requests made for this //@@ model. Each inference in a batched request is counted as an //@@ individual inference. For example, if a client sends a single //@@ inference request with batch size 64, "inference_count" will be //@@ incremented by 64. Similarly, if a clients sends 64 individual //@@ requests each with batch size 1, "inference_count" will be //@@ incremented by 64. The "inference_count" value DOES NOT include //@@ cache hits. //@@ uint64 inference_count = 4; //@@ .. cpp:var:: uint64 last_inference //@@ //@@ The cumulative count of the number of successful inference executions //@@ performed for the model. When dynamic batching is enabled, a single //@@ model execution can perform inferencing for more than one inference //@@ request. For example, if a clients sends 64 individual requests each //@@ with batch size 1 and the dynamic batcher batches them into a single //@@ large batch for model execution then "execution_count" will be //@@ incremented by 1. If, on the other hand, the dynamic batcher is not //@@ enabled for that each of the 64 individual requests is executed //@@ independently, then "execution_count" will be incremented by 64. //@@ The "execution_count" value DOES NOT include cache hits. //@@ uint64 execution_count = 5; //@@ .. cpp:var:: InferStatistics inference_stats //@@ //@@ The aggregate statistics for the model/version. //@@ InferStatistics inference_stats = 6; //@@ .. cpp:var:: InferBatchStatistics batch_stats (repeated) //@@ //@@ The aggregate statistics for each different batch size that is //@@ executed in the model. The batch statistics indicate how many actual //@@ model executions were performed and show differences due to different //@@ batch size (for example, larger batches typically take longer to //@@ compute). //@@ repeated InferBatchStatistics batch_stats = 7; //@@ .. cpp:var:: MemoryUsage memory_usage (repeated) //@@ //@@ The memory usage detected during model loading, which may be used to //@@ estimate the memory to be released once the model is unloaded. Note //@@ that the estimation is inferenced by the profiling tools and //@@ framework's memory schema, therefore it is advised to perform //@@ experiments to understand the scenario that the reported memory usage //@@ can be relied on. As a starting point, the GPU memory usage for //@@ models in ONNX Runtime backend and TensorRT backend is usually //@@ aligned. //@@ repeated MemoryUsage memory_usage = 8; //@@ .. cpp:var:: map response_stats //@@ //@@ The key and value pairs for all responses statistics. The key is a //@@ string identifying a set of response statistics aggregated together //@@ (i.e. index of the response sent). The value is the aggregated //@@ response statistics. //@@ map response_stats = 9; } //@@ //@@.. cpp:var:: message ModelStatisticsResponse //@@ //@@ Response message for ModelStatistics. //@@ message ModelStatisticsResponse { //@@ .. cpp:var:: ModelStatistics model_stats (repeated) //@@ //@@ Statistics for each requested model. //@@ repeated ModelStatistics model_stats = 1; } //@@ //@@.. cpp:var:: message ModelRepositoryParameter //@@ //@@ An model repository parameter value. //@@ message ModelRepositoryParameter { //@@ .. cpp:var:: oneof parameter_choice //@@ //@@ The parameter value can be a string, an int64 or //@@ a boolean //@@ oneof parameter_choice { //@@ .. cpp:var:: bool bool_param //@@ //@@ A boolean parameter value. //@@ bool bool_param = 1; //@@ .. cpp:var:: int64 int64_param //@@ //@@ An int64 parameter value. //@@ int64 int64_param = 2; //@@ .. cpp:var:: string string_param //@@ //@@ A string parameter value. //@@ string string_param = 3; //@@ .. cpp:var:: bytes bytes_param //@@ //@@ A bytes parameter value. //@@ bytes bytes_param = 4; } } //@@ //@@.. cpp:var:: message RepositoryIndexRequest //@@ //@@ Request message for RepositoryIndex. //@@ message RepositoryIndexRequest { //@@ .. cpp:var:: string repository_name //@@ //@@ The name of the repository. If empty the index is returned //@@ for all repositories. //@@ string repository_name = 1; //@@ .. cpp:var:: bool ready //@@ //@@ If true returned only models currently ready for inferencing. //@@ bool ready = 2; } //@@ //@@.. cpp:var:: message RepositoryIndexResponse //@@ //@@ Response message for RepositoryIndex. //@@ message RepositoryIndexResponse { //@@ //@@ .. cpp:var:: message ModelIndex //@@ //@@ Index entry for a model. //@@ message ModelIndex { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the model. //@@ string name = 1; //@@ .. cpp:var:: string version //@@ //@@ The version of the model. //@@ string version = 2; //@@ //@@ .. cpp:var:: string state //@@ //@@ The state of the model. //@@ string state = 3; //@@ //@@ .. cpp:var:: string reason //@@ //@@ The reason, if any, that the model is in the given state. //@@ string reason = 4; } //@@ //@@ .. cpp:var:: ModelIndex models (repeated) //@@ //@@ An index entry for each model. //@@ repeated ModelIndex models = 1; } //@@ //@@.. cpp:var:: message RepositoryModelLoadRequest //@@ //@@ Request message for RepositoryModelLoad. //@@ message RepositoryModelLoadRequest { //@@ .. cpp:var:: string repository_name //@@ //@@ The name of the repository to load from. If empty the model //@@ is loaded from any repository. //@@ string repository_name = 1; //@@ .. cpp:var:: string repository_name //@@ //@@ The name of the model to load, or reload. //@@ string model_name = 2; //@@ .. cpp:var:: map parameters //@@ //@@ Optional model repository request parameters. //@@ map parameters = 3; } //@@ //@@.. cpp:var:: message RepositoryModelLoadResponse //@@ //@@ Response message for RepositoryModelLoad. //@@ message RepositoryModelLoadResponse {} //@@ //@@.. cpp:var:: message RepositoryModelUnloadRequest //@@ //@@ Request message for RepositoryModelUnload. //@@ message RepositoryModelUnloadRequest { //@@ .. cpp:var:: string repository_name //@@ //@@ The name of the repository from which the model was originally //@@ loaded. If empty the repository is not considered. //@@ string repository_name = 1; //@@ .. cpp:var:: string repository_name //@@ //@@ The name of the model to unload. //@@ string model_name = 2; //@@ .. cpp:var:: map parameters //@@ //@@ Optional model repository request parameters. //@@ map parameters = 3; } //@@ //@@.. cpp:var:: message RepositoryModelUnloadResponse //@@ //@@ Response message for RepositoryModelUnload. //@@ message RepositoryModelUnloadResponse {} //@@ //@@.. cpp:var:: message SystemSharedMemoryStatusRequest //@@ //@@ Request message for SystemSharedMemoryStatus. //@@ message SystemSharedMemoryStatusRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the region to get status for. If empty the //@@ status is returned for all registered regions. //@@ string name = 1; } //@@ //@@.. cpp:var:: message SystemSharedMemoryStatusResponse //@@ //@@ Response message for SystemSharedMemoryStatus. //@@ message SystemSharedMemoryStatusResponse { //@@ //@@ .. cpp:var:: message RegionStatus //@@ //@@ Status for a shared memory region. //@@ message RegionStatus { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name for the shared memory region. //@@ string name = 1; //@@ .. cpp:var:: string shared_memory_key //@@ //@@ The key of the underlying memory object that contains the //@@ shared memory region. //@@ string key = 2; //@@ .. cpp:var:: uint64 offset //@@ //@@ Offset, in bytes, within the underlying memory object to //@@ the start of the shared memory region. //@@ uint64 offset = 3; //@@ .. cpp:var:: uint64 byte_size //@@ //@@ Size of the shared memory region, in bytes. //@@ uint64 byte_size = 4; } //@@ //@@ .. cpp:var:: map regions //@@ //@@ Status for each of the registered regions, indexed by //@@ region name. //@@ map regions = 1; } //@@ //@@.. cpp:var:: message SystemSharedMemoryRegisterRequest //@@ //@@ Request message for SystemSharedMemoryRegister. //@@ message SystemSharedMemoryRegisterRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the region to register. //@@ string name = 1; //@@ .. cpp:var:: string shared_memory_key //@@ //@@ The key of the underlying memory object that contains the //@@ shared memory region. //@@ string key = 2; //@@ .. cpp:var:: uint64 offset //@@ //@@ Offset, in bytes, within the underlying memory object to //@@ the start of the shared memory region. //@@ uint64 offset = 3; //@@ .. cpp:var:: uint64 byte_size //@@ //@@ Size of the shared memory region, in bytes. //@@ uint64 byte_size = 4; } //@@ //@@.. cpp:var:: message SystemSharedMemoryRegisterResponse //@@ //@@ Response message for SystemSharedMemoryRegister. //@@ message SystemSharedMemoryRegisterResponse {} //@@ //@@.. cpp:var:: message SystemSharedMemoryUnregisterRequest //@@ //@@ Request message for SystemSharedMemoryUnregister. //@@ message SystemSharedMemoryUnregisterRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the system region to unregister. If empty //@@ all system shared-memory regions are unregistered. //@@ string name = 1; } //@@ //@@.. cpp:var:: message SystemSharedMemoryUnregisterResponse //@@ //@@ Response message for SystemSharedMemoryUnregister. //@@ message SystemSharedMemoryUnregisterResponse {} //@@ //@@.. cpp:var:: message CudaSharedMemoryStatusRequest //@@ //@@ Request message for CudaSharedMemoryStatus. //@@ message CudaSharedMemoryStatusRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the region to get status for. If empty the //@@ status is returned for all registered regions. //@@ string name = 1; } //@@ //@@.. cpp:var:: message CudaSharedMemoryStatusResponse //@@ //@@ Response message for CudaSharedMemoryStatus. //@@ message CudaSharedMemoryStatusResponse { //@@ //@@ .. cpp:var:: message RegionStatus //@@ //@@ Status for a shared memory region. //@@ message RegionStatus { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name for the shared memory region. //@@ string name = 1; //@@ .. cpp:var:: uin64 device_id //@@ //@@ The GPU device ID where the cudaIPC handle was created. //@@ uint64 device_id = 2; //@@ .. cpp:var:: uint64 byte_size //@@ //@@ Size of the shared memory region, in bytes. //@@ uint64 byte_size = 3; } //@@ //@@ .. cpp:var:: map regions //@@ //@@ Status for each of the registered regions, indexed by //@@ region name. //@@ map regions = 1; } //@@ //@@.. cpp:var:: message CudaSharedMemoryRegisterRequest //@@ //@@ Request message for CudaSharedMemoryRegister. //@@ message CudaSharedMemoryRegisterRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the region to register. //@@ string name = 1; //@@ .. cpp:var:: bytes raw_handle //@@ //@@ The raw serialized cudaIPC handle. //@@ bytes raw_handle = 2; //@@ .. cpp:var:: int64 device_id //@@ //@@ The GPU device ID on which the cudaIPC handle was created. //@@ int64 device_id = 3; //@@ .. cpp:var:: uint64 byte_size //@@ //@@ Size of the shared memory block, in bytes. //@@ uint64 byte_size = 4; } //@@ //@@.. cpp:var:: message CudaSharedMemoryRegisterResponse //@@ //@@ Response message for CudaSharedMemoryRegister. //@@ message CudaSharedMemoryRegisterResponse {} //@@ //@@.. cpp:var:: message CudaSharedMemoryUnregisterRequest //@@ //@@ Request message for CudaSharedMemoryUnregister. //@@ message CudaSharedMemoryUnregisterRequest { //@@ //@@ .. cpp:var:: string name //@@ //@@ The name of the cuda region to unregister. If empty //@@ all cuda shared-memory regions are unregistered. //@@ string name = 1; } //@@ //@@.. cpp:var:: message CudaSharedMemoryUnregisterResponse //@@ //@@ Response message for CudaSharedMemoryUnregister. //@@ message CudaSharedMemoryUnregisterResponse {} //@@ //@@.. cpp:var:: message TraceSettingRequest //@@ //@@ Request message for TraceSetting. //@@ message TraceSettingRequest { //@@ //@@ .. cpp:var:: message SettingValue //@@ //@@ The values to be associated with a trace setting. //@@ If no value is provided, the setting will be clear and //@@ the global setting value will be used. //@@ message SettingValue { //@@ //@@ .. cpp:var:: string value (repeated) //@@ //@@ The value. //@@ repeated string value = 1; } //@@ .. cpp:var:: map settings //@@ //@@ The new setting values to be updated, //@@ settings that are not specified will remain unchanged. //@@ map settings = 1; //@@ //@@ .. cpp:var:: string model_name //@@ //@@ The name of the model to apply the new trace settings. //@@ If not given, the new settings will be applied globally. //@@ string model_name = 2; } //@@ //@@.. cpp:var:: message TraceSettingResponse //@@ //@@ Response message for TraceSetting. //@@ message TraceSettingResponse { //@@ //@@ .. cpp:var:: message SettingValue //@@ //@@ The values to be associated with a trace setting. //@@ message SettingValue { //@@ //@@ .. cpp:var:: string value (repeated) //@@ //@@ The value. //@@ repeated string value = 1; } //@@ .. cpp:var:: map settings //@@ //@@ The current trace settings, including any changes specified //@@ by TraceSettingRequest. //@@ map settings = 1; } //@@ //@@.. cpp:var:: message LogSettingsRequest //@@ //@@ Request message for LogSettings. //@@ message LogSettingsRequest { message SettingValue { oneof parameter_choice { //@@ .. cpp:var:: bool bool_param //@@ //@@ A boolean parameter value. //@@ bool bool_param = 1; //@@ .. cpp:var:: uint32 uint32_param //@@ //@@ An uint32 parameter value. //@@ uint32 uint32_param = 2; //@@ .. cpp:var:: string string_param //@@ //@@ A string parameter value. //@@ string string_param = 3; } } //@@ .. cpp:var:: map settings //@@ //@@ The current log settings. //@@ map settings = 1; } //@@ //@@.. cpp:var:: message LogSettingsResponse //@@ //@@ Response message for LogSettings. //@@ message LogSettingsResponse { message SettingValue { oneof parameter_choice { //@@ .. cpp:var:: bool bool_param //@@ //@@ A boolean parameter value. //@@ bool bool_param = 1; //@@ .. cpp:var:: uint32 uint32_param //@@ //@@ An int32 parameter value. //@@ uint32 uint32_param = 2; //@@ .. cpp:var:: string string_param //@@ //@@ A string parameter value. //@@ string string_param = 3; } } //@@ .. cpp:var:: map settings //@@ //@@ The current log settings. //@@ map settings = 1; } kytensor-2.49.0.7/common/protobuf/health.proto000066400000000000000000000047211527052371200213230ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. syntax = "proto3"; package grpc.health.v1; //@@.. cpp:namespace:: grpc.health.v1 //@@ //@@.. cpp:var:: message HealthCheckRequest //@@ //@@ Request message for HealthCheck //@@ message HealthCheckRequest { string service = 1; } //@@ //@@.. cpp:var:: message HealthCheckResponse //@@ //@@ Response message for HealthCheck //@@ message HealthCheckResponse { //@@ //@@.. cpp:enum:: ServingStatus //@@ //@@ Statuses supported by GRPC's health check. //@@ enum ServingStatus { UNKNOWN = 0; SERVING = 1; NOT_SERVING = 2; SERVICE_UNKNOWN = 3; } ServingStatus status = 1; } //@@ //@@.. cpp:var:: service Health //@@ //@@ Health service for GRPC endpoints. //@@ service Health { //@@ .. cpp:var:: rpc Check(HealthCheckRequest) returns //@@ (HealthCheckResponse) //@@ //@@ Get serving status of the inference server. //@@ rpc Check(HealthCheckRequest) returns (HealthCheckResponse); } kytensor-2.49.0.7/common/protobuf/model_config.proto000066400000000000000000002156771527052371200225210ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Copyright (c) 2018, TensorFlow Authors. All rights reserved. syntax = "proto3"; package inference; //@@.. cpp:namespace:: inference //@@ //@@.. cpp:enum:: DataType //@@ //@@ Data types supported for input and output tensors. //@@ enum DataType { //@@ .. cpp:enumerator:: DataType::INVALID = 0 TYPE_INVALID = 0; //@@ .. cpp:enumerator:: DataType::BOOL = 1 TYPE_BOOL = 1; //@@ .. cpp:enumerator:: DataType::UINT8 = 2 TYPE_UINT8 = 2; //@@ .. cpp:enumerator:: DataType::UINT16 = 3 TYPE_UINT16 = 3; //@@ .. cpp:enumerator:: DataType::UINT32 = 4 TYPE_UINT32 = 4; //@@ .. cpp:enumerator:: DataType::UINT64 = 5 TYPE_UINT64 = 5; //@@ .. cpp:enumerator:: DataType::INT8 = 6 TYPE_INT8 = 6; //@@ .. cpp:enumerator:: DataType::INT16 = 7 TYPE_INT16 = 7; //@@ .. cpp:enumerator:: DataType::INT32 = 8 TYPE_INT32 = 8; //@@ .. cpp:enumerator:: DataType::INT64 = 9 TYPE_INT64 = 9; //@@ .. cpp:enumerator:: DataType::FP16 = 10 TYPE_FP16 = 10; //@@ .. cpp:enumerator:: DataType::FP32 = 11 TYPE_FP32 = 11; //@@ .. cpp:enumerator:: DataType::FP64 = 12 TYPE_FP64 = 12; //@@ .. cpp:enumerator:: DataType::STRING = 13 TYPE_STRING = 13; //@@ .. cpp:enumerator:: DataType::BF16 = 14 TYPE_BF16 = 14; } //@@ //@@ .. cpp:var:: message ModelRateLimiter //@@ //@@ The specifications required by the rate limiter to properly //@@ schedule the inference requests across the different models //@@ and their instances. //@@ message ModelRateLimiter { //@@ .. cpp:var:: message Resource //@@ //@@ The resource property. //@@ message Resource { //@@ .. cpp:var:: string name //@@ //@@ The name associated with the resource. //@@ string name = 1; //@@ .. cpp:var:: bool global //@@ //@@ Whether or not the resource is global. If true then the resource //@@ is assumed to be shared among the devices otherwise specified //@@ count of the resource is assumed for each device associated //@@ with the instance. //@@ bool global = 2; //@@ .. cpp:var:: uint32 count //@@ //@@ The number of resources required for the execution of the model //@@ instance. //@@ uint32 count = 3; } //@@ .. cpp:var:: Resource resources (repeated) //@@ //@@ The resources required to execute the request on a model instance. //@@ Resources are just names with a corresponding count. The execution //@@ of the instance will be blocked until the specified resources are //@@ available. By default an instance uses no rate-limiter resources. //@@ repeated Resource resources = 1; //@@ .. cpp:var:: uint32 priority //@@ //@@ The optional weighting value to be used for prioritizing across //@@ instances. An instance with priority 2 will be given 1/2 the //@@ number of scheduling chances as an instance_group with priority //@@ 1. The default priority is 1. The priority of value 0 will be //@@ treated as priority 1. //@@ uint32 priority = 2; } //@@ //@@.. cpp:var:: message ModelInstanceGroup //@@ //@@ A group of one or more instances of a model and resources made //@@ available for those instances. //@@ message ModelInstanceGroup { //@@ //@@ .. cpp:enum:: Kind //@@ //@@ Kind of this instance group. //@@ enum Kind { //@@ .. cpp:enumerator:: Kind::KIND_AUTO = 0 //@@ //@@ This instance group represents instances that can run on either //@@ CPU or GPU. If all GPUs listed in 'gpus' are available then //@@ instances will be created on GPU(s), otherwise instances will //@@ be created on CPU. //@@ KIND_AUTO = 0; //@@ .. cpp:enumerator:: Kind::KIND_GPU = 1 //@@ //@@ This instance group represents instances that must run on the //@@ GPU. //@@ KIND_GPU = 1; //@@ .. cpp:enumerator:: Kind::KIND_CPU = 2 //@@ //@@ This instance group represents instances that must run on the //@@ CPU. //@@ KIND_CPU = 2; //@@ .. cpp:enumerator:: Kind::KIND_MODEL = 3 //@@ //@@ This instance group represents instances that should run on the //@@ CPU and/or GPU(s) as specified by the model or backend itself. //@@ The inference server will not override the model/backend //@@ settings. //@@ KIND_MODEL = 3; } //@@ //@@ .. cpp:var:: message SecondaryDevice //@@ //@@ A secondary device required for a model instance. //@@ message SecondaryDevice { //@@ //@@ .. cpp:enum:: SecondaryDeviceKind //@@ //@@ The kind of the secondary device. //@@ enum SecondaryDeviceKind { //@@ .. cpp:enumerator:: SecondaryDeviceKind::KIND_NVDLA = 0 //@@ //@@ An NVDLA core. http://nvdla.org //@@ Currently KIND_NVDLA is only supported by the TensorRT backend. //@@ KIND_NVDLA = 0; } //@@ .. cpp:var:: SecondaryDeviceKind kind //@@ //@@ The secondary device kind. //@@ SecondaryDeviceKind kind = 1; //@@ .. cpp:var:: int64 device_id //@@ //@@ Identifier for the secondary device. //@@ int64 device_id = 2; } //@@ .. cpp:var:: string name //@@ //@@ Optional name of this group of instances. If not specified the //@@ name will be formed as _. The name of //@@ individual instances will be further formed by a unique instance //@@ number and GPU index: //@@ string name = 1; //@@ .. cpp:var:: Kind kind //@@ //@@ The kind of this instance group. Default is KIND_AUTO. If //@@ KIND_AUTO or KIND_GPU then both 'count' and 'gpu' are valid and //@@ may be specified. If KIND_CPU or KIND_MODEL only 'count' is valid //@@ and 'gpu' cannot be specified. //@@ Kind kind = 4; //@@ .. cpp:var:: int32 count //@@ //@@ For a group assigned to GPU, the number of instances created for //@@ each GPU listed in 'gpus'. For a group assigned to CPU the number //@@ of instances created. Default is 1. int32 count = 2; //@@ .. cpp:var:: ModelRateLimiter rate_limiter //@@ //@@ The rate limiter specific settings to be associated with this //@@ instance group. Optional, if not specified no rate limiting //@@ will be applied to this instance group. //@@ ModelRateLimiter rate_limiter = 6; //@@ .. cpp:var:: int32 gpus (repeated) //@@ //@@ GPU(s) where instances should be available. For each GPU listed, //@@ 'count' instances of the model will be available. Setting 'gpus' //@@ to empty (or not specifying at all) is equivalent to listing all //@@ available GPUs. //@@ repeated int32 gpus = 3; //@@ .. cpp:var:: SecondaryDevice secondary_devices (repeated) //@@ //@@ Secondary devices that are required by instances specified by this //@@ instance group. Optional. //@@ repeated SecondaryDevice secondary_devices = 8; //@@ .. cpp:var:: string profile (repeated) //@@ //@@ For TensorRT models containing multiple optimization profile, this //@@ parameter specifies a set of optimization profiles available to this //@@ instance group. The inference server will choose the optimal profile //@@ based on the shapes of the input tensors. This field should lie //@@ between 0 and - 1 //@@ and be specified only for TensorRT backend, otherwise an error will //@@ be generated. If not specified, the server will select the first //@@ optimization profile by default. //@@ repeated string profile = 5; //@@ .. cpp:var:: bool passive //@@ //@@ Whether the instances within this instance group will be accepting //@@ inference requests from the scheduler. If true, the instances will //@@ not be added to the scheduler. Default value is false. //@@ bool passive = 7; //@@ .. cpp:var:: string host_policy //@@ //@@ The host policy name that the instance to be associated with. //@@ The default value is set to reflect the device kind of the instance, //@@ for instance, KIND_CPU is "cpu", KIND_MODEL is "model" and //@@ KIND_GPU is "gpu_". //@@ string host_policy = 9; } //@@ //@@.. cpp:var:: message ModelTensorReshape //@@ //@@ Reshape specification for input and output tensors. //@@ message ModelTensorReshape { //@@ .. cpp:var:: int64 shape (repeated) //@@ //@@ The shape to use for reshaping. //@@ repeated int64 shape = 1; } //@@ //@@.. cpp:var:: message ModelInput //@@ //@@ An input required by the model. //@@ message ModelInput { //@@ //@@ .. cpp:enum:: Format //@@ //@@ The format for the input. //@@ enum Format { //@@ .. cpp:enumerator:: Format::FORMAT_NONE = 0 //@@ //@@ The input has no specific format. This is the default. //@@ FORMAT_NONE = 0; //@@ .. cpp:enumerator:: Format::FORMAT_NHWC = 1 //@@ //@@ HWC image format. Tensors with this format require 3 dimensions //@@ if the model does not support batching (max_batch_size = 0) or 4 //@@ dimensions if the model does support batching (max_batch_size //@@ >= 1). In either case the 'dims' below should only specify the //@@ 3 non-batch dimensions (i.e. HWC or CHW). //@@ FORMAT_NHWC = 1; //@@ .. cpp:enumerator:: Format::FORMAT_NCHW = 2 //@@ //@@ CHW image format. Tensors with this format require 3 dimensions //@@ if the model does not support batching (max_batch_size = 0) or 4 //@@ dimensions if the model does support batching (max_batch_size //@@ >= 1). In either case the 'dims' below should only specify the //@@ 3 non-batch dimensions (i.e. HWC or CHW). //@@ FORMAT_NCHW = 2; } //@@ .. cpp:var:: string name //@@ //@@ The name of the input. //@@ string name = 1; //@@ .. cpp:var:: DataType data_type //@@ //@@ The data-type of the input. //@@ DataType data_type = 2; //@@ .. cpp:var:: Format format //@@ //@@ The format of the input. Optional. //@@ Format format = 3; //@@ .. cpp:var:: int64 dims (repeated) //@@ //@@ The dimensions/shape of the input tensor that must be provided //@@ when invoking the inference API for this model. //@@ repeated int64 dims = 4; //@@ .. cpp:var:: ModelTensorReshape reshape //@@ //@@ The shape expected for this input by the backend. The input will //@@ be reshaped to this before being presented to the backend. The //@@ reshape must have the same number of elements as the input shape //@@ specified by 'dims'. Optional. //@@ ModelTensorReshape reshape = 5; //@@ .. cpp:var:: bool is_shape_tensor //@@ //@@ Whether or not the input is a shape tensor to the model. This field //@@ is currently supported only for the TensorRT model. An error will be //@@ generated if this specification does not comply with underlying //@@ model. //@@ bool is_shape_tensor = 6; //@@ .. cpp:var:: bool allow_ragged_batch //@@ //@@ Whether or not the input is allowed to be "ragged" in a dynamically //@@ created batch. Default is false indicating that two requests will //@@ only be batched if this tensor has the same shape in both requests. //@@ True indicates that two requests can be batched even if this tensor //@@ has a different shape in each request. //@@ bool allow_ragged_batch = 7; //@@ .. cpp:var:: bool optional //@@ //@@ Whether or not the input is optional for the model execution. //@@ If true, the input is not required in the inference request. //@@ Default value is false. //@@ bool optional = 8; //@@ .. cpp:var:: bool is_non_linear_format_io //@@ //@@ Indicates whether the input tensor uses a non-linear IO format. This //@@ field is currently supported only for TensorRT models. An error will //@@ be generated if this specification does not comply with the //@@ underlying model. //@@ bool is_non_linear_format_io = 9; } //@@ //@@.. cpp:var:: message ModelOutput //@@ //@@ An output produced by the model. //@@ message ModelOutput { //@@ .. cpp:var:: string name //@@ //@@ The name of the output. //@@ string name = 1; //@@ .. cpp:var:: DataType data_type //@@ //@@ The data-type of the output. //@@ DataType data_type = 2; //@@ .. cpp:var:: int64 dims (repeated) //@@ //@@ The dimensions/shape of the output tensor. //@@ repeated int64 dims = 3; //@@ .. cpp:var:: ModelTensorReshape reshape //@@ //@@ The shape produced for this output by the backend. The output will //@@ be reshaped from this to the shape specified in 'dims' before being //@@ returned in the inference response. The reshape must have the same //@@ number of elements as the output shape specified by 'dims'. Optional. //@@ ModelTensorReshape reshape = 5; //@@ .. cpp:var:: string label_filename //@@ //@@ The label file associated with this output. Should be specified only //@@ for outputs that represent classifications. Optional. //@@ string label_filename = 4; //@@ .. cpp:var:: bool is_shape_tensor //@@ //@@ Whether or not the output is a shape tensor to the model. This field //@@ is currently supported only for the TensorRT model. An error will be //@@ generated if this specification does not comply with underlying //@@ model. //@@ bool is_shape_tensor = 6; //@@ .. cpp:var:: bool is_non_linear_format_io //@@ //@@ Indicates whether the output tensor uses a non-linear IO format. This //@@ field is currently supported only for TensorRT models. An error will //@@ be generated if this specification does not comply with the //@@ underlying model. //@@ bool is_non_linear_format_io = 7; } //@@ .. cpp:var:: message BatchInput //@@ //@@ A batch input is an additional input that must be added by //@@ the backend based on all the requests in a batch. //@@ message BatchInput { //@@ //@@ .. cpp:enum:: Kind //@@ //@@ The kind of the batch input. //@@ enum Kind { //@@ .. cpp:enumerator:: Kind::BATCH_ELEMENT_COUNT = 0 //@@ //@@ The element count of the 'source_input' will be added as //@@ input with shape [1]. //@@ BATCH_ELEMENT_COUNT = 0; //@@ .. cpp:enumerator:: Kind::BATCH_ACCUMULATED_ELEMENT_COUNT = 1 //@@ //@@ The accumulated element count of the 'source_input' will be //@@ added as input with shape [1]. For example, if there is a //@@ batch of two request, each with 2 elements, an input of value //@@ 2 will be added to the first request, and an input of value //@@ 4 will be added to the second request. //@@ BATCH_ACCUMULATED_ELEMENT_COUNT = 1; //@@ .. cpp:enumerator:: //@@ Kind::BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO = 2 //@@ //@@ The accumulated element count of the 'source_input' will be //@@ added as input with shape [1], except for the first request //@@ in the batch. For the first request in the batch, the input //@@ will have shape [2] where the first element is value 0. //@@ BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO = 2; //@@ .. cpp:enumerator:: Kind::BATCH_MAX_ELEMENT_COUNT_AS_SHAPE = 3 //@@ //@@ Among the requests in the batch, the max element count of the //@@ 'source_input' will be added as input with shape //@@ [max_element_count] for the first request in the batch. //@@ For other requests, such input will be with shape [0]. //@@ The data of the tensor will be uninitialized. //@@ BATCH_MAX_ELEMENT_COUNT_AS_SHAPE = 3; //@@ .. cpp:enumerator:: Kind::BATCH_ITEM_SHAPE = 4 //@@ //@@ Among the requests in the batch, the shape of the //@@ 'source_input' will be added as input with shape //@@ [batch_size, len(input_dim)]. For example, if one //@@ batch-2 input with shape [3, 1] and batch-1 input //@@ with shape [2, 2] are batched, the batch input will //@@ have shape [3, 2] and value [ [3, 1], [3, 1], [2, 2]]. //@@ BATCH_ITEM_SHAPE = 4; //@@ .. cpp:enumerator:: Kind::BATCH_ITEM_SHAPE_FLATTEN = 5 //@@ //@@ Among the requests in the batch, the shape of the //@@ 'source_input' will be added as input with single dimensional //@@ shape [batch_size * len(input_dim)]. For example, if one //@@ batch-2 input with shape [3, 1] and batch-1 input //@@ with shape [2, 2] are batched, the batch input will //@@ have shape [6] and value [3, 1, 3, 1, 2, 2]. //@@ BATCH_ITEM_SHAPE_FLATTEN = 5; } //@@ .. cpp:var:: Kind kind //@@ //@@ The kind of this batch input. //@@ Kind kind = 1; //@@ .. cpp:var:: string target_name (repeated) //@@ //@@ The name of the model inputs that the backend will create //@@ for this batch input. //@@ repeated string target_name = 2; //@@ .. cpp:var:: DataType data_type //@@ //@@ The input's datatype. The data type can be TYPE_INT32 or //@@ TYPE_FP32. //@@ DataType data_type = 3; //@@ .. cpp:var:: string source_input (repeated) //@@ //@@ The backend derives the value for each batch input from one or //@@ more other inputs. 'source_input' gives the names of those //@@ inputs. //@@ repeated string source_input = 4; } //@@.. cpp:var:: message BatchOutput //@@ //@@ A batch output is an output produced by the model that must be handled //@@ differently by the backend based on all the requests in a batch. //@@ message BatchOutput { //@@ //@@ .. cpp:enum:: Kind //@@ //@@ The kind of the batch output. //@@ enum Kind { //@@ .. cpp:enumerator:: Kind::BATCH_SCATTER_WITH_INPUT_SHAPE = 0 //@@ //@@ The output should be scattered according to the shape of //@@ 'source_input'. The dynamic dimension of the output will //@@ be set to the value of the same dimension in the input. //@@ BATCH_SCATTER_WITH_INPUT_SHAPE = 0; } //@@ .. cpp:var:: string target_name (repeated) //@@ //@@ The name of the outputs to be produced by this batch output //@@ specification. //@@ repeated string target_name = 1; //@@ .. cpp:var:: Kind kind //@@ //@@ The kind of this batch output. //@@ Kind kind = 2; //@@ .. cpp:var:: string source_input (repeated) //@@ //@@ The backend derives each batch output from one or more inputs. //@@ 'source_input' gives the names of those inputs. //@@ repeated string source_input = 3; } //@@ //@@.. cpp:var:: message ModelVersionPolicy //@@ //@@ Policy indicating which versions of a model should be made //@@ available by the inference server. //@@ message ModelVersionPolicy { //@@ .. cpp:var:: message Latest //@@ //@@ Serve only the latest version(s) of a model. This is //@@ the default policy. //@@ message Latest { //@@ .. cpp:var:: uint32 num_versions //@@ //@@ Serve only the 'num_versions' highest-numbered versions. T //@@ The default value of 'num_versions' is 1, indicating that by //@@ default only the single highest-number version of a //@@ model will be served. //@@ uint32 num_versions = 1; } //@@ .. cpp:var:: message All //@@ //@@ Serve all versions of the model. //@@ message All {} //@@ .. cpp:var:: message Specific //@@ //@@ Serve only specific versions of the model. //@@ message Specific { //@@ .. cpp:var:: int64 versions (repeated) //@@ //@@ The specific versions of the model that will be served. //@@ repeated int64 versions = 1; } //@@ .. cpp:var:: oneof policy_choice //@@ //@@ Each model must implement only a single version policy. The //@@ default policy is 'Latest'. //@@ oneof policy_choice { //@@ .. cpp:var:: Latest latest //@@ //@@ Serve only latest version(s) of the model. //@@ Latest latest = 1; //@@ .. cpp:var:: All all //@@ //@@ Serve all versions of the model. //@@ All all = 2; //@@ .. cpp:var:: Specific specific //@@ //@@ Serve only specific version(s) of the model. //@@ Specific specific = 3; } } //@@ //@@.. cpp:var:: message ModelOptimizationPolicy //@@ //@@ Optimization settings for a model. These settings control if/how a //@@ model is optimized and prioritized by the backend framework when //@@ it is loaded. //@@ message ModelOptimizationPolicy { //@@ //@@ .. cpp:var:: message Graph //@@ //@@ Enable generic graph optimization of the model. If not specified //@@ the framework's default level of optimization is used. Supports //@@ TensorFlow graphdef and savedmodel and Onnx models. For TensorFlow //@@ causes XLA to be enabled/disabled for the model. For Onnx defaults //@@ to enabling all optimizations, -1 enables only basic optimizations, //@@ +1 enables only basic and extended optimizations. //@@ message Graph { //@@ .. cpp:var:: int32 level //@@ //@@ The optimization level. Defaults to 0 (zero) if not specified. //@@ //@@ - -1: Disabled //@@ - 0: Framework default //@@ - 1+: Enable optimization level (greater values indicate //@@ higher optimization levels) //@@ int32 level = 1; } //@@ //@@ .. cpp:enum:: ModelPriority //@@ //@@ Model priorities. A model will be given scheduling and execution //@@ preference over models at lower priorities. Current model //@@ priorities only work for TensorRT models. //@@ enum ModelPriority { //@@ .. cpp:enumerator:: ModelPriority::PRIORITY_DEFAULT = 0 //@@ //@@ The default model priority. //@@ PRIORITY_DEFAULT = 0; //@@ .. cpp:enumerator:: ModelPriority::PRIORITY_MAX = 1 //@@ //@@ The maximum model priority. //@@ PRIORITY_MAX = 1; //@@ .. cpp:enumerator:: ModelPriority::PRIORITY_MIN = 2 //@@ //@@ The minimum model priority. //@@ PRIORITY_MIN = 2; } //@@ //@@ .. cpp:var:: message Cuda //@@ //@@ CUDA-specific optimization settings. //@@ message Cuda { //@@ .. cpp:var:: message GraphSpec //@@ //@@ Specification of the CUDA graph to be captured. //@@ message GraphSpec { //@@ .. cpp:var:: message Dims //@@ //@@ Specification of tensor dimension. //@@ message Shape { //@@ .. cpp:var:: int64 dim (repeated) //@@ //@@ The dimension. //@@ repeated int64 dim = 1; } message LowerBound { //@@ .. cpp:var:: int32 batch_size //@@ //@@ The batch size of the CUDA graph. If 'max_batch_size' is 0, //@@ 'batch_size' must be set to 0. Otherwise, 'batch_size' must //@@ be set to value between 1 and 'max_batch_size'. //@@ int32 batch_size = 1; //@@ .. cpp:var:: map input //@@ //@@ The specification of the inputs. 'Shape' is the shape of //@@ the input without batching dimension. //@@ map input = 2; } //@@ .. cpp:var:: int32 batch_size //@@ //@@ The batch size of the CUDA graph. If 'max_batch_size' is 0, //@@ 'batch_size' must be set to 0. Otherwise, 'batch_size' must //@@ be set to value between 1 and 'max_batch_size'. //@@ int32 batch_size = 1; //@@ .. cpp:var:: map input //@@ //@@ The specification of the inputs. 'Shape' is the shape of the //@@ input without batching dimension. //@@ map input = 2; //@@ .. cpp:var:: LowerBound graph_lower_bound //@@ //@@ Specify the lower bound of the CUDA graph. Optional. //@@ If specified, the graph can be used for input shapes and //@@ batch sizes that are in closed interval between the lower //@@ bound specification and graph specification. For dynamic //@@ shape model, this allows CUDA graphs to be launched //@@ frequently without capturing all possible shape combinations. //@@ However, using graph for shape combinations different from //@@ the one used for capturing introduces uninitialized data for //@@ execution and it may distort the inference result if //@@ the model is sensitive to uninitialized data. //@@ LowerBound graph_lower_bound = 3; } //@@ .. cpp:var:: bool graphs //@@ //@@ Use CUDA graphs API to capture model operations and execute //@@ them more efficiently. Default value is false. //@@ Currently only recognized by TensorRT backend. //@@ bool graphs = 1; //@@ .. cpp:var:: bool busy_wait_events //@@ //@@ Use busy-waiting to synchronize CUDA events to achieve minimum //@@ latency from event complete to host thread to be notified, with //@@ the cost of high CPU load. Default value is false. //@@ Currently only recognized by TensorRT backend. //@@ bool busy_wait_events = 2; //@@ .. cpp:var:: GraphSpec graph_spec (repeated) //@@ //@@ Specification of the CUDA graph to be captured. If not specified //@@ and 'graphs' is true, the default CUDA graphs will be captured //@@ based on model settings. //@@ Currently only recognized by TensorRT backend. //@@ repeated GraphSpec graph_spec = 3; //@@ .. cpp:var:: bool output_copy_stream //@@ //@@ Uses a CUDA stream separate from the inference stream to copy the //@@ output to host. However, be aware that setting this option to //@@ true will lead to an increase in the memory consumption of the //@@ model as Triton will allocate twice as much GPU memory for its //@@ I/O tensor buffers. Default value is false. //@@ Currently only recognized by TensorRT backend. //@@ bool output_copy_stream = 4; } //@@ //@@ .. cpp:var:: message ExecutionAccelerators //@@ //@@ Specify the preferred execution accelerators to be used to execute //@@ the model. Currently only recognized by ONNX Runtime backend and //@@ TensorFlow backend. //@@ //@@ For ONNX Runtime backend, it will deploy the model with the execution //@@ accelerators by priority, the priority is determined based on the //@@ order that they are set, i.e. the provider at the front has highest //@@ priority. Overall, the priority will be in the following order: //@@ (if instance is on GPU) //@@ CUDA Execution Provider (if instance is on GPU) //@@ //@@ Default CPU Execution Provider //@@ message ExecutionAccelerators { //@@ //@@ .. cpp:var:: message Accelerator //@@ //@@ Specify the accelerator to be used to execute the model. //@@ Accelerator with the same name may accept different parameters //@@ depending on the backends. //@@ message Accelerator { //@@ .. cpp:var:: string name //@@ //@@ The name of the execution accelerator. //@@ string name = 1; //@@ .. cpp:var:: map parameters //@@ //@@ Additional parameters used to configure the accelerator. //@@ map parameters = 2; } //@@ .. cpp:var:: Accelerator gpu_execution_accelerator (repeated) //@@ //@@ The preferred execution provider to be used if the model instance //@@ is deployed on GPU. //@@ //@@ For ONNX Runtime backend, possible value is "tensorrt" as name, //@@ and no parameters are required. //@@ //@@ For TensorFlow backend, possible values are "tensorrt", //@@ "auto_mixed_precision", "gpu_io". //@@ //@@ For "tensorrt", the following parameters can be specified: //@@ "precision_mode": The precision used for optimization. //@@ Allowed values are "FP32" and "FP16". Default value is "FP32". //@@ //@@ "max_cached_engines": The maximum number of cached TensorRT //@@ engines in dynamic TensorRT ops. Default value is 100. //@@ //@@ "minimum_segment_size": The smallest model subgraph that will //@@ be considered for optimization by TensorRT. Default value is 3. //@@ //@@ "max_workspace_size_bytes": The maximum GPU memory the model //@@ can use temporarily during execution. Default value is 1GB. //@@ //@@ For "auto_mixed_precision", no parameters are required. If set, //@@ the model will try to use FP16 for better performance. //@@ This optimization can not be set with "tensorrt". //@@ //@@ For "gpu_io", no parameters are required. If set, the model will //@@ be executed using TensorFlow Callable API to set input and output //@@ tensors in GPU memory if possible, which can reduce data transfer //@@ overhead if the model is used in ensemble. However, the Callable //@@ object will be created on model creation and it will request all //@@ outputs for every model execution, which may impact the //@@ performance if a request does not require all outputs. This //@@ optimization will only take affect if the model instance is //@@ created with KIND_GPU. //@@ repeated Accelerator gpu_execution_accelerator = 1; //@@ .. cpp:var:: Accelerator cpu_execution_accelerator (repeated) //@@ //@@ The preferred execution provider to be used if the model instance //@@ is deployed on CPU. //@@ //@@ For ONNX Runtime backend, possible value is "openvino" as name, //@@ and no parameters are required. //@@ repeated Accelerator cpu_execution_accelerator = 2; } //@@ //@@ .. cpp:var:: message PinnedMemoryBuffer //@@ //@@ Specify whether to use a pinned memory buffer when transferring data //@@ between non-pinned system memory and GPU memory. Using a pinned //@@ memory buffer for system from/to GPU transfers will typically provide //@@ increased performance. For example, in the common use case where the //@@ request provides inputs and delivers outputs via non-pinned system //@@ memory, if the model instance accepts GPU IOs, the inputs will be //@@ processed by two copies: from non-pinned system memory to pinned //@@ memory, and from pinned memory to GPU memory. Similarly, pinned //@@ memory will be used for delivering the outputs. //@@ message PinnedMemoryBuffer { //@@ .. cpp:var:: bool enable //@@ //@@ Use pinned memory buffer. Default is true. //@@ bool enable = 1; } //@@ .. cpp:var:: Graph graph //@@ //@@ The graph optimization setting for the model. Optional. //@@ Graph graph = 1; //@@ .. cpp:var:: ModelPriority priority //@@ //@@ The priority setting for the model. Optional. //@@ ModelPriority priority = 2; //@@ .. cpp:var:: Cuda cuda //@@ //@@ CUDA-specific optimization settings. Optional. //@@ Cuda cuda = 3; //@@ .. cpp:var:: ExecutionAccelerators execution_accelerators //@@ //@@ The accelerators used for the model. Optional. //@@ ExecutionAccelerators execution_accelerators = 4; //@@ .. cpp:var:: PinnedMemoryBuffer input_pinned_memory //@@ //@@ Use pinned memory buffer when the data transfer for inputs //@@ is between GPU memory and non-pinned system memory. //@@ Default is true. //@@ PinnedMemoryBuffer input_pinned_memory = 5; //@@ .. cpp:var:: PinnedMemoryBuffer output_pinned_memory //@@ //@@ Use pinned memory buffer when the data transfer for outputs //@@ is between GPU memory and non-pinned system memory. //@@ Default is true. //@@ PinnedMemoryBuffer output_pinned_memory = 6; //@@ .. cpp:var:: uint32 gather_kernel_buffer_threshold //@@ //@@ The backend may use a gather kernel to gather input data if the //@@ device has direct access to the source buffer and the destination //@@ buffer. In such case, the gather kernel will be used only if the //@@ number of buffers to be gathered is greater or equal to //@@ the specified value. If 0, the gather kernel will be disabled. //@@ Default value is 0. //@@ Currently only recognized by TensorRT backend. //@@ uint32 gather_kernel_buffer_threshold = 7; //@@ .. cpp:var:: bool eager_batching //@@ //@@ Start preparing the next batch before the model instance is ready //@@ for the next inference. This option can be used to overlap the //@@ batch preparation with model execution, with the trade-off that //@@ the next batch might be smaller than what it could have been. //@@ Default value is false. //@@ Currently only recognized by TensorRT backend. //@@ bool eager_batching = 8; } //@@ //@@.. cpp:var:: message ModelQueuePolicy //@@ //@@ Queue policy for inference requests. //@@ message ModelQueuePolicy { //@@ //@@ .. cpp:enum:: TimeoutAction //@@ //@@ The action applied to timed-out requests. //@@ enum TimeoutAction { //@@ .. cpp:enumerator:: Action::REJECT = 0 //@@ //@@ Reject the request and return error message accordingly. //@@ REJECT = 0; //@@ .. cpp:enumerator:: Action::DELAY = 1 //@@ //@@ Delay the request until all other requests at the same //@@ (or higher) priority levels that have not reached their timeouts //@@ are processed. A delayed request will eventually be processed, //@@ but may be delayed indefinitely due to newly arriving requests. //@@ DELAY = 1; } //@@ //@@ .. cpp:var:: TimeoutAction timeout_action //@@ //@@ The action applied to timed-out request. //@@ The default action is REJECT. //@@ TimeoutAction timeout_action = 1; //@@ //@@ .. cpp:var:: uint64 default_timeout_microseconds //@@ //@@ The default timeout for every request, in microseconds. //@@ The default value is 0 which indicates that no timeout is set. //@@ uint64 default_timeout_microseconds = 2; //@@ //@@ .. cpp:var:: bool allow_timeout_override //@@ //@@ Whether individual request can override the default timeout value. //@@ When true, individual requests can set a timeout that is less than //@@ the default timeout value but may not increase the timeout. //@@ The default value is false. //@@ bool allow_timeout_override = 3; //@@ //@@ .. cpp:var:: uint32 max_queue_size //@@ //@@ The maximum queue size for holding requests. A request will be //@@ rejected immediately if it can't be enqueued because the queue is //@@ full. The default value is 0 which indicates that no maximum //@@ queue size is enforced. //@@ uint32 max_queue_size = 4; } //@@ //@@.. cpp:var:: message ModelDynamicBatching //@@ //@@ Dynamic batching configuration. These settings control how dynamic //@@ batching operates for the model. //@@ message ModelDynamicBatching { //@@ .. cpp:var:: int32 preferred_batch_size (repeated) //@@ //@@ Preferred batch sizes for dynamic batching. If a batch of one of //@@ these sizes can be formed it will be executed immediately. If //@@ not specified a preferred batch size will be chosen automatically //@@ based on model and GPU characteristics. //@@ repeated int32 preferred_batch_size = 1; //@@ .. cpp:var:: uint64 max_queue_delay_microseconds //@@ //@@ The maximum time, in microseconds, a request will be delayed in //@@ the scheduling queue to wait for additional requests for //@@ batching. Default is 0. //@@ uint64 max_queue_delay_microseconds = 2; //@@ .. cpp:var:: bool preserve_ordering //@@ //@@ Should the dynamic batcher preserve the ordering of responses to //@@ match the order of requests received by the scheduler. Default is //@@ false. If true, the responses will be returned in the same order as //@@ the order of requests sent to the scheduler. If false, the responses //@@ may be returned in arbitrary order. This option is specifically //@@ needed when a sequence of related inference requests (i.e. inference //@@ requests with the same correlation ID) are sent to the dynamic //@@ batcher to ensure that the sequence responses are in the correct //@@ order. //@@ bool preserve_ordering = 3; //@@ .. cpp:var:: uint64 priority_levels //@@ //@@ The number of priority levels to be enabled for the model, //@@ the priority level starts from 1 and 1 is the highest priority. //@@ Requests are handled in priority order with all priority 1 requests //@@ processed before priority 2, all priority 2 requests processed before //@@ priority 3, etc. Requests with the same priority level will be //@@ handled in the order that they are received. //@@ uint64 priority_levels = 4; //@@ .. cpp:var:: uint64 default_priority_level //@@ //@@ The priority level used for requests that don't specify their //@@ priority. The value must be in the range [ 1, 'priority_levels' ]. //@@ uint64 default_priority_level = 5; //@@ .. cpp:var:: ModelQueuePolicy default_queue_policy //@@ //@@ The default queue policy used for requests that don't require //@@ priority handling and requests that specify priority levels where //@@ there is no specific policy given. If not specified, a policy with //@@ default field values will be used. //@@ ModelQueuePolicy default_queue_policy = 6; //@@ .. cpp:var:: map priority_queue_policy //@@ //@@ Specify the queue policy for the priority level. The default queue //@@ policy will be used if a priority level doesn't specify a queue //@@ policy. //@@ map priority_queue_policy = 7; } //@@ //@@.. cpp:var:: message ModelSequenceBatching //@@ //@@ Sequence batching configuration. These settings control how sequence //@@ batching operates for the model. //@@ message ModelSequenceBatching { //@@ .. cpp:var:: message Control //@@ //@@ A control is a signal that the sequence batcher uses to //@@ communicate with a backend. //@@ message Control { //@@ //@@ .. cpp:enum:: Kind //@@ //@@ The kind of the control. //@@ enum Kind { //@@ .. cpp:enumerator:: Kind::CONTROL_SEQUENCE_START = 0 //@@ //@@ A new sequence is/is-not starting. If true a sequence is //@@ starting, if false a sequence is continuing. Must //@@ specify either int32_false_true, fp32_false_true or //@@ bool_false_true for this control. This control is optional. //@@ CONTROL_SEQUENCE_START = 0; //@@ .. cpp:enumerator:: Kind::CONTROL_SEQUENCE_READY = 1 //@@ //@@ A sequence is/is-not ready for inference. If true the //@@ input tensor data is valid and should be used. If false //@@ the input tensor data is invalid and inferencing should //@@ be "skipped". Must specify either int32_false_true, //@@ fp32_false_true or bool_false_true for this control. This //@@ control is optional. //@@ CONTROL_SEQUENCE_READY = 1; //@@ .. cpp:enumerator:: Kind::CONTROL_SEQUENCE_END = 2 //@@ //@@ A sequence is/is-not ending. If true a sequence is //@@ ending, if false a sequence is continuing. Must specify //@@ either int32_false_true, fp32_false_true or bool_false_true //@@ for this control. This control is optional. //@@ CONTROL_SEQUENCE_END = 2; //@@ .. cpp:enumerator:: Kind::CONTROL_SEQUENCE_CORRID = 3 //@@ //@@ The correlation ID of the sequence. The correlation ID //@@ is an uint64_t value that is communicated in whole or //@@ in part by the tensor. The tensor's datatype must be //@@ specified by data_type and must be TYPE_UINT64, TYPE_INT64, //@@ TYPE_UINT32 or TYPE_INT32. If a 32-bit datatype is specified //@@ the correlation ID will be truncated to the low-order 32 //@@ bits. This control is optional. //@@ CONTROL_SEQUENCE_CORRID = 3; } //@@ .. cpp:var:: Kind kind //@@ //@@ The kind of this control. //@@ Kind kind = 1; //@@ .. cpp:var:: int32 int32_false_true (repeated) //@@ //@@ The control's true and false setting is indicated by setting //@@ a value in an int32 tensor. The tensor must be a //@@ 1-dimensional tensor with size equal to the batch size of //@@ the request. 'int32_false_true' must have two entries: the //@@ first the false value and the second the true value. //@@ repeated int32 int32_false_true = 2; //@@ .. cpp:var:: float fp32_false_true (repeated) //@@ //@@ The control's true and false setting is indicated by setting //@@ a value in a fp32 tensor. The tensor must be a //@@ 1-dimensional tensor with size equal to the batch size of //@@ the request. 'fp32_false_true' must have two entries: the //@@ first the false value and the second the true value. //@@ repeated float fp32_false_true = 3; //@@ .. cpp:var:: bool bool_false_true (repeated) //@@ //@@ The control's true and false setting is indicated by setting //@@ a value in a bool tensor. The tensor must be a //@@ 1-dimensional tensor with size equal to the batch size of //@@ the request. 'bool_false_true' must have two entries: the //@@ first the false value and the second the true value. //@@ repeated bool bool_false_true = 5; //@@ .. cpp:var:: DataType data_type //@@ //@@ The control's datatype. //@@ DataType data_type = 4; } //@@ .. cpp:var:: message ControlInput //@@ //@@ The sequence control values to communicate by a model input. //@@ message ControlInput { //@@ .. cpp:var:: string name //@@ //@@ The name of the model input. //@@ string name = 1; //@@ .. cpp:var:: Control control (repeated) //@@ //@@ The control value(s) that should be communicated to the //@@ model using this model input. //@@ repeated Control control = 2; } //@@ //@@ .. cpp:var:: message InitialState //@@ //@@ Settings used to initialize data for implicit state. //@@ message InitialState { //@@ .. cpp:var:: DataType data_type //@@ //@@ The data-type of the state. //@@ DataType data_type = 1; //@@ .. cpp:var:: int64 dims (repeated) //@@ //@@ The shape of the state tensor, not including the batch //@@ dimension. //@@ repeated int64 dims = 2; //@@ .. cpp:var:: oneof state_data //@@ //@@ Specify how the initial state data is generated. //@@ oneof state_data { //@@ //@@ .. cpp:var:: bool zero_data //@@ //@@ The identifier for using zeros as initial state data. //@@ Note that the value of 'zero_data' will not be checked, //@@ instead, zero data will be used as long as the field is set. //@@ bool zero_data = 3; //@@ .. cpp:var:: string data_file //@@ //@@ The file whose content will be used as the initial data for //@@ the state in row-major order. The file must be provided in //@@ sub-directory 'initial_state' under the model directory. //@@ string data_file = 4; } //@@ .. cpp:var:: string name //@@ //@@ The name of the state initialization. //@@ string name = 5; } //@@ .. cpp:var:: message State //@@ //@@ An input / output pair of tensors that carry state for the sequence. //@@ message State { //@@ .. cpp:var:: string input_name //@@ //@@ The name of the model state input. //@@ string input_name = 1; //@@ .. cpp:var:: string output_name //@@ //@@ The name of the model state output. //@@ string output_name = 2; //@@ .. cpp:var:: DataType data_type //@@ //@@ The data-type of the state. //@@ DataType data_type = 3; //@@ .. cpp:var:: int64 dim (repeated) //@@ //@@ The dimension. //@@ repeated int64 dims = 4; //@@ .. cpp:var:: InitialState initial_state (repeated) //@@ //@@ The optional field to specify the initial state for the model. //@@ repeated InitialState initial_state = 5; //@@ .. cpp:var:: bool use_same_buffer_for_input_output //@@ //@@ The optional field to use a single buffer for both input and output //@@ state. Without this option, Triton allocates separate buffers //@@ for input and output state //@@ which can be problematic if the state size is //@@ large. This option reduces the memory usage by allocating a single //@@ buffer. Enabling this option is recommended whenever //@@ the input state is processed before the output state is written. //@@ When enabled the state //@@ will always be updated independent of whether //@@ TRITONBACKEND_StateUpdate is called //@@ (however TRITONBACKEND_StateUpdate should still be called for //@@ completeness). //@@ //@@ The default value is false. //@@ bool use_same_buffer_for_input_output = 6; //@@ .. cpp:var:: bool use_growable_memory //@@ //@@ The optional field to enable an implicit state buffer to grow //@@ without reallocating or copying existing memory. //@@ Additional memory will be appended to the end of the buffer and //@@ existing data will be preserved. //@@ This option is only available for CUDA memory and requires enabling //@@ use_same_buffer_for_input_output. When using this option, //@@ StateBuffer call will always return CUDA memory even if CPU memory //@@ is requested. //@@ //@@ The default value is false. //@@ bool use_growable_memory = 7; } //@@ .. cpp:var:: message StrategyDirect //@@ //@@ The sequence batcher uses a specific, unique batch //@@ slot for each sequence. All inference requests in a //@@ sequence are directed to the same batch slot in the same //@@ model instance over the lifetime of the sequence. This //@@ is the default strategy. //@@ message StrategyDirect { //@@ .. cpp:var:: uint64 max_queue_delay_microseconds //@@ //@@ The maximum time, in microseconds, a candidate request //@@ will be delayed in the sequence batch scheduling queue to //@@ wait for additional requests for batching. Default is 0. //@@ uint64 max_queue_delay_microseconds = 1; //@@ .. cpp:var:: float minimum_slot_utilization //@@ //@@ The minimum slot utilization that must be satisfied to //@@ execute the batch before 'max_queue_delay_microseconds' expires. //@@ For example, a value of 0.5 indicates that the batch should be //@@ executed as soon as 50% or more of the slots are ready even if //@@ the 'max_queue_delay_microseconds' timeout has not expired. //@@ The default is 0.0, indicating that a batch will be executed //@@ before 'max_queue_delay_microseconds' timeout expires if at least //@@ one batch slot is ready. 'max_queue_delay_microseconds' will be //@@ ignored unless minimum_slot_utilization is set to a non-zero //@@ value. //@@ float minimum_slot_utilization = 2; } //@@ .. cpp:var:: message StrategyOldest //@@ //@@ The sequence batcher maintains up to 'max_candidate_sequences' //@@ candidate sequences. 'max_candidate_sequences' can be greater //@@ than the model's 'max_batch_size'. For inferencing the batcher //@@ chooses from the candidate sequences up to 'max_batch_size' //@@ inference requests. Requests are chosen in an oldest-first //@@ manner across all candidate sequences. A given sequence is //@@ not guaranteed to be assigned to the same batch slot for //@@ all inference requests of that sequence. //@@ message StrategyOldest { //@@ .. cpp:var:: int32 max_candidate_sequences //@@ //@@ Maximum number of candidate sequences that the batcher //@@ maintains. Excess sequences are kept in an ordered backlog //@@ and become candidates when existing candidate sequences //@@ complete. //@@ int32 max_candidate_sequences = 1; //@@ .. cpp:var:: int32 preferred_batch_size (repeated) //@@ //@@ Preferred batch sizes for dynamic batching of candidate //@@ sequences. If a batch of one of these sizes can be formed //@@ it will be executed immediately. If not specified a //@@ preferred batch size will be chosen automatically //@@ based on model and GPU characteristics. //@@ repeated int32 preferred_batch_size = 2; //@@ .. cpp:var:: uint64 max_queue_delay_microseconds //@@ //@@ The maximum time, in microseconds, a candidate request //@@ will be delayed in the dynamic batch scheduling queue to //@@ wait for additional requests for batching. Default is 0. //@@ uint64 max_queue_delay_microseconds = 3; //@@ .. cpp:var:: bool preserve_ordering //@@ //@@ Should the dynamic batcher preserve the ordering of responses to //@@ match the order of requests received by the scheduler. Default is //@@ false. If true, the responses will be returned in the same order //@@ as the order of requests sent to the scheduler. If false, the //@@ responses may be returned in arbitrary order. This option is //@@ specifically needed when a sequence of related inference requests //@@ (i.e. inference requests with the same correlation ID) are sent //@@ to the dynamic batcher to ensure that the sequence responses are //@@ in the correct order. //@@ //@@ When using decoupled models, setting this to true may block the //@@ responses from independent sequences from being returned to the //@@ client until the previous request completes, hurting overall //@@ performance. If using GRPC streaming protocol, the stream //@@ ordering guarantee may be sufficient alone to ensure the //@@ responses for each sequence are returned in sequence-order //@@ without blocking based on independent requests, depending on the //@@ use case. //@@ bool preserve_ordering = 4; } //@@ .. cpp:var:: oneof strategy_choice //@@ //@@ The strategy used by the sequence batcher. Default strategy //@@ is 'direct'. //@@ oneof strategy_choice { //@@ .. cpp:var:: StrategyDirect direct //@@ //@@ StrategyDirect scheduling strategy. //@@ StrategyDirect direct = 3; //@@ .. cpp:var:: StrategyOldest oldest //@@ //@@ StrategyOldest scheduling strategy. //@@ StrategyOldest oldest = 4; } //@@ .. cpp:var:: uint64 max_sequence_idle_microseconds //@@ //@@ The maximum time, in microseconds, that a sequence is allowed to //@@ be idle before it is aborted. The inference server considers a //@@ sequence idle when it does not have any inference request queued //@@ for the sequence. If this limit is exceeded, the inference server //@@ will free the sequence slot allocated by the sequence and make it //@@ available for another sequence. If not specified (or specified as //@@ zero) a default value of 1000000 (1 second) is used. //@@ uint64 max_sequence_idle_microseconds = 1; //@@ .. cpp:var:: ControlInput control_input (repeated) //@@ //@@ The model input(s) that the server should use to communicate //@@ sequence start, stop, ready and similar control values to the //@@ model. //@@ repeated ControlInput control_input = 2; //@@ .. cpp:var:: State state (repeated) //@@ //@@ The optional state that can be stored in Triton for performing //@@ inference requests on a sequence. Each sequence holds an implicit //@@ state local to itself. The output state tensor provided by the //@@ model in 'output_name' field of the current inference request will //@@ be transferred as an input tensor named 'input_name' in the next //@@ request of the same sequence. The input state of the first request //@@ in the sequence contains garbage data. //@@ repeated State state = 5; //@@ .. cpp:var:: bool iterative_sequence //@@ //@@ Requests for iterative sequences are processed over a number //@@ of iterations. An iterative sequence is initiated by a single //@@ request and is "rescheduled" by the model until completion. //@@ Requests for inflight requests will be batched together //@@ and can complete independently. Note this feature //@@ requires backend support. Default value is false. bool iterative_sequence = 6; } //@@ //@@.. cpp:var:: message ModelEnsembling //@@ //@@ Model ensembling configuration. These settings specify the models that //@@ compose the ensemble and how data flows between the models. //@@ message ModelEnsembling { //@@ .. cpp:var:: message Step //@@ //@@ Each step specifies a model included in the ensemble, //@@ maps ensemble tensor names to the model input tensors, //@@ and maps model output tensors to ensemble tensor names //@@ message Step { //@@ .. cpp:var:: string model_name //@@ //@@ The name of the model to execute for this step of the ensemble. //@@ string model_name = 1; //@@ .. cpp:var:: int64 model_version //@@ //@@ The version of the model to use for inference. If -1 //@@ the latest/most-recent version of the model is used. //@@ int64 model_version = 2; //@@ .. cpp:var:: map input_map //@@ //@@ Map from name of an input tensor on this step's model to ensemble //@@ tensor name. The ensemble tensor must have the same data type and //@@ shape as the model input. Each model input must be assigned to //@@ one ensemble tensor, but the same ensemble tensor can be assigned //@@ to multiple model inputs. //@@ map input_map = 3; //@@ .. cpp:var:: map output_map //@@ //@@ Map from name of an output tensor on this step's model to ensemble //@@ tensor name. The data type and shape of the ensemble tensor will //@@ be inferred from the model output. It is optional to assign all //@@ model outputs to ensemble tensors. One ensemble tensor name //@@ can appear in an output map only once. //@@ map output_map = 4; //@@ .. cpp:var:: string model_namespace //@@ //@@ [RESERVED] currently this field is reserved for internal use, users //@@ must not set any value to this field to avoid unexpected behavior. //@@ string model_namespace = 5; } //@@ .. cpp:var:: Step step (repeated) //@@ //@@ The models and the input / output mappings used within the ensemble. //@@ repeated Step step = 1; } //@@ //@@.. cpp:var:: message ModelParameter //@@ //@@ A model parameter. //@@ message ModelParameter { //@@ .. cpp:var:: string string_value //@@ //@@ The string value of the parameter. //@@ string string_value = 1; } //@@ //@@.. cpp:var:: message ModelWarmup //@@ //@@ Settings used to construct the request sample for model warmup. //@@ message ModelWarmup { //@@ //@@ .. cpp:var:: message Input //@@ //@@ Meta data associated with an input. //@@ message Input { //@@ .. cpp:var:: DataType data_type //@@ //@@ The data-type of the input. //@@ DataType data_type = 1; //@@ .. cpp:var:: int64 dims (repeated) //@@ //@@ The shape of the input tensor, not including the batch dimension. //@@ repeated int64 dims = 2; //@@ .. cpp:var:: oneof input_data_type //@@ //@@ Specify how the input data is generated. If the input has STRING //@@ data type and 'random_data' is set, the data generation will fall //@@ back to 'zero_data'. //@@ oneof input_data_type { //@@ //@@ .. cpp:var:: bool zero_data //@@ //@@ The identifier for using zeros as input data. Note that the //@@ value of 'zero_data' will not be checked, instead, zero data //@@ will be used as long as the field is set. //@@ bool zero_data = 3; //@@ //@@ .. cpp:var:: bool random_data //@@ //@@ The identifier for using random data as input data. Note that //@@ the value of 'random_data' will not be checked, instead, //@@ random data will be used as long as the field is set. //@@ bool random_data = 4; //@@ .. cpp:var:: string input_data_file //@@ //@@ The file whose content will be used as raw input data in //@@ row-major order. The file must be provided in a sub-directory //@@ 'warmup' under the model directory. The file contents should be //@@ in binary format. For TYPE_STRING data-type, an element is //@@ represented by a 4-byte unsigned integer giving the length //@@ followed by the actual bytes. //@@ string input_data_file = 5; } } //@@ .. cpp:var:: string name //@@ //@@ The name of the request sample. //@@ string name = 1; //@@ .. cpp:var:: uint32 batch_size //@@ //@@ The batch size of the inference request. This must be >= 1. For //@@ models that don't support batching, batch_size must be 1. If //@@ batch_size > 1, the 'inputs' specified below will be duplicated to //@@ match the batch size requested. //@@ uint32 batch_size = 2; //@@ .. cpp:var:: map inputs //@@ //@@ The warmup meta data associated with every model input, including //@@ control tensors. //@@ map inputs = 3; //@@ .. cpp:var:: uint32 count //@@ //@@ The number of iterations that this warmup sample will be executed. //@@ For example, if this field is set to 2, 2 model executions using this //@@ sample will be scheduled for warmup. Default value is 0 which //@@ indicates that this sample will be used only once. //@@ Note that for sequence model, 'count' may not work well //@@ because the model often expect a valid sequence of requests which //@@ should be represented by a series of warmup samples. 'count > 1' //@@ essentially "resends" one of the sample, which may invalidate the //@@ sequence and result in unexpected warmup failure. //@@ uint32 count = 4; } //@@ //@@ .. cpp:var:: message ModelOperations //@@ //@@ The metadata of libraries providing custom operations for this model. //@@ message ModelOperations { //@@ .. cpp:var:: string op_library_filename (repeated) //@@ //@@ Optional paths of the libraries providing custom operations for //@@ this model. Valid only for ONNX models. //@@ repeated string op_library_filename = 1; } //@@ //@@ .. cpp:var:: message ModelTransactionPolicy //@@ //@@ The specification that describes the nature of transactions //@@ to be expected from the model. //@@ message ModelTransactionPolicy { //@@ .. cpp:var:: bool decoupled //@@ //@@ Indicates whether responses generated by the model are decoupled with //@@ the requests issued to it, which means the number of responses //@@ generated by model may differ from number of requests issued, and //@@ that the responses may be out of order relative to the order of //@@ requests. The default is false, which means the model will generate //@@ exactly one response for each request. //@@ bool decoupled = 1; } //@@ //@@.. cpp:var:: message ModelRepositoryAgents //@@ //@@ The repository agents for the model. //@@ message ModelRepositoryAgents { //@@ //@@ .. cpp:var:: message Agent //@@ //@@ A repository agent that should be invoked for the specified //@@ repository actions for this model. //@@ message Agent { //@@ .. cpp:var:: string name //@@ //@@ The name of the agent. //@@ string name = 1; //@@ .. cpp:var:: map parameters //@@ //@@ The parameters for the agent. //@@ map parameters = 2; } //@@ //@@ .. cpp:var:: Agent agents (repeated) //@@ //@@ The ordered list of agents for the model. These agents will be //@@ invoked in order to respond to repository actions occurring for the //@@ model. //@@ repeated Agent agents = 1; } //@@ //@@.. cpp:var:: message ModelResponseCache //@@ //@@ The response cache setting for the model. //@@ message ModelResponseCache { //@@ //@@ .. cpp::var:: bool enable //@@ //@@ Whether or not to use response cache for the model. If True, the //@@ responses from the model are cached and when identical request //@@ is encountered, instead of going through the model execution, //@@ the response from the cache is utilized. By default, response //@@ cache is disabled for the models. //@@ bool enable = 1; } //@@ //@@.. cpp:var:: message ModelConfig //@@ //@@ A model configuration. //@@ message ModelConfig { //@@ .. cpp:var:: string name //@@ //@@ The name of the model. //@@ string name = 1; //@@ .. cpp:var:: string platform //@@ //@@ Additional backend-specific configuration for the model. //@@ Please refer to the backend documentation on whether this field //@@ should be specified. //@@ string platform = 2; //@@ .. cpp:var:: string backend //@@ //@@ The backend used by the model. //@@ string backend = 17; //@@ .. cpp:var:: string runtime //@@ //@@ The name of the backend library file used by the model. //@@ string runtime = 25; //@@ .. cpp:var:: ModelVersionPolicy version_policy //@@ //@@ Policy indicating which version(s) of the model will be served. //@@ ModelVersionPolicy version_policy = 3; //@@ .. cpp:var:: int32 max_batch_size //@@ //@@ Maximum batch size allowed for inference. This can only decrease //@@ what is allowed by the model itself. A max_batch_size value of 0 //@@ indicates that batching is not allowed for the model and the //@@ dimension/shape of the input and output tensors must exactly //@@ match what is specified in the input and output configuration. A //@@ max_batch_size value > 0 indicates that batching is allowed and //@@ so the model expects the input tensors to have an additional //@@ initial dimension for the batching that is not specified in the //@@ input (for example, if the model supports batched inputs of //@@ 2-dimensional tensors then the model configuration will specify //@@ the input shape as [ X, Y ] but the model will expect the actual //@@ input tensors to have shape [ N, X, Y ]). For max_batch_size > 0 //@@ returned outputs will also have an additional initial dimension //@@ for the batch. //@@ int32 max_batch_size = 4; //@@ .. cpp:var:: ModelInput input (repeated) //@@ //@@ The inputs request by the model. //@@ repeated ModelInput input = 5; //@@ .. cpp:var:: ModelOutput output (repeated) //@@ //@@ The outputs produced by the model. //@@ repeated ModelOutput output = 6; //@@ .. cpp:var:: BatchInput batch_input (repeated) //@@ //@@ The model input(s) that the server should use to communicate //@@ batch related values to the model. //@@ repeated BatchInput batch_input = 20; //@@ .. cpp:var:: BatchOutput batch_output (repeated) //@@ //@@ The outputs produced by the model that requires special handling //@@ by the model backend. //@@ repeated BatchOutput batch_output = 21; //@@ .. cpp:var:: ModelOptimizationPolicy optimization //@@ //@@ Optimization configuration for the model. If not specified //@@ then default optimization policy is used. //@@ ModelOptimizationPolicy optimization = 12; //@@ .. cpp:var:: oneof scheduling_choice //@@ //@@ The scheduling policy for the model. If not specified the //@@ default scheduling policy is used for the model. The default //@@ policy is to execute each inference request independently. //@@ oneof scheduling_choice { //@@ .. cpp:var:: ModelDynamicBatching dynamic_batching //@@ //@@ If specified, enables the dynamic-batching scheduling //@@ policy. With dynamic-batching the scheduler may group //@@ together independent requests into a single batch to //@@ improve inference throughput. //@@ ModelDynamicBatching dynamic_batching = 11; //@@ .. cpp:var:: ModelSequenceBatching sequence_batching //@@ //@@ If specified, enables the sequence-batching scheduling //@@ policy. With sequence-batching, inference requests //@@ with the same correlation ID are routed to the same //@@ model instance. Multiple sequences of inference requests //@@ may be batched together into a single batch to //@@ improve inference throughput. //@@ ModelSequenceBatching sequence_batching = 13; //@@ .. cpp:var:: ModelEnsembling ensemble_scheduling //@@ //@@ If specified, enables the model-ensembling scheduling //@@ policy. With model-ensembling, inference requests //@@ will be processed according to the specification, such as an //@@ execution sequence of models. The input specified in this model //@@ config will be the input for the ensemble, and the output //@@ specified will be the output of the ensemble. //@@ ModelEnsembling ensemble_scheduling = 15; } //@@ .. cpp:var:: ModelInstanceGroup instance_group (repeated) //@@ //@@ Instances of this model. If not specified, one instance //@@ of the model will be instantiated on each available GPU. //@@ repeated ModelInstanceGroup instance_group = 7; //@@ .. cpp:var:: string default_model_filename //@@ //@@ Optional filename of the model file to use if a //@@ compute-capability specific model is not specified in //@@ :cpp:var:`cc_model_filenames`. If not specified the default name //@@ is 'model.graphdef', 'model.savedmodel', 'model.plan' or //@@ 'model.pt' depending on the model type. //@@ string default_model_filename = 8; //@@ .. cpp:var:: map cc_model_filenames //@@ //@@ Optional map from CUDA compute capability to the filename of //@@ the model that supports that compute capability. The filename //@@ refers to a file within the model version directory. //@@ map cc_model_filenames = 9; //@@ .. cpp:var:: map metric_tags //@@ //@@ Optional metric tags. User-specific key-value pairs for metrics //@@ reported for this model. These tags are applied to the metrics //@@ reported on the HTTP metrics port. //@@ map metric_tags = 10; //@@ .. cpp:var:: map parameters //@@ //@@ Optional model parameters. User-specified parameter values. //@@ map parameters = 14; //@@ .. cpp:var:: ModelWarmup model_warmup (repeated) //@@ //@@ Warmup setting of this model. If specified, all instances //@@ will be run with the request samples in sequence before //@@ serving the model. //@@ This field can only be specified if the model is not an ensemble //@@ model. //@@ repeated ModelWarmup model_warmup = 16; //@@ .. cpp:var:: ModelOperations model_operations //@@ //@@ Optional metadata of the libraries providing custom operations for //@@ this model. //@@ ModelOperations model_operations = 18; //@@ .. cpp:var:: ModelTransactionPolicy model_transaction_policy //@@ //@@ Optional specification that describes the nature of transactions //@@ to be expected from the model. //@@ ModelTransactionPolicy model_transaction_policy = 19; //@@ .. cpp:var:: ModelRepositoryAgents model_repository_agents //@@ //@@ Optional specification of the agent(s) that should be invoked //@@ with repository actions are performed for this model. //@@ ModelRepositoryAgents model_repository_agents = 23; //@@ .. cpp:var:: ModelResponseCache response_cache //@@ //@@ Optional setting for utilizing the response cache for this //@@ model. //@@ ModelResponseCache response_cache = 24; } kytensor-2.49.0.7/common/pyproject.toml000066400000000000000000000043021527052371200200400ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [tool.codespell] # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - # this is only to allow you to run codespell interactively skip = "./.git,./.github" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" # use the 'clear' dictionary for unambiguous spelling mistakes builtin = "clear" # disable warnings about binary files and wrong encoding quiet-level = 3 [tool.isort] profile = "black" use_parentheses = true multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 ensure_newline_before_comments = true line_length = 88 balanced_wrapping = true indent = " " skip = ["build"] kytensor-2.49.0.7/common/src/000077500000000000000000000000001527052371200157145ustar00rootroot00000000000000kytensor-2.49.0.7/common/src/CMakeLists.txt000066400000000000000000000114161527052371200204570ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Error # add_library( triton-common-error error.cc ) add_library( TritonCommon::triton-common-error ALIAS triton-common-error ) target_include_directories( triton-common-error PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) target_link_libraries(triton-common-error PRIVATE common-compile-settings) # # Logging # add_library( triton-common-logging logging.cc ) add_library( TritonCommon::triton-common-logging ALIAS triton-common-logging ) target_include_directories( triton-common-logging PUBLIC $ $ PRIVATE ${RAPIDJSON_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR} ) if(${TRITON_ENABLE_LOGGING}) target_compile_definitions( triton-common-logging PRIVATE TRITON_ENABLE_LOGGING=1 ) endif() # TRITON_ENABLE_LOGGING target_link_libraries(triton-common-logging PRIVATE common-compile-settings) # # Async Work Queue # add_library( triton-common-async-work-queue async_work_queue.cc error.cc thread_pool.cc ) add_library( TritonCommon::triton-common-async-work-queue ALIAS triton-common-async-work-queue ) target_include_directories( triton-common-async-work-queue PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) target_link_libraries(triton-common-async-work-queue PUBLIC Threads::Threads PRIVATE common-compile-settings ) # # Thread Pool # add_library( triton-common-thread-pool thread_pool.cc ) add_library( TritonCommon::triton-common-thread-pool ALIAS triton-common-thread-pool ) target_include_directories( triton-common-thread-pool PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) target_link_libraries(triton-common-thread-pool PUBLIC Threads::Threads PRIVATE common-compile-settings ) # # Table Printer # add_library( triton-common-table-printer table_printer.cc ) add_library( TritonBackend::triton-common-table-printer ALIAS triton-common-table-printer ) target_include_directories( triton-common-table-printer PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_link_libraries(triton-common-table-printer PRIVATE common-compile-settings) set_target_properties( triton-common-async-work-queue triton-common-error triton-common-logging triton-common-table-printer triton-common-thread-pool PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE POSITION_INDEPENDENT_CODE ON ) set_target_properties( triton-common-async-work-queue PROPERTIES OUTPUT_NAME tritonasyncworkqueue ) set_target_properties( triton-common-thread-pool PROPERTIES OUTPUT_NAME tritonthreadpool ) set_target_properties( triton-common-error PROPERTIES OUTPUT_NAME tritoncommonerror ) set_target_properties( triton-common-logging PROPERTIES OUTPUT_NAME tritoncommonlogging ) set_target_properties( triton-common-table-printer PROPERTIES OUTPUT_NAME tritontableprinter ) #add_subdirectory(test) kytensor-2.49.0.7/common/src/async_work_queue.cc000066400000000000000000000060761527052371200216170ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/common/async_work_queue.h" namespace triton { namespace common { AsyncWorkQueue::~AsyncWorkQueue() { GetSingleton()->thread_pool_.reset(); } AsyncWorkQueue* AsyncWorkQueue::GetSingleton() { static AsyncWorkQueue singleton; return &singleton; } Error AsyncWorkQueue::Initialize(size_t worker_count) { if (worker_count < 1) { return Error( Error::Code::INVALID_ARG, "Async work queue must be initialized with positive 'worker_count'"); } static std::mutex init_mtx; std::lock_guard lk(init_mtx); if (GetSingleton()->thread_pool_) { return Error( Error::Code::ALREADY_EXISTS, "Async work queue has been initialized with " + std::to_string(GetSingleton()->thread_pool_->Size()) + " 'worker_count'"); } GetSingleton()->thread_pool_.reset(new ThreadPool(worker_count)); return Error::Success; } size_t AsyncWorkQueue::WorkerCount() { if (!GetSingleton()->thread_pool_) { return 0; } return GetSingleton()->thread_pool_->Size(); } Error AsyncWorkQueue::AddTask(std::function&& task) { if (!GetSingleton()->thread_pool_) { return Error( Error::Code::UNAVAILABLE, "Async work queue must be initialized before adding task"); } GetSingleton()->thread_pool_->Enqueue(std::move(task)); return Error::Success; } void AsyncWorkQueue::Reset() { // Reconstruct the singleton to reset it GetSingleton()->~AsyncWorkQueue(); new (GetSingleton()) AsyncWorkQueue(); } }} // namespace triton::common kytensor-2.49.0.7/common/src/error.cc000066400000000000000000000046121527052371200173570ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/common/error.h" namespace triton { namespace common { const Error Error::Success(Error::Code::SUCCESS); std::string Error::AsString() const { std::string str(CodeString(code_)); str += ": " + msg_; return str; } const char* Error::CodeString(const Code code) { switch (code) { case Error::Code::SUCCESS: return "OK"; case Error::Code::UNKNOWN: return "Unknown"; case Error::Code::INTERNAL: return "Internal"; case Error::Code::NOT_FOUND: return "Not found"; case Error::Code::INVALID_ARG: return "Invalid argument"; case Error::Code::UNAVAILABLE: return "Unavailable"; case Error::Code::UNSUPPORTED: return "Unsupported"; case Error::Code::ALREADY_EXISTS: return "Already exists"; default: break; } return ""; } }} // namespace triton::common kytensor-2.49.0.7/common/src/logging.cc000066400000000000000000000125061527052371200176550ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include // Defined but not used #define TRITONJSON_STATUSTYPE uint8_t #define TRITONJSON_STATUSRETURN(M) #define TRITONJSON_STATUSSUCCESS 0 #include "triton/common/logging.h" #include "triton/common/triton_json.h" namespace triton { namespace common { Logger gLogger_; Logger::Logger() : enables_{true, true, true}, vlevel_(0), format_(Format::kDEFAULT) { const char* value = std::getenv(Logger::ESCAPE_ENVIRONMENT_VARIABLE); escape_log_messages_ = (value && std::strcmp(value, "0") == 0) ? false : true; } void Logger::Log(const std::string& msg) { const std::lock_guard lock(mutex_); if (file_stream_.is_open()) { file_stream_ << msg << std::endl; } else { std::cerr << msg << std::endl; } } void Logger::Flush() { std::cerr << std::flush; } #ifdef _WIN32 void LogMessage::LogTimestamp(std::stringstream& stream) { switch (gLogger_.LogFormat()) { case Logger::Format::kDEFAULT: { stream << std::setfill('0') << std::setw(2) << timestamp_.wMonth << std::setw(2) << timestamp_.wDay << ' ' << std::setw(2) << timestamp_.wHour << ':' << std::setw(2) << timestamp_.wMinute << ':' << std::setw(2) << timestamp_.wSecond << '.' << std::setw(6) << timestamp_.wMilliseconds * 1000; break; } case Logger::Format::kISO8601: { stream << timestamp_.wYear << '-' << std::setfill('0') << std::setw(2) << timestamp_.wMonth << '-' << std::setw(2) << timestamp_.wDay << 'T' << std::setw(2) << timestamp_.wHour << ':' << std::setw(2) << timestamp_.wMinute << ':' << std::setw(2) << timestamp_.wSecond << "Z"; break; } } } #else void LogMessage::LogTimestamp(std::stringstream& stream) { struct tm tm_time; gmtime_r(((time_t*)&(timestamp_.tv_sec)), &tm_time); switch (gLogger_.LogFormat()) { case Logger::Format::kDEFAULT: { stream << std::setfill('0') << std::setw(2) << (tm_time.tm_mon + 1) << std::setw(2) << tm_time.tm_mday << ' ' << std::setw(2) << tm_time.tm_hour << ':' << std::setw(2) << tm_time.tm_min << ':' << std::setw(2) << tm_time.tm_sec << '.' << std::setw(6) << timestamp_.tv_usec; break; } case Logger::Format::kISO8601: { stream << (tm_time.tm_year + 1900) << '-' << std::setfill('0') << std::setw(2) << (tm_time.tm_mon + 1) << '-' << std::setw(2) << tm_time.tm_mday << 'T' << std::setw(2) << tm_time.tm_hour << ':' << std::setw(2) << tm_time.tm_min << ':' << std::setw(2) << tm_time.tm_sec << "Z"; break; } } } #endif void LogMessage::LogPreamble(std::stringstream& stream) { switch (gLogger_.LogFormat()) { case Logger::Format::kDEFAULT: { stream << Logger::LEVEL_NAMES[static_cast(level_)]; LogTimestamp(stream); stream << ' ' << pid_ << ' ' << path_ << ':' << line_ << "] "; break; } case Logger::Format::kISO8601: { LogTimestamp(stream); stream << " " << Logger::LEVEL_NAMES[static_cast(level_)] << ' ' << pid_ << ' ' << path_ << ':' << line_ << "] "; break; } } } LogMessage::~LogMessage() { std::stringstream log_record; LogPreamble(log_record); std::string escaped_message = escape_log_messages_ ? TritonJson::SerializeString(message_.str()) : message_.str(); if (heading_ != nullptr) { std::string escaped_heading = gLogger_.EscapeLogMessages() ? TritonJson::SerializeString(heading_) : heading_; log_record << escaped_heading << '\n'; } log_record << escaped_message; gLogger_.Log(log_record.str()); } }} // namespace triton::common kytensor-2.49.0.7/common/src/model_config.cc000066400000000000000000000243731527052371200206610ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/common/model_config.h" namespace triton { namespace common { bool IsFixedSizeDataType(const inference::DataType dtype) { return dtype != inference::DataType::TYPE_STRING; } size_t GetDataTypeByteSize(const inference::DataType dtype) { switch (dtype) { case inference::DataType::TYPE_BOOL: return 1; case inference::DataType::TYPE_UINT8: return 1; case inference::DataType::TYPE_UINT16: return 2; case inference::DataType::TYPE_UINT32: return 4; case inference::DataType::TYPE_UINT64: return 8; case inference::DataType::TYPE_INT8: return 1; case inference::DataType::TYPE_INT16: return 2; case inference::DataType::TYPE_INT32: return 4; case inference::DataType::TYPE_INT64: return 8; case inference::DataType::TYPE_FP16: return 2; case inference::DataType::TYPE_FP32: return 4; case inference::DataType::TYPE_FP64: return 8; case inference::DataType::TYPE_STRING: return 0; case inference::DataType::TYPE_BF16: return 2; default: break; } return 0; } int64_t GetElementCount(const DimsList& dims) { bool first = true; int64_t cnt = 0; for (auto dim : dims) { if (dim == WILDCARD_DIM) { return -1; } if (first) { cnt = dim; first = false; } else { cnt *= dim; } } return cnt; } int64_t GetElementCount(const std::vector& dims) { bool first = true; int64_t cnt = 0; for (auto dim : dims) { if (dim == WILDCARD_DIM) { return -1; } if (first) { cnt = dim; first = false; } else { cnt *= dim; } } return cnt; } int64_t GetElementCount(const inference::ModelInput& mio) { return GetElementCount(mio.dims()); } int64_t GetElementCount(const inference::ModelOutput& mio) { return GetElementCount(mio.dims()); } int64_t GetByteSize(const inference::DataType& dtype, const DimsList& dims) { size_t dt_size = GetDataTypeByteSize(dtype); if (dt_size == 0) { return -1; } int64_t cnt = GetElementCount(dims); if (cnt == -1) { return -1; } return cnt * dt_size; } int64_t GetByteSize(const inference::DataType& dtype, const std::vector& dims) { size_t dt_size = GetDataTypeByteSize(dtype); if (dt_size == 0) { return -1; } int64_t cnt = GetElementCount(dims); if (cnt == -1) { return -1; } return cnt * dt_size; } int64_t GetByteSize( const int batch_size, const inference::DataType& dtype, const DimsList& dims) { if (dims.size() == 0) { return batch_size * GetDataTypeByteSize(dtype); } int64_t bs = GetByteSize(dtype, dims); if (bs == -1) { return -1; } return std::max(1, batch_size) * bs; } int64_t GetByteSize( const int batch_size, const inference::DataType& dtype, const std::vector& dims) { if (dims.size() == 0) { return batch_size * GetDataTypeByteSize(dtype); } int64_t bs = GetByteSize(dtype, dims); if (bs == -1) { return -1; } return std::max(1, batch_size) * bs; } int64_t GetByteSize(const inference::ModelInput& mio) { return GetByteSize(mio.data_type(), mio.dims()); } int64_t GetByteSize(const inference::ModelOutput& mio) { return GetByteSize(mio.data_type(), mio.dims()); } int GetCpuNiceLevel(const inference::ModelConfig& config) { int nice = SCHEDULER_DEFAULT_NICE; if (config.has_optimization()) { switch (config.optimization().priority()) { case inference::ModelOptimizationPolicy::PRIORITY_MAX: nice = 0; break; case inference::ModelOptimizationPolicy::PRIORITY_MIN: nice = 19; break; default: nice = SCHEDULER_DEFAULT_NICE; break; } } return nice; } bool CompareDims(const DimsList& dims0, const DimsList& dims1) { if (dims0.size() != dims1.size()) { return false; } for (int i = 0; i < dims0.size(); ++i) { if (dims0[i] != dims1[i]) { return false; } } return true; } bool CompareDims( const std::vector& dims0, const std::vector& dims1) { if (dims0.size() != dims1.size()) { return false; } for (size_t i = 0; i < dims0.size(); ++i) { if (dims0[i] != dims1[i]) { return false; } } return true; } bool CompareDimsWithWildcard(const DimsList& dims0, const DimsList& dims1) { if (dims0.size() != dims1.size()) { return false; } for (int i = 0; i < dims0.size(); ++i) { if ((dims0[i] != WILDCARD_DIM) && (dims1[i] != WILDCARD_DIM) && (dims0[i] != dims1[i])) { return false; } } return true; } bool CompareDimsWithWildcard( const DimsList& dims0, const std::vector& dims1) { if (dims0.size() != (int64_t)dims1.size()) { return false; } for (int i = 0; i < dims0.size(); ++i) { if ((dims0[i] != WILDCARD_DIM) && (dims1[i] != WILDCARD_DIM) && (dims0[i] != dims1[i])) { return false; } } return true; } std::string DimsListToString(const DimsList& dims) { bool first = true; std::string str("["); for (const auto& dim : dims) { if (!first) { str += ","; } str += std::to_string(dim); first = false; } str += "]"; return str; } std::string DimsListToString(const std::vector& dims, const int start_idx) { int idx = 0; std::string str("["); for (const auto& dim : dims) { if (idx >= start_idx) { if (idx > start_idx) { str += ","; } str += std::to_string(dim); } idx++; } str += "]"; return str; } const char* DataTypeToProtocolString(const inference::DataType dtype) { switch (dtype) { case inference::DataType::TYPE_BOOL: return "BOOL"; case inference::DataType::TYPE_UINT8: return "UINT8"; case inference::DataType::TYPE_UINT16: return "UINT16"; case inference::DataType::TYPE_UINT32: return "UINT32"; case inference::DataType::TYPE_UINT64: return "UINT64"; case inference::DataType::TYPE_INT8: return "INT8"; case inference::DataType::TYPE_INT16: return "INT16"; case inference::DataType::TYPE_INT32: return "INT32"; case inference::DataType::TYPE_INT64: return "INT64"; case inference::DataType::TYPE_FP16: return "FP16"; case inference::DataType::TYPE_FP32: return "FP32"; case inference::DataType::TYPE_FP64: return "FP64"; case inference::DataType::TYPE_STRING: return "BYTES"; case inference::DataType::TYPE_BF16: return "BF16"; default: break; } return ""; } inference::DataType ProtocolStringToDataType(const std::string& dtype) { return ProtocolStringToDataType(dtype.c_str(), dtype.size()); } inference::DataType ProtocolStringToDataType(const char* dtype, size_t len) { if (len < 4 || len > 6) { return inference::DataType::TYPE_INVALID; } if ((*dtype == 'I') && (len != 6)) { if ((dtype[1] == 'N') && (dtype[2] == 'T')) { if ((dtype[3] == '8') && (len == 4)) { return inference::DataType::TYPE_INT8; } else if ((dtype[3] == '1') && (dtype[4] == '6')) { return inference::DataType::TYPE_INT16; } else if ((dtype[3] == '3') && (dtype[4] == '2')) { return inference::DataType::TYPE_INT32; } else if ((dtype[3] == '6') && (dtype[4] == '4')) { return inference::DataType::TYPE_INT64; } } } else if ((*dtype == 'U') && (len != 4)) { if ((dtype[1] == 'I') && (dtype[2] == 'N') && (dtype[3] == 'T')) { if ((dtype[4] == '8') && (len == 5)) { return inference::DataType::TYPE_UINT8; } else if ((dtype[4] == '1') && (dtype[5] == '6')) { return inference::DataType::TYPE_UINT16; } else if ((dtype[4] == '3') && (dtype[5] == '2')) { return inference::DataType::TYPE_UINT32; } else if ((dtype[4] == '6') && (dtype[5] == '4')) { return inference::DataType::TYPE_UINT64; } } } else if ((*dtype == 'F') && (dtype[1] == 'P') && (len == 4)) { if ((dtype[2] == '1') && (dtype[3] == '6')) { return inference::DataType::TYPE_FP16; } else if ((dtype[2] == '3') && (dtype[3] == '2')) { return inference::DataType::TYPE_FP32; } else if ((dtype[2] == '6') && (dtype[3] == '4')) { return inference::DataType::TYPE_FP64; } } else if (*dtype == 'B') { switch (dtype[1]) { case 'Y': if (!strcmp(dtype + 2, "TES")) { return inference::DataType::TYPE_STRING; } break; case 'O': if (!strcmp(dtype + 2, "OL")) { return inference::DataType::TYPE_BOOL; } break; case 'F': if (!strcmp(dtype + 2, "16")) { return inference::DataType::TYPE_BF16; } break; } } return inference::DataType::TYPE_INVALID; } }} // namespace triton::common kytensor-2.49.0.7/common/src/table_printer.cc000066400000000000000000000173441527052371200210660ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/common/table_printer.h" #ifdef _WIN32 // suppress the min and max definitions in Windef.h. #define NOMINMAX #include #else #include #include #endif #include #include #include #include #include #include #include #include namespace triton { namespace common { // // ASCII table printer. // void TablePrinter::InsertRow(const std::vector& row) { std::vector> table_row; // Number of lines in each field in the record size_t max_height = 0; // Update max length of data items in each row for (size_t i = 0; i < row.size(); ++i) { table_row.push_back(std::vector{}); std::stringstream ss(row[i]); std::string line; size_t max_width = 0; while (std::getline(ss, line, '\n')) { table_row[i].push_back(line); if (line.size() > max_width) max_width = line.size(); } if (max_width > max_widths_[i]) max_widths_[i] = max_width; size_t number_of_lines = table_row[i].size(); if (max_height < number_of_lines) max_height = number_of_lines; } max_heights_.push_back(max_height); data_.emplace_back(table_row); } void TablePrinter::FairShare() { // initialize original index locations size_t array_size = max_widths_.size(); std::vector idx(array_size); iota(idx.begin(), idx.end(), 0); stable_sort(idx.begin(), idx.end(), [this](size_t i1, size_t i2) { return this->max_widths_[i1] < this->max_widths_[i2]; }); size_t loop_index = 1; for (auto itr = idx.begin(); itr != idx.end(); ++itr) { // If a column is not using all the space allocated to it if (max_widths_[*itr] < shares_[*itr]) { float excess = shares_[*itr] - max_widths_[*itr]; shares_[*itr] -= excess; if (itr == idx.end() - 1) break; auto update_itr = idx.begin() + (itr - idx.begin() + 1); // excess amount of unused space that must be distributed evenly to the // next columns float excess_per_column = excess / (array_size - loop_index); for (; update_itr != idx.end(); ++update_itr) { shares_[*update_itr] += excess_per_column; excess -= excess_per_column; } } ++loop_index; } // Remove any decimal shares for (auto itr = idx.begin(); itr != idx.end(); ++itr) { shares_[*itr] = (size_t)shares_[*itr]; } // For each record for (size_t i = 0; i < data_.size(); i++) { auto current_row = data_[i]; // For each field in the record for (size_t j = 0; j < current_row.size(); j++) { // For each line in the record for (size_t line_index = 0; line_index < current_row[j].size(); line_index++) { std::string line = current_row[j][line_index]; size_t num_rows = (line.size() + shares_[j] - 1) / shares_[j]; // If the number of rows required for this record is larger than 1, we // will break that line and put it in multiple lines if (num_rows > 1) { // Remove the multi-line field, it will be replaced by the line // that can fits the column size data_[i][j].erase(data_[i][j].begin() + line_index); for (size_t k = 0; k < num_rows; k++) { size_t start_index = std::min((size_t)(k * shares_[j]), line.size()); size_t end_index = std::min((size_t)((k + 1) * shares_[j]), line.size()); data_[i][j].insert( data_[i][j].begin() + line_index + k, line.substr(start_index, end_index - start_index)); } // We need to advance the index for the split lines. line_index += num_rows - 1; } if (max_heights_[i] < (num_rows - 1 + current_row[j].size())) max_heights_[i] += num_rows - 1; } } } } void TablePrinter::AddRow(std::stringstream& table, size_t row_index) { auto row = data_[row_index]; size_t max_height = max_heights_[row_index]; for (size_t j = 0; j < max_height; j++) { table << "|" << std::left; for (size_t i = 0; i < row.size(); i++) { if (j < row[i].size()) table << " " << std::setw(shares_[i]) << row[i][j] << " |"; else table << " " << std::setw(shares_[i]) << " " << " |"; } // Do not add new line if this is the last row of this record if (j != max_height - 1) table << "\n"; } table << "\n"; } void TablePrinter::AddRowDivider(std::stringstream& table) { table << "+"; for (const auto& share : shares_) { for (size_t i = 0; i < share + 2; i++) table << "-"; table << "+"; } table << "\n"; } std::string TablePrinter::PrintTable() { std::stringstream table; table << "\n"; FairShare(); AddRowDivider(table); // Add table headers AddRow(table, 0); AddRowDivider(table); for (size_t j = 1; j < data_.size(); j++) { AddRow(table, j); } AddRowDivider(table); return table.str(); } // TablePrinter will take the ownership of `headers`. TablePrinter::TablePrinter(const std::vector& headers) { // terminal size size_t column_size = 500; #ifdef _WIN32 CONSOLE_SCREEN_BUFFER_INFO csbi; int ret = GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi); if (ret && (csbi.dwSize.X != 0)) { column_size = csbi.dwSize.X; } #else struct winsize terminal_size; int status = ioctl(STDOUT_FILENO, TIOCGWINSZ, &terminal_size); if ((status == 0) && (terminal_size.ws_col != 0)) { column_size = terminal_size.ws_col; } #endif for (size_t i = 0; i < headers.size(); ++i) { max_widths_.emplace_back(0); } // Calculate fair share of every column size_t number_of_columns = headers.size(); // Terminal width is the actual terminal width minus two times spaces // required before and after each column and number of columns plus 1 for // the pipes between the columns size_t terminal_width = column_size - (2 * number_of_columns) - (number_of_columns + 1); int equal_share = terminal_width / headers.size(); for (size_t i = 0; i < headers.size(); ++i) { shares_.emplace_back(equal_share); terminal_width -= equal_share; } InsertRow(headers); } }} // namespace triton::common kytensor-2.49.0.7/common/src/test/000077500000000000000000000000001527052371200166735ustar00rootroot00000000000000kytensor-2.49.0.7/common/src/test/CMakeLists.txt000066400000000000000000000035311527052371200214350ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) # # Unit tests # include(FetchContent) FetchContent_Declare( googletest URL https://github.com/google/googletest/archive/9406a60c7839052e4944ea4dbc8344762a89f9bd.zip ) FetchContent_MakeAvailable(googletest) if (TRITON_COMMON_ENABLE_JSON) add_subdirectory(triton_json triton_json) endif() kytensor-2.49.0.7/common/src/test/triton_json/000077500000000000000000000000001527052371200212435ustar00rootroot00000000000000kytensor-2.49.0.7/common/src/test/triton_json/CMakeLists.txt000066400000000000000000000041331527052371200240040ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) add_executable(triton-json-test triton_json_test.cc ../../error.cc) target_link_libraries( triton-json-test GTest::gtest GTest::gtest_main) target_include_directories( triton-json-test PUBLIC $ $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) set_target_properties( triton-json-test PROPERTIES OUTPUT_NAME triton_json_test ) install( TARGETS triton-json-test RUNTIME DESTINATION bin ) kytensor-2.49.0.7/common/src/test/triton_json/triton_json_test.cc000066400000000000000000000103451527052371200251640ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/common/error.h" #define TRITONJSON_STATUSTYPE Error #define TRITONJSON_STATUSRETURN(M) \ return Error(Error::Code::INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS Error() #include "gtest/gtest.h" #include "triton/common/triton_json.h" namespace { TEST(JsonTypeCheck, TestIsBool) { triton::common::TritonJson::Value value; value.Parse("{\"x\": true}"); triton::common::TritonJson::Value x; ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsBool()); value.Parse("{\"x\": false}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsBool()); value.Parse("{\"x\": \"x\"}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_FALSE(x.IsBool()); } TEST(JsonTypeCheck, TestIsObject) { triton::common::TritonJson::Value value; value.Parse("{\"x\": {}}"); triton::common::TritonJson::Value x; ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsObject()); value.Parse("{\"x\": 2}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_FALSE(x.IsObject()); } TEST(JsonTypeCheck, TestIsString) { triton::common::TritonJson::Value value; value.Parse("{\"x\": \"123\"}"); triton::common::TritonJson::Value x; ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsString()); value.Parse("{\"x\": 2}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_FALSE(x.IsString()); } TEST(JsonTypeCheck, TestIsArray) { triton::common::TritonJson::Value value; value.Parse("{\"x\": []}"); triton::common::TritonJson::Value x; ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsArray()); value.Parse("{\"x\": 2}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_FALSE(x.IsArray()); } TEST(JsonTypeCheck, TestIsNumber) { triton::common::TritonJson::Value value; value.Parse("{\"x\": 2.0}"); triton::common::TritonJson::Value x; ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsNumber()); value.Parse("{\"x\": 2.001}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsNumber()); value.Parse("{\"x\": 2}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsNumber()); value.Parse("{\"x\": \"a\"}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_FALSE(x.IsNumber()); } TEST(JsonTypeCheck, TestIsInt) { triton::common::TritonJson::Value value; value.Parse("{\"x\": 2.0}"); triton::common::TritonJson::Value x; ASSERT_TRUE(value.Find("x", &x)); ASSERT_FALSE(x.IsInt()); value.Parse("{\"x\": 2}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsInt()); value.Parse("{\"x\": -2}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_TRUE(x.IsInt()); value.Parse("{\"x\": \"a\"}"); ASSERT_TRUE(value.Find("x", &x)); ASSERT_FALSE(x.IsInt()); } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/common/src/thread_pool.cc000066400000000000000000000062511527052371200205270ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/common/thread_pool.h" #include namespace triton { namespace common { ThreadPool::ThreadPool(size_t thread_count) { if (!thread_count) { throw std::invalid_argument("Thread count must be greater than zero."); } // Define infinite loop for each thread to wait for a task to complete const auto worker_loop = [this]() { while (true) { Task task; { std::unique_lock lk(queue_mtx_); // Wake if there's a task to do, or the pool has been stopped. cv_.wait(lk, [&]() { return !task_queue_.empty() || stop_; }); // Exit condition if (stop_ && task_queue_.empty()) { break; } task = std::move(task_queue_.front()); task_queue_.pop(); } // Execute task - ensure function has a valid target if (task) { task(); } } }; workers_.reserve(thread_count); for (size_t i = 0; i < thread_count; ++i) { workers_.emplace_back(worker_loop); } } ThreadPool::~ThreadPool() { { std::lock_guard lk(queue_mtx_); // Signal to each worker that it should exit loop when tasks are finished stop_ = true; } // Wake all threads to clean up cv_.notify_all(); for (auto& t : workers_) { t.join(); } } void ThreadPool::Enqueue(Task&& task) { { std::lock_guard lk(queue_mtx_); // Don't accept more work if pool is shutting down if (stop_) { return; } task_queue_.push(std::move(task)); } // Only wake one thread per task // Todo: DLIS-3859 if ThreadPool gets used more. cv_.notify_one(); } }} // namespace triton::common kytensor-2.49.0.7/core/000077500000000000000000000000001527052371200145655ustar00rootroot00000000000000kytensor-2.49.0.7/core/.clang-format000066400000000000000000000014271527052371200171440ustar00rootroot00000000000000--- BasedOnStyle: Google IndentWidth: 2 ColumnLimit: 80 ContinuationIndentWidth: 4 UseTab: Never MaxEmptyLinesToKeep: 2 SortIncludes: true CompactNamespaces: true ReflowComments: true DerivePointerAlignment: false PointerAlignment: Left AllowShortIfStatementsOnASingleLine: false AllowShortBlocksOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AlwaysBreakAfterReturnType: TopLevelDefinitions AlignAfterOpenBracket: AlwaysBreak BreakBeforeBraces: Custom BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: true AfterNamespace: false AfterStruct: false AfterUnion: false BeforeCatch: true BinPackArguments: true BinPackParameters: true ConstructorInitializerAllOnOneLineOrOnePerLine: false IndentCaseLabels: true kytensor-2.49.0.7/core/.github/000077500000000000000000000000001527052371200161255ustar00rootroot00000000000000kytensor-2.49.0.7/core/.github/workflows/000077500000000000000000000000001527052371200201625ustar00rootroot00000000000000kytensor-2.49.0.7/core/.github/workflows/pre-commit.yml000066400000000000000000000033201527052371200227570ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: pre-commit on: pull_request: jobs: pre-commit: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - uses: pre-commit/action@v3.0.0 kytensor-2.49.0.7/core/.pre-commit-config.yaml000066400000000000000000000056111527052371200210510ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black rev: 23.1.0 hooks: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] types_or: [python, cython] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.5 hooks: - id: clang-format types_or: [c, c++, cuda, proto, textproto, java] args: ["-fallback-style=none", "-style=file", "-i"] - repo: https://github.com/codespell-project/codespell rev: v2.2.4 hooks: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$) # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-json - id: check-toml - id: check-yaml - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace kytensor-2.49.0.7/core/CMakeLists.txt000066400000000000000000000373011527052371200173310ustar00rootroot00000000000000# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritoncore LANGUAGES C CXX) # Control building of shared library vs. only headers and stub. By # default only the headers and library stub is built. Set # TRITON_CORE_HEADERS_ONLY=OFF to also build libtritonserver.so. option(TRITON_CORE_HEADERS_ONLY "Build only headers and stub" ON) # # Specifying min required C++ standard # set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.") # # Triton Server API # add_library( triton-core-serverapi INTERFACE ) add_library( TritonCore::triton-core-serverapi ALIAS triton-core-serverapi ) target_include_directories( triton-core-serverapi INTERFACE $ $ ) # # Triton Backend API # add_library( triton-core-backendapi INTERFACE ) add_library( TritonCore::triton-core-backendapi ALIAS triton-core-backendapi ) target_include_directories( triton-core-backendapi INTERFACE $ $ ) # # Triton RepoAgent API # add_library( triton-core-repoagentapi INTERFACE ) add_library( TritonCore::triton-core-repoagentapi ALIAS triton-core-repoagentapi ) target_include_directories( triton-core-repoagentapi INTERFACE $ $ ) # # Triton Cache API # add_library( triton-core-cacheapi INTERFACE ) add_library( TritonCore::triton-core-cacheapi ALIAS triton-core-cacheapi ) target_include_directories( triton-core-cacheapi INTERFACE $ $ ) # # Stub library for libtritonserver.so that stubs Triton Server API and # Triton Backend API # add_library( triton-core-serverstub SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/tritonserver_stub.cc ) add_library( TritonCore::triton-core-serverstub ALIAS triton-core-serverstub ) target_compile_features(triton-core-serverstub PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message("Using MSVC as compiler, default target on Windows 10. " "If the target system is not Windows 10, please update _WIN32_WINNT " "to corresponding value.") target_compile_options( triton-core-serverstub PRIVATE /Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor ) else() target_compile_options( triton-core-serverstub PRIVATE -Wall -Wextra -Wno-unused-parameter -Werror ) endif() set_target_properties( triton-core-serverstub PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME tritonserver ) # # Shared library implementing Triton Server API # if(NOT TRITON_CORE_HEADERS_ONLY) include(CMakeDependentOption) set(TRITON_VERSION "0.0.0" CACHE STRING "The version of the Triton shared library" ) option(TRITON_ENABLE_LOGGING "Include logging support in server" ON) option(TRITON_ENABLE_STATS "Include statistics collections in server" ON) option(TRITON_ENABLE_TRACING "Include tracing support in server" OFF) option(TRITON_ENABLE_NVTX "Include NVTX support in server" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in server" ON) option(TRITON_ENABLE_MALI_GPU "Enable Arm Mali GPU support in server" OFF) set(TRITON_MIN_COMPUTE_CAPABILITY "6.0" CACHE STRING "The minimum CUDA compute capability supported by Triton" ) set(TRITON_EXTRA_LIB_PATHS "" CACHE PATH "Extra library paths for Triton Server build") # Ensemble option(TRITON_ENABLE_ENSEMBLE "Include ensemble support in server" OFF) # Metrics option(TRITON_ENABLE_METRICS "Include metrics support in server" ON) option(TRITON_ENABLE_METRICS_GPU "Include GPU metrics support in server" ON) option(TRITON_ENABLE_METRICS_CPU "Include CPU metrics support in server" ON) # Cloud storage option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF) option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF) option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF) # Repo tags set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_THIRD_PARTY_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/third_party repo") # Triton module source addr set(TRITON_MODULE_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/.." CACHE STRING "Addr for triton source") # Third-party location set(TRITON_THIRD_PARTY_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/third-party" CACHE STRING "Location of third-party build") set(TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/third-party-src" CACHE STRING "Location of third-party source") if(TRITON_ENABLE_METRICS AND NOT TRITON_ENABLE_STATS) message(FATAL_ERROR "TRITON_ENABLE_METRICS=ON requires TRITON_ENABLE_STATS=ON") endif() if(TRITON_ENABLE_TRACING AND NOT TRITON_ENABLE_STATS) message(FATAL_ERROR "TRITON_ENABLE_TRACING=ON requires TRITON_ENABLE_STATS=ON") endif() if (TRITON_ENABLE_METRICS_CPU AND NOT TRITON_ENABLE_METRICS) message(FATAL_ERROR "TRITON_ENABLE_METRICS_CPU=ON requires TRITON_ENABLE_METRICS=ON") endif() if (TRITON_ENABLE_METRICS_GPU AND NOT TRITON_ENABLE_METRICS) message(FATAL_ERROR "TRITON_ENABLE_METRICS_GPU=ON requires TRITON_ENABLE_METRICS=ON") endif() if (TRITON_ENABLE_METRICS_GPU AND NOT TRITON_ENABLE_GPU) message(FATAL_ERROR "TRITON_ENABLE_METRICS_GPU=ON requires TRITON_ENABLE_GPU=ON") endif() # include(FetchContent) # FetchContent_Declare( # repo-third-party # SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/third_party # ) # FetchContent_MakeAvailable(repo-third-party) # Some libs are installed to ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib64 instead # of ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib on Centos set (LIB_DIR "lib") if(LINUX) file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE") if(${DISTRO_ID_LIKE} MATCHES "rhel|centos") set (LIB_DIR "lib64") endif(${DISTRO_ID_LIKE} MATCHES "rhel|centos") endif(LINUX) set(TRITON_CORE_HEADERS_ONLY OFF) # Need to use ExternalProject for our builds so that we can get the # correct dependencies between Triton shared library components and # the ExternalProject dependencies (found in the third_party repo) include(ExternalProject) # If CMAKE_TOOLCHAIN_FILE is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "") if (CMAKE_TOOLCHAIN_FILE) set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "-DCMAKE_TOOLCHAIN_FILE:PATH=${CMAKE_TOOLCHAIN_FILE}") endif() # If VCPKG_TARGET_TRIPLET is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "") if (VCPKG_TARGET_TRIPLET) set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "-DVCPKG_TARGET_TRIPLET:STRING=${VCPKG_TARGET_TRIPLET}") endif() # If OPENSSL_ROOT_DIR is set, propagate that hint path to the external # projects with OpenSSL dependency. set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "") if (OPENSSL_ROOT_DIR) set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") endif() # Location where protobuf-config.cmake will be installed varies by # platform if (WIN32) set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/cmake") else() set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf") endif() if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(TRITON_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/install) else() set(TRITON_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) endif() set(TRITON_DEPENDS googletest protobuf re2 Boost) if(${TRITON_ENABLE_GCS}) set(TRITON_DEPENDS ${TRITON_DEPENDS} google-cloud-cpp) endif() # TRITON_ENABLE_GCS if(${TRITON_ENABLE_S3}) set(TRITON_DEPENDS ${TRITON_DEPENDS} aws-sdk-cpp) # Add where to find all S3 dependencies to CMAKE_PREFIX_PATH set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/lib/cmake ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/lib) endif() # TRITON_ENABLE_S3 if(${TRITON_ENABLE_AZURE_STORAGE}) set(TRITON_DEPENDS ${TRITON_DEPENDS} azure-sdk) set(TRITON_AZURE_STORAGE_PACKAGE_DIRS -Dazure-storage-blobs-cpp_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/azure-sdk/share/azure-storage-blobs-cpp -Dazure-storage-common-cpp_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/azure-sdk/share/azure-storage-common-cpp -Dazure-core-cpp_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/azure-sdk/share/azure-core-cpp ) endif() # TRITON_ENABLE_AZURE_STORAGE if(${TRITON_ENABLE_METRICS}) set(TRITON_DEPENDS ${TRITON_DEPENDS} prometheus-cpp) endif() # TRITON_ENABLE_METRICS if(${TRITON_ENABLE_GPU}) set(TRITON_DEPENDS ${TRITON_DEPENDS} cnmem) endif() # TRITON_ENABLE_GPU ExternalProject_Add(triton-core PREFIX triton-core SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/triton-core" CMAKE_CACHE_ARGS -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DBoost_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/boost -DGTEST_ROOT:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/googletest -DgRPC_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc/lib/cmake/grpc -Dc-ares_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares/${LIB_DIR}/cmake/c-ares -Dabsl_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl/${LIB_DIR}/cmake/absl -Dre2_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/re2/${LIB_DIR}/cmake/re2 -Dnlohmann_json_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/nlohmann_json/${LIB_DIR}/cmake/nlohmann_json -Dprometheus-cpp_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/prometheus-cpp/${LIB_DIR}/cmake/prometheus-cpp -Dgoogle_cloud_cpp_storage_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/google-cloud-cpp/${LIB_DIR}/cmake/google_cloud_cpp_storage -Dgoogle_cloud_cpp_rest_internal_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/google-cloud-cpp/${LIB_DIR}/cmake/google_cloud_cpp_rest_internal -Dgoogle_cloud_cpp_common_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/google-cloud-cpp/${LIB_DIR}/cmake/google_cloud_cpp_common ${TRITON_AZURE_STORAGE_PACKAGE_DIRS} -DCrc32c_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/crc32c/${LIB_DIR}/cmake/Crc32c -DAWSSDK_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/AWSSDK -Daws-cpp-sdk-core_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/aws-cpp-sdk-core -Daws-cpp-sdk-s3_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/aws-cpp-sdk-s3 -Daws-c-event-stream_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-c-event-stream/cmake -Daws-c-common_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-c-common/cmake -Daws-checksums_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-checksums/cmake -DCMAKE_PREFIX_PATH:PATH=${CMAKE_PREFIX_PATH} -DCNMEM_PATH:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/cnmem -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} -DTRITON_MODULE_SOURCE_DIR:STRING=${TRITON_MODULE_SOURCE_DIR} -DTRITON_EXTRA_LIB_PATHS:PATH=${TRITON_EXTRA_LIB_PATHS} -DTRITON_ENABLE_NVTX:BOOL=${TRITON_ENABLE_NVTX} -DTRITON_ENABLE_TRACING:BOOL=${TRITON_ENABLE_TRACING} -DTRITON_ENABLE_LOGGING:BOOL=${TRITON_ENABLE_LOGGING} -DTRITON_ENABLE_STATS:BOOL=${TRITON_ENABLE_STATS} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DTRITON_ENABLE_MALI_GPU:BOOL=${TRITON_ENABLE_MALI_GPU} -DTRITON_MIN_COMPUTE_CAPABILITY:STRING=${TRITON_MIN_COMPUTE_CAPABILITY} -DTRITON_ENABLE_METRICS:BOOL=${TRITON_ENABLE_METRICS} -DTRITON_ENABLE_METRICS_GPU:BOOL=${TRITON_ENABLE_METRICS_GPU} -DTRITON_ENABLE_METRICS_CPU:BOOL=${TRITON_ENABLE_METRICS_CPU} -DTRITON_ENABLE_GCS:BOOL=${TRITON_ENABLE_GCS} -DTRITON_ENABLE_AZURE_STORAGE:BOOL=${TRITON_ENABLE_AZURE_STORAGE} -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3} -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} -DTRITON_VERSION:STRING=${TRITON_VERSION} INSTALL_COMMAND "" #DEPENDS ${TRITON_DEPENDS} ) endif() # NOT TRITON_CORE_HEADERS_ONLY # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonCore) install( TARGETS triton-core-backendapi triton-core-repoagentapi triton-core-cacheapi triton-core-serverapi EXPORT triton-core-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) install( TARGETS triton-core-serverstub EXPORT triton-core-targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR} ) install( DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) install( EXPORT triton-core-targets FILE TritonCoreTargets.cmake NAMESPACE TritonCore:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonCoreConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonCoreConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonCoreConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-core-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonCoreTargets.cmake NAMESPACE TritonCore:: ) export(PACKAGE TritonCore) # do not build tritonserver python #if(NOT TRITON_CORE_HEADERS_ONLY) # add_subdirectory(python python) #endif() kytensor-2.49.0.7/core/LICENSE000066400000000000000000000027261527052371200156010ustar00rootroot00000000000000Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/core/README.md000066400000000000000000000113441527052371200160470ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Triton Inference Server Core This repository holds the source code and headers for the library that implements the core functionality of Triton. The *core* library can be built as described below and used directly via its [C API](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/inference_protocols.md#in-process-triton-server-api). To be useful the core library must be paired with one or more backends. You can learn more about backends in the [backend repo](https://github.com/triton-inference-server/backend). Typically you do not build or use the core library on its own, but as part of the *tritonserver* executable. The *tritonserver* executable is built in the [server repo](https://github.com/triton-inference-server/server) as described in the [server build documentation](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/build.md). Ask questions or report problems in the main Triton [issues page](https://github.com/triton-inference-server/server/issues). ## Build the Triton Core Library Before building the Triton core library, your build system must install the required dependencies described in the [build documentation](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/build.md). For example, if you are building the core library with GPU support (-DTRITON_ENABLE_GPU=ON), then you must install the CUDA, cuDNN, and TensorRT dependencies required for the version of Triton you are building. To build, first clone the release branch matching the Triton release you are interest in (*rxx.yy*), or the *main* branch to build the top-of-tree. The Triton core library is built with CMake. ``` $ mkdir build $ cd build $ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_CORE_HEADERS_ONLY=OFF .. $ make install ``` When the build completes, the install directory will contain the Triton core shared library (install/lib/libtritonserver.so on Linux, install/bin/tritonserver.dll on Windows), and the core library headers files in install/include/triton/core. ### Build a Release Branch The following required Triton repositories will be pulled and used in the build. By default the "main" branch/tag will be used for each repo but the listed CMake argument can be used to override. * triton-inference-server/third_party: -DTRITON_THIRD_PARTY_REPO_TAG=[tag] * triton-inference-server/common: -DTRITON_COMMON_REPO_TAG=[tag] You will need to override if you are building from a release branch. For example, if you are building the r23.04 version of Triton, you would clone the r23.04 branch of the core repo and use the following cmake command. ``` $ cmake -DTRITON_THIRD_PARTY_REPO_TAG=r23.04 -DTRITON_COMMON_REPO_TAG=r23.04 -DTRITON_CORE_HEADERS_ONLY=OFF .. ``` ### Build Options The [CMakeLists.txt](CMakeLists.txt) file contains the options available when build the core library. For example, to build the core library with the default settings plus S3 cloud storage and ensembling support use the following command. ``` $ cmake -DTRITON_CORE_HEADERS_ONLY=OFF -DTRITON_ENABLE_S3=ON -DTRITON_ENABLE_ENSEMBLE=ON .. ``` kytensor-2.49.0.7/core/cmake/000077500000000000000000000000001527052371200156455ustar00rootroot00000000000000kytensor-2.49.0.7/core/cmake/TritonCoreConfig.cmake.in000066400000000000000000000034431527052371200224760ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONCORE_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONCORE_CMAKE_DIR}) if(NOT TARGET TritonCore::triton-core-serverapi) include("${TRITONCORE_CMAKE_DIR}/TritonCoreTargets.cmake") endif() kytensor-2.49.0.7/core/include/000077500000000000000000000000001527052371200162105ustar00rootroot00000000000000kytensor-2.49.0.7/core/include/triton/000077500000000000000000000000001527052371200175275ustar00rootroot00000000000000kytensor-2.49.0.7/core/include/triton/core/000077500000000000000000000000001527052371200204575ustar00rootroot00000000000000kytensor-2.49.0.7/core/include/triton/core/tritonbackend.h000066400000000000000000002470211527052371200234650ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "triton/core/tritonserver.h" #ifdef __cplusplus extern "C" { #endif #ifdef _COMPILING_TRITONBACKEND #if defined(_MSC_VER) #define TRITONBACKEND_DECLSPEC __declspec(dllexport) #define TRITONBACKEND_ISPEC __declspec(dllimport) #elif defined(__GNUC__) #define TRITONBACKEND_DECLSPEC __attribute__((__visibility__("default"))) #define TRITONBACKEND_ISPEC #else #define TRITONBACKEND_DECLSPEC #define TRITONBACKEND_ISPEC #endif #else #if defined(_MSC_VER) #define TRITONBACKEND_DECLSPEC __declspec(dllimport) #define TRITONBACKEND_ISPEC __declspec(dllexport) #else #define TRITONBACKEND_DECLSPEC #define TRITONBACKEND_ISPEC #endif #endif struct TRITONBACKEND_MemoryManager; struct TRITONBACKEND_Input; struct TRITONBACKEND_Output; struct TRITONBACKEND_State; struct TRITONBACKEND_Request; struct TRITONBACKEND_ResponseFactory; struct TRITONBACKEND_Response; struct TRITONBACKEND_Backend; struct TRITONBACKEND_Model; struct TRITONBACKEND_ModelInstance; struct TRITONBACKEND_ModelInstanceResponseStatistics; struct TRITONBACKEND_BackendAttribute; struct TRITONBACKEND_Batcher; /// /// TRITONBACKEND API Version /// /// The TRITONBACKEND API is versioned with major and minor version /// numbers. Any change to the API that does not impact backwards /// compatibility (for example, adding a non-required function) /// increases the minor version number. Any change that breaks /// backwards compatibility (for example, deleting or changing the /// behavior of a function) increases the major version number. A /// backend should check that the API version used to compile the /// backend is compatible with the API version of the Triton server /// that it is running in. This is typically done by code similar to /// the following which makes sure that the major versions are equal /// and that the minor version of Triton is >= the minor version used /// to build the backend. /// /// uint32_t api_version_major, api_version_minor; /// TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor); /// if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || /// (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { /// return TRITONSERVER_ErrorNew( /// TRITONSERVER_ERROR_UNSUPPORTED, /// "triton backend API version does not support this backend"); /// } /// #define TRITONBACKEND_API_VERSION_MAJOR 1 #define TRITONBACKEND_API_VERSION_MINOR 19 /// Get the TRITONBACKEND API version supported by Triton. This value /// can be compared against the TRITONBACKEND_API_VERSION_MAJOR and /// TRITONBACKEND_API_VERSION_MINOR used to build the backend to /// ensure that Triton is compatible with the backend. /// /// \param major Returns the TRITONBACKEND API major version supported /// by Triton. /// \param minor Returns the TRITONBACKEND API minor version supported /// by Triton. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ApiVersion( uint32_t* major, uint32_t* minor); /// TRITONBACKEND_ArtifactType /// /// The ways that the files that make up a backend or model are /// communicated to the backend. /// /// TRITONBACKEND_ARTIFACT_FILESYSTEM: The model or backend /// artifacts are made available to Triton via a locally /// accessible filesystem. The backend can access these files /// using an appropriate system API. /// typedef enum TRITONBACKEND_artifacttype_enum { TRITONBACKEND_ARTIFACT_FILESYSTEM } TRITONBACKEND_ArtifactType; /// /// TRITONBACKEND_MemoryManager /// /// Object representing an memory manager that is capable of /// allocating and otherwise managing different memory types. For /// improved performance Triton maintains pools for GPU and CPU-pinned /// memory and the memory manager allows backends to access those /// pools. /// /// Allocate a contiguous block of memory of a specific type using a /// memory manager. Two error codes have specific interpretations for /// this function: /// /// TRITONSERVER_ERROR_UNSUPPORTED: Indicates that Triton is /// incapable of allocating the requested memory type and memory /// type ID. Requests for the memory type and ID will always fail /// no matter 'byte_size' of the request. /// /// TRITONSERVER_ERROR_UNAVAILABLE: Indicates that Triton can /// allocate the memory type and ID but that currently it cannot /// allocate a contiguous block of memory of the requested /// 'byte_size'. /// /// \param manager The memory manager. /// \param buffer Returns the allocated memory. /// \param memory_type The type of memory to allocate. /// \param memory_type_id The ID associated with the memory type to /// allocate. For GPU memory this indicates the device ID of the GPU /// to allocate from. /// \param byte_size The size of memory to allocate, in bytes. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_MemoryManagerAllocate( TRITONBACKEND_MemoryManager* manager, void** buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id, const uint64_t byte_size); /// Free a buffer that was previously allocated with /// TRITONBACKEND_MemoryManagerAllocate. The call must provide the /// same values for 'memory_type' and 'memory_type_id' as were used /// when the buffer was allocate or else the behavior is undefined. /// /// \param manager The memory manager. /// \param buffer The allocated memory buffer to free. /// \param memory_type The type of memory of the buffer. /// \param memory_type_id The ID associated with the memory type of /// the buffer. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_MemoryManagerFree( TRITONBACKEND_MemoryManager* manager, void* buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id); /// /// TRITONBACKEND_Input /// /// Object representing an input tensor. /// /// Get the name and properties of an input tensor. The returned /// strings and other properties are owned by the input, not the /// caller, and so should not be modified or freed. /// /// \param input The input tensor. /// \param name If non-nullptr, returns the tensor name. /// \param datatype If non-nullptr, returns the tensor datatype. /// \param shape If non-nullptr, returns the tensor shape. /// \param dim_count If non-nullptr, returns the number of dimensions /// in the tensor shape. /// \param byte_size If non-nullptr, returns the size of the available /// data for the tensor, in bytes. This size reflects the actual data /// available, and does not necessarily match what is /// expected/required for the tensor given its shape and datatype. It /// is the responsibility of the backend to handle mismatches in these /// sizes appropriately. /// \param buffer_count If non-nullptr, returns the number of buffers /// holding the contents of the tensor. These buffers are accessed /// using TRITONBACKEND_InputBuffer. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputProperties( TRITONBACKEND_Input* input, const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint32_t* dims_count, uint64_t* byte_size, uint32_t* buffer_count); /// Get the name and properties of an input tensor associated with a given /// host policy. If there are no input buffers for the specified host policy, /// the properties of the fallback input buffers are returned. The returned /// strings and other properties are owned by the input, not the caller, and so /// should not be modified or freed. /// /// \param input The input tensor. /// \param host_policy_name The host policy name. Fallback input properties /// will be return if nullptr is provided. /// \param name If non-nullptr, returns the tensor name. /// \param datatype If non-nullptr, returns the tensor datatype. /// \param shape If non-nullptr, returns the tensor shape. /// \param dim_count If non-nullptr, returns the number of dimensions /// in the tensor shape. /// \param byte_size If non-nullptr, returns the size of the available /// data for the tensor, in bytes. This size reflects the actual data /// available, and does not necessarily match what is /// expected/required for the tensor given its shape and datatype. It /// is the responsibility of the backend to handle mismatches in these /// sizes appropriately. /// \param buffer_count If non-nullptr, returns the number of buffers /// holding the contents of the tensor. These buffers are accessed /// using TRITONBACKEND_InputBufferForHostPolicy. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputPropertiesForHostPolicy( TRITONBACKEND_Input* input, const char* host_policy_name, const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint32_t* dims_count, uint64_t* byte_size, uint32_t* buffer_count); /// Get a buffer holding (part of) the tensor data for an input. For a /// given input the number of buffers composing the input are found /// from 'buffer_count' returned by TRITONBACKEND_InputProperties. The /// returned buffer is owned by the input and so should not be /// modified or freed by the caller. The lifetime of the buffer /// matches that of the input and so the buffer should not be accessed /// after the input tensor object is released. /// /// \param input The input tensor. /// \param index The index of the buffer. Must be 0 <= index < /// buffer_count, where buffer_count is the value returned by /// TRITONBACKEND_InputProperties. /// \param buffer Returns a pointer to a contiguous block of data for /// the named input. /// \param buffer_byte_size Returns the size, in bytes, of 'buffer'. /// \param memory_type Acts as both input and output. On input gives /// the buffer memory type preferred by the function caller. Returns /// the actual memory type of 'buffer'. /// \param memory_type_id Acts as both input and output. On input /// gives the buffer memory type id preferred by the function caller. /// Returns the actual memory type id of 'buffer'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputBuffer( TRITONBACKEND_Input* input, const uint32_t index, const void** buffer, uint64_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); /// Get a buffer holding (part of) the tensor data for an input for a specific /// host policy. If there are no input buffers specified for this host policy, /// the fallback input buffer is returned. /// For a given input the number of buffers composing the input are found /// from 'buffer_count' returned by TRITONBACKEND_InputPropertiesForHostPolicy. /// The returned buffer is owned by the input and so should not be modified or /// freed by the caller. The lifetime of the buffer matches that of the input /// and so the buffer should not be accessed after the input tensor object is /// released. /// /// \param input The input tensor. /// \param host_policy_name The host policy name. Fallback input buffer /// will be return if nullptr is provided. /// \param index The index of the buffer. Must be 0 <= index < /// buffer_count, where buffer_count is the value returned by /// TRITONBACKEND_InputPropertiesForHostPolicy. /// \param buffer Returns a pointer to a contiguous block of data for /// the named input. /// \param buffer_byte_size Returns the size, in bytes, of 'buffer'. /// \param memory_type Acts as both input and output. On input gives /// the buffer memory type preferred by the function caller. Returns /// the actual memory type of 'buffer'. /// \param memory_type_id Acts as both input and output. On input /// gives the buffer memory type id preferred by the function caller. /// Returns the actual memory type id of 'buffer'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputBufferForHostPolicy( TRITONBACKEND_Input* input, const char* host_policy_name, const uint32_t index, const void** buffer, uint64_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); /// Get the buffer attributes associated with the given input buffer. For a /// given input the number of buffers composing the input are found from /// 'buffer_count' returned by TRITONBACKEND_InputProperties. The returned /// 'buffer_attributes' is owned by the input and so should not be modified or /// freed by the caller. The lifetime of the 'buffer_attributes' matches that of /// the input and so the 'buffer_attributes' should not be accessed after the /// input tensor object is released. /// /// \param input The input tensor. /// \param index The index of the buffer. Must be 0 <= index < buffer_count, /// where buffer_count is the value returned by TRITONBACKEND_InputProperties. /// \param buffer Returns a pointer to a contiguous block of data for /// the named input. /// \param buffer_attributes Returns the attributes for the given buffer. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputBufferAttributes( TRITONBACKEND_Input* input, const uint32_t index, const void** buffer, TRITONSERVER_BufferAttributes** buffer_attributes); /// /// TRITONBACKEND_Output /// /// Object representing a response output tensor. /// /// Get a buffer to use to hold the tensor data for the output. The /// returned buffer is owned by the output and so should not be freed /// by the caller. The caller can and should fill the buffer with the /// output data for the tensor. The lifetime of the buffer matches /// that of the output and so the buffer should not be accessed after /// the output tensor object is released. /// /// \param buffer Returns a pointer to a buffer where the contents of /// the output tensor should be placed. /// \param buffer_byte_size The size, in bytes, of the buffer required /// by the caller. /// \param memory_type Acts as both input and output. On input gives /// the buffer memory type preferred by the caller. Returns the /// actual memory type of 'buffer'. /// \param memory_type_id Acts as both input and output. On input /// gives the buffer memory type id preferred by the caller. Returns /// the actual memory type id of 'buffer'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_OutputBuffer( TRITONBACKEND_Output* output, void** buffer, const uint64_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); /// Get the buffer attributes associated with the given output buffer. The /// returned 'buffer_attributes' is owned by the output and so should not be /// modified or freed by the caller. The lifetime of the 'buffer_attributes' /// matches that of the output and so the 'buffer_attributes' should not be /// accessed after the output tensor object is released. This function must be /// called after the TRITONBACKEND_OutputBuffer otherwise it might contain /// incorrect data. /// /// \param output The output tensor. /// \param buffer_attributes Returns the attributes for the output buffer. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_OutputBufferAttributes( TRITONBACKEND_Output* output, TRITONSERVER_BufferAttributes** buffer_attributes); /// /// TRITONBACKEND_Request /// /// Object representing an inference request. /// /// Get the ID of the request. Can be nullptr if request doesn't have /// an ID. The returned string is owned by the request, not the /// caller, and so should not be modified or freed. /// /// \param request The inference request. /// \param id Returns the ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestId( TRITONBACKEND_Request* request, const char** id); /// Query whether the request is cancelled or not. /// /// If possible the backend should terminate any processing and /// send an error response with cancelled status. /// /// \param request The inference request. /// \param is_cancelled Returns true if the request is cancelled otherwise it /// would return false. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestIsCancelled( TRITONBACKEND_Request* request, bool* is_cancelled); /// Query whether the response factory is cancelled or not. /// /// If possible the backend should terminate any processing and /// send an error response with cancelled status. /// /// \param factory The response factory /// \param is_cancelled Returns true if the request is cancelled otherwise it /// would return false. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactoryIsCancelled( TRITONBACKEND_ResponseFactory* factory, bool* is_cancelled); /// Get the correlation ID of the request if it is an unsigned integer. /// Zero indicates that the request does not have a correlation ID. /// Returns failure if correlation ID for given request is not an unsigned /// integer. /// /// \param request The inference request. /// \param id Returns the correlation ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestCorrelationId( TRITONBACKEND_Request* request, uint64_t* id); /// Get the timeout of the request. A value of 0 indicates no timeout was /// specified for the request. provided, the timeout will default to zero. /// /// \param request The inference request. /// \param timeout Returns the timeout value for the request. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InferenceRequestTimeoutMicroseconds( TRITONBACKEND_Request* request, uint64_t* timeout); /// Get the correlation ID of the request if it is a string. /// Empty string indicates that the request does not have a correlation ID. /// Returns error if correlation ID for given request is not a string. /// /// \param request The inference request. /// \param id Returns the correlation ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestCorrelationIdString( TRITONBACKEND_Request* request, const char** id); /// Get the flag(s) associated with a request. On return 'flags' holds /// a bitwise-or of all flag values, see TRITONSERVER_RequestFlag for /// available flags. /// /// \param request The inference request. /// \param flags Returns the flags. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestFlags( TRITONBACKEND_Request* request, uint32_t* flags); /// Get the number of parameters specified in the inference request. /// /// \param request The inference request. /// \param count Returns the number of parameters. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestParameterCount( TRITONBACKEND_Request* request, uint32_t* count); /// Get a request parameters by index. The order of parameters in a given /// request is not necessarily consistent with other requests, even if /// the requests are in the same batch. As a result, you can not /// assume that an index obtained from one request will point to the /// same parameter in a different request. /// /// The lifetime of the returned parameter object matches that of the /// request and so the parameter object should not be accessed after the /// request object is released. /// /// \param request The inference request. /// \param index The index of the parameter. Must be 0 <= index < /// count, where count is the value returned by /// TRITONBACKEND_RequestParameterCount. /// \param key Returns the key of the parameter. /// \param type Returns the type of the parameter. /// \param vvalue Returns a pointer to the parameter value. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestParameter( TRITONBACKEND_Request* request, const uint32_t index, const char** key, TRITONSERVER_ParameterType* type, const void** vvalue); /// Get the number of input tensors specified in the request. /// /// \param request The inference request. /// \param count Returns the number of input tensors. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputCount( TRITONBACKEND_Request* request, uint32_t* count); /// Get the name of an input tensor. The caller does not own /// the returned string and must not modify or delete it. The lifetime /// of the returned string extends only as long as 'request'. /// /// \param request The inference request. /// \param index The index of the input tensor. Must be 0 <= index < /// count, where count is the value returned by /// TRITONBACKEND_RequestInputCount. /// \param input_name Returns the name of the input tensor /// corresponding to the index. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputName( TRITONBACKEND_Request* request, const uint32_t index, const char** input_name); /// Get a named request input. The lifetime of the returned input /// object matches that of the request and so the input object should /// not be accessed after the request object is released. /// /// \param request The inference request. /// \param name The name of the input. /// \param input Returns the input corresponding to the name. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInput( TRITONBACKEND_Request* request, const char* name, TRITONBACKEND_Input** input); /// Get a request input by index. The order of inputs in a given /// request is not necessarily consistent with other requests, even if /// the requests are in the same batch. As a result, you can not /// assume that an index obtained from one request will point to the /// same input in a different request. /// /// The lifetime of the returned input object matches that of the /// request and so the input object should not be accessed after the /// request object is released. /// /// \param request The inference request. /// \param index The index of the input tensor. Must be 0 <= index < /// count, where count is the value returned by /// TRITONBACKEND_RequestInputCount. /// \param input Returns the input corresponding to the index. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputByIndex( TRITONBACKEND_Request* request, const uint32_t index, TRITONBACKEND_Input** input); /// Get the number of output tensors requested to be returned in the /// request. /// /// \param request The inference request. /// \param count Returns the number of output tensors. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestOutputCount( TRITONBACKEND_Request* request, uint32_t* count); /// Get the name of a requested output tensor. The caller does not own /// the returned string and must not modify or delete it. The lifetime /// of the returned string extends only as long as 'request'. /// /// \param request The inference request. /// \param index The index of the requested output tensor. Must be 0 /// <= index < count, where count is the value returned by /// TRITONBACKEND_RequestOutputCount. /// \param output_name Returns the name of the requested output tensor /// corresponding to the index. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestOutputName( TRITONBACKEND_Request* request, const uint32_t index, const char** output_name); /// Returns the preferred memory type and memory type ID of the output buffer /// for the request. As much as possible, Triton will attempt to return /// the same memory_type and memory_type_id values that will be returned by /// the subsequent call to TRITONBACKEND_OutputBuffer, however, the backend must /// be capable of handling cases where the values differ. /// /// \param request The request. /// \param name The name of the output tensor. This is optional /// and it should be set to nullptr to indicate that the tensor name has /// not determined. /// \param byte_size The expected size of the buffer. This is optional /// and it should be set to nullptr to indicate that the byte size has /// not determined. /// \param memory_type Acts as both input and output. On input gives /// the memory type preferred by the caller. Returns memory type preferred /// by Triton, taken account of the caller preferred type. /// \param memory_type_id Acts as both input and output. On input gives /// the memory type ID preferred by the caller. Returns memory type ID preferred /// by Triton, taken account of the caller preferred type ID. /// \return a TRITONSERVER_Error object if a failure occurs. /// A TRITONSERVER_ERROR_UNAVAILABLE error indicates that the properties are not /// available, other error codes indicate an error. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestOutputBufferProperties( TRITONBACKEND_Request* request, const char* name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); /// Release the request. The request should be released when it is no /// longer needed by the backend. If this call returns with an error /// (i.e. non-nullptr) then the request was not released and ownership /// remains with the backend. If this call returns with success, the /// 'request' object is no longer owned by the backend and must not be /// used. Any tensor names, data types, shapes, input tensors, /// etc. returned by TRITONBACKEND_Request* functions for this request /// are no longer valid. If a persistent copy of that data is required /// it must be created before calling this function. /// /// \param request The inference request. /// \param release_flags Flags indicating what type of request release /// should be performed. \see TRITONSERVER_RequestReleaseFlag. \see /// TRITONSERVER_InferenceRequestReleaseFn_t. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestRelease( TRITONBACKEND_Request* request, uint32_t release_flags); /// Get the trace associated with a request. The returned trace is owned by the /// request, not the caller, and so should not be modified or freed. /// If the request is not being traced, then `nullptr` will be returned. /// /// \param request The inference request. /// \param trace Returns the trace associated with the request. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestTrace( TRITONBACKEND_Request* request, TRITONSERVER_InferenceTrace** trace); /// /// TRITONBACKEND_ResponseFactory /// /// Object representing an inference response factory. Using a /// response factory is not required; instead a response can be /// generated directly from a TRITONBACKEND_Request object using /// TRITONBACKEND_ResponseNew(). A response factory allows a request /// to be released before all responses have been sent. Releasing a /// request as early as possible releases all input tensor data and /// therefore may be desirable in some cases. /// Create the response factory associated with a request. /// /// \param factory Returns the new response factory. /// \param request The inference request. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactoryNew( TRITONBACKEND_ResponseFactory** factory, TRITONBACKEND_Request* request); /// Destroy a response factory. /// /// \param factory The response factory. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactoryDelete( TRITONBACKEND_ResponseFactory* factory); /// Send response flags without a corresponding response. /// /// \param factory The response factory. /// \param send_flags Flags to send. \see /// TRITONSERVER_ResponseCompleteFlag. \see /// TRITONSERVER_InferenceResponseCompleteFn_t. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactorySendFlags( TRITONBACKEND_ResponseFactory* factory, const uint32_t send_flags); /// /// TRITONBACKEND_Response /// /// Object representing an inference response. For a given request, /// the backend must carefully manage the lifecycle of responses /// generated for that request to ensure that the output tensor /// buffers are allocated correctly. When a response is created with /// TRITONBACKEND_ResponseNew or TRITONBACKEND_ResponseNewFromFactory, /// all the outputs and corresponding buffers must be created for that /// response using TRITONBACKEND_ResponseOutput and /// TRITONBACKEND_OutputBuffer *before* another response is created /// for the request. For a given response, outputs can be created in /// any order but they must be created sequentially/synchronously (for /// example, the backend cannot use multiple threads to simultaneously /// add multiple outputs to a response). /// /// The above requirement applies only to responses being generated /// for a given request. The backend may generate responses in /// parallel on multiple threads as long as those responses are for /// different requests. /// /// This order of response creation must be strictly followed. But, /// once response(s) are created they do not need to be sent /// immediately, nor do they need to be sent in the order they were /// created. The backend may even delete a created response instead of /// sending it by using TRITONBACKEND_ResponseDelete. /// Create a response for a request. /// /// \param response Returns the new response. /// \param request The request. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseNew( TRITONBACKEND_Response** response, TRITONBACKEND_Request* request); /// Create a response using a factory. /// /// \param response Returns the new response. /// \param factory The response factory. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseNewFromFactory( TRITONBACKEND_Response** response, TRITONBACKEND_ResponseFactory* factory); /// Destroy a response. It is not necessary to delete a response if /// TRITONBACKEND_ResponseSend is called as that function transfers /// ownership of the response object to Triton. /// /// \param response The response. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseDelete( TRITONBACKEND_Response* response); /// Set a string parameter in the response. /// /// \param response The response. /// \param name The name of the parameter. /// \param value The value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSetStringParameter( TRITONBACKEND_Response* response, const char* name, const char* value); /// Set an integer parameter in the response. /// /// \param response The response. /// \param name The name of the parameter. /// \param value The value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSetIntParameter( TRITONBACKEND_Response* response, const char* name, const int64_t value); /// Set a boolean parameter in the response. /// /// \param response The response. /// \param name The name of the parameter. /// \param value The value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSetBoolParameter( TRITONBACKEND_Response* response, const char* name, const bool value); /// Set a double parameter in the response. /// /// \param response The response. /// \param name The name of the parameter. /// \param value The value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSetDoubleParameter( TRITONBACKEND_Response* response, const char* name, const double value); /// Create an output tensor in the response. The lifetime of the /// returned output tensor object matches that of the response and so /// the output tensor object should not be accessed after the response /// object is deleted. /// /// \param response The response. /// \param output Returns the new response output. /// \param name The name of the output tensor. /// \param datatype The datatype of the output tensor. /// \param shape The shape of the output tensor. /// \param dims_count The number of dimensions in the output tensor /// shape. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseOutput( TRITONBACKEND_Response* response, TRITONBACKEND_Output** output, const char* name, const TRITONSERVER_DataType datatype, const int64_t* shape, const uint32_t dims_count); /// Send a response. Calling this function transfers ownership of the /// response object to Triton. The caller must not access or delete /// the response object after calling this function. /// /// \param response The response. /// \param send_flags Flags associated with the response. \see /// TRITONSERVER_ResponseCompleteFlag. \see /// TRITONSERVER_InferenceResponseCompleteFn_t. /// \param error The TRITONSERVER_Error to send if the response is an error, or /// nullptr if the response is successful. The caller retains ownership to the /// error object and must free it with TRITONSERVER_ErrorDelete. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSend( TRITONBACKEND_Response* response, const uint32_t send_flags, TRITONSERVER_Error* error); /// /// TRITONBACKEND_State /// /// Object representing a state. /// /// Create a state in the request. The returned state object is only valid /// before the TRITONBACKEND_StateUpdate is called. The state should not be /// freed by the caller. If TRITONBACKEND_StateUpdate is not called, the /// lifetime of the state matches the lifetime of the request. If the state name /// does not exist in the "state" section of the model configuration, the state /// will not be created and an error will be returned. If this function is /// called when sequence batching is not enabled or there is no 'states' section /// in the sequence batching section of the model configuration, this call will /// return an error. /// /// \param state Returns the new state. /// \param request The request. /// \param name The name of the state. /// \param datatype The datatype of the state. /// \param shape The shape of the state. /// \param dims_count The number of dimensions in the state shape. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateNew( TRITONBACKEND_State** state, TRITONBACKEND_Request* request, const char* name, const TRITONSERVER_DataType datatype, const int64_t* shape, const uint32_t dims_count); /// Update the state for the sequence. Calling this function will replace the /// state stored for this sequence in Triton with 'state' provided in the /// function argument. If this function is called when sequence batching is not /// enabled or there is no 'states' section in the sequence batching section of /// the model configuration, this call will return an error. The backend is not /// required to call this function. If the backend doesn't call /// TRITONBACKEND_StateUpdate function, this particular state for the sequence /// will not be updated and the next inference request in the sequence will use /// the same state as the current inference request. /// /// \param state The state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateUpdate( TRITONBACKEND_State* state); /// Get a buffer to use to hold the tensor data for the state. The returned /// buffer is owned by the state and so should not be freed by the caller. The /// caller can and should fill the buffer with the state data. The buffer must /// not be accessed by the backend after TRITONBACKEND_StateUpdate is called. /// The caller should fill the buffer before calling TRITONBACKEND_StateUpdate. /// /// \param state The state. /// \param buffer Returns a pointer to a buffer where the contents of the state /// should be placed. /// \param buffer_byte_size The size, in bytes, of the buffer required /// by the caller. /// \param memory_type Acts as both input and output. On input gives /// the buffer memory type preferred by the caller. Returns the /// actual memory type of 'buffer'. /// \param memory_type_id Acts as both input and output. On input /// gives the buffer memory type id preferred by the caller. Returns /// the actual memory type id of 'buffer'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateBuffer( TRITONBACKEND_State* state, void** buffer, const uint64_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); /// Get the buffer attributes associated with the given state buffer. /// The returned 'buffer_attributes' is owned by the state and so should not be /// modified or freed by the caller. The lifetime of the 'buffer_attributes' /// matches that of the state. /// /// \param state The state. /// \param buffer_attributes Returns the buffer attributes for the given state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateBufferAttributes( TRITONBACKEND_State* state, TRITONSERVER_BufferAttributes** buffer_attributes); /// /// TRITONBACKEND_Backend /// /// Object representing a backend. /// /// TRITONBACKEND_ExecutionPolicy /// /// Types of execution policy that can be implemented by a backend. /// /// TRITONBACKEND_EXECUTION_BLOCKING: An instance of the model /// blocks in TRITONBACKEND_ModelInstanceExecute until it is ready /// to handle another inference. Upon returning from /// TRITONBACKEND_ModelInstanceExecute, Triton may immediately /// call TRITONBACKEND_ModelInstanceExecute for the same instance /// to execute a new batch of requests. Thus, most backends using /// this policy will not return from /// TRITONBACKEND_ModelInstanceExecute until all responses have /// been sent and all requests have been released. This is the /// default execution policy. /// /// TRITONBACKEND_EXECUTION_DEVICE_BLOCKING: An instance, A, of the /// model blocks in TRITONBACKEND_ModelInstanceExecute if the /// device associated with the instance is unable to handle /// another inference. Even if another instance, B, associated /// with the device, is available and ready to perform an /// inference, Triton will not invoke /// TRITONBACKEND_ModeInstanceExecute for B until A returns from /// TRITONBACKEND_ModelInstanceExecute. Triton will not be blocked /// from calling TRITONBACKEND_ModelInstanceExecute for instance /// C, which is associated with a different device than A and B, /// even if A or B has not returned from /// TRITONBACKEND_ModelInstanceExecute. This execution policy is /// typically used by a backend that can cooperatively execute /// multiple model instances on the same device. /// typedef enum TRITONBACKEND_execpolicy_enum { TRITONBACKEND_EXECUTION_BLOCKING, TRITONBACKEND_EXECUTION_DEVICE_BLOCKING } TRITONBACKEND_ExecutionPolicy; /// Get the name of the backend. The caller does not own the returned /// string and must not modify or delete it. The lifetime of the /// returned string extends only as long as 'backend'. /// /// \param backend The backend. /// \param name Returns the name of the backend. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendName( TRITONBACKEND_Backend* backend, const char** name); /// Get the backend configuration. The 'backend_config' message is /// owned by Triton and should not be modified or freed by the caller. /// /// The backend configuration, as JSON, is: /// /// { /// "cmdline" : { /// "" : "", /// ... /// } /// } /// /// \param backend The backend. /// \param backend_config Returns the backend configuration as a message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendConfig( TRITONBACKEND_Backend* backend, TRITONSERVER_Message** backend_config); /// Get the execution policy for this backend. By default the /// execution policy is TRITONBACKEND_EXECUTION_BLOCKING. /// /// \param backend The backend. /// \param policy Returns the execution policy. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendExecutionPolicy( TRITONBACKEND_Backend* backend, TRITONBACKEND_ExecutionPolicy* policy); /// Set the execution policy for this backend. By default the /// execution policy is TRITONBACKEND_EXECUTION_BLOCKING. Triton reads /// the backend's execution policy after calling /// TRITONBACKEND_Initialize, so to be recognized changes to the /// execution policy must be made in TRITONBACKEND_Initialize. /// Also, note that if using sequence batcher for the model, Triton will /// use TRITONBACKEND_EXECUTION_BLOCKING policy irrespective of the /// policy specified by this setter function. /// /// \param backend The backend. /// \param policy The execution policy. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendSetExecutionPolicy( TRITONBACKEND_Backend* backend, TRITONBACKEND_ExecutionPolicy policy); /// Get the location of the files that make up the backend /// implementation. This location contains the backend shared library /// and any other files located with the shared library. The /// 'location' communicated depends on how the backend is being /// communicated to Triton as indicated by 'artifact_type'. /// /// TRITONBACKEND_ARTIFACT_FILESYSTEM: The backend artifacts are /// made available to Triton via the local filesystem. 'location' /// returns the full path to the directory containing this /// backend's artifacts. The returned string is owned by Triton, /// not the caller, and so should not be modified or freed. /// /// \param backend The backend. /// \param artifact_type Returns the artifact type for the backend. /// \param path Returns the location. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendArtifacts( TRITONBACKEND_Backend* backend, TRITONBACKEND_ArtifactType* artifact_type, const char** location); /// Get the memory manager associated with a backend. /// /// \param backend The backend. /// \param manager Returns the memory manager. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendMemoryManager( TRITONBACKEND_Backend* backend, TRITONBACKEND_MemoryManager** manager); /// Get the user-specified state associated with the backend. The /// state is completely owned and managed by the backend. /// /// \param backend The backend. /// \param state Returns the user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendState( TRITONBACKEND_Backend* backend, void** state); /// Set the user-specified state associated with the backend. The /// state is completely owned and managed by the backend. /// /// \param backend The backend. /// \param state The user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendSetState( TRITONBACKEND_Backend* backend, void* state); /// /// TRITONBACKEND_Model /// /// Object representing a model implemented using the backend. /// /// Get the name of the model. The returned string is owned by the /// model object, not the caller, and so should not be modified or /// freed. /// /// \param model The model. /// \param name Returns the model name. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelName( TRITONBACKEND_Model* model, const char** name); /// Get the version of the model. /// /// \param model The model. /// \param version Returns the model version. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelVersion( TRITONBACKEND_Model* model, uint64_t* version); /// Get the location of the files that make up the model. The /// 'location' communicated depends on how the model is being /// communicated to Triton as indicated by 'artifact_type'. /// /// TRITONBACKEND_ARTIFACT_FILESYSTEM: The model artifacts are made /// available to Triton via the local filesystem. 'location' /// returns the full path to the directory in the model repository /// that contains this model's artifacts. The returned string is /// owned by Triton, not the caller, and so should not be modified /// or freed. /// /// \param model The model. /// \param artifact_type Returns the artifact type for the model. /// \param path Returns the location. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelRepository( TRITONBACKEND_Model* model, TRITONBACKEND_ArtifactType* artifact_type, const char** location); /// Get the model configuration. The caller takes ownership of the /// message object and must call TRITONSERVER_MessageDelete to release /// the object. The configuration is available via this call even /// before the model is loaded and so can be used in /// TRITONBACKEND_ModelInitialize. TRITONSERVER_ServerModelConfig /// returns equivalent information but is not usable until after the /// model loads. /// /// \param model The model. /// \param config_version The model configuration will be returned in /// a format matching this version. If the configuration cannot be /// represented in the requested version's format then an error will /// be returned. Currently only version 1 is supported. /// \param model_config Returns the model configuration as a message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelConfig( TRITONBACKEND_Model* model, const uint32_t config_version, TRITONSERVER_Message** model_config); /// Whether the backend should attempt to auto-complete the model configuration. /// If true, the model should fill the inputs, outputs, and max batch size in /// the model configuration if incomplete. If the model configuration is /// changed, the new configuration must be reported to Triton using /// TRITONBACKEND_ModelSetConfig. /// /// \param model The model. /// \param auto_complete_config Returns whether the backend should auto-complete /// the model configuration. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelAutoCompleteConfig( TRITONBACKEND_Model* model, bool* auto_complete_config); /// Set the model configuration in Triton server. This API should only be called /// when the backend implements the auto-completion of model configuration /// and TRITONBACKEND_ModelAutoCompleteConfig returns true in /// auto_complete_config. Only the inputs, outputs, max batch size, and /// scheduling choice can be changed. A caveat being scheduling choice can only /// be changed if none is previously set. Any other changes to the model /// configuration will be ignored by Triton. This function can only be called /// from TRITONBACKEND_ModelInitialize, calling in any other context will result /// in an error being returned. Additionally, Triton server can add some of the /// missing fields in the provided config with this call. The backend must get /// the complete configuration again by using TRITONBACKEND_ModelConfig. /// TRITONBACKEND_ModelSetConfig does not take ownership of the message object /// and so the caller should call TRITONSERVER_MessageDelete to release the /// object once the function returns. /// /// \param model The model. /// \param config_version The format version of the model configuration. /// If the configuration is not represented in the version's format /// then an error will be returned. Currently only version 1 is supported. /// \param model_config The updated model configuration as a message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelSetConfig( TRITONBACKEND_Model* model, const uint32_t config_version, TRITONSERVER_Message* model_config); /// Get the TRITONSERVER_Server object that this model is being served /// by. /// /// \param model The model. /// \param server Returns the server. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelServer( TRITONBACKEND_Model* model, TRITONSERVER_Server** server); /// Get the backend used by the model. /// /// \param model The model. /// \param model Returns the backend object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelBackend( TRITONBACKEND_Model* model, TRITONBACKEND_Backend** backend); /// Get the user-specified state associated with the model. The /// state is completely owned and managed by the backend. /// /// \param model The model. /// \param state Returns the user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelState( TRITONBACKEND_Model* model, void** state); /// Set the user-specified state associated with the model. The /// state is completely owned and managed by the backend. /// /// \param model The model. /// \param state The user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelSetState( TRITONBACKEND_Model* model, void* state); /// Report the memory usage of the model that will be released on /// TRITONBACKEND_ModelFinalize. The backend may call this function within the /// lifecycle of the TRITONBACKEND_Model object (between /// TRITONBACKEND_ModelInitialize and TRITONBACKEND_ModelFinalize) to report the /// latest usage. To report the memory usage of a model instance, /// see TRITONBACKEND_ModelInstanceReportMemoryUsage. /// /// \param model The model. /// \param usage The list of buffer attributes that records the memory usage, /// each entry should record the total memory usage of a given memory type and /// id. For example, if the model itself occupies 64 bytes on each of /// CUDA device 0 and CUDA device 1. Then 'usage' should have first two entries /// set, one has the buffer attributes of "type GPU, id 0, 64 bytes" and the /// other has "type GPU, id 1, 64 bytes". 'usage' is owned by the backend and /// may be released after the function returns. /// \param usage_size The number of entries in 'usage'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelReportMemoryUsage( TRITONBACKEND_Model* model, TRITONSERVER_BufferAttributes** usage, uint32_t usage_size); /// /// TRITONBACKEND_ModelInstance /// /// Object representing a model instance implemented using the /// backend. /// /// Get the name of the model instance. The returned string is owned by the /// model object, not the caller, and so should not be modified or /// freed. /// /// \param instance The model instance. /// \param name Returns the instance name. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceName( TRITONBACKEND_ModelInstance* instance, const char** name); /// Get the kind of the model instance. /// /// \param instance The model instance. /// \param kind Returns the instance kind. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceKind( TRITONBACKEND_ModelInstance* instance, TRITONSERVER_InstanceGroupKind* kind); /// Get the device ID of the model instance. /// /// \param instance The model instance. /// \param device_id Returns the instance device ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceDeviceId( TRITONBACKEND_ModelInstance* instance, int32_t* device_id); /// Get the host policy setting. The 'host_policy' message is /// owned by Triton and should not be modified or freed by the caller. /// /// The host policy setting, as JSON, is: /// /// { /// "" : { /// "" : "", /// ... /// } /// } /// /// \param instance The model instance. /// \param host_policy Returns the host policy setting as a message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceHostPolicy( TRITONBACKEND_ModelInstance* instance, TRITONSERVER_Message** host_policy); /// Whether the model instance is passive. /// /// \param instance The model instance. /// \param is_passive Returns true if the instance is passive, false otherwise /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceIsPassive( TRITONBACKEND_ModelInstance* instance, bool* is_passive); /// Get the number of optimization profiles to be loaded for the instance. /// /// \param instance The model instance. /// \param count Returns the number of optimization profiles. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceProfileCount( TRITONBACKEND_ModelInstance* instance, uint32_t* count); /// Get the name of optimization profile. The caller does not own /// the returned string and must not modify or delete it. The lifetime /// of the returned string extends only as long as 'instance'. /// /// \param instance The model instance. /// \param index The index of the optimization profile. Must be 0 /// <= index < count, where count is the value returned by /// TRITONBACKEND_ModelInstanceProfileCount. /// \param profile_name Returns the name of the optimization profile /// corresponding to the index. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceProfileName( TRITONBACKEND_ModelInstance* instance, const uint32_t index, const char** profile_name); /// Get the number of secondary devices configured for the instance. /// /// \param instance The model instance. /// \param count Returns the number of secondary devices. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceSecondaryDeviceCount( TRITONBACKEND_ModelInstance* instance, uint32_t* count); /// Get the properties of indexed secondary device. The returned /// strings and other properties are owned by the instance, not the /// caller, and so should not be modified or freed. /// /// \param instance The model instance. /// \param index The index of the secondary device. Must be 0 /// <= index < count, where count is the value returned by /// TRITONBACKEND_ModelInstanceSecondaryDeviceCount. /// \param kind Returns the kind of secondary device corresponding /// to the index. /// \param id Returns the id of secondary device corresponding to the index. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( TRITONBACKEND_ModelInstance* instance, uint32_t index, const char** kind, int64_t* id); /// Get the model associated with a model instance. /// /// \param instance The model instance. /// \param backend Returns the model object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceModel( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Model** model); /// Get the user-specified state associated with the model /// instance. The state is completely owned and managed by the /// backend. /// /// \param instance The model instance. /// \param state Returns the user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceState( TRITONBACKEND_ModelInstance* instance, void** state); /// Set the user-specified state associated with the model /// instance. The state is completely owned and managed by the /// backend. /// /// \param instance The model instance. /// \param state The user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceSetState( TRITONBACKEND_ModelInstance* instance, void* state); /// Report the memory usage of the model instance that will be released on /// TRITONBACKEND_ModelInstanceFinalize. The backend may call this function /// within the lifecycle of the TRITONBACKEND_Model object (between /// TRITONBACKEND_ModelInstanceInitialize and /// TRITONBACKEND_ModelInstanceFinalize) to report the latest usage. To report /// the memory usage of the model, see TRITONBACKEND_ModelReportMemoryUsage. /// /// \param instance The model instance. /// \param usage The list of buffer attributes that records the memory usage, /// each entry should record the total memory usage of a given memory type and /// id. For example, if the instance itself occupies 64 bytes on each of /// CUDA device 0 and CUDA device 1. Then 'usage' should have first two entries /// set, one has the buffer attributes of "type GPU, id 0, 64 bytes" and the /// other has "type GPU, id 1, 64 bytes". 'usage' is owned by the backend and /// may be released after the function returns. /// \param usage_size The number of entries in 'usage'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceReportMemoryUsage( TRITONBACKEND_ModelInstance* instance, TRITONSERVER_BufferAttributes** usage, uint32_t usage_size); /// Record statistics for an inference request. /// /// Set 'success' true to indicate that the inference request /// completed successfully. In this case all timestamps should be /// non-zero values reported in nanoseconds and should be collected /// using std::chrono::steady_clock::now().time_since_epoch() or the equivalent. /// Set 'success' to false to indicate that the inference request failed /// to complete successfully. In this case all timestamps values are /// ignored. /// /// For consistency of measurement across different backends, the /// timestamps should be collected at the following points during /// TRITONBACKEND_ModelInstanceExecute. /// /// TRITONBACKEND_ModelInstanceExecute() /// CAPTURE TIMESTAMP (exec_start_ns) /// < process input tensors to prepare them for inference /// execution, including copying the tensors to/from GPU if /// necessary> /// CAPTURE TIMESTAMP (compute_start_ns) /// < perform inference computations to produce outputs > /// CAPTURE TIMESTAMP (compute_end_ns) /// < allocate output buffers and extract output tensors, including /// copying the tensors to/from GPU if necessary> /// CAPTURE TIMESTAMP (exec_end_ns) /// return /// /// Note that these statistics are associated with a valid /// TRITONBACKEND_Request object and so must be reported before the /// request is released. For backends that release the request before /// all response(s) are sent, these statistics cannot capture /// information about the time required to produce the response. /// /// \param instance The model instance. /// \param request The inference request that statistics are being /// reported for. /// \param success True if the inference request completed /// successfully, false if it failed to complete. /// \param exec_start_ns Timestamp for the start of execution. /// \param compute_start_ns Timestamp for the start of execution /// computations. /// \param compute_end_ns Timestamp for the end of execution /// computations. /// \param exec_end_ns Timestamp for the end of execution. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceReportStatistics( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request* request, const bool success, const uint64_t exec_start_ns, const uint64_t compute_start_ns, const uint64_t compute_end_ns, const uint64_t exec_end_ns); /// Create a new inference response statistics object. /// /// \param response_statistics The new response statistics object to be created. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsNew( TRITONBACKEND_ModelInstanceResponseStatistics** response_statistics); /// Delete an inference response statistics object. /// /// The caller retains ownership to the objects set on the deleted response /// statistics object and must free them separately. /// /// \param response_statistics The response statistics object to be deleted. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsDelete( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics); /// Set model instance to an inference response statistics object. /// /// \param response_statistics The response statistics object. /// \param model_instance The model instance. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetModelInstance( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, TRITONBACKEND_ModelInstance* model_instance); /// Set response factory to an inference response statistics object. /// /// \param response_statistics The response statistics object. /// \param response_factory The response factory. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseFactory( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, TRITONBACKEND_ResponseFactory* response_factory); /// Set response start time to an inference response statistics object. /// /// All timestamps should be reported in nanonseconds and collected using /// std::chrono::steady_clock::now().time_since_epoch() or the equivalent. /// /// For consistency of measurement across different backends, the timestamps /// should be collected at the following points during /// TRITONBACKEND_ModelInstanceExecute. /// /// TRITONBACKEND_ModelInstanceExecute() /// < start of this response > /// CAPTURE TIMESTAMP (response_start) /// < generate this response > /// CAPTURE TIMESTAMP (compute_output_start) /// < allocate output buffers and extract output tensors, including copying /// the tensors to/from GPU if necessary > /// CAPTURE TIMESTAMP (response_end) /// < end of this response > /// return /// /// \param response_statistics The response statistics object. /// \param response_start The response start time. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseStart( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, uint64_t response_start); /// Set compute output start time to an inference response statistics object. /// /// Do NOT set this compute output start time (or set it to 0), if reporting an /// empty response. /// /// All timestamps should be reported in nanonseconds and collected using /// std::chrono::steady_clock::now().time_since_epoch() or the equivalent. /// /// For consistency of measurement across different backends, the timestamps /// should be collected at the following points during /// TRITONBACKEND_ModelInstanceExecute. /// /// TRITONBACKEND_ModelInstanceExecute() /// < start of this response > /// CAPTURE TIMESTAMP (response_start) /// < generate this response > /// CAPTURE TIMESTAMP (compute_output_start) /// < allocate output buffers and extract output tensors, including copying /// the tensors to/from GPU if necessary > /// CAPTURE TIMESTAMP (response_end) /// < end of this response > /// return /// /// \param response_statistics The response statistics object. /// \param compute_output_start The compute output start time. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetComputeOutputStart( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, uint64_t compute_output_start); /// Set response end time to an inference response statistics object. /// /// All timestamps should be reported in nanonseconds and collected using /// std::chrono::steady_clock::now().time_since_epoch() or the equivalent. /// /// For consistency of measurement across different backends, the timestamps /// should be collected at the following points during /// TRITONBACKEND_ModelInstanceExecute. /// /// TRITONBACKEND_ModelInstanceExecute() /// < start of this response > /// CAPTURE TIMESTAMP (response_start) /// < generate this response > /// CAPTURE TIMESTAMP (compute_output_start) /// < allocate output buffers and extract output tensors, including copying /// the tensors to/from GPU if necessary > /// CAPTURE TIMESTAMP (response_end) /// < end of this response > /// return /// /// \param response_statistics The response statistics object. /// \param response_end The response end time. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseEnd( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, uint64_t response_end); /// Set error to an inference response statistics object. /// /// Use the same error object passed to the TRITONBACKEND_ResponseSend. \see /// TRITONBACKEND_ResponseSend. /// /// \param response_statistics The response statistics object. /// \param error The error object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetError( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, TRITONSERVER_Error* error); /// Record statistics for an inference response. /// /// The caller retains ownership to the response statistics and must free it /// after this function returns. /// /// \param response_statistics The statistics to be recorded. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceReportResponseStatistics( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics); /// Record statistics for the execution of an entire batch of /// inference requests. /// /// All timestamps should be non-zero values reported in nanoseconds /// and should be collected using /// std::chrono::steady_clock::now().time_since_epoch() or the equivalent. /// See TRITONBACKEND_ModelInstanceReportStatistics for more information about /// the timestamps. /// /// 'batch_size' is the sum of the batch sizes for the individual /// requests that were delivered together in the call to /// TRITONBACKEND_ModelInstanceExecute. For example, if three requests /// are passed to TRITONBACKEND_ModelInstanceExecute and those /// requests have batch size 1, 2, and 3; then 'batch_size' should be /// set to 6. /// /// \param instance The model instance. /// \param batch_size Combined batch size of all the individual /// requests executed in the batch. /// \param exec_start_ns Timestamp for the start of execution. /// \param compute_start_ns Timestamp for the start of execution /// computations. /// \param compute_end_ns Timestamp for the end of execution /// computations. /// \param exec_end_ns Timestamp for the end of execution. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceReportBatchStatistics( TRITONBACKEND_ModelInstance* instance, const uint64_t batch_size, const uint64_t exec_start_ns, const uint64_t compute_start_ns, const uint64_t compute_end_ns, const uint64_t exec_end_ns); /// /// The following functions can be implemented by a backend. Functions /// indicated as required must be implemented or the backend will fail /// to load. /// /// Initialize a backend. This function is optional, a backend is not /// required to implement it. This function is called once when a /// backend is loaded to allow the backend to initialize any state /// associated with the backend. A backend has a single state that is /// shared across all models that use the backend. /// /// \param backend The backend. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Initialize( TRITONBACKEND_Backend* backend); /// Finalize for a backend. This function is optional, a backend is /// not required to implement it. This function is called once, just /// before the backend is unloaded. All state associated with the /// backend should be freed and any threads created for the backend /// should be exited/joined before returning from this function. /// /// \param backend The backend. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Finalize( TRITONBACKEND_Backend* backend); /// Initialize for a model. This function is optional, a backend is /// not required to implement it. This function is called once when a /// model that uses the backend is loaded to allow the backend to /// initialize any state associated with the model. The backend should /// also examine the model configuration to determine if the /// configuration is suitable for the backend. Any errors reported by /// this function will prevent the model from loading. /// /// \param model The model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInitialize( TRITONBACKEND_Model* model); /// Finalize for a model. This function is optional, a backend is not /// required to implement it. This function is called once for a /// model, just before the model is unloaded from Triton. All state /// associated with the model should be freed and any threads created /// for the model should be exited/joined before returning from this /// function. /// /// \param model The model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelFinalize( TRITONBACKEND_Model* model); /// Initialize for a model instance. This function is optional, a /// backend is not required to implement it. This function is called /// once when a model instance is created to allow the backend to /// initialize any state associated with the instance. /// /// \param instance The model instance. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( TRITONBACKEND_ModelInstance* instance); /// Finalize for a model instance. This function is optional, a /// backend is not required to implement it. This function is called /// once for an instance, just before the corresponding model is /// unloaded from Triton. All state associated with the instance /// should be freed and any threads created for the instance should be /// exited/joined before returning from this function. /// /// \param instance The model instance. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( TRITONBACKEND_ModelInstance* instance); /// Execute a batch of one or more requests on a model instance. This /// function is required. Triton will not perform multiple /// simultaneous calls to this function for a given model 'instance'; /// however, there may be simultaneous calls for different model /// instances (for the same or different models). /// /// If an error is returned the ownership of the request objects /// remains with Triton and the backend must not retain references to /// the request objects or access them in any way. /// /// If success is returned, ownership of the request objects is /// transferred to the backend and it is then responsible for creating /// responses and releasing the request objects. Note that even though /// ownership of the request objects is transferred to the backend, the /// ownership of the buffer holding request pointers is returned back /// to Triton upon return from TRITONBACKEND_ModelInstanceExecute. If /// any request objects need to be maintained beyond /// TRITONBACKEND_ModelInstanceExecute, then the pointers must be copied /// out of the array within TRITONBACKEND_ModelInstanceExecute. /// /// \param instance The model instance. /// \param requests The requests. /// \param request_count The number of requests in the batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count); /// Query the backend for different model attributes. This function is optional, /// a backend is not required to implement it. The backend is also not required /// to set all backend attribute listed. This function is called when /// Triton requires further backend / model information to perform operations. /// This function may be called multiple times within the lifetime of the /// backend (between TRITONBACKEND_Initialize and TRITONBACKEND_Finalize). /// The backend may return error to indicate failure to set the backend /// attributes, and the attributes specified in the same function call will be /// ignored. Triton will update the specified attributes if 'nullptr' is /// returned. /// /// \param backend The backend. /// \param backend_attributes Return the backend attribute. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_GetBackendAttribute( TRITONBACKEND_Backend* backend, TRITONBACKEND_BackendAttribute* backend_attributes); /// TRITONBACKEND_BackendAttribute /// /// API to modify attributes associated with a backend. /// /// Add the preferred instance group of the backend. This function /// can be called multiple times to cover different instance group kinds that /// the backend supports, given the priority order that the first call describes /// the most preferred group. In the case where instance group are not /// explicitly provided, Triton will use this attribute to create model /// deployment that aligns more with the backend preference. /// /// \param backend_attributes The backend attributes object. /// \param kind The kind of the instance group. /// \param count The number of instances per device. Triton default will be used /// if 0 is provided. /// \param device_ids The devices where instances should be available. Triton /// default will be used if 'nullptr' is provided. /// \param id_count The number of devices in 'device_ids'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( TRITONBACKEND_BackendAttribute* backend_attributes, const TRITONSERVER_InstanceGroupKind kind, const uint64_t count, const uint64_t* device_ids, const uint64_t id_count); /// Sets whether or not the backend supports concurrently loading multiple /// TRITONBACKEND_ModelInstances in a thread-safe manner. /// /// Most backends are thread-safe for parallel execution of model instances as /// that is the primary use of concurrency in backends. However, not all /// backends are thread-safe when initializing or finalizing model instances. In /// order for Triton to know that it can safely load instances concurrently, the /// backend needs to opt-in by setting this backend attribute to true. By /// default, this attribute is false and calls to the /// TRITONBACKEND_ModelInstanceInitialize function will be made serially. If /// this attribute is set to true, then Triton will make calls to /// TRITONBACKEND_ModelInstanceInitialize concurrently. /// /// \param backend_attributes The backend attributes object. /// \param enabled Whether or not the backend supports loading model instances /// in parallel. TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendAttributeSetParallelModelInstanceLoading( TRITONBACKEND_BackendAttribute* backend_attributes, bool enabled); /// TRITONBACKEND Batching /// /// API to add custom batching strategy /// /// The following functions can be implemented by a backend to add custom /// batching conditionals on top of the existing Triton batching strategy. The /// functions are optional but all or none must be implemented. /// /// Create a new batcher for use with custom batching. This is called during /// model loading. The batcher will point to a user-defined data structure that /// holds read-only data used for custom batching. /// /// \param batcher User-defined placeholder for backend to store and /// retrieve information about the batching strategy for this /// model. Returns a TRITONSERVER_Error indicating success /// or failure. \param model The backend model for which Triton is forming a /// batch. \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelBatcherInitialize( TRITONBACKEND_Batcher** batcher, TRITONBACKEND_Model* model); /// Free memory associated with batcher. This is called during model unloading. /// /// \param batcher User-defined placeholder for backend to store and /// retrieve information about the batching strategy for this model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelBatcherFinalize( TRITONBACKEND_Batcher* batcher); /// Check whether a request should be added to the pending model batch. /// /// \param request The request to be added to the pending batch. /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. When the callback returns, this should reflect /// the latest batch information. /// \param should_include The pointer to be updated on whether the request /// should be included in the batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelBatchIncludeRequest( TRITONBACKEND_Request* request, void* userp, bool* should_include); /// Callback to be invoked when Triton has begun forming a batch. /// /// \param batcher The read-only placeholder for backend to retrieve // information about the batching strategy for this model. /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelBatchInitialize( const TRITONBACKEND_Batcher* batcher, void** userp); /// Callback to be invoked when Triton has finishing forming a batch. /// /// \param userp The placeholder for backend to store and retrieve information /// about this pending batch. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelBatchFinalize( void* userp); /// Get all information about an output tensor by its name. The caller does /// not own any of the referenced return values and must not modify or delete /// them. The lifetime of all returned values extends until 'response' is /// deleted. /// /// \param response The response object. /// \param name The name of the output. /// \param datatype Returns the type of the output. /// \param shape Returns the shape of the output. /// \param dim_count Returns the number of dimensions of the returned /// shape. /// \return a TRITONSERVER_Error indicating success or failure. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_InferenceResponseOutputByName( TRITONBACKEND_Response* response, const char* name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint64_t* dim_count); /// Get all information about an output tensor by its index. The caller does /// not own any of the referenced return values and must not modify or delete /// them. The lifetime of all returned values extends until 'response' is /// deleted. /// /// \param response The response object. /// \param index The index of the output tensor, must be 0 <= index < /// count, where 'count' is the value returned by /// TRITONSERVER_InferenceResponseOutputCount. /// \param name Returns the name of the output. /// \param datatype Returns the type of the output. /// \param shape Returns the shape of the output. /// \param dim_count Returns the number of dimensions of the returned /// shape. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InferenceResponseOutput( TRITONBACKEND_Response* response, const uint32_t index, const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint64_t* dim_count); #ifdef __cplusplus } #endif kytensor-2.49.0.7/core/include/triton/core/tritoncache.h000066400000000000000000000333331527052371200231400ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "triton/core/tritonserver.h" #ifdef __cplusplus extern "C" { #endif #ifdef _COMPILING_TRITONCACHE #if defined(_MSC_VER) #define TRITONCACHE_DECLSPEC __declspec(dllexport) #define TRITONCACHE_ISPEC __declspec(dllimport) #elif defined(__GNUC__) #define TRITONCACHE_DECLSPEC __attribute__((__visibility__("default"))) #define TRITONCACHE_ISPEC #else #define TRITONCACHE_DECLSPEC #define TRITONCACHE_ISPEC #endif #else #if defined(_MSC_VER) #define TRITONCACHE_DECLSPEC __declspec(dllimport) #define TRITONCACHE_ISPEC __declspec(dllexport) #else #define TRITONCACHE_DECLSPEC #define TRITONCACHE_ISPEC #endif #endif struct TRITONCACHE_Cache; struct TRITONCACHE_CacheEntry; struct TRITONCACHE_Allocator; /// /// TRITONCACHE API Version /// /// The TRITONCACHE API is versioned with major and minor version /// numbers. Any change to the API that does not impact backwards /// compatibility (for example, adding a non-required function) /// increases the minor version number. Any change that breaks /// backwards compatibility (for example, deleting or changing the /// behavior of a function) increases the major version number. A /// cache implementation should check that the API version used to compile /// the cache is compatible with the API version of the Triton server /// that it is running in. This is typically done by code similar to /// the following which makes sure that the major versions are equal /// and that the minor version of Triton is >= the minor version used /// to build the cache. /// /// uint32_t api_version_major, api_version_minor; /// TRITONCACHE_ApiVersion(&api_version_major, &api_version_minor); /// if ((api_version_major != TRITONCACHE_API_VERSION_MAJOR) || /// (api_version_minor < TRITONCACHE_API_VERSION_MINOR)) { /// return TRITONSERVER_ErrorNew( /// TRITONSERVER_ERROR_UNSUPPORTED, /// "triton cache API version does not support this cache"); /// } /// #define TRITONCACHE_API_VERSION_MAJOR 0 #define TRITONCACHE_API_VERSION_MINOR 1 /// Get the TRITONCACHE API version supported by Triton. This /// value can be compared against the /// TRITONCACHE_API_VERSION_MAJOR and /// TRITONCACHE_API_VERSION_MINOR used to build the cache to /// ensure that Triton is compatible with the cache. /// /// \param major Returns the TRITONCACHE API major version supported /// by Triton. /// \param minor Returns the TRITONCACHE API minor version supported /// by Triton. /// \return a TRITONSERVER_Error indicating success or failure. TRITONCACHE_DECLSPEC TRITONSERVER_Error* TRITONCACHE_ApiVersion( uint32_t* major, uint32_t* minor); /// Get the number of buffers held by entry /// /// \param entry The CacheEntry object to query. /// \param count Returns the number of buffers in entry. /// \return a TRITONSERVER_Error indicating success or failure. TRITONCACHE_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntryBufferCount( TRITONCACHE_CacheEntry* entry, size_t* count); /// Adds buffer to entry. /// /// NOTE: (DLIS-2673) Only buffers in CPU memory supported currently. /// /// \param entry The CacheEntry object to add buffer to. /// \param base The base address of the buffer to add. /// \param buffer_attributes The buffer attributes associated with the buffer. /// The caller must create the buffer attributes object, and set the relevant /// fields through the BufferAttributes API. /// \return a TRITONSERVER_Error indicating success or failure. TRITONCACHE_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntryAddBuffer( TRITONCACHE_CacheEntry* entry, void* base, TRITONSERVER_BufferAttributes* buffer_attributes); /// Gets the buffer at index from entry. /// /// The caller does not own the returned buffer and must not modify or delete /// it. The lifetime of the buffer extends until 'entry' is deleted. If the /// buffer needs to persist long term, the caller should make a copy. /// /// NOTE: Currently in the context of Triton, this API is used for the cache /// implementation to access the buffers from the opaque entry object passed by /// Triton in TRITONCACHE_CacheInsert. It is expected that the cache /// will get the buffer, and perform any necessary copy within the /// TRITONCACHE_CacheInsert implementation. After TRITONCACHE_CacheInsert /// returns, there is no guarantee that Triton won't delete the entry holding /// the buffer. This is also why the caller is expected to create and own the /// BufferAttributes object, as a copy would be needed otherwise anyway. /// /// \param entry The CacheEntry object owning the buffer. /// \param index The index of the buffer, must be 0 <= index < count, where /// 'count' is the value returned by TRITONCACHE_CacheEntryBufferCount. /// \param base The base address of the buffer at index that is returned. /// \param buffer_attributes The buffer attributes associated with the buffer. /// The caller must create the buffer attributes object, then Triton will /// internally set the relevant fields on this object through the /// BufferAttributes API. /// \return a TRITONSERVER_Error indicating success or failure. TRITONCACHE_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntryGetBuffer( TRITONCACHE_CacheEntry* entry, size_t index, void** base, TRITONSERVER_BufferAttributes* buffer_attributes); /// Sets the buffer at index in entry. /// /// The entry does not own the buffer and will generally not modify or delete /// it. /// /// NOTE: Currently in the context of Triton, this API is used to allow the /// cache implementation to allocate/provide buffers for Triton to copy /// directly into in TRITONCACHE_CacheInsert to avoid intermediate copies. /// /// \param entry The CacheEntry object owning the buffer. /// \param index The index of the buffer, must be 0 <= index < count, where /// 'count' is the value returned by TRITONCACHE_CacheEntryBufferCount. /// \param base The base address of the new buffer to set at index. /// \param buffer_attributes (optional) buffer attributes associated with the /// buffer to overwrite existing attributes. If the entry already has a buffer /// with the same attributes, there is no need to provide a new one. /// \return a TRITONSERVER_Error indicating success or failure. TRITONCACHE_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntrySetBuffer( TRITONCACHE_CacheEntry* entry, size_t index, void* new_base, TRITONSERVER_BufferAttributes* buffer_attributes); /// Callback that can be used with a custom allocator to prepare buffers, /// and copy to/from the entry. /// /// For example: /// TRITONCACHE_CacheLookup: /// The cache can provide cache-allocated buffers directly in the entry /// object, and can use this callback + allocator to allocate buffers /// on the Triton side that require some cache metadata before allocation /// (ex: size of cached data). After, Triton can copy directly from the /// cache buffers into the Triton-allocated buffers. /// TRITONCACHE_CacheInsert: /// The cache can provide cache-allocated buffers directly in the entry /// object through the TRITONCACHE_CacheEntrySetBuffer API, then use this /// callback to copy buffers from Triton into the cache-allocated buffers. /// /// \param allocator Allocator that prepares buffers to copy to or from. /// \param entry The entry containing buffers and buffer attributes to copy from TRITONCACHE_DECLSPEC TRITONSERVER_Error* TRITONCACHE_Copy( TRITONCACHE_Allocator* allocator, TRITONCACHE_CacheEntry* entry); /// /// The following functions can be implemented by a cache. Functions /// indicated as required must be implemented or the cache will fail /// to load. /// /// Initialize a new cache object. /// /// This function is required to be implemented by the cache. /// /// The caller takes ownership of the /// TRITONCACHE_Cache object and must call /// TRITONCACHE_CacheFinalize to cleanup and release the object. /// /// This API is implemented by the user-provided cache implementation, /// so specific details will be found within the cache implementation. /// /// \param cache Returns the new cache object. /// \param config The config options to initialize the cache with. /// This will be passed as-is to the cache implementation, so /// the expected format and parsing is up to the cache as well. /// \return a TRITONSERVER_Error indicating success or failure. TRITONCACHE_ISPEC TRITONSERVER_Error* TRITONCACHE_CacheInitialize( TRITONCACHE_Cache** cache, const char* config); /// Cleanup a cache object. /// /// This function is required to be implemented by the cache. /// /// This API is implemented by the user-provided cache implementation, /// so specific details will be found within the cache implementation. /// /// \param cache The cache object to delete. /// \return a TRITONSERVER_Error indicating success or failure. TRITONCACHE_ISPEC TRITONSERVER_Error* TRITONCACHE_CacheFinalize( TRITONCACHE_Cache* cache); /// Inserts entry into cache at specified key. Typically this will fail /// if the key already exists, but a cache implementation may decide to allow /// overwriting entries for existing keys. /// /// This function is required to be implemented by the cache. /// /// This API is implemented by the user-provided cache implementation, /// so specific details will be found within the cache implementation. /// /// \param cache The object that is used to communicate with the cache /// implementation through a shared library. /// \param key The key used to access the cache. Generally, this is some /// unique value or hash representing the entry to avoid collisions. /// \param entry The entry to be inserted into the cache. /// \param allocator TritonCacheAllocator that is used to copy data directly /// into cache-provided buffers to avoid intermediate copies. /// The cache implementation expects that the entry will hold /// the requested buffer sizes. Then the cache implementation /// is expected to set the buffer addresses in the entry to /// the cache-allocated buffers through the /// TRITONCACHE_CacheEntrySetBuffer API. /// \return a TRITONSERVER_Error indicating success or failure. /// Specific errors will be up the cache implementation, but general /// error best practices that should be followed are as follows: /// - TRITONSERVER_ERROR_INVALID_ARG /// - bad argument passed, nullptr, etc. /// - TRITONSERVER_ERROR_ALREADY_EXISTS /// - key already exists and will not be inserted again /// - TRITONSERVER_ERROR_INTERNAL /// - internal errors /// - nullptr /// - success TRITONCACHE_ISPEC TRITONSERVER_Error* TRITONCACHE_CacheInsert( TRITONCACHE_Cache* cache, const char* key, TRITONCACHE_CacheEntry* entry, TRITONCACHE_Allocator* allocator); /// Retrieves entry from cache at specified key, if key exists. /// /// This function is required to be implemented by the cache. /// /// This API is implemented by the user-provided cache implementation, /// so specific details will be found within the cache implementation. /// /// \param cache The object that is used to communicate with the cache /// implementation through a shared library. /// \param key The key used to access the cache. Generally, this is some /// unique value or hash representing the entry to avoid collisions. /// \param entry The entry to be retrieved from the cache. /// \param allocator TritonCacheAllocator that is used to copy cache data into /// into user provided buffers to avoid intermediate copies. /// \return a TRITONSERVER_Error indicating success or failure. /// Specific errors will be up the cache implementation, but general /// error best practices that should be followed are as follows: /// - TRITONSERVER_ERROR_INVALID_ARG /// - bad argument passed, nullptr, etc. /// - TRITONSERVER_ERROR_NOT_FOUND /// - key not found in cache /// - TRITONSERVER_ERROR_INTERNAL /// - other internal errors /// - nullptr /// - success TRITONCACHE_ISPEC TRITONSERVER_Error* TRITONCACHE_CacheLookup( TRITONCACHE_Cache* cache, const char* key, TRITONCACHE_CacheEntry* entry, TRITONCACHE_Allocator* allocator); #ifdef __cplusplus } // extern C #endif kytensor-2.49.0.7/core/include/triton/core/tritonrepoagent.h000066400000000000000000000460261527052371200240640ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "triton/core/tritonserver.h" #ifdef __cplusplus extern "C" { #endif #ifdef _COMPILING_TRITONREPOAGENT #if defined(_MSC_VER) #define TRITONREPOAGENT_DECLSPEC __declspec(dllexport) #define TRITONREPOAGENT_ISPEC __declspec(dllimport) #elif defined(__GNUC__) #define TRITONREPOAGENT_DECLSPEC __attribute__((__visibility__("default"))) #define TRITONREPOAGENT_ISPEC #else #define TRITONREPOAGENT_DECLSPEC #define TRITONREPOAGENT_ISPEC #endif #else #if defined(_MSC_VER) #define TRITONREPOAGENT_DECLSPEC __declspec(dllimport) #define TRITONREPOAGENT_ISPEC __declspec(dllexport) #else #define TRITONREPOAGENT_DECLSPEC #define TRITONREPOAGENT_ISPEC #endif #endif struct TRITONREPOAGENT_Agent; struct TRITONREPOAGENT_AgentModel; /// /// TRITONREPOAGENT API Version /// /// The TRITONREPOAGENT API is versioned with major and minor version /// numbers. Any change to the API that does not impact backwards /// compatibility (for example, adding a non-required function) /// increases the minor version number. Any change that breaks /// backwards compatibility (for example, deleting or changing the /// behavior of a function) increases the major version number. A /// repository agent should check that the API version used to compile /// the agent is compatible with the API version of the Triton server /// that it is running in. This is typically done by code similar to /// the following which makes sure that the major versions are equal /// and that the minor version of Triton is >= the minor version used /// to build the agent. /// /// uint32_t api_version_major, api_version_minor; /// TRITONREPOAGENT_ApiVersion(&api_version_major, &api_version_minor); /// if ((api_version_major != TRITONREPOAGENT_API_VERSION_MAJOR) || /// (api_version_minor < TRITONREPOAGENT_API_VERSION_MINOR)) { /// return TRITONSERVER_ErrorNew( /// TRITONSERVER_ERROR_UNSUPPORTED, /// "triton repository agent API version does not support this agent"); /// } /// #define TRITONREPOAGENT_API_VERSION_MAJOR 0 #define TRITONREPOAGENT_API_VERSION_MINOR 1 /// Get the TRITONREPOAGENT API version supported by Triton. This /// value can be compared against the /// TRITONREPOAGENT_API_VERSION_MAJOR and /// TRITONREPOAGENT_API_VERSION_MINOR used to build the agent to /// ensure that Triton is compatible with the agent. /// /// \param major Returns the TRITONREPOAGENT API major version supported /// by Triton. /// \param minor Returns the TRITONREPOAGENT API minor version supported /// by Triton. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ApiVersion( uint32_t* major, uint32_t* minor); /// TRITONREPOAGENT_ArtifactType /// /// The ways that the files that make up a model's repository content /// are communicated between Triton and the agent. /// /// TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are /// communicated to and from the repository agent via a locally /// accessible filesystem. The agent can access these files using /// an appropriate filesystem API. /// /// TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are /// communicated to and from the repository agent via a remote filesystem. /// The remote filesystem path follows the same convention as is used for /// repository paths, for example, "s3://" prefix indicates an S3 path. /// typedef enum TRITONREPOAGENT_artifacttype_enum { TRITONREPOAGENT_ARTIFACT_FILESYSTEM, TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM } TRITONREPOAGENT_ArtifactType; /// TRITONREPOAGENT_ActionType /// /// Types of repository actions that can be handled by an agent. /// The lifecycle of a TRITONREPOAGENT_AgentModel begins with a call to /// TRITONREPOAGENT_ModelInitialize and ends with a call to /// TRITONREPOAGENT_ModelFinalize. Between those calls the current lifecycle /// state of the model is communicated by calls to TRITONREPOAGENT_ModelAction. /// Possible lifecycles are: /// /// LOAD -> LOAD_COMPLETE -> UNLOAD -> UNLOAD_COMPLETE /// LOAD -> LOAD_FAIL /// /// TRITONREPOAGENT_ACTION_LOAD: A model is being loaded. /// /// TRITONREPOAGENT_ACTION_LOAD_COMPLETE: The model load completed /// successfully and the model is now loaded. /// /// TRITONREPOAGENT_ACTION_LOAD_FAIL: The model load did not complete /// successfully. The model is not loaded. /// /// TRITONREPOAGENT_ACTION_UNLOAD: The model is being unloaded. /// /// TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: The model unload is complete. /// typedef enum TRITONREPOAGENT_actiontype_enum { TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_COMPLETE, TRITONREPOAGENT_ACTION_LOAD_FAIL, TRITONREPOAGENT_ACTION_UNLOAD, TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE } TRITONREPOAGENT_ActionType; /// Get the location of the files that make up the model. The /// 'location' communicated depends on how the model is being /// communicated to the agent as indicated by 'artifact_type'. /// /// TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are /// made available to the agent via the local /// filesystem. 'location' returns the full path to the directory /// in the model repository that contains the model's /// artifacts. The returned location string is owned by Triton, /// not the caller, and so should not be modified or freed. The /// contents of the directory are owned by Triton, not the agent, /// and so the agent should not delete or modify the contents. Use /// TRITONREPOAGENT_RepositoryAcquire to get a location that can be /// used to modify the model repository contents. /// /// TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are /// made available to the agent via a remote filesystem. /// 'location' returns the full path to the remote directory that contains /// the model's artifacts. The returned location string is owned by Triton, /// not the caller, and so should not be modified or freed. The contents of /// the remote directory are owned by Triton, not the agent, /// and so the agent should not delete or modify the contents. /// Use TRITONREPOAGENT_ModelRepositoryLocationAcquire to get a location /// that can be used to write updated model repository contents. /// /// \param agent The agent. /// \param model The model. /// \param artifact_type Returns the artifact type for the location. /// \param path Returns the location. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelRepositoryLocation( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, TRITONREPOAGENT_ArtifactType* artifact_type, const char** location); /// Acquire a location where the agent can produce a new version of /// the model repository files. This is a convenience method to create /// a temporary directory for the agent. The agent is responsible for /// calling TRITONREPOAGENT_ModelRepositoryLocationDelete in /// TRITONREPOAGENT_ModelFinalize to delete the location. Initially the /// acquired location is empty. The 'location' communicated depends on /// the requested 'artifact_type'. /// /// TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The location is a directory /// on the local filesystem. 'location' returns the full path to /// an empty directory that the agent should populate with the /// model's artifacts. The returned location string is owned by /// Triton, not the agent, and so should not be modified or freed. /// /// \param agent The agent. /// \param model The model. /// \param artifact_type The artifact type for the location. /// \param path Returns the location. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelRepositoryLocationAcquire( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ArtifactType artifact_type, const char** location); /// Discard and release ownership of a previously acquired location /// and its contents. The agent must not access or modify the location /// or its contents after this call. /// /// \param agent The agent. /// \param model The model. /// \param path The location to release. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelRepositoryLocationRelease( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const char* location); /// Inform Triton that the specified repository location should be used for /// the model in place of the original model repository. This method can only be /// called when TRITONREPOAGENT_ModelAction is invoked with /// TRITONREPOAGENT_ACTION_LOAD. The 'location' The 'location' /// communicated depends on how the repository is being /// communicated to Triton as indicated by 'artifact_type'. /// /// TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are /// made available to Triton via the local filesystem. 'location' returns /// the full path to the directory. Ownership of the contents of the /// returned directory are transferred to Triton and the agent should not /// modified or freed the contents until TRITONREPOAGENT_ModelFinalize. /// The local filesystem directory can be created using /// TRITONREPOAGENT_ModelReopsitroyLocationAcquire or the agent can use /// its own local filesystem API. /// /// TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are /// made available to Triton via a remote filesystem. 'location' returns /// the full path to the remote filesystem directory. Ownership of the /// contents of the returned directory are transferred to Triton and /// the agent should not modified or freed the contents until /// TRITONREPOAGENT_ModelFinalize. /// /// \param agent The agent. /// \param model The model. /// \param artifact_type The artifact type for the location. /// \param path Returns the location. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelRepositoryUpdate( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ArtifactType artifact_type, const char* location); /// Get the number of agent parameters defined for a model. /// /// \param agent The agent. /// \param model The model. /// \param count Returns the number of input tensors. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelParameterCount( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, uint32_t* count); /// Get a parameter name and value. The caller does not own the /// returned strings and must not modify or delete them. /// /// \param agent The agent. /// \param model The model. /// \param index The index of the parameter. Must be 0 <= index < /// count, where count is the value returned by /// TRITONREPOAGENT_ModelParameterCount. /// \param parameter_name Returns the name of the parameter. /// \param parameter_value Returns the value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelParameter( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const uint32_t index, const char** parameter_name, const char** parameter_value); /// Get the model configuration. The caller takes ownership of the /// message object and must call TRITONSERVER_MessageDelete to release /// the object. If the model repository does not contain a /// config.pbtxt file then 'model_config' is returned as nullptr. /// /// \param agent The agent. /// \param model The model. /// \param config_version The model configuration will be returned in /// a format matching this version. If the configuration cannot be /// represented in the requested version's format then an error will /// be returned. Currently only version 1 is supported. /// \param model_config Returns the model configuration as a message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelConfig( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const uint32_t config_version, TRITONSERVER_Message** model_config); /// Get the user-specified state associated with the model. /// /// \param model The agent model. /// \param state Returns the user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelState( TRITONREPOAGENT_AgentModel* model, void** state); /// Set the user-specified state associated with the model. /// /// \param model The agent model. /// \param state The user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelSetState( TRITONREPOAGENT_AgentModel* model, void* state); /// Get the user-specified state associated with the agent. /// /// \param agent The agent. /// \param state Returns the user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_State( TRITONREPOAGENT_Agent* agent, void** state); /// Set the user-specified state associated with the agent. /// /// \param agent The agent. /// \param state The user state, or nullptr if no user state. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_SetState( TRITONREPOAGENT_Agent* agent, void* state); /// /// The following functions can be implemented by an agent. Functions /// indicated as required must be implemented or the agent will fail /// to load. /// /// Initialize an agent. This function is optional. This function is /// called once when an agent is loaded to allow the agent to /// initialize any state associated with the agent. An agent has a /// single state that is shared across all invocations of the agent. /// /// \param agent The agent. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_Initialize( TRITONREPOAGENT_Agent* agent); /// Finalize for an agent. This function is optional. This function is /// called once, just before the agent is unloaded. All state /// associated with the agent should be freed and any threads created /// for the agent should be exited/joined before returning from this /// function. /// /// \param agent The agent. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_Finalize( TRITONREPOAGENT_Agent* agent); /// Initialize a model associated with an agent. This function is optional. /// This function is called once when an agent model's lifecycle begins to allow /// the agent model to initialize any state associated with it. An agent model /// has a single state that is shared across all the lifecycle of the agent /// model. /// /// \param agent The agent to be associated with the model. /// \param model The model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelInitialize( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model); /// Finalize for a model. This function is optional. This function is /// called once, just before the end of the agent model's lifecycle. All state /// associated with the agent model should be freed and any threads created /// for the agent model should be exited/joined before returning from this /// function. If the model acquired a model location using /// TRITONREPOAGENT_ModelRepositoryLocationAcquire, it must call /// TRITONREPOAGENT_ModelRepositoryLocationRelease to release that location. /// /// \param agent The agent associated with the model. /// \param model The model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelFinalize( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model); /// Handle an action for a specified model. This function is /// required. Triton will not perform multiple simultaneous calls to /// this function for a given agent and model; however, there may be /// simultaneous calls for the agent for different models. /// /// If the agent does not handle the action the agent should /// immediately return success (nullptr). /// /// Any modification to the model's repository must be made when 'action_type' /// is TRITONREPOAGENT_ACTION_LOAD. /// To modify the model's repository the agent must either acquire a mutable /// location via TRITONREPOAGENT_ModelRepositoryLocationAcquire /// or its own managed location, report the location to Triton via /// TRITONREPOAGENT_ModelRepositoryUpdate, and then return /// success (nullptr). If the agent does not need to make any changes /// to the model repository it should not call /// TRITONREPOAGENT_ModelRepositoryUpdate and then return success. /// To indicate that a model load should fail return a non-success status. /// /// \param agent The agent. /// \param model The model that is the target of the action. /// \action_type The type of action the agent should handle for the model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelAction( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type); #ifdef __cplusplus } #endif kytensor-2.49.0.7/core/include/triton/core/tritonserver.h000066400000000000000000003670141527052371200234110ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once /// \file #include #include #include #ifdef __cplusplus extern "C" { #endif #ifdef _COMPILING_TRITONSERVER #if defined(_MSC_VER) #define TRITONSERVER_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONSERVER_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONSERVER_DECLSPEC #endif #else #if defined(_MSC_VER) #define TRITONSERVER_DECLSPEC __declspec(dllimport) #else #define TRITONSERVER_DECLSPEC #endif #endif struct TRITONSERVER_BufferAttributes; struct TRITONSERVER_Error; struct TRITONSERVER_InferenceRequest; struct TRITONSERVER_InferenceResponse; struct TRITONSERVER_InferenceTrace; struct TRITONSERVER_Message; struct TRITONSERVER_Metrics; struct TRITONSERVER_Parameter; struct TRITONSERVER_ResponseAllocator; struct TRITONSERVER_Server; struct TRITONSERVER_ServerOptions; struct TRITONSERVER_Metric; struct TRITONSERVER_MetricFamily; struct TRITONSERVER_MetricArgs; /// /// TRITONSERVER API Version /// /// The TRITONSERVER API is versioned with major and minor version /// numbers. Any change to the API that does not impact backwards /// compatibility (for example, adding a non-required function) /// increases the minor version number. Any change that breaks /// backwards compatibility (for example, deleting or changing the /// behavior of a function) increases the major version number. A /// client should check that the API version used to compile the /// client is compatible with the API version of the Triton shared /// library that it is linking against. This is typically done by code /// similar to the following which makes sure that the major versions /// are equal and that the minor version of the Triton shared library /// is >= the minor version used to build the client. /// /// uint32_t api_version_major, api_version_minor; /// TRITONSERVER_ApiVersion(&api_version_major, &api_version_minor); /// if ((api_version_major != TRITONSERVER_API_VERSION_MAJOR) || /// (api_version_minor < TRITONSERVER_API_VERSION_MINOR)) { /// return TRITONSERVER_ErrorNew( /// TRITONSERVER_ERROR_UNSUPPORTED, /// "triton server API version does not support this client"); /// } /// #define TRITONSERVER_API_VERSION_MAJOR 1 #define TRITONSERVER_API_VERSION_MINOR 34 /// Get the TRITONBACKEND API version supported by the Triton shared /// library. This value can be compared against the /// TRITONSERVER_API_VERSION_MAJOR and TRITONSERVER_API_VERSION_MINOR /// used to build the client to ensure that Triton shared library is /// compatible with the client. /// /// \param major Returns the TRITONSERVER API major version supported /// by Triton. /// \param minor Returns the TRITONSERVER API minor version supported /// by Triton. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ApiVersion( uint32_t* major, uint32_t* minor); /// TRITONSERVER_DataType /// /// Tensor data types recognized by TRITONSERVER. /// typedef enum TRITONSERVER_datatype_enum { TRITONSERVER_TYPE_INVALID, TRITONSERVER_TYPE_BOOL, TRITONSERVER_TYPE_UINT8, TRITONSERVER_TYPE_UINT16, TRITONSERVER_TYPE_UINT32, TRITONSERVER_TYPE_UINT64, TRITONSERVER_TYPE_INT8, TRITONSERVER_TYPE_INT16, TRITONSERVER_TYPE_INT32, TRITONSERVER_TYPE_INT64, TRITONSERVER_TYPE_FP16, TRITONSERVER_TYPE_FP32, TRITONSERVER_TYPE_FP64, TRITONSERVER_TYPE_BYTES, TRITONSERVER_TYPE_BF16 } TRITONSERVER_DataType; /// Get the string representation of a data type. The returned string /// is not owned by the caller and so should not be modified or freed. /// /// \param datatype The data type. /// \return The string representation of the data type. TRITONSERVER_DECLSPEC const char* TRITONSERVER_DataTypeString( TRITONSERVER_DataType datatype); /// Get the Triton datatype corresponding to a string representation /// of a datatype. /// /// \param dtype The datatype string representation. /// \return The Triton data type or TRITONSERVER_TYPE_INVALID if the /// string does not represent a data type. TRITONSERVER_DECLSPEC TRITONSERVER_DataType TRITONSERVER_StringToDataType(const char* dtype); /// Get the size of a Triton datatype in bytes. Zero is returned for /// TRITONSERVER_TYPE_BYTES because it have variable size. Zero is /// returned for TRITONSERVER_TYPE_INVALID. /// /// \param dtype The datatype. /// \return The size of the datatype. TRITONSERVER_DECLSPEC uint32_t TRITONSERVER_DataTypeByteSize(TRITONSERVER_DataType datatype); /// TRITONSERVER_MemoryType /// /// Types of memory recognized by TRITONSERVER. /// typedef enum TRITONSERVER_memorytype_enum { TRITONSERVER_MEMORY_CPU, TRITONSERVER_MEMORY_CPU_PINNED, TRITONSERVER_MEMORY_GPU } TRITONSERVER_MemoryType; /// Get the string representation of a memory type. The returned /// string is not owned by the caller and so should not be modified or /// freed. /// /// \param memtype The memory type. /// \return The string representation of the memory type. TRITONSERVER_DECLSPEC const char* TRITONSERVER_MemoryTypeString( TRITONSERVER_MemoryType memtype); /// TRITONSERVER_ParameterType /// /// Types of parameters recognized by TRITONSERVER. /// typedef enum TRITONSERVER_parametertype_enum { TRITONSERVER_PARAMETER_STRING, TRITONSERVER_PARAMETER_INT, TRITONSERVER_PARAMETER_BOOL, TRITONSERVER_PARAMETER_DOUBLE, TRITONSERVER_PARAMETER_BYTES } TRITONSERVER_ParameterType; /// Get the string representation of a parameter type. The returned /// string is not owned by the caller and so should not be modified or /// freed. /// /// \param paramtype The parameter type. /// \return The string representation of the parameter type. TRITONSERVER_DECLSPEC const char* TRITONSERVER_ParameterTypeString( TRITONSERVER_ParameterType paramtype); /// Create a new parameter object. The caller takes ownership of the /// TRITONSERVER_Parameter object and must call TRITONSERVER_ParameterDelete to /// release the object. The object will maintain its own copy of the 'value' /// /// \param name The parameter name. /// \param type The parameter type. /// \param value The pointer to the value. /// \return A new TRITONSERVER_Parameter object. 'nullptr' will be returned if /// 'type' is 'TRITONSERVER_PARAMETER_BYTES'. The caller should use /// TRITONSERVER_ParameterBytesNew to create parameter with bytes type. TRITONSERVER_DECLSPEC struct TRITONSERVER_Parameter* TRITONSERVER_ParameterNew( const char* name, const TRITONSERVER_ParameterType type, const void* value); /// Create a new parameter object with type TRITONSERVER_PARAMETER_BYTES. /// The caller takes ownership of the TRITONSERVER_Parameter object and must /// call TRITONSERVER_ParameterDelete to release the object. The object only /// maintains a shallow copy of the 'byte_ptr' so the data content must be /// valid until the parameter object is deleted. /// /// \param name The parameter name. /// \param byte_ptr The pointer to the data content. /// \param size The size of the data content. /// \return A new TRITONSERVER_Error object. TRITONSERVER_DECLSPEC struct TRITONSERVER_Parameter* TRITONSERVER_ParameterBytesNew( const char* name, const void* byte_ptr, const uint64_t size); /// Delete an parameter object. /// /// \param parameter The parameter object. TRITONSERVER_DECLSPEC void TRITONSERVER_ParameterDelete( struct TRITONSERVER_Parameter* parameter); /// TRITONSERVER_InstanceGroupKind /// /// Kinds of instance groups recognized by TRITONSERVER. /// typedef enum TRITONSERVER_instancegroupkind_enum { TRITONSERVER_INSTANCEGROUPKIND_AUTO, TRITONSERVER_INSTANCEGROUPKIND_CPU, TRITONSERVER_INSTANCEGROUPKIND_GPU, TRITONSERVER_INSTANCEGROUPKIND_MODEL } TRITONSERVER_InstanceGroupKind; /// Get the string representation of an instance-group kind. The /// returned string is not owned by the caller and so should not be /// modified or freed. /// /// \param kind The instance-group kind. /// \return The string representation of the kind. TRITONSERVER_DECLSPEC const char* TRITONSERVER_InstanceGroupKindString( TRITONSERVER_InstanceGroupKind kind); /// TRITONSERVER_Logging /// /// Types/levels of logging. /// typedef enum TRITONSERVER_loglevel_enum { TRITONSERVER_LOG_INFO, TRITONSERVER_LOG_WARN, TRITONSERVER_LOG_ERROR, TRITONSERVER_LOG_VERBOSE } TRITONSERVER_LogLevel; /// Logging Formats /// /// The TRITONSERVER API offers two logging formats. The formats have /// a common set of fields but differ in how the timestamp for a log /// entry is represented. Messages are serialized according to JSON /// encoding rules by default. This behavior can be disabled by /// setting the environment variable TRITON_SERVER_ESCAPE_LOG_MESSAGES /// to "0". /// /// /// 1. TRITONSERVER_LOG_DEFAULT /// /// ::. :] /// /// Example: /// /// I0520 20:03:25.829575 3355 model_lifecycle.cc:441] "AsyncLoad() 'simple'" /// /// 2. TRITONSERVER_LOG_ISO8601 /// /// --T::Z :] /// /// Example: /// /// 2024-05-20T20:03:26Z I 3415 model_lifecycle.cc:441] "AsyncLoad() 'simple'" /// typedef enum TRITONSERVER_logformat_enum { TRITONSERVER_LOG_DEFAULT, TRITONSERVER_LOG_ISO8601 } TRITONSERVER_LogFormat; /// Is a log level enabled? /// /// \param level The log level. /// \return True if the log level is enabled, false if not enabled. TRITONSERVER_DECLSPEC bool TRITONSERVER_LogIsEnabled( TRITONSERVER_LogLevel level); /// Log a message at a given log level if that level is enabled. /// /// \param level The log level. /// \param filename The file name of the location of the log message. /// \param line The line number of the log message. /// \param msg The log message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_LogMessage( TRITONSERVER_LogLevel level, const char* filename, const int line, const char* msg); /// TRITONSERVER_Error /// /// Errors are reported by a TRITONSERVER_Error object. A NULL /// TRITONSERVER_Error indicates no error, a non-NULL TRITONSERVER_Error /// indicates error and the code and message for the error can be /// retrieved from the object. /// /// The caller takes ownership of a TRITONSERVER_Error object returned by /// the API and must call TRITONSERVER_ErrorDelete to release the object. /// /// The TRITONSERVER_Error error codes typedef enum TRITONSERVER_errorcode_enum { TRITONSERVER_ERROR_UNKNOWN, TRITONSERVER_ERROR_INTERNAL, TRITONSERVER_ERROR_NOT_FOUND, TRITONSERVER_ERROR_INVALID_ARG, TRITONSERVER_ERROR_UNAVAILABLE, TRITONSERVER_ERROR_UNSUPPORTED, TRITONSERVER_ERROR_ALREADY_EXISTS, TRITONSERVER_ERROR_CANCELLED } TRITONSERVER_Error_Code; /// Create a new error object. The caller takes ownership of the /// TRITONSERVER_Error object and must call TRITONSERVER_ErrorDelete to /// release the object. /// /// \param code The error code. /// \param msg The error message. /// \return A new TRITONSERVER_Error object. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ErrorNew( TRITONSERVER_Error_Code code, const char* msg); /// Delete an error object. /// /// \param error The error object. TRITONSERVER_DECLSPEC void TRITONSERVER_ErrorDelete( struct TRITONSERVER_Error* error); /// Get the error code. /// /// \param error The error object. /// \return The error code. TRITONSERVER_DECLSPEC TRITONSERVER_Error_Code TRITONSERVER_ErrorCode(struct TRITONSERVER_Error* error); /// Get the string representation of an error code. The returned /// string is not owned by the caller and so should not be modified or /// freed. The lifetime of the returned string extends only as long as /// 'error' and must not be accessed once 'error' is deleted. /// /// \param error The error object. /// \return The string representation of the error code. TRITONSERVER_DECLSPEC const char* TRITONSERVER_ErrorCodeString( struct TRITONSERVER_Error* error); /// Get the error message. The returned string is not owned by the /// caller and so should not be modified or freed. The lifetime of the /// returned string extends only as long as 'error' and must not be /// accessed once 'error' is deleted. /// /// \param error The error object. /// \return The error message. TRITONSERVER_DECLSPEC const char* TRITONSERVER_ErrorMessage( struct TRITONSERVER_Error* error); /// TRITONSERVER_ResponseAllocator /// /// Object representing a memory allocator for output tensors in an /// inference response. /// /// Type for allocation function that allocates a buffer to hold an /// output tensor. /// /// \param allocator The allocator that is provided in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \param tensor_name The name of the output tensor to allocate for. /// \param byte_size The size of the buffer to allocate. /// \param memory_type The type of memory that the caller prefers for /// the buffer allocation. /// \param memory_type_id The ID of the memory that the caller prefers /// for the buffer allocation. /// \param userp The user data pointer that is provided as /// 'response_allocator_userp' in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \param buffer Returns a pointer to the allocated memory. /// \param buffer_userp Returns a user-specified value to associate /// with the buffer, or nullptr if no user-specified value should be /// associated with the buffer. This value will be provided in the /// call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer /// is released and will also be returned by /// TRITONSERVER_InferenceResponseOutput. /// \param actual_memory_type Returns the type of memory where the /// allocation resides. May be different than the type of memory /// requested by 'memory_type'. /// \param actual_memory_type_id Returns the ID of the memory where /// the allocation resides. May be different than the ID of the memory /// requested by 'memory_type_id'. /// \return a TRITONSERVER_Error object if a failure occurs while /// attempting an allocation. If an error is returned all other return /// values will be ignored. typedef struct TRITONSERVER_Error* (*TRITONSERVER_ResponseAllocatorAllocFn_t)( struct TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id); /// Type for allocation function that allocates a buffer to hold an /// output tensor with buffer attributes. The callback function must fill in the /// appropriate buffer attributes information related to this buffer. If set, /// this function is always called after TRITONSERVER_ResponseAllocatorAllocFn_t /// function. /// /// \param allocator The allocator that is provided in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \param tensor_name The name of the output tensor to allocate for. /// \param buffer_attributes The buffer attributes associated with the buffer. /// \param userp The user data pointer that is provided as /// 'response_allocator_userp' in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \param buffer_userp Returns a user-specified value to associate /// with the buffer, or nullptr if no user-specified value should be /// associated with the buffer. This value will be provided in the /// call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer /// is released and will also be returned by /// TRITONSERVER_InferenceResponseOutput. /// \return a TRITONSERVER_Error object if a failure occurs while /// attempting an allocation. If an error is returned all other return /// values will be ignored. typedef struct TRITONSERVER_Error* ( *TRITONSERVER_ResponseAllocatorBufferAttributesFn_t)( struct TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, struct TRITONSERVER_BufferAttributes* buffer_attributes, void* userp, void* buffer_userp); /// Type for function that is called to query the allocator's preferred memory /// type and memory type ID. As much as possible, the allocator should attempt /// to return the same memory_type and memory_type_id values that will be /// returned by the subsequent call to TRITONSERVER_ResponseAllocatorAllocFn_t. /// But the allocator is not required to do so. /// /// \param allocator The allocator that is provided in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \param userp The user data pointer that is provided as /// 'response_allocator_userp' in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \param tensor_name The name of the output tensor. This is optional /// and it should be set to nullptr to indicate that the tensor name has /// not determined. /// \param byte_size The expected size of the buffer. This is optional /// and it should be set to nullptr to indicate that the byte size has /// not determined. /// \param memory_type Acts as both input and output. On input gives /// the memory type preferred by the caller. Returns memory type preferred /// by the allocator, taken account of the caller preferred type. /// \param memory_type_id Acts as both input and output. On input gives /// the memory type ID preferred by the caller. Returns memory type ID preferred /// by the allocator, taken account of the caller preferred type ID. /// \return a TRITONSERVER_Error object if a failure occurs. typedef struct TRITONSERVER_Error* (*TRITONSERVER_ResponseAllocatorQueryFn_t)( struct TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); /// Type for function that is called when the server no longer holds /// any reference to a buffer allocated by /// TRITONSERVER_ResponseAllocatorAllocFn_t. In practice this function /// is typically called when the response object associated with the /// buffer is deleted by TRITONSERVER_InferenceResponseDelete. /// /// \param allocator The allocator that is provided in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \param buffer Pointer to the buffer to be freed. /// \param buffer_userp The user-specified value associated /// with the buffer in TRITONSERVER_ResponseAllocatorAllocFn_t. /// \param byte_size The size of the buffer. /// \param memory_type The type of memory holding the buffer. /// \param memory_type_id The ID of the memory holding the buffer. /// \return a TRITONSERVER_Error object if a failure occurs while /// attempting the release. If an error is returned Triton will not /// attempt to release the buffer again. typedef struct TRITONSERVER_Error* (*TRITONSERVER_ResponseAllocatorReleaseFn_t)( struct TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); /// Type for function that is called to indicate that subsequent /// allocation requests will refer to a new response. /// /// \param allocator The allocator that is provided in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \param userp The user data pointer that is provided as /// 'response_allocator_userp' in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// \return a TRITONSERVER_Error object if a failure occurs. typedef struct TRITONSERVER_Error* (*TRITONSERVER_ResponseAllocatorStartFn_t)( struct TRITONSERVER_ResponseAllocator* allocator, void* userp); /// Create a new response allocator object. /// /// The response allocator object is used by Triton to allocate /// buffers to hold the output tensors in inference responses. Most /// models generate a single response for each inference request /// (TRITONSERVER_TXN_ONE_TO_ONE). For these models the order of /// callbacks will be: /// /// TRITONSERVER_ServerInferAsync called /// - start_fn : optional (and typically not required) /// - alloc_fn : called once for each output tensor in response /// TRITONSERVER_InferenceResponseDelete called /// - release_fn: called once for each output tensor in response /// /// For models that generate multiple responses for each inference /// request (TRITONSERVER_TXN_DECOUPLED), the start_fn callback can be /// used to determine sets of alloc_fn callbacks that belong to the /// same response: /// /// TRITONSERVER_ServerInferAsync called /// - start_fn /// - alloc_fn : called once for each output tensor in response /// - start_fn /// - alloc_fn : called once for each output tensor in response /// ... /// For each response, TRITONSERVER_InferenceResponseDelete called /// - release_fn: called once for each output tensor in the response /// /// In all cases the start_fn, alloc_fn and release_fn callback /// functions must be thread-safe. Typically making these functions /// thread-safe does not require explicit locking. The recommended way /// to implement these functions is to have each inference request /// provide a 'response_allocator_userp' object that is unique to that /// request with TRITONSERVER_InferenceRequestSetResponseCallback. The /// callback functions then operate only on this unique state. Locking /// is required only when the callback function needs to access state /// that is shared across inference requests (for example, a common /// allocation pool). /// /// \param allocator Returns the new response allocator object. /// \param alloc_fn The function to call to allocate buffers for result /// tensors. /// \param release_fn The function to call when the server no longer /// holds a reference to an allocated buffer. /// \param start_fn The function to call to indicate that the /// subsequent 'alloc_fn' calls are for a new response. This callback /// is optional (use nullptr to indicate that it should not be /// invoked). /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorNew( struct TRITONSERVER_ResponseAllocator** allocator, TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, TRITONSERVER_ResponseAllocatorStartFn_t start_fn); /// Set the buffer attributes function for a response allocator object. /// The function will be called after alloc_fn to set the buffer attributes /// associated with the output buffer. /// /// The thread-safy requirement for buffer_attributes_fn is the same as other /// allocator callbacks. /// /// \param allocator The response allocator object. /// \param buffer_attributes_fn The function to call to get the buffer /// attributes information for an allocated buffer. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( struct TRITONSERVER_ResponseAllocator* allocator, TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn); /// Set the query function to a response allocator object. Usually the /// function will be called before alloc_fn to understand what is the /// allocator's preferred memory type and memory type ID at the current /// situation to make different execution decision. /// /// The thread-safy requirement for query_fn is the same as other allocator /// callbacks. /// /// \param allocator The response allocator object. /// \param query_fn The function to call to query allocator's preferred memory /// type and memory type ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorSetQueryFunction( struct TRITONSERVER_ResponseAllocator* allocator, TRITONSERVER_ResponseAllocatorQueryFn_t query_fn); /// Delete a response allocator. /// /// \param allocator The response allocator object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorDelete( struct TRITONSERVER_ResponseAllocator* allocator); /// TRITONSERVER_Message /// /// Object representing a Triton Server message. /// /// Create a new message object from serialized JSON string. /// /// \param message The message object. /// \param base The base of the serialized JSON. /// \param byte_size The size, in bytes, of the serialized message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MessageNewFromSerializedJson( struct TRITONSERVER_Message** message, const char* base, size_t byte_size); /// Delete a message object. /// /// \param message The message object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MessageDelete( struct TRITONSERVER_Message* message); /// Get the base and size of the buffer containing the serialized /// message in JSON format. The buffer is owned by the /// TRITONSERVER_Message object and should not be modified or freed by /// the caller. The lifetime of the buffer extends only as long as /// 'message' and must not be accessed once 'message' is deleted. /// /// \param message The message object. /// \param base Returns the base of the serialized message. /// \param byte_size Returns the size, in bytes, of the serialized /// message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MessageSerializeToJson( struct TRITONSERVER_Message* message, const char** base, size_t* byte_size); /// TRITONSERVER_Metrics /// /// Object representing metrics. /// /// Metric format types typedef enum tritonserver_metricformat_enum { TRITONSERVER_METRIC_PROMETHEUS } TRITONSERVER_MetricFormat; /// Delete a metrics object. /// /// \param metrics The metrics object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricsDelete( struct TRITONSERVER_Metrics* metrics); /// Get a buffer containing the metrics in the specified format. For /// each format the buffer contains the following: /// /// TRITONSERVER_METRIC_PROMETHEUS: 'base' points to a single multiline /// string (char*) that gives a text representation of the metrics in /// prometheus format. 'byte_size' returns the length of the string /// in bytes. /// /// The buffer is owned by the 'metrics' object and should not be /// modified or freed by the caller. The lifetime of the buffer /// extends only as long as 'metrics' and must not be accessed once /// 'metrics' is deleted. /// /// \param metrics The metrics object. /// \param format The format to use for the returned metrics. /// \param base Returns a pointer to the base of the formatted /// metrics, as described above. /// \param byte_size Returns the size, in bytes, of the formatted /// metrics. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricsFormatted( struct TRITONSERVER_Metrics* metrics, TRITONSERVER_MetricFormat format, const char** base, size_t* byte_size); /// TRITONSERVER_InferenceTrace /// /// Object that represents tracing for an inference request. /// /// Trace levels. The trace level controls the type of trace /// activities that are reported for an inference request. /// /// Trace level values are power-of-2 and can be combined to trace /// multiple types of activities. For example, use /// (TRITONSERVER_TRACE_LEVEL_TIMESTAMPS | /// TRITONSERVER_TRACE_LEVEL_TENSORS) to trace both timestamps and /// tensors for an inference request. /// /// TRITONSERVER_TRACE_LEVEL_MIN and TRITONSERVER_TRACE_LEVEL_MAX are /// deprecated and should not be used. typedef enum tritonserver_tracelevel_enum { /// Tracing disabled. No trace activities are reported. TRITONSERVER_TRACE_LEVEL_DISABLED = 0, /// Deprecated. Use TRITONSERVER_TRACE_LEVEL_TIMESTAMPS. TRITONSERVER_TRACE_LEVEL_MIN = 1, /// Deprecated. Use TRITONSERVER_TRACE_LEVEL_TIMESTAMPS. TRITONSERVER_TRACE_LEVEL_MAX = 2, /// Record timestamps for the inference request. TRITONSERVER_TRACE_LEVEL_TIMESTAMPS = 0x4, /// Record input and output tensor values for the inference request. TRITONSERVER_TRACE_LEVEL_TENSORS = 0x8 } TRITONSERVER_InferenceTraceLevel; /// Get the string representation of a trace level. The returned /// string is not owned by the caller and so should not be modified or /// freed. /// /// \param level The trace level. /// \return The string representation of the trace level. TRITONSERVER_DECLSPEC const char* TRITONSERVER_InferenceTraceLevelString( TRITONSERVER_InferenceTraceLevel level); /// Trace activities typedef enum tritonserver_traceactivity_enum { TRITONSERVER_TRACE_REQUEST_START = 0, TRITONSERVER_TRACE_QUEUE_START = 1, TRITONSERVER_TRACE_COMPUTE_START = 2, TRITONSERVER_TRACE_COMPUTE_INPUT_END = 3, TRITONSERVER_TRACE_COMPUTE_OUTPUT_START = 4, TRITONSERVER_TRACE_COMPUTE_END = 5, TRITONSERVER_TRACE_REQUEST_END = 6, TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT = 7, TRITONSERVER_TRACE_TENSOR_BACKEND_INPUT = 8, TRITONSERVER_TRACE_TENSOR_BACKEND_OUTPUT = 9, TRITONSERVER_TRACE_CUSTOM_ACTIVITY = 10 } TRITONSERVER_InferenceTraceActivity; /// Get the string representation of a trace activity. The returned /// string is not owned by the caller and so should not be modified or /// freed. /// /// \param activity The trace activity. /// \return The string representation of the trace activity. TRITONSERVER_DECLSPEC const char* TRITONSERVER_InferenceTraceActivityString( TRITONSERVER_InferenceTraceActivity activity); /// Type for trace timeline activity callback function. This callback function /// is used to report activity occurring for a trace. This function /// does not take ownership of 'trace' and so any information needed /// from that object must be copied before returning. The 'userp' data /// is the same as what is supplied in the call to /// TRITONSERVER_InferenceTraceNew. typedef void (*TRITONSERVER_InferenceTraceActivityFn_t)( struct TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, void* userp); /// Type for trace tensor activity callback function. This callback function /// is used to report tensor activity occurring for a trace. This function /// does not take ownership of 'trace' and so any information needed /// from that object must be copied before returning. The 'userp' data /// is the same as what is supplied in the call to /// TRITONSERVER_InferenceTraceTensorNew. typedef void (*TRITONSERVER_InferenceTraceTensorActivityFn_t)( struct TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, const char* name, TRITONSERVER_DataType datatype, const void* base, size_t byte_size, const int64_t* shape, uint64_t dim_count, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* userp); /// Type for trace release callback function. This callback function /// is called when all activity for the trace has completed. The /// callback function takes ownership of the /// TRITONSERVER_InferenceTrace object. The 'userp' data is the same /// as what is supplied in the call to TRITONSERVER_InferenceTraceNew. typedef void (*TRITONSERVER_InferenceTraceReleaseFn_t)( struct TRITONSERVER_InferenceTrace* trace, void* userp); /// Create a new inference trace object. The caller takes ownership of /// the TRITONSERVER_InferenceTrace object and must call /// TRITONSERVER_InferenceTraceDelete to release the object. /// /// The activity callback function will be called to report activity /// for 'trace' as well as for any child traces that are spawned by /// 'trace', and so the activity callback must check the trace object /// to determine specifically what activity is being reported. /// /// The release callback is called for both 'trace' and for any child /// traces spawned by 'trace'. /// /// \param trace Returns the new inference trace object. /// \param level The tracing level. /// \param parent_id The parent trace id for this trace. A value of 0 /// indicates that there is not parent trace. /// \param activity_fn The callback function where activity for the /// trace is reported. /// \param release_fn The callback function called when all activity /// is complete for the trace. /// \param trace_userp User-provided pointer that is delivered to /// the activity and release callback functions. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceNew( struct TRITONSERVER_InferenceTrace** trace, TRITONSERVER_InferenceTraceLevel level, uint64_t parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, TRITONSERVER_InferenceTraceReleaseFn_t release_fn, void* trace_userp); /// Create a new inference trace object. The caller takes ownership of /// the TRITONSERVER_InferenceTrace object and must call /// TRITONSERVER_InferenceTraceDelete to release the object. /// /// The timeline and tensor activity callback function will be called to report /// activity for 'trace' as well as for any child traces that are spawned by /// 'trace', and so the activity callback must check the trace object /// to determine specifically what activity is being reported. /// /// The release callback is called for both 'trace' and for any child /// traces spawned by 'trace'. /// /// \param trace Returns the new inference trace object. /// \param level The tracing level. /// \param parent_id The parent trace id for this trace. A value of 0 /// indicates that there is not parent trace. /// \param activity_fn The callback function where timeline activity for the /// trace is reported. /// \param tensor_activity_fn The callback function where tensor activity for /// the trace is reported. /// \param release_fn The callback function called when all activity /// is complete for the trace. /// \param trace_userp User-provided pointer that is delivered to /// the activity and release callback functions. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceTensorNew( struct TRITONSERVER_InferenceTrace** trace, TRITONSERVER_InferenceTraceLevel level, uint64_t parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn, TRITONSERVER_InferenceTraceReleaseFn_t release_fn, void* trace_userp); /// Report a trace activity. All the traces reported using this API will be /// using TRITONSERVER_TRACE_CUSTOM_ACTIVITY type. /// /// \param trace The trace object. /// \param timestamp The timestamp associated with the trace activity. /// \param name The trace activity name. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceReportActivity( TRITONSERVER_InferenceTrace* trace, uint64_t timestamp, const char* activity_name); /// Delete a trace object. /// /// \param trace The trace object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceDelete(struct TRITONSERVER_InferenceTrace* trace); /// Get the id associated with a trace. Every trace is assigned an id /// that is unique across all traces created for a Triton server. /// /// \param trace The trace. /// \param id Returns the id associated with the trace. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceId( struct TRITONSERVER_InferenceTrace* trace, uint64_t* id); /// Get the parent id associated with a trace. The parent id indicates /// a parent-child relationship between two traces. A parent id value /// of 0 indicates that there is no parent trace. /// /// \param trace The trace. /// \param id Returns the parent id associated with the trace. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceParentId( struct TRITONSERVER_InferenceTrace* trace, uint64_t* parent_id); /// Get the name of the model associated with a trace. The caller does /// not own the returned string and must not modify or delete it. The /// lifetime of the returned string extends only as long as 'trace'. /// /// \param trace The trace. /// \param model_name Returns the name of the model associated with /// the trace. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceModelName( struct TRITONSERVER_InferenceTrace* trace, const char** model_name); /// Get the version of the model associated with a trace. /// /// \param trace The trace. /// \param model_version Returns the version of the model associated /// with the trace. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceModelVersion( struct TRITONSERVER_InferenceTrace* trace, int64_t* model_version); /// Get the request id associated with a trace. The caller does /// not own the returned string and must not modify or delete it. The /// lifetime of the returned string extends only as long as 'trace'. /// /// \param trace The trace. /// \param request_id Returns the version of the model associated /// with the trace. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceRequestId( struct TRITONSERVER_InferenceTrace* trace, const char** request_id); /// Get the child trace, spawned from the parent trace. The caller owns /// the returned object and must call TRITONSERVER_InferenceTraceDelete /// to release the object, unless ownership is transferred through /// other APIs (see TRITONSERVER_ServerInferAsync). /// /// \param trace The trace. /// \param child_trace Returns the child trace, spawned from the trace. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceSpawnChildTrace( struct TRITONSERVER_InferenceTrace* trace, struct TRITONSERVER_InferenceTrace** child_trace); /// Set TRITONSERVER_InferenceTrace context. /// /// \param trace The trace. /// \param trace_context A new trace context to associate with the trace. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceSetContext( struct TRITONSERVER_InferenceTrace* trace, const char* trace_context); /// Get TRITONSERVER_InferenceTrace context. /// /// \param trace The trace. /// \param trace_context Returns the context associated with the trace. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceTraceContext( struct TRITONSERVER_InferenceTrace* trace, const char** trace_context); /// TRITONSERVER_InferenceRequest /// /// Object representing an inference request. The inference request /// provides the meta-data and input tensor values needed for an /// inference and returns the inference result meta-data and output /// tensors. An inference request object can be modified and reused /// multiple times. /// /// Inference request flags. The enum values must be power-of-2 values. typedef enum tritonserver_requestflag_enum { TRITONSERVER_REQUEST_FLAG_SEQUENCE_START = 1, TRITONSERVER_REQUEST_FLAG_SEQUENCE_END = 2 } TRITONSERVER_RequestFlag; /// Inference request release flags. The enum values must be /// power-of-2 values. typedef enum tritonserver_requestreleaseflag_enum { TRITONSERVER_REQUEST_RELEASE_ALL = 1, TRITONSERVER_REQUEST_RELEASE_RESCHEDULE = 2 } TRITONSERVER_RequestReleaseFlag; /// Inference response complete flags. The enum values must be /// power-of-2 values. typedef enum tritonserver_responsecompleteflag_enum { TRITONSERVER_RESPONSE_COMPLETE_FINAL = 1 } TRITONSERVER_ResponseCompleteFlag; /// Type for inference request release callback function. The callback /// indicates what type of release is being performed on the request /// and for some of these the callback function takes ownership of the /// TRITONSERVER_InferenceRequest object. The 'userp' data is the data /// provided as 'request_release_userp' in the call to /// TRITONSERVER_InferenceRequestSetReleaseCallback. /// /// One or more flags will be specified when the callback is invoked, /// and the callback must take the following actions: /// /// - TRITONSERVER_REQUEST_RELEASE_ALL: The entire inference request /// is being released and ownership is passed to the callback /// function. Triton will not longer access the 'request' object /// itself nor any input tensor data associated with the /// request. The callback should free or otherwise manage the /// 'request' object and all associated tensor data. /// - TRITONSERVER_REQUEST_RELEASE_RESCHEDULE: This flag is currently being /// consumed internally and the callback is not expected to receive nor /// process this kind of release. The backend will call /// TRITONBACKEND_RequestRelease with this flag when it wishes to reschedule /// the request back to the model. An example is that the model is /// recursively performing inference of the request and use the rescheduling /// to proceed the recursive execution. /// /// Note that currently TRITONSERVER_REQUEST_RELEASE_ALL should always /// be set when the callback is invoked but in the future that may /// change, so the callback should explicitly check for the flag /// before taking ownership of the request object. /// typedef void (*TRITONSERVER_InferenceRequestReleaseFn_t)( struct TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp); /// Type for callback function indicating that an inference response /// has completed. The callback function takes ownership of the /// TRITONSERVER_InferenceResponse object. The 'userp' data is the /// data provided as 'response_userp' in the call to /// TRITONSERVER_InferenceRequestSetResponseCallback. /// /// One or more flags may be specified when the callback is invoked: /// /// - TRITONSERVER_RESPONSE_COMPLETE_FINAL: Indicates that no more /// responses will be generated for a given request (more /// specifically, that no more responses will be generated for the /// inference request that set this callback and 'userp'). When /// this flag is set 'response' may be a response object or may be /// nullptr. If 'response' is not nullptr, then 'response' is the /// last response that Triton will produce for the request. If /// 'response' is nullptr then Triton is indicating that no more /// responses will be produced for the request. typedef void (*TRITONSERVER_InferenceResponseCompleteFn_t)( struct TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); /// Create a new inference request object. /// /// \param inference_request Returns the new request object. /// \param server the inference server object. /// \param model_name The name of the model to use for the request. /// \param model_version The version of the model to use for the /// request. If -1 then the server will choose a version based on the /// model's policy. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestNew( struct TRITONSERVER_InferenceRequest** inference_request, struct TRITONSERVER_Server* server, const char* model_name, const int64_t model_version); /// Delete an inference request object. /// /// \param inference_request The request object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestDelete( struct TRITONSERVER_InferenceRequest* inference_request); /// Get the ID for a request. The returned ID is owned by /// 'inference_request' and must not be modified or freed by the /// caller. /// /// \param inference_request The request object. /// \param id Returns the ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestId( struct TRITONSERVER_InferenceRequest* inference_request, const char** id); /// Set the ID for a request. /// /// \param inference_request The request object. /// \param id The ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetId( struct TRITONSERVER_InferenceRequest* inference_request, const char* id); /// Get the flag(s) associated with a request. On return 'flags' holds /// a bitwise-or of all flag values, see TRITONSERVER_RequestFlag for /// available flags. /// /// \param inference_request The request object. /// \param flags Returns the flags. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestFlags( struct TRITONSERVER_InferenceRequest* inference_request, uint32_t* flags); /// Set the flag(s) associated with a request. 'flags' should hold a /// bitwise-or of all flag values, see TRITONSERVER_RequestFlag for /// available flags. /// /// \param inference_request The request object. /// \param flags The flags. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetFlags( struct TRITONSERVER_InferenceRequest* inference_request, uint32_t flags); /// Get the correlation ID of the inference request as an unsigned integer. /// Default is 0, which indicates that the request has no correlation ID. /// If the correlation id associated with the inference request is a string, /// this function will return a failure. The correlation ID is used /// to indicate two or more inference request are related to each other. /// How this relationship is handled by the inference server is determined by /// the model's scheduling policy. /// /// \param inference_request The request object. /// \param correlation_id Returns the correlation ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestCorrelationId( struct TRITONSERVER_InferenceRequest* inference_request, uint64_t* correlation_id); /// Get the correlation ID of the inference request as a string. /// Default is empty "", which indicates that the request has no correlation ID. /// If the correlation id associated with the inference request is an unsigned /// integer, then this function will return a failure. The correlation ID /// is used to indicate two or more inference request are related to each other. /// How this relationship is handled by the inference server is determined by /// the model's scheduling policy. /// /// \param inference_request The request object. /// \param correlation_id Returns the correlation ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestCorrelationIdString( struct TRITONSERVER_InferenceRequest* inference_request, const char** correlation_id); /// Set the correlation ID of the inference request to be an unsigned integer. /// Default is 0, which indicates that the request has no correlation ID. /// The correlation ID is used to indicate two or more inference request /// are related to each other. How this relationship is handled by the /// inference server is determined by the model's scheduling policy. /// /// \param inference_request The request object. /// \param correlation_id The correlation ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetCorrelationId( struct TRITONSERVER_InferenceRequest* inference_request, uint64_t correlation_id); /// Set the correlation ID of the inference request to be a string. /// The correlation ID is used to indicate two or more inference /// request are related to each other. How this relationship is /// handled by the inference server is determined by the model's /// scheduling policy. /// /// \param inference_request The request object. /// \param correlation_id The correlation ID. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetCorrelationIdString( struct TRITONSERVER_InferenceRequest* inference_request, const char* correlation_id); /// Cancel an inference request. Requests are canceled on a best /// effort basis and no guarantee is provided that cancelling a /// request will result in early termination. Note that the /// inference request cancellation status will be reset after /// TRITONSERVER_InferAsync is run. This means that if you cancel /// the request before calling TRITONSERVER_InferAsync /// the request will not be cancelled. /// /// \param inference_request The request object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestCancel( struct TRITONSERVER_InferenceRequest* inference_request); /// Query whether the request is cancelled or not. /// /// If possible the backend should terminate any processing and /// send an error response with cancelled status. /// /// \param inference_request The request object. /// \param is_cancelled Returns whether the inference request is cancelled or /// not. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestIsCancelled( struct TRITONSERVER_InferenceRequest* inference_request, bool* is_cancelled); /// Deprecated. See TRITONSERVER_InferenceRequestPriorityUInt64 instead. /// /// Get the priority for a request. The default is 0 indicating that /// the request does not specify a priority and so will use the /// model's default priority. /// /// \param inference_request The request object. /// \param priority Returns the priority level. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestPriority( struct TRITONSERVER_InferenceRequest* inference_request, uint32_t* priority); /// Get the priority for a request. The default is 0 indicating that /// the request does not specify a priority and so will use the /// model's default priority. /// /// \param inference_request The request object. /// \param priority Returns the priority level. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestPriorityUInt64( struct TRITONSERVER_InferenceRequest* inference_request, uint64_t* priority); /// Deprecated. See TRITONSERVER_InferenceRequestSetPriorityUInt64 instead. /// /// Set the priority for a request. The default is 0 indicating that /// the request does not specify a priority and so will use the /// model's default priority. /// /// \param inference_request The request object. /// \param priority The priority level. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetPriority( struct TRITONSERVER_InferenceRequest* inference_request, uint32_t priority); /// Set the priority for a request. The default is 0 indicating that /// the request does not specify a priority and so will use the /// model's default priority. /// /// \param inference_request The request object. /// \param priority The priority level. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetPriorityUInt64( struct TRITONSERVER_InferenceRequest* inference_request, uint64_t priority); /// Get the timeout for a request, in microseconds. The default is 0 /// which indicates that the request has no timeout. /// /// \param inference_request The request object. /// \param timeout_us Returns the timeout, in microseconds. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestTimeoutMicroseconds( struct TRITONSERVER_InferenceRequest* inference_request, uint64_t* timeout_us); /// Set the timeout for a request, in microseconds. The default is 0 /// which indicates that the request has no timeout. /// /// \param inference_request The request object. /// \param timeout_us The timeout, in microseconds. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetTimeoutMicroseconds( struct TRITONSERVER_InferenceRequest* inference_request, uint64_t timeout_us); /// Add an input to a request. /// /// \param inference_request The request object. /// \param name The name of the input. /// \param datatype The type of the input. Valid type names are BOOL, /// UINT8, UINT16, UINT32, UINT64, INT8, INT16, INT32, INT64, FP16, /// FP32, FP64, and BYTES. /// \param shape The shape of the input. /// \param dim_count The number of dimensions of 'shape'. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestAddInput( struct TRITONSERVER_InferenceRequest* inference_request, const char* name, const TRITONSERVER_DataType datatype, const int64_t* shape, uint64_t dim_count); /// Add a raw input to a request. The name recognized by the model, data type /// and shape of the input will be deduced from model configuration. /// This function must be called at most once on request with no other input to /// ensure the deduction is accurate. /// /// \param inference_request The request object. /// \param name The name of the input. This name is only used as a reference /// of the raw input in other Tritonserver APIs. It doesn't associate with the /// name used in the model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestAddRawInput( struct TRITONSERVER_InferenceRequest* inference_request, const char* name); /// Remove an input from a request. /// /// \param inference_request The request object. /// \param name The name of the input. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveInput( struct TRITONSERVER_InferenceRequest* inference_request, const char* name); /// Remove all inputs from a request. /// /// \param inference_request The request object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveAllInputs( struct TRITONSERVER_InferenceRequest* inference_request); /// Assign a buffer of data to an input. The buffer will be appended /// to any existing buffers for that input. The 'inference_request' /// object takes ownership of the buffer and so the caller should not /// modify or free the buffer until that ownership is released by /// 'inference_request' being deleted or by the input being removed /// from 'inference_request'. /// /// \param inference_request The request object. /// \param name The name of the input. /// \param base The base address of the input data. /// \param byte_size The size, in bytes, of the input data. /// \param memory_type The memory type of the input data. /// \param memory_type_id The memory type id of the input data. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestAppendInputData( struct TRITONSERVER_InferenceRequest* inference_request, const char* name, const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); /// Assign a buffer of data to an input for execution on all model instances /// with the specified host policy. The buffer will be appended to any existing /// buffers for that input on all devices with this host policy. The /// 'inference_request' object takes ownership of the buffer and so the caller /// should not modify or free the buffer until that ownership is released by /// 'inference_request' being deleted or by the input being removed from /// 'inference_request'. If the execution is scheduled on a device that does not /// have a input buffer specified using this function, then the input buffer /// specified with TRITONSERVER_InferenceRequestAppendInputData will be used so /// a non-host policy specific version of data must be added using that API. /// \param inference_request The request object. /// \param name The name of the input. /// \param base The base address of the input data. /// \param byte_size The size, in bytes, of the input data. /// \param memory_type The memory type of the input data. /// \param memory_type_id The memory type id of the input data. /// \param host_policy_name All model instances executing with this host_policy /// will use this input buffer for execution. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( struct TRITONSERVER_InferenceRequest* inference_request, const char* name, const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, const char* host_policy_name); /// Assign a buffer of data to an input. The buffer will be appended /// to any existing buffers for that input. The 'inference_request' /// object takes ownership of the buffer and so the caller should not /// modify or free the buffer until that ownership is released by /// 'inference_request' being deleted or by the input being removed /// from 'inference_request'. /// /// \param inference_request The request object. /// \param name The name of the input. /// \param base The base address of the input data. /// \param buffer_attributes The buffer attributes of the input. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( struct TRITONSERVER_InferenceRequest* inference_request, const char* name, const void* base, struct TRITONSERVER_BufferAttributes* buffer_attributes); /// Clear all input data from an input, releasing ownership of the /// buffer(s) that were appended to the input with /// TRITONSERVER_InferenceRequestAppendInputData or /// TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy /// \param inference_request The request object. /// \param name The name of the input. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveAllInputData( struct TRITONSERVER_InferenceRequest* inference_request, const char* name); /// Add an output request to an inference request. /// /// \param inference_request The request object. /// \param name The name of the output. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestAddRequestedOutput( struct TRITONSERVER_InferenceRequest* inference_request, const char* name); /// Remove an output request from an inference request. /// /// \param inference_request The request object. /// \param name The name of the output. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveRequestedOutput( struct TRITONSERVER_InferenceRequest* inference_request, const char* name); /// Remove all output requests from an inference request. /// /// \param inference_request The request object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveAllRequestedOutputs( struct TRITONSERVER_InferenceRequest* inference_request); /// Set the release callback for an inference request. The release /// callback is called by Triton to return ownership of the request /// object. /// /// \param inference_request The request object. /// \param request_release_fn The function called to return ownership /// of the 'inference_request' object. /// \param request_release_userp User-provided pointer that is /// delivered to the 'request_release_fn' callback. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetReleaseCallback( struct TRITONSERVER_InferenceRequest* inference_request, TRITONSERVER_InferenceRequestReleaseFn_t request_release_fn, void* request_release_userp); /// Set the allocator and response callback for an inference /// request. The allocator is used to allocate buffers for any output /// tensors included in responses that are produced for this /// request. The response callback is called to return response /// objects representing responses produced for this request. /// Typically 'response_allocator_userp' and 'response_userp' will no /// longer be referenced after 'response_fn' is invoked with /// 'TRITONSERVER_RESPONSE_COMPLETE_FINAL' flag, therefore the user may /// release 'response_allocator_userp' and 'response_userp' at that point. /// /// \param inference_request The request object. /// \param response_allocator The TRITONSERVER_ResponseAllocator to use /// to allocate buffers to hold inference results. /// \param response_allocator_userp User-provided pointer that is /// delivered to the response allocator's start and allocation functions. /// \param response_fn The function called to deliver an inference /// response for this request. /// \param response_userp User-provided pointer that is delivered to /// the 'response_fn' callback. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetResponseCallback( struct TRITONSERVER_InferenceRequest* inference_request, struct TRITONSERVER_ResponseAllocator* response_allocator, void* response_allocator_userp, TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp); /// Set a string parameter in the request. /// /// \param request The request. /// \param key The name of the parameter. /// \param value The value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetStringParameter( struct TRITONSERVER_InferenceRequest* request, const char* key, const char* value); /// Set an integer parameter in the request. /// /// \param request The request. /// \param key The name of the parameter. /// \param value The value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetIntParameter( struct TRITONSERVER_InferenceRequest* request, const char* key, const int64_t value); /// Set a boolean parameter in the request. /// /// \param request The request. /// \param key The name of the parameter. /// \param value The value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetBoolParameter( struct TRITONSERVER_InferenceRequest* request, const char* key, const bool value); /// Set a double parameter in the request. /// /// \param request The request. /// \param key The name of the parameter. /// \param value The value of the parameter. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetDoubleParameter( struct TRITONSERVER_InferenceRequest* request, const char* key, const double value); /// TRITONSERVER_InferenceResponse /// /// Object representing an inference response. The inference response /// provides the meta-data and output tensor values calculated by the /// inference. /// /// Delete an inference response object. /// /// \param inference_response The response object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseDelete( struct TRITONSERVER_InferenceResponse* inference_response); /// Return the error status of an inference response. Return a /// TRITONSERVER_Error object on failure, return nullptr on success. /// The returned error object is owned by 'inference_response' and so /// should not be deleted by the caller. /// /// \param inference_response The response object. /// \return a TRITONSERVER_Error indicating the success or failure /// status of the response. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseError( struct TRITONSERVER_InferenceResponse* inference_response); /// Get model used to produce a response. The caller does not own the /// returned model name value and must not modify or delete it. The /// lifetime of all returned values extends until 'inference_response' /// is deleted. /// /// \param inference_response The response object. /// \param model_name Returns the name of the model. /// \param model_version Returns the version of the model. /// this response. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseModel( struct TRITONSERVER_InferenceResponse* inference_response, const char** model_name, int64_t* model_version); /// Get the ID of the request corresponding to a response. The caller /// does not own the returned ID and must not modify or delete it. The /// lifetime of all returned values extends until 'inference_response' /// is deleted. /// /// \param inference_response The response object. /// \param request_id Returns the ID of the request corresponding to /// this response. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseId( struct TRITONSERVER_InferenceResponse* inference_response, const char** request_id); /// Get the number of parameters available in the response. /// /// \param inference_response The response object. /// \param count Returns the number of parameters. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseParameterCount( struct TRITONSERVER_InferenceResponse* inference_response, uint32_t* count); /// Get all information about a parameter. The caller does not own any /// of the returned values and must not modify or delete them. The /// lifetime of all returned values extends until 'inference_response' /// is deleted. /// /// The 'vvalue' returns a void* pointer that must be cast /// appropriately based on 'type'. For example: /// /// void* vvalue; /// TRITONSERVER_ParameterType type; /// TRITONSERVER_InferenceResponseParameter( /// response, index, &name, &type, &vvalue); /// switch (type) { /// case TRITONSERVER_PARAMETER_BOOL: /// bool value = *(reinterpret_cast(vvalue)); /// ... /// case TRITONSERVER_PARAMETER_INT: /// int64_t value = *(reinterpret_cast(vvalue)); /// ... /// case TRITONSERVER_PARAMETER_STRING: /// const char* value = reinterpret_cast(vvalue); /// ... /// /// \param inference_response The response object. /// \param index The index of the parameter, must be 0 <= index < /// count, where 'count' is the value returned by /// TRITONSERVER_InferenceResponseParameterCount. /// \param name Returns the name of the parameter. /// \param type Returns the type of the parameter. /// \param vvalue Returns a pointer to the parameter value. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseParameter( struct TRITONSERVER_InferenceResponse* inference_response, const uint32_t index, const char** name, TRITONSERVER_ParameterType* type, const void** vvalue); /// Get the number of outputs available in the response. /// /// \param inference_response The response object. /// \param count Returns the number of output tensors. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseOutputCount( struct TRITONSERVER_InferenceResponse* inference_response, uint32_t* count); /// Get all information about an output tensor. The tensor data is /// returned as the base pointer to the data and the size, in bytes, /// of the data. The caller does not own any of the returned values /// and must not modify or delete them. The lifetime of all returned /// values extends until 'inference_response' is deleted. /// /// \param inference_response The response object. /// \param index The index of the output tensor, must be 0 <= index < /// count, where 'count' is the value returned by /// TRITONSERVER_InferenceResponseOutputCount. /// \param name Returns the name of the output. /// \param datatype Returns the type of the output. /// \param shape Returns the shape of the output. /// \param dim_count Returns the number of dimensions of the returned /// shape. /// \param base Returns the tensor data for the output. /// \param byte_size Returns the size, in bytes, of the data. /// \param memory_type Returns the memory type of the data. /// \param memory_type_id Returns the memory type id of the data. /// \param userp The user-specified value associated with the buffer /// in TRITONSERVER_ResponseAllocatorAllocFn_t. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseOutput( struct TRITONSERVER_InferenceResponse* inference_response, const uint32_t index, const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint64_t* dim_count, const void** base, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id, void** userp); /// Get a classification label associated with an output for a given /// index. The caller does not own the returned label and must not /// modify or delete it. The lifetime of all returned label extends /// until 'inference_response' is deleted. /// /// \param inference_response The response object. /// \param index The index of the output tensor, must be 0 <= index < /// count, where 'count' is the value returned by /// TRITONSERVER_InferenceResponseOutputCount. /// \param class_index The index of the class. /// \param name Returns the label corresponding to 'class_index' or /// nullptr if no label. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_InferenceResponseOutputClassificationLabel( struct TRITONSERVER_InferenceResponse* inference_response, const uint32_t index, const size_t class_index, const char** label); /// TRITONSERVER_BufferAttributes /// /// API to create, modify, or retrieve attributes associated with a buffer. /// /// Create a new buffer attributes object. The caller takes ownership of /// the TRITONSERVER_BufferAttributes object and must call /// TRITONSERVER_BufferAttributesDelete to release the object. /// /// \param buffer_attributes Returns the new buffer attributes object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesNew( struct TRITONSERVER_BufferAttributes** buffer_attributes); /// Delete a buffer attributes object. /// /// \param buffer_attributes The buffer_attributes object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesDelete( struct TRITONSERVER_BufferAttributes* buffer_attributes); /// Set the memory type id field of the buffer attributes. /// /// \param buffer_attributes The buffer attributes object. /// \param memory_type_id Memory type id to assign to the buffer attributes /// object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesSetMemoryTypeId( struct TRITONSERVER_BufferAttributes* buffer_attributes, int64_t memory_type_id); /// Set the memory type field of the buffer attributes. /// /// \param buffer_attributes The buffer attributes object. /// \param memory_type Memory type to assign to the buffer attributes object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesSetMemoryType( struct TRITONSERVER_BufferAttributes* buffer_attributes, TRITONSERVER_MemoryType memory_type); /// Set the CudaIpcHandle field of the buffer attributes. /// /// \param buffer_attributes The buffer attributes object. /// \param cuda_ipc_handle The CudaIpcHandle to assign to the buffer attributes /// object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesSetCudaIpcHandle( struct TRITONSERVER_BufferAttributes* buffer_attributes, void* cuda_ipc_handle); /// Set the byte size field of the buffer attributes. /// /// \param buffer_attributes The buffer attributes object. /// \param byte_size Byte size to assign to the buffer attributes object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesSetByteSize( struct TRITONSERVER_BufferAttributes* buffer_attributes, size_t byte_size); /// Get the memory type id field of the buffer attributes. /// /// \param buffer_attributes The buffer attributes object. /// \param memory_type_id Returns the memory type id associated with the buffer /// attributes object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesMemoryTypeId( struct TRITONSERVER_BufferAttributes* buffer_attributes, int64_t* memory_type_id); /// Get the memory type field of the buffer attributes. /// /// \param buffer_attributes The buffer attributes object. /// \param memory_type Returns the memory type associated with the buffer /// attributes object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesMemoryType( struct TRITONSERVER_BufferAttributes* buffer_attributes, TRITONSERVER_MemoryType* memory_type); /// Get the CudaIpcHandle field of the buffer attributes object. /// /// \param buffer_attributes The buffer attributes object. /// \param cuda_ipc_handle Returns the memory type associated with the buffer /// attributes object. If the cudaIpcHandle does not exist for the buffer, /// nullptr will be returned. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesCudaIpcHandle( struct TRITONSERVER_BufferAttributes* buffer_attributes, void** cuda_ipc_handle); /// Get the byte size field of the buffer attributes. /// /// \param buffer_attributes The buffer attributes object. /// \param byte_size Returns the byte size associated with the buffer attributes /// object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_BufferAttributesByteSize( struct TRITONSERVER_BufferAttributes* buffer_attributes, size_t* byte_size); /// TRITONSERVER_ServerOptions /// /// Options to use when creating an inference server. /// /// Model control modes typedef enum tritonserver_modelcontrolmode_enum { TRITONSERVER_MODEL_CONTROL_NONE, TRITONSERVER_MODEL_CONTROL_POLL, TRITONSERVER_MODEL_CONTROL_EXPLICIT } TRITONSERVER_ModelControlMode; /// Rate limit modes typedef enum tritonserver_ratelimitmode_enum { TRITONSERVER_RATE_LIMIT_OFF, TRITONSERVER_RATE_LIMIT_EXEC_COUNT } TRITONSERVER_RateLimitMode; /// Create a new server options object. The caller takes ownership of /// the TRITONSERVER_ServerOptions object and must call /// TRITONSERVER_ServerOptionsDelete to release the object. /// /// \param options Returns the new server options object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsNew( struct TRITONSERVER_ServerOptions** options); /// Delete a server options object. /// /// \param options The server options object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsDelete(struct TRITONSERVER_ServerOptions* options); /// Set the textual ID for the server in a server options. The ID is a /// name that identifies the server. /// /// \param options The server options object. /// \param server_id The server identifier. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetServerId( struct TRITONSERVER_ServerOptions* options, const char* server_id); /// Set the model repository path in a server options. The path must be /// the full absolute path to the model repository. This function can be called /// multiple times with different paths to set multiple model repositories. /// Note that if a model is not unique across all model repositories /// at any time, the model will not be available. /// /// \param options The server options object. /// \param model_repository_path The full path to the model repository. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelRepositoryPath( struct TRITONSERVER_ServerOptions* options, const char* model_repository_path); /// Set the model control mode in a server options. For each mode the models /// will be managed as the following: /// /// TRITONSERVER_MODEL_CONTROL_NONE: the models in model repository will be /// loaded on startup. After startup any changes to the model repository will /// be ignored. Calling TRITONSERVER_ServerPollModelRepository will result in /// an error. /// /// TRITONSERVER_MODEL_CONTROL_POLL: the models in model repository will be /// loaded on startup. The model repository can be polled periodically using /// TRITONSERVER_ServerPollModelRepository and the server will load, unload, /// and updated models according to changes in the model repository. /// /// TRITONSERVER_MODEL_CONTROL_EXPLICIT: the models in model repository will /// not be loaded on startup. The corresponding model control APIs must be /// called to load / unload a model in the model repository. /// /// \param options The server options object. /// \param mode The mode to use for the model control. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelControlMode( struct TRITONSERVER_ServerOptions* options, TRITONSERVER_ModelControlMode mode); /// Set the model to be loaded at startup in a server options. The model must be /// present in one, and only one, of the specified model repositories. /// This function can be called multiple times with different model name /// to set multiple startup models. /// Note that it only takes affect on TRITONSERVER_MODEL_CONTROL_EXPLICIT mode. /// /// \param options The server options object. /// \param mode_name The name of the model to load on startup. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetStartupModel( struct TRITONSERVER_ServerOptions* options, const char* model_name); /// Enable or disable strict model configuration handling in a server /// options. /// /// \param options The server options object. /// \param strict True to enable strict model configuration handling, /// false to disable. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetStrictModelConfig( struct TRITONSERVER_ServerOptions* options, bool strict); /// Set the custom model configuration name to load for all models. /// Fall back to default config file if empty. /// /// \param options The server options object. /// \param config_name The name of the config file to load for all models. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelConfigName( struct TRITONSERVER_ServerOptions* options, const char* model_config_name); /// Set the rate limit mode in a server options. /// /// TRITONSERVER_RATE_LIMIT_EXEC_COUNT: The rate limiting prioritizes the /// inference execution using the number of times each instance has got a /// chance to run. The execution gets to run only when its resource /// constraints are satisfied. /// /// TRITONSERVER_RATE_LIMIT_OFF: The rate limiting is turned off and the /// inference gets executed whenever an instance is available. /// /// \param options The server options object. /// \param mode The mode to use for the rate limiting. By default, execution /// count is used to determine the priorities. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetRateLimiterMode( struct TRITONSERVER_ServerOptions* options, TRITONSERVER_RateLimitMode mode); /// Add resource count for rate limiting. /// /// \param options The server options object. /// \param name The name of the resource. /// \param count The count of the resource. /// \param device The device identifier for the resource. A value of -1 /// indicates that the specified number of resources are available on every /// device. The device value is ignored for a global resource. The server /// will use the rate limiter configuration specified for instance groups /// in model config to determine whether resource is global. In case of /// conflicting resource type in different model configurations, server /// will raise an appropriate error while loading model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsAddRateLimiterResource( struct TRITONSERVER_ServerOptions* options, const char* resource_name, const size_t resource_count, const int device); /// Set the total pinned memory byte size that the server can allocate /// in a server options. The pinned memory pool will be shared across /// Triton itself and the backends that use /// TRITONBACKEND_MemoryManager to allocate memory. /// /// \param options The server options object. /// \param size The pinned memory pool byte size. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize( struct TRITONSERVER_ServerOptions* options, uint64_t size); /// Set the total CUDA memory byte size that the server can allocate /// on given GPU device in a server options. The pinned memory pool /// will be shared across Triton itself and the backends that use /// TRITONBACKEND_MemoryManager to allocate memory. /// /// \param options The server options object. /// \param gpu_device The GPU device to allocate the memory pool. /// \param size The CUDA memory pool byte size. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize( struct TRITONSERVER_ServerOptions* options, int gpu_device, uint64_t size); /// Set the size of the virtual address space that will be used /// for growable memory in implicit state. /// /// \param options The server options object. /// \param gpu_device The GPU device to set the CUDA virtual address space size /// \param size The size of the CUDA virtual address space. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCudaVirtualAddressSize( TRITONSERVER_ServerOptions* options, int gpu_device, size_t cuda_virtual_address_size); /// Deprecated. See TRITONSERVER_ServerOptionsSetCacheConfig instead. /// /// Set the total response cache byte size that the server can allocate in CPU /// memory. The response cache will be shared across all inference requests and /// across all models. /// /// \param options The server options object. /// \param size The total response cache byte size. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetResponseCacheByteSize( struct TRITONSERVER_ServerOptions* options, uint64_t size); /// Set the cache config that will be used to initialize the cache /// implementation for "cache_name". /// /// It is expected that the "cache_name" provided matches a directory inside /// the "cache_dir" used for TRITONSERVER_ServerOptionsSetCacheDirectory. The /// default "cache_dir" is "/opt/tritonserver/caches", so for a "cache_name" of /// "local", Triton would expect to find the "local" cache implementation at /// "/opt/tritonserver/caches/local/libtritoncache_local.so" /// /// Altogether an example for the "local" cache implementation would look like: /// std::string cache_name = "local"; /// std::string config_json = R"({"size": 1048576})" /// auto err = TRITONSERVER_ServerOptionsSetCacheConfig( /// options, cache_name, config_json); /// /// \param options The server options object. /// \param cache_name The name of the cache. Example names would be /// "local", "redis", or the name of a custom cache implementation. /// \param config_json The string representation of config JSON that is /// used to initialize the cache implementation. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCacheConfig( struct TRITONSERVER_ServerOptions* options, const char* cache_name, const char* config_json); /// Set the directory containing cache shared libraries. This /// directory is searched when looking for cache implementations. /// /// \param options The server options object. /// \param cache_dir The full path of the cache directory. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCacheDirectory( struct TRITONSERVER_ServerOptions* options, const char* cache_dir); /// Set the minimum support CUDA compute capability in a server /// options. /// /// \param options The server options object. /// \param cc The minimum CUDA compute capability. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( struct TRITONSERVER_ServerOptions* options, double cc); /// Enable or disable exit-on-error in a server options. /// /// \param options The server options object. /// \param exit True to enable exiting on initialization error, false /// to continue. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetExitOnError( struct TRITONSERVER_ServerOptions* options, bool exit); /// Enable or disable strict readiness handling in a server options. /// /// \param options The server options object. /// \param strict True to enable strict readiness handling, false to /// disable. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetStrictReadiness( struct TRITONSERVER_ServerOptions* options, bool strict); /// Set the exit timeout, in seconds, for the server in a server /// options. /// /// \param options The server options object. /// \param timeout The exit timeout, in seconds. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetExitTimeout( struct TRITONSERVER_ServerOptions* options, unsigned int timeout); /// Set the number of threads used in buffer manager in a server options. /// /// \param options The server options object. /// \param thread_count The number of threads. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetBufferManagerThreadCount( struct TRITONSERVER_ServerOptions* options, unsigned int thread_count); /// Set the number of threads to concurrently load models in a server options. /// /// \param options The server options object. /// \param thread_count The number of threads. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelLoadThreadCount( struct TRITONSERVER_ServerOptions* options, unsigned int thread_count); /// Set the number of retry to load a model in a server options. /// /// \param options The server options object. /// \param retry_count The number of retry. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelLoadRetryCount( struct TRITONSERVER_ServerOptions* options, unsigned int retry_count); /// Enable model namespacing to allow serving models with the same name if /// they are in different namespaces. /// /// \param options The server options object. /// \param enable_namespace Whether to enable model namespacing or not. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelNamespacing( struct TRITONSERVER_ServerOptions* options, bool enable_namespace); /// Enable peer access to allow GPU device to directly access the memory of /// another GPU device. Note that even when this option is set to True, Triton /// will only try to enable peer access and might fail to enable it if the /// underlying system doesn't support peer access. /// /// \param options The server options object. /// \param enable_peer_access Whether to enable peer access or not. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetEnablePeerAccess( struct TRITONSERVER_ServerOptions* options, bool enable_peer_access); /// Provide a log output file. /// /// \param options The server options object. /// \param file a string defining the file where the log outputs will be saved. /// An empty string for the file name will cause triton to direct logging /// facilities to the console /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogFile( struct TRITONSERVER_ServerOptions* options, const char* file); /// Enable or disable info level logging. /// /// \param options The server options object. /// \param log True to enable info logging, false to disable. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogInfo( struct TRITONSERVER_ServerOptions* options, bool log); /// Enable or disable warning level logging. /// /// \param options The server options object. /// \param log True to enable warning logging, false to disable. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogWarn( struct TRITONSERVER_ServerOptions* options, bool log); /// Enable or disable error level logging. /// /// \param options The server options object. /// \param log True to enable error logging, false to disable. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogError( struct TRITONSERVER_ServerOptions* options, bool log); /// Set the logging format. /// /// \param options The server options object. /// \param format The logging format. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogFormat( struct TRITONSERVER_ServerOptions* options, const TRITONSERVER_LogFormat format); /// Set verbose logging level. Level zero disables verbose logging. /// /// \param options The server options object. /// \param level The verbose logging level. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogVerbose( struct TRITONSERVER_ServerOptions* options, int level); /// Enable or disable metrics collection in a server options. /// /// \param options The server options object. /// \param metrics True to enable metrics, false to disable. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMetrics( struct TRITONSERVER_ServerOptions* options, bool metrics); /// Enable or disable GPU metrics collection in a server options. GPU /// metrics are collected if both this option and /// TRITONSERVER_ServerOptionsSetMetrics are true. /// /// \param options The server options object. /// \param gpu_metrics True to enable GPU metrics, false to disable. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetGpuMetrics( struct TRITONSERVER_ServerOptions* options, bool gpu_metrics); /// Enable or disable CPU metrics collection in a server options. CPU /// metrics are collected if both this option and /// TRITONSERVER_ServerOptionsSetMetrics are true. /// /// \param options The server options object. /// \param cpu_metrics True to enable CPU metrics, false to disable. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCpuMetrics( struct TRITONSERVER_ServerOptions* options, bool cpu_metrics); /// Set the interval for metrics collection in a server options. /// This is 2000 milliseconds by default. /// /// \param options The server options object. /// \param metrics_interval_ms The time interval in ms between /// successive metrics updates. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMetricsInterval( struct TRITONSERVER_ServerOptions* options, uint64_t metrics_interval_ms); /// Set the directory containing backend shared libraries. This /// directory is searched last after the version and model directory /// in the model repository when looking for the backend shared /// library for a model. If the backend is named 'be' the directory /// searched is 'backend_dir'/be/libtriton_be.so. /// /// \param options The server options object. /// \param backend_dir The full path of the backend directory. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetBackendDirectory( struct TRITONSERVER_ServerOptions* options, const char* backend_dir); /// Set the directory containing repository agent shared libraries. This /// directory is searched when looking for the repository agent shared /// library for a model. If the repo agent is named 'ra' the directory /// searched is 'repoagent_dir'/ra/libtritonrepoagent_ra.so. /// /// \param options The server options object. /// \param repoagent_dir The full path of the repository agent directory. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetRepoAgentDirectory( struct TRITONSERVER_ServerOptions* options, const char* repoagent_dir); /// Specify the limit on memory usage as a fraction on the device identified by /// 'kind' and 'device_id'. If model loading on the device is requested and the /// current memory usage exceeds the limit, the load will be rejected. If not /// specified, the limit will not be set. /// /// Currently support TRITONSERVER_INSTANCEGROUPKIND_GPU /// /// \param options The server options object. /// \param kind The kind of the device. /// \param device_id The id of the device. /// \param fraction The limit on memory usage as a fraction /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit( struct TRITONSERVER_ServerOptions* options, const TRITONSERVER_InstanceGroupKind kind, const int device_id, const double fraction); /// Set a configuration setting for a named backend in a server /// options. /// /// \param options The server options object. /// \param backend_name The name of the backend. /// \param setting The name of the setting. /// \param value The setting value. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetBackendConfig( struct TRITONSERVER_ServerOptions* options, const char* backend_name, const char* setting, const char* value); /// Set a host policy setting for a given policy name in a server options. /// /// \param options The server options object. /// \param policy_name The name of the policy. /// \param setting The name of the setting. /// \param value The setting value. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetHostPolicy( struct TRITONSERVER_ServerOptions* options, const char* policy_name, const char* setting, const char* value); /// Set a configuration setting for metrics in server options. /// /// \param options The server options object. /// \param name The name of the configuration group. An empty string indicates /// a global configuration option. /// \param setting The name of the setting. /// \param value The setting value. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMetricsConfig( struct TRITONSERVER_ServerOptions* options, const char* name, const char* setting, const char* value); /// TRITONSERVER_Server /// /// An inference server. /// /// Model batch flags. The enum values must be power-of-2 values. typedef enum tritonserver_batchflag_enum { TRITONSERVER_BATCH_UNKNOWN = 1, TRITONSERVER_BATCH_FIRST_DIM = 2 } TRITONSERVER_ModelBatchFlag; /// Model index flags. The enum values must be power-of-2 values. typedef enum tritonserver_modelindexflag_enum { TRITONSERVER_INDEX_FLAG_READY = 1 } TRITONSERVER_ModelIndexFlag; /// Model transaction policy flags. The enum values must be /// power-of-2 values. typedef enum tritonserver_txn_property_flag_enum { TRITONSERVER_TXN_ONE_TO_ONE = 1, TRITONSERVER_TXN_DECOUPLED = 2 } TRITONSERVER_ModelTxnPropertyFlag; /// Create a new server object. The caller takes ownership of the /// TRITONSERVER_Server object and must call TRITONSERVER_ServerDelete /// to release the object. /// /// \param server Returns the new inference server object. /// \param options The inference server options object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerNew( struct TRITONSERVER_Server** server, struct TRITONSERVER_ServerOptions* options); /// Delete a server object. If server is not already stopped it is /// stopped before being deleted. /// /// \param server The inference server object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerDelete( struct TRITONSERVER_Server* server); /// Stop a server object. A server can't be restarted once it is /// stopped. /// /// \param server The inference server object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerStop( struct TRITONSERVER_Server* server); /// Set the exit timeout on the server object. This value overrides the value /// initially set through server options and provides a mechanism to update the /// exit timeout while the serving is running. /// /// \param server The inference server object. /// \param timeout The exit timeout, in seconds. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerSetExitTimeout( struct TRITONSERVER_Server* server, unsigned int timeout); /// Register a new model repository. Not available in polling mode. /// /// \param server The inference server object. /// \param repository_path The full path to the model repository. /// \param name_mapping List of name_mapping parameters. Each mapping has /// the model directory name as its key, overridden model name as its value. /// \param model_count Number of mappings provided. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerRegisterModelRepository( struct TRITONSERVER_Server* server, const char* repository_path, const struct TRITONSERVER_Parameter** name_mapping, const uint32_t mapping_count); /// Unregister a model repository. Not available in polling mode. /// /// \param server The inference server object. /// \param repository_path The full path to the model repository. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerUnregisterModelRepository( struct TRITONSERVER_Server* server, const char* repository_path); /// Check the model repository for changes and update server state /// based on those changes. /// /// \param server The inference server object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerPollModelRepository(struct TRITONSERVER_Server* server); /// Is the server live? /// /// \param server The inference server object. /// \param live Returns true if server is live, false otherwise. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerIsLive( struct TRITONSERVER_Server* server, bool* live); /// Is the server ready? /// /// \param server The inference server object. /// \param ready Returns true if server is ready, false otherwise. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerIsReady( struct TRITONSERVER_Server* server, bool* ready); /// Is the model ready? /// /// \param server The inference server object. /// \param model_name The name of the model to get readiness for. /// \param model_version The version of the model to get readiness /// for. If -1 then the server will choose a version based on the /// model's policy. /// \param ready Returns true if server is ready, false otherwise. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerModelIsReady( struct TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, bool* ready); /// Get the batch properties of the model. The properties are /// communicated by a flags value and an (optional) object returned by /// 'voidp'. /// /// - TRITONSERVER_BATCH_UNKNOWN: Triton cannot determine the /// batching properties of the model. This means that the model /// does not support batching in any way that is usable by /// Triton. The returned 'voidp' value is nullptr. /// /// - TRITONSERVER_BATCH_FIRST_DIM: The model supports batching /// along the first dimension of every input and output /// tensor. Triton schedulers that perform batching can /// automatically batch inference requests along this dimension. /// The returned 'voidp' value is nullptr. /// /// \param server The inference server object. /// \param model_name The name of the model. /// \param model_version The version of the model. If -1 then the /// server will choose a version based on the model's policy. /// \param flags Returns flags indicating the batch properties of the /// model. /// \param voidp If non-nullptr, returns a point specific to the /// 'flags' value. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerModelBatchProperties( struct TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, uint32_t* flags, void** voidp); /// Get the transaction policy of the model. The policy is /// communicated by a flags value. /// /// - TRITONSERVER_TXN_ONE_TO_ONE: The model generates exactly /// one response per request. /// /// - TRITONSERVER_TXN_DECOUPLED: The model may generate zero /// to many responses per request. /// /// \param server The inference server object. /// \param model_name The name of the model. /// \param model_version The version of the model. If -1 then the /// server will choose a version based on the model's policy. /// \param txn_flags Returns flags indicating the transaction policy of the /// model. /// \param voidp If non-nullptr, returns a point specific to the 'flags' value. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerModelTransactionProperties( struct TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, uint32_t* txn_flags, void** voidp); /// Get the metadata of the server as a TRITONSERVER_Message object. /// The caller takes ownership of the message object and must call /// TRITONSERVER_MessageDelete to release the object. /// /// \param server The inference server object. /// \param server_metadata Returns the server metadata message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerMetadata( struct TRITONSERVER_Server* server, struct TRITONSERVER_Message** server_metadata); /// Get the metadata of a model as a TRITONSERVER_Message /// object. The caller takes ownership of the message object and must /// call TRITONSERVER_MessageDelete to release the object. /// /// \param server The inference server object. /// \param model_name The name of the model. /// \param model_version The version of the model. /// If -1 then the server will choose a version based on the model's /// policy. /// \param model_metadata Returns the model metadata message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerModelMetadata( struct TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, struct TRITONSERVER_Message** model_metadata); /// Get the statistics of a model as a TRITONSERVER_Message /// object. The caller takes ownership of the object and must call /// TRITONSERVER_MessageDelete to release the object. /// /// \param server The inference server object. /// \param model_name The name of the model. /// If empty, then statistics for all available models will be returned, /// and the server will choose a version based on those models' policies. /// \param model_version The version of the model. If -1 then the /// server will choose a version based on the model's policy. /// \param model_stats Returns the model statistics message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerModelStatistics( struct TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, struct TRITONSERVER_Message** model_stats); /// Get the configuration of a model as a TRITONSERVER_Message object. /// The caller takes ownership of the message object and must call /// TRITONSERVER_MessageDelete to release the object. /// /// \param server The inference server object. /// \param model_name The name of the model. /// \param model_version The version of the model. If -1 then the /// server will choose a version based on the model's policy. /// \param config_version The model configuration will be returned in /// a format matching this version. If the configuration cannot be /// represented in the requested version's format then an error will /// be returned. Currently only version 1 is supported. /// \param model_config Returns the model config message. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerModelConfig( struct TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, const uint32_t config_version, struct TRITONSERVER_Message** model_config); /// Get the index of all unique models in the model repositories as a /// TRITONSERVER_Message object. The caller takes ownership of the /// message object and must call TRITONSERVER_MessageDelete to release /// the object. /// /// If TRITONSERVER_INDEX_FLAG_READY is set in 'flags' only the models /// that are loaded into the server and ready for inferencing are /// returned. /// /// \param server The inference server object. /// \param flags TRITONSERVER_ModelIndexFlag flags that control how to /// collect the index. /// \param model_index Return the model index message that holds the /// index of all models contained in the server's model repository(s). /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerModelIndex( struct TRITONSERVER_Server* server, uint32_t flags, struct TRITONSERVER_Message** model_index); /// Load the requested model or reload the model if it is already /// loaded. The function does not return until the model is loaded or /// fails to load. Returned error indicates if model loaded /// successfully or not. /// /// \param server The inference server object. /// \param model_name The name of the model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerLoadModel( struct TRITONSERVER_Server* server, const char* model_name); /// Load the requested model or reload the model if it is already /// loaded, with load parameters provided. The function does not return until /// the model is loaded or fails to load. Returned error indicates if model /// loaded successfully or not. /// Currently the below parameter names are recognized: /// - "config" : string parameter that contains a JSON representation of the /// model configuration. This config will be used for loading the model instead /// of the one in the model directory. /// /// \param server The inference server object. /// \param model_name The name of the model. /// \param parameters The array of load parameters. /// \param parameter_count The number of parameters. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerLoadModelWithParameters( struct TRITONSERVER_Server* server, const char* model_name, const struct TRITONSERVER_Parameter** parameters, const uint64_t parameter_count); /// Unload the requested model. Unloading a model that is not loaded /// on server has no affect and success code will be returned. /// The function does not wait for the requested model to be fully unload /// and success code will be returned. /// Returned error indicates if model unloaded successfully or not. /// /// \param server The inference server object. /// \param model_name The name of the model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerUnloadModel( struct TRITONSERVER_Server* server, const char* model_name); /// Unload the requested model, and also unload any dependent model that /// was loaded along with the requested model (for example, the models composing /// an ensemble). Unloading a model that is not loaded /// on server has no affect and success code will be returned. /// The function does not wait for the requested model and all dependent /// models to be fully unload and success code will be returned. /// Returned error indicates if model unloaded successfully or not. /// /// \param server The inference server object. /// \param model_name The name of the model. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerUnloadModelAndDependents( struct TRITONSERVER_Server* server, const char* model_name); /// Get the current metrics for the server. The caller takes ownership /// of the metrics object and must call TRITONSERVER_MetricsDelete to /// release the object. /// /// \param server The inference server object. /// \param metrics Returns the metrics. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerMetrics( struct TRITONSERVER_Server* server, struct TRITONSERVER_Metrics** metrics); /// Perform inference using the meta-data and inputs supplied by the /// 'inference_request'. If the function returns success, then the /// caller releases ownership of 'inference_request' and must not /// access it in any way after this call, until ownership is returned /// via the 'request_release_fn' callback registered in the request /// object with TRITONSERVER_InferenceRequestSetReleaseCallback. /// /// The function unconditionally takes ownership of 'trace' and so the /// caller must not access it in any way after this call (except in /// the trace activity callbacks) until ownership is returned via the /// trace's release_fn callback. /// /// Responses produced for this request are returned using the /// allocator and callback registered with the request by /// TRITONSERVER_InferenceRequestSetResponseCallback. /// /// \param server The inference server object. /// \param inference_request The request object. /// \param trace The trace object for this request, or nullptr if no /// tracing. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerInferAsync( struct TRITONSERVER_Server* server, struct TRITONSERVER_InferenceRequest* inference_request, struct TRITONSERVER_InferenceTrace* trace); /// TRITONSERVER_MetricKind /// /// Types of metrics recognized by TRITONSERVER. /// typedef enum TRITONSERVER_metrickind_enum { TRITONSERVER_METRIC_KIND_COUNTER, TRITONSERVER_METRIC_KIND_GAUGE, TRITONSERVER_METRIC_KIND_HISTOGRAM } TRITONSERVER_MetricKind; /// Create a new metric family object. The caller takes ownership of the /// TRITONSERVER_MetricFamily object and must call /// TRITONSERVER_MetricFamilyDelete to release the object. /// /// \param family Returns the new metric family object. /// \param kind The type of metric family to create. /// \param name The name of the metric family seen when calling the metrics /// endpoint. /// \param description The description of the metric family seen when /// calling the metrics endpoint. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricFamilyNew( struct TRITONSERVER_MetricFamily** family, const TRITONSERVER_MetricKind kind, const char* name, const char* description); /// Delete a metric family object. /// A TRITONSERVER_MetricFamily* object should be deleted AFTER its /// corresponding TRITONSERVER_Metric* objects have been deleted. /// Attempting to delete a family before its metrics will return an error. /// /// \param family The metric family object to delete. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricFamilyDelete(struct TRITONSERVER_MetricFamily* family); /// Get the TRITONSERVER_MetricKind of the metric family. /// /// \param family The metric family object to query. /// \param kind Returns the TRITONSERVER_MetricKind of metric. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_GetMetricFamilyKind( struct TRITONSERVER_MetricFamily* family, TRITONSERVER_MetricKind* kind); /// Create a new metric args object. The caller takes ownership of the /// TRITONSERVER_MetricArgs object and must call TRITONSERVER_MetricArgsDelete /// to release the object. /// /// \param args Returns the new metric args object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricArgsNew( struct TRITONSERVER_MetricArgs** args); /// Set metric args with histogram metric parameter. /// /// \param args The metric args object to set. /// \param buckets The array of bucket boundaries for the expected range of /// observed values. /// /// \param buckets_count The number of bucket boundaries. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricArgsSetHistogram( struct TRITONSERVER_MetricArgs* args, const double* buckets, const uint64_t buckets_count); /// Delete a metric args object. /// /// \param args The metric args object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricArgsDelete( struct TRITONSERVER_MetricArgs* args); /// Create a new metric object. The caller takes ownership of the /// TRITONSERVER_Metric object and must call /// TRITONSERVER_MetricDelete to release the object. The caller is also /// responsible for ownership of the labels passed in. Each label can be deleted /// immediately after creating the metric with TRITONSERVER_ParameterDelete /// if not re-using the labels. /// /// \param metric Returns the new metric object. /// \param family The metric family to add this new metric to. /// \param labels The array of labels to associate with this new metric. /// \param label_count The number of labels. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricNew( struct TRITONSERVER_Metric** metric, struct TRITONSERVER_MetricFamily* family, const struct TRITONSERVER_Parameter** labels, const uint64_t label_count); /// Create a new metric object. The caller takes ownership of the /// TRITONSERVER_Metric object and must call /// TRITONSERVER_MetricDelete to release the object. The caller is also /// responsible for ownership of the labels passed in. /// Each label can be deleted immediately after creating the metric with /// TRITONSERVER_ParameterDelete if not re-using the labels. /// Metric args can be deleted immediately after creating the metric with /// TRITONSERVER_MetricArgsDelete if not re-using the metric args. /// /// \param metric Returns the new metric object. /// \param family The metric family to add this new metric to. /// \param labels The array of labels to associate with this new metric. /// \param label_count The number of labels. /// \param args Metric args that store additional arguments to construct /// particular metric types, e.g. histogram. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricNewWithArgs( struct TRITONSERVER_Metric** metric, struct TRITONSERVER_MetricFamily* family, const struct TRITONSERVER_Parameter** labels, const uint64_t label_count, const struct TRITONSERVER_MetricArgs* args); /// Delete a metric object. /// All TRITONSERVER_Metric* objects should be deleted BEFORE their /// corresponding TRITONSERVER_MetricFamily* objects have been deleted. /// If a family is deleted before its metrics, an error will be returned. /// /// \param metric The metric object to delete. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricDelete( struct TRITONSERVER_Metric* metric); /// Get the current value of a metric object. /// Supports metrics of kind TRITONSERVER_METRIC_KIND_COUNTER /// and TRITONSERVER_METRIC_KIND_GAUGE, and returns /// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind. /// /// \param metric The metric object to query. /// \param value Returns the current value of the metric object. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricValue( struct TRITONSERVER_Metric* metric, double* value); /// Increment the current value of metric by value. /// Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE for any value, /// and TRITONSERVER_METRIC_KIND_COUNTER for non-negative values. Returns /// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind /// and TRITONSERVER_ERROR_INVALID_ARG for negative values on a /// TRITONSERVER_METRIC_KIND_COUNTER metric. /// /// \param metric The metric object to update. /// \param value The amount to increment the metric's value by. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricIncrement( struct TRITONSERVER_Metric* metric, double value); /// Set the current value of metric to value. /// Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE and returns /// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind. /// /// \param metric The metric object to update. /// \param value The amount to set metric's value to. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricSet( struct TRITONSERVER_Metric* metric, double value); /// Sample an observation and count it to the appropriate bucket of a metric. /// Supports metrics of kind TRITONSERVER_METRIC_KIND_HISTOGRAM and returns /// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind. /// /// \param metric The metric object to update. /// \param value The amount for metric to sample observation. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricObserve( struct TRITONSERVER_Metric* metric, double value); /// Get the TRITONSERVER_MetricKind of metric of its corresponding family. /// /// \param metric The metric object to query. /// \param kind Returns the TRITONSERVER_MetricKind of metric. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_GetMetricKind( struct TRITONSERVER_Metric* metric, TRITONSERVER_MetricKind* kind); #ifdef __cplusplus } #endif kytensor-2.49.0.7/core/pyproject.toml000066400000000000000000000043021527052371200175000ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [tool.codespell] # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - # this is only to allow you to run codespell interactively skip = "./.git,./.github" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" # use the 'clear' dictionary for unambiguous spelling mistakes builtin = "clear" # disable warnings about binary files and wrong encoding quiet-level = 3 [tool.isort] profile = "black" use_parentheses = true multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 ensure_newline_before_comments = true line_length = 88 balanced_wrapping = true indent = " " skip = ["build"] kytensor-2.49.0.7/core/python/000077500000000000000000000000001527052371200161065ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/CMakeLists.txt000066400000000000000000000052511527052371200206510ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) add_subdirectory(tritonserver) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION ${TRITON_VERSION}) configure_file(../LICENSE LICENSE.txt COPYONLY) configure_file(setup.py setup.py @ONLY) file(COPY test/ DESTINATION ./test/.) set(WHEEL_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION ${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt ${CMAKE_CURRENT_BINARY_DIR}/setup.py ${CMAKE_CURRENT_BINARY_DIR}/tritonserver python-bindings ) set(wheel_stamp_file "stamp.whl") add_custom_command( OUTPUT "${wheel_stamp_file}" COMMAND python3 ARGS "${CMAKE_CURRENT_SOURCE_DIR}/build_wheel.py" --dest-dir "${CMAKE_CURRENT_BINARY_DIR}/generic" --binding-path $ DEPENDS ${WHEEL_DEPENDS} ) add_custom_target( generic-server-wheel ALL DEPENDS "${wheel_stamp_file}" ) # Wheel set(WHEEL_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/generic/wheel/dist/") install( DIRECTORY ${WHEEL_OUT_DIR} DESTINATION "${CMAKE_INSTALL_PREFIX}/python" ) # Tests set(TEST_DIR "${CMAKE_CURRENT_BINARY_DIR}/test") install( DIRECTORY ${TEST_DIR} DESTINATION "${CMAKE_INSTALL_PREFIX}/python" ) kytensor-2.49.0.7/core/python/build_wheel.py000077500000000000000000000104561527052371200207540ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os import pathlib import re import shutil import subprocess import sys from distutils.dir_util import copy_tree from tempfile import mkstemp def fail_if(p, msg): if p: print("error: {}".format(msg), file=sys.stderr) sys.exit(1) def mkdir(path): pathlib.Path(path).mkdir(parents=True, exist_ok=True) def touch(path): pathlib.Path(path).touch() def cpdir(src, dest): copy_tree(src, dest, preserve_symlinks=1) def sed(pattern, replace, source, dest=None): fin = open(source, "r") if dest: fout = open(dest, "w") else: fd, name = mkstemp() fout = open(name, "w") for line in fin: out = re.sub(pattern, replace, line) fout.write(out) fin.close() fout.close() if not dest: shutil.copyfile(name, source) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--dest-dir", type=str, required=True, help="Destination directory." ) parser.add_argument( "--binding-path", type=str, required=True, help="Path to Triton Python binding." ) FLAGS = parser.parse_args() FLAGS.triton_version = None with open("TRITON_VERSION", "r") as vfile: FLAGS.triton_version = vfile.readline().strip() FLAGS.whl_dir = os.path.join(FLAGS.dest_dir, "wheel") print("=== Building in: {}".format(os.getcwd())) print("=== Using builddir: {}".format(FLAGS.whl_dir)) print("Adding package files") mkdir(os.path.join(FLAGS.whl_dir, "tritonserver")) shutil.copy("tritonserver/__init__.py", os.path.join(FLAGS.whl_dir, "tritonserver")) # Type checking marker file indicating support for type checkers. # https://peps.python.org/pep-0561/ shutil.copy("tritonserver/py.typed", os.path.join(FLAGS.whl_dir, "tritonserver")) cpdir("tritonserver/_c", os.path.join(FLAGS.whl_dir, "tritonserver", "_c")) cpdir("tritonserver/_api", os.path.join(FLAGS.whl_dir, "tritonserver", "_api")) PYBIND_LIB = os.path.basename(FLAGS.binding_path) shutil.copyfile( FLAGS.binding_path, os.path.join(FLAGS.whl_dir, "tritonserver", "_c", PYBIND_LIB), ) shutil.copyfile("LICENSE.txt", os.path.join(FLAGS.whl_dir, "LICENSE.txt")) shutil.copyfile("setup.py", os.path.join(FLAGS.whl_dir, "setup.py")) os.chdir(FLAGS.whl_dir) print("=== Building wheel") args = ["python3", "setup.py", "bdist_wheel"] wenv = os.environ.copy() wenv["VERSION"] = FLAGS.triton_version wenv["TRITON_PYBIND"] = PYBIND_LIB p = subprocess.Popen(args, env=wenv) p.wait() fail_if(p.returncode != 0, "setup.py failed") cpdir("dist", FLAGS.dest_dir) print("=== Output wheel file is in: {}".format(FLAGS.dest_dir)) touch(os.path.join(FLAGS.dest_dir, "stamp.whl")) kytensor-2.49.0.7/core/python/setup.py000077500000000000000000000100301527052371200176150ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os import sys from itertools import chain from setuptools import find_packages, setup if "--plat-name" in sys.argv: PLATFORM_FLAG = sys.argv[sys.argv.index("--plat-name") + 1] else: PLATFORM_FLAG = "any" if "VERSION" not in os.environ: raise Exception("envvar VERSION must be specified") VERSION = os.environ["VERSION"] try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel class bdist_wheel(_bdist_wheel): def finalize_options(self): _bdist_wheel.finalize_options(self) self.root_is_pure = False def get_tag(self): pyver, abi, plat = "py3", "none", PLATFORM_FLAG return pyver, abi, plat except ImportError: bdist_wheel = None this_directory = os.path.abspath(os.path.dirname(__file__)) data_files = [ ("", ["LICENSE.txt"]), ] # Type checking marker file indicating support for type checkers. # https://peps.python.org/pep-0561/ # Type hints for c extension generated by mypy platform_package_data = [ os.environ["TRITON_PYBIND"], "py.typed", "_c/__init__.pyi", "_c/triton_bindings.pyi", ] gpu_extras = ["cupy-cuda12x"] test_extras = ["pytest"] all_extras = gpu_extras + test_extras setup( name="tritonserver", version=VERSION, author="NVIDIA Inc.", author_email="sw-dl-triton@nvidia.com", description="Triton Inference Server In-Process Python API", license="BSD", url="https://developer.nvidia.com/nvidia-triton-inference-server", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Intended Audience :: Information Technology", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Image Recognition", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries", "Topic :: Utilities", "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Environment :: Console", "Natural Language :: English", "Operating System :: OS Independent", ], packages=find_packages(), package_data={ "": platform_package_data, }, zip_safe=False, cmdclass={"bdist_wheel": bdist_wheel}, data_files=data_files, install_requires=["numpy<2"], extras_require={"GPU": gpu_extras, "test": test_extras, "all": all_extras}, ) kytensor-2.49.0.7/core/python/test/000077500000000000000000000000001527052371200170655ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/test/test_api.py000066400000000000000000000457061527052371200212630ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio import copy import json import os import queue import shutil import time import unittest import numpy import pytest import tritonserver try: import cupy except ImportError: cupy = None try: import torch if not torch.cuda.is_available(): torch = None except ImportError: torch = None module_directory = os.path.split(os.path.abspath(__file__))[0] test_model_directory = os.path.abspath( os.path.join(module_directory, "test_api_models") ) test_logs_directory = os.path.abspath(os.path.join(module_directory, "test_api_logs")) shutil.rmtree(test_logs_directory, ignore_errors=True) os.makedirs(test_logs_directory) server_options = tritonserver.Options( server_id="TestServer", model_repository=test_model_directory, log_verbose=6, log_error=True, log_warn=True, log_info=True, exit_on_error=True, strict_model_config=False, model_control_mode=tritonserver.ModelControlMode.EXPLICIT, exit_timeout=30, ) class ModelTests(unittest.TestCase): def setup_method(self, method): self._server_options = copy.copy(server_options) self._server_options.log_file = os.path.join( test_logs_directory, method.__name__ + ".server.log" ) def test_create_request(self): server = tritonserver.Server(self._server_options).start(wait_until_ready=True) request = server.models()["test"].create_request() request = tritonserver.InferenceRequest(server.model("test")) class AllocatorTests(unittest.TestCase): class MockMemoryAllocator(tritonserver.MemoryAllocator): def __init__(self): pass def allocate(self, *args, **kwargs): raise Exception("foo") def setup_method(self, method): self._server_options = copy.copy(server_options) self._server_options.log_file = os.path.join( test_logs_directory, method.__name__ + ".server.log" ) @pytest.mark.skipif(cupy is None, reason="Skipping gpu memory, cupy not installed") def test_memory_fallback_to_cpu(self): server = tritonserver.Server(self._server_options).start(wait_until_ready=True) self.assertTrue(server.ready()) allocator = tritonserver.default_memory_allocators[tritonserver.MemoryType.GPU] del tritonserver.default_memory_allocators[tritonserver.MemoryType.GPU] server.load( "test", { "config": json.dumps( { "backend": "python", "parameters": { "decoupled": {"string_value": "False"}, "request_gpu_memory": {"string_value": "True"}, }, } ) }, ) fp16_input = numpy.random.rand(1, 100).astype(dtype=numpy.float16) for response in server.model("test").infer( inputs={"fp16_input": fp16_input}, ): self.assertEqual( response.outputs["fp16_output"].memory_type, tritonserver.MemoryType.CPU ) fp16_output = numpy.from_dlpack(response.outputs["fp16_output"]) self.assertEqual(fp16_input[0][0], fp16_output[0][0]) tritonserver.default_memory_allocators[tritonserver.MemoryType.GPU] = allocator def test_memory_allocator_exception(self): server = tritonserver.Server(self._server_options).start(wait_until_ready=True) self.assertTrue(server.ready()) server.load( "test", { "config": json.dumps( { "backend": "python", "parameters": {"decoupled": {"string_value": "False"}}, } ) }, ) with self.assertRaises(tritonserver.InternalError): for response in server.model("test").infer( inputs={ "string_input": tritonserver.Tensor.from_string_array([["hello"]]) }, output_memory_type="gpu", output_memory_allocator=AllocatorTests.MockMemoryAllocator(), ): pass def test_unsupported_memory_type(self): server = tritonserver.Server(self._server_options).start(wait_until_ready=True) self.assertTrue(server.ready()) server.load( "test", { "config": json.dumps( { "backend": "python", "parameters": {"decoupled": {"string_value": "False"}}, } ) }, ) if tritonserver.MemoryType.GPU in tritonserver.default_memory_allocators: allocator = tritonserver.default_memory_allocators[ tritonserver.MemoryType.GPU ] del tritonserver.default_memory_allocators[tritonserver.MemoryType.GPU] else: allocator = None with self.assertRaises(tritonserver.InvalidArgumentError): for response in server.model("test").infer( inputs={ "string_input": tritonserver.Tensor.from_string_array([["hello"]]) }, output_memory_type="gpu", ): pass if allocator is not None: tritonserver.default_memory_allocators[ tritonserver.MemoryType.GPU ] = allocator @pytest.mark.skipif(torch is None, reason="Skipping test, torch not installed") def test_allocate_on_cpu_and_reshape(self): allocator = tritonserver.default_memory_allocators[tritonserver.MemoryType.CPU] memory_buffer = allocator.allocate( memory_type=tritonserver.MemoryType.CPU, memory_type_id=0, size=200 ) cpu_array = memory_buffer.owner self.assertEqual(memory_buffer.size, 200) fp32_size = int(memory_buffer.size / 4) tensor = tritonserver.Tensor( tritonserver.DataType.FP32, shape=[fp32_size], memory_buffer=memory_buffer ) cpu_fp32_array = numpy.from_dlpack(tensor) self.assertEqual(cpu_array.ctypes.data, cpu_fp32_array.ctypes.data) self.assertEqual(cpu_fp32_array.dtype, numpy.float32) self.assertEqual(cpu_fp32_array.nbytes, 200) @pytest.mark.skipif(cupy is None, reason="Skipping gpu memory, cupy not installed") @pytest.mark.skipif(torch is None, reason="Skipping test, torch not installed") def test_allocate_on_gpu_and_reshape(self): if cupy is None: return allocator = tritonserver.default_memory_allocators[tritonserver.MemoryType.GPU] memory_buffer = allocator.allocate( memory_type=tritonserver.MemoryType.GPU, memory_type_id=0, size=200 ) gpu_array = memory_buffer.owner gpu_array = cupy.empty([10, 20], dtype=cupy.uint8) memory_buffer = tritonserver.MemoryBuffer.from_dlpack(gpu_array) self.assertEqual(memory_buffer.size, 200) fp32_size = int(memory_buffer.size / 4) tensor = tritonserver.Tensor( tritonserver.DataType.FP32, shape=[fp32_size], memory_buffer=memory_buffer ) gpu_fp32_array = cupy.from_dlpack(tensor) self.assertEqual( gpu_array.__cuda_array_interface__["data"][0], gpu_fp32_array.__cuda_array_interface__["data"][0], ) self.assertEqual(gpu_fp32_array.dtype, cupy.float32) self.assertEqual(gpu_fp32_array.nbytes, 200) torch_fp32_tensor = torch.from_dlpack(tensor) self.assertEqual(torch_fp32_tensor.dtype, torch.float32) self.assertEqual( torch_fp32_tensor.data_ptr(), gpu_array.__cuda_array_interface__["data"][0] ) self.assertEqual(torch_fp32_tensor.nbytes, 200) class TensorTests(unittest.TestCase): @pytest.mark.skipif(cupy is None, reason="Skipping gpu memory, cupy not installed") def test_cpu_to_gpu(self): if cupy is None: return cpu_array = numpy.random.rand(1, 3, 100, 100).astype(numpy.float32) cpu_tensor = tritonserver.Tensor.from_dlpack(cpu_array) gpu_tensor = cpu_tensor.to_device("gpu:0") gpu_array = cupy.from_dlpack(gpu_tensor) self.assertEqual(gpu_array.device, cupy.cuda.Device(0)) numpy.testing.assert_array_equal(cpu_array, gpu_array.get()) memory_buffer = tritonserver.MemoryBuffer.from_dlpack(gpu_array) self.assertEqual( gpu_array.__cuda_array_interface__["data"][0], memory_buffer.data_ptr ) @pytest.mark.skipif( torch is None, reason="Skipping gpu memory, torch not installed" ) @pytest.mark.skipif(cupy is None, reason="Skipping gpu memory, cupy not installed") def test_gpu_tensor_from_dl_pack(self): if cupy is None or torch is None: return cupy_array = cupy.ones([100]).astype(cupy.float64) tensor = tritonserver.Tensor.from_dlpack(cupy_array) torch_tensor = torch.from_dlpack(cupy_array) self.assertEqual(torch_tensor.data_ptr(), tensor.data_ptr) self.assertEqual(torch_tensor.nbytes, tensor.size) self.assertEqual(torch_tensor.__dlpack_device__(), tensor.__dlpack_device__()) @pytest.mark.skipif(torch is None, reason="Skipping test, torch not installed") def test_tensor_from_numpy(self): cpu_array = numpy.random.rand(1, 3, 100, 100).astype(numpy.float32) tensor = tritonserver.Tensor.from_dlpack(cpu_array) torch_tensor = torch.from_dlpack(tensor) numpy.testing.assert_array_equal(torch_tensor.numpy(), cpu_array) self.assertEqual(torch_tensor.data_ptr(), cpu_array.ctypes.data) class ServerTests(unittest.TestCase): def setup_method(self, method): self._server_options = copy.copy(server_options) self._server_options.log_file = os.path.join( test_logs_directory, method.__name__ + ".server.log" ) def test_not_started(self): server = tritonserver.Server() with self.assertRaises(tritonserver.InvalidArgumentError): server.ready() def test_invalid_option_type(self): server = tritonserver.Server(server_id=1) with self.assertRaises(TypeError): server.start() server = tritonserver.Server(model_repository=1) with self.assertRaises(TypeError): server.start() def test_invalid_repo(self): with self.assertRaises(tritonserver.InternalError): tritonserver.Server(model_repository="foo").start() def test_ready(self): server = tritonserver.Server(self._server_options).start() self.assertTrue(server.ready()) def test_stop(self): server = tritonserver.Server(self._server_options).start(wait_until_ready=True) self.assertTrue(server.ready()) server.load( "test", { "config": json.dumps( { "backend": "python", "parameters": {"decoupled": {"string_value": "False"}}, # Keep instance count low for fast startup/cleanup. # Alternatively can use KIND_CPU here, but keeping gpus/count explicit. "instance_group": [ {"kind": "KIND_GPU", "gpus": [0], "count": 1} ], } ) }, ) fp16_input = numpy.random.rand(1, 100).astype(dtype=numpy.float16) for response in server.model("test").infer( inputs={"fp16_input": fp16_input}, output_memory_type="cpu", raise_on_error=True, ): fp16_output = numpy.from_dlpack(response.outputs["fp16_output"]) numpy.testing.assert_array_equal(fp16_input, fp16_output) server.stop() def test_model_repository_not_specified(self): with self.assertRaises(tritonserver.InvalidArgumentError): tritonserver.Server(model_repository=None).start() class InferenceTests(unittest.TestCase): def setup_method(self, method): self._server_options = copy.copy(server_options) self._server_options.log_file = os.path.join( test_logs_directory, method.__name__ + ".server.log" ) @pytest.mark.skipif(cupy is None, reason="Skipping gpu memory, cupy not installed") def test_gpu_output(self): server = tritonserver.Server(self._server_options).start(wait_until_ready=True) self.assertTrue(server.ready()) server.load( "test", { "config": json.dumps( { "backend": "python", "parameters": {"decoupled": {"string_value": "False"}}, } ) }, ) fp16_input = numpy.random.rand(1, 100).astype(dtype=numpy.float16) for response in server.model("test").infer( inputs={"fp16_input": fp16_input}, output_memory_type="gpu", ): fp16_output = cupy.from_dlpack(response.outputs["fp16_output"]) self.assertEqual(fp16_input[0][0], fp16_output[0][0]) for response in server.model("test").infer( inputs={"string_input": [["hello"]]}, output_memory_type="gpu", ): text_output = response.outputs["string_output"].to_string_array() self.assertEqual(text_output[0][0], "hello") for response in server.model("test").infer( inputs={"string_input": tritonserver.Tensor.from_string_array([["hello"]])}, output_memory_type="gpu", ): text_output = response.outputs["string_output"].to_string_array() text_output = response.outputs["string_output"].to_string_array() self.assertEqual(text_output[0][0], "hello") def test_basic_inference(self): server = tritonserver.Server(self._server_options).start(wait_until_ready=True) self.assertTrue(server.ready()) server.load( "test", { "config": json.dumps( { "backend": "python", "parameters": {"decoupled": {"string_value": "False"}}, } ) }, ) inputs = { "fp16_input": numpy.random.rand(1, 100).astype(dtype=numpy.float16), "bool_input": numpy.random.rand(1, 100).astype(dtype=numpy.bool_), } for response in server.model("test").infer( inputs=inputs, output_memory_type="cpu", raise_on_error=True, ): for input_name, input_value in inputs.items(): output_value = response.outputs[input_name.replace("input", "output")] output_value = numpy.from_dlpack(output_value) numpy.testing.assert_array_equal(input_value, output_value) # test normal bool inputs = {"bool_input": [[True, False, False, True]]} for response in server.model("test").infer( inputs=inputs, output_memory_type="cpu", raise_on_error=True, ): for input_name, input_value in inputs.items(): output_value = numpy.from_dlpack( response.outputs[input_name.replace("input", "output")] ) numpy.testing.assert_array_equal(input_value, output_value) def test_parameters(self): server = tritonserver.Server(self._server_options).start(wait_until_ready=True) self.assertTrue(server.ready()) server.load( "test", { "config": json.dumps( { "backend": "python", "parameters": {"decoupled": {"string_value": "False"}}, } ) }, ) fp16_input = numpy.random.rand(1, 100).astype(dtype=numpy.float16) input_parameters = { "int_parameter": 0, "float_parameter": 0.5, "bool_parameter": False, "string_parameter": "test", } for response in server.model("test").infer( inputs={"fp16_input": fp16_input}, parameters=input_parameters, output_memory_type="cpu", raise_on_error=True, ): fp16_output = numpy.from_dlpack(response.outputs["fp16_output"]) numpy.testing.assert_array_equal(fp16_input, fp16_output) output_parameters = json.loads( response.outputs["output_parameters"].to_string_array()[0] ) assert input_parameters == output_parameters with self.assertRaises(tritonserver.InvalidArgumentError): input_parameters = { "invalid": {"test": 1}, } server.model("test").infer( inputs={"fp16_input": fp16_input}, parameters=input_parameters, output_memory_type="cpu", raise_on_error=True, ) with self.assertRaises(tritonserver.InvalidArgumentError): input_parameters = { "invalid": None, } server.model("test").infer( inputs={"fp16_input": fp16_input}, parameters=input_parameters, output_memory_type="cpu", raise_on_error=True, ) kytensor-2.49.0.7/core/python/test/test_api_models/000077500000000000000000000000001527052371200222405ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/test/test_api_models/test/000077500000000000000000000000001527052371200232175ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/test/test_api_models/test/1/000077500000000000000000000000001527052371200233575ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/test/test_api_models/test/1/model.py000066400000000000000000000075151527052371200250410ustar00rootroot00000000000000import json import numpy as np import triton_python_backend_utils as pb_utils try: import cupy except: cupy = None class TritonPythonModel: @staticmethod def auto_complete_config(auto_complete_model_config): inputs = [] outputs = [] dims = [-1, -1] optional = True config = auto_complete_model_config.as_dict() for data_type in pb_utils.TRITON_STRING_TO_NUMPY.keys(): type_name = data_type.split("_")[1].lower() input_name = f"{type_name}_input" output_name = f"{type_name}_output" inputs.append( { "name": input_name, "data_type": data_type, "dims": dims, "optional": optional, } ) outputs.append({"name": output_name, "data_type": data_type, "dims": dims}) outputs.append( {"name": "output_parameters", "data_type": "TYPE_STRING", "dims": [1]} ) for input_ in inputs: auto_complete_model_config.add_input(input_) for output in outputs: auto_complete_model_config.add_output(output) auto_complete_model_config.set_max_batch_size(0) if config["parameters"]["decoupled"]["string_value"] == "True": auto_complete_model_config.set_model_transaction_policy({"decoupled": True}) return auto_complete_model_config def initialize(self, args): self._model_config = json.loads(args["model_config"]) self._decoupled = self._model_config.get("model_transaction_policy", {}).get( "decoupled" ) self._request_gpu_memory = False if "parameters" in self._model_config: parameters = self._model_config["parameters"] if ( "request_gpu_memory" in parameters and parameters["request_gpu_memory"]["string_value"] == "True" ): self._request_gpu_memory = True def execute_decoupled(self, requests): for request in requests: sender = request.get_response_sender() output_tensors = [] for input_tensor in request.inputs(): input_value = input_tensor.as_numpy() output_tensor = pb_utils.Tensor( input_tensor.name().replace("input", "output"), input_value ) output_tensors.append(output_tensor) sender.send(pb_utils.InferenceResponse(output_tensors=output_tensors)) sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) return None def execute(self, requests): if self._decoupled: return self.execute_decoupled(requests) responses = [] for request in requests: output_tensors = [] parameters = [] for input_tensor in request.inputs(): input_value = input_tensor.as_numpy() if self._request_gpu_memory: input_value = cupy.array(input_value) output_tensor = pb_utils.Tensor.from_dlpack( input_tensor.name().replace("input", "output"), input_value ) else: output_tensor = pb_utils.Tensor( input_tensor.name().replace("input", "output"), input_value ) output_tensors.append(output_tensor) output_parameters = np.array([request.parameters()]).astype(np.object_) output_tensors.append( pb_utils.Tensor("output_parameters", output_parameters) ) responses.append( pb_utils.InferenceResponse( output_tensors=output_tensors, ) ) return responses kytensor-2.49.0.7/core/python/test/test_api_models/test/config.pbtxt000066400000000000000000000000221527052371200255410ustar00rootroot00000000000000backend: "python" kytensor-2.49.0.7/core/python/test/test_binding.py000066400000000000000000001337441527052371200221240ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import gc import json import os import queue import shutil import unittest import numpy from tritonserver import _c as triton_bindings # Callback functions used in inference pipeline # 'user_object' is a per-request counter of how many times the # callback is invoked def g_alloc_fn( allocator, tensor_name, byte_size, memory_type, memory_type_id, user_object ): if "alloc" not in user_object: user_object["alloc"] = 0 user_object["alloc"] += 1 buffer = numpy.empty(byte_size, numpy.byte) return (buffer.ctypes.data, buffer, triton_bindings.TRITONSERVER_MemoryType.CPU, 0) def g_release_fn( allocator, buffer, buffer_user_object, byte_size, memory_type, memory_type_id ): # No-op, buffer ('buffer_user_object') will be garbage collected # only sanity check that the objects are expected if (not isinstance(buffer_user_object, numpy.ndarray)) or ( buffer_user_object.ctypes.data != buffer ): raise Exception("Misaligned parameters in allocator release callback") pass def g_start_fn(allocator, user_object): if "start" not in user_object: user_object["start"] = 0 user_object["start"] += 1 pass def g_query_fn( allocator, user_object, tensor_name, byte_size, memory_type, memory_type_id ): if "query" not in user_object: user_object["query"] = 0 user_object["query"] += 1 return (triton_bindings.TRITONSERVER_MemoryType.CPU, 0) def g_buffer_fn( allocator, tensor_name, buffer_attribute, user_object, buffer_user_object ): if "buffer" not in user_object: user_object["buffer"] = 0 user_object["buffer"] += 1 buffer_attribute.memory_type = triton_bindings.TRITONSERVER_MemoryType.CPU buffer_attribute.memory_type_id = 0 buffer_attribute.byte_size = buffer_user_object.size return buffer_attribute def g_timestamp_fn(trace, activity, timestamp_ns, user_object): if trace.id not in user_object: user_object[trace.id] = [] # not owning trace, so must read property out trace_log = { "id": trace.id, "parent_id": trace.parent_id, "model_name": trace.model_name, "model_version": trace.model_version, "request_id": trace.request_id, "activity": activity, "timestamp": timestamp_ns, } user_object[trace.id].append(trace_log) def g_tensor_fn( trace, activity, tensor_name, data_type, buffer, byte_size, shape, memory_type, memory_type_id, user_object, ): if trace.id not in user_object: user_object[trace.id] = [] # not owning trace, so must read property out trace_log = { "id": trace.id, "parent_id": trace.parent_id, "model_name": trace.model_name, "model_version": trace.model_version, "request_id": trace.request_id, "activity": activity, "tensor": { "name": tensor_name, "data_type": data_type, # skip 'buffer' "byte_size": byte_size, "shape": shape, "memory_type": memory_type, "memory_type_id": memory_type_id, }, } user_object[trace.id].append(trace_log) def g_trace_release_fn(trace, user_object): # sanity check that 'trace' has been tracked, the object # will be released on garbage collection if trace.id not in user_object: raise Exception("Releasing unseen trace") user_object["signal_queue"].put("TRACE_RELEASED") def g_response_fn(response, flags, user_object): user_object.put((flags, response)) def g_request_fn(request, flags, user_object): if flags != 1: raise Exception("Unexpected request release flag") # counter of "inflight" requests user_object.put(request) # Python model file string to fastly deploy test model, depends on # 'TRITONSERVER_Server' operates properly to load model with content passed # through the load API. g_python_addsub = b''' import json import numpy as np import triton_python_backend_utils as pb_utils class TritonPythonModel: @staticmethod def auto_complete_config(auto_complete_model_config): input0 = {"name": "INPUT0", "data_type": "TYPE_FP32", "dims": [4]} input1 = {"name": "INPUT1", "data_type": "TYPE_FP32", "dims": [4]} output0 = {"name": "OUTPUT0", "data_type": "TYPE_FP32", "dims": [4]} output1 = {"name": "OUTPUT1", "data_type": "TYPE_FP32", "dims": [4]} auto_complete_model_config.set_max_batch_size(0) auto_complete_model_config.add_input(input0) auto_complete_model_config.add_input(input1) auto_complete_model_config.add_output(output0) auto_complete_model_config.add_output(output1) # [WARNING] Specify specific dynamic batching field by knowing # the implementation detail auto_complete_model_config.set_dynamic_batching() auto_complete_model_config._model_config["dynamic_batching"]["priority_levels"] = 20 auto_complete_model_config._model_config["dynamic_batching"]["default_priority_level"] = 10 return auto_complete_model_config def initialize(self, args): self.model_config = model_config = json.loads(args["model_config"]) output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") self.output0_dtype = pb_utils.triton_string_to_numpy( output0_config["data_type"] ) self.output1_dtype = pb_utils.triton_string_to_numpy( output1_config["data_type"] ) def execute(self, requests): """This function is called on inference request.""" output0_dtype = self.output0_dtype output1_dtype = self.output1_dtype responses = [] for request in requests: in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") out_0, out_1 = ( in_0.as_numpy() + in_1.as_numpy(), in_0.as_numpy() - in_1.as_numpy(), ) out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(output0_dtype)) out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(output1_dtype)) responses.append(pb_utils.InferenceResponse([out_tensor_0, out_tensor_1])) return responses ''' # ======================================= Test cases =========================== class BindingTest(unittest.TestCase): def setUp(self): self._test_model_repo = os.path.join(os.getcwd(), "binding_test_repo") # clear model repository that may be created for testing. if os.path.exists(self._test_model_repo): shutil.rmtree(self._test_model_repo) os.makedirs(self._test_model_repo) self._model_name = "addsub" self._version = "1" self._file_name = "model.py" def tearDown(self): gc.collect() # clear model repository that may be created for testing. if os.path.exists(self._test_model_repo): shutil.rmtree(self._test_model_repo) # helper functions def _to_pyobject(self, triton_message): return json.loads(triton_message.serialize_to_json()) # prepare a model repository with "addsub" model def _create_model_repository(self): os.makedirs( os.path.join(self._test_model_repo, self._model_name, self._version) ) with open( os.path.join( self._test_model_repo, self._model_name, self._version, self._file_name ), "wb", ) as f: f.write(g_python_addsub) # create a Triton instance with POLL mode on repository prepared by # '_create_model_repository' def _start_polling_server(self): # prepare model repository self._create_model_repository() options = triton_bindings.TRITONSERVER_ServerOptions() options.set_model_repository_path(self._test_model_repo) options.set_model_control_mode( triton_bindings.TRITONSERVER_ModelControlMode.POLL ) # enable "auto-complete" to skip providing config.pbtxt options.set_strict_model_config(False) options.set_server_id("testing_server") # [FIXME] Need to fix coupling of response and server options.set_exit_timeout(5) return triton_bindings.TRITONSERVER_Server(options) def _prepare_inference_request(self, server): allocator = triton_bindings.TRITONSERVER_ResponseAllocator( g_alloc_fn, g_release_fn, g_start_fn ) allocator.set_buffer_attributes_function(g_buffer_fn) allocator.set_query_function(g_query_fn) request_counter = queue.Queue() response_queue = queue.Queue() allocator_counter = {} request = triton_bindings.TRITONSERVER_InferenceRequest( server, self._model_name, -1 ) request.id = "req_0" request.set_release_callback(g_request_fn, request_counter) request.set_response_callback( allocator, allocator_counter, g_response_fn, response_queue ) input = numpy.ones([4], dtype=numpy.float32) input_buffer = input.ctypes.data ba = triton_bindings.TRITONSERVER_BufferAttributes() ba.memory_type = triton_bindings.TRITONSERVER_MemoryType.CPU ba.memory_type_id = 0 ba.byte_size = input.itemsize * input.size request.add_input( "INPUT0", triton_bindings.TRITONSERVER_DataType.FP32, input.shape ) request.add_input( "INPUT1", triton_bindings.TRITONSERVER_DataType.FP32, input.shape ) request.append_input_data_with_buffer_attributes("INPUT0", input_buffer, ba) request.append_input_data_with_buffer_attributes("INPUT1", input_buffer, ba) return request, allocator, response_queue, request_counter def test_exceptions(self): ex_list = [ triton_bindings.UnknownError, triton_bindings.InternalError, triton_bindings.NotFoundError, triton_bindings.InvalidArgumentError, triton_bindings.UnavailableError, triton_bindings.UnsupportedError, triton_bindings.AlreadyExistsError, ] for ex_type in ex_list: with self.assertRaises(triton_bindings.TritonError) as ctx: raise ex_type("Error message") self.assertTrue(isinstance(ctx.exception, ex_type)) self.assertEqual(str(ctx.exception), "Error message") def test_data_type(self): t_list = [ (triton_bindings.TRITONSERVER_DataType.INVALID, "", 0), (triton_bindings.TRITONSERVER_DataType.BOOL, "BOOL", 1), (triton_bindings.TRITONSERVER_DataType.UINT8, "UINT8", 1), (triton_bindings.TRITONSERVER_DataType.UINT16, "UINT16", 2), (triton_bindings.TRITONSERVER_DataType.UINT32, "UINT32", 4), (triton_bindings.TRITONSERVER_DataType.UINT64, "UINT64", 8), (triton_bindings.TRITONSERVER_DataType.INT8, "INT8", 1), (triton_bindings.TRITONSERVER_DataType.INT16, "INT16", 2), (triton_bindings.TRITONSERVER_DataType.INT32, "INT32", 4), (triton_bindings.TRITONSERVER_DataType.INT64, "INT64", 8), (triton_bindings.TRITONSERVER_DataType.FP16, "FP16", 2), (triton_bindings.TRITONSERVER_DataType.FP32, "FP32", 4), (triton_bindings.TRITONSERVER_DataType.FP64, "FP64", 8), (triton_bindings.TRITONSERVER_DataType.BYTES, "BYTES", 0), (triton_bindings.TRITONSERVER_DataType.BF16, "BF16", 2), ] for t, t_str, t_size in t_list: self.assertEqual(triton_bindings.TRITONSERVER_DataTypeString(t), t_str) self.assertEqual(triton_bindings.TRITONSERVER_StringToDataType(t_str), t) self.assertEqual(triton_bindings.TRITONSERVER_DataTypeByteSize(t), t_size) def test_memory_type(self): t_list = [ (triton_bindings.TRITONSERVER_MemoryType.CPU, "CPU"), (triton_bindings.TRITONSERVER_MemoryType.CPU_PINNED, "CPU_PINNED"), (triton_bindings.TRITONSERVER_MemoryType.GPU, "GPU"), ] for t, t_str in t_list: self.assertEqual(triton_bindings.TRITONSERVER_MemoryTypeString(t), t_str) def test_parameter_type(self): t_list = [ (triton_bindings.TRITONSERVER_ParameterType.STRING, "STRING"), (triton_bindings.TRITONSERVER_ParameterType.INT, "INT"), (triton_bindings.TRITONSERVER_ParameterType.BOOL, "BOOL"), (triton_bindings.TRITONSERVER_ParameterType.BYTES, "BYTES"), ] for t, t_str in t_list: self.assertEqual(triton_bindings.TRITONSERVER_ParameterTypeString(t), t_str) def test_parameter(self): # C API doesn't provide additional API for parameter, can only test # New/Delete unless we mock the implementation to expose more info. str_param = triton_bindings.TRITONSERVER_Parameter("str_key", "str_value") int_param = triton_bindings.TRITONSERVER_Parameter("int_key", 123) bool_param = triton_bindings.TRITONSERVER_Parameter("bool_key", True) # bytes parameter doesn't own the buffer b = bytes("abc", "utf-8") bytes_param = triton_bindings.TRITONSERVER_Parameter("bytes_key", b) del str_param del int_param del bool_param del bytes_param gc.collect() def test_instance_kind(self): t_list = [ (triton_bindings.TRITONSERVER_InstanceGroupKind.AUTO, "AUTO"), (triton_bindings.TRITONSERVER_InstanceGroupKind.CPU, "CPU"), (triton_bindings.TRITONSERVER_InstanceGroupKind.GPU, "GPU"), (triton_bindings.TRITONSERVER_InstanceGroupKind.MODEL, "MODEL"), ] for t, t_str in t_list: self.assertEqual( triton_bindings.TRITONSERVER_InstanceGroupKindString(t), t_str ) def test_log(self): # This test depends on 'TRITONSERVER_ServerOptions' operates properly # to modify log settings. # Direct Triton to log message into a file so that the log may be # retrieved on the Python side. Otherwise the log will be default # on stderr and Python utils can not redirect the pipe on Triton side. log_file = "triton_binding_test_log_output.txt" default_format_regex = r"[0-9][0-9][0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9][0-9][0-9][0-9]" iso8601_format_regex = r"[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]Z" try: options = triton_bindings.TRITONSERVER_ServerOptions() # Enable subset of log levels options.set_log_file(log_file) options.set_log_info(True) options.set_log_warn(False) options.set_log_error(True) options.set_log_verbose(0) options.set_log_format(triton_bindings.TRITONSERVER_LogFormat.DEFAULT) for ll, enabled in [ (triton_bindings.TRITONSERVER_LogLevel.INFO, True), (triton_bindings.TRITONSERVER_LogLevel.WARN, False), (triton_bindings.TRITONSERVER_LogLevel.ERROR, True), (triton_bindings.TRITONSERVER_LogLevel.VERBOSE, False), ]: self.assertEqual(triton_bindings.TRITONSERVER_LogIsEnabled(ll), enabled) # Write message to each of the log level triton_bindings.TRITONSERVER_LogMessage( triton_bindings.TRITONSERVER_LogLevel.INFO, "filename", 123, "info_message", ) triton_bindings.TRITONSERVER_LogMessage( triton_bindings.TRITONSERVER_LogLevel.WARN, "filename", 456, "warn_message", ) triton_bindings.TRITONSERVER_LogMessage( triton_bindings.TRITONSERVER_LogLevel.ERROR, "filename", 789, "error_message", ) triton_bindings.TRITONSERVER_LogMessage( triton_bindings.TRITONSERVER_LogLevel.VERBOSE, "filename", 147, "verbose_message", ) with open(log_file, "r") as f: log = f.read() # Check level self.assertRegex(log, r"filename:123.*info_message") self.assertNotRegex(log, r"filename:456.*warn_message") self.assertRegex(log, r"filename:789.*error_message") self.assertNotRegex(log, r"filename:147.*verbose_message") # Check format "MMDD hh:mm:ss.ssssss". self.assertRegex(log, default_format_regex) # sanity check that there is no log with other format "YYYY-MM-DDThh:mm:ssZ L" self.assertNotRegex(log, iso8601_format_regex) # Test different format options.set_log_format(triton_bindings.TRITONSERVER_LogFormat.ISO8601) triton_bindings.TRITONSERVER_LogMessage( triton_bindings.TRITONSERVER_LogLevel.INFO, "fn", 258, "info_message" ) with open(log_file, "r") as f: log = f.read() self.assertRegex(log, r"fn:258.*info_message") self.assertRegex(log, iso8601_format_regex) finally: # Must make sure the log settings are reset as the logger is unique # within the process options.set_log_file("") options.set_log_info(False) options.set_log_warn(False) options.set_log_error(False) options.set_log_verbose(0) options.set_log_format(triton_bindings.TRITONSERVER_LogFormat.DEFAULT) os.remove(log_file) def test_buffer_attributes(self): expected_memory_type = triton_bindings.TRITONSERVER_MemoryType.CPU_PINNED expected_memory_type_id = 4 expected_byte_size = 1024 buffer_attributes = triton_bindings.TRITONSERVER_BufferAttributes() buffer_attributes.memory_type_id = expected_memory_type_id self.assertEqual(buffer_attributes.memory_type_id, expected_memory_type_id) buffer_attributes.memory_type = expected_memory_type self.assertEqual(buffer_attributes.memory_type, expected_memory_type) buffer_attributes.byte_size = expected_byte_size self.assertEqual(buffer_attributes.byte_size, expected_byte_size) # cuda_ipc_handle is supposed to be cudaIpcMemHandle_t, must initialize buffer # of that size to avoid segfault. The handle getter/setter is different from other # attributes that different pointers may be returned from the getter, but the byte # content pointed by the pointer should be the same import ctypes from array import array handle_byte_size = 64 mock_handle = array("b", [i for i in range(handle_byte_size)]) buffer_attributes.cuda_ipc_handle = mock_handle.buffer_info()[0] res_arr = (ctypes.c_char * handle_byte_size).from_address( buffer_attributes.cuda_ipc_handle ) for i in range(handle_byte_size): self.assertEqual(int.from_bytes(res_arr[i], "big"), mock_handle[i]) def test_allocator(self): def alloc_fn( allocator, tensor_name, byte_size, memory_type, memory_type_id, user_object ): return (123, None, triton_bindings.TRITONSERVER_MemoryType.GPU, 1) def release_fn( allocator, buffer, buffer_user_object, byte_size, memory_type, memory_type_id, ): pass def start_fn(allocator, user_object): pass def query_fn( allocator, user_object, tensor_name, byte_size, memory_type, memory_type_id ): return (triton_bindings.TRITONSERVER_MemoryType.GPU, 1) def buffer_fn( allocator, tensor_name, buffer_attribute, user_object, buffer_user_object ): return buffer_attribute # allocator without start_fn allocator = triton_bindings.TRITONSERVER_ResponseAllocator(alloc_fn, release_fn) del allocator gc.collect() # allocator with start_fn allocator = triton_bindings.TRITONSERVER_ResponseAllocator( alloc_fn, release_fn, start_fn ) allocator.set_buffer_attributes_function(buffer_fn) allocator.set_query_function(query_fn) def test_message(self): expected_dict = {"key_0": [1, 2, "3"], "key_1": {"nested_key": "nested_value"}} message = triton_bindings.TRITONSERVER_Message(json.dumps(expected_dict)) self.assertEqual(expected_dict, json.loads(message.serialize_to_json())) def test_metrics(self): # This test depends on 'TRITONSERVER_Server' operates properly # to access metrics. # Create server in EXPLICIT mode so we don't need to ensure # a model repository is proper repository options = triton_bindings.TRITONSERVER_ServerOptions() options.set_model_repository_path(self._test_model_repo) options.set_model_control_mode( triton_bindings.TRITONSERVER_ModelControlMode.EXPLICIT ) server = triton_bindings.TRITONSERVER_Server(options) metrics = server.metrics() # Check one of the metrics is reported self.assertTrue( "nv_cpu_memory_used_bytes" in metrics.formatted(triton_bindings.TRITONSERVER_MetricFormat.PROMETHEUS) ) def test_trace_enum(self): t_list = [ (triton_bindings.TRITONSERVER_InferenceTraceLevel.DISABLED, "DISABLED"), (triton_bindings.TRITONSERVER_InferenceTraceLevel.MIN, "MIN"), (triton_bindings.TRITONSERVER_InferenceTraceLevel.MAX, "MAX"), (triton_bindings.TRITONSERVER_InferenceTraceLevel.TIMESTAMPS, "TIMESTAMPS"), (triton_bindings.TRITONSERVER_InferenceTraceLevel.TENSORS, "TENSORS"), ] for t, t_str in t_list: self.assertEqual( triton_bindings.TRITONSERVER_InferenceTraceLevelString(t), t_str ) # bit-wise operation level = int(triton_bindings.TRITONSERVER_InferenceTraceLevel.TIMESTAMPS) | int( triton_bindings.TRITONSERVER_InferenceTraceLevel.TENSORS ) self.assertNotEqual( level & int(triton_bindings.TRITONSERVER_InferenceTraceLevel.TIMESTAMPS), 0 ) self.assertNotEqual( level & int(triton_bindings.TRITONSERVER_InferenceTraceLevel.TENSORS), 0 ) t_list = [ ( triton_bindings.TRITONSERVER_InferenceTraceActivity.REQUEST_START, "REQUEST_START", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.QUEUE_START, "QUEUE_START", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.COMPUTE_START, "COMPUTE_START", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.COMPUTE_INPUT_END, "COMPUTE_INPUT_END", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.COMPUTE_OUTPUT_START, "COMPUTE_OUTPUT_START", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.COMPUTE_END, "COMPUTE_END", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.REQUEST_END, "REQUEST_END", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.TENSOR_QUEUE_INPUT, "TENSOR_QUEUE_INPUT", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.TENSOR_BACKEND_INPUT, "TENSOR_BACKEND_INPUT", ), ( triton_bindings.TRITONSERVER_InferenceTraceActivity.TENSOR_BACKEND_OUTPUT, "TENSOR_BACKEND_OUTPUT", ), ] for t, t_str in t_list: self.assertEqual( triton_bindings.TRITONSERVER_InferenceTraceActivityString(t), t_str ) def test_trace(self): # This test depends on 'test_infer_async' test to capture # the trace level = int(triton_bindings.TRITONSERVER_InferenceTraceLevel.TIMESTAMPS) | int( triton_bindings.TRITONSERVER_InferenceTraceLevel.TENSORS ) trace_dict = {"signal_queue": queue.Queue()} trace = triton_bindings.TRITONSERVER_InferenceTrace( level, 123, g_timestamp_fn, g_tensor_fn, g_trace_release_fn, trace_dict ) # [FIXME] get a copy of trace id due to potential issue of 'trace' # lifecycle trace_id = trace.id # Send and wait for inference, not care about result. server = self._start_polling_server() ( request, allocator, response_queue, request_counter, ) = self._prepare_inference_request(server) server.infer_async(request, trace) # [FIXME] WAR due to trace lifecycle is tied to response in Triton core, # trace reference should drop on response send.. res = response_queue.get(block=True, timeout=10) del res gc.collect() _ = trace_dict["signal_queue"].get(block=True, timeout=10) # check 'trace_dict' self.assertTrue(trace_id in trace_dict) # check activity are logged correctly, # value of 0 indicate it is timestamp trace, # non-zero is tensor trace and the value is how many times this # particular activity should be logged expected_activities = { # timestamp triton_bindings.TRITONSERVER_InferenceTraceActivity.REQUEST_START: 0, triton_bindings.TRITONSERVER_InferenceTraceActivity.QUEUE_START: 0, triton_bindings.TRITONSERVER_InferenceTraceActivity.COMPUTE_START: 0, triton_bindings.TRITONSERVER_InferenceTraceActivity.COMPUTE_INPUT_END: 0, triton_bindings.TRITONSERVER_InferenceTraceActivity.COMPUTE_OUTPUT_START: 0, triton_bindings.TRITONSERVER_InferenceTraceActivity.COMPUTE_END: 0, triton_bindings.TRITONSERVER_InferenceTraceActivity.REQUEST_END: 0, # not timestamp triton_bindings.TRITONSERVER_InferenceTraceActivity.TENSOR_QUEUE_INPUT: 2, # TENSOR_BACKEND_INPUT never get called with in Triton core # triton_bindings.TRITONSERVER_InferenceTraceActivity.TENSOR_BACKEND_INPUT : 2, triton_bindings.TRITONSERVER_InferenceTraceActivity.TENSOR_BACKEND_OUTPUT: 2, } for tl in trace_dict[trace_id]: # basic check self.assertEqual(tl["id"], trace_id) self.assertEqual(tl["parent_id"], 123) self.assertEqual(tl["model_name"], self._model_name) self.assertEqual(tl["model_version"], 1) self.assertEqual(tl["request_id"], "req_0") self.assertTrue(tl["activity"] in expected_activities) if expected_activities[tl["activity"]] == 0: self.assertTrue("timestamp" in tl) else: self.assertTrue("tensor" in tl) expected_activities[tl["activity"]] -= 1 if expected_activities[tl["activity"]] == 0: del expected_activities[tl["activity"]] # check if dict is empty to ensure the activity are logged in correct # amount. self.assertFalse(bool(expected_activities)) request_counter.get() def test_options(self): options = triton_bindings.TRITONSERVER_ServerOptions() # Generic options.set_server_id("server_id") options.set_min_supported_compute_capability(7.0) options.set_exit_on_error(False) options.set_strict_readiness(False) options.set_exit_timeout(30) # Models options.set_model_repository_path("model_repo_0") options.set_model_repository_path("model_repo_1") for m in [ triton_bindings.TRITONSERVER_ModelControlMode.NONE, triton_bindings.TRITONSERVER_ModelControlMode.POLL, triton_bindings.TRITONSERVER_ModelControlMode.EXPLICIT, ]: options.set_model_control_mode(m) options.set_startup_model("*") options.set_strict_model_config(True) options.set_model_load_thread_count(2) options.set_model_namespacing(True) # Only support Kind GPU for now options.set_model_load_device_limit( triton_bindings.TRITONSERVER_InstanceGroupKind.GPU, 0, 0.5 ) for k in [ triton_bindings.TRITONSERVER_InstanceGroupKind.AUTO, triton_bindings.TRITONSERVER_InstanceGroupKind.CPU, triton_bindings.TRITONSERVER_InstanceGroupKind.MODEL, ]: with self.assertRaises(triton_bindings.TritonError) as context: options.set_model_load_device_limit(k, 0, 0) self.assertTrue("not supported" in str(context.exception)) # Backend options.set_backend_directory("backend_dir_0") options.set_backend_directory("backend_dir_1") options.set_backend_config("backend_name", "setting", "value") # Rate limiter for r in [ triton_bindings.TRITONSERVER_RateLimitMode.OFF, triton_bindings.TRITONSERVER_RateLimitMode.EXEC_COUNT, ]: options.set_rate_limiter_mode(r) options.add_rate_limiter_resource("shared_resource", 4, -1) options.add_rate_limiter_resource("device_resource", 1, 0) # memory pools options.set_pinned_memory_pool_byte_size(1024) options.set_cuda_memory_pool_byte_size(0, 2048) # cache options.set_response_cache_byte_size(4096) options.set_cache_config( "cache_name", json.dumps({"config_0": "value_0", "config_1": "value_1"}) ) options.set_cache_directory("cache_dir_0") options.set_cache_directory("cache_dir_1") # Log try: options.set_log_file("some_file") options.set_log_info(True) options.set_log_warn(True) options.set_log_error(True) options.set_log_verbose(2) for f in [ triton_bindings.TRITONSERVER_LogFormat.DEFAULT, triton_bindings.TRITONSERVER_LogFormat.ISO8601, ]: options.set_log_format(f) finally: # Must make sure the log settings are reset as the logger is unique # within the process options.set_log_file("") options.set_log_info(False) options.set_log_warn(False) options.set_log_error(False) options.set_log_verbose(0) options.set_log_format(triton_bindings.TRITONSERVER_LogFormat.DEFAULT) # Metrics options.set_gpu_metrics(True) options.set_cpu_metrics(True) options.set_metrics_interval(5) options.set_metrics_config("metrics_group", "setting", "value") # Misc.. with self.assertRaises(triton_bindings.TritonError) as context: options.set_host_policy("policy_name", "setting", "value") self.assertTrue("Unsupported host policy setting" in str(context.exception)) options.set_repo_agent_directory("repo_agent_dir_0") options.set_repo_agent_directory("repo_agent_dir_1") options.set_buffer_manager_thread_count(4) def test_server(self): server = self._start_polling_server() # is_live self.assertTrue(server.is_live()) # is_ready self.assertTrue(server.is_ready()) # model_is_ready self.assertTrue(server.model_is_ready(self._model_name, -1)) # model_batch_properties expected_batch_properties = ( int(triton_bindings.TRITONSERVER_ModelBatchFlag.UNKNOWN), 0, ) self.assertEqual( server.model_batch_properties(self._model_name, -1), expected_batch_properties, ) # model_transaction_properties expected_transaction_policy = ( int(triton_bindings.TRITONSERVER_ModelTxnPropertyFlag.ONE_TO_ONE), 0, ) self.assertEqual( server.model_transaction_properties(self._model_name, -1), expected_transaction_policy, ) # metadata server_meta_data = self._to_pyobject(server.metadata()) self.assertTrue("name" in server_meta_data) self.assertEqual(server_meta_data["name"], "testing_server") # model_metadata model_meta_data = self._to_pyobject(server.model_metadata(self._model_name, -1)) self.assertTrue("name" in model_meta_data) self.assertEqual(model_meta_data["name"], self._model_name) # model_statistics model_statistics = self._to_pyobject( server.model_statistics(self._model_name, -1) ) self.assertTrue("model_stats" in model_statistics) # model_config model_config = self._to_pyobject(server.model_config(self._model_name, -1, 1)) self.assertTrue("input" in model_config) # model_index model_index = self._to_pyobject(server.model_index(0)) self.assertEqual(model_index[0]["name"], self._model_name) # metrics (see test_metrics) # infer_async (see test_infer_async) def test_request(self): # This test depends on 'TRITONSERVER_Server' operates properly to initialize # the request server = self._start_polling_server() with self.assertRaises(triton_bindings.NotFoundError) as ctx: _ = triton_bindings.TRITONSERVER_InferenceRequest( server, "not_existing_model", -1 ) self.assertTrue("unknown model" in str(ctx.exception)) expected_request_id = "request" expected_flags = int( triton_bindings.TRITONSERVER_RequestFlag.SEQUENCE_START ) | int(triton_bindings.TRITONSERVER_RequestFlag.SEQUENCE_END) expected_correlation_id = 2 expected_correlation_id_string = "123" expected_priority = 19 # larger value than model max priority level, # will be set to default (10, see 'g_python_addsub' for config detail) expected_priority_uint64 = 67 expected_timeout_microseconds = 222 request = triton_bindings.TRITONSERVER_InferenceRequest(server, "addsub", -1) # request metadata request.id = expected_request_id self.assertEqual(request.id, expected_request_id) request.flags = expected_flags self.assertEqual(request.flags, expected_flags) request.correlation_id = expected_correlation_id self.assertEqual(request.correlation_id, expected_correlation_id) request.correlation_id_string = expected_correlation_id_string self.assertEqual(request.correlation_id_string, expected_correlation_id_string) # Expect error from retrieving correlation id in a wrong type, # wrap in lambda function to avoid early evaluation that raises # exception before assert self.assertRaises(triton_bindings.TritonError, lambda: request.correlation_id) request.priority = expected_priority self.assertEqual(request.priority, expected_priority) request.priority_uint64 = expected_priority_uint64 self.assertEqual(request.priority_uint64, 10) request.timeout_microseconds = expected_timeout_microseconds self.assertEqual(request.timeout_microseconds, expected_timeout_microseconds) request.set_string_parameter("str_key", "str_val") request.set_int_parameter("int_key", 567) request.set_bool_parameter("bool_key", False) # I/O input = numpy.ones([2, 3], dtype=numpy.float32) buffer = input.ctypes.data ba = triton_bindings.TRITONSERVER_BufferAttributes() ba.memory_type = triton_bindings.TRITONSERVER_MemoryType.CPU ba.memory_type_id = 0 ba.byte_size = input.itemsize * input.size request.add_input( "INPUT0", triton_bindings.TRITONSERVER_DataType.FP32, input.shape ) self.assertRaises(triton_bindings.TritonError, request.remove_input, "INPUT2") # raw input assumes single input self.assertRaises(triton_bindings.TritonError, request.add_raw_input, "INPUT1") request.remove_input("INPUT0") request.add_raw_input("INPUT1") request.remove_all_inputs() # all inputs are removed, all 'append' functions should raise exceptions aid_args = ["INPUT0", buffer, ba.byte_size, ba.memory_type, ba.memory_type_id] self.assertRaises( triton_bindings.TritonError, request.append_input_data, *aid_args ) self.assertRaises( triton_bindings.TritonError, request.append_input_data_with_host_policy, *aid_args, "host_policy_name" ) self.assertRaises( triton_bindings.TritonError, request.append_input_data_with_buffer_attributes, "INPUT0", buffer, ba, ) self.assertRaises( triton_bindings.TritonError, request.remove_all_input_data, "INPUT0" ) # Add back input request.add_input( "INPUT0", triton_bindings.TRITONSERVER_DataType.FP32, input.shape ) request.append_input_data(*aid_args) request.remove_all_input_data("INPUT0") request.add_requested_output("OUTPUT0") request.remove_requested_output("OUTPUT1") request.remove_all_requested_outputs() def test_infer_async(self): # start server server = self._start_polling_server() # prepare for infer allocator = triton_bindings.TRITONSERVER_ResponseAllocator( g_alloc_fn, g_release_fn, g_start_fn ) allocator.set_buffer_attributes_function(g_buffer_fn) allocator.set_query_function(g_query_fn) request_counter = queue.Queue() response_queue = queue.Queue() allocator_counter = {} request = triton_bindings.TRITONSERVER_InferenceRequest( server, self._model_name, -1 ) request.id = "req_0" request.set_release_callback(g_request_fn, request_counter) request.set_response_callback( allocator, allocator_counter, g_response_fn, response_queue ) input = numpy.ones([4], dtype=numpy.float32) input_buffer = input.ctypes.data ba = triton_bindings.TRITONSERVER_BufferAttributes() ba.memory_type = triton_bindings.TRITONSERVER_MemoryType.CPU ba.memory_type_id = 0 ba.byte_size = input.itemsize * input.size request.add_input( "INPUT0", triton_bindings.TRITONSERVER_DataType.FP32, input.shape ) request.add_input( "INPUT1", triton_bindings.TRITONSERVER_DataType.FP32, input.shape ) request.append_input_data_with_buffer_attributes("INPUT0", input_buffer, ba) request.append_input_data_with_buffer_attributes("INPUT1", input_buffer, ba) # non-blocking, wait on response complete server.infer_async(request) # Expect every response to be returned in 10 seconds flags, res = response_queue.get(block=True, timeout=10) self.assertEqual( flags, int(triton_bindings.TRITONSERVER_ResponseCompleteFlag.FINAL) ) # expect no error res.throw_if_response_error() # version will be actual model version self.assertEqual(res.model, (self._model_name, 1)) self.assertEqual(res.id, request.id) self.assertEqual(res.parameter_count, 0) # out of range access self.assertRaises(triton_bindings.TritonError, res.parameter, 0) # read output tensor self.assertEqual(res.output_count, 2) for out, expected_name, expected_data in [ (res.output(0), "OUTPUT0", input + input), (res.output(1), "OUTPUT1", input - input), ]: ( name, data_type, shape, out_buffer, byte_size, memory_type, memory_type_id, numpy_buffer, ) = out self.assertEqual(name, expected_name) self.assertEqual(data_type, triton_bindings.TRITONSERVER_DataType.FP32) self.assertEqual(shape, expected_data.shape) self.assertEqual(out_buffer, numpy_buffer.ctypes.data) # buffer attribute used for input doesn't necessarily to # match output buffer attributes, this is just knowing the detail. self.assertEqual(byte_size, ba.byte_size) self.assertEqual(memory_type, ba.memory_type) self.assertEqual(memory_type_id, ba.memory_type_id) self.assertTrue( numpy.allclose( numpy_buffer.view(dtype=expected_data.dtype).reshape(shape), expected_data, ) ) # label (no label so empty) self.assertEqual(len(res.output_classification_label(0, 1)), 0) # [FIXME] keep alive behavior is not established between response # and server, so must explicitly handle the destruction order for now. del res # sanity check on user objects self.assertEqual(allocator_counter["start"], 1) self.assertEqual(allocator_counter["alloc"], 2) # Knowing implementation detail that the backend doesn't use query API self.assertTrue("query" not in allocator_counter) self.assertEqual(allocator_counter["buffer"], 2) # Expect request to be released in 10 seconds request = request_counter.get(block=True, timeout=10) def test_server_explicit(self): self._create_model_repository() # explicit : load with params options = triton_bindings.TRITONSERVER_ServerOptions() options.set_model_repository_path(self._test_model_repo) options.set_model_control_mode( triton_bindings.TRITONSERVER_ModelControlMode.EXPLICIT ) options.set_strict_model_config(False) server = triton_bindings.TRITONSERVER_Server(options) load_file_params = [ triton_bindings.TRITONSERVER_Parameter("config", r"{}"), triton_bindings.TRITONSERVER_Parameter( "file:" + os.path.join(self._version, self._file_name), g_python_addsub ), ] server.load_model_with_parameters("wired_addsub", load_file_params) self.assertTrue(server.model_is_ready("wired_addsub", -1)) # Model Repository self.assertFalse(server.model_is_ready(self._model_name, -1)) # unregister server.unregister_model_repository(self._test_model_repo) self.assertRaises( triton_bindings.TritonError, server.load_model, self._model_name ) # register server.register_model_repository(self._test_model_repo, []) server.load_model(self._model_name) self.assertTrue(server.model_is_ready(self._model_name, -1)) # unload server.unload_model("wired_addsub") self.assertFalse(server.model_is_ready("wired_addsub", -1)) server.unload_model_and_dependents(self._model_name) self.assertFalse(server.model_is_ready(self._model_name, -1)) def test_custom_metric(self): options = triton_bindings.TRITONSERVER_ServerOptions() options.set_model_repository_path(self._test_model_repo) options.set_model_control_mode( triton_bindings.TRITONSERVER_ModelControlMode.EXPLICIT ) server = triton_bindings.TRITONSERVER_Server(options) # create custom metric mf = triton_bindings.TRITONSERVER_MetricFamily( triton_bindings.TRITONSERVER_MetricKind.COUNTER, "custom_metric_familiy", "custom metric example", ) m = triton_bindings.TRITONSERVER_Metric(mf, []) m.increment(2) self.assertEqual(m.kind, triton_bindings.TRITONSERVER_MetricKind.COUNTER) self.assertEqual(m.value, 2) # can't use 'set_value' due to wrong kind self.assertRaises(triton_bindings.TritonError, m.set_value, 5) # Check custom metric is reported metrics = server.metrics() self.assertTrue( "custom_metric_familiy" in metrics.formatted(triton_bindings.TRITONSERVER_MetricFormat.PROMETHEUS) ) if __name__ == "__main__": unittest.main() kytensor-2.49.0.7/core/python/tritonserver/000077500000000000000000000000001527052371200206545ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/tritonserver/CMakeLists.txt000066400000000000000000000056121527052371200234200ustar00rootroot00000000000000# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) project(triton-bindings LANGUAGES C CXX) # Top level module entry point and typed marker file(COPY __init__.py DESTINATION .) file(COPY py.typed DESTINATION .) # Copy the '__init__.py' for the '_c' module file(COPY _c/__init__.py DESTINATION ./_c/.) file(COPY _c/__init__.pyi DESTINATION ./_c/.) file(COPY _c/triton_bindings.pyi DESTINATION ./_c/.) # Find and copy _api modules file(GLOB PYTHON_MODULE_FILES ./_api/*.py) file(COPY ${PYTHON_MODULE_FILES} DESTINATION ./_api/.) include(FetchContent) FetchContent_Declare( pybind11 SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/third_party/pybind11 #GIT_REPOSITORY "https://github.com/pybind/pybind11" # COMMIT ID for v2.10.0 #GIT_TAG "aa304c9c7d725ffb9d10af08a3b34cb372307020" #GIT_SHALLOW ON ) FetchContent_MakeAvailable(pybind11) set( PYTHON_BINDING_SRCS _c/tritonserver_pybind.cc ) pybind11_add_module(python-bindings SHARED ${PYTHON_BINDING_SRCS}) target_link_libraries( python-bindings PRIVATE triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) target_compile_features(python-bindings PRIVATE cxx_std_17) set_property(TARGET python-bindings PROPERTY OUTPUT_NAME triton_bindings) # Add Triton library default path in 'rpath' for runtime library lookup set_target_properties(python-bindings PROPERTIES BUILD_RPATH "$ORIGIN:/opt/tritonserver/lib") kytensor-2.49.0.7/core/python/tritonserver/__init__.py000066400000000000000000000114131527052371200227650ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Triton Inference Server In-Process Python API The Triton Inference Server In-Process Python API enables developers to easily embed a Triton inference server within their python application. Developers can load and interact with models, send inference requests, query metrics, etc. Everything available through the C API for emebedding a Triton Inference Server in C / C++ applications has been provided within a Python API. Note: The Python API is currently in BETA and interfaces and capabilities are subject to change. Any feedback is welcome. Note: Any objects not explicitly exported here are considered private. Note: Any methods, properties, or arguments prefixed with `_` are considered private. """ from importlib.metadata import PackageNotFoundError, version from tritonserver._api._allocators import MemoryAllocator as MemoryAllocator from tritonserver._api._allocators import MemoryBuffer as MemoryBuffer from tritonserver._api._allocators import MemoryType as MemoryType from tritonserver._api._allocators import ( default_memory_allocators as default_memory_allocators, ) from tritonserver._api._dlpack import DLDeviceType as DLDeviceType from tritonserver._api._model import Model as Model from tritonserver._api._model import ModelBatchFlag as ModelBatchFlag from tritonserver._api._model import ModelTxnPropertyFlag as ModelTxnPropertyFlag from tritonserver._api._request import InferenceRequest as InferenceRequest from tritonserver._api._server import InstanceGroupKind as InstanceGroupKind from tritonserver._api._server import LogFormat as LogFormat from tritonserver._api._server import Metric as Metric from tritonserver._api._server import MetricFamily as MetricFamily from tritonserver._api._server import MetricFormat as MetricFormat from tritonserver._api._server import MetricKind as MetricKind from tritonserver._api._server import ModelControlMode as ModelControlMode from tritonserver._api._server import Options as Options from tritonserver._api._server import RateLimiterResource as RateLimiterResource from tritonserver._api._server import RateLimitMode as RateLimitMode from tritonserver._api._server import Server as Server from tritonserver._api._tensor import DataType as DataType from tritonserver._api._tensor import Tensor as Tensor from tritonserver._c import AlreadyExistsError as AlreadyExistsError from tritonserver._c import InternalError as InternalError from tritonserver._c import InvalidArgumentError as InvalidArgumentError from tritonserver._c import NotFoundError as NotFoundError from tritonserver._c import TritonError as TritonError from tritonserver._c import UnavailableError as UnavailableError from tritonserver._c import UnknownError as UnknownError from tritonserver._c import UnsupportedError as UnsupportedError _exceptions = [ TritonError, NotFoundError, UnknownError, InternalError, InvalidArgumentError, UnavailableError, AlreadyExistsError, UnsupportedError, ] # Rename module for exceptions to simplify stack trace for exception in _exceptions: exception.__module__ = "tritonserver" globals()[exception.__name__] = exception __all__ = [] __version__ = "UNKNOWN" try: __version__ = version("tritonserver") except PackageNotFoundError: pass del _exceptions del version del PackageNotFoundError kytensor-2.49.0.7/core/python/tritonserver/_api/000077500000000000000000000000001527052371200215645ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/tritonserver/_api/__init__.py000066400000000000000000000000001527052371200236630ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/tritonserver/_api/_allocators.py000066400000000000000000000246331527052371200244500ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Default / Example Allocators for Tensor Memory""" from __future__ import annotations from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, Optional import numpy from tritonserver._api._datautils import ( DeviceOrMemoryType, DLPackObject, parse_device_or_memory_type, ) from tritonserver._api._logging import LogLevel, LogMessage from tritonserver._c import TRITONSERVER_ResponseAllocator from tritonserver._c.triton_bindings import ( InvalidArgumentError, TRITONSERVER_BufferAttributes, ) from tritonserver._c.triton_bindings import TRITONSERVER_MemoryType as MemoryType try: import cupy except ImportError: cupy = None default_memory_allocators: dict[MemoryType, MemoryAllocator] = dict({}) @dataclass class MemoryBuffer: """Memory allocated for a Tensor. This object does not own the memory but holds a reference to the owner. Parameters ---------- data_ptr : int Pointer to the allocated memory. memory_type : MemoryType memory type memory_type_id : int memory type id (typically the same as device id) size : int Size of the allocated memory in bytes. owner : Any Object that owns or manages the memory buffer. Allocated memory must not be freed while a reference to the owner is held. Examples -------- >>> buffer = MemoryBuffer.from_dlpack(numpy.array([100],dtype=numpy.uint8)) """ data_ptr: int memory_type: MemoryType memory_type_id: int size: int owner: Any @staticmethod def from_dlpack(owner: Any) -> MemoryBuffer: if not hasattr(owner, "__dlpack__"): raise InvalidArgumentError("Object does not support DLpack protocol") dlpack_object = DLPackObject(owner) return MemoryBuffer._from_dlpack_object(owner, dlpack_object) @staticmethod def _from_dlpack_object(owner: Any, dlpack_object: DLPackObject) -> MemoryBuffer: if not dlpack_object.contiguous: raise InvalidArgumentError("Only contiguous memory is supported") return MemoryBuffer( int(dlpack_object.data_ptr), dlpack_object.memory_type, dlpack_object.memory_type_id, dlpack_object.byte_size, owner, ) def _create_tritonserver_buffer_attributes(self) -> TRITONSERVER_BufferAttributes: buffer_attributes = TRITONSERVER_BufferAttributes() buffer_attributes.memory_type = self.memory_type buffer_attributes.memory_type_id = self.memory_type_id buffer_attributes.byte_size = self.size # TODO: Support allocation / use of cuda shared memory # buffer_attributes.cuda_ipc_handle = None return buffer_attributes class MemoryAllocator(ABC): """Abstract interface to allow for custom memory allocation strategies Classes implementing the MemoryAllocator interface have to provide an allocate method returning MemoryBuffer objects. A memory allocator implementation does not need to match the requested memory type or memory type id. Examples -------- class TorchAllocator(tritonserver.MemoryAllocator): def allocate(self, size, memory_type, memory_type_id): device = "cpu" if memory_type == tritonserver.MemoryType.GPU: device = "cuda" tensor = torch.zeros(size,dtype=torch.uint8,device=device) return tritonserver.MemoryBuffer.from_dlpack(tensor) """ @abstractmethod def allocate( self, size: int, memory_type: MemoryType, memory_type_id: int ) -> MemoryBuffer: """Allocate memory buffer for tensor. Note: A memory allocator implementation does not need to honor the requested memory type or memory type id Parameters ---------- size : int number of bytes requested memory_type : MemoryType type of memory requested (CPU, GPU, etc.) memory_type_id : int memory type id requested (typically device id) Returns ------- MemoryBuffer memory buffer with requested size Examples -------- memory_buffer = allocator.allocate(100, MemoryType.CPU, 0) """ pass class CPUAllocator(MemoryAllocator): def __init__(self): pass def allocate( self, size: int, memory_type: MemoryType, memory_type_id: int ) -> MemoryBuffer: ndarray = numpy.empty(size, numpy.byte) return MemoryBuffer.from_dlpack(ndarray) default_memory_allocators[MemoryType.CPU] = CPUAllocator() if cupy is not None: class GPUAllocator(MemoryAllocator): def __init__(self): pass def allocate( self, size: int, memory_type: MemoryType, memory_type_id: int, ) -> MemoryBuffer: with cupy.cuda.Device(memory_type_id): ndarray = cupy.empty(size, cupy.byte) return MemoryBuffer.from_dlpack(ndarray) default_memory_allocators[MemoryType.GPU] = GPUAllocator() class ResponseAllocator: def __init__( self, memory_allocator: Optional[MemoryAllocator] = None, device_or_memory_type: Optional[DeviceOrMemoryType] = None, ): self._memory_allocator = memory_allocator self._memory_type: Optional[MemoryType] = None self._memory_type_id: int = 0 self._response_allocator = None if device_or_memory_type is not None: self._memory_type, self._memory_type_id = parse_device_or_memory_type( device_or_memory_type ) if ( self._memory_type is not None and self._memory_allocator is None and self._memory_type not in default_memory_allocators ): raise InvalidArgumentError( f"Memory type {self._memory_type} not supported by default_memory_allocators: {default_memory_allocators}" ) def allocate( self, _allocator, tensor_name, byte_size, memory_type, memory_type_id, _user_object, ): try: if self._memory_type is not None: memory_type = self._memory_type memory_type_id = self._memory_type_id memory_allocator = self._memory_allocator if memory_allocator is None: if memory_type in default_memory_allocators: memory_allocator = default_memory_allocators[memory_type] else: LogMessage( LogLevel.WARN, f"Requested memory type {memory_type} is not supported, falling back to {MemoryType.CPU}", ) memory_type = MemoryType.CPU memory_type_id = 0 memory_allocator = default_memory_allocators[memory_type] memory_buffer = memory_allocator.allocate( byte_size, memory_type, memory_type_id ) return ( memory_buffer.data_ptr, memory_buffer, memory_buffer.memory_type, memory_buffer.memory_type_id, ) except Exception as e: message = f"Catastrophic failure in allocator: {e}, returning NULL" LogMessage(LogLevel.ERROR, message) return (0, None, MemoryType.CPU, 0) def release( self, _allocator, _buffer_, _buffer_user_object, _byte_size, _memory_type, _memory_type_id, ): pass def start(self, _allocator, _user_object): pass def query_preferred_memory_type( self, _allocator, _user_object, _tensor_name, _byte_size, memory_type: MemoryType, memory_type_id, ): if self._memory_type is not None: memory_type = self._memory_type memory_type_id = self._memory_type_id return (memory_type, memory_type_id) def set_buffer_attributes( self, _allocator, _tensor_name, buffer_attributes, _user_object, _buffer_user_object, ): if self._memory_type is not None: buffer_attributes.memory_type = self._memory_type buffer_attributes.memory_type_id = self._memory_type_id return buffer_attributes def create_tritonserver_response_allocator(self): self._response_allocator = TRITONSERVER_ResponseAllocator( self.allocate, self.release, self.start ) self._response_allocator.set_query_function(self.query_preferred_memory_type) self._response_allocator.set_buffer_attributes_function( self.set_buffer_attributes ) return self._response_allocator kytensor-2.49.0.7/core/python/tritonserver/_api/_datautils.py000066400000000000000000000247171527052371200243020ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations import ctypes import struct from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass from types import ModuleType from typing import Any, Callable, ClassVar, Optional, Sequence, Type import numpy from tritonserver._c import InvalidArgumentError from tritonserver._c import TRITONSERVER_DataType as DataType from tritonserver._c import TRITONSERVER_MemoryType as MemoryType from tritonserver._c import TRITONSERVER_ResponseAllocator, UnsupportedError from . import _dlpack try: import cupy except ImportError: cupy = None DeviceOrMemoryType = ( tuple[MemoryType, int] | MemoryType | tuple[_dlpack.DLDeviceType, int] | str ) class CustomKeyErrorDict(dict): def __init__( self, from_name: str, to_name: str, *args, exception: Type[Exception] = InvalidArgumentError, **kwargs, ): super().__init__(*args, **kwargs) self._to_name = to_name self._from_name = from_name self._exception = exception def __getitem__(self, key): try: return super().__getitem__(key) except KeyError: raise self._exception( f"Unsupported {self._from_name}. Can't convert {key} to {self._to_name}" ) from None STRING_TO_TRITON_MEMORY_TYPE: dict[str, MemoryType] = CustomKeyErrorDict( "Memory Type String", "Triton server memory type", {"CPU": MemoryType.CPU, "CPU_PINNED": MemoryType.CPU_PINNED, "GPU": MemoryType.GPU}, ) DLPACK_DEVICE_TYPE_TO_TRITON_MEMORY_TYPE: dict[ _dlpack.DLDeviceType, MemoryType ] = CustomKeyErrorDict( "DLPack device type", "Triton server memory type", { _dlpack.DLDeviceType.kDLCUDA: MemoryType.GPU, _dlpack.DLDeviceType.kDLCPU: MemoryType.CPU, }, ) TRITON_MEMORY_TYPE_TO_DLPACK_DEVICE_TYPE: dict[ MemoryType, _dlpack.DLDeviceType ] = CustomKeyErrorDict( "Triton server memory type", "DLPack device type", { **{ value: key for key, value in DLPACK_DEVICE_TYPE_TO_TRITON_MEMORY_TYPE.items() }, **{MemoryType.CPU_PINNED: _dlpack.DLDeviceType.kDLCPU}, }, ) DLPACK_TO_TRITON_DTYPE: dict[ tuple[_dlpack.DLDataTypeCode, int], DataType ] = CustomKeyErrorDict( "DLPack data type", "Triton server data type", { (_dlpack.DLDataTypeCode.kDLBool, 8): DataType.BOOL, (_dlpack.DLDataTypeCode.kDLInt, 8): DataType.INT8, ( _dlpack.DLDataTypeCode.kDLInt, 16, ): DataType.INT16, ( _dlpack.DLDataTypeCode.kDLInt, 32, ): DataType.INT32, ( _dlpack.DLDataTypeCode.kDLInt, 64, ): DataType.INT64, ( _dlpack.DLDataTypeCode.kDLUInt, 8, ): DataType.UINT8, ( _dlpack.DLDataTypeCode.kDLUInt, 16, ): DataType.UINT16, ( _dlpack.DLDataTypeCode.kDLUInt, 32, ): DataType.UINT32, ( _dlpack.DLDataTypeCode.kDLUInt, 64, ): DataType.UINT64, ( _dlpack.DLDataTypeCode.kDLFloat, 16, ): DataType.FP16, ( _dlpack.DLDataTypeCode.kDLFloat, 32, ): DataType.FP32, ( _dlpack.DLDataTypeCode.kDLFloat, 64, ): DataType.FP64, ( _dlpack.DLDataTypeCode.kDLBfloat, 16, ): DataType.BF16, }, ) TRITON_TO_DLPACK_DTYPE: dict[DataType, _dlpack.DLDataType] = CustomKeyErrorDict( "Triton server data type", "DLPack data type", { value: _dlpack.DLDataType(type_code=key[0], bits=key[1], lanes=1) for key, value in DLPACK_TO_TRITON_DTYPE.items() }, ) NUMPY_TO_TRITON_DTYPE: dict[type, DataType] = CustomKeyErrorDict( "Numpy data type", "Triton server data type", { bool: DataType.BOOL, numpy.bool_: DataType.BOOL, numpy.int8: DataType.INT8, numpy.int16: DataType.INT16, numpy.int32: DataType.INT32, numpy.int64: DataType.INT64, numpy.uint8: DataType.UINT8, numpy.uint16: DataType.UINT16, numpy.uint32: DataType.UINT32, numpy.uint64: DataType.UINT64, numpy.float16: DataType.FP16, numpy.float32: DataType.FP32, numpy.float64: DataType.FP64, numpy.bytes_: DataType.BYTES, numpy.str_: DataType.BYTES, numpy.object_: DataType.BYTES, }, ) TRITON_TO_NUMPY_DTYPE: dict[DataType, type] = CustomKeyErrorDict( "Triton data type", "Numpy data type", { **{value: key for key, value in NUMPY_TO_TRITON_DTYPE.items()}, **{DataType.BYTES: numpy.object_}, **{DataType.BOOL: numpy.bool_}, }, ) def parse_device_or_memory_type( device_or_memory_type: DeviceOrMemoryType, ) -> tuple[MemoryType, int]: if isinstance(device_or_memory_type, tuple): if isinstance(device_or_memory_type[0], MemoryType): memory_type = device_or_memory_type[0] memory_type_id = device_or_memory_type[1] elif isinstance(device_or_memory_type[0], _dlpack.DLDeviceType): memory_type = DLPACK_DEVICE_TYPE_TO_TRITON_MEMORY_TYPE[ device_or_memory_type[0] ] memory_type_id = device_or_memory_type[1] else: raise InvalidArgumentError(f"Invalid memory type {device_or_memory_type}") elif isinstance(device_or_memory_type, MemoryType): memory_type = device_or_memory_type memory_type_id = 0 elif isinstance(device_or_memory_type, str): memory_str_tuple = device_or_memory_type.split(":") if len(memory_str_tuple) > 2: raise InvalidArgumentError( f"Invalid memory type string {device_or_memory_type}" ) memory_type = STRING_TO_TRITON_MEMORY_TYPE[memory_str_tuple[0].upper()] if len(memory_str_tuple) == 2: try: memory_type_id = int(memory_str_tuple[1]) except ValueError: raise InvalidArgumentError( f"Invalid memory type string {device_or_memory_type}" ) from None else: memory_type_id = 0 return (memory_type, memory_type_id) class DLPackObject: def __init__(self, value) -> None: try: stream = None device, device_id = value.__dlpack_device__() if device == _dlpack.DLDeviceType.kDLCUDA: if cupy is None: raise UnsupportedError( f"DLPack synchronization on device {device,device_id} not supported" ) with cupy.cuda.Device(device_id): stream = 1 # legacy default stream self._capsule = _dlpack.get_dlpack_capsule(value, stream) self._tensor = _dlpack.get_managed_tensor(self._capsule).dl_tensor else: self._capsule = _dlpack.get_dlpack_capsule(value) self._tensor = _dlpack.get_managed_tensor(self._capsule).dl_tensor except Exception as e: raise InvalidArgumentError( f"Object does not support DLPack protocol: {e}" ) from None def __eq__(self, other) -> bool: if not isinstance(other, DLPackObject): return False if self.byte_size != other.byte_size: return False if self.memory_type != other.memory_type: return False if self.memory_type_id != other.memory_type_id: return False if self.shape != other.shape: return False if self.data_ptr != other.data_ptr: return False if self.contiguous != other.contiguous: return False if self.triton_data_type != other.triton_data_type: return False return True @property def byte_size(self) -> int: return _dlpack.get_byte_size( self._tensor.dtype, self._tensor.ndim, self._tensor.shape ) @property def memory_type(self) -> MemoryType: return DLPACK_DEVICE_TYPE_TO_TRITON_MEMORY_TYPE[self._tensor.device.device_type] @property def memory_type_id(self) -> int: return self._tensor.device.device_id @property def shape(self) -> list[int]: return [self._tensor.shape[i] for i in range(self._tensor.ndim)] @property def triton_data_type(self) -> DataType: return DLPACK_TO_TRITON_DTYPE[self.data_type] @property def data_type(self) -> tuple[_dlpack.DLDataTypeCode, int]: return (self._tensor.dtype.type_code, self._tensor.dtype.bits) @property def data_ptr(self) -> ctypes.c_void_p: return self._tensor.data + self._tensor.byte_offset @property def contiguous(self) -> bool: return _dlpack.is_contiguous_data( self._tensor.ndim, self._tensor.shape, self._tensor.strides ) kytensor-2.49.0.7/core/python/tritonserver/_api/_dlpack.py000066400000000000000000000213301527052371200235320ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ################################################################################ # This file contains the DLPack API wrapped in Python style (see # 'dlpack.h' for detail) and the utilities for Triton client to interact # with DLPack # # Ref: # https://github.com/dmlc/dlpack/blob/main/include/dlpack/dlpack.h # https://github.com/dmlc/dlpack/blob/main/apps/numpy_dlpack/dlpack/from_numpy.py ################################################################################ import ctypes # Need to explicit set the res / arg types for pythonapi functions to # work properly ctypes.pythonapi.PyMem_RawMalloc.restype = ctypes.c_void_p ctypes.pythonapi.PyMem_RawFree.argtypes = [ctypes.c_void_p] ctypes.pythonapi.PyCapsule_New.restype = ctypes.py_object ctypes.pythonapi.PyCapsule_New.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p, ] ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object, ctypes.c_char_p] c_str_dltensor = b"dltensor" class DLDeviceType(ctypes.c_int): kDLCPU = 1 kDLCUDA = 2 kDLCUDAHost = 3 kDLOpenCL = 4 kDLVulkan = 7 kDLMetal = 8 kDLVPI = 9 kDLROCM = 10 kDLROCMHost = 11 kDLExtDev = 12 kDLCUDAManaged = 13 kDLOneAPI = 14 kDLWebGPU = 15 kDLHexagon = 16 class DLDevice(ctypes.Structure): _fields_ = [ ("device_type", ctypes.c_int), ("device_id", ctypes.c_int), ] class DLDataTypeCode(ctypes.c_uint8): kDLInt = 0 kDLUInt = 1 kDLFloat = 2 kDLOpaquePointer = 3 kDLBfloat = 4 kDLComplex = 5 kDLBool = 6 class DLDataType(ctypes.Structure): _fields_ = [ ("type_code", ctypes.c_uint8), ("bits", ctypes.c_uint8), ("lanes", ctypes.c_uint16), ] class DLTensor(ctypes.Structure): _fields_ = [ ("data", ctypes.c_void_p), ("device", DLDevice), ("ndim", ctypes.c_int), ("dtype", DLDataType), ("shape", ctypes.POINTER(ctypes.c_int64)), ("strides", ctypes.POINTER(ctypes.c_int64)), ("byte_offset", ctypes.c_uint64), ] class DLManagedTensor(ctypes.Structure): _fields_ = [ ("dl_tensor", DLTensor), ("manager_ctx", ctypes.c_void_p), ("deleter", ctypes.CFUNCTYPE(None, ctypes.c_void_p)), ] # Utilities def _raise_error(msg): """ Raise error with the provided message """ raise Exception(msg=msg) from None # Use as managed context in DLPack that doesn't hold ownership of the # data content. class DataViewContext: def __init__(self, shape) -> None: # Convert the Python object to ctypes objects expected by # DLPack self._shape = (ctypes.c_int64 * len(shape))(*shape) # No strides: compact and row-major self._strides = ctypes.POINTER(ctypes.c_int64)() def as_manager_ctx(self) -> ctypes.c_void_p: py_obj = ctypes.py_object(self) py_obj_ptr = ctypes.pointer(py_obj) ctypes.pythonapi.Py_IncRef(py_obj) ctypes.pythonapi.Py_IncRef(ctypes.py_object(py_obj_ptr)) return ctypes.cast(py_obj_ptr, ctypes.c_void_p) @ctypes.CFUNCTYPE(None, ctypes.c_void_p) def managed_tensor_deleter(handle: ctypes.c_void_p) -> None: dl_managed_tensor = DLManagedTensor.from_address(handle) py_obj_ptr = ctypes.cast( dl_managed_tensor.manager_ctx, ctypes.POINTER(ctypes.py_object) ) py_obj = py_obj_ptr.contents ctypes.pythonapi.Py_DecRef(py_obj) ctypes.pythonapi.Py_DecRef(ctypes.py_object(py_obj_ptr)) ctypes.pythonapi.PyMem_RawFree(handle) @ctypes.CFUNCTYPE(None, ctypes.c_void_p) def pycapsule_deleter(handle: ctypes.c_void_p) -> None: pycapsule: ctypes.py_object = ctypes.cast(handle, ctypes.py_object) if ctypes.pythonapi.PyCapsule_IsValid(pycapsule, c_str_dltensor): dl_managed_tensor = ctypes.pythonapi.PyCapsule_GetPointer( pycapsule, c_str_dltensor ) managed_tensor_deleter(dl_managed_tensor) ctypes.pythonapi.PyCapsule_SetDestructor(pycapsule, None) def triton_to_dlpack_dtype(dtype): if dtype == "BOOL": type_code = DLDataTypeCode.kDLBool bits = 8 elif dtype == "INT8": type_code = DLDataTypeCode.kDLInt bits = 8 elif dtype == "INT16": type_code = DLDataTypeCode.kDLInt bits = 16 elif dtype == "INT32": type_code = DLDataTypeCode.kDLInt bits = 32 elif dtype == "INT64": type_code = DLDataTypeCode.kDLInt bits = 64 elif dtype == "UINT8": type_code = DLDataTypeCode.kDLUInt bits = 8 elif dtype == "UINT16": type_code = DLDataTypeCode.kDLUInt bits = 16 elif dtype == "UINT32": type_code = DLDataTypeCode.kDLUInt bits = 32 elif dtype == "UINT64": type_code = DLDataTypeCode.kDLUInt bits = 64 elif dtype == "FP16": type_code = DLDataTypeCode.kDLFloat bits = 16 elif dtype == "FP32": type_code = DLDataTypeCode.kDLFloat bits = 32 elif dtype == "FP64": type_code = DLDataTypeCode.kDLFloat bits = 64 elif dtype == "BF16": type_code = DLDataTypeCode.kDLBfloat bits = 16 elif dtype == "BYTES": _raise_error("DLPack currently doesn't support BYTES type") else: _raise_error( "Can not convert unknown data type '{}' to DLPack data type".format(dtype) ) return DLDataType(type_code, bits, 1) def is_contiguous_data( ndim: ctypes.c_int, shape: ctypes.POINTER(ctypes.c_int64), stride: ctypes.POINTER(ctypes.c_int64), ): # If 'stride' doesn't capture valid value if (stride is None) or (not bool(stride)): return True calculated_stride = 1 # iterate stride in reverse order [ndim-1, -1) for i in reversed(range(ndim)): if stride[i] != calculated_stride: return False calculated_stride *= shape[i] return True def get_byte_size( dtype: DLDataType, ndim: ctypes.c_int, shape: ctypes.POINTER(ctypes.c_int64) ): element_byte_size = dtype.bits * dtype.lanes // 8 # Assume 8 bits in a byte for i in range(ndim): element_byte_size *= shape[i] return element_byte_size def get_dlpack_capsule(dlpack_obj, stream=None): # Extract PyCapsule of the DLPack object if hasattr(dlpack_obj, "__dlpack__"): if not hasattr(dlpack_obj, "__dlpack_device__"): _raise_error( "DLPack expects '__dlpack_device__' if '__dlpack__' has been defined" ) device = dlpack_obj.__dlpack_device__() # Have to condition on the device type as, using numpy as example, # some DLPack implementation doesn't accept 'stream' as arguments if device != DLDeviceType.kDLCUDA: return dlpack_obj.__dlpack__() else: return dlpack_obj.__dlpack__(stream) else: # Old interface where PyCapsule object is passed directly return dlpack_obj def get_dlpack_device(dlpack_obj): if hasattr(dlpack_obj, "__dlpack_device__"): return dlpack_obj.__dlpack_device__() return None def get_managed_tensor(dlcapsule): ptr = ctypes.pythonapi.PyCapsule_GetPointer(dlcapsule, c_str_dltensor) return DLManagedTensor.from_address(ptr) kytensor-2.49.0.7/core/python/tritonserver/_api/_logging.py000066400000000000000000000046201527052371200237250ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logging Utilities""" import inspect import os from tritonserver._c.triton_bindings import TRITONSERVER_LogLevel as LogLevel from tritonserver._c.triton_bindings import TRITONSERVER_LogMessage def LogMessage(level: LogLevel, message: str): """Log Message using Triton Inference Server Logger Parameters ---------- level : LogLevel log level one of LogLevel.WARN, LogLevel.ERROR, LogLevel.INFO message : str message Examples -------- LogMessage(LogLevel.ERROR,"I've got a bad feeling about this ...") """ filename, line_number = "unknown", -1 try: current_frame = inspect.stack()[-1] filename, line_number = ( os.path.basename(current_frame.filename), current_frame.lineno, ) except Exception as e: pass TRITONSERVER_LogMessage(level, filename, line_number, message) kytensor-2.49.0.7/core/python/tritonserver/_api/_model.py000066400000000000000000000414551527052371200234060ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Class for interacting with Triton Inference Server Models""" from __future__ import annotations import asyncio import json import queue from typing import TYPE_CHECKING, Any, Optional from tritonserver._api._allocators import ResponseAllocator from tritonserver._api._request import InferenceRequest from tritonserver._api._response import AsyncResponseIterator, ResponseIterator from tritonserver._c.triton_bindings import InvalidArgumentError from tritonserver._c.triton_bindings import ( TRITONSERVER_ModelBatchFlag as ModelBatchFlag, ) from tritonserver._c.triton_bindings import ( TRITONSERVER_ModelTxnPropertyFlag as ModelTxnPropertyFlag, ) from tritonserver._c.triton_bindings import TRITONSERVER_Server if TYPE_CHECKING: from tritonserver._api._server import Server class Model: """Class for interacting with Triton Inference Server models Model objects are returned from server factory methods and allow users to query metadata and execute inference requests. """ def __init__( self, server: Server, name: str, version: int = -1, state: Optional[str] = None, reason: Optional[str] = None, ) -> None: """Initialize model Model objects should be obtainted from Server factory methods and not instantiated directly. See `Server` documentation. Parameters ---------- server : Server Server instance. name : str model name version : int model version state : Optional[str] state of model (if known) reason : Optional[str] reason for model state (if known) Examples -------- >>> server.model("test") server.model("test") {'name': 'test', 'version': -1, 'state': None} """ self._server = server._server if not isinstance(self._server, TRITONSERVER_Server): raise InvalidArgumentError("Server not started") self.name = name self.version = version self.state = state self.reason = reason def create_request(self, **kwargs: Unpack[InferenceRequest]) -> InferenceRequest: """Inference request factory method Return an inference request object that can be used with model.infer() ro model.async_infer() Parameters ---------- kwargs : Unpack[InferenceRequest] Keyword arguments passed to `InferenceRequest` constructor. See `InferenceRequest` documentation for details. Returns ------- InferenceRequest Inference request associated with this model Examples -------- >>> server.model("test").create_request() server.model("test").create_request() InferenceRequest(model={'name': 'test', 'version': -1, 'state': None}, _server=, request_id=None, flags=0, correlation_id=None, priority=0, timeout=0, inputs={}, parameters={}, output_memory_type=None, output_memory_allocator=None, response_queue=None, _serialized_inputs={}) """ if "model" in kwargs: kwargs.pop("model") return InferenceRequest(model=self, **kwargs) def async_infer( self, inference_request: Optional[InferenceRequest] = None, raise_on_error: bool = True, **kwargs: Unpack[InferenceRequest], ) -> AsyncResponseIterator: """Send an inference request to the model for execution Sends an inference request to the model. Responses are returned using an asyncio compatible iterator. See c:func:`TRITONSERVER_ServerInferAsync` Parameters ---------- inference_request : Optional[InferenceRequest] inference request object. If not provided inference request will be created using remaining keyword arguments. raise_on_error : bool, default True if True iterator will raise an error on any response errors returned from the model. If False errors will be returned as part of the response object. kwargs : Unpack[InferenceRequest] If a request object is not provided, a new object will be created with remaining keyword arguments. See `InferenceRequest` documentation for valid arguments. Returns ------- AsyncResponseIterator asyncio compatible iterator Raises ------ InvalidArgumentError if any invalid arguments are provided """ if inference_request is None: inference_request = InferenceRequest(model=self, **kwargs) if (inference_request.response_queue is not None) and ( not isinstance(inference_request.response_queue, asyncio.Queue) ): raise InvalidArgumentError( "asyncio.Queue must be used for async response iterator" ) request = inference_request._create_tritonserver_inference_request() response_iterator = AsyncResponseIterator( self, request, inference_request.response_queue, raise_on_error, ) response_allocator = ResponseAllocator( inference_request.output_memory_allocator, inference_request.output_memory_type, ).create_tritonserver_response_allocator() request.set_response_callback( response_allocator, None, response_iterator._response_callback, None ) self._server.infer_async(request) return response_iterator def infer( self, inference_request: Optional[InferenceRequest] = None, raise_on_error: bool = True, **kwargs: Unpack[InferenceRequest], ) -> ResponseIterator: """Send an inference request to the model for execution Sends an inference request to the model. Responses are returned asynchronously using an iterator. See c:func:`TRITONSERVER_ServerInferAsync` Parameters ---------- inference_request : Optional[InferenceRequest] inference request object. If not provided inference request will be created using remaining keyword arguments. raise_on_error : bool, default True if True iterator will raise an error on any response errors returned from the model. If False errors will be returned as part of the response object. kwargs : Unpack[InferenceRequest] If a request object is not provided, a new object will be created with remaining keyword arguments. See `InferenceRequest` documentation for valid arguments. Returns ------- ResponseIterator Response iterator Raises ------ InvalidArgumentError if any invalid arguments are provided Examples -------- >>> responses = server.model("test_2").infer(inputs={"text_input":["hello"]}) responses = list(server.model("test_2").infer(inputs={"text_input":["hello"]})) >>> response = responses[0] print(response) InferenceResponse(model={'name': 'test_2', 'version': 1, 'state': None}, _server=, request_id='', parameters={}, outputs={'text_output': Tensor(data_type=, shape=array([1]), memory_buffer=MemoryBuffer(data_ptr=140003384498080, memory_type=, memory_type_id=0, size=9, owner=array([ 5, 0, 0, 0, 104, 101, 108, 108, 111], dtype=int8)))}, error=None, classification_label=None, final=False) >>> response.outputs["text_output"].to_bytes_array() response.outputs["text_output"].to_bytes_array() array([b'hello'], dtype=object) """ if inference_request is None: inference_request = InferenceRequest(model=self, **kwargs) if (inference_request.response_queue is not None) and ( not isinstance(inference_request.response_queue, queue.SimpleQueue) ): raise InvalidArgumentError( "queue.SimpleQueue must be used for response iterator" ) request = inference_request._create_tritonserver_inference_request() response_iterator = ResponseIterator( self, request, inference_request.response_queue, raise_on_error, ) response_allocator = ResponseAllocator( inference_request.output_memory_allocator, inference_request.output_memory_type, ).create_tritonserver_response_allocator() request.set_response_callback( response_allocator, None, response_iterator._response_callback, None ) self._server.infer_async(request) return response_iterator def metadata(self) -> dict[str, Any]: """Returns medatadata about a model and its inputs and outputs See c:func:`TRITONSERVER_ServerModelMetadata()` Returns ------- dict[str, Any] Model metadata as a dictionary of key value pairs Examples -------- server.model("test").metadata() {'name': 'test', 'versions': ['1'], 'platform': 'python', 'inputs': [{'name': 'text_input', 'datatype': 'BYTES', 'shape': [-1]}, {'name': 'fp16_input', 'datatype': 'FP16', 'shape': [-1, 1]}], 'outputs': [{'name': 'text_output', 'datatype': 'BYTES', 'shape': [-1]}, {'name': 'fp16_output', 'datatype': 'FP16', 'shape': [-1, 1]}]} """ return json.loads( self._server.model_metadata(self.name, self.version).serialize_to_json() ) def ready(self) -> bool: """Returns whether a model is ready to accept requests See :c:func:`TRITONSERVER_ServerModelIsReady()` Returns ------- bool True if model is ready. False otherwise. Examples -------- >>> server.model("test").ready() server.model("test").ready() True """ return self._server.model_is_ready(self.name, self.version) def batch_properties(self) -> ModelBatchFlag: """Returns the batch properties of the model See :c:func:`TRITONSERVER_ServerModelBatchProperties` Returns ------- ModelBatchFlag ModelBatchFlag.UNKNOWN or ModelBatchFlag.FIRST_DIM Examples -------- >>> server.model("resnet50_libtorch").batch_properties() server.model("resnet50_libtorch").batch_properties() """ flags, _ = self._server.model_batch_properties(self.name, self.version) return ModelBatchFlag(flags) def transaction_properties(self) -> ModelTxnPropertyFlag: """Returns the transaction properties of the model See :c:func:`TRITONSERVER_ServerModelTransactionProperties` Returns ------- ModelTxnPropertyFlag ModelTxnPropertyFlag.ONE_TO_ONE or ModelTxnPropertyFlag.DECOUPLED Examples -------- >>> server.model("resnet50_libtorch").transaction_properties() server.model("resnet50_libtorch").transaction_properties() """ txn_properties, _ = self._server.model_transaction_properties( self.name, self.version ) return ModelTxnPropertyFlag(txn_properties) def statistics(self) -> dict[str, Any]: """Returns model statistics See :c:func:`TRITONSERVER_ServerModelStatistics` Returns ------- dict[str, Any] Dictionary of key value pairs representing model statistics Examples -------- >>> server.model("test").statistics() server.model("test").statistics() {'model_stats': [{'name': 'test', 'version': '1', 'last_inference': 1704731597736, 'inference_count': 2, 'execution_count': 2, 'inference_stats': {'success': {'count': 2, 'ns': 3079473}, 'fail': {'count': 0, 'ns': 0}, 'queue': {'count': 2, 'ns': 145165}, 'compute_input': {'count': 2, 'ns': 124645}, 'compute_infer': {'count': 2, 'ns': 2791809}, 'compute_output': {'count': 2, 'ns': 10240}, 'cache_hit': {'count': 0, 'ns': 0}, 'cache_miss': {'count': 0, 'ns': 0}}, 'batch_stats': [{'batch_size': 1, 'compute_input': {'count': 2, 'ns': 124645}, 'compute_infer': {'count': 2, 'ns': 2791809}, 'compute_output': {'count': 2, 'ns': 10240}}], 'memory_usage': []}]} """ return json.loads( self._server.model_statistics(self.name, self.version).serialize_to_json() ) def config(self, config_version: int = 1) -> dict[str, Any]: """Returns model configuration See :c:func:`TRITONSERVER_ServerModelConfiguration` Parameters ---------- config_version : int configuration version in case multiple are supported Returns ------- dict[str, Any] Dictionary of key value pairs for model configuration Examples -------- >>> server.model("test").config() server.model("test").config() {'name': 'test', 'platform': '', 'backend': 'python', 'version_policy': {'latest': {'num_versions': 1}}, 'max_batch_size': 0, 'input': [{'name': 'text_input', 'data_type': 'TYPE_STRING', 'format': 'FORMAT_NONE', 'dims': [-1], 'is_shape_tensor': False, 'allow_ragged_batch': False, 'optional': True}, {'name': 'fp16_input', 'data_type': 'TYPE_FP16', 'format': 'FORMAT_NONE', 'dims': [-1, 1], 'is_shape_tensor': False, 'allow_ragged_batch': False, 'optional': True}], 'output': [{'name': 'text_output', 'data_type': 'TYPE_STRING', 'dims': [-1], 'label_filename': '', 'is_shape_tensor': False}, {'name': 'fp16_output', 'data_type': 'TYPE_FP16', 'dims': [-1, 1], 'label_filename': '', 'is_shape_tensor': False}], 'batch_input': [], 'batch_output': [], 'optimization': {'priority': 'PRIORITY_DEFAULT', 'input_pinned_memory': {'enable': True}, 'output_pinned_memory': {'enable': True}, 'gather_kernel_buffer_threshold': 0, 'eager_batching': False}, 'instance_group': [{'name': 'test_2', 'kind': 'KIND_GPU', 'count': 1, 'gpus': [0], 'secondary_devices': [], 'profile': [], 'passive': False, 'host_policy': ''}], 'default_model_filename': 'model.py', 'cc_model_filenames': {}, 'metric_tags': {}, 'parameters': {}, 'model_warmup': [], 'model_transaction_policy': {'decoupled': True}} """ return json.loads( self._server.model_config( self.name, self.version, config_version ).serialize_to_json() ) def __repr__(self): return str(self) def __str__(self): return "%s" % ( {"name": self.name, "version": self.version, "state": self.state} ) kytensor-2.49.0.7/core/python/tritonserver/_api/_request.py000066400000000000000000000202341527052371200237660ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Class for sending inference requests to Triton Inference Server Models""" from __future__ import annotations import asyncio import queue from dataclasses import dataclass, field from typing import Any, Optional from tritonserver._api import _model from tritonserver._api._allocators import MemoryAllocator from tritonserver._api._datautils import CustomKeyErrorDict from tritonserver._api._dlpack import DLDeviceType as DLDeviceType from tritonserver._api._tensor import Tensor from tritonserver._c.triton_bindings import InvalidArgumentError from tritonserver._c.triton_bindings import TRITONSERVER_DataType as DataType from tritonserver._c.triton_bindings import TRITONSERVER_InferenceRequest from tritonserver._c.triton_bindings import TRITONSERVER_MemoryType as MemoryType from tritonserver._c.triton_bindings import TRITONSERVER_Server DeviceOrMemoryType = ( tuple[MemoryType, int] | MemoryType | tuple[DLDeviceType, int] | str ) @dataclass class InferenceRequest: """Dataclass representing an inference request. Inference request objects are created using Model factory methods. They contain input parameters and input data as well as configuration for response output memory allocation. See c:func:`TRITONSERVER_InferenceRequest` for more details Parameters ---------- model : Model Model instance associated with the inference request. request_id : Optional[str], default None Unique identifier for the inference request. flags : int, default 0 Flags indicating options for the inference request. correlation_id : Optional[Union[int, str]], default None Correlation ID associated with the inference request. priority : int, default 0 Priority of the inference request. timeout : int, default 0 Timeout for the inference request in microseconds. inputs : Dict[str, Union[Tensor, Any]], default {} Dictionary of input names and corresponding input tensors or data. parameters : Dict[str, Union[str, int, bool, float]], default {} Dictionary of parameters for the inference request. output_memory_type : Optional[DeviceOrMemoryType], default None output_memory_type : Optional[DeviceOrMemoryType], default None Type of memory to allocate for inference response output. If not provided memory type will be chosen based on backend / model preference with MemoryType.CPU as fallback. Memory type can be given as a string, MemoryType, tuple [MemoryType, memory_type__id], or tuple[DLDeviceType, device_id]. output_memory_allocator : Optional[MemoryAllocator], default None Memory allocator to use for inference response output. If not provided default allocators will be used to allocate MemoryType.GPU or MemoryType.CPU memory as set in output_memory_type or as requested by backend / model preference. response_queue : Optional[Union[queue.SimpleQueue, asyncio.Queue]], default None Queue for asynchronous handling of inference responses. If provided Inference responses will be added to the queue in addition to the response iterator. Must be queue.SimpleQueue for non asyncio requests and asyncio.Queue for asyncio requests. Examples -------- # Creating a request explicitly >>> request = server.model("test").create_request() request = server.model("test").create_request() request.inputs["fp16_input"] = numpy.array([[1.0]]).astype(numpy.float16) for response in server.model("test_2").infer(request): print(numpy.from_dlpack(response.outputs["fp16_output"])) [[1.]] # Creating a request implicitly for response in server.model("test_2").infer( inputs={"fp16_input": numpy.array([[1.0]]).astype(numpy.float16)} ): print(numpy.from_dlpack(response.outputs["fp16_output"])) [[1.]] """ model: _model.Model request_id: Optional[str] = None flags: int = 0 correlation_id: Optional[int | str] = None priority: int = 0 timeout: int = 0 inputs: dict[str, Tensor | Any] = field(default_factory=dict) parameters: dict[str, str | int | bool | float] = field(default_factory=dict) output_memory_type: Optional[DeviceOrMemoryType] = None output_memory_allocator: Optional[MemoryAllocator] = None response_queue: Optional[queue.SimpleQueue | asyncio.Queue] = None _serialized_inputs: dict[str, Tensor] = field(init=False, default_factory=dict) _server: TRITONSERVER_Server = field(init=False) _set_parameter_methods = CustomKeyErrorDict( "Value", "Request Parameter", { str: TRITONSERVER_InferenceRequest.set_string_parameter, int: TRITONSERVER_InferenceRequest.set_int_parameter, float: TRITONSERVER_InferenceRequest.set_double_parameter, bool: TRITONSERVER_InferenceRequest.set_bool_parameter, }, ) def __post_init__(self): self._server = self.model._server def _release_request(self, _request, _flags, _user_object): pass def _add_inputs(self, request): for name, value in self.inputs.items(): if not isinstance(value, Tensor): tensor = Tensor._from_object(value) else: tensor = value if tensor.data_type == DataType.BYTES: # to ensure lifetime of array self._serialized_inputs[name] = tensor request.add_input(name, tensor.data_type, tensor.shape) request.append_input_data_with_buffer_attributes( name, tensor.data_ptr, tensor.memory_buffer._create_tritonserver_buffer_attributes(), ) def _set_parameters(self, request): for key, value in self.parameters.items(): InferenceRequest._set_parameter_methods[type(value)](request, key, value) def _create_tritonserver_inference_request(self): request = TRITONSERVER_InferenceRequest( self._server, self.model.name, self.model.version ) if self.request_id is not None: request.id = self.request_id request.priority_uint64 = self.priority request.timeout_microseconds = self.timeout if self.correlation_id is not None: if isinstance(self.correlation_id, int): request.correlation_id = self.correlation_id else: request.correlation_id_string = self.correlation_id request.flags = self.flags self._add_inputs(request) self._set_parameters(request) request.set_release_callback(self._release_request, None) return request kytensor-2.49.0.7/core/python/tritonserver/_api/_response.py000066400000000000000000000332641527052371200241430ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Class for receiving inference responses to Triton Inference Server Models""" from __future__ import annotations import asyncio import inspect import queue from dataclasses import asdict, dataclass, field from typing import TYPE_CHECKING, Optional from tritonserver._api import _model if TYPE_CHECKING: from tritonserver._api._model import Model from tritonserver._api._logging import LogMessage from tritonserver._api._tensor import Tensor from tritonserver._c.triton_bindings import ( InternalError, TritonError, TRITONSERVER_InferenceRequest, ) from tritonserver._c.triton_bindings import TRITONSERVER_LogLevel as LogLevel from tritonserver._c.triton_bindings import ( TRITONSERVER_ResponseCompleteFlag, TRITONSERVER_Server, ) class AsyncResponseIterator: """Asyncio compatible response iterator Response iterators are returned from model inference methods and allow users to process inference responses in the order they were received for a request. """ def __init__( self, model: _model.Model, request: TRITONSERVER_InferenceRequest, user_queue: Optional[asyncio.Queue] = None, raise_on_error: bool = False, loop: Optional[asyncio.AbstractEventLoop] = None, ) -> None: """Initialize AsyncResponseIterator AsyncResponseIterator objects are obtained from Model inference methods and not instantiated directly. See `Model` documentation. Parameters ---------- model : Model model associated with inference request request : TRITONSERVER_InferenceRequest Underlying C binding TRITONSERVER_InferenceRequest object. Private. user_queue : Optional[asyncio.Queue] Optional user queue for responses in addition to internal iterator queue. raise_on_error : bool if True response errors will be raised as exceptions. loop : Optional[asyncio.AbstractEventLoop] asyncio loop object """ if loop is None: loop = asyncio.get_running_loop() self._loop = loop self._queue = asyncio.Queue() self._user_queue = user_queue self._complete = False self._request = request self._model = model self._raise_on_error = raise_on_error def __aiter__(self) -> AsyncResponseIterator: """Return async iterator. For use with async for loops. Returns ------- AsyncResponseIterator Examples -------- responses = server.model("test").async_infer(inputs={"fp16_input":numpy.array([[1]],dtype=numpy.float16)}) async for response in responses: print(nummpy.from_dlpack(response.outputs["fp16_output"])) """ return self async def __anext__(self): """Returns the next response received for a request Returns the next response received for a request as an awaitable object. Raises ------ response.error If raise_on_error is set to True, response errors are raised as exceptions StopAsyncIteration Indicates all responses for a request have been received. Final responses may or may not include outputs and must be checked. """ if self._complete: raise StopAsyncIteration response = await self._queue.get() self._complete = response.final if response.error is not None and self._raise_on_error: raise response.error return response def cancel(self) -> None: """Cancel an inflight request Cancels an in-flight request. Cancellation is handled on a best effort basis and may not prevent execution of a request if it is already started or completed. See c:func:`TRITONSERVER_ServerInferenceRequestCancel` Examples -------- responses = server.model("test").infer(inputs={"text_input":["hello"]}) responses.cancel() """ if self._request is not None: self._request.cancel() def _response_callback(self, response, flags, unused): try: if self._request is None: raise InternalError("Response received after final response flag") response = InferenceResponse._from_tritonserver_inference_response( self._model, self._request, response, flags ) asyncio.run_coroutine_threadsafe(self._queue.put(response), self._loop) if self._user_queue is not None: asyncio.run_coroutine_threadsafe( self._user_queue.put(response), self._loop ) if flags == TRITONSERVER_ResponseCompleteFlag.FINAL: del self._request self._request = None except Exception as e: message = f"Catastrophic failure in response callback: {e}" LogMessage(LogLevel.ERROR, message) # catastrophic failure raise e from None class ResponseIterator: """Response iterator Response iterators are returned from model inference methods and allow users to process inference responses in the order they were received for a request. """ def __init__( self, model: Model, request: TRITONSERVER_InferenceRequest, user_queue: Optional[queue.SimpleQueue] = None, raise_on_error: bool = False, ): """Initialize ResponseIterator ResponseIterator objects are obtained from Model inference methods and not instantiated directly. See `Model` documentation. Parameters ---------- model : Model model associated with inference request request : TRITONSERVER_InferenceRequest Underlying C binding TRITONSERVER_InferenceRequest object. Private. user_queue : Optional[asyncio.Queue] Optional user queue for responses in addition to internal iterator queue. raise_on_error : bool if True response errors will be raised as exceptions. """ self._queue = queue.SimpleQueue() self._user_queue = user_queue self._complete = False self._request = request self._model = model self._raise_on_error = raise_on_error def __iter__(self) -> ResponseIterator: """Return response iterator. Returns ------- ResponseIterator Examples -------- responses = server.model("test").infer(inputs={"fp16_input":numpy.array([[1]],dtype=numpy.float16)}) for response in responses: print(nummpy.from_dlpack(response.outputs["fp16_output"])) """ return self def __next__(self): """Returns the next response received for a request Raises ------ response.error If raise_on_error is set to True, response errors are raised as exceptions StopIteration Indicates all responses for a request have been received. Final responses may or may not include outputs and must be checked. """ if self._complete: raise StopIteration response = self._queue.get() self._complete = response.final if response.error is not None and self._raise_on_error: raise response.error return response def cancel(self): """Cancel an inflight request Cancels an in-flight request. Cancellation is handled on a best effort basis and may not prevent execution of a request if it is already started or completed. See c:func:`TRITONSERVER_ServerInferenceRequestCancel` Examples -------- responses = server.model("test").infer(inputs={"text_input":["hello"]}) responses.cancel() """ if self._request is not None: self._request.cancel() def _response_callback(self, response, flags, unused): try: if self._request is None: raise InternalError("Response received after final response flag") response = InferenceResponse._from_tritonserver_inference_response( self._model, self._request, response, flags ) self._queue.put(response) if self._user_queue is not None: self._user_queue.put(response) if flags == TRITONSERVER_ResponseCompleteFlag.FINAL: del self._request self._request = None except Exception as e: message = f"Catastrophic failure in response callback: {e}" LogMessage(LogLevel.ERROR, message) # catastrophic failure raise e from None @dataclass class InferenceResponse: """Dataclass representing an inference response. Inference response objects are returned from response iterators which are in turn returned from model inference methods. They contain output data, output parameters, any potential errors reported and a flag to indicate if the response is the final one for a request. See c:func:`TRITONSERVER_InferenceResponse` for more details Parameters ---------- model : Model Model instance associated with the response. request_id : Optional[str], default None Unique identifier for the inference request (if provided) parameters : dict[str, str | int | bool], default {} Additional parameters associated with the response. outputs : dict [str, Tensor], default {} Output tensors for the inference. error : Optional[TritonError], default None Error (if any) that occurred in the processing of the request. classification_label : Optional[str], default None Classification label associated with the inference. Not currently supported. final : bool, default False Flag indicating if the response is final """ model: _model.Model request_id: Optional[str] = None parameters: dict[str, str | int | bool] = field(default_factory=dict) outputs: dict[str, Tensor] = field(default_factory=dict) error: Optional[TritonError] = None classification_label: Optional[str] = None final: bool = False @staticmethod def _from_tritonserver_inference_response( model: _model.Model, request: TRITONSERVER_InferenceRequest, response, flags: TRITONSERVER_ResponseCompleteFlag, ): result = InferenceResponse( model, request.id, final=(flags == TRITONSERVER_ResponseCompleteFlag.FINAL), ) try: if response is None: return result try: response.throw_if_response_error() except TritonError as error: error.args += (result,) result.error = error name, version = response.model result.model.name = name result.model.version = version result.request_id = response.id parameters = {} for parameter_index in range(response.parameter_count): name, type_, value = response.parameter(parameter_index) parameters[name] = value result.parameters = parameters outputs = {} for output_index in range(response.output_count): ( name, data_type, shape, _data_ptr, _byte_size, _memory_type, _memory_type_id, memory_buffer, ) = response.output(output_index) tensor = Tensor(data_type, shape, memory_buffer) outputs[name] = tensor result.outputs = outputs except Exception as e: error = InternalError(f"Unexpected error in creating response object: {e}") error.args += (result,) result.error = error # TODO: support classification # values["classification_label"] = response.output_classification_label() return result kytensor-2.49.0.7/core/python/tritonserver/_api/_server.py000066400000000000000000001113221527052371200236030ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Classes for configuring, instantiating, and interacting with an in-process Triton Inference Server.""" from __future__ import annotations import ctypes import json import time from dataclasses import dataclass, field from typing import Annotated, Any, Optional from tritonserver._api._model import Model from tritonserver._c.triton_bindings import InvalidArgumentError from tritonserver._c.triton_bindings import ( TRITONSERVER_InstanceGroupKind as InstanceGroupKind, ) from tritonserver._c.triton_bindings import TRITONSERVER_LogFormat as LogFormat from tritonserver._c.triton_bindings import TRITONSERVER_LogLevel as LogLevel from tritonserver._c.triton_bindings import TRITONSERVER_Metric from tritonserver._c.triton_bindings import TRITONSERVER_MetricFamily as MetricFamily from tritonserver._c.triton_bindings import TRITONSERVER_MetricFormat as MetricFormat from tritonserver._c.triton_bindings import TRITONSERVER_MetricKind as MetricKind from tritonserver._c.triton_bindings import ( TRITONSERVER_ModelControlMode as ModelControlMode, ) from tritonserver._c.triton_bindings import TRITONSERVER_Parameter from tritonserver._c.triton_bindings import TRITONSERVER_RateLimitMode as RateLimitMode from tritonserver._c.triton_bindings import ( TRITONSERVER_Server, TRITONSERVER_ServerOptions, UnavailableError, ) uint = Annotated[int, ctypes.c_uint] @dataclass(slots=True) class RateLimiterResource: """Resource count for rate limiting. The amount of a resource available. See :c:func:`TRITONSERVER_ServerOptionsAddRateLimiterResource` Parameters ---------- name : str Name of resource count : uint Count of resource available device : uint The id of the device """ name: str count: uint device: uint @dataclass(slots=True) class ModelLoadDeviceLimit: """Memory limit for loading models on a device. See :c:func:`TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit` Parameters ---------- kind : InstanceGroupKind The kind of device device : uint The id of the device fraction : float The limit on memory usage as a fraction of device memory. """ kind: InstanceGroupKind device: uint fraction: float @dataclass(slots=True) class Options: """Server Options. Parameters ---------- model_repository : str | list[str], default [] Model repository path(s). At least one path is required. See :c:func:`TRITONSERVER_ServerOptionsSetModelRepositoryPath` server_id : str, default 'triton' Textural ID for the server. See :c:func:`TRITONSERVER_ServerOptionsSetServerId` model_control_mode : ModelControlMode, default ModelControlModel.NONE Model control mode. ModelControlMode.NONE : All models in the repository are loaded on startup. ModelControlMode.POLL : All models in the repository are loaded on startup. Model repository changes can be applied using `poll_model_repository`. ModelControlMode.EXPLICIT : Models will not be loaded at startup and need to be explicitly loaded and unloaded using model control APIs `load_model`, `unload_model`. See :c:func:`TRITONSERVER_ServerOptionsSetModelControlMode` startup_models : list[str], default [] List of models to load at startup. Only relevant with ModelControlMode.EXPLICIT. See :c:func:`TRITONSERVER_ServerOptionsSetStartupModel` strict_model_config : bool, default False Enable or disable strict model configuration. See :c:func:`TRITONSERVER_ServerOptionsSetStrictModelConfig` rate_limiter_mode : RateLimitMode, default RateLimitMode.OFF Rate limit mode. RateLimitMode.EXEC_COUNT : Rate limiting prioritizes execution based on the number of times each instance has run and if resource constraints can be satisfied. RateLimitMode.OFF : Rate limiting is disabled. See :c:func:`TRITONSERVER_ServerOptionsSetRateLimiterMode` rate_limiter_resources : list[RateLimiterResource], default [] Resource counts for rate limiting. See :c:func:`TRITONSERVER_ServerOptionsAddRateLimiterResource` pinned_memory_pool_size : uint, default 1 << 28 Total pinned memory size. See :c:func:`TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize` cuda_memory_pool_sizes : dict[int, uint], default {} Total CUDA memory pool size per device. See :c:func:`TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize` cache_config : dict[str, dict[str, Any]], default {} Key-value configuration parameters for the cache provider. See :c:func:`TRITONSERVER_ServerOptionsSetCacheConfig` cache_directory : str, default "/opt/tritonserver/caches" Directory for cache provider implementations. See :c:func:`TRITONSERVER_ServerOptionsSetCacheDirectory` min_supported_compute_capability : float, default 6.0 Minimum required CUDA compute capability. See :c:func:`TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability` exit_on_error : bool, default True Whether to exit on an initialization error. See :c:func:`TRITONSERVER_ServerOptionsSetExitOnError` strict_readiness : bool, default True Enable or disable strict readiness handling. See :c:func:`TRITONSERVER_ServerOptionsSetStrictReadiness` exit_timeout : uint, default 30 Exit timeout for the server in seconds. See :c:func:`TRITONSERVER_ServerOptionsSetExitTimeout` buffer_manager_thread_count : uint, default 0 Number of threads used by the buffer manager. See :c:func:`TRITONSERVER_ServerOptionsSetBufferManagerThreadCount` model_load_thread_count : uint, default 4 Number of threads used to load models concurrently. See :c:func:`TRITONSERVER_ServerOptionsSetModelLoadThreadCount` model_load_retry_count : uint, default 0 Number of threads used to load models concurrently. See :c:func:`TRITONSERVER_ServerOptionsSetModelLoadRetryCount` model_namespacing : bool, default False Enable or disable model namespacing. See :c:func:`TRITONSERVER_ServerOptionsSetModelNamespacing` enable_peer_access : bool, default True Enable or disable GPU peer access. See :c:func:`TRITONSERVER_ServerOptionsSetEnablePeerAccess` log_file : Optional[str], default None Path to the log file. If None, logs are written to stdout. See :c:func:`TRITONSERVER_ServerOptionsSetLogVerbose` log_info : bool, default False Enable or disable logging of INFO level messages. See :c:func:`TRITONSERVER_ServerOptionsSetLogInfo` log_warn : bool, default False Enable or disable logging of WARNING level messages. See :c:func:`TRITONSERVER_ServerOptionsSetLogWarn` log_error : bool, default False Enable or disable logging of ERROR level messages. See :c:func:`TRITONSERVER_ServerOptionsSetLogError` log_format : LogFormat, default LogFormat.DEFAULT Log message format. See :c:func:`TRITONSERVER_ServerOptionsSetLogFormat` log_verbose : uint, default 0 Verbose logging level. Level zero disables logging. See :c:func:`TRITONSERVER_ServerOptionsSetLogVerbose` metrics : bool, default True Enable or disable metric collection. See :c:func:`TRITONSERVER_ServerOptionsSetMetrics` gpu_metrics : bool, default True Enable or disable GPU metric collection. See :c:func:`TRITONSERVER_ServerOptionsSetGpuMetrics` cpu_metrics : bool, default True Enable or disable CPU metric collection. See :c:func:`TRITONSERVER_ServerOptionsSetCpuMetrics` metrics_interval : uint, default 2000 Interval, in milliseconds, for metric collection. See :c:func:`TRITONSERVER_ServerOptionsSetMetricsInterval` backend_directory : str, default "/opt/tritonserver/backends" Directory containing backend implementations. See :c:func:`TRITONSERVER_ServerOptionsSetBackendDirectory` repo_agent_directory : str, default "/opt/tritonserver/repoagents" Directory containing repository agent implementations. See :c:func:`TRITONSERVER_ServerOptionsSetRepoAgentDirectory` model_load_device_limits : list[ModelLoadDeviceLimit], default [] Device memory limits for model loading. See :c:func:`TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit` backend_configuration : dict[str, dict[str, str]], default {} Configuration for backend providers. See :c:func:`TRITONSERVER_ServerOptionsSetBackendConfig` host_policies : dict[str, dict[str, str]], default {} Host policies for given policy name See :c:func:`TRITONSERVER_ServerOptionsSetHostPolicy` metrics_configuration : dict[str, dict[str, str]], default {} Configuration for metric reporting. See :c:func:`TRITONSERVER_ServerOptionsSetMetricsConfig` Notes ----- The class represents server options with various configurable parameters for Triton Inference Server. Please refer to the Triton Inference Server documentation for more details on each option. """ model_repository: str | list[str] = field(default_factory=list[str]) server_id: str = "triton" model_control_mode: ModelControlMode = ModelControlMode.NONE startup_models: list[str] = field(default_factory=list[str]) strict_model_config: bool = False rate_limiter_mode: RateLimitMode = RateLimitMode.OFF rate_limiter_resources: list[RateLimiterResource] = field( default_factory=list[RateLimiterResource] ) pinned_memory_pool_size: uint = 1 << 28 cuda_memory_pool_sizes: dict[int, uint] = field(default_factory=dict[int, uint]) # response_cache_size: Annotated[int, ctypes.c_uint] = 0 cache_config: dict[str, dict[str, Any]] = field( default_factory=dict[str, dict[str, Any]] ) cache_directory: str = "/opt/tritonserver/caches" min_supported_compute_capability: float = 6.0 exit_on_error: bool = True strict_readiness: bool = True exit_timeout: uint = 30 buffer_manager_thread_count: uint = 0 model_load_thread_count: uint = 4 model_load_retry_count: uint = 0 model_namespacing: bool = False enable_peer_access: bool = True log_file: Optional[str] = None log_info: bool = False log_warn: bool = False log_error: bool = False log_format: LogFormat = LogFormat.DEFAULT log_verbose: uint = 0 metrics: bool = True gpu_metrics: bool = True cpu_metrics: bool = True metrics_interval: uint = 2000 backend_directory: str = "/opt/tritonserver/backends" repo_agent_directory: str = "/opt/tritonserver/repoagents" model_load_device_limits: list[ModelLoadDeviceLimit] = field( default_factory=list[ModelLoadDeviceLimit] ) backend_configuration: dict[str, dict[str, str]] = field( default_factory=dict[str, dict[str, str]] ) host_policies: dict[str, dict[str, str]] = field( default_factory=dict[str, dict[str, str]] ) metrics_configuration: dict[str, dict[str, str]] = field( default_factory=dict[str, dict[str, str]] ) def _cascade_log_levels(self) -> None: if self.log_verbose > 0: self.log_info = True if self.log_info: self.log_warn = True if self.log_warn: self.log_error = True def _create_tritonserver_server_options( self, ) -> TRITONSERVER_ServerOptions: options = TRITONSERVER_ServerOptions() options.set_server_id(self.server_id) if self.model_repository is None: raise InvalidArgumentError("Model repository must be specified.") if not isinstance(self.model_repository, list): self.model_repository = [self.model_repository] for model_repository_path in self.model_repository: options.set_model_repository_path(model_repository_path) options.set_model_control_mode(self.model_control_mode) for startup_model in self.startup_models: options.set_startup_model(startup_model) options.set_strict_model_config(self.strict_model_config) options.set_rate_limiter_mode(self.rate_limiter_mode) for rate_limiter_resource in self.rate_limiter_resources: options.add_rate_limiter_resource( rate_limiter_resource.name, rate_limiter_resource.count, rate_limiter_resource.device, ) options.set_pinned_memory_pool_byte_size(self.pinned_memory_pool_size) for device, memory_size in self.cuda_memory_pool_sizes.items(): options.set_cuda_memory_pool_byte_size(device, memory_size) for cache_name, settings in self.cache_config: options.set_cache_config(cache_name, json.dumps(settings)) options.set_cache_directory(self.cache_directory) options.set_min_supported_compute_capability( self.min_supported_compute_capability ) options.set_exit_on_error(self.exit_on_error) options.set_strict_readiness(self.strict_readiness) options.set_exit_timeout(self.exit_timeout) options.set_buffer_manager_thread_count(self.buffer_manager_thread_count) options.set_model_load_thread_count(self.model_load_thread_count) options.set_model_load_retry_count(self.model_load_retry_count) options.set_model_namespacing(self.model_namespacing) options.set_enable_peer_access(self.enable_peer_access) if self.log_file: options.set_log_file(self.log_file) self._cascade_log_levels() options.set_log_info(self.log_info) options.set_log_warn(self.log_warn) options.set_log_error(self.log_error) options.set_log_format(self.log_format) options.set_log_verbose(self.log_verbose) options.set_metrics(self.metrics) options.set_cpu_metrics(self.cpu_metrics) options.set_gpu_metrics(self.gpu_metrics) options.set_metrics_interval(self.metrics_interval) options.set_backend_directory(self.backend_directory) options.set_repo_agent_directory(self.repo_agent_directory) for model_load_device_limit in self.model_load_device_limits: options.set_model_load_device_limit( model_load_device_limit.kind, model_load_device_limit.device, model_load_device_limit.fraction, ) for host_policy, settings in self.host_policies.items(): for setting_name, setting_value in settings.items(): options.set_host_policy(host_policy, setting_name, setting_value) for config_name, settings in self.metrics_configuration.items(): for setting_name, setting_value in settings.items(): options.set_metrics_config(config_name, setting_name, setting_value) for backend, settings in self.backend_configuration.items(): for setting_name, setting_value in settings.items(): options.set_backend_config(backend, setting_name, setting_value) return options class ModelDictionary(dict): """Model dictionary associating model name, version tuples to model objects ModelDictionary objects returned from Server.models(). Not intendended to be instantiated directly. Parameters ---------- dict : [str | tuple [str, int], Model] Raises ------ KeyError Model not found Examples -------- >>> server.models() server.models() {('resnet50_libtorch', 1): {'name': 'resnet50_libtorch', 'version': 1, 'state': 'READY'}} >>> server.models()['resnet50_libtorch'] server.models()['resnet50_libtorch'] {'name': 'resnet50_libtorch', 'version': -1, 'state': None} """ def __init__(self, server: Server, models: list[Model]) -> None: super().__init__() for model in models: self[(model.name, model.version)] = model self._server = server self._model_names = [x[0] for x in self.keys()] def __getitem__(self, key): if isinstance(key, tuple): try: return dict.__getitem__(self, key) except KeyError: raise KeyError(f"Unknown Model: {key}") from None else: if key in self._model_names: return Model(self._server, name=key, version=-1) else: raise KeyError(f"Unknown Model: {key}") class Server: """Triton Inference Server Server objects allow users to instantiate and interact with an in-process Triton Inference Server. Server objects provide methods to access models, server metadata, and metrics. """ def __init__( self, options: Optional[Options] = None, **kwargs: Unpack[Options] ) -> None: """Initialize Triton Inference Server Initialize Triton Inference Server based on configuration options. Options can be passed as an object or key, value pairs that will be used to construct an object. Note: options will be validated on start(). Parameters ---------- options : Optional[Options] Server configuration options. kwargs : Unpack[Options] Keyname arguments passed to `Options` constructor. See `Options` documentation for details. Examples -------- >>> server = tritonserver.Server(model_repository="/workspace/models") server = tritonserver.Server(model_repository="/workspace/models") >>> tritonserver.Options(model_repository="/workspace/models") tritonserver.Options(model_repository="/workspace/models") Options(server_id='triton', model_repository='/workspace/models', model_control_mode=, startup_models=[], strict_model_config=False, rate_limiter_mode=, rate_limiter_resources=[], pinned_memory_pool_size=268435456, cuda_memory_pool_sizes={}, cache_config={}, cache_directory='/opt/tritonserver/caches', min_supported_compute_capability=6.0, exit_on_error=True, strict_readiness=True, exit_timeout=30, buffer_manager_thread_count=0, model_load_thread_count=4, model_namespacing=False, enable_peer_access=True, log_file=None, log_info=False, log_warn=False, log_error=False, log_format=, log_verbose=False, metrics=True, gpu_metrics=True, cpu_metrics=True, metrics_interval=2000, backend_directory='/opt/tritonserver/backends', repo_agent_directory='/opt/tritonserver/repoagents', model_load_device_limits=[], backend_configuration={}, host_policies={}, metrics_configuration={}) """ if options is None: options = Options(**kwargs) self.options: Options = options self._server = Server._UnstartedServer() def start( self, wait_until_ready: bool = False, polling_interval: float = 0.1, timeout: Optional[float] = None, ) -> Server: """Start the in-process server Starts the in-process server and loads models (depending on the ModelControlMode setting). Configuration options are validated and any errors raised as exceptions. Parameters ---------- wait_until_ready : bool, default False Wait for the server to reach a ready state before returning. polling_interval : float, default 0.1 Time to sleep between polling for server ready. Only applicable if `wait_until_ready` is set to True. timeout : Optional[float] Timeout when waiting for server to be ready. Only applicable if `wait_until_ready` is set to True. Returns ------- Server Raises ------ UnavailableError If timeout reached before server ready. InvalidArgumentError Raised on invalid configuration or if server already started. Examples -------- >>> server = tritonserver.Server(model_repository="/workspace/models") server = tritonserver.Server(model_repository="/workspace/models") >>> server.start() server.start() """ if not isinstance(self._server, Server._UnstartedServer): raise InvalidArgumentError("Server already started") self._server = TRITONSERVER_Server( self.options._create_tritonserver_server_options() ) start_time = time.time() while ( wait_until_ready and not self.ready() and ((timeout is None) or (time.time() - start_time) < timeout) ): time.sleep(polling_interval) if wait_until_ready and not self.ready(): raise UnavailableError("Timeout before ready") return self def stop(self) -> None: """Stop server and unload models See c:func:`TRITONSERVER_ServerStop` Returns ------- Server Examples -------- >>> server.stop() server.stop() """ self._server.stop() self._server = Server._UnstartedServer() def unregister_model_repository(self, repository_path: str) -> None: """Unregister model repository Only available when ModelControlMode is set to explicit See c:func:`TRITONSERVER_ServerUnregisterModelRepository` Parameters ---------- repository_path : str path to unregister Returns ------- Server Examples -------- >>> options = tritonserver.Options() >>> options.model_control_mode=tritonserver.ModelControlMode.EXPLICIT >>> options.model_repository="/workspace/models" >>> options.startup_models=["test"] >>> server = tritonserver.Server(options) >>> server.start() >>> server.models() {('resnet50_libtorch', -1): {'name': 'resnet50_libtorch', 'version': -1, 'state': None}, ('test', 1): {'name': 'test', 'version': 1, 'state': 'READY'}, ('test_2', -1): {'name': 'test_2', 'version': -1, 'state': None}} >>> server.unregister_model_repository("/workspace/models") >>> server.models() {} """ self._server.unregister_model_repository(repository_path) def register_model_repository( self, repository_path: str, name_mapping: Optional[dict[str, str]] = None ) -> None: """Add a new model repository. Adds a new model repository. Only available when ModelControlMode is set to explicit See :c:func:`TRITONSERVER_ServerRegisterModelRepository` Parameters ---------- repository_path : str repository path name_mapping : Optional[dict[str, str]] override model names Examples -------- >>> options = tritonserver.Options() >>> options.model_control_mode=tritonserver.ModelControlMode.EXPLICIT >>> options.model_repository="/workspace/models" >>> options.startup_models=["test"] >>> server = tritonserver.Server(options) >>> server.start() >>> server.models() {('resnet50_libtorch', -1): {'name': 'resnet50_libtorch', 'version': -1, 'state': None}, ('test', 1): {'name': 'test', 'version': 1, 'state': 'READY'}, ('test_2', -1): {'name': 'test_2', 'version': -1, 'state': None}} >>> server.unregister_model_repository("/workspace/models") >>> server.models() {} >>> server.register_model_repository("/workspace/models",{"test":"new_model"}) >>> server.models() {('new_name', -1): {'name': 'new_name', 'version': -1, 'state': None}, ('resnet50_libtorch', -1): {'name': 'resnet50_libtorch', 'version': -1, 'state': None}, ('test_2', -1): {'name': 'test_2', 'version': -1, 'state': None}} """ if name_mapping is None: name_mapping = {} name_mapping_list = [ TRITONSERVER_Parameter(name, value) for name, value in name_mapping.items() ] self._server.register_model_repository(repository_path, name_mapping_list) def poll_model_repository(self) -> None: """Poll model repository for changes Only available if ModelControlMode.POLL is enabled. See c:func:`TRITONSERVER_ServerPollModelRepository` Returns ------- Server """ self._server.poll_model_repository() def metadata(self) -> dict[str, Any]: """Returns metadata for server Returns metadata for server including name, version and enabled extensions. See :c:func`TRITONSERVER_ServerMetadata` Returns ------- dict[str, Any] Dictionary of key value pairs of metadata information Examples -------- >>> server.metadata() server.metadata() {'name': 'triton', 'version': '2.41.0', 'extensions': ['classification', 'sequence', 'model_repository', 'model_repository(unload_dependents)', 'schedule_policy', 'model_configuration', 'system_shared_memory', 'cuda_shared_memory', 'binary_tensor_data', 'parameters', 'statistics', 'trace', 'logging']} """ return json.loads(self._server.metadata().serialize_to_json()) def live(self) -> bool: """Returns true if the server is live. See :c:func`TRITONSERVER_ServerIsLive()` Returns ------- bool True if server is live. False otherwise. Examples -------- >>> server.live() server.live() True """ return self._server.is_live() def ready(self) -> bool: """Returns True if the server is ready See c:func:`TRITONSERVER_ServerIsReady()` Returns ------- bool True if server is ready. False otherwise. Examples -------- >>> server.ready() server.ready() True """ return self._server.is_ready() def model(self, model_name: str, model_version: int = -1) -> Model: """Factory method for creating Model objects Creates and returns a Model object that can be used to interact with a model. See `Model` documentation for more details. Note: Model is not validated until it is used. Parameters ---------- model_name : str name of model model_version : int model version, default -1 Returns ------- Model Model object Raises ------ InvalidArgumentError If server isn't started. Examples -------- >>> server.model("test") server.model("test") {'name': 'test', 'version': -1, 'state': None} >>> server.model("test").metadata() server.model("test").metadata() {'name': 'test', 'versions': ['1'], 'platform': 'python', 'inputs': [{'name': 'text_input', 'datatype': 'BYTES', 'shape': [-1]}, {'name': 'fp16_input', 'datatype': 'FP16', 'shape': [-1, 1]}], 'outputs': [{'name': 'text_output', 'datatype': 'BYTES', 'shape': [-1]}, {'name': 'fp16_output', 'datatype': 'FP16', 'shape': [-1, 1]}]} """ if isinstance(self._server, Server._UnstartedServer): raise InvalidArgumentError("Server not started") return Model(self, model_name, model_version) def models(self, exclude_not_ready: bool = False) -> ModelDictionary: """Returns a dictionary of known models in the model repository See c:func:`TRTIONSERVER_ServerModelIndex()` Parameters ---------- exclude_not_ready : bool exclude any models which are not in a ready state Returns ------- ModelDictionary Dictionary mapping model name, version to Model objects Raises ------ InvalidArgumentError If server is not started Examples -------- >>> server.models() server.models() {('new_name', -1): {'name': 'new_name', 'version': -1, 'state': None}, ('resnet50_libtorch', -1): {'name': 'resnet50_libtorch', 'version': -1, 'state': None}, ('test_2', -1): {'name': 'test_2', 'version': -1, 'state': None}} >>> server.models(exclude_not_ready=True) server.models(exclude_not_ready=True) {} """ if isinstance(self._server, Server._UnstartedServer): raise InvalidArgumentError("Server not started") return ModelDictionary(self, self._model_index(exclude_not_ready)) def load( self, model_name: str, parameters: Optional[dict[str, str | int | bool | bytes]] = None, ) -> Model: """Load a model Load a model from the repository and wait for it to be ready. Only available if ModelControlMode is EXPLICIT. See c:func:`TRITONSERVER_ServerLoadModel` Parameters ---------- model_name : str model name parameters : Optional[dict[str, str | int | bool | bytes]] parameters to override settings and upload model artifacts Returns ------- Model Model object Examples -------- >>> server.load("new_name") server.load("new_name") {'name': 'new_name', 'version': -1, 'state': None} >>> server.models() server.models() {('new_name', 1): {'name': 'new_name', 'version': 1, 'state': 'READY'}, ('resnet50_libtorch', -1): {'name': 'resnet50_libtorch', 'version': -1, 'state': None}, ('test_2', -1): {'name': 'test_2', 'version': -1, 'state': None}} """ if parameters is not None: parameter_list = [ TRITONSERVER_Parameter(name, value) for name, value in parameters.items() ] self._server.load_model_with_parameters(model_name, parameter_list) else: self._server.load_model(model_name) return self.model(model_name) def unload( self, model: str | Model, unload_dependents: bool = False, wait_until_unloaded: bool = False, polling_interval: float = 0.1, timeout: Optional[float] = None, ) -> None: """Unload model Unloads a model and its dependents (optional). See c:func:`TRITONSERVER_ServerUnloadModel()` Parameters ---------- model : str | Model model name or model object unload_dependents : bool if True dependent models will also be unloaded wait_until_unloaded : bool if True call will wait until model is unloaded before returning. polling_interval : float time to wait in between polling if model is unloaded timeout : Optional[float] timeout to wait for the model to become unloaded Raises ------ InvalidArgumentError if server is not started Examples -------- >>> server.unload("new_name", wait_for_unloaded=True) server.unload("new_name", wait_for_unloaded=True) >>> server.models() server.models() {('new_name', 1): {'name': 'new_name', 'version': 1, 'state': 'UNAVAILABLE'}, ('resnet50_libtorch', -1): {'name': 'resnet50_libtorch', 'version': -1, 'state': None}, ('test_2', -1): {'name': 'test_2', 'version': -1, 'state': None}} """ if isinstance(self._server, Server._UnstartedServer): raise InvalidArgumentError("Server not started") if isinstance(model, str): model = Model(self, model) if unload_dependents: self._server.unload_model_and_dependents(model.name) else: self._server.unload_model(model.name) if wait_until_unloaded: model_versions = [ key for key in self.models().keys() if key[0] == model.name ] start_time = time.time() while not self._model_unloaded(model_versions) and ( (timeout is None) or (time.time() - start_time < timeout) ): time.sleep(polling_interval) def metrics(self, metric_format: MetricFormat = MetricFormat.PROMETHEUS) -> str: """Return server and custom metrics See c:func:`TRITONSERVER_ServerMetrics()` Parameters ---------- metric_format : MetricFormat format for metrics Returns ------- str string containing metrics in specified format """ return self._server.metrics().formatted(metric_format) class _UnstartedServer(object): def __init__(self): pass def __getattribute__(self, name): raise InvalidArgumentError("Server not started") def __setattr__(self, name, value): raise InvalidArgumentError("Server not started") def _model_unloaded(self, model_versions: list[tuple[str, int]]) -> bool: model_states = self.models() for model_version in model_versions: if model_states[model_version].state not in Server._UNLOADED_STATES: return False return True def _model_index(self, exclude_not_ready=False) -> list[Model]: if isinstance(self._server, Server._UnstartedServer): raise InvalidArgumentError("Server not started") models = json.loads( self._server.model_index(exclude_not_ready).serialize_to_json() ) for model in models: if "version" in model: model["version"] = int(model["version"]) return [Model(self, **model) for model in models] _UNLOADED_STATES = [None, "UNAVAILABLE"] class Metric(TRITONSERVER_Metric): """Class for adding a custom metric to Triton inference server metrics reporting Metric objects are created as part of a MetricFamily where the MetricFamily defines the type and name of the metric. When a metric is added to a family it can further add additional labels allowing for multiple metrics to be associated with the same family. For more details see c:func:`TRITONSERVER_Metric` documentation. """ def __init__(self, family: MetricFamily, labels: Optional[dict[str, str]] = None): """Initialize Metric object Parameters ---------- family : MetricFamily Metric family that includes MetricKind, name, and description. labels : Optional[dict[str, str]] Additional labels for the metric (returned in reporting to distinguish multiple metrics within a family) """ if labels is not None: parameters = [ TRITONSERVER_Parameter(name, value) for name, value in labels.items() ] else: parameters = [] TRITONSERVER_Metric.__init__(self, family, parameters) kytensor-2.49.0.7/core/python/tritonserver/_api/_tensor.py000066400000000000000000000520141527052371200236110ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations import ctypes import struct from dataclasses import dataclass from typing import Any, Callable, ClassVar, Sequence import numpy from tritonserver._api._allocators import MemoryBuffer from tritonserver._api._datautils import ( NUMPY_TO_TRITON_DTYPE, TRITON_MEMORY_TYPE_TO_DLPACK_DEVICE_TYPE, TRITON_TO_DLPACK_DTYPE, DLPackObject, parse_device_or_memory_type, ) from tritonserver._api._dlpack import ( DLDevice, DLDeviceType, DLManagedTensor, c_str_dltensor, ) from tritonserver._c.triton_bindings import ( InvalidArgumentError, TRITONSERVER_BufferAttributes, ) from tritonserver._c.triton_bindings import TRITONSERVER_DataType as DataType from tritonserver._c.triton_bindings import TRITONSERVER_MemoryType as MemoryType from tritonserver._c.triton_bindings import UnsupportedError DeviceOrMemoryType = ( tuple[MemoryType, int] | MemoryType | tuple[DLDeviceType, int] | str ) try: import cupy except ImportError: cupy = None @dataclass class Tensor: """Class representing a Tensor. Parameters ---------- data_type : DataType Data type of the tensor. shape : Sequence[int] Shape of the tensor. memory_buffer : MemoryBuffer Memory buffer containing the tensor data. """ data_type: DataType shape: Sequence[int] memory_buffer: MemoryBuffer @property def data_ptr(self) -> int: """Get the pointer to the tensor's data. Returns ------- int The pointer to the tensor's data. """ return self.memory_buffer.data_ptr @property def memory_type(self) -> MemoryType: """Get the memory type of the tensor. Returns ------- MemoryType The memory type of the tensor. """ return self.memory_buffer.memory_type @property def memory_type_id(self) -> int: """Get the ID representing the memory type of the tensor. Returns ------- int The ID representing the memory type of the tensor. """ return self.memory_buffer.memory_type_id @property def size(self) -> int: """Get the size of the tensor's data in bytes. Returns ------- int The size of the tensor's data in bytes. """ return self.memory_buffer.size def _sync_on_requested_stream(self, requested_stream_ptr): """Stream synchronization based on cupy implementation. Record event on requested stream if different from current stream. See `https://data-apis.org/array-api/latest/API_specification/generated/array_api.array.__dlpack__.html` and `https://github.com/cupy/cupy/blob/f9563bcd5c674623f80ce975b590f9c860b44ed6/cupy/_core/core.pyx#L281`. for more details. Parameters ---------- requested_stream_ptr : requested stream as defined by DLPack protocol Raises ------ unsupported If synchronization can not be done """ current_stream_ptr = None current_stream = None unsupported = UnsupportedError( f"DLPack stream synchronization on memory type {self.memory_type} and stream {requested_stream_ptr} not supported" ) if requested_stream_ptr is not None and not isinstance( requested_stream_ptr, int ): raise unsupported if self.memory_type != MemoryType.GPU: if requested_stream_ptr not in (None, 0): raise unsupported return if cupy is None: raise unsupported # NOTE: Technically this is not required by the protocol. It is the # responsibility of the caller(consumer) to ensure that # we are on the correct device. Added to ensure # the semantics are correct - but should be a no-op. # May be removed in the future. with cupy.cuda.Device(self.memory_type_id): current_stream = cupy.cuda.get_current_stream() curr_stream_ptr = current_stream.ptr # Based on cupy documentation # cupy.cuda.Stream.null.ptr is legacy default stream # cupy.cuda.Stream.ptds.ptr is per thread default stream if curr_stream_ptr == 0: curr_stream_ptr = cupy.cuda.Stream.null.ptr if requested_stream_ptr in (None, 0, 1): requested_stream_ptr = cupy.cuda.Stream.null.ptr if requested_stream_ptr in (2,): requested_stream_ptr = cupy.cuda.Stream.ptds.ptr if requested_stream_ptr >= 0 and curr_stream_ptr != requested_stream_ptr: next_stream = cupy.cuda.ExternalStream(requested_stream_ptr) event = current_stream.record() next_stream.wait_event(event) def __dlpack__(self, *, stream=None): """Convert the tensor to a DLPack-compatible object. Parameters ---------- stream : Any, optional Currently Ignored parameter, by default None. Returns ------- Any A DLPack-compatible object representing the tensor. """ self._sync_on_requested_stream(stream) dl_managed_tensor = Tensor._create_managed_tensor() dl_managed_tensor.dl_tensor.data = self.data_ptr dl_managed_tensor.dl_tensor.device = DLDevice( TRITON_MEMORY_TYPE_TO_DLPACK_DEVICE_TYPE[self.memory_type], self.memory_type_id, ) dl_managed_tensor.dl_tensor.dtype = TRITON_TO_DLPACK_DTYPE[self.data_type] dl_managed_tensor.dl_tensor.ndim = len(self.shape) dl_managed_tensor.dl_tensor.shape = (ctypes.c_int64 * len(self.shape))( *self.shape ) dl_managed_tensor.dl_tensor.strides = ctypes.POINTER(ctypes.c_int64)() dl_managed_tensor.dl_tensor.byte_offset = 0 dl_managed_tensor.deleter = Tensor._managed_tensor_deleter self._set_dlpack_manager_ctx(dl_managed_tensor) pycapsule = ctypes.pythonapi.PyCapsule_New( ctypes.byref(dl_managed_tensor), c_str_dltensor, Tensor._pycapsule_deleter, ) return pycapsule def __dlpack_device__(self) -> tuple[DLDeviceType, int]: """Get the DLPack device information for the tensor. Returns ------- tuple[DLDeviceType, int] A tuple representing the DLPack device information (device type, device ID). """ return ( TRITON_MEMORY_TYPE_TO_DLPACK_DEVICE_TYPE[self.memory_type], self.memory_type_id, ) def to_string_array(self) -> numpy.ndarray: """Deserialize Triton BYTES Tensor into numpy array of strings. If memory is not on the host the tensor data will be copied to the host before deserialization. For more details on the format of Triton BYTES Tensors please see Triton Inference Server documentation. Returns ------- numpy.ndarray A numpy array of objects representing the BYTES tensor. Examples -------- numpy_ndarray = response.outputs["text_output"].to_string_array() """ return self.to_bytes_array().astype(str) def to_bytes_array(self) -> numpy.ndarray: """Deserialize Triton BYTES Tensor into numpy array. If memory is not on the host the tensor data will be copied to the host before deserialization. For more details on the format of Triton BYTES Tensors please see Triton Inference Server documentation. Returns ------- numpy.ndarray A numpy array of objects representing the BYTES tensor. Examples -------- numpy_ndarray = response.outputs["text_output"].to_bytes_array() """ if self.data_type != DataType.BYTES: raise InvalidArgumentError( f"Tensor has data type {self.data_type} not {DataType.BYTES}" ) # Reshape into 1d array of bytes on host original_data_type = self.data_type original_shape = self.shape self.data_type = DataType.UINT8 self.shape = [self.size] numpy_ndarray = self._to_numpy_on_host() # Deserialize bytes array and reshape self.shape = original_shape self.data_type = original_data_type return Tensor._deserialize_bytes_array(numpy_ndarray).reshape(self.shape) @staticmethod def from_string_array(string_array: list[str] | numpy.ndarray) -> Tensor: """Create Triton BYTES Tensor from numpy array of strings or list of strings. Creates a Triton tensor of type BYTES from a list of strings, or numpy array of type str_. The method allocates new host memory to store the serialized tensor. For more details on the format of Triton BYTES Tensors please see Triton Inference Server documentation. Parameters ---------- string_array : list[str] | numpy.ndarray an array like object to convert Returns ------- Tensor Raises ------ InvalidArgumentError If the given object can not be converted. Examples -------- tensor = Tensor.from_string_array(numpy.array(["hello"])) tensor = Tensor.from_string_array(["hello"]) """ return Tensor.from_bytes_array(string_array) @staticmethod def from_bytes_array( bytes_array: list[str] | list[bytes] | numpy.ndarray, ) -> Tensor: """Create Triton BYTES Tensor from numpy array or list Creates a Triton tensor of type BYTES from a list of strings, bytes or a numpy array of type object_, bytes_, or str_. The method allocates new host memory to store the serialized tensor. For more details on the format of Triton BYTES Tensors please see Triton Inference Server documentation. Parameters ---------- bytes_array : list[str | bytes] | numpy.ndarray an array like object to convert Returns ------- Tensor Raises ------ InvalidArgumentError If the given object can not be converted. Examples -------- tensor = Tensor.from_bytes_array(numpy.array(["hello"])) tensor = Tensor.from_bytes_array(["hello"]) """ result = Tensor._from_object(bytes_array) if result.data_type != DataType.BYTES: raise InvalidArgumentError( f"Unsupported conversion from {bytes_array} to Triton BYTES Tensor. Got {result.data_type}" ) return result @staticmethod def _from_object(obj: list[Any] | numpy.ndarray | Any) -> Tensor: """Create a tensor from an object. Creates a tensor from an object using specific conversion methods if available or falls back to using __from_dlpack__. Specific conversions are currently supported for: list[obj: Any] : implicitly converted to numpy.array() numpy.ndarray : serialized if required to BYTES tensor Parameters ---------- obj : list[Any] | numpy.ndarray | Any The input object to create the tensor from. Returns ------- Tensor A new tensor created from the specified object. Examples -------- tensor = Tensor.from_object(numpy.array(["hello"])) tensor = Tensor.from_object(["hello"]) """ if type(obj) in Tensor._from_converters: return Tensor._from_converters[type(obj)](obj) elif hasattr(obj, "__dlpack__"): return Tensor.from_dlpack(obj) else: raise InvalidArgumentError( f"Input type {type(obj)} not supported. Must be one of {list(Tensor._from_converters.keys())} or the type must support __dlpack__" ) @staticmethod def from_dlpack(obj: Any) -> Tensor: """Create a tensor from a DLPack-compatible object. Parameters ---------- obj : Any The DLPack-compatible object. Returns ------- Tensor A new tensor created from the DLPack-compatible object. Examples -------- tensor = Tensor.from_dlpack(numpy.array([0,1,2], dtype=numpy.float16)) tensor = Tensor.from_dlpack(torch.zeros(100, dtype=torch.float16)) """ dlpack_object = DLPackObject(obj) data_type = dlpack_object.triton_data_type shape = dlpack_object.shape memory_buffer = MemoryBuffer._from_dlpack_object( obj, dlpack_object=dlpack_object ) return Tensor(data_type, shape, memory_buffer) def to_host(self) -> Tensor: """Move the tensor to CPU memory from device memory Returns ------- Tensor The tensor moved to the CPU. Examples -------- tensor = Tensor.from_dlpack(torch.zeros(100, dtype=torch.float16).to("cuda")) numpy_nd_array = numpy.array(tensor.to_host()) """ return self.to_device("cpu") def to_device(self, device: DeviceOrMemoryType) -> Tensor: """Move the tensor to the specified device. Parameters ---------- device : DeviceOrMemoryType The target device. Device can be specified as a string, MemoryType, tuple [MemoryType, memory_type__id], or tuple[DLDeviceType, device_id]. Returns ------- Tensor The tensor moved to the specified device. Examples -------- tensor_cpu = tritonserver.Tensor.from_dlpack(numpy.array([0,1,2], dtype=numpy.float16)) # Different ways to specify the device tensor_gpu = tensor_cpu.to_device(MemoryType.GPU) tensor_gpu = tensor_cpu.to_device((MemoryType.GPU,0)) tensor_gpu = tensor_cpu.to_device((DLDeviceType.kDLCUDA,0)) tensor_gpu = tensor_cpu.to_device("gpu") tensor_gpu = tensor_cpu.to_device("gpu:0") ndarray_gpu = cupy.from_dlpack(tensor_gpu) ndarray_gpu[0] = ndarray_gpu.mean() tensor_cpu = tensor_gpu.to_device("cpu") ndarray_cpu = numpy.from_dlpack(tensor_cpu) assert ndarray_cpu[0] == ndarray_gpu[0] """ memory_type, memory_type_id = parse_device_or_memory_type(device) if self.memory_type == memory_type and self.memory_type_id == memory_type_id: return self if self.memory_type == MemoryType.CPU_PINNED and memory_type == MemoryType.CPU: return self if cupy is not None: if self.memory_type in (MemoryType.CPU, MemoryType.CPU_PINNED): ndarray = numpy.from_dlpack(self) else: ndarray = cupy.from_dlpack(self) if memory_type == MemoryType.CPU: return Tensor.from_dlpack(cupy.asnumpy(ndarray)) if memory_type == MemoryType.GPU: with cupy.cuda.Device(memory_type_id): return Tensor.from_dlpack(cupy.asarray(ndarray)) raise UnsupportedError( f"Conversion from {(self.memory_type,self.memory_type_id)} to {(memory_type, memory_type_id)} not supported." ) def _to_numpy_on_host(self) -> numpy.ndarray: if self.memory_type in (MemoryType.CPU, MemoryType.CPU_PINNED): return numpy.from_dlpack(self) if cupy is not None: return cupy.asnumpy(cupy.from_dlpack(self)) raise UnsupportedError( f"Conversion from {self.memory_type} to numpy array not supported." ) @staticmethod def _deserialize_bytes_array(numpy_ndarray: numpy.ndarray) -> numpy.ndarray: result = [] _buffer = memoryview(numpy_ndarray) offset = 0 while offset < len(_buffer): (item_length,) = struct.unpack_from("@I", _buffer, offset) offset += 4 result.append(bytes(_buffer[offset : offset + item_length])) offset += item_length return numpy.array(result, dtype=numpy.object_) @staticmethod def _serialize_numpy_bytes_array(array: numpy.ndarray) -> numpy.ndarray: result = [] for array_item in numpy.nditer(array, flags=["refs_ok"], order="C"): item = array_item.item() if not isinstance(item, bytes): item = str(item).encode("utf-8") result.append(struct.pack("@I", len(item))) result.append(item) return numpy.frombuffer(b"".join(result), dtype=numpy.byte) @staticmethod def _from_list(obj: list[Any]) -> Tensor: try: return Tensor._from_numpy(numpy.array(obj)) except Exception as e: raise InvalidArgumentError( f"Conversion from {obj} to tensor not supported." ) from e @staticmethod def _from_numpy(obj: numpy.ndarray | numpy.generic) -> Tensor: data_type = NUMPY_TO_TRITON_DTYPE[obj.dtype.type] shape = obj.shape if isinstance(obj, numpy.generic): obj = numpy.asarray(obj) if data_type == DataType.BYTES: obj = Tensor._serialize_numpy_bytes_array(obj) memory_buffer = MemoryBuffer( data_ptr=obj.ctypes.data, memory_type=MemoryType.CPU, memory_type_id=0, size=obj.itemsize * obj.size, owner=obj, ) return Tensor(data_type, shape, memory_buffer) @staticmethod def _create_managed_tensor(): size = ctypes.c_size_t(ctypes.sizeof(DLManagedTensor)) address = ctypes.pythonapi.PyMem_RawMalloc(size) return DLManagedTensor.from_address(address) @staticmethod @ctypes.CFUNCTYPE(None, ctypes.c_void_p) def _managed_tensor_deleter(handle: int) -> None: dl_managed_tensor = DLManagedTensor.from_address(handle) tensor_obj_ptr = ctypes.cast( dl_managed_tensor.manager_ctx, ctypes.POINTER(ctypes.py_object) ) tensor_obj = tensor_obj_ptr.contents ctypes.pythonapi.Py_DecRef(tensor_obj) shape_obj = ctypes.py_object(dl_managed_tensor.dl_tensor.shape) ctypes.pythonapi.Py_DecRef(shape_obj) ctypes.pythonapi.PyMem_RawFree(handle) @staticmethod @ctypes.CFUNCTYPE(None, ctypes.c_void_p) def _pycapsule_deleter(handle: ctypes.c_void_p) -> None: try: pycapsule: ctypes.py_object = ctypes.cast(handle, ctypes.py_object) if ctypes.pythonapi.PyCapsule_IsValid(pycapsule, c_str_dltensor): dl_managed_tensor = ctypes.pythonapi.PyCapsule_GetPointer( pycapsule, c_str_dltensor ) Tensor._managed_tensor_deleter(dl_managed_tensor) ctypes.pythonapi.PyCapsule_SetDestructor(pycapsule, None) except Exception as e: print(f"Exception occurred while deleting capsule: {e}") raise e def _set_dlpack_manager_ctx(self, dl_managed_tensor): tensor_obj = ctypes.py_object(self) tensor_obj_ptr = ctypes.pointer(tensor_obj) dl_managed_tensor.manager_ctx = ctypes.cast(tensor_obj_ptr, ctypes.c_void_p) shape_obj = ctypes.py_object(dl_managed_tensor.dl_tensor.shape) ctypes.pythonapi.Py_IncRef(tensor_obj) ctypes.pythonapi.Py_IncRef(shape_obj) _from_converters: ClassVar[dict[type, Callable[[Any], Tensor]]] = dict( {numpy.ndarray: _from_numpy, numpy.generic: _from_numpy, list: _from_list}, ) kytensor-2.49.0.7/core/python/tritonserver/_c/000077500000000000000000000000001527052371200212355ustar00rootroot00000000000000kytensor-2.49.0.7/core/python/tritonserver/_c/__init__.py000066400000000000000000000030451527052371200233500ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from .triton_bindings import * kytensor-2.49.0.7/core/python/tritonserver/_c/__init__.pyi000066400000000000000000000035621527052371200235250ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Type information for Triton _c bindings.""" # Note: this file was generated using mypy with an empty __init__.py # file in the tritonserver package directory to avoid any renaming / # aliasing done by the wrapper # # mypy 1.8.0 (compiled: yes) # # stubgen -p tritonserver._c # # Todo: add stub generation to build process from .triton_bindings import * kytensor-2.49.0.7/core/python/tritonserver/_c/triton_bindings.pyi000066400000000000000000000647471527052371200251760ustar00rootroot00000000000000# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Type information for Triton _c bindings.""" # Note: this file was generated using mypy with an empty __init__.py # file in the tritonserver package directory to avoid any renaming / # aliasing done by the wrapper # # mypy 1.8.0 (compiled: yes) # # stubgen -p tritonserver._c # # Todo: add stub generation to build process from typing import Callable, ClassVar, List, Optional, Tuple, overload import numpy ALL: TRITONSERVER_RequestReleaseFlag COMPUTE_END: TRITONSERVER_InferenceTraceActivity COMPUTE_INPUT_END: TRITONSERVER_InferenceTraceActivity COMPUTE_OUTPUT_START: TRITONSERVER_InferenceTraceActivity COMPUTE_START: TRITONSERVER_InferenceTraceActivity DECOUPLED: TRITONSERVER_ModelTxnPropertyFlag DISABLED: TRITONSERVER_InferenceTraceLevel FINAL: TRITONSERVER_ResponseCompleteFlag FIRST_DIM: TRITONSERVER_ModelBatchFlag MAX: TRITONSERVER_InferenceTraceLevel MIN: TRITONSERVER_InferenceTraceLevel ONE_TO_ONE: TRITONSERVER_ModelTxnPropertyFlag QUEUE_START: TRITONSERVER_InferenceTraceActivity READY: TRITONSERVER_ModelIndexFlag REQUEST_END: TRITONSERVER_InferenceTraceActivity REQUEST_START: TRITONSERVER_InferenceTraceActivity SEQUENCE_END: TRITONSERVER_RequestFlag SEQUENCE_START: TRITONSERVER_RequestFlag TENSORS: TRITONSERVER_InferenceTraceLevel TENSOR_BACKEND_INPUT: TRITONSERVER_InferenceTraceActivity TENSOR_BACKEND_OUTPUT: TRITONSERVER_InferenceTraceActivity TENSOR_QUEUE_INPUT: TRITONSERVER_InferenceTraceActivity TIMESTAMPS: TRITONSERVER_InferenceTraceLevel UNKNOWN: TRITONSERVER_ModelBatchFlag class AlreadyExistsError(TritonError): ... class InternalError(TritonError): ... class InvalidArgumentError(TritonError): ... class NotFoundError(TritonError): ... class TRITONSERVER_BufferAttributes: byte_size: int cuda_ipc_handle: int memory_type: TRITONSERVER_MemoryType memory_type_id: int def __init__(self) -> None: ... class TRITONSERVER_DataType: __members__: ClassVar[dict] = ... # read-only BF16: ClassVar[TRITONSERVER_DataType] = ... BOOL: ClassVar[TRITONSERVER_DataType] = ... BYTES: ClassVar[TRITONSERVER_DataType] = ... FP16: ClassVar[TRITONSERVER_DataType] = ... FP32: ClassVar[TRITONSERVER_DataType] = ... FP64: ClassVar[TRITONSERVER_DataType] = ... INT16: ClassVar[TRITONSERVER_DataType] = ... INT32: ClassVar[TRITONSERVER_DataType] = ... INT64: ClassVar[TRITONSERVER_DataType] = ... INT8: ClassVar[TRITONSERVER_DataType] = ... INVALID: ClassVar[TRITONSERVER_DataType] = ... UINT16: ClassVar[TRITONSERVER_DataType] = ... UINT32: ClassVar[TRITONSERVER_DataType] = ... UINT64: ClassVar[TRITONSERVER_DataType] = ... UINT8: ClassVar[TRITONSERVER_DataType] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_InferenceRequest: correlation_id: int correlation_id_string: str flags: int id: str priority: int priority_uint64: int timeout_microseconds: int def __init__(self, arg0, arg1: str, arg2: int) -> None: ... def add_input( self, arg0: str, arg1: TRITONSERVER_DataType, arg2: List[int] ) -> None: ... def add_raw_input(self, arg0: str) -> None: ... def add_requested_output(self, arg0: str) -> None: ... def append_input_data( self, arg0: str, arg1: int, arg2: int, arg3: TRITONSERVER_MemoryType, arg4: int ) -> None: ... def append_input_data_with_buffer_attributes( self, arg0: str, arg1: int, arg2: TRITONSERVER_BufferAttributes ) -> None: ... def append_input_data_with_host_policy( self, arg0: str, arg1: int, arg2: int, arg3: TRITONSERVER_MemoryType, arg4: int, arg5: str, ) -> None: ... def cancel(self) -> None: ... def remove_all_input_data(self, arg0: str) -> None: ... def remove_all_inputs(self) -> None: ... def remove_all_requested_outputs(self) -> None: ... def remove_input(self, arg0: str) -> None: ... def remove_requested_output(self, arg0: str) -> None: ... def set_bool_parameter(self, arg0: str, arg1: bool) -> None: ... def set_int_parameter(self, arg0: str, arg1: int) -> None: ... def set_release_callback( self, arg0: Callable[[TRITONSERVER_InferenceRequest, int, object], None], arg1: object, ) -> None: ... def set_response_callback( self, arg0: object, arg1: object, arg2: Callable[[object, int, object], None], arg3: object, ) -> None: ... def set_string_parameter(self, arg0: str, arg1: str) -> None: ... def set_double_parameter(self, arg0: str, arg1: float) -> None: ... class TRITONSERVER_InferenceResponse: def __init__(self, *args, **kwargs) -> None: ... def output( self, arg0: int ) -> Tuple[ str, TRITONSERVER_DataType, numpy.ndarray[numpy.int64], int, int, TRITONSERVER_MemoryType, int, object, ]: ... def output_classification_label(self, arg0: int, arg1: int) -> str: ... def parameter( self, arg0: int ) -> Tuple[str, TRITONSERVER_ParameterType, object]: ... def throw_if_response_error(self) -> None: ... @property def id(self) -> str: ... @property def model(self) -> Tuple[str, int]: ... @property def output_count(self) -> int: ... @property def parameter_count(self) -> int: ... class TRITONSERVER_InferenceTrace: @overload def __init__( self, level: int, parent_id: int, activity_function: Callable[ [object, TRITONSERVER_InferenceTraceActivity, int, object], None ], tensor_activity_function: Callable[ [ object, TRITONSERVER_InferenceTraceActivity, str, TRITONSERVER_DataType, int, int, numpy.ndarray[numpy.int64], TRITONSERVER_MemoryType, int, object, ], None, ], release_function: Callable[[TRITONSERVER_InferenceTrace, object], None], trace_userp: object, ) -> None: ... @overload def __init__( self, level: int, parent_id: int, activity_function: Callable[ [object, TRITONSERVER_InferenceTraceActivity, int, object], None ], release_function: Callable[[TRITONSERVER_InferenceTrace, object], None], trace_userp: object, ) -> None: ... @property def id(self) -> int: ... @property def model_name(self) -> str: ... @property def model_version(self) -> int: ... @property def parent_id(self) -> int: ... @property def request_id(self) -> str: ... class TRITONSERVER_InferenceTraceActivity: __members__: ClassVar[dict] = ... # read-only COMPUTE_END: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... COMPUTE_INPUT_END: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... COMPUTE_OUTPUT_START: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... COMPUTE_START: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... QUEUE_START: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... REQUEST_END: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... REQUEST_START: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... TENSOR_BACKEND_INPUT: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... TENSOR_BACKEND_OUTPUT: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... TENSOR_QUEUE_INPUT: ClassVar[TRITONSERVER_InferenceTraceActivity] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_InferenceTraceLevel: __members__: ClassVar[dict] = ... # read-only DISABLED: ClassVar[TRITONSERVER_InferenceTraceLevel] = ... MAX: ClassVar[TRITONSERVER_InferenceTraceLevel] = ... MIN: ClassVar[TRITONSERVER_InferenceTraceLevel] = ... TENSORS: ClassVar[TRITONSERVER_InferenceTraceLevel] = ... TIMESTAMPS: ClassVar[TRITONSERVER_InferenceTraceLevel] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_InstanceGroupKind: __members__: ClassVar[dict] = ... # read-only AUTO: ClassVar[TRITONSERVER_InstanceGroupKind] = ... CPU: ClassVar[TRITONSERVER_InstanceGroupKind] = ... GPU: ClassVar[TRITONSERVER_InstanceGroupKind] = ... MODEL: ClassVar[TRITONSERVER_InstanceGroupKind] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_LogFormat: __members__: ClassVar[dict] = ... # read-only DEFAULT: ClassVar[TRITONSERVER_LogFormat] = ... ISO8601: ClassVar[TRITONSERVER_LogFormat] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_LogLevel: __members__: ClassVar[dict] = ... # read-only ERROR: ClassVar[TRITONSERVER_LogLevel] = ... INFO: ClassVar[TRITONSERVER_LogLevel] = ... VERBOSE: ClassVar[TRITONSERVER_LogLevel] = ... WARN: ClassVar[TRITONSERVER_LogLevel] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_MemoryType: __members__: ClassVar[dict] = ... # read-only CPU: ClassVar[TRITONSERVER_MemoryType] = ... CPU_PINNED: ClassVar[TRITONSERVER_MemoryType] = ... GPU: ClassVar[TRITONSERVER_MemoryType] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_Message: def __init__(self, arg0: str) -> None: ... def serialize_to_json(self) -> str: ... class TRITONSERVER_Metric: def __init__( self, arg0: TRITONSERVER_MetricFamily, arg1: List[TRITONSERVER_Parameter] ) -> None: ... def increment(self, arg0: float) -> None: ... def set_value(self, arg0: float) -> None: ... @property def kind(self) -> TRITONSERVER_MetricKind: ... @property def value(self) -> float: ... class TRITONSERVER_MetricFamily: def __init__(self, arg0: TRITONSERVER_MetricKind, arg1: str, arg2: str) -> None: ... class TRITONSERVER_MetricFormat: __members__: ClassVar[dict] = ... # read-only PROMETHEUS: ClassVar[TRITONSERVER_MetricFormat] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_MetricKind: __members__: ClassVar[dict] = ... # read-only COUNTER: ClassVar[TRITONSERVER_MetricKind] = ... GAUGE: ClassVar[TRITONSERVER_MetricKind] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_Metrics: def __init__(self, *args, **kwargs) -> None: ... def formatted(self, arg0: TRITONSERVER_MetricFormat) -> str: ... class TRITONSERVER_ModelBatchFlag: __members__: ClassVar[dict] = ... # read-only FIRST_DIM: ClassVar[TRITONSERVER_ModelBatchFlag] = ... UNKNOWN: ClassVar[TRITONSERVER_ModelBatchFlag] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_ModelControlMode: __members__: ClassVar[dict] = ... # read-only EXPLICIT: ClassVar[TRITONSERVER_ModelControlMode] = ... NONE: ClassVar[TRITONSERVER_ModelControlMode] = ... POLL: ClassVar[TRITONSERVER_ModelControlMode] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_ModelIndexFlag: __members__: ClassVar[dict] = ... # read-only READY: ClassVar[TRITONSERVER_ModelIndexFlag] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_ModelTxnPropertyFlag: __members__: ClassVar[dict] = ... # read-only DECOUPLED: ClassVar[TRITONSERVER_ModelTxnPropertyFlag] = ... ONE_TO_ONE: ClassVar[TRITONSERVER_ModelTxnPropertyFlag] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_Parameter: @overload def __init__(self, arg0: str, arg1: bytes) -> None: ... @overload def __init__(self, arg0: str, arg1: str) -> None: ... @overload def __init__(self, arg0: str, arg1: int) -> None: ... @overload def __init__(self, arg0: str, arg1: bool) -> None: ... class TRITONSERVER_ParameterType: __members__: ClassVar[dict] = ... # read-only BOOL: ClassVar[TRITONSERVER_ParameterType] = ... BYTES: ClassVar[TRITONSERVER_ParameterType] = ... INT: ClassVar[TRITONSERVER_ParameterType] = ... STRING: ClassVar[TRITONSERVER_ParameterType] = ... DOUBLE: ClassVar[TRITONSERVER_ParameterType] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_RateLimitMode: __members__: ClassVar[dict] = ... # read-only EXEC_COUNT: ClassVar[TRITONSERVER_RateLimitMode] = ... OFF: ClassVar[TRITONSERVER_RateLimitMode] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_RequestFlag: __members__: ClassVar[dict] = ... # read-only SEQUENCE_END: ClassVar[TRITONSERVER_RequestFlag] = ... SEQUENCE_START: ClassVar[TRITONSERVER_RequestFlag] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_RequestReleaseFlag: __members__: ClassVar[dict] = ... # read-only ALL: ClassVar[TRITONSERVER_RequestReleaseFlag] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_ResponseAllocator: @overload def __init__( self, alloc_function: Callable[ [object, str, int, TRITONSERVER_MemoryType, int, object], Tuple[int, object, TRITONSERVER_MemoryType, int], ], release_function: Callable[ [object, int, object, int, TRITONSERVER_MemoryType, int], None ], start_function: Callable[[object, object], None], ) -> None: ... @overload def __init__( self, alloc_function: Callable[ [object, str, int, TRITONSERVER_MemoryType, int, object], Tuple[int, object, TRITONSERVER_MemoryType, int], ], release_function: Callable[ [object, int, object, int, TRITONSERVER_MemoryType, int], None ], ) -> None: ... def set_buffer_attributes_function( self, buffer_attributes_function: Callable[ [object, str, object, object, object], object ], ) -> None: ... def set_query_function( self, query_function: Callable[ [object, object, str, Optional[int], TRITONSERVER_MemoryType, int], Tuple[TRITONSERVER_MemoryType, int], ], ) -> None: ... class TRITONSERVER_ResponseCompleteFlag: __members__: ClassVar[dict] = ... # read-only FINAL: ClassVar[TRITONSERVER_ResponseCompleteFlag] = ... __entries: ClassVar[dict] = ... def __init__(self, value: int) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... def __ne__(self, other: object) -> bool: ... @property def name(self) -> str: ... @property def value(self) -> int: ... class TRITONSERVER_Server: def __init__(self, arg0: TRITONSERVER_ServerOptions) -> None: ... @overload def infer_async( self, arg0: TRITONSERVER_InferenceRequest, arg1: TRITONSERVER_InferenceTrace ) -> None: ... @overload def infer_async(self, arg0: TRITONSERVER_InferenceRequest) -> None: ... def is_live(self) -> bool: ... def is_ready(self) -> bool: ... def load_model(self, arg0: str) -> None: ... def load_model_with_parameters( self, arg0: str, arg1: List[TRITONSERVER_Parameter] ) -> None: ... def metadata(self) -> TRITONSERVER_Message: ... def metrics(self) -> TRITONSERVER_Metrics: ... def model_batch_properties(self, arg0: str, arg1: int) -> Tuple[int, int]: ... def model_config(self, arg0: str, arg1: int, arg2: int) -> TRITONSERVER_Message: ... def model_index(self, arg0: int) -> TRITONSERVER_Message: ... def model_is_ready(self, arg0: str, arg1: int) -> bool: ... def model_metadata(self, arg0: str, arg1: int) -> TRITONSERVER_Message: ... def model_statistics(self, arg0: str, arg1: int) -> TRITONSERVER_Message: ... def model_transaction_properties(self, arg0: str, arg1: int) -> Tuple[int, int]: ... @overload def poll_model_repository(self) -> None: ... @overload def poll_model_repository(self) -> None: ... def register_model_repository( self, arg0: str, arg1: List[TRITONSERVER_Parameter] ) -> None: ... def stop(self) -> None: ... def unload_model(self, arg0: str) -> None: ... def unload_model_and_dependents(self, arg0: str) -> None: ... def unregister_model_repository(self, arg0: str) -> None: ... class TRITONSERVER_ServerOptions: def __init__(self) -> None: ... def add_rate_limiter_resource(self, arg0: str, arg1: int, arg2: int) -> None: ... def set_backend_config(self, arg0: str, arg1: str, arg2: str) -> None: ... def set_backend_directory(self, arg0: str) -> None: ... def set_buffer_manager_thread_count(self, arg0: int) -> None: ... def set_cache_config(self, arg0: str, arg1: str) -> None: ... def set_cache_directory(self, arg0: str) -> None: ... def set_cpu_metrics(self, arg0: bool) -> None: ... def set_cuda_memory_pool_byte_size(self, arg0: int, arg1: int) -> None: ... def set_exit_on_error(self, arg0: bool) -> None: ... def set_exit_timeout(self, arg0: int) -> None: ... def set_gpu_metrics(self, arg0: bool) -> None: ... def set_host_policy(self, arg0: str, arg1: str, arg2: str) -> None: ... def set_log_error(self, arg0: bool) -> None: ... def set_log_file(self, arg0: str) -> None: ... def set_log_format(self, arg0: TRITONSERVER_LogFormat) -> None: ... def set_log_info(self, arg0: bool) -> None: ... def set_log_verbose(self, arg0: int) -> None: ... def set_log_warn(self, arg0: bool) -> None: ... def set_metrics(self, arg0: bool) -> None: ... def set_metrics_config(self, arg0: str, arg1: str, arg2: str) -> None: ... def set_metrics_interval(self, arg0: int) -> None: ... def set_min_supported_compute_capability(self, arg0: float) -> None: ... def set_model_control_mode(self, arg0: TRITONSERVER_ModelControlMode) -> None: ... def set_model_load_device_limit( self, arg0: TRITONSERVER_InstanceGroupKind, arg1: int, arg2: float ) -> None: ... def set_model_load_thread_count(self, arg0: int) -> None: ... def set_model_load_retry_count(self, arg0: int) -> None: ... def set_model_namespacing(self, arg0: bool) -> None: ... def set_enable_peer_access(self, arg0: bool) -> None: ... def set_model_repository_path(self, arg0: str) -> None: ... def set_pinned_memory_pool_byte_size(self, arg0: int) -> None: ... def set_rate_limiter_mode(self, arg0: TRITONSERVER_RateLimitMode) -> None: ... def set_repo_agent_directory(self, arg0: str) -> None: ... def set_response_cache_byte_size(self, arg0: int) -> None: ... def set_server_id(self, arg0: str) -> None: ... def set_startup_model(self, arg0: str) -> None: ... def set_strict_model_config(self, arg0: bool) -> None: ... def set_strict_readiness(self, arg0: bool) -> None: ... class TritonError(Exception): ... class UnavailableError(TritonError): ... class UnknownError(TritonError): ... class UnsupportedError(TritonError): ... def TRITONSERVER_DataTypeByteSize(arg0: TRITONSERVER_DataType) -> int: ... def TRITONSERVER_DataTypeString(arg0: TRITONSERVER_DataType) -> str: ... def TRITONSERVER_InferenceTraceActivityString( arg0: TRITONSERVER_InferenceTraceActivity, ) -> str: ... def TRITONSERVER_InferenceTraceLevelString( arg0: TRITONSERVER_InferenceTraceLevel, ) -> str: ... def TRITONSERVER_InstanceGroupKindString( arg0: TRITONSERVER_InstanceGroupKind, ) -> str: ... def TRITONSERVER_LogIsEnabled(arg0: TRITONSERVER_LogLevel) -> bool: ... def TRITONSERVER_LogMessage( arg0: TRITONSERVER_LogLevel, arg1: str, arg2: int, arg3: str ) -> None: ... def TRITONSERVER_MemoryTypeString(arg0: TRITONSERVER_MemoryType) -> str: ... def TRITONSERVER_ParameterTypeString(arg0: TRITONSERVER_ParameterType) -> str: ... def TRITONSERVER_StringToDataType(arg0: str) -> TRITONSERVER_DataType: ... def api_version() -> tuple: ... kytensor-2.49.0.7/core/python/tritonserver/_c/tritonserver_pybind.cc000066400000000000000000002334161527052371200256700ustar00rootroot00000000000000// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include // This binding is merely used to map Triton C API into Python equivalent, // and therefore, the naming will be the same as the one used in corresponding // sections. However, there are a few exceptions to better transit to Python: // Structs: // * Triton structs are encapsulated in a thin wrapper to isolate raw pointer // operations which is not supported in pure Python. A thin 'PyWrapper' base // class is defined with common utilities // * Trivial getters and setters are grouped to be a Python class property. // However, this creates asymmetry that some APIs are called like function // while some like member variables. So I am open to expose getter / setter // if it may be more intuitive. // * The wrapper is only served as communication between Python and C, it will // be unwrapped when control reaches C API and the C struct will be wrapped // when control reaches Python side. Python binding user should respect the // "ownership" and lifetime of the wrapper in the same way as described in // the C API. Python binding user must not assume the same C struct will // always be referred through the same wrapper object. // Enums: // * In C API, the enum values are prefixed by the enum name. The Python // equivalent is an enum class and thus the prefix is removed to avoid // duplication, i.e. Python user may specify a value by // 'TRITONSERVER_ResponseCompleteFlag.FINAL'. // Functions / Callbacks: // * Output parameters are converted to return value. APIs that return an error // will be thrown as an exception. The same applies to callbacks. // ** Note that in the C API, the inference response may carry an error object // that represent an inference failure. The equivalent Python API will raise // the corresponding exception if the response contains error object. // * The function parameters and return values are exposed in Python style, // for example, object pointer becomes py::object, C array and length // condenses into Python array. namespace py = pybind11; namespace triton { namespace core { namespace python { // Macro used by PyWrapper #define DISALLOW_COPY(TypeName) TypeName(const TypeName&) = delete; #define DISALLOW_ASSIGN(TypeName) void operator=(const TypeName&) = delete; #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ DISALLOW_COPY(TypeName) \ DISALLOW_ASSIGN(TypeName) #define DESTRUCTOR_WITH_LOG(TypeName, DeleteFunction) \ ~TypeName() \ { \ if (owned_ && triton_object_) { \ auto err__ = (DeleteFunction(triton_object_)); \ if (err__) { \ std::shared_ptr managed_err( \ err__, TRITONSERVER_ErrorDelete); \ py::print(TRITONSERVER_ErrorMessage(err__)); \ } \ }} // base exception for all Triton error code struct TritonError : public std::runtime_error { explicit TritonError(const std::string& what) : std::runtime_error(what) {} }; // triton::core::python exceptions map 1:1 to TRITONSERVER_Error_Code. struct UnknownError : public TritonError { explicit UnknownError(const std::string& what) : TritonError(what) {} }; struct InternalError : public TritonError { explicit InternalError(const std::string& what) : TritonError(what) {} }; struct NotFoundError : public TritonError { explicit NotFoundError(const std::string& what) : TritonError(what) {} }; struct InvalidArgumentError : public TritonError { explicit InvalidArgumentError(const std::string& what) : TritonError(what) {} }; struct UnavailableError : public TritonError { explicit UnavailableError(const std::string& what) : TritonError(what) {} }; struct UnsupportedError : public TritonError { explicit UnsupportedError(const std::string& what) : TritonError(what) {} }; struct AlreadyExistsError : public TritonError { explicit AlreadyExistsError(const std::string& what) : TritonError(what) {} }; TRITONSERVER_Error* CreateTRITONSERVER_ErrorFrom(const py::error_already_set& ex) { // Reserved lookup to get Python type of the exceptions, // 'TRITONSERVER_ERROR_UNKNOWN' is the fallback error code. // static auto uk = // py::module::import("triton_bindings").attr("UnknownError"); static auto it = py::module::import("triton_bindings").attr("InternalError"); static auto nf = py::module::import("triton_bindings").attr("NotFoundError"); static auto ia = py::module::import("triton_bindings").attr("InvalidArgumentError"); static auto ua = py::module::import("triton_bindings").attr("UnavailableError"); static auto us = py::module::import("triton_bindings").attr("UnsupportedError"); static auto ae = py::module::import("triton_bindings").attr("AlreadyExistsError"); TRITONSERVER_Error_Code code = TRITONSERVER_ERROR_UNKNOWN; if (ex.matches(it.ptr())) { code = TRITONSERVER_ERROR_INTERNAL; } else if (ex.matches(nf.ptr())) { code = TRITONSERVER_ERROR_NOT_FOUND; } else if (ex.matches(ia.ptr())) { code = TRITONSERVER_ERROR_INVALID_ARG; } else if (ex.matches(ua.ptr())) { code = TRITONSERVER_ERROR_UNAVAILABLE; } else if (ex.matches(us.ptr())) { code = TRITONSERVER_ERROR_UNSUPPORTED; } else if (ex.matches(ae.ptr())) { code = TRITONSERVER_ERROR_ALREADY_EXISTS; } return TRITONSERVER_ErrorNew(code, ex.what()); } void ThrowIfError(TRITONSERVER_Error* err) { if (err == nullptr) { return; } std::shared_ptr managed_err( err, TRITONSERVER_ErrorDelete); std::string msg = TRITONSERVER_ErrorMessage(err); switch (TRITONSERVER_ErrorCode(err)) { case TRITONSERVER_ERROR_INTERNAL: throw InternalError(std::move(msg)); case TRITONSERVER_ERROR_NOT_FOUND: throw NotFoundError(std::move(msg)); case TRITONSERVER_ERROR_INVALID_ARG: throw InvalidArgumentError(std::move(msg)); case TRITONSERVER_ERROR_UNAVAILABLE: throw UnavailableError(std::move(msg)); case TRITONSERVER_ERROR_UNSUPPORTED: throw UnsupportedError(std::move(msg)); case TRITONSERVER_ERROR_ALREADY_EXISTS: throw AlreadyExistsError(std::move(msg)); default: throw UnknownError(std::move(msg)); } } template class PyWrapper { public: explicit PyWrapper(TritonStruct* triton_object, bool owned) : triton_object_(triton_object), owned_(owned) { } PyWrapper() = default; // Destructor will be defined per specialization for now as a few // Triton object delete functions have different signatures, which // requires a function wrapper to generalize the destructor. // Use internally to get the pointer of the underlying Triton object TritonStruct* Ptr() { return triton_object_; } DISALLOW_COPY_AND_ASSIGN(PyWrapper); protected: TritonStruct* triton_object_{nullptr}; bool owned_{false}; }; class PyParameter : public PyWrapper { public: explicit PyParameter(struct TRITONSERVER_Parameter* p, const bool owned) : PyWrapper(p, owned) { } PyParameter(const char* name, const std::string& val) : PyWrapper( TRITONSERVER_ParameterNew( name, TRITONSERVER_PARAMETER_STRING, val.c_str()), true) { } PyParameter(const char* name, int64_t val) : PyWrapper( TRITONSERVER_ParameterNew(name, TRITONSERVER_PARAMETER_INT, &val), true) { } PyParameter(const char* name, double val) : PyWrapper( TRITONSERVER_ParameterNew( name, TRITONSERVER_PARAMETER_DOUBLE, &val), true) { } PyParameter(const char* name, bool val) : PyWrapper( TRITONSERVER_ParameterNew(name, TRITONSERVER_PARAMETER_BOOL, &val), true) { } PyParameter(const char* name, const void* byte_ptr, uint64_t size) : PyWrapper(TRITONSERVER_ParameterBytesNew(name, byte_ptr, size), true) { } ~PyParameter() { if (owned_ && triton_object_) { TRITONSERVER_ParameterDelete(triton_object_); } } }; class PyBufferAttributes : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyBufferAttributes, TRITONSERVER_BufferAttributesDelete); PyBufferAttributes() { ThrowIfError(TRITONSERVER_BufferAttributesNew(&triton_object_)); owned_ = true; } explicit PyBufferAttributes( struct TRITONSERVER_BufferAttributes* ba, const bool owned) : PyWrapper(ba, owned) { } void SetMemoryTypeId(int64_t memory_type_id) { ThrowIfError(TRITONSERVER_BufferAttributesSetMemoryTypeId( triton_object_, memory_type_id)); } void SetMemoryType(TRITONSERVER_MemoryType memory_type) { ThrowIfError(TRITONSERVER_BufferAttributesSetMemoryType( triton_object_, memory_type)); } void SetCudaIpcHandle(uintptr_t cuda_ipc_handle) { ThrowIfError(TRITONSERVER_BufferAttributesSetCudaIpcHandle( triton_object_, reinterpret_cast(cuda_ipc_handle))); } void SetByteSize(size_t byte_size) { ThrowIfError( TRITONSERVER_BufferAttributesSetByteSize(triton_object_, byte_size)); } // Define methods to get buffer attribute fields int64_t MemoryTypeId() { int64_t memory_type_id = 0; ThrowIfError(TRITONSERVER_BufferAttributesMemoryTypeId( triton_object_, &memory_type_id)); return memory_type_id; } TRITONSERVER_MemoryType MemoryType() { TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; ThrowIfError( TRITONSERVER_BufferAttributesMemoryType(triton_object_, &memory_type)); return memory_type; } uintptr_t CudaIpcHandle() { void* cuda_ipc_handle = nullptr; ThrowIfError(TRITONSERVER_BufferAttributesCudaIpcHandle( triton_object_, &cuda_ipc_handle)); return reinterpret_cast(cuda_ipc_handle); } size_t ByteSize() { size_t byte_size; ThrowIfError( TRITONSERVER_BufferAttributesByteSize(triton_object_, &byte_size)); return byte_size; } }; class PyResponseAllocator : public PyWrapper { public: DESTRUCTOR_WITH_LOG( PyResponseAllocator, TRITONSERVER_ResponseAllocatorDelete); // Callback resource that holds Python user provided buffer and // Triton C callback wrappers. This struct will be used for both // 'allocator_userp' and 'buffer_userp' struct CallbackResource { CallbackResource(const py::object& a, const py::object& uo) : allocator(a), user_object(uo) { } // Storing the py::object of PyResponseAllocator to have convenient access // to callbacks. py::object allocator; py::object user_object; }; using AllocFn = std::function< std::tuple( py::object, std::string, size_t, TRITONSERVER_MemoryType, int64_t, py::object)>; using ReleaseFn = std::function; using StartFn = std::function; // size as input, optional? using QueryFn = std::function( py::object, py::object, std::string, std::optional, TRITONSERVER_MemoryType, int64_t)>; using BufferAttributesFn = std::function; PyResponseAllocator(AllocFn alloc, ReleaseFn release) : alloc_fn_(alloc), release_fn_(release) { ThrowIfError(TRITONSERVER_ResponseAllocatorNew( &triton_object_, PyTritonAllocFn, PyTritonReleaseFn, nullptr)); owned_ = true; } PyResponseAllocator(AllocFn alloc, ReleaseFn release, StartFn start) : alloc_fn_(alloc), release_fn_(release), start_fn_(start) { ThrowIfError(TRITONSERVER_ResponseAllocatorNew( &triton_object_, PyTritonAllocFn, PyTritonReleaseFn, PyTritonStartFn)); owned_ = true; } // Below implements the Triton callbacks, note that when registering the // callbacks in Triton, an wrapped 'CallbackResource' must be used to bridge // the gap between the Python API and C API. static TRITONSERVER_Error* PyTritonAllocFn( struct TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { py::gil_scoped_acquire gil; struct TRITONSERVER_Error* err = nullptr; auto cr = reinterpret_cast(userp); try { auto res = cr->allocator.cast()->alloc_fn_( cr->allocator, tensor_name, byte_size, memory_type, memory_type_id, cr->user_object); *buffer = reinterpret_cast(std::get<0>(res)); { // In C API usage, its typical to allocate user object within the // callback and place the release logic in release callback. The same // logic can't trivially ported to Python as user object is scoped, // therefore the binding needs to wrap the object to ensure the user // object will not be garbage collected until after release callback. *buffer_userp = new CallbackResource(cr->allocator, std::get<1>(res)); } *actual_memory_type = std::get<2>(res); *actual_memory_type_id = std::get<3>(res); } catch (py::error_already_set& ex) { err = CreateTRITONSERVER_ErrorFrom(ex); } return err; } static TRITONSERVER_Error* PyTritonReleaseFn( struct TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { py::gil_scoped_acquire gil; struct TRITONSERVER_Error* err = nullptr; auto cr = reinterpret_cast(buffer_userp); try { cr->allocator.cast()->release_fn_( cr->allocator, reinterpret_cast(buffer), cr->user_object, byte_size, memory_type, memory_type_id); } catch (py::error_already_set& ex) { err = CreateTRITONSERVER_ErrorFrom(ex); } // Done with CallbackResource associated with this buffer delete cr; return err; } static TRITONSERVER_Error* PyTritonStartFn( struct TRITONSERVER_ResponseAllocator* allocator, void* userp) { py::gil_scoped_acquire gil; struct TRITONSERVER_Error* err = nullptr; auto cr = reinterpret_cast(userp); try { cr->allocator.cast()->start_fn_( cr->allocator, cr->user_object); } catch (py::error_already_set& ex) { err = CreateTRITONSERVER_ErrorFrom(ex); } return err; } static TRITONSERVER_Error* PyTritonQueryFn( struct TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { py::gil_scoped_acquire gil; struct TRITONSERVER_Error* err = nullptr; auto cr = reinterpret_cast(userp); try { std::optional bs; if (byte_size) { bs = *byte_size; } auto res = cr->allocator.cast()->query_fn_( cr->allocator, cr->user_object, tensor_name, bs, *memory_type, *memory_type_id); *memory_type = std::get<0>(res); *memory_type_id = std::get<1>(res); } catch (py::error_already_set& ex) { err = CreateTRITONSERVER_ErrorFrom(ex); } return err; } static TRITONSERVER_Error* PyTritonBufferAttributesFn( struct TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, struct TRITONSERVER_BufferAttributes* buffer_attributes, void* userp, void* buffer_userp) { py::gil_scoped_acquire gil; struct TRITONSERVER_Error* err = nullptr; auto cr = reinterpret_cast(userp); auto bcr = reinterpret_cast(buffer_userp); PyBufferAttributes pba{buffer_attributes, false /* owned_ */}; try { // Python version of BufferAttributes callback has return value // to be the filled buffer attributes. The callback implementation // should modify the passed PyBufferAttributes object and return it. // However, the implementation may construct new PyBufferAttributes // which requires additional checking to properly return the attributes // through C API. auto res = cr->allocator.cast()->buffer_attributes_fn_( cr->allocator, tensor_name, py::cast(pba, py::return_value_policy::reference), cr->user_object, bcr->user_object); // Copy if 'res' is new object, otherwise the attributes have been set. auto res_pba = res.cast(); if (res_pba->Ptr() != buffer_attributes) { pba.SetMemoryTypeId(res_pba->MemoryTypeId()); pba.SetMemoryType(res_pba->MemoryType()); pba.SetCudaIpcHandle(res_pba->CudaIpcHandle()); pba.SetByteSize(res_pba->ByteSize()); } } catch (py::error_already_set& ex) { err = CreateTRITONSERVER_ErrorFrom(ex); } return err; } void SetBufferAttributesFunction(BufferAttributesFn baf) { buffer_attributes_fn_ = baf; ThrowIfError(TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( triton_object_, PyTritonBufferAttributesFn)); } void SetQueryFunction(QueryFn qf) { query_fn_ = qf; ThrowIfError(TRITONSERVER_ResponseAllocatorSetQueryFunction( triton_object_, PyTritonQueryFn)); } private: AllocFn alloc_fn_{nullptr}; ReleaseFn release_fn_{nullptr}; StartFn start_fn_{nullptr}; QueryFn query_fn_{nullptr}; BufferAttributesFn buffer_attributes_fn_{nullptr}; }; class PyMessage : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyMessage, TRITONSERVER_MessageDelete); PyMessage(const std::string& serialized_json) { ThrowIfError(TRITONSERVER_MessageNewFromSerializedJson( &triton_object_, serialized_json.c_str(), serialized_json.size())); owned_ = true; } explicit PyMessage(struct TRITONSERVER_Message* m, const bool owned) : PyWrapper(m, owned) { } std::string SerializeToJson() { const char* base = nullptr; size_t byte_size = 0; ThrowIfError( TRITONSERVER_MessageSerializeToJson(triton_object_, &base, &byte_size)); return std::string(base, byte_size); } }; class PyMetrics : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyMetrics, TRITONSERVER_MetricsDelete); explicit PyMetrics(struct TRITONSERVER_Metrics* metrics, bool owned) : PyWrapper(metrics, owned) { } std::string Formatted(TRITONSERVER_MetricFormat format) { const char* base = nullptr; size_t byte_size = 0; ThrowIfError(TRITONSERVER_MetricsFormatted( triton_object_, format, &base, &byte_size)); return std::string(base, byte_size); } }; class PyTrace : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyTrace, TRITONSERVER_InferenceTraceDelete); using TimestampActivityFn = std::function; using TensorActivityFn = std::function, TRITONSERVER_MemoryType, int64_t, py::object)>; using ReleaseFn = std::function, py::object)>; struct CallbackResource { CallbackResource( TimestampActivityFn ts, TensorActivityFn t, ReleaseFn r, const py::object& uo) : timestamp_fn(ts), tensor_fn(t), release_fn(r), user_object(uo) { } TimestampActivityFn timestamp_fn{nullptr}; TensorActivityFn tensor_fn{nullptr}; ReleaseFn release_fn{nullptr}; py::object user_object; // The trace API will use the same 'trace_userp' for all traces associated // with the request, and because there is no guarantee that the root trace // must be released last, need to track all trace seen / released to // determine whether this CallbackResource may be released. std::set seen_traces; }; // Use internally when interacting with C APIs that takes ownership, // this function will also release the ownership of the callback resource // because once the ownership is transferred, the callback resource // will be accessed in the callback pipeline and should not be tied to the // PyWrapper's lifecycle. The callback resource will be released in the // Triton C callback wrapper. struct TRITONSERVER_InferenceTrace* Release() { owned_ = false; callback_resource_.release(); return triton_object_; } PyTrace( int level, uint64_t parent_id, TimestampActivityFn timestamp, ReleaseFn release, const py::object& user_object) : callback_resource_( new CallbackResource(timestamp, nullptr, release, user_object)) { ThrowIfError(TRITONSERVER_InferenceTraceNew( &triton_object_, static_cast(level), parent_id, PyTritonTraceTimestampActivityFn, PyTritonTraceRelease, callback_resource_.get())); owned_ = true; } PyTrace( int level, uint64_t parent_id, TimestampActivityFn timestamp, TensorActivityFn tensor, ReleaseFn release, const py::object& user_object) : callback_resource_( new CallbackResource(timestamp, tensor, release, user_object)) { ThrowIfError(TRITONSERVER_InferenceTraceTensorNew( &triton_object_, static_cast(level), parent_id, PyTritonTraceTimestampActivityFn, PyTritonTraceTensorActivityFn, PyTritonTraceRelease, callback_resource_.get())); owned_ = true; } explicit PyTrace(struct TRITONSERVER_InferenceTrace* t, const bool owned) : PyWrapper(t, owned) { } CallbackResource* ReleaseCallbackResource() { return callback_resource_.release(); } uint64_t Id() { uint64_t val = 0; ThrowIfError(TRITONSERVER_InferenceTraceId(triton_object_, &val)); return val; } uint64_t ParentId() { uint64_t val = 0; ThrowIfError(TRITONSERVER_InferenceTraceParentId(triton_object_, &val)); return val; } std::string ModelName() { const char* val = nullptr; ThrowIfError(TRITONSERVER_InferenceTraceModelName(triton_object_, &val)); return val; } int64_t ModelVersion() { int64_t val = 0; ThrowIfError(TRITONSERVER_InferenceTraceModelVersion(triton_object_, &val)); return val; } std::string RequestId() { const char* val = nullptr; ThrowIfError(TRITONSERVER_InferenceTraceRequestId(triton_object_, &val)); return val; } // Below implements the Triton callbacks, note that when registering the // callbacks in Triton, an wrapped 'CallbackResource' must be used to bridge // the gap between the Python API and C API. static void PyTritonTraceTimestampActivityFn( struct TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, void* userp) { py::gil_scoped_acquire gil; // Note that 'trace' associated with the activity is not necessary the // root trace captured in Callback Resource, so need to always wrap 'trace' // in PyTrace for the Python callback to interact with the correct trace. PyTrace pt(trace, false /* owned */); auto cr = reinterpret_cast(userp); cr->seen_traces.insert(reinterpret_cast(trace)); cr->timestamp_fn( py::cast(pt, py::return_value_policy::reference), activity, timestamp_ns, cr->user_object); } static void PyTritonTraceTensorActivityFn( struct TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, const char* name, TRITONSERVER_DataType datatype, const void* base, size_t byte_size, const int64_t* shape, uint64_t dim_count, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* userp) { py::gil_scoped_acquire gil; // See 'PyTritonTraceTimestampActivityFn' for 'pt' explanation. PyTrace pt(trace, false /* owned */); auto cr = reinterpret_cast(userp); cr->seen_traces.insert(reinterpret_cast(trace)); cr->tensor_fn( py::cast(pt, py::return_value_policy::reference), activity, name, datatype, reinterpret_cast(base), byte_size, py::array_t(dim_count, shape), memory_type, memory_type_id, cr->user_object); } static void PyTritonTraceRelease( struct TRITONSERVER_InferenceTrace* trace, void* userp) { py::gil_scoped_acquire gil; // See 'PyTritonTraceTimestampActivityFn' for 'pt' explanation. // wrap in shared_ptr to transfer ownership to Python auto managed_pt = std::make_shared(trace, true /* owned */); auto cr = reinterpret_cast(userp); cr->release_fn(managed_pt, cr->user_object); cr->seen_traces.erase(reinterpret_cast(trace)); if (cr->seen_traces.empty()) { delete cr; } } private: std::unique_ptr callback_resource_{nullptr}; }; class PyInferenceResponse : public PyWrapper { public: DESTRUCTOR_WITH_LOG( PyInferenceResponse, TRITONSERVER_InferenceResponseDelete); using CompleteFn = std::function; struct CallbackResource { CallbackResource( CompleteFn c, PyResponseAllocator::CallbackResource* a, const py::object& u) : complete_fn(c), allocator_resource(a), user_object(u) { } CompleteFn complete_fn; // During 'TRITONSERVER_InferenceRequestSetResponseCallback', a // PyResponseAllocator::CallbackResource is allocated and passed as // 'response_allocator_userp', which is used during any output buffer // allocation of the requests. However, unlike other 'userp', there is no // dedicated release callback to signal that the allocator resource may be // released. So we deduce the point of time is deduced based on the // following: 'TRITONSERVER_InferenceResponseCompleteFn_t' invoked with // 'TRITONSERVER_RESPONSE_COMPLETE_FINAL' flag indicates there is no more // responses to be generated and so does output allocation, therefore // 'allocator_resource' may be released as part of releasing // 'PyInferenceResponse::CallbackResource' PyResponseAllocator::CallbackResource* allocator_resource; py::object user_object; }; explicit PyInferenceResponse( struct TRITONSERVER_InferenceResponse* response, bool owned) : PyWrapper(response, owned) { } void ThrowIfResponseError() { ThrowIfError(TRITONSERVER_InferenceResponseError(triton_object_)); } std::tuple Model() { const char* model_name = nullptr; int64_t model_version = 0; ThrowIfError(TRITONSERVER_InferenceResponseModel( triton_object_, &model_name, &model_version)); return {model_name, model_version}; } std::string Id() { const char* val = nullptr; ThrowIfError(TRITONSERVER_InferenceResponseId(triton_object_, &val)); return val; } uint32_t ParameterCount() { uint32_t val = 0; ThrowIfError( TRITONSERVER_InferenceResponseParameterCount(triton_object_, &val)); return val; } std::tuple Parameter( uint32_t index) { const char* name = nullptr; TRITONSERVER_ParameterType type = TRITONSERVER_PARAMETER_STRING; const void* value = nullptr; ThrowIfError(TRITONSERVER_InferenceResponseParameter( triton_object_, index, &name, &type, &value)); py::object py_value; switch (type) { case TRITONSERVER_PARAMETER_STRING: py_value = py::str(reinterpret_cast(value)); break; case TRITONSERVER_PARAMETER_INT: py_value = py::int_(*reinterpret_cast(value)); break; case TRITONSERVER_PARAMETER_BOOL: py_value = py::bool_(*reinterpret_cast(value)); break; case TRITONSERVER_PARAMETER_DOUBLE: py_value = py::float_(*reinterpret_cast(value)); break; default: throw UnsupportedError( std::string("Unexpected type '") + TRITONSERVER_ParameterTypeString(type) + "' received as response parameter"); break; } return {name, type, py_value}; } uint32_t OutputCount() { uint32_t val = 0; ThrowIfError( TRITONSERVER_InferenceResponseOutputCount(triton_object_, &val)); return val; } std::tuple< std::string, TRITONSERVER_DataType, py::array_t, uintptr_t, size_t, TRITONSERVER_MemoryType, int64_t, py::object> Output(uint32_t index) { const char* name = nullptr; TRITONSERVER_DataType datatype = TRITONSERVER_TYPE_INVALID; const int64_t* shape = nullptr; uint64_t dim_count = 0; const void* base = nullptr; size_t byte_size = 0; TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; int64_t memory_type_id = 0; void* userp = nullptr; ThrowIfError(TRITONSERVER_InferenceResponseOutput( triton_object_, index, &name, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); return { name, datatype, py::array_t(dim_count, shape), reinterpret_cast(base), byte_size, memory_type, memory_type_id, reinterpret_cast(userp) ->user_object}; } std::string OutputClassificationLabel(uint32_t index, size_t class_index) { const char* val = nullptr; ThrowIfError(TRITONSERVER_InferenceResponseOutputClassificationLabel( triton_object_, index, class_index, &val)); return (val == nullptr) ? "" : val; } }; // forward declaration class PyServer; class PyInferenceRequest : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyInferenceRequest, TRITONSERVER_InferenceRequestDelete); using ReleaseFn = std::function, uint32_t, py::object)>; // Defer definition until PyServer is defined PyInferenceRequest( PyServer& server, const std::string& model_name, const int64_t model_version); explicit PyInferenceRequest( struct TRITONSERVER_InferenceRequest* r, const bool owned) : PyWrapper(r, owned) { } // Use internally when interacting with C APIs that takes ownership, // this function will also release the ownership of the callback resource // because once the ownership is transferred, the callback resource // will be accessed in the callback pipeline and should not be tied to the // PyWrapper's lifecycle. The callback resource will be released in the // Triton C callback wrapper. struct TRITONSERVER_InferenceRequest* Release() { // Note that Release() doesn't change ownership as the // same PyInferenceRequest will be passed along the life cycle. allocator_callback_resource_.release(); response_callback_resource_.release(); return triton_object_; } struct CallbackResource { CallbackResource(ReleaseFn r, const py::object& uo) : release_fn(r), user_object(uo) { } ReleaseFn release_fn; py::object user_object; // Unsafe handling to ensure the same PyInferenceRequest object // goes through the request release cycle. This is due to // a 'keep_alive' relationship is built between 'PyInferenceRequest' // and 'PyServer': a request is associated with a server and the server // should be kept alive until all associated requests is properly released. // And here we exploit the 'keep_alive' utility in PyBind to guarantee so. // See PyServer::InferAsync on how this field is set to avoid potential // circular inclusion. std::shared_ptr request; }; void SetReleaseCallback(ReleaseFn release, const py::object& user_object) { request_callback_resource_.reset( new CallbackResource(release, user_object)); ThrowIfError(TRITONSERVER_InferenceRequestSetReleaseCallback( triton_object_, PyTritonRequestReleaseCallback, request_callback_resource_.get())); } static void PyTritonRequestReleaseCallback( struct TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { py::gil_scoped_acquire gil; auto cr = reinterpret_cast(userp); cr->release_fn(cr->request, flags, cr->user_object); delete cr; } void SetResponseCallback( const py::object& allocator, const py::object& allocater_user_object, PyInferenceResponse::CompleteFn response, const py::object& response_user_object) { allocator_callback_resource_.reset( new PyResponseAllocator::CallbackResource( allocator, allocater_user_object)); response_callback_resource_.reset(new PyInferenceResponse::CallbackResource( response, allocator_callback_resource_.get(), response_user_object)); ThrowIfError(TRITONSERVER_InferenceRequestSetResponseCallback( triton_object_, allocator.cast()->Ptr(), allocator_callback_resource_.get(), PyTritonResponseCompleteCallback, response_callback_resource_.get())); } static void PyTritonResponseCompleteCallback( struct TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { py::gil_scoped_acquire gil; auto managed_pt = std::make_shared(response, true /* owned */); auto cr = reinterpret_cast(userp); if (response == nullptr) { cr->complete_fn(py::none(), flags, cr->user_object); } else { cr->complete_fn(py::cast(managed_pt), flags, cr->user_object); } if (flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) { delete cr->allocator_resource; delete cr; } } // Trivial setters / getters void SetId(const std::string& id) { ThrowIfError( TRITONSERVER_InferenceRequestSetId(triton_object_, id.c_str())); } std::string Id() { const char* val = nullptr; ThrowIfError(TRITONSERVER_InferenceRequestId(triton_object_, &val)); return val; } void SetFlags(uint32_t flags) { ThrowIfError(TRITONSERVER_InferenceRequestSetFlags(triton_object_, flags)); } uint32_t Flags() { uint32_t val = 0; ThrowIfError(TRITONSERVER_InferenceRequestFlags(triton_object_, &val)); return val; } void SetCorrelationId(uint64_t correlation_id) { ThrowIfError(TRITONSERVER_InferenceRequestSetCorrelationId( triton_object_, correlation_id)); } uint64_t CorrelationId() { uint64_t val = 0; ThrowIfError( TRITONSERVER_InferenceRequestCorrelationId(triton_object_, &val)); return val; } void SetCorrelationIdString(const std::string& correlation_id) { ThrowIfError(TRITONSERVER_InferenceRequestSetCorrelationIdString( triton_object_, correlation_id.c_str())); } std::string CorrelationIdString() { const char* val = nullptr; ThrowIfError( TRITONSERVER_InferenceRequestCorrelationIdString(triton_object_, &val)); return val; } void SetPriority(uint32_t priority) { ThrowIfError( TRITONSERVER_InferenceRequestSetPriority(triton_object_, priority)); } void SetPriorityUint64(uint64_t priority) { ThrowIfError(TRITONSERVER_InferenceRequestSetPriorityUInt64( triton_object_, priority)); } uint32_t Priority() { uint32_t val = 0; ThrowIfError(TRITONSERVER_InferenceRequestPriority(triton_object_, &val)); return val; } uint64_t PriorityUint64() { uint64_t val = 0; ThrowIfError( TRITONSERVER_InferenceRequestPriorityUInt64(triton_object_, &val)); return val; } void SetTimeoutMicroseconds(uint64_t timeout_us) { ThrowIfError(TRITONSERVER_InferenceRequestSetTimeoutMicroseconds( triton_object_, timeout_us)); } uint64_t TimeoutMicroseconds() { uint64_t val = 0; ThrowIfError( TRITONSERVER_InferenceRequestTimeoutMicroseconds(triton_object_, &val)); return val; } void AddInput( const std::string& name, TRITONSERVER_DataType data_type, std::vector shape) { ThrowIfError(TRITONSERVER_InferenceRequestAddInput( triton_object_, name.c_str(), data_type, shape.data(), shape.size())); } void AddRawInput(const std::string& name) { ThrowIfError( TRITONSERVER_InferenceRequestAddRawInput(triton_object_, name.c_str())); } void RemoveInput(const std::string& name) { ThrowIfError( TRITONSERVER_InferenceRequestRemoveInput(triton_object_, name.c_str())); } void RemoveAllInputs() { ThrowIfError(TRITONSERVER_InferenceRequestRemoveAllInputs(triton_object_)); } void AppendInputData( const std::string& name, uintptr_t base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { ThrowIfError(TRITONSERVER_InferenceRequestAppendInputData( triton_object_, name.c_str(), reinterpret_cast(base), byte_size, memory_type, memory_type_id)); } void AppendInputDataWithHostPolicy( const std::string name, uintptr_t base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, const std::string& host_policy_name) { ThrowIfError(TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( triton_object_, name.c_str(), reinterpret_cast(base), byte_size, memory_type, memory_type_id, host_policy_name.c_str())); } void AppendInputDataWithBufferAttributes( const std::string& name, uintptr_t base, PyBufferAttributes* buffer_attributes) { ThrowIfError( TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( triton_object_, name.c_str(), reinterpret_cast(base), buffer_attributes->Ptr())); } void RemoveAllInputData(const std::string& name) { ThrowIfError(TRITONSERVER_InferenceRequestRemoveAllInputData( triton_object_, name.c_str())); } void AddRequestedOutput(const std::string& name) { ThrowIfError(TRITONSERVER_InferenceRequestAddRequestedOutput( triton_object_, name.c_str())); } void RemoveRequestedOutput(const std::string& name) { ThrowIfError(TRITONSERVER_InferenceRequestRemoveRequestedOutput( triton_object_, name.c_str())); } void RemoveAllRequestedOutputs() { ThrowIfError( TRITONSERVER_InferenceRequestRemoveAllRequestedOutputs(triton_object_)); } void SetStringParameter(const std::string& key, const std::string& value) { ThrowIfError(TRITONSERVER_InferenceRequestSetStringParameter( triton_object_, key.c_str(), value.c_str())); } void SetIntParameter(const std::string& key, int64_t value) { ThrowIfError(TRITONSERVER_InferenceRequestSetIntParameter( triton_object_, key.c_str(), value)); } void SetBoolParameter(const std::string& key, bool value) { ThrowIfError(TRITONSERVER_InferenceRequestSetBoolParameter( triton_object_, key.c_str(), value)); } void SetDoubleParameter(const std::string& key, double value) { ThrowIfError(TRITONSERVER_InferenceRequestSetDoubleParameter( triton_object_, key.c_str(), value)); } void Cancel() { ThrowIfError(TRITONSERVER_InferenceRequestCancel(triton_object_)); } public: std::unique_ptr request_callback_resource_{nullptr}; private: std::unique_ptr allocator_callback_resource_{nullptr}; std::unique_ptr response_callback_resource_{nullptr}; }; class PyServerOptions : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyServerOptions, TRITONSERVER_ServerOptionsDelete); PyServerOptions() { ThrowIfError(TRITONSERVER_ServerOptionsNew(&triton_object_)); owned_ = true; } void SetServerId(const std::string& server_id) { ThrowIfError(TRITONSERVER_ServerOptionsSetServerId( triton_object_, server_id.c_str())); } void SetModelRepositoryPath(const std::string& model_repository_path) { ThrowIfError(TRITONSERVER_ServerOptionsSetModelRepositoryPath( triton_object_, model_repository_path.c_str())); } void SetModelControlMode(TRITONSERVER_ModelControlMode mode) { ThrowIfError( TRITONSERVER_ServerOptionsSetModelControlMode(triton_object_, mode)); } void SetStartupModel(const std::string& model_name) { ThrowIfError(TRITONSERVER_ServerOptionsSetStartupModel( triton_object_, model_name.c_str())); } void SetStrictModelConfig(bool strict) { ThrowIfError( TRITONSERVER_ServerOptionsSetStrictModelConfig(triton_object_, strict)); } void SetRateLimiterMode(TRITONSERVER_RateLimitMode mode) { ThrowIfError( TRITONSERVER_ServerOptionsSetRateLimiterMode(triton_object_, mode)); } void AddRateLimiterResource( const std::string& resource_name, size_t resource_count, int device) { ThrowIfError(TRITONSERVER_ServerOptionsAddRateLimiterResource( triton_object_, resource_name.c_str(), resource_count, device)); } void SetPinnedMemoryPoolByteSize(uint64_t size) { ThrowIfError(TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize( triton_object_, size)); } void SetCudaMemoryPoolByteSize(int gpu_device, uint64_t size) { ThrowIfError(TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize( triton_object_, gpu_device, size)); } void SetResponseCacheByteSize(uint64_t size) { ThrowIfError(TRITONSERVER_ServerOptionsSetResponseCacheByteSize( triton_object_, size)); } void SetCacheConfig( const std::string& cache_name, const std::string& config_json) { ThrowIfError(TRITONSERVER_ServerOptionsSetCacheConfig( triton_object_, cache_name.c_str(), config_json.c_str())); } void SetCacheDirectory(const std::string& cache_dir) { ThrowIfError(TRITONSERVER_ServerOptionsSetCacheDirectory( triton_object_, cache_dir.c_str())); } void SetMinSupportedComputeCapability(double cc) { ThrowIfError(TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( triton_object_, cc)); } void SetExitOnError(bool exit) { ThrowIfError( TRITONSERVER_ServerOptionsSetExitOnError(triton_object_, exit)); } void SetStrictReadiness(bool strict) { ThrowIfError( TRITONSERVER_ServerOptionsSetStrictReadiness(triton_object_, strict)); } void SetExitTimeout(unsigned int timeout) { ThrowIfError( TRITONSERVER_ServerOptionsSetExitTimeout(triton_object_, timeout)); } void SetBufferManagerThreadCount(unsigned int thread_count) { ThrowIfError(TRITONSERVER_ServerOptionsSetBufferManagerThreadCount( triton_object_, thread_count)); } void SetModelLoadThreadCount(unsigned int thread_count) { ThrowIfError(TRITONSERVER_ServerOptionsSetModelLoadThreadCount( triton_object_, thread_count)); } void SetModelLoadRetryCount(unsigned int retry_count) { ThrowIfError(TRITONSERVER_ServerOptionsSetModelLoadRetryCount( triton_object_, retry_count)); } void SetModelNamespacing(bool enable_namespace) { ThrowIfError(TRITONSERVER_ServerOptionsSetModelNamespacing( triton_object_, enable_namespace)); } void SetEnablePeerAccess(bool enable_peer_access) { ThrowIfError(TRITONSERVER_ServerOptionsSetEnablePeerAccess( triton_object_, enable_peer_access)); } void SetLogFile(const std::string& file) { ThrowIfError( TRITONSERVER_ServerOptionsSetLogFile(triton_object_, file.c_str())); } void SetLogInfo(bool log) { ThrowIfError(TRITONSERVER_ServerOptionsSetLogInfo(triton_object_, log)); } void SetLogWarn(bool log) { ThrowIfError(TRITONSERVER_ServerOptionsSetLogWarn(triton_object_, log)); } void SetLogError(bool log) { ThrowIfError(TRITONSERVER_ServerOptionsSetLogError(triton_object_, log)); } void SetLogFormat(TRITONSERVER_LogFormat format) { ThrowIfError( TRITONSERVER_ServerOptionsSetLogFormat(triton_object_, format)); } void SetLogVerbose(int level) { ThrowIfError( TRITONSERVER_ServerOptionsSetLogVerbose(triton_object_, level)); } void SetMetrics(bool metrics) { ThrowIfError(TRITONSERVER_ServerOptionsSetMetrics(triton_object_, metrics)); } void SetGpuMetrics(bool gpu_metrics) { ThrowIfError( TRITONSERVER_ServerOptionsSetGpuMetrics(triton_object_, gpu_metrics)); } void SetCpuMetrics(bool cpu_metrics) { ThrowIfError( TRITONSERVER_ServerOptionsSetCpuMetrics(triton_object_, cpu_metrics)); } void SetMetricsInterval(uint64_t metrics_interval_ms) { ThrowIfError(TRITONSERVER_ServerOptionsSetMetricsInterval( triton_object_, metrics_interval_ms)); } void SetBackendDirectory(const std::string& backend_dir) { ThrowIfError(TRITONSERVER_ServerOptionsSetBackendDirectory( triton_object_, backend_dir.c_str())); } void SetRepoAgentDirectory(const std::string& repoagent_dir) { ThrowIfError(TRITONSERVER_ServerOptionsSetRepoAgentDirectory( triton_object_, repoagent_dir.c_str())); } void SetModelLoadDeviceLimit( TRITONSERVER_InstanceGroupKind kind, int device_id, double fraction) { ThrowIfError(TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit( triton_object_, kind, device_id, fraction)); } void SetBackendConfig( const std::string& backend_name, const std::string& setting, const std::string& value) { ThrowIfError(TRITONSERVER_ServerOptionsSetBackendConfig( triton_object_, backend_name.c_str(), setting.c_str(), value.c_str())); } void SetHostPolicy( const std::string& policy_name, const std::string& setting, const std::string& value) { ThrowIfError(TRITONSERVER_ServerOptionsSetHostPolicy( triton_object_, policy_name.c_str(), setting.c_str(), value.c_str())); } void SetMetricsConfig( const std::string& name, const std::string& setting, const std::string& value) { ThrowIfError(TRITONSERVER_ServerOptionsSetMetricsConfig( triton_object_, name.c_str(), setting.c_str(), value.c_str())); } }; class PyServer : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyServer, TRITONSERVER_ServerDelete); PyServer(PyServerOptions& options) { ThrowIfError(TRITONSERVER_ServerNew(&triton_object_, options.Ptr())); owned_ = true; } void Stop() const { // ServerStop is blocking for the duration of the server exit timeout, so // ensure to release the GIL. This can allow request release callbacks // to be interleaved while server is waiting for live requests/models // to complete. Without releasing GIL, this function may acquire the GIL // first and block the Triton request from being released/freed, thus // blocking the server's shutdown in a circular manner thinking a model is // still alive. py::gil_scoped_release release; ThrowIfError(TRITONSERVER_ServerStop(triton_object_)); } void RegisterModelRepository( const std::string& repository_path, const std::vector>& name_mapping) const { std::vector params; for (const auto& nm : name_mapping) { params.emplace_back(nm->Ptr()); } ThrowIfError(TRITONSERVER_ServerRegisterModelRepository( triton_object_, repository_path.c_str(), params.data(), params.size())); } void UnregisterModelRepository(const std::string& repository_path) const { ThrowIfError(TRITONSERVER_ServerUnregisterModelRepository( triton_object_, repository_path.c_str())); } void PollModelRepository() const { ThrowIfError(TRITONSERVER_ServerPollModelRepository(triton_object_)); } bool IsLive() const { bool live; ThrowIfError(TRITONSERVER_ServerIsLive(triton_object_, &live)); return live; } bool IsReady() const { bool ready; ThrowIfError(TRITONSERVER_ServerIsReady(triton_object_, &ready)); return ready; } bool ModelIsReady(const std::string& model_name, int64_t model_version) const { bool ready; ThrowIfError(TRITONSERVER_ServerModelIsReady( triton_object_, model_name.c_str(), model_version, &ready)); return ready; } std::tuple ModelBatchProperties( const std::string& model_name, int64_t model_version) const { uint32_t flags; void* voidp; ThrowIfError(TRITONSERVER_ServerModelBatchProperties( triton_object_, model_name.c_str(), model_version, &flags, &voidp)); return {flags, reinterpret_cast(voidp)}; } std::tuple ModelTransactionProperties( const std::string& model_name, int64_t model_version) const { uint32_t txn_flags; void* voidp; ThrowIfError(TRITONSERVER_ServerModelTransactionProperties( triton_object_, model_name.c_str(), model_version, &txn_flags, &voidp)); return {txn_flags, reinterpret_cast(voidp)}; } std::shared_ptr Metadata() const { struct TRITONSERVER_Message* server_metadata; ThrowIfError(TRITONSERVER_ServerMetadata(triton_object_, &server_metadata)); return std::make_shared(server_metadata, true /* owned */); } std::shared_ptr ModelMetadata( const std::string& model_name, int64_t model_version) const { struct TRITONSERVER_Message* model_metadata; ThrowIfError(TRITONSERVER_ServerModelMetadata( triton_object_, model_name.c_str(), model_version, &model_metadata)); return std::make_shared(model_metadata, true /* owned */); } std::shared_ptr ModelStatistics( const std::string& model_name, int64_t model_version) const { struct TRITONSERVER_Message* model_stats; ThrowIfError(TRITONSERVER_ServerModelStatistics( triton_object_, model_name.c_str(), model_version, &model_stats)); return std::make_shared(model_stats, true /* owned */); } std::shared_ptr ModelConfig( const std::string& model_name, int64_t model_version, uint32_t config_version = 1) const { struct TRITONSERVER_Message* model_config; ThrowIfError(TRITONSERVER_ServerModelConfig( triton_object_, model_name.c_str(), model_version, config_version, &model_config)); return std::make_shared(model_config, true /* owned */); } std::shared_ptr ModelIndex(uint32_t flags) const { struct TRITONSERVER_Message* model_index; ThrowIfError( TRITONSERVER_ServerModelIndex(triton_object_, flags, &model_index)); return std::make_shared(model_index, true /* owned */); } void LoadModel(const std::string& model_name) { // load model is blocking, ensure to release GIL py::gil_scoped_release release; ThrowIfError( TRITONSERVER_ServerLoadModel(triton_object_, model_name.c_str())); } void LoadModelWithParameters( const std::string& model_name, const std::vector>& parameters) const { std::vector params; for (const auto& p : parameters) { params.emplace_back(p->Ptr()); } // load model is blocking, ensure to release GIL py::gil_scoped_release release; ThrowIfError(TRITONSERVER_ServerLoadModelWithParameters( triton_object_, model_name.c_str(), params.data(), params.size())); } void UnloadModel(const std::string& model_name) { ThrowIfError( TRITONSERVER_ServerUnloadModel(triton_object_, model_name.c_str())); } void UnloadModelAndDependents(const std::string& model_name) { ThrowIfError(TRITONSERVER_ServerUnloadModelAndDependents( triton_object_, model_name.c_str())); } std::shared_ptr Metrics() const { struct TRITONSERVER_Metrics* metrics; ThrowIfError(TRITONSERVER_ServerMetrics(triton_object_, &metrics)); return std::make_shared(metrics, true /* owned */); } void InferAsync( const std::shared_ptr& request, PyTrace& trace) { // Extra handling to avoid circular inclusion: // request -> request_callback_resource_ -> request // 1. extract 'request_callback_resource_' out and provide // scoped handler to place resource back to request if not released, // TRITONSERVER_ServerInferAsync failed in other words. // 2. add 'request' into resource so request release callback can access it. // 3. call TRITONSERVER_ServerInferAsync. // 4. release the extracted resource if TRITONSERVER_ServerInferAsync // returns. static auto resource_handler = [](PyInferenceRequest::CallbackResource* cr) { if (cr != nullptr) { cr->request->request_callback_resource_.reset(cr); cr->request.reset(); } }; std::unique_ptr< PyInferenceRequest::CallbackResource, decltype(resource_handler)> scoped_rh( request->request_callback_resource_.release(), resource_handler); scoped_rh->request = request; ThrowIfError(TRITONSERVER_ServerInferAsync( triton_object_, request->Ptr(), trace.Ptr())); // Ownership of the internal C object is transferred. scoped_rh.release(); request->Release(); trace.Release(); } void InferAsync(const std::shared_ptr& request) { static auto resource_handler = [](PyInferenceRequest::CallbackResource* cr) { if (cr != nullptr) { cr->request->request_callback_resource_.reset(cr); cr->request.reset(); } }; std::unique_ptr< PyInferenceRequest::CallbackResource, decltype(resource_handler)> scoped_rh( request->request_callback_resource_.release(), resource_handler); scoped_rh->request = request; ThrowIfError( TRITONSERVER_ServerInferAsync(triton_object_, request->Ptr(), nullptr)); // Ownership of the internal C object is transferred. scoped_rh.release(); request->Release(); } }; class PyMetricFamily : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyMetricFamily, TRITONSERVER_MetricFamilyDelete); PyMetricFamily( TRITONSERVER_MetricKind kind, const std::string& name, const std::string& description) { TRITONSERVER_MetricFamilyNew( &triton_object_, kind, name.c_str(), description.c_str()); owned_ = true; } }; class PyMetric : public PyWrapper { public: DESTRUCTOR_WITH_LOG(PyMetric, TRITONSERVER_MetricDelete); PyMetric( PyMetricFamily& family, const std::vector>& labels) { std::vector params; for (const auto& label : labels) { params.emplace_back(label->Ptr()); } ThrowIfError(TRITONSERVER_MetricNew( &triton_object_, family.Ptr(), params.data(), params.size())); owned_ = true; } double Value() const { double val = 0; ThrowIfError(TRITONSERVER_MetricValue(triton_object_, &val)); return val; } void Increment(double val) const { ThrowIfError(TRITONSERVER_MetricIncrement(triton_object_, val)); } void SetValue(double val) const { ThrowIfError(TRITONSERVER_MetricSet(triton_object_, val)); } TRITONSERVER_MetricKind Kind() const { TRITONSERVER_MetricKind val = TRITONSERVER_METRIC_KIND_COUNTER; ThrowIfError(TRITONSERVER_GetMetricKind(triton_object_, &val)); return val; } }; // Deferred definitions.. PyInferenceRequest::PyInferenceRequest( PyServer& server, const std::string& model_name, const int64_t model_version) { ThrowIfError(TRITONSERVER_InferenceRequestNew( &triton_object_, server.Ptr(), model_name.c_str(), model_version)); owned_ = true; } // [FIXME] module name? PYBIND11_MODULE(triton_bindings, m) { m.doc() = "Python bindings for Triton Inference Server"; // [FIXME] if dynamic linking, should have version check here as well to // make sure the binding is compatible with the Triton library loaded m.def("api_version", []() { uint32_t major = 0, minor = 0; ThrowIfError(TRITONSERVER_ApiVersion(&major, &minor)); return py::make_tuple(major, minor); }); // TRITONSERVER_Error... converted to 'TritonError' exception // Implement exception inheritance in PyBind: // https://github.com/jagerman/pybind11/blob/master/tests/test_exceptions.cpp#L149-L152 auto te = pybind11::register_exception(m, "TritonError"); pybind11::register_exception(m, "UnknownError", te.ptr()); pybind11::register_exception(m, "InternalError", te.ptr()); pybind11::register_exception(m, "NotFoundError", te.ptr()); pybind11::register_exception( m, "InvalidArgumentError", te.ptr()); pybind11::register_exception( m, "UnavailableError", te.ptr()); pybind11::register_exception( m, "UnsupportedError", te.ptr()); pybind11::register_exception( m, "AlreadyExistsError", te.ptr()); // TRITONSERVER_DataType py::enum_(m, "TRITONSERVER_DataType") .value("INVALID", TRITONSERVER_TYPE_INVALID) .value("BOOL", TRITONSERVER_TYPE_BOOL) .value("UINT8", TRITONSERVER_TYPE_UINT8) .value("UINT16", TRITONSERVER_TYPE_UINT16) .value("UINT32", TRITONSERVER_TYPE_UINT32) .value("UINT64", TRITONSERVER_TYPE_UINT64) .value("INT8", TRITONSERVER_TYPE_INT8) .value("INT16", TRITONSERVER_TYPE_INT16) .value("INT32", TRITONSERVER_TYPE_INT32) .value("INT64", TRITONSERVER_TYPE_INT64) .value("FP16", TRITONSERVER_TYPE_FP16) .value("FP32", TRITONSERVER_TYPE_FP32) .value("FP64", TRITONSERVER_TYPE_FP64) .value("BYTES", TRITONSERVER_TYPE_BYTES) .value("BF16", TRITONSERVER_TYPE_BF16); // helper functions m.def("TRITONSERVER_DataTypeString", [](TRITONSERVER_DataType datatype) { return TRITONSERVER_DataTypeString(datatype); }); m.def("TRITONSERVER_StringToDataType", [](const char* dtype) { return TRITONSERVER_StringToDataType(dtype); }); m.def("TRITONSERVER_DataTypeByteSize", [](TRITONSERVER_DataType datatype) { return TRITONSERVER_DataTypeByteSize(datatype); }); // TRITONSERVER_MemoryType py::enum_(m, "TRITONSERVER_MemoryType") .value("CPU", TRITONSERVER_MEMORY_CPU) .value("CPU_PINNED", TRITONSERVER_MEMORY_CPU_PINNED) .value("GPU", TRITONSERVER_MEMORY_GPU); // helper functions m.def("TRITONSERVER_MemoryTypeString", [](TRITONSERVER_MemoryType memtype) { return TRITONSERVER_MemoryTypeString(memtype); }); // TRITONSERVER_ParameterType py::enum_(m, "TRITONSERVER_ParameterType") .value("STRING", TRITONSERVER_PARAMETER_STRING) .value("INT", TRITONSERVER_PARAMETER_INT) .value("DOUBLE", TRITONSERVER_PARAMETER_DOUBLE) .value("BOOL", TRITONSERVER_PARAMETER_BOOL) .value("BYTES", TRITONSERVER_PARAMETER_BYTES); // helper functions m.def( "TRITONSERVER_ParameterTypeString", [](TRITONSERVER_ParameterType paramtype) { return TRITONSERVER_ParameterTypeString(paramtype); }); // TRITONSERVER_Parameter py::class_>( m, "TRITONSERVER_Parameter") // Python bytes can be consumed by function accepting string, so order // the py::bytes constructor before string to ensure correct overload // constructor is used .def(py::init([](const char* name, py::bytes bytes) { // [FIXME] does not own 'bytes' in the same way as C API, but can also // hold 'bytes' to make sure it will not be invalidated while in use. // i.e. safe to perform // a = triton_bindings.TRITONSERVER_Parameter("abc", b'abc') // # 'a' still points to valid buffer at this line. // Note that even holding 'bytes', it is the user's responsibility not // to modify 'bytes' while the parameter is in use. py::buffer_info info(py::buffer(bytes).request()); return std::make_unique(name, info.ptr, info.size); })) .def(py::init()) .def(py::init()) .def(py::init()); // TRITONSERVER_InstanceGroupKind py::enum_(m, "TRITONSERVER_InstanceGroupKind") .value("AUTO", TRITONSERVER_INSTANCEGROUPKIND_AUTO) .value("CPU", TRITONSERVER_INSTANCEGROUPKIND_CPU) .value("GPU", TRITONSERVER_INSTANCEGROUPKIND_GPU) .value("MODEL", TRITONSERVER_INSTANCEGROUPKIND_MODEL); m.def( "TRITONSERVER_InstanceGroupKindString", [](TRITONSERVER_InstanceGroupKind kind) { return TRITONSERVER_InstanceGroupKindString(kind); }); // TRITONSERVER_Log py::enum_(m, "TRITONSERVER_LogLevel") .value("INFO", TRITONSERVER_LOG_INFO) .value("WARN", TRITONSERVER_LOG_WARN) .value("ERROR", TRITONSERVER_LOG_ERROR) .value("VERBOSE", TRITONSERVER_LOG_VERBOSE); py::enum_(m, "TRITONSERVER_LogFormat") .value("DEFAULT", TRITONSERVER_LOG_DEFAULT) .value("ISO8601", TRITONSERVER_LOG_ISO8601); m.def("TRITONSERVER_LogIsEnabled", [](TRITONSERVER_LogLevel level) { return TRITONSERVER_LogIsEnabled(level); }); m.def( "TRITONSERVER_LogMessage", [](TRITONSERVER_LogLevel level, const char* filename, const int line, const char* msg) { ThrowIfError(TRITONSERVER_LogMessage(level, filename, line, msg)); }); py::class_(m, "TRITONSERVER_BufferAttributes") .def(py::init<>()) .def_property( "memory_type_id", &PyBufferAttributes::MemoryTypeId, &PyBufferAttributes::SetMemoryTypeId) .def_property( "memory_type", &PyBufferAttributes::MemoryType, &PyBufferAttributes::SetMemoryType) .def_property( "cuda_ipc_handle", &PyBufferAttributes::CudaIpcHandle, &PyBufferAttributes::SetCudaIpcHandle) .def_property( "byte_size", &PyBufferAttributes::ByteSize, &PyBufferAttributes::SetByteSize); py::class_(m, "TRITONSERVER_ResponseAllocator") .def( py::init< PyResponseAllocator::AllocFn, PyResponseAllocator::ReleaseFn, PyResponseAllocator::StartFn>(), py::arg("alloc_function"), py::arg("release_function"), py::arg("start_function")) .def( py::init< PyResponseAllocator::AllocFn, PyResponseAllocator::ReleaseFn>(), py::arg("alloc_function"), py::arg("release_function")) .def( "set_buffer_attributes_function", &PyResponseAllocator::SetBufferAttributesFunction, py::arg("buffer_attributes_function")) .def( "set_query_function", &PyResponseAllocator::SetQueryFunction, py::arg("query_function")); // TRITONSERVER_Message py::class_>(m, "TRITONSERVER_Message") .def(py::init()) .def("serialize_to_json", &PyMessage::SerializeToJson); // TRITONSERVER_Metrics py::enum_(m, "TRITONSERVER_MetricFormat") .value("PROMETHEUS", TRITONSERVER_METRIC_PROMETHEUS); py::class_>(m, "TRITONSERVER_Metrics") .def("formatted", &PyMetrics::Formatted); // TRITONSERVER_InferenceTrace py::enum_( m, "TRITONSERVER_InferenceTraceLevel") .value("DISABLED", TRITONSERVER_TRACE_LEVEL_DISABLED) .value("MIN", TRITONSERVER_TRACE_LEVEL_MIN) .value("MAX", TRITONSERVER_TRACE_LEVEL_MAX) .value("TIMESTAMPS", TRITONSERVER_TRACE_LEVEL_TIMESTAMPS) .value("TENSORS", TRITONSERVER_TRACE_LEVEL_TENSORS) .export_values(); m.def( "TRITONSERVER_InferenceTraceLevelString", &TRITONSERVER_InferenceTraceLevelString); py::enum_( m, "TRITONSERVER_InferenceTraceActivity") .value("REQUEST_START", TRITONSERVER_TRACE_REQUEST_START) .value("QUEUE_START", TRITONSERVER_TRACE_QUEUE_START) .value("COMPUTE_START", TRITONSERVER_TRACE_COMPUTE_START) .value("COMPUTE_INPUT_END", TRITONSERVER_TRACE_COMPUTE_INPUT_END) .value("COMPUTE_OUTPUT_START", TRITONSERVER_TRACE_COMPUTE_OUTPUT_START) .value("COMPUTE_END", TRITONSERVER_TRACE_COMPUTE_END) .value("REQUEST_END", TRITONSERVER_TRACE_REQUEST_END) .value("TENSOR_QUEUE_INPUT", TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT) .value("TENSOR_BACKEND_INPUT", TRITONSERVER_TRACE_TENSOR_BACKEND_INPUT) .value("TENSOR_BACKEND_OUTPUT", TRITONSERVER_TRACE_TENSOR_BACKEND_OUTPUT) .export_values(); m.def( "TRITONSERVER_InferenceTraceActivityString", &TRITONSERVER_InferenceTraceActivityString); py::class_>( m, "TRITONSERVER_InferenceTrace") .def( py::init< int, uint64_t, PyTrace::TimestampActivityFn, PyTrace::TensorActivityFn, PyTrace::ReleaseFn, const py::object&>(), py::arg("level"), py::arg("parent_id"), py::arg("activity_function"), py::arg("tensor_activity_function"), py::arg("release_function"), py::arg("trace_userp")) .def( py::init< int, uint64_t, PyTrace::TimestampActivityFn, PyTrace::ReleaseFn, const py::object&>(), py::arg("level"), py::arg("parent_id"), py::arg("activity_function"), py::arg("release_function"), py::arg("trace_userp")) .def_property_readonly("id", &PyTrace::Id) .def_property_readonly("parent_id", &PyTrace::ParentId) .def_property_readonly("model_name", &PyTrace::ModelName) .def_property_readonly("model_version", &PyTrace::ModelVersion) .def_property_readonly("request_id", &PyTrace::RequestId); // TRITONSERVER_InferenceRequest py::enum_(m, "TRITONSERVER_RequestFlag") .value("SEQUENCE_START", TRITONSERVER_REQUEST_FLAG_SEQUENCE_START) .value("SEQUENCE_END", TRITONSERVER_REQUEST_FLAG_SEQUENCE_END) .export_values(); py::enum_( m, "TRITONSERVER_RequestReleaseFlag") .value("ALL", TRITONSERVER_REQUEST_RELEASE_ALL) .export_values(); py::class_>( m, "TRITONSERVER_InferenceRequest") .def( py::init(), py::keep_alive<1, 2>()) .def("set_release_callback", &PyInferenceRequest::SetReleaseCallback) .def("set_response_callback", &PyInferenceRequest::SetResponseCallback) .def_property("id", &PyInferenceRequest::Id, &PyInferenceRequest::SetId) .def_property( "flags", &PyInferenceRequest::Flags, &PyInferenceRequest::SetFlags) .def_property( "correlation_id", &PyInferenceRequest::CorrelationId, &PyInferenceRequest::SetCorrelationId) .def_property( "correlation_id_string", &PyInferenceRequest::CorrelationIdString, &PyInferenceRequest::SetCorrelationIdString) .def_property( "priority", &PyInferenceRequest::Priority, &PyInferenceRequest::SetPriority) .def_property( "priority_uint64", &PyInferenceRequest::PriorityUint64, &PyInferenceRequest::SetPriorityUint64) .def_property( "timeout_microseconds", &PyInferenceRequest::TimeoutMicroseconds, &PyInferenceRequest::SetTimeoutMicroseconds) .def("add_input", &PyInferenceRequest::AddInput) .def("add_raw_input", &PyInferenceRequest::AddRawInput) .def("remove_input", &PyInferenceRequest::RemoveInput) .def("remove_all_inputs", &PyInferenceRequest::RemoveAllInputs) .def("append_input_data", &PyInferenceRequest::AppendInputData) .def( "append_input_data_with_host_policy", &PyInferenceRequest::AppendInputDataWithHostPolicy) .def( "append_input_data_with_buffer_attributes", &PyInferenceRequest::AppendInputDataWithBufferAttributes) .def("remove_all_input_data", &PyInferenceRequest::RemoveAllInputData) .def("add_requested_output", &PyInferenceRequest::AddRequestedOutput) .def( "remove_requested_output", &PyInferenceRequest::RemoveRequestedOutput) .def( "remove_all_requested_outputs", &PyInferenceRequest::RemoveAllRequestedOutputs) .def("set_string_parameter", &PyInferenceRequest::SetStringParameter) .def("set_int_parameter", &PyInferenceRequest::SetIntParameter) .def("set_bool_parameter", &PyInferenceRequest::SetBoolParameter) .def("set_double_parameter", &PyInferenceRequest::SetDoubleParameter) .def("cancel", &PyInferenceRequest::Cancel); // TRITONSERVER_InferenceResponse py::enum_( m, "TRITONSERVER_ResponseCompleteFlag") .value("FINAL", TRITONSERVER_RESPONSE_COMPLETE_FINAL) .export_values(); py::class_>( m, "TRITONSERVER_InferenceResponse") .def( "throw_if_response_error", &PyInferenceResponse::ThrowIfResponseError) .def_property_readonly("model", &PyInferenceResponse::Model) .def_property_readonly("id", &PyInferenceResponse::Id) .def_property_readonly( "parameter_count", &PyInferenceResponse::ParameterCount) .def("parameter", &PyInferenceResponse::Parameter) .def_property_readonly("output_count", &PyInferenceResponse::OutputCount) .def("output", &PyInferenceResponse::Output) .def( "output_classification_label", &PyInferenceResponse::OutputClassificationLabel); // TRITONSERVER_ServerOptions py::enum_(m, "TRITONSERVER_ModelControlMode") .value("NONE", TRITONSERVER_MODEL_CONTROL_NONE) .value("POLL", TRITONSERVER_MODEL_CONTROL_POLL) .value("EXPLICIT", TRITONSERVER_MODEL_CONTROL_EXPLICIT); py::enum_(m, "TRITONSERVER_RateLimitMode") .value("OFF", TRITONSERVER_RATE_LIMIT_OFF) .value("EXEC_COUNT", TRITONSERVER_RATE_LIMIT_EXEC_COUNT); py::class_(m, "TRITONSERVER_ServerOptions") .def(py::init<>()) .def("set_server_id", &PyServerOptions::SetServerId) .def( "set_model_repository_path", &PyServerOptions::SetModelRepositoryPath) .def("set_model_control_mode", &PyServerOptions::SetModelControlMode) .def("set_startup_model", &PyServerOptions::SetStartupModel) .def("set_strict_model_config", &PyServerOptions::SetStrictModelConfig) .def("set_rate_limiter_mode", &PyServerOptions::SetRateLimiterMode) .def( "add_rate_limiter_resource", &PyServerOptions::AddRateLimiterResource) .def( "set_pinned_memory_pool_byte_size", &PyServerOptions::SetPinnedMemoryPoolByteSize) .def( "set_cuda_memory_pool_byte_size", &PyServerOptions::SetCudaMemoryPoolByteSize) .def( "set_response_cache_byte_size", &PyServerOptions::SetResponseCacheByteSize) .def("set_cache_config", &PyServerOptions::SetCacheConfig) .def("set_cache_directory", &PyServerOptions::SetCacheDirectory) .def( "set_min_supported_compute_capability", &PyServerOptions::SetMinSupportedComputeCapability) .def("set_exit_on_error", &PyServerOptions::SetExitOnError) .def("set_strict_readiness", &PyServerOptions::SetStrictReadiness) .def("set_exit_timeout", &PyServerOptions::SetExitTimeout) .def( "set_buffer_manager_thread_count", &PyServerOptions::SetBufferManagerThreadCount) .def( "set_model_load_thread_count", &PyServerOptions::SetModelLoadThreadCount) .def( "set_model_load_retry_count", &PyServerOptions::SetModelLoadRetryCount) .def("set_model_namespacing", &PyServerOptions::SetModelNamespacing) .def("set_enable_peer_access", &PyServerOptions::SetEnablePeerAccess) .def("set_log_file", &PyServerOptions::SetLogFile) .def("set_log_info", &PyServerOptions::SetLogInfo) .def("set_log_warn", &PyServerOptions::SetLogWarn) .def("set_log_error", &PyServerOptions::SetLogError) .def("set_log_format", &PyServerOptions::SetLogFormat) .def("set_log_verbose", &PyServerOptions::SetLogVerbose) .def("set_metrics", &PyServerOptions::SetMetrics) .def("set_gpu_metrics", &PyServerOptions::SetGpuMetrics) .def("set_cpu_metrics", &PyServerOptions::SetCpuMetrics) .def("set_metrics_interval", &PyServerOptions::SetMetricsInterval) .def("set_backend_directory", &PyServerOptions::SetBackendDirectory) .def("set_repo_agent_directory", &PyServerOptions::SetRepoAgentDirectory) .def( "set_model_load_device_limit", &PyServerOptions::SetModelLoadDeviceLimit) .def("set_backend_config", &PyServerOptions::SetBackendConfig) .def("set_host_policy", &PyServerOptions::SetHostPolicy) .def("set_metrics_config", &PyServerOptions::SetMetricsConfig); // TRITONSERVER_Server py::enum_(m, "TRITONSERVER_ModelBatchFlag") .value("UNKNOWN", TRITONSERVER_BATCH_UNKNOWN) .value("FIRST_DIM", TRITONSERVER_BATCH_FIRST_DIM) .export_values(); py::enum_(m, "TRITONSERVER_ModelIndexFlag") .value("READY", TRITONSERVER_INDEX_FLAG_READY) .export_values(); py::enum_( m, "TRITONSERVER_ModelTxnPropertyFlag") .value("ONE_TO_ONE", TRITONSERVER_TXN_ONE_TO_ONE) .value("DECOUPLED", TRITONSERVER_TXN_DECOUPLED) .export_values(); py::class_(m, "TRITONSERVER_Server") .def(py::init()) .def("stop", &PyServer::Stop) .def("register_model_repository", &PyServer::RegisterModelRepository) .def("unregister_model_repository", &PyServer::UnregisterModelRepository) .def("poll_model_repository", &PyServer::PollModelRepository) .def("is_live", &PyServer::IsLive) .def("is_ready", &PyServer::IsReady) .def("model_is_ready", &PyServer::ModelIsReady) .def("model_batch_properties", &PyServer::ModelBatchProperties) .def( "model_transaction_properties", &PyServer::ModelTransactionProperties) .def("metadata", &PyServer::Metadata) .def("model_metadata", &PyServer::ModelMetadata) .def("model_statistics", &PyServer::ModelStatistics) .def("model_config", &PyServer::ModelConfig) .def("model_index", &PyServer::ModelIndex) .def("load_model", &PyServer::LoadModel) .def("load_model_with_parameters", &PyServer::LoadModelWithParameters) .def("unload_model", &PyServer::UnloadModel) .def("unload_model_and_dependents", &PyServer::UnloadModelAndDependents) .def("metrics", &PyServer::Metrics) .def( "infer_async", py::overload_cast< const std::shared_ptr&, PyTrace&>( &PyServer::InferAsync)) .def( "infer_async", py::overload_cast&>( &PyServer::InferAsync)); // TRITONSERVER_MetricKind py::enum_(m, "TRITONSERVER_MetricKind") .value("COUNTER", TRITONSERVER_METRIC_KIND_COUNTER) .value("GAUGE", TRITONSERVER_METRIC_KIND_GAUGE); // TRITONSERVER_MetricFamily py::class_(m, "TRITONSERVER_MetricFamily") .def(py::init< TRITONSERVER_MetricKind, const std::string&, const std::string&>()); // TRITONSERVER_Metric py::class_(m, "TRITONSERVER_Metric") .def( py::init< PyMetricFamily&, const std::vector>&>(), py::keep_alive<1, 2>()) .def_property_readonly("value", &PyMetric::Value) .def("increment", &PyMetric::Increment) .def("set_value", &PyMetric::SetValue) .def_property_readonly("kind", &PyMetric::Kind); } }}} // namespace triton::core::python kytensor-2.49.0.7/core/python/tritonserver/py.typed000066400000000000000000000000001527052371200223410ustar00rootroot00000000000000kytensor-2.49.0.7/core/src/000077500000000000000000000000001527052371200153545ustar00rootroot00000000000000kytensor-2.49.0.7/core/src/CMakeLists.txt000066400000000000000000000314101527052371200201130ustar00rootroot00000000000000# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(libtritonserver LANGUAGES C CXX) if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++17") # # Dependencies # # We must include the transitive closure of all repos so that we can # override the tag. # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) set(TRITON_COMMON_ENABLE_PROTOBUF ON) FetchContent_MakeAvailable(repo-common) find_package(PkgConfig REQUIRED) # # CUDA # if(${TRITON_ENABLE_GPU}) find_package(CUDAToolkit REQUIRED) find_package(CUDA REQUIRED) message(STATUS "Using CUDA ${CUDA_VERSION}") endif() # TRITON_ENABLE_GPU # # Boost # # Minimum of 1.78 required for use of boost::span. This can eventually be # relaxed and replaced with std::span in C++20. # set(BOOST_ROOT ${Boost_DIR}) # set(Boost_NO_SYSTEM_PATHS ON) # find_package(Boost 1.78 REQUIRED COMPONENTS) # message(STATUS "Using Boost ${Boost_VERSION}") # # Protobuf # set(protobuf_MODULE_COMPATIBLE TRUE CACHE BOOL "protobuf_MODULE_COMPATIBLE" FORCE) find_package(Protobuf REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") # # Prometheus # if(${TRITON_ENABLE_METRICS}) find_package(prometheus-cpp CONFIG REQUIRED) message(STATUS "Using prometheus-cpp ${prometheus-cpp_VERSION}") endif() # TRITON_ENABLE_METRICS # # GCS # if(${TRITON_ENABLE_GCS}) find_package(google_cloud_cpp_storage REQUIRED) message(STATUS "Using google-cloud-cpp ${google_cloud_cpp_storage_VERSION}") endif() # TRITON_ENABLE_GCS # # S3 # if(${TRITON_ENABLE_S3}) find_package(ZLIB REQUIRED) find_package(AWSSDK REQUIRED COMPONENTS s3) message(STATUS "Using aws-sdk-cpp ${AWSSDK_VERSION}") endif() # # Azure Storage # if(${TRITON_ENABLE_AZURE_STORAGE}) find_package(azure-storage-blobs-cpp CONFIG REQUIRED) message(STATUS "Using Azure storage blobs ${azure-storage-blobs-cpp_VERSION}") endif() configure_file(libtritonserver.ldscript libtritonserver.ldscript COPYONLY) set( SERVER_SRCS backend_config.cc backend_manager.cc backend_memory_manager.cc backend_model.cc backend_model_instance.cc buffer_attributes.cc cache_entry.cc cache_manager.cc cuda_utils.cc dynamic_batch_scheduler.cc ensemble_scheduler/ensemble_scheduler.cc ensemble_scheduler/ensemble_utils.cc filesystem/api.cc infer_parameter.cc infer_request.cc infer_response.cc infer_stats.cc infer_trace.cc instance_queue.cc label_provider.cc memory.cc metric_model_reporter.cc metrics.cc metric_family.cc model.cc model_config_utils.cc model_repository_manager/model_lifecycle.cc model_repository_manager/model_repository_manager.cc numa_utils.cc payload.cc pinned_memory_manager.cc rate_limiter.cc repo_agent.cc scheduler_utils.cc sequence_batch_scheduler/sequence_batch_scheduler.cc sequence_batch_scheduler/sequence_utils.cc sequence_state.cc server.cc shared_library.cc status.cc tritoncache.cc tritonserver.cc ) set( SERVER_HDRS backend_config.h backend_manager.h backend_memory_manager.h backend_model.h backend_model_instance.h buffer_attributes.h cache_entry.h cache_manager.h constants.h cuda_utils.h dynamic_batch_scheduler.h ensemble_scheduler/ensemble_scheduler.h ensemble_scheduler/ensemble_utils.h filesystem/api.h infer_parameter.h infer_request.h infer_response.h infer_stats.h infer_trace.h instance_queue.h label_provider.h memory.h metric_model_reporter.h metrics.h metric_family.h model_config_utils.h model.h model_repository_manager/model_lifecycle.h model_repository_manager/model_repository_manager.h numa_utils.h payload.h pinned_memory_manager.h rate_limiter.h repo_agent.h response_allocator.h scheduler.h scheduler_utils.h sequence_batch_scheduler/sequence_batch_scheduler.h sequence_batch_scheduler/sequence_utils.h sequence_state.h server.h server_message.h shared_library.h status.h tritonserver_apis.h ) if(${TRITON_ENABLE_GPU}) set( SERVER_SRCS ${SERVER_SRCS} cuda_memory_manager.cc cuda_block_manager.h model_config_cuda.cc ) set( SERVER_HDRS ${SERVER_HDRS} cuda_memory_manager.h cuda_block_manager.cc model_config_cuda.h ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_ENSEMBLE}) set( SERVER_SRCS ${SERVER_SRCS} ensemble_scheduler/ensemble_model.cc ) set( SERVER_HDRS ${SERVER_HDRS} ensemble_scheduler/ensemble_model.h ) endif() # TRITON_ENABLE_ENSEMBLE add_library( triton-core SHARED ${SERVER_SRCS} ${SERVER_HDRS} ) add_library( TritonCore::triton-core ALIAS triton-core ) target_compile_features(triton-core PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message("Using MSVC as compiler, default target on Windows 10. " "If the target system is not Windows 10, please update _WIN32_WINNT " "to corresponding value.") target_compile_options( triton-core PRIVATE /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor ) else() target_compile_options( triton-core PRIVATE -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Werror ) set_target_properties( triton-core PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtritonserver.ldscript LINK_FLAGS "-Wl,--version-script libtritonserver.ldscript" ) endif() set_target_properties( triton-core PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME tritonserver SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "$\{ORIGIN\}:$\{ORIGIN\}/pytorch" ) target_include_directories( triton-core PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../include ${CMAKE_CURRENT_SOURCE_DIR} ${Boost_INCLUDE_DIRS} ${Protobuf_INCLUDE_DIRS} ) if(${TRITON_ENABLE_GPU}) target_include_directories( triton-core PRIVATE ${CNMEM_PATH}/include ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_METRICS}) target_include_directories( triton-core PRIVATE $ ) endif() # TRITON_ENABLE_METRICS if(${TRITON_ENABLE_GCS}) target_include_directories( triton-core PRIVATE $ ) if (NOT WIN32) # [WIP] still needed? set_source_files_properties( filesystem/api.cc PROPERTIES COMPILE_FLAGS -Wno-missing-field-initializers ) endif() # NOT WIN32 endif() # TRITON_ENABLE_GCS if(${TRITON_ENABLE_S3}) target_include_directories( triton-core PRIVATE $ ) endif() if(${TRITON_ENABLE_AZURE_STORAGE}) target_include_directories( triton-core PRIVATE $ ) endif() # TRITON_ENABLE_AZURE_STORAGE target_compile_definitions( triton-core PRIVATE TRITON_VERSION="${TRITON_VERSION}" ) if(${TRITON_ENABLE_NVTX}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_NVTX=1 ) endif() # TRITON_ENABLE_NVTX if(${TRITON_ENABLE_TRACING}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_TRACING=1 ) endif() # TRITON_ENABLE_TRACING if(${TRITON_ENABLE_LOGGING}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_LOGGING=1 ) endif() # TRITON_ENABLE_LOGGING if(${TRITON_ENABLE_STATS}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_STATS=1 ) endif() # TRITON_ENABLE_STATS if(${TRITON_ENABLE_GPU}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_GPU=1 PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) if(CUDA_VERSION VERSION_GREATER "10.1" OR CUDA_VERSION VERSION_EQUAL "10.1") target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_CUDA_GRAPH=1 ) else() message(WARNING "CUDA ${CUDA_VERSION} does not support CUDA graphs.") endif() endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_MALI_GPU}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_MALI_GPU=1 ) endif() # TRITON_ENABLE_MALI_GPU if(${TRITON_ENABLE_METRICS}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_METRICS=1 ) endif() # TRITON_ENABLE_METRICS if(${TRITON_ENABLE_METRICS_GPU}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_METRICS_GPU=1 ) endif() # TRITON_ENABLE_METRICS_GPU if(${TRITON_ENABLE_METRICS_CPU}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_METRICS_CPU=1 ) endif() # TRITON_ENABLE_METRICS_CPU if(${TRITON_ENABLE_GCS}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_GCS=1 ) endif() # TRITON_ENABLE_GCS if(${TRITON_ENABLE_AZURE_STORAGE}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_AZURE_STORAGE=1 ) endif() # TRITON_ENABLE_AZURE_STORAGE if(${TRITON_ENABLE_S3}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_S3=1 ) endif() # TRITON_ENABLE_S3 if(${TRITON_ENABLE_ENSEMBLE}) target_compile_definitions( triton-core PRIVATE TRITON_ENABLE_ENSEMBLE=1 ) endif() # TRITON_ENABLE_ENSEMBLE FOREACH(p ${TRITON_EXTRA_LIB_PATHS}) target_link_directories( triton-core PRIVATE ${p} ) ENDFOREACH(p) # re2 requires Threads::Threads to be available in the # list of targets find_package(Threads) pkg_check_modules(Re2 REQUIRED re2) message(STATUS "Using re2 ${Re2_VERSION}") target_link_libraries( triton-core PRIVATE proto-library # from repo-common triton-common-async-work-queue # from repo-common triton-common-thread-pool # from repo-common triton-common-error # from repo-common triton-common-model-config # from repo-common triton-common-logging # from repo-common triton-common-json # from repo-common triton-common-table-printer # from repo-common protobuf::libprotobuf libre2.a ) if (NOT WIN32) target_link_libraries( triton-core PRIVATE dl numa ) endif() if(${TRITON_ENABLE_METRICS}) target_link_libraries( triton-core PRIVATE prometheus-cpp::core ) endif() # TRITON_ENABLE_METRICS if(${TRITON_ENABLE_GCS}) target_link_libraries( triton-core PRIVATE google-cloud-cpp::storage ) endif() # TRITON_ENABLE_GCS if(${TRITON_ENABLE_S3}) target_link_libraries( triton-core PRIVATE aws-cpp-sdk-s3 ) endif() # TRITON_ENABLE_S3 if(${TRITON_ENABLE_AZURE_STORAGE}) target_link_libraries( triton-core PRIVATE Azure::azure-storage-blobs ) endif() # TRITON_ENABLE_AZURE_STORAGE if(${TRITON_ENABLE_GPU}) find_library(CNMEM_LIBRARY NAMES cnmem PATHS ${CNMEM_PATH}/lib) target_link_libraries( triton-core PRIVATE ${CNMEM_LIBRARY} CUDA::cudart ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_METRICS_GPU}) target_link_libraries( triton-core PRIVATE dcgm ) endif() # TRITON_ENABLE_METRICS_GPU install( TARGETS triton-core LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) # Currently unit tests do not build for windows... #if (NOT WIN32) # add_subdirectory(test test) #endif() # NOT WIN32 kytensor-2.49.0.7/core/src/backend_config.cc000066400000000000000000000151021527052371200205760ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "backend_config.h" #include "status.h" #include "triton/common/logging.h" #include "triton/common/model_config.h" namespace triton { namespace core { namespace { Status CheckTFSpecializedBackendName( const triton::common::BackendCmdlineConfigMap& config_map) { std::string tf_version_str = "2"; const auto& itr = config_map.find("tensorflow"); if (itr != config_map.end()) { if (BackendConfiguration(itr->second, "version", &tf_version_str).IsOk()) { if (tf_version_str == "1") { return Status( Status::Code::INVALID_ARG, "starting from 23.04, Triton no longer supports Tensorflow 1. " "Please switch to Tensorflow 2."); } if (tf_version_str != "2") { return Status( Status::Code::INVALID_ARG, "unexpected TensorFlow library version '" + tf_version_str + "', expects 2."); } } } return Status::Success; } } // namespace Status BackendConfiguration( const triton::common::BackendCmdlineConfig& config, const std::string& key, std::string* val) { for (const auto& pr : config) { if (pr.first == key) { *val = pr.second; return Status::Success; } } return Status( Status::Code::INTERNAL, std::string("unable to find common backend configuration for '") + key + "'"); } Status BackendConfigurationParseStringToDouble(const std::string& str, double* val) { try { *val = std::stod(str); } catch (...) { return Status( Status::Code::INTERNAL, "unable to parse common backend configuration as double"); } return Status::Success; } Status BackendConfigurationParseStringToBool(const std::string& str, bool* val) { try { std::string lowercase_str{str}; std::transform( lowercase_str.begin(), lowercase_str.end(), lowercase_str.begin(), [](unsigned char c) { return std::tolower(c); }); *val = (lowercase_str == "true"); } catch (...) { return Status( Status::Code::INTERNAL, "unable to parse common backend configuration as bool"); } return Status::Success; } Status BackendConfigurationGlobalBackendsDirectory( const triton::common::BackendCmdlineConfigMap& config_map, std::string* dir) { const auto& itr = config_map.find(std::string()); if (itr == config_map.end()) { return Status( Status::Code::INTERNAL, "unable to find global backends directory configuration"); } RETURN_IF_ERROR(BackendConfiguration(itr->second, "backend-directory", dir)); return Status::Success; } Status BackendConfigurationMinComputeCapability( const triton::common::BackendCmdlineConfigMap& config_map, double* mcc) { #ifdef TRITON_ENABLE_GPU *mcc = TRITON_MIN_COMPUTE_CAPABILITY; #else *mcc = 0; #endif // TRITON_ENABLE_GPU const auto& itr = config_map.find(std::string()); if (itr == config_map.end()) { return Status( Status::Code::INTERNAL, "unable to find common backend configuration"); } std::string min_compute_capability_str; RETURN_IF_ERROR(BackendConfiguration( itr->second, "min-compute-capability", &min_compute_capability_str)); RETURN_IF_ERROR( BackendConfigurationParseStringToDouble(min_compute_capability_str, mcc)); return Status::Success; } Status BackendConfigurationAutoCompleteConfig( const triton::common::BackendCmdlineConfigMap& config_map, bool* acc) { const auto& itr = config_map.find(std::string()); if (itr == config_map.end()) { return Status( Status::Code::INTERNAL, "unable to find auto-complete configuration"); } std::string auto_complete_config_str; RETURN_IF_ERROR(BackendConfiguration( itr->second, "auto-complete-config", &auto_complete_config_str)); RETURN_IF_ERROR( BackendConfigurationParseStringToBool(auto_complete_config_str, acc)); return Status::Success; } Status BackendConfigurationSpecializeBackendName( const triton::common::BackendCmdlineConfigMap& config_map, const std::string& backend_name, std::string* specialized_name) { *specialized_name = backend_name; if (backend_name == "tensorflow") { RETURN_IF_ERROR(CheckTFSpecializedBackendName(config_map)); } return Status::Success; } Status BackendConfigurationModelLoadGpuFraction( const triton::common::BackendCmdlineConfigMap& config_map, const int device_id, double* memory_limit) { *memory_limit = 1.0; const auto& itr = config_map.find(std::string()); if (itr == config_map.end()) { return Status( Status::Code::INTERNAL, "unable to find global backends directory configuration"); } static std::string key_prefix = "model-load-gpu-limit-device-"; std::string memory_limit_str; auto status = BackendConfiguration( itr->second, key_prefix + std::to_string(device_id), &memory_limit_str); // Allow missing key, default to 1.0 (no limit) if the limit is not specified if (status.IsOk()) { RETURN_IF_ERROR(BackendConfigurationParseStringToDouble( memory_limit_str, memory_limit)); } return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_config.h000066400000000000000000000066441527052371200204530ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "status.h" #include "triton/common/model_config.h" namespace triton { namespace core { /// Get a key's string value from a backend configuration. Status BackendConfiguration( const triton::common::BackendCmdlineConfig& config, const std::string& key, std::string* val); /// Convert a backend configuration string value into a double. Status BackendConfigurationParseStringToDouble( const std::string& str, double* val); /// Convert a backend configuration string value into a bool. Status BackendConfigurationParseStringToBool(const std::string& str, bool* val); /// Get the global backends directory from the backend configuration. Status BackendConfigurationGlobalBackendsDirectory( const triton::common::BackendCmdlineConfigMap& config_map, std::string* dir); /// Get the minimum compute capability from the backend configuration. Status BackendConfigurationMinComputeCapability( const triton::common::BackendCmdlineConfigMap& config_map, double* mcc); /// Get the model configuration auto-complete setting from the backend /// configuration. Status BackendConfigurationAutoCompleteConfig( const triton::common::BackendCmdlineConfigMap& config_map, bool* acc); /// Convert a backend name to the specialized version of that name /// based on the backend configuration. For example, "tensorflow" will /// convert to either "tensorflow1" or "tensorflow2" depending on how /// tritonserver is run. Status BackendConfigurationSpecializeBackendName( const triton::common::BackendCmdlineConfigMap& config_map, const std::string& backend_name, std::string* specialized_name); /// Get GPU memory limit fraction for model loading /// from the backend configuration. Status BackendConfigurationModelLoadGpuFraction( const triton::common::BackendCmdlineConfigMap& config_map, const int device_id, double* memory_limit); }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_manager.cc000066400000000000000000000335221527052371200207510ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "backend_manager.h" #include "backend_memory_manager.h" #include "server_message.h" #include "shared_library.h" #include "triton/common/logging.h" // For unknown reason, windows will not export the TRITONBACKEND_* // functions declared with dllexport in tritonbackend.h. To get those // functions exported it is (also?) necessary to mark the definitions // in this file with dllexport as well. #if defined(_MSC_VER) #define TRITONAPI_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONAPI_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONAPI_DECLSPEC #endif namespace triton { namespace core { // // TritonBackend // Status TritonBackend::Create( const std::string& name, const std::string& dir, const std::string& libpath, const triton::common::BackendCmdlineConfig& backend_cmdline_config, std::shared_ptr* backend) { // Create the JSON representation of the backend configuration. triton::common::TritonJson::Value backend_config_json( triton::common::TritonJson::ValueType::OBJECT); if (!backend_cmdline_config.empty()) { triton::common::TritonJson::Value cmdline_json( backend_config_json, triton::common::TritonJson::ValueType::OBJECT); for (const auto& pr : backend_cmdline_config) { RETURN_IF_ERROR(cmdline_json.AddString(pr.first.c_str(), pr.second)); } RETURN_IF_ERROR( backend_config_json.Add("cmdline", std::move(cmdline_json))); } TritonServerMessage backend_config(backend_config_json); auto local_backend = std::shared_ptr( new TritonBackend(name, dir, libpath, backend_config)); // Load the library and initialize all the entrypoints RETURN_IF_ERROR(local_backend->LoadBackendLibrary()); // Backend initialization is optional... The TRITONBACKEND_Backend // object is this TritonBackend object. We must set set shared // library path to point to the backend directory in case the // backend library attempts to load additional shared libraries. if (local_backend->backend_init_fn_ != nullptr) { std::unique_ptr slib; RETURN_IF_ERROR(SharedLibrary::Acquire(&slib)); RETURN_IF_ERROR(slib->SetLibraryDirectory(local_backend->dir_)); TRITONSERVER_Error* err = local_backend->backend_init_fn_( reinterpret_cast(local_backend.get())); RETURN_IF_ERROR(slib->ResetLibraryDirectory()); RETURN_IF_TRITONSERVER_ERROR(err); } local_backend->UpdateAttributes(); *backend = std::move(local_backend); return Status::Success; } Status TritonBackend::UpdateAttributes() { if (backend_attri_fn_ == nullptr) { return Status::Success; } // Create an Attribute object for the backend to fill, note that it copies // some fields from 'attributes_' while the others use default value. This // is an ad hoc way to determine whether the attribute is set by the backend // and keep / update current value. Attribute latest; latest.exec_policy_ = attributes_.exec_policy_; RETURN_IF_TRITONSERVER_ERROR(backend_attri_fn_( reinterpret_cast(this), reinterpret_cast(&latest))); // Update attributes that were set attributes_.exec_policy_ = latest.exec_policy_; if (!latest.preferred_groups_.empty()) { attributes_.preferred_groups_ = latest.preferred_groups_; } attributes_.parallel_instance_loading_ = latest.parallel_instance_loading_; return Status::Success; } TritonBackend::TritonBackend( const std::string& name, const std::string& dir, const std::string& libpath, const TritonServerMessage& backend_config) : name_(name), dir_(dir), libpath_(libpath), backend_config_(backend_config), state_(nullptr) { ClearHandles(); } TritonBackend::~TritonBackend() { LOG_VERBOSE(1) << "unloading backend '" << name_ << "'"; // Backend finalization is optional... The TRITONBACKEND_Backend // object is this TritonBackend object. if (backend_fini_fn_ != nullptr) { LOG_TRITONSERVER_ERROR( backend_fini_fn_(reinterpret_cast(this)), "failed finalizing backend"); } ClearHandles(); } void TritonBackend::ClearHandles() { dlhandle_ = nullptr; backend_init_fn_ = nullptr; backend_fini_fn_ = nullptr; backend_attri_fn_ = nullptr; model_init_fn_ = nullptr; model_fini_fn_ = nullptr; inst_init_fn_ = nullptr; inst_fini_fn_ = nullptr; inst_exec_fn_ = nullptr; } Status TritonBackend::LoadBackendLibrary() { TritonBackendInitFn_t bifn; TritonBackendFiniFn_t bffn; TritonBackendAttriFn_t bafn; TritonModelInitFn_t mifn; TritonModelFiniFn_t mffn; TritonModelInstanceInitFn_t iifn; TritonModelInstanceFiniFn_t iffn; TritonModelInstanceExecFn_t iefn; { std::unique_ptr slib; RETURN_IF_ERROR(SharedLibrary::Acquire(&slib)); RETURN_IF_ERROR(slib->OpenLibraryHandle(libpath_, &dlhandle_)); // Backend initialize and finalize functions, optional RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONBACKEND_Initialize", true /* optional */, reinterpret_cast(&bifn))); RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONBACKEND_Finalize", true /* optional */, reinterpret_cast(&bffn))); // Backend attribute function, optional RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONBACKEND_GetBackendAttribute", true /* optional */, reinterpret_cast(&bafn))); // Model initialize and finalize functions, optional RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONBACKEND_ModelInitialize", true /* optional */, reinterpret_cast(&mifn))); RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONBACKEND_ModelFinalize", true /* optional */, reinterpret_cast(&mffn))); // Model instance initialize and finalize functions, optional RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONBACKEND_ModelInstanceInitialize", true /* optional */, reinterpret_cast(&iifn))); RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONBACKEND_ModelInstanceFinalize", true /* optional */, reinterpret_cast(&iffn))); // Model instance execute function, required RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONBACKEND_ModelInstanceExecute", false /* optional */, reinterpret_cast(&iefn))); } backend_init_fn_ = bifn; backend_fini_fn_ = bffn; backend_attri_fn_ = bafn; model_init_fn_ = mifn; model_fini_fn_ = mffn; inst_init_fn_ = iifn; inst_fini_fn_ = iffn; inst_exec_fn_ = iefn; return Status::Success; } extern "C" { TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ApiVersion(uint32_t* major, uint32_t* minor) { *major = TRITONBACKEND_API_VERSION_MAJOR; *minor = TRITONBACKEND_API_VERSION_MINOR; return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendName(TRITONBACKEND_Backend* backend, const char** name) { TritonBackend* tb = reinterpret_cast(backend); *name = tb->Name().c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendConfig( TRITONBACKEND_Backend* backend, TRITONSERVER_Message** backend_config) { TritonBackend* tb = reinterpret_cast(backend); *backend_config = const_cast( reinterpret_cast(&tb->BackendConfig())); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendExecutionPolicy( TRITONBACKEND_Backend* backend, TRITONBACKEND_ExecutionPolicy* policy) { TritonBackend* tb = reinterpret_cast(backend); *policy = tb->ExecutionPolicy(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendSetExecutionPolicy( TRITONBACKEND_Backend* backend, TRITONBACKEND_ExecutionPolicy policy) { TritonBackend* tb = reinterpret_cast(backend); tb->SetExecutionPolicy(policy); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendArtifacts( TRITONBACKEND_Backend* backend, TRITONBACKEND_ArtifactType* artifact_type, const char** location) { TritonBackend* tb = reinterpret_cast(backend); *artifact_type = TRITONBACKEND_ARTIFACT_FILESYSTEM; *location = tb->Directory().c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendMemoryManager( TRITONBACKEND_Backend* backend, TRITONBACKEND_MemoryManager** manager) { static TritonMemoryManager gMemoryManager; *manager = reinterpret_cast(&gMemoryManager); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendState(TRITONBACKEND_Backend* backend, void** state) { TritonBackend* tb = reinterpret_cast(backend); *state = tb->State(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendSetState(TRITONBACKEND_Backend* backend, void* state) { TritonBackend* tb = reinterpret_cast(backend); tb->SetState(state); return nullptr; // success } } // extern C // // TritonBackendManager // static std::weak_ptr backend_manager_; static std::mutex mu_; Status TritonBackendManager::Create(std::shared_ptr* manager) { std::lock_guard lock(mu_); // If there is already a manager then we just use it... *manager = backend_manager_.lock(); if (*manager != nullptr) { return Status::Success; } manager->reset(new TritonBackendManager()); backend_manager_ = *manager; return Status::Success; } Status TritonBackendManager::CreateBackend( const std::string& name, const std::string& dir, const std::string& libpath, const triton::common::BackendCmdlineConfig& backend_cmdline_config, bool is_python_based_backend, std::shared_ptr* backend) { std::lock_guard lock(mu_); const auto python_based_backend_path = JoinPath({dir, kPythonFilename}); std::vector paths = {libpath, python_based_backend_path}; for (const auto& path : paths) { const auto& itr = backend_map_.find(path); // If backend already exists, re-use it. if (itr != backend_map_.end()) { *backend = itr->second; // Python based backends use the same shared library as python backend. // If libpath to libtriton_python.so is already found, we need to check // if backend names match. If not, we create a new python based backend. if ((*backend)->Name() == name) { return Status::Success; } } } RETURN_IF_ERROR(TritonBackend::Create( name, dir, libpath, backend_cmdline_config, backend)); (*backend)->SetPythonBasedBackendFlag(is_python_based_backend); if (is_python_based_backend) { backend_map_.insert({python_based_backend_path, *backend}); } else { backend_map_.insert({libpath, *backend}); } return Status::Success; } Status TritonBackendManager::BackendState( std::unique_ptr>>* backend_state) { std::lock_guard lock(mu_); std::unique_ptr>> backend_state_map( new std::unordered_map>); std::vector python_backend_config{}; bool has_python_backend = false; for (const auto& backend_pair : backend_map_) { auto& libpath = backend_pair.first; auto backend = backend_pair.second; if (backend->Name() == kPythonBackend) { has_python_backend = true; } const char* backend_config; size_t backend_config_size; backend->BackendConfig().Serialize(&backend_config, &backend_config_size); backend_state_map->insert( {backend->Name(), std::vector{libpath, backend_config}}); if (backend->IsPythonBackendBased() && python_backend_config.empty()) { python_backend_config.emplace_back(backend->LibPath()); python_backend_config.emplace_back(std::string(backend_config)); } } if (!has_python_backend && !python_backend_config.empty()) { backend_state_map->insert({kPythonBackend, python_backend_config}); } *backend_state = std::move(backend_state_map); return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_manager.h000066400000000000000000000147771527052371200206260ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include "constants.h" #include "filesystem/api.h" #include "server_message.h" #include "status.h" #include "triton/common/model_config.h" #include "tritonserver_apis.h" namespace triton { namespace core { // // Proxy to a backend shared library. // class TritonBackend { public: struct Attribute { Attribute() : exec_policy_(TRITONBACKEND_EXECUTION_BLOCKING), parallel_instance_loading_(false) { } TRITONBACKEND_ExecutionPolicy exec_policy_; std::vector preferred_groups_; // Whether the backend supports loading model instances in parallel bool parallel_instance_loading_; }; typedef TRITONSERVER_Error* (*TritonModelInitFn_t)( TRITONBACKEND_Model* model); typedef TRITONSERVER_Error* (*TritonModelFiniFn_t)( TRITONBACKEND_Model* model); typedef TRITONSERVER_Error* (*TritonModelInstanceInitFn_t)( TRITONBACKEND_ModelInstance* instance); typedef TRITONSERVER_Error* (*TritonModelInstanceFiniFn_t)( TRITONBACKEND_ModelInstance* instance); typedef TRITONSERVER_Error* (*TritonModelInstanceExecFn_t)( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_cnt); static Status Create( const std::string& name, const std::string& dir, const std::string& libpath, const triton::common::BackendCmdlineConfig& backend_cmdline_config, std::shared_ptr* backend); ~TritonBackend(); const std::string& Name() const { return name_; } const std::string& Directory() const { return dir_; } const std::string& LibPath() const { return libpath_; } const TritonServerMessage& BackendConfig() const { return backend_config_; } const Attribute& BackendAttributes() const { return attributes_; } TRITONBACKEND_ExecutionPolicy ExecutionPolicy() const { return attributes_.exec_policy_; } void SetExecutionPolicy(const TRITONBACKEND_ExecutionPolicy policy) { attributes_.exec_policy_ = policy; } void* State() { return state_; } void SetState(void* state) { state_ = state; } bool IsPythonBackendBased() { return is_python_based_backend_; } void SetPythonBasedBackendFlag(bool is_python_based_backend) { is_python_based_backend_ = is_python_based_backend; } TritonModelInitFn_t ModelInitFn() const { return model_init_fn_; } TritonModelFiniFn_t ModelFiniFn() const { return model_fini_fn_; } TritonModelInstanceInitFn_t ModelInstanceInitFn() const { return inst_init_fn_; } TritonModelInstanceFiniFn_t ModelInstanceFiniFn() const { return inst_fini_fn_; } TritonModelInstanceExecFn_t ModelInstanceExecFn() const { return inst_exec_fn_; } private: typedef TRITONSERVER_Error* (*TritonBackendInitFn_t)( TRITONBACKEND_Backend* backend); typedef TRITONSERVER_Error* (*TritonBackendFiniFn_t)( TRITONBACKEND_Backend* backend); typedef TRITONSERVER_Error* (*TritonBackendAttriFn_t)( TRITONBACKEND_Backend* backend, TRITONBACKEND_BackendAttribute* backend_attributes); TritonBackend( const std::string& name, const std::string& dir, const std::string& libpath, const TritonServerMessage& backend_config); void ClearHandles(); Status LoadBackendLibrary(); Status UpdateAttributes(); // The name of the backend. const std::string name_; // Full path to the directory holding backend shared library and // other artifacts. const std::string dir_; // Full path to the backend shared library. const std::string libpath_; bool is_python_based_backend_; // Backend configuration as JSON TritonServerMessage backend_config_; // backend attributes Attribute attributes_; // dlopen / dlsym handles void* dlhandle_; TritonBackendInitFn_t backend_init_fn_; TritonBackendFiniFn_t backend_fini_fn_; TritonBackendAttriFn_t backend_attri_fn_; TritonModelInitFn_t model_init_fn_; TritonModelFiniFn_t model_fini_fn_; TritonModelInstanceInitFn_t inst_init_fn_; TritonModelInstanceFiniFn_t inst_fini_fn_; TritonModelInstanceExecFn_t inst_exec_fn_; // Opaque state associated with the backend. void* state_; }; // // Manage communication with Triton backends and their lifecycle. // class TritonBackendManager { public: static Status Create(std::shared_ptr* manager); Status CreateBackend( const std::string& name, const std::string& dir, const std::string& libpath, const triton::common::BackendCmdlineConfig& backend_cmdline_config, bool is_python_based_backend, std::shared_ptr* backend); Status BackendState( std::unique_ptr< std::unordered_map>>* backend_state); private: DISALLOW_COPY_AND_ASSIGN(TritonBackendManager); TritonBackendManager() = default; std::unordered_map> backend_map_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_memory_manager.cc000066400000000000000000000115671527052371200223460ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "backend_memory_manager.h" #include "pinned_memory_manager.h" #include "status.h" #include "tritonserver_apis.h" #ifdef TRITON_ENABLE_GPU #include #include "cuda_memory_manager.h" #endif // TRITON_ENABLE_GPU // For unknown reason, windows will not export the TRITONBACKEND_* // functions declared with dllexport in tritonbackend.h. To get those // functions exported it is (also?) necessary to mark the definitions // in this file with dllexport as well. #if defined(_MSC_VER) #define TRITONAPI_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONAPI_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONAPI_DECLSPEC #endif namespace triton { namespace core { extern "C" { TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_MemoryManagerAllocate( TRITONBACKEND_MemoryManager* manager, void** buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id, const uint64_t byte_size) { switch (memory_type) { case TRITONSERVER_MEMORY_GPU: #ifdef TRITON_ENABLE_GPU { auto status = CudaMemoryManager::Alloc(buffer, byte_size, memory_type_id); if (!status.IsOk()) { return TRITONSERVER_ErrorNew( StatusCodeToTritonCode(status.ErrorCode()), status.Message().c_str()); } break; } #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "GPU memory allocation not supported"); #endif // TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_CPU_PINNED: #ifdef TRITON_ENABLE_GPU { TRITONSERVER_MemoryType mt = memory_type; auto status = PinnedMemoryManager::Alloc(buffer, byte_size, &mt, false); if (!status.IsOk()) { return TRITONSERVER_ErrorNew( StatusCodeToTritonCode(status.ErrorCode()), status.Message().c_str()); } break; } #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Pinned memory allocation not supported"); #endif // TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_CPU: { *buffer = malloc(byte_size); if (*buffer == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "CPU memory allocation failed"); } break; } } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_MemoryManagerFree( TRITONBACKEND_MemoryManager* manager, void* buffer, const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id) { switch (memory_type) { case TRITONSERVER_MEMORY_GPU: { #ifdef TRITON_ENABLE_GPU auto status = CudaMemoryManager::Free(buffer, memory_type_id); if (!status.IsOk()) { return TRITONSERVER_ErrorNew( StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str()); } #endif // TRITON_ENABLE_GPU break; } case TRITONSERVER_MEMORY_CPU_PINNED: { #ifdef TRITON_ENABLE_GPU auto status = PinnedMemoryManager::Free(buffer); if (!status.IsOk()) { return TRITONSERVER_ErrorNew( StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str()); } #endif // TRITON_ENABLE_GPU break; } case TRITONSERVER_MEMORY_CPU: free(buffer); break; } return nullptr; // success } } // extern C }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_memory_manager.h000066400000000000000000000034401527052371200221770ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once namespace triton { namespace core { // Currently there is just a global memory manager that is used for // all backends and which simply forwards requests on to the core // memory manager. struct TritonMemoryManager {}; }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_model.cc000066400000000000000000002067671527052371200204540ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "backend_model.h" #include #include #include "backend_config.h" #include "dynamic_batch_scheduler.h" #include "filesystem/api.h" #include "model_config_utils.h" #include "numa_utils.h" #include "sequence_batch_scheduler/sequence_batch_scheduler.h" #include "sequence_state.h" #include "server.h" #include "server_message.h" #include "shared_library.h" #include "triton/common/logging.h" #include "tritonserver_apis.h" // For unknown reason, windows will not export the TRITONBACKEND_* // functions declared with dllexport in tritonbackend.h. To get those // functions exported it is (also?) necessary to mark the definitions // in this file with dllexport as well. #if defined(_MSC_VER) #define TRITONAPI_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONAPI_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONAPI_DECLSPEC #endif namespace triton { namespace core { Status TritonModel::Create( InferenceServer* server, const std::string& model_path, const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map, const triton::common::HostPolicyCmdlineConfigMap& host_policy_map, const ModelIdentifier& model_id, const int64_t version, inference::ModelConfig model_config, const bool is_config_provided, std::unique_ptr* model) { model->reset(); // The model configuration must specify a backend. const std::string& backend_name = model_config.backend(); if (backend_name.empty()) { return Status( Status::Code::INVALID_ARG, "must specify 'backend' for '" + model_config.name() + "'"); } // Localize the content of the model repository corresponding to // 'model_path'. This model holds a handle to the localized content // so that it persists as long as the model is loaded. std::shared_ptr localized_model_dir; RETURN_IF_ERROR(LocalizePath(model_path, &localized_model_dir)); // Localize paths in backend model config // [FIXME] Remove once a more permanent solution is implemented (DLIS-4211) RETURN_IF_ERROR(LocalizePythonBackendExecutionEnvironmentPath( model_path, &model_config, &localized_model_dir)); // Get some internal configuration values needed for initialization. std::string backend_dir; RETURN_IF_ERROR(BackendConfigurationGlobalBackendsDirectory( backend_cmdline_config_map, &backend_dir)); bool auto_complete_config = false; RETURN_IF_ERROR(BackendConfigurationAutoCompleteConfig( backend_cmdline_config_map, &auto_complete_config)); double min_compute_capability = 0; RETURN_IF_ERROR(BackendConfigurationMinComputeCapability( backend_cmdline_config_map, &min_compute_capability)); std::string specialized_backend_name; RETURN_IF_ERROR(BackendConfigurationSpecializeBackendName( backend_cmdline_config_map, backend_name, &specialized_backend_name)); bool is_python_based_backend = false; std::vector search_paths = GetBackendLibrarySearchPaths( model_path, version, backend_dir, backend_name); std::string backend_libdir, backend_libpath; RETURN_IF_ERROR(GetBackendLibraryProperties( localized_model_dir->Path(), version, backend_dir, specialized_backend_name, &model_config, &is_python_based_backend, &search_paths, &backend_libdir, &backend_libpath)); if (is_python_based_backend) { RETURN_IF_ERROR(SetPythonBasedBackendExecutionEnvironment( backend_libdir, &model_config)); } // Resolve the global backend configuration with the specific backend // configuration triton::common::BackendCmdlineConfig config; RETURN_IF_ERROR(ResolveBackendConfigs( backend_cmdline_config_map, (is_python_based_backend ? kPythonBackend : backend_name), config)); RETURN_IF_ERROR(SetBackendConfigDefaults(config)); std::shared_ptr backend; RETURN_IF_ERROR(server->BackendManager()->CreateBackend( backend_name, backend_libdir, backend_libpath, config, is_python_based_backend, &backend)); // Normalize backend-dependent config { const auto& attributes = backend->BackendAttributes(); // [WIP] formalize config normalization / validation RETURN_IF_ERROR(NormalizeInstanceGroup( min_compute_capability, attributes.preferred_groups_, &model_config)); RETURN_IF_ERROR( ValidateInstanceGroup(model_config, min_compute_capability)); } // Create and initialize the model. std::unique_ptr local_model(new TritonModel( server, localized_model_dir, backend, min_compute_capability, model_id, version, model_config, auto_complete_config, backend_cmdline_config_map, host_policy_map)); TritonModel* raw_local_model = local_model.get(); // Model initialization is optional... The TRITONBACKEND_Model object is this // TritonModel object. if (backend->ModelInitFn() != nullptr) { // We must set set shared library path to point to the backend directory in // case the backend library attempts to load additional shared libraries. // Currently, the set and reset function is effective only on Windows, so // there is no need to set path on non-Windows. // However, parallel model loading will not see any speedup on Windows and // the global lock inside the SharedLibrary is a WAR. // [FIXME] Reduce lock WAR on SharedLibrary (DLIS-4300) #ifdef _WIN32 std::unique_ptr slib; RETURN_IF_ERROR(SharedLibrary::Acquire(&slib)); RETURN_IF_ERROR(slib->SetLibraryDirectory(backend->Directory())); #endif TRITONSERVER_Error* err = backend->ModelInitFn()( reinterpret_cast(raw_local_model)); #ifdef _WIN32 RETURN_IF_ERROR(slib->ResetLibraryDirectory()); #endif RETURN_IF_TRITONSERVER_ERROR(err); } // Initialize the model for Triton core usage RETURN_IF_ERROR(local_model->Init(is_config_provided)); RETURN_IF_ERROR(local_model->GetExecutionPolicy(model_config)); // Initialize the custom batching library for the model, if provided. if (model_config.has_sequence_batching()) { if (model_config.parameters().contains("TRITON_BATCH_STRATEGY_PATH")) { return Status( Status::Code::INVALID_ARG, "TRITON_BATCH_STRATEGY_PATH cannot be specified with " "sequence batcher, using default batching strategy"); } } else { std::string batch_libpath; if (model_config.parameters().contains("TRITON_BATCH_STRATEGY_PATH")) { batch_libpath = model_config.parameters() .at("TRITON_BATCH_STRATEGY_PATH") .string_value(); bool exists = false; RETURN_IF_ERROR(FileExists(batch_libpath, &exists)); if (!exists) { return Status( triton::common::Error::Code::NOT_FOUND, ("Batching library path not found: " + batch_libpath).c_str()); } } else { const std::string batch_libname = "batchstrategy.so"; for (const auto& path : search_paths) { const auto full_path = JoinPath({path, batch_libname}); bool exists = false; RETURN_IF_ERROR(FileExists(full_path, &exists)); if (exists) { batch_libpath = full_path; break; } } } if (!batch_libpath.empty()) { LOG_INFO << "Loading custom batching strategy library " << batch_libpath << " for model " << model_config.name(); RETURN_IF_ERROR(local_model->SetBatchingStrategy(batch_libpath)); } } // Create or update the model instances for this model. std::vector> added_instances, removed_instances; RETURN_IF_ERROR(local_model->PrepareInstances( model_config, &added_instances, &removed_instances)); RETURN_IF_ERROR(local_model->SetConfiguredScheduler(added_instances)); local_model->CommitInstances(); *model = std::move(local_model); return Status::Success; } Status TritonModel::UpdateInstanceGroup(const inference::ModelConfig& new_model_config) { // Generate normalized model config with new instance group. inference::ModelConfig model_config = config_; model_config.clear_instance_group(); model_config.mutable_instance_group()->Add( new_model_config.instance_group().begin(), new_model_config.instance_group().end()); RETURN_IF_ERROR(NormalizeInstanceGroup( min_compute_capability_, backend_->BackendAttributes().preferred_groups_, &model_config)); RETURN_IF_ERROR(ValidateInstanceGroup(model_config, min_compute_capability_)); // Prepare the new instances on the new config. std::vector> added_instances, removed_instances; Status status = PrepareInstances(model_config, &added_instances, &removed_instances); if (!status.IsOk()) { ClearBackgroundInstances(); return status; } // Update the scheduler. status = UpdateConfiguredScheduler(added_instances, removed_instances); if (!status.IsOk()) { ClearBackgroundInstances(); return status; } // Commit the instance update. CommitInstances(); *config_.mutable_instance_group() = model_config.instance_group(); return Status::Success; } Status TritonModel::GetExecutionPolicy(const inference::ModelConfig& model_config) { // Set 'device_blocking_' device_blocking_ = false; if (backend_->ExecutionPolicy() == TRITONBACKEND_EXECUTION_DEVICE_BLOCKING) { if (model_config.has_sequence_batching()) { LOG_INFO << "Overriding execution policy to " "\"TRITONBACKEND_EXECUTION_BLOCKING\" for sequence model \"" << model_config.name() << "\""; } else { device_blocking_ = true; } } return Status::Success; } std::vector TritonModel::GetBackendLibrarySearchPaths( const std::string& model_path, int64_t version, const std::string& backend_dir, const std::string& backend_name) { const auto version_path = JoinPath({model_path, std::to_string(version)}); const auto backend_path = JoinPath({backend_dir, backend_name}); std::vector search_paths = { version_path, model_path, backend_path}; return search_paths; } Status TritonModel::GetBackendLibraryProperties( const std::string& model_path, int64_t version, const std::string& backend_dir, const std::string& backend_name, inference::ModelConfig* model_config, bool* is_python_based_backend, std::vector* search_paths, std::string* backend_libdir, std::string* backend_libpath) { std::string python_based_backend_libdir; std::string backend_libname = model_config->runtime(); if (backend_libname.empty()) { RETURN_IF_ERROR(GetBackendRuntimeLibraryName( backend_dir, backend_name, *search_paths, &backend_libname, backend_libdir, backend_libpath, is_python_based_backend)); if (!*is_python_based_backend) { // All variables are correctly set for C++ backends on initial search. return Status::Success; } python_based_backend_libdir = *backend_libdir; model_config->set_runtime(backend_libname); } else { *is_python_based_backend = backend_libname == kPythonFilename; } std::string cpp_backend_libname = backend_libname; if (*is_python_based_backend) { // Set C++ library name to Python backend. cpp_backend_libname = AssembleCPPRuntimeLibraryName(kPythonBackend); // The search paths only contain locations related to the Python backend // based backend, the global Python backend location has to be added. search_paths->emplace_back(JoinPath({backend_dir, kPythonBackend})); } RETURN_IF_ERROR(FindBackendLibraryPath( *search_paths, cpp_backend_libname, backend_libdir, backend_libpath)); if (backend_libpath->empty()) { std::string search_paths_str = ""; for (const auto& path : *search_paths) { search_paths_str += "'" + path + "' "; } return Status( Status::Code::INVALID_ARG, "unable to find backend library '" + cpp_backend_libname + "' for model '" + model_config->name() + "', searched: " + search_paths_str); } if (IsChildPathEscapingParentPath( *backend_libpath /* child_path */, *backend_libdir /* parent_path */)) { return Status( Status::Code::INVALID_ARG, "backend library name '" + cpp_backend_libname + "' escapes backend directory '" + *backend_libdir + "', for model '" + model_config->name() + "', check model config runtime field"); } // Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++ // backend library, 'backend_libdir' needs adjustment for Python based // backend. if (*is_python_based_backend) { if (python_based_backend_libdir.empty()) { python_based_backend_libdir = JoinPath({backend_dir, backend_name}); // Make sure the library and its directory exist. std::string path = JoinPath({python_based_backend_libdir, kPythonFilename}); bool path_exist; RETURN_IF_ERROR(FileExists(path, &path_exist)); if (!path_exist) { return Status( Status::Code::INVALID_ARG, "unable to find Python backend based backend library '" + backend_libname + "' for model '" + model_config->name() + "', searched: '" + path + "'"); } } *backend_libdir = python_based_backend_libdir; } return Status::Success; } Status TritonModel::GetBackendRuntimeLibraryName( const std::string& backend_dir, const std::string& backend_name, const std::vector& search_paths, std::string* backend_libname, std::string* backend_libdir, std::string* backend_libpath, bool* is_python_based_backend) { // Try C++ runtime *backend_libname = AssembleCPPRuntimeLibraryName(backend_name); RETURN_IF_ERROR(FindBackendLibraryPath( search_paths, *backend_libname, backend_libdir, backend_libpath)); if (!backend_libpath->empty()) { *is_python_based_backend = false; return Status::Success; } // Try Python runtime std::vector python_search_paths = { JoinPath({backend_dir, backend_name})}; *backend_libname = kPythonFilename; RETURN_IF_ERROR(FindBackendLibraryPath( python_search_paths, *backend_libname, backend_libdir, backend_libpath)); if (!backend_libpath->empty()) { *is_python_based_backend = true; return Status::Success; } // Cannot find runtime return Status( Status::Code::INVALID_ARG, "unable to find backend library for backend '" + backend_name + "', try specifying runtime on the model configuration."); } Status TritonModel::FindBackendLibraryPath( const std::vector& search_paths, const std::string& backend_libname, std::string* backend_libdir, std::string* backend_libpath) { backend_libpath->clear(); for (const auto& path : search_paths) { const auto full_path = JoinPath({path, backend_libname}); bool exists = false; RETURN_IF_ERROR(FileExists(full_path, &exists)); if (exists) { *backend_libdir = path; *backend_libpath = full_path; break; } } return Status::Success; } std::string TritonModel::AssembleCPPRuntimeLibraryName(const std::string& backend_name) { #ifdef _WIN32 return "triton_" + backend_name + ".dll"; #else return "libtriton_" + backend_name + ".so"; #endif } Status TritonModel::ResolveBackendConfigs( const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map, const std::string& backend_name, triton::common::BackendCmdlineConfig& config) { const auto& global_itr = backend_cmdline_config_map.find(std::string()); const auto& specific_itr = backend_cmdline_config_map.find(backend_name); std::map lconfig; if (global_itr != backend_cmdline_config_map.end()) { // Accumulate all global settings for (auto& setting : global_itr->second) { lconfig[setting.first] = setting.second; } } if (specific_itr != backend_cmdline_config_map.end()) { // Accumulate backend specific settings and override // global settings with specific configs if needed for (auto& setting : specific_itr->second) { lconfig[setting.first] = setting.second; } } for (auto& final_setting : lconfig) { config.emplace_back(final_setting); } return Status::Success; } const std::unordered_map backend_config_defaults( {{"default-max-batch-size", "4"}}); Status TritonModel::SetBackendConfigDefaults( triton::common::BackendCmdlineConfig& config) { auto backend_config_defaults_copy = backend_config_defaults; for (auto& setting : config) { if (setting.first.compare("default-max-batch-size") == 0) { LOG_VERBOSE(1) << "Found overwritten default setting: " << setting.first << "," << setting.second; backend_config_defaults_copy.erase(setting.first); } if (backend_config_defaults_copy.empty()) { break; } } // Anything left should be added to the config for (const auto& default_setting : backend_config_defaults_copy) { LOG_VERBOSE(1) << "Adding default backend config setting: " << default_setting.first << "," << default_setting.second; config.push_back( std::make_pair(default_setting.first, default_setting.second)); } return Status::Success; } std::unordered_map< TritonModelInstance::Signature, std::vector>> TritonModel::IndexInstances() const { std::unordered_map< TritonModelInstance::Signature, std::vector>> mapped_instances; for (auto* instances : {&instances_, &passive_instances_}) { for (auto& instance : (*instances)) { auto itr = mapped_instances .emplace( instance->GetSignature(), std::vector>()) .first; itr->second.push_back(instance); } } return mapped_instances; } Status TritonModel::PrepareInstances( const inference::ModelConfig& model_config, std::vector>* added_instances, std::vector>* removed_instances) { added_instances->clear(); removed_instances->clear(); std::unordered_map< TritonModelInstance::Signature, std::vector>> existing_instances = IndexInstances(); std::vector> creation_results; // Used to protect shared states for parallel instance loading std::mutex instance_mu; // Deferred will be lazily evaluated when the result is requested. Since the // creation_results are requested serially below, this is equivalent to making // the calls serially. auto launch_policy = std::launch::deferred; // Override for testing/debugging purposes bool parallel = backend_->BackendAttributes().parallel_instance_loading_; const char* env = std::getenv("TRITON_PARALLEL_INSTANCE_LOADING"); if (env != nullptr) { std::string s_env = std::string(env); if (!s_env.empty()) { parallel = (s_env == "1") ? true : false; LOG_VERBOSE(1) << "Using TRITON_PARALLEL_INSTANCE_LOADING environment variable " "override: " << parallel; } } // If the backend supports it, std::launch::async will allow concurrent calls if (parallel) { launch_policy = std::launch::async; } // Iterates over all the requested instances on the model config, and decides // if each requested instance can reuse an existing instance or a new instance // is needed. for (const auto& group : model_config.instance_group()) { std::vector profile_names; for (const auto& profile_name : group.profile()) { profile_names.push_back(profile_name); } std::vector secondary_devices; for (const auto& secondary_device : group.secondary_devices()) { secondary_devices.emplace_back( inference:: ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_Name( secondary_device.kind()), secondary_device.device_id()); } for (int32_t c = 0; c < group.count(); ++c) { std::string instance_name{group.name() + "_" + std::to_string(c)}; const bool passive = group.passive(); struct InstanceSetting { InstanceSetting( const std::string& policy_name, TRITONSERVER_InstanceGroupKind kind, int32_t device_id, const inference::ModelRateLimiter* rate_limiter_config) : policy_name_(policy_name), kind_(kind), device_id_(device_id), rate_limiter_config_(rate_limiter_config) { } const std::string policy_name_; const TRITONSERVER_InstanceGroupKind kind_; const int32_t device_id_; const inference::ModelRateLimiter* rate_limiter_config_; }; std::vector instance_settings; if (group.kind() == inference::ModelInstanceGroup::KIND_CPU) { instance_settings.emplace_back( group.host_policy().empty() ? "cpu" : group.host_policy(), TRITONSERVER_INSTANCEGROUPKIND_CPU, 0 /* device_id */, &group.rate_limiter()); } else if (group.kind() == inference::ModelInstanceGroup::KIND_GPU) { for (const int32_t device_id : group.gpus()) { instance_settings.emplace_back( group.host_policy().empty() ? ("gpu_" + std::to_string(device_id)) : group.host_policy(), TRITONSERVER_INSTANCEGROUPKIND_GPU, device_id, &group.rate_limiter()); } } else if (group.kind() == inference::ModelInstanceGroup::KIND_MODEL) { instance_settings.emplace_back( group.host_policy().empty() ? "model" : group.host_policy(), TRITONSERVER_INSTANCEGROUPKIND_MODEL, 0 /* device_id */, &group.rate_limiter()); } else { return Status( Status::Code::INVALID_ARG, std::string("instance_group kind ") + ModelInstanceGroup_Kind_Name(group.kind()) + " not supported"); } for (const auto& is : instance_settings) { // All the information for the requested instance is ready. Create a // signature that identifies the requested instance. const TritonModelInstance::Signature signature(group, is.device_id_); // Check if the requested instance can reuse an existing instance. if (!TritonModelInstance::ShareBackendThread( DeviceBlocking(), is.kind_)) { auto itr = existing_instances.find(signature); if (itr != existing_instances.end() && !itr->second.empty()) { auto existing_instance = itr->second.back(); itr->second.pop_back(); LOG_VERBOSE(2) << "Re-using model instance named '" << existing_instance->Name() << "' with device id '" << existing_instance->DeviceId() << "'"; RegisterBackgroundInstance(std::move(existing_instance), passive); continue; } } // Note that the local variables should be captured by value creation_results.emplace_back( std::async(launch_policy, [=, &instance_mu]() { // The requested instance did not match an existing instance. // Create a new instance. std::shared_ptr new_instance; RETURN_IF_ERROR(TritonModelInstance::CreateInstance( this, instance_name, signature, is.kind_, is.device_id_, profile_names, passive, is.policy_name_, *is.rate_limiter_config_, secondary_devices, &new_instance)); { std::lock_guard lk(instance_mu); added_instances->push_back(new_instance); RegisterBackgroundInstance(std::move(new_instance), passive); } // Keep logging to a single stream operator to avoid interweaving const auto msg = "Created model instance named '" + instance_name + "' with device id '" + std::to_string(is.device_id_) + "'"; LOG_VERBOSE(2) << msg; return Status::Success; })); } } } // Any existing instances not reused will be removed. for (auto pair : existing_instances) { removed_instances->insert( removed_instances->end(), pair.second.begin(), pair.second.end()); } auto status = Status::Success; for (auto& cr : creation_results) { auto lstatus = cr.get(); if (!lstatus.IsOk()) { LOG_ERROR << "ERROR: Failed to create instance: " << lstatus.Message(); status = lstatus; } } return status; } void TritonModel::CommitInstances() { instances_.swap(bg_instances_); passive_instances_.swap(bg_passive_instances_); ClearBackgroundInstances(); } void TritonModel::RegisterBackgroundInstance( std::shared_ptr&& instance, const bool passive) { if (passive) { bg_passive_instances_.emplace_back(std::move(instance)); } else { bg_instances_.emplace_back(std::move(instance)); } } void TritonModel::ClearBackgroundInstances() { bg_instances_.clear(); bg_passive_instances_.clear(); } std::vector> TritonModel::GetInstancesByDevice(int32_t device_id) const { std::vector> result; // Do not match passive instances, as they do not have a backend thread. // Do not match foreground instances, as backend threads cannot be updated. for (auto& instance : bg_instances_) { if (instance->DeviceId() == device_id) { result.push_back(instance); } } return result; } Status TritonModel::UpdateModelConfig( const uint32_t config_version, TRITONSERVER_Message* updated_config_message) { const char* buffer; size_t byte_size; RETURN_IF_TRITONSERVER_ERROR(TRITONSERVER_MessageSerializeToJson( updated_config_message, &buffer, &byte_size)); inference::ModelConfig updated_config; RETURN_IF_ERROR( JsonToModelConfig({buffer, byte_size}, config_version, &updated_config)); auto config = Config(); config.set_max_batch_size(updated_config.max_batch_size()); auto inputs_config = config.mutable_input(); *inputs_config = updated_config.input(); auto outputs_config = config.mutable_output(); *outputs_config = updated_config.output(); if (!config.scheduling_choice_case()) { if (updated_config.has_dynamic_batching()) { auto dynamic_batching_config = config.mutable_dynamic_batching(); *dynamic_batching_config = updated_config.dynamic_batching(); } else if (updated_config.has_sequence_batching()) { auto sequence_batching_config = config.mutable_sequence_batching(); *sequence_batching_config = updated_config.sequence_batching(); } else if (updated_config.has_ensemble_scheduling()) { auto ensemble_scheduling_config = config.mutable_ensemble_scheduling(); *ensemble_scheduling_config = updated_config.ensemble_scheduling(); } // else do nothing } else if ( config.scheduling_choice_case() != updated_config.scheduling_choice_case()) { return Status( triton::common::Error::Code::INTERNAL, (std::string("Cannot update scheduling choice from ") + std::to_string(config.scheduling_choice_case()) + std::string(" to ") + std::to_string(config.scheduling_choice_case()) + std::string(" when auto-completing.")) .c_str()); } // else do nothing // Update model_transaction_policy if needed if (updated_config.has_model_transaction_policy()) { bool is_decoupled = updated_config.model_transaction_policy().decoupled(); config.mutable_model_transaction_policy()->set_decoupled(is_decoupled); } // Need to normalize the model configuration for // populating missing fields. RETURN_IF_ERROR(NormalizeModelConfig(min_compute_capability_, &config)); RETURN_IF_ERROR(SetModelConfig(config)); return Status::Success; } Status TritonModel::SetConfiguredScheduler( const std::vector>& new_instances) { std::unique_ptr scheduler; // Need to enforce equal shape batches (i.e. non-ragged batches) if // the model 1) allows one or more variable-size input tensors that // are not marked as 'allow_ragged_batch' or 2) has one or more // shape-tensor inputs. This is not needed if all input shapes are // non-variable and if there are no shape tensors... so we don't // enable it in that case for efficiency reasons. std::unordered_map enforce_equal_shape_tensors; for (const auto& input : config_.input()) { if (input.is_shape_tensor()) { enforce_equal_shape_tensors.insert({input.name(), true}); } else if ( !input.allow_ragged_batch() && (triton::common::GetElementCount(input) == -1)) { enforce_equal_shape_tensors.insert({input.name(), false}); } } // If 'sequence_batching' is configured, then use the SequenceBatchScheduler, // otherwise use the default DynamicBatchScheduler. if (config_.has_sequence_batching()) { // Sequence batcher if (config_.parameters().contains("TRITON_BATCH_STRATEGY_PATH")) { LOG_ERROR << "TRITON_BATCH_STRATEGY_PATH cannot be specified with " "sequence batcher, using default batching strategy"; } RETURN_IF_ERROR(SequenceBatchScheduler::Create( this, new_instances, enforce_equal_shape_tensors, &scheduler)); } else if (config_.has_dynamic_batching()) { // Dynamic batcher RETURN_IF_ERROR(DynamicBatchScheduler::Create( this, nullptr, 0 /*nice*/, true /* dynamic_batching_enabled */, config_.max_batch_size(), enforce_equal_shape_tensors, config_.dynamic_batching(), &scheduler)); } else { // Default scheduler. Use dynamic batch scheduler (with batching // disabled) as the default scheduler. RETURN_IF_ERROR(DynamicBatchScheduler::Create( this, nullptr, 0 /*nice*/, false /* dynamic_batching_enabled */, 1 /* max_batch_size */, std::unordered_map< std::string, bool>() /* enforce_equal_shape_tensors */, false /* preserve_ordering */, std::set() /* preferred_batch_sizes */, 0 /* max_queue_delay_microseconds */, &scheduler)); } return SetScheduler(std::move(scheduler)); } Status TritonModel::UpdateConfiguredScheduler( const std::vector>& added_instances, const std::vector>& removed_instances) { if (config_.has_sequence_batching()) { SequenceBatchScheduler* sched = dynamic_cast(scheduler_.get()); if (sched == nullptr) { return Status( Status::Code::INTERNAL, "Unable to downcast from 'Scheduler' to 'SequenceBatchScheduler' " "during scheduler update"); } return sched->Update(added_instances, removed_instances); } // Non-sequence scheduler does not need to be updated, because other // schedulers do not require the information on model instances to function, // and only interact with the rate limiter. return Status::Success; } Status TritonModel::SetBatchingStrategy(const std::string& batch_libpath) { // Load library and functions. std::unique_ptr slib; RETURN_IF_ERROR(SharedLibrary::Acquire(&slib)); RETURN_IF_ERROR(slib->OpenLibraryHandle(batch_libpath, &batch_dlhandle_)); RETURN_IF_ERROR(slib->GetEntrypoint( batch_dlhandle_, "TRITONBACKEND_ModelBatchIncludeRequest", true /* optional */, reinterpret_cast(&batch_incl_fn_))); RETURN_IF_ERROR(slib->GetEntrypoint( batch_dlhandle_, "TRITONBACKEND_ModelBatchInitialize", true /* optional */, reinterpret_cast(&batch_init_fn_))); RETURN_IF_ERROR(slib->GetEntrypoint( batch_dlhandle_, "TRITONBACKEND_ModelBatchFinalize", true /* optional */, reinterpret_cast(&batch_fini_fn_))); RETURN_IF_ERROR(slib->GetEntrypoint( batch_dlhandle_, "TRITONBACKEND_ModelBatcherFinalize", true /* optional */, reinterpret_cast(&batcher_fini_fn_))); RETURN_IF_ERROR(slib->GetEntrypoint( batch_dlhandle_, "TRITONBACKEND_ModelBatcherInitialize", true /* optional */, reinterpret_cast(&batcher_init_fn_))); // If one custom batching function is defined, all must be. const bool defined_some = batch_incl_fn_ || batch_init_fn_ || batch_fini_fn_ || batcher_init_fn_ || batcher_fini_fn_; const bool defined_all = batch_incl_fn_ && batch_init_fn_ && batch_fini_fn_ && batcher_init_fn_ && batcher_fini_fn_; if (defined_some && !defined_all) { ClearHandles(); return Status( Status::Code::INVALID_ARG, batch_libpath + " does not define all " "required custom batching functions for model " + config_.name()); } // If a custom batcher is provided, initialize it. if (batcher_init_fn_) { TRITONSERVER_Error* err = batcher_init_fn_( Batcher(), reinterpret_cast(this)); if (err) { auto status = Status( TritonCodeToStatusCode(TRITONSERVER_ErrorCode(err)), TRITONSERVER_ErrorMessage(err)); TRITONSERVER_ErrorDelete(err); return status; } } return Status::Success; } TritonModel::TritonModel( InferenceServer* server, const std::shared_ptr& localized_model_dir, const std::shared_ptr& backend, const double min_compute_capability, const ModelIdentifier& model_id, const int64_t version, const inference::ModelConfig& config, const bool auto_complete_config, const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map, const triton::common::HostPolicyCmdlineConfigMap& host_policy_map) : Model( min_compute_capability, localized_model_dir->Path(), model_id, version, config), server_(server), min_compute_capability_(min_compute_capability), auto_complete_config_(auto_complete_config), backend_cmdline_config_map_(backend_cmdline_config_map), host_policy_map_(host_policy_map), device_blocking_(false), localized_model_dir_(localized_model_dir), backend_(backend), state_(nullptr) { } TritonModel::~TritonModel() { // If there is a custom batcher, finalize it. if (batcher_fini_fn_) { TRITONSERVER_Error* err = batcher_fini_fn_(*Batcher()); *Batcher() = nullptr; if (err) { LOG_ERROR << "Custom batcher finalization failed for model " << config_.name() << ": " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } } // Clear library handles. ClearHandles(); // Explicitly delete/finalize the scheduler before the model instances. scheduler_.reset(nullptr); // Explicitly delete/finalize all model instances before finalizing // the model itself. instances_.clear(); passive_instances_.clear(); ClearBackgroundInstances(); // Unregister itself from the rate limiter. Note this should happen // after all instances are destructed. Destrucing instances ensures // there are no instance threads waiting on rate limiter for // receiving their payloads. server_->GetRateLimiter()->UnregisterModel(this); // Model finalization is optional... The TRITONBACKEND_Model // object is this TritonModel object. if (backend_->ModelFiniFn() != nullptr) { LOG_TRITONSERVER_ERROR( backend_->ModelFiniFn()(reinterpret_cast(this)), "failed finalizing model"); } } void TritonModel::ClearHandles() { if (batch_dlhandle_ == nullptr) { return; } { std::unique_ptr slib; LOG_STATUS_ERROR( SharedLibrary::Acquire(&slib), "~TritonModel::ClearHandles"); LOG_STATUS_ERROR( slib->CloseLibraryHandle(batch_dlhandle_), "TritonModel::ClearHandles"); } batch_dlhandle_ = nullptr; batch_incl_fn_ = nullptr; batch_init_fn_ = nullptr; batch_fini_fn_ = nullptr; batcher_init_fn_ = nullptr; batcher_fini_fn_ = nullptr; } extern "C" { // // TRITONBACKEND_Model // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelName(TRITONBACKEND_Model* model, const char** name) { TritonModel* tm = reinterpret_cast(model); *name = tm->Name().c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelVersion(TRITONBACKEND_Model* model, uint64_t* version) { TritonModel* tm = reinterpret_cast(model); *version = tm->Version(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelRepository( TRITONBACKEND_Model* model, TRITONBACKEND_ArtifactType* artifact_type, const char** location) { TritonModel* tm = reinterpret_cast(model); *artifact_type = TRITONBACKEND_ARTIFACT_FILESYSTEM; *location = tm->LocalizedModelPath().c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelConfig( TRITONBACKEND_Model* model, const uint32_t config_version, TRITONSERVER_Message** model_config) { TritonModel* tm = reinterpret_cast(model); std::string model_config_json; RETURN_TRITONSERVER_ERROR_IF_ERROR( ModelConfigToJson(tm->Config(), config_version, &model_config_json)); *model_config = reinterpret_cast( new TritonServerMessage(std::move(model_config_json))); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelAutoCompleteConfig( TRITONBACKEND_Model* model, bool* auto_complete_config) { TritonModel* tm = reinterpret_cast(model); *auto_complete_config = tm->AutoCompleteConfig(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelSetConfig( TRITONBACKEND_Model* model, const uint32_t config_version, TRITONSERVER_Message* model_config) { TritonModel* tm = reinterpret_cast(model); RETURN_TRITONSERVER_ERROR_IF_ERROR( tm->UpdateModelConfig(config_version, model_config)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelServer( TRITONBACKEND_Model* model, TRITONSERVER_Server** server) { TritonModel* tm = reinterpret_cast(model); *server = reinterpret_cast(tm->Server()); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelBackend( TRITONBACKEND_Model* model, TRITONBACKEND_Backend** backend) { TritonModel* tm = reinterpret_cast(model); *backend = reinterpret_cast(tm->Backend().get()); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelState(TRITONBACKEND_Model* model, void** state) { TritonModel* tm = reinterpret_cast(model); *state = tm->State(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelSetState(TRITONBACKEND_Model* model, void* state) { TritonModel* tm = reinterpret_cast(model); tm->SetState(state); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelReportMemoryUsage( TRITONBACKEND_Model* model, TRITONSERVER_BufferAttributes** usage, uint32_t usage_size) { TritonModel* tm = reinterpret_cast(model); tm->SetMemoryUsage({reinterpret_cast(usage), usage_size}); return nullptr; // success } /// /// TRITONBACKEND_Request /// TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestId(TRITONBACKEND_Request* request, const char** id) { InferenceRequest* tr = reinterpret_cast(request); *id = tr->Id().c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestCorrelationId(TRITONBACKEND_Request* request, uint64_t* id) { InferenceRequest* tr = reinterpret_cast(request); const InferenceRequest::SequenceId& correlation_id = tr->CorrelationId(); if (correlation_id.Type() != InferenceRequest::SequenceId::DataType::UINT64) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (tr->LogRequest() + "correlation ID in request is not an unsigned int") .c_str()); } *id = correlation_id.UnsignedIntValue(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InferenceRequestTimeoutMicroseconds( TRITONBACKEND_Request* request, uint64_t* timeout) { InferenceRequest* tr = reinterpret_cast(request); *timeout = tr->TimeoutMicroseconds(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestFlags(TRITONBACKEND_Request* request, uint32_t* flags) { InferenceRequest* tr = reinterpret_cast(request); *flags = tr->Flags(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestIsCancelled( TRITONBACKEND_Request* request, bool* is_cancelled) { InferenceRequest* tr = reinterpret_cast(request); RETURN_TRITONSERVER_ERROR_IF_ERROR(tr->IsCancelled(is_cancelled)); return nullptr; } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestCorrelationIdString( TRITONBACKEND_Request* request, const char** id) { InferenceRequest* tr = reinterpret_cast(request); const InferenceRequest::SequenceId& correlation_id = tr->CorrelationId(); if (correlation_id.Type() != InferenceRequest::SequenceId::DataType::STRING) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (tr->LogRequest() + "correlation ID in request is not a string") .c_str()); } *id = correlation_id.StringValue().c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputCount(TRITONBACKEND_Request* request, uint32_t* count) { InferenceRequest* tr = reinterpret_cast(request); *count = tr->ImmutableInputs().size(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputName( TRITONBACKEND_Request* request, const uint32_t index, const char** input_name) { *input_name = nullptr; InferenceRequest* tr = reinterpret_cast(request); const auto& inputs = tr->ImmutableInputs(); if (index >= inputs.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (tr->LogRequest() + "out of bounds index " + std::to_string(index) + ": request has " + std::to_string(inputs.size()) + " inputs") .c_str()); } // The request inputs are not allowed to change once the request // makes it to the backend, so it is ok to just iterate through the // map. This linear search is the best we can do given the need for // the inputs to be in a map and given the typical small number of // inputs is better than having every request maintain the inputs as // both map and vector. uint32_t cnt = 0; for (const auto& pr : inputs) { if (cnt++ == index) { InferenceRequest::Input* in = pr.second; *input_name = in->Name().c_str(); break; } } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInput( TRITONBACKEND_Request* request, const char* name, TRITONBACKEND_Input** input) { InferenceRequest* tr = reinterpret_cast(request); const auto& inputs = tr->ImmutableInputs(); const auto& itr = inputs.find(name); if (itr == inputs.end()) { *input = nullptr; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (tr->LogRequest() + "unknown request input name " + name).c_str()); } InferenceRequest::Input* in = itr->second; *input = reinterpret_cast(in); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputByIndex( TRITONBACKEND_Request* request, const uint32_t index, TRITONBACKEND_Input** input) { InferenceRequest* tr = reinterpret_cast(request); const auto& inputs = tr->ImmutableInputs(); if (index >= inputs.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (tr->LogRequest() + "out of bounds index " + std::to_string(index) + ": request has " + std::to_string(inputs.size()) + " inputs") .c_str()); } // The request inputs are not allowed to change once the request // makes it to the backend, so it is ok to just iterate through the // map. This linear search is the best we can do given the need for // the inputs to be in a map and given the typical small number of // inputs is better than having every request maintain the inputs as // both map and vector. uint32_t cnt = 0; for (const auto& pr : inputs) { if (cnt++ == index) { InferenceRequest::Input* in = pr.second; *input = reinterpret_cast(in); break; } } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestOutputCount( TRITONBACKEND_Request* request, uint32_t* count) { InferenceRequest* tr = reinterpret_cast(request); *count = tr->ImmutableRequestedOutputs().size(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestOutputName( TRITONBACKEND_Request* request, const uint32_t index, const char** output_name) { *output_name = nullptr; InferenceRequest* tr = reinterpret_cast(request); const auto& routputs = tr->ImmutableRequestedOutputs(); if (index >= routputs.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (tr->LogRequest() + "out of bounds index " + std::to_string(index) + ": request has " + std::to_string(routputs.size()) + " requested outputs") .c_str()); } // The requested outputs are not allowed to change once the request // makes it to the backend, so it is ok to just iterate through the // set. This linear search is the best we can do given the requested // outputs being in a set and given the typical small number of // requested outputs it should not be a performance issue. uint32_t cnt = 0; for (const auto& rout : routputs) { if (cnt++ == index) { *output_name = rout.c_str(); break; } } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestOutputBufferProperties( TRITONBACKEND_Request* request, const char* name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { InferenceRequest* tr = reinterpret_cast(request); RETURN_TRITONSERVER_ERROR_IF_ERROR( tr->OutputBufferProperties(name, byte_size, memory_type, memory_type_id)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestRelease( TRITONBACKEND_Request* request, uint32_t release_flags) { InferenceRequest* tr = reinterpret_cast(request); std::unique_ptr ur(tr); auto status = InferenceRequest::Release(std::move(ur), release_flags); if (!status.IsOk()) { // On error, ownership of request is not taken and should not be // managed by unique pointer. ur.release(); RETURN_TRITONSERVER_ERROR_IF_ERROR(status); } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestTrace( TRITONBACKEND_Request* request, TRITONSERVER_InferenceTrace** trace) { #ifdef TRITON_ENABLE_TRACING InferenceRequest* tr = reinterpret_cast(request); if (tr->TraceProxy() != nullptr) { *trace = reinterpret_cast( tr->TraceProxy()->Trace()); } else { *trace = nullptr; } return nullptr; // success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "tracing is not supported"); #endif // TRITON_ENABLE_TRACING } /// /// TRITONBACKEND_State /// TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateUpdate(TRITONBACKEND_State* state) { SequenceState* ts = reinterpret_cast(state); RETURN_TRITONSERVER_ERROR_IF_ERROR(ts->Update()); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateNew( TRITONBACKEND_State** state, TRITONBACKEND_Request* request, const char* name, const TRITONSERVER_DataType datatype, const int64_t* shape, const uint32_t dims_count) { InferenceRequest* tr = reinterpret_cast(request); SequenceState* lstate; std::vector lshape(shape, shape + dims_count); auto& sequence_state = tr->GetSequenceStates(); if (sequence_state == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to add state '") + name + "'. State configuration is missing for model '" + tr->ModelName() + "'.") .c_str()); } RETURN_TRITONSERVER_ERROR_IF_ERROR(sequence_state->OutputState( name, TritonToDataType(datatype), lshape, &lstate)); *state = reinterpret_cast(lstate); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateBuffer( TRITONBACKEND_State* state, void** buffer, const uint64_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { SequenceState* to = reinterpret_cast(state); Status status = Status::Success; TRITONSERVER_MemoryType current_memory_type; int64_t current_memory_type_id; void* lbuffer = nullptr; MutableMemory* mutable_memory = reinterpret_cast(to->Data().get()); lbuffer = mutable_memory->MutableBuffer( ¤t_memory_type, ¤t_memory_type_id); // If the buffer size exactly matches the buffer available and is requesting // the same memory type and memory type id, reuse the currently allocated // buffer. if (to->Data()->TotalByteSize() == buffer_byte_size && current_memory_type == *memory_type && current_memory_type_id == *memory_type_id) { *buffer = lbuffer; } else { RETURN_TRITONSERVER_ERROR_IF_ERROR(to->ResizeOrReallocate( buffer, buffer_byte_size, memory_type, memory_type_id)); } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateBufferAttributes( TRITONBACKEND_State* state, TRITONSERVER_BufferAttributes** buffer_attributes) { SequenceState* to = reinterpret_cast(state); to->Data()->BufferAt( 0, reinterpret_cast(buffer_attributes)); return nullptr; // success } // // TRITONBACKEND_ResponseFactory // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactoryNew( TRITONBACKEND_ResponseFactory** factory, TRITONBACKEND_Request* request) { InferenceRequest* tr = reinterpret_cast(request); std::shared_ptr* response_factory = new std::shared_ptr(tr->ResponseFactory()); *factory = reinterpret_cast(response_factory); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactoryDelete(TRITONBACKEND_ResponseFactory* factory) { std::shared_ptr* response_factory = reinterpret_cast*>(factory); delete response_factory; return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactorySendFlags( TRITONBACKEND_ResponseFactory* factory, const uint32_t send_flags) { std::shared_ptr* response_factory = reinterpret_cast*>(factory); RETURN_TRITONSERVER_ERROR_IF_ERROR( (*response_factory)->SendFlags(send_flags)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactoryIsCancelled( TRITONBACKEND_ResponseFactory* factory, bool* is_cancelled) { std::shared_ptr* response_factory = reinterpret_cast*>(factory); *is_cancelled = (*response_factory)->IsCancelled(); return nullptr; // success } /// /// TRITONBACKEND_Response /// TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseNew( TRITONBACKEND_Response** response, TRITONBACKEND_Request* request) { *response = nullptr; InferenceRequest* tr = reinterpret_cast(request); std::unique_ptr tresp; RETURN_TRITONSERVER_ERROR_IF_ERROR( tr->ResponseFactory()->CreateResponse(&tresp)); *response = reinterpret_cast(tresp.release()); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseNewFromFactory( TRITONBACKEND_Response** response, TRITONBACKEND_ResponseFactory* factory) { *response = nullptr; std::shared_ptr* response_factory = reinterpret_cast*>(factory); std::unique_ptr tr; RETURN_TRITONSERVER_ERROR_IF_ERROR((*response_factory)->CreateResponse(&tr)); *response = reinterpret_cast(tr.release()); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseDelete(TRITONBACKEND_Response* response) { InferenceResponse* tr = reinterpret_cast(response); delete tr; return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSetStringParameter( TRITONBACKEND_Response* response, const char* name, const char* value) { if (!response) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "response was nullptr"); } InferenceResponse* tr = reinterpret_cast(response); RETURN_TRITONSERVER_ERROR_IF_ERROR(tr->AddParameter(name, value)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSetIntParameter( TRITONBACKEND_Response* response, const char* name, const int64_t value) { if (!response) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "response was nullptr"); } InferenceResponse* tr = reinterpret_cast(response); RETURN_TRITONSERVER_ERROR_IF_ERROR(tr->AddParameter(name, value)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSetBoolParameter( TRITONBACKEND_Response* response, const char* name, const bool value) { if (!response) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "response was nullptr"); } InferenceResponse* tr = reinterpret_cast(response); RETURN_TRITONSERVER_ERROR_IF_ERROR(tr->AddParameter(name, value)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSetDoubleParameter( TRITONBACKEND_Response* response, const char* name, const double value) { if (!response) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "response was nullptr"); } InferenceResponse* tr = reinterpret_cast(response); RETURN_TRITONSERVER_ERROR_IF_ERROR(tr->AddParameter(name, value)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseOutput( TRITONBACKEND_Response* response, TRITONBACKEND_Output** output, const char* name, const TRITONSERVER_DataType datatype, const int64_t* shape, const uint32_t dims_count) { if (!response) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "response was nullptr"); } *output = nullptr; InferenceResponse* tr = reinterpret_cast(response); std::vector lshape(shape, shape + dims_count); InferenceResponse::Output* loutput; RETURN_TRITONSERVER_ERROR_IF_ERROR(tr->AddOutput( name, TritonToDataType(datatype), std::move(lshape), &loutput)); *output = reinterpret_cast(loutput); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSend( TRITONBACKEND_Response* response, const uint32_t send_flags, TRITONSERVER_Error* error) { if (!response) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "response was nullptr"); } InferenceResponse* tr = reinterpret_cast(response); std::unique_ptr utr(tr); if (error == nullptr) { RETURN_TRITONSERVER_ERROR_IF_ERROR( InferenceResponse::Send(std::move(utr), send_flags)); } else { RETURN_TRITONSERVER_ERROR_IF_ERROR(InferenceResponse::SendWithStatus( std::move(utr), send_flags, Status( TritonCodeToStatusCode(TRITONSERVER_ErrorCode(error)), TRITONSERVER_ErrorMessage(error)))); } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestParameterCount( TRITONBACKEND_Request* request, uint32_t* count) { InferenceRequest* lrequest = reinterpret_cast(request); const auto& parameters = lrequest->Parameters(); *count = parameters.size(); return nullptr; // Success } TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestParameter( TRITONBACKEND_Request* request, const uint32_t index, const char** key, TRITONSERVER_ParameterType* type, const void** vvalue) { InferenceRequest* lrequest = reinterpret_cast(request); const auto& parameters = lrequest->Parameters(); if (index >= parameters.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, ("out of bounds index " + std::to_string(index) + std::string(": request has ") + std::to_string(parameters.size()) + " parameters") .c_str()); } const InferenceParameter& param = parameters[index]; *key = param.Name().c_str(); *type = param.Type(); *vvalue = param.ValuePointer(); return nullptr; // Success } /// /// TRITONBACKEND_Input /// TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputProperties( TRITONBACKEND_Input* input, const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint32_t* dims_count, uint64_t* byte_size, uint32_t* buffer_count) { InferenceRequest::Input* ti = reinterpret_cast(input); if (name != nullptr) { *name = ti->Name().c_str(); } if (datatype != nullptr) { *datatype = DataTypeToTriton(ti->DType()); } if (shape != nullptr) { *shape = ti->ShapeWithBatchDim().data(); } if (dims_count != nullptr) { *dims_count = ti->ShapeWithBatchDim().size(); } if (byte_size != nullptr) { *byte_size = ti->Data()->TotalByteSize(); } if (buffer_count != nullptr) { *buffer_count = ti->DataBufferCount(); } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputPropertiesForHostPolicy( TRITONBACKEND_Input* input, const char* host_policy_name, const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint32_t* dims_count, uint64_t* byte_size, uint32_t* buffer_count) { InferenceRequest::Input* ti = reinterpret_cast(input); if (name != nullptr) { *name = ti->Name().c_str(); } if (datatype != nullptr) { *datatype = DataTypeToTriton(ti->DType()); } if (shape != nullptr) { *shape = ti->ShapeWithBatchDim().data(); } if (dims_count != nullptr) { *dims_count = ti->ShapeWithBatchDim().size(); } if (host_policy_name != nullptr) { if (byte_size != nullptr) { *byte_size = ti->Data(host_policy_name)->TotalByteSize(); } if (buffer_count != nullptr) { *buffer_count = ti->DataBufferCountForHostPolicy(host_policy_name); } } else { if (byte_size != nullptr) { *byte_size = ti->Data()->TotalByteSize(); } if (buffer_count != nullptr) { *buffer_count = ti->DataBufferCount(); } } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputBuffer( TRITONBACKEND_Input* input, const uint32_t index, const void** buffer, uint64_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { InferenceRequest::Input* ti = reinterpret_cast(input); Status status = ti->DataBuffer( index, buffer, reinterpret_cast(buffer_byte_size), memory_type, memory_type_id); if (!status.IsOk()) { *buffer = nullptr; *buffer_byte_size = 0; RETURN_TRITONSERVER_ERROR_IF_ERROR(status); } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputBufferAttributes( TRITONBACKEND_Input* input, const uint32_t index, const void** buffer, TRITONSERVER_BufferAttributes** buffer_attributes) { InferenceRequest::Input* ti = reinterpret_cast(input); Status status = ti->DataBufferAttributes( index, buffer, reinterpret_cast(buffer_attributes)); if (!status.IsOk()) { *buffer = nullptr; *buffer_attributes = nullptr; RETURN_TRITONSERVER_ERROR_IF_ERROR(status); } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputBufferForHostPolicy( TRITONBACKEND_Input* input, const char* host_policy_name, const uint32_t index, const void** buffer, uint64_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { InferenceRequest::Input* ti = reinterpret_cast(input); Status status = (host_policy_name == nullptr) ? ti->DataBuffer( index, buffer, reinterpret_cast(buffer_byte_size), memory_type, memory_type_id) : ti->DataBufferForHostPolicy( index, buffer, reinterpret_cast(buffer_byte_size), memory_type, memory_type_id, host_policy_name); if (!status.IsOk()) { *buffer = nullptr; *buffer_byte_size = 0; RETURN_TRITONSERVER_ERROR_IF_ERROR(status); } return nullptr; // success } /// /// TRITONBACKEND_Output /// TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_OutputBuffer( TRITONBACKEND_Output* output, void** buffer, const uint64_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { InferenceResponse::Output* to = reinterpret_cast(output); Status status = to->AllocateDataBuffer( buffer, buffer_byte_size, memory_type, memory_type_id); if (!status.IsOk()) { *buffer = nullptr; RETURN_TRITONSERVER_ERROR_IF_ERROR(status); } return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_OutputBufferAttributes( TRITONBACKEND_Output* output, TRITONSERVER_BufferAttributes** buffer_attributes) { InferenceResponse::Output* to = reinterpret_cast(output); *buffer_attributes = reinterpret_cast( to->GetBufferAttributes()); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( TRITONBACKEND_BackendAttribute* backend_attributes, const TRITONSERVER_InstanceGroupKind kind, const uint64_t count, const uint64_t* device_ids, const uint64_t id_count) { auto ba = reinterpret_cast(backend_attributes); ba->preferred_groups_.emplace_back(); auto& pg = ba->preferred_groups_.back(); switch (kind) { case TRITONSERVER_INSTANCEGROUPKIND_AUTO: pg.set_kind(inference::ModelInstanceGroup::KIND_AUTO); break; case TRITONSERVER_INSTANCEGROUPKIND_CPU: pg.set_kind(inference::ModelInstanceGroup::KIND_CPU); break; case TRITONSERVER_INSTANCEGROUPKIND_GPU: pg.set_kind(inference::ModelInstanceGroup::KIND_GPU); break; case TRITONSERVER_INSTANCEGROUPKIND_MODEL: pg.set_kind(inference::ModelInstanceGroup::KIND_MODEL); break; } pg.set_count(count); if (device_ids != nullptr) { for (size_t i = 0; i < id_count; ++i) { pg.add_gpus(device_ids[i]); } } return nullptr; } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendAttributeSetParallelModelInstanceLoading( TRITONBACKEND_BackendAttribute* backend_attributes, bool enabled) { auto ba = reinterpret_cast(backend_attributes); ba->parallel_instance_loading_ = enabled; return nullptr; } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InferenceResponseOutputByName( TRITONBACKEND_Response* response, const char* name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint64_t* dim_count) { InferenceResponse* tr = reinterpret_cast(response); const auto& outputs = tr->Outputs(); uint32_t output_count = outputs.size(); std::string output_name = std::string(name); for (uint32_t idx = 0; idx < output_count; ++idx) { if (outputs[idx].Name() == output_name) { *datatype = DataTypeToTriton(outputs[idx].DType()); const std::vector& oshape = outputs[idx].Shape(); *shape = &oshape[0]; *dim_count = oshape.size(); return nullptr; // success } } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, ("Output name " + output_name + "not found.").c_str()); } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InferenceResponseOutput( TRITONBACKEND_Response* response, const uint32_t index, const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint64_t* dim_count) { InferenceResponse* tr = reinterpret_cast(response); const auto& outputs = tr->Outputs(); if (index >= outputs.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, ("out of bounds index " + std::to_string(index) + std::string(": response has ") + std::to_string(outputs.size()) + " outputs") .c_str()); } const InferenceResponse::Output& output = outputs[index]; *name = output.Name().c_str(); *datatype = DataTypeToTriton(output.DType()); const std::vector& oshape = output.Shape(); *shape = &oshape[0]; *dim_count = oshape.size(); return nullptr; // success } } // extern C }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_model.h000066400000000000000000000305251527052371200203010ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "backend_manager.h" #include "backend_model_instance.h" #include "filesystem/api.h" #include "infer_request.h" #include "model.h" #include "model_config.pb.h" #include "status.h" namespace triton { namespace core { class InferenceServer; // // Represents a model. // // Inheriting from Model to implement backend APIs // class TritonModel : public Model { public: typedef TRITONSERVER_Error* (*TritonModelBatchInclFn_t)( TRITONBACKEND_Request* request, void* userp, bool* should_include); typedef TRITONSERVER_Error* (*TritonModelBatchInitFn_t)( TRITONBACKEND_Batcher* batcher, void** userp); typedef TRITONSERVER_Error* (*TritonModelBatchFiniFn_t)(void* userp); typedef TRITONSERVER_Error* (*TritonModelBatcherInitFn_t)( TRITONBACKEND_Batcher** batcher, TRITONBACKEND_Model* model); typedef TRITONSERVER_Error* (*TritonModelBatcherFiniFn_t)( TRITONBACKEND_Batcher* batcher); static Status Create( InferenceServer* server, const std::string& model_path, const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map, const triton::common::HostPolicyCmdlineConfigMap& host_policy_map, const ModelIdentifier& model_id, const int64_t version, inference::ModelConfig model_config, const bool is_config_provided, std::unique_ptr* model); ~TritonModel(); // Return path to the localized model directory. const std::string& LocalizedModelPath() const { return localized_model_dir_->Path(); } // Return pointer to the underlying server. InferenceServer* Server() { return server_; } // Return whether the backend should attempt to auto-complete the model config bool AutoCompleteConfig() const { return auto_complete_config_; } // Called by TRITONBACKEND_ModelSetConfig() C-API. Status UpdateModelConfig( const uint32_t config_version, TRITONSERVER_Message* updated_config_message); // Return the underlying backend. const std::shared_ptr& Backend() const { return backend_; } // Return the backend command line config map. const triton::common::BackendCmdlineConfigMap& BackendConfigMap() const { return backend_cmdline_config_map_; } // Return the host policy command line config map. const triton::common::HostPolicyCmdlineConfigMap& HostPolicyMap() const { return host_policy_map_; } // True if different instances should be grouped by device; false otherwise. bool DeviceBlocking() const { return device_blocking_; } // Get a vector of non-passive background instances that share the device id. std::vector> GetInstancesByDevice( int32_t device_id) const; // Manipulate the opaque state associated with this model. void* State() { return state_; } void SetState(void* state) { state_ = state; } // Update instance group. Status UpdateInstanceGroup(const inference::ModelConfig& new_model_config); // Custom batching function getters. TritonModelBatchInclFn_t ModelBatchInclFn() const { return batch_incl_fn_; } TritonModelBatchInitFn_t ModelBatchInitFn() const { return batch_init_fn_; } TritonModelBatchFiniFn_t ModelBatchFiniFn() const { return batch_fini_fn_; } TRITONBACKEND_Batcher** Batcher() { return &batcher_; } private: DISALLOW_COPY_AND_ASSIGN(TritonModel); TritonModel( InferenceServer* server, const std::shared_ptr& localized_model_dir, const std::shared_ptr& backend, const double min_compute_capability, const ModelIdentifier& model_id, const int64_t version, const inference::ModelConfig& config, const bool auto_complete_config, const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map, const triton::common::HostPolicyCmdlineConfigMap& host_policy_map); // Prepare the next set of instances on the background. Returns the instances // that will be added and removed if the next set of instances is to be // committed. Status PrepareInstances( const inference::ModelConfig& model_config, std::vector>* added_instances, std::vector>* removed_instances); // Replace the foreground instances with background instances. void CommitInstances(); // Return all foreground instances indexed by its respective signature. std::unordered_map< TritonModelInstance::Signature, std::vector>> IndexInstances() const; // Add a new instance into the background. void RegisterBackgroundInstance( std::shared_ptr&& instance, const bool passive); // Clear all background instances. void ClearBackgroundInstances(); // Gets the execution policy setting from the backend. Status GetExecutionPolicy(const inference::ModelConfig& model_config); std::map> AccumulatedInstanceMemoryUsage() const override { std::map> res; // [FIXME] thread-safety on encountering instance change for (const auto& instances : {&instances_, &passive_instances_}) { for (const auto& instance : *instances) { const auto& imu = instance->MemoryUsage(); for (const auto& mem_type_map : imu) { const auto& mem_type = mem_type_map.first; for (const auto& mem_id_map : mem_type_map.second) { const auto& mem_id = mem_id_map.first; const auto& byte_size = mem_id_map.second; res[mem_type][mem_id] += byte_size; } } } } return res; } // Set the scheduler based on the model configuration and the provided // instances. Status SetConfiguredScheduler( const std::vector>& new_instances); // Update the set scheduler to the new set of instances. Status UpdateConfiguredScheduler( const std::vector>& added_instances, const std::vector>& removed_instances); // Set the batching strategy, if custom functions provided by user. // This function should only be called with the dynamic batcher. Status SetBatchingStrategy(const std::string& batch_libpath); // Merges the global backend configs with the specific // backend configs. static Status ResolveBackendConfigs( const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map, const std::string& backend_name, triton::common::BackendCmdlineConfig& config); // Sets defaults for some backend configurations when none are specified on // the command line. static Status SetBackendConfigDefaults( triton::common::BackendCmdlineConfig& config); // Get the search paths to the backend shared library. static std::vector GetBackendLibrarySearchPaths( const std::string& model_path, int64_t version, const std::string& backend_dir, const std::string& backend_name); // Get backend library directory and path, and search paths for the library // and whether the backend is based on Python backend. The model configuration // runtime field will be updated if left empty. static Status GetBackendLibraryProperties( const std::string& model_path, int64_t version, const std::string& backend_dir, const std::string& backend_name, inference::ModelConfig* model_config, bool* is_python_based_backend, std::vector* search_paths, std::string* backend_libdir, std::string* backend_libpath); // Get 'backend_libname', 'backend_libdir', 'backend_libpath' and // 'is_python_based_backend' by searching for different possible backend // library names on 'search_paths'. Searching for Python based backend // runtime is limited to 'backend_dir'. static Status GetBackendRuntimeLibraryName( const std::string& backend_dir, const std::string& backend_name, const std::vector& search_paths, std::string* backend_libname, std::string* backend_libdir, std::string* backend_libpath, bool* is_python_based_backend); // Search for 'backend_libname' on 'search_paths'. If found, the matching // search path will be stored in 'backend_libdir' and the backend library path // will be stored in 'backend_libpath'. If not found, 'backend_libpath' will // be set to empty. static Status FindBackendLibraryPath( const std::vector& search_paths, const std::string& backend_libname, std::string* backend_libdir, std::string* backend_libpath); // Assemble the C++ runtime library name. static std::string AssembleCPPRuntimeLibraryName( const std::string& backend_name); // Clear library handles. void ClearHandles(); // The server object that owns this model. The model holds this as a // raw pointer because the lifetime of the server is guaranteed to // be longer than the lifetime of a model owned by the server. InferenceServer* server_; // The minimum supported compute capability on device. const double min_compute_capability_; // Whether the backend should attempt to auto-complete the model config. const bool auto_complete_config_; // The backend cmdline config. const triton::common::BackendCmdlineConfigMap backend_cmdline_config_map_; // The host policy map. const triton::common::HostPolicyCmdlineConfigMap host_policy_map_; // The device blocking. It should not be changed after the model is created. bool device_blocking_; // The localized repo directory holding the model. If localization // required creation of a temporary local copy then that copy will // persist as along as this object is retained by this model. std::shared_ptr localized_model_dir_; // Backend used by this model. std::shared_ptr backend_; // The model instances for this model. Passive instances are loaded but not // added to the scheduler. std::vector> instances_; std::vector> passive_instances_; // They are the background 'instances_' and 'passive_instances_', not yet // effective until committed. std::vector> bg_instances_; std::vector> bg_passive_instances_; // Opaque state associated with this model. void* state_; // Custom batching shared object handle, function pointers, and batcher // pointer. void* batch_dlhandle_ = nullptr; TritonModelBatchInclFn_t batch_incl_fn_ = nullptr; TritonModelBatchInitFn_t batch_init_fn_ = nullptr; TritonModelBatchFiniFn_t batch_fini_fn_ = nullptr; TritonModelBatcherInitFn_t batcher_init_fn_ = nullptr; TritonModelBatcherFiniFn_t batcher_fini_fn_ = nullptr; TRITONBACKEND_Batcher* batcher_ = nullptr; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_model_instance.cc000066400000000000000000001242511527052371200223230ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "backend_model_instance.h" #include "status.h" #ifndef _WIN32 #include #include #include #endif #include "backend_config.h" #include "backend_model.h" #include "cuda_utils.h" #include "infer_stats.h" #include "metrics.h" #include "model_config.pb.h" #include "numa_utils.h" #include "server.h" #include "shared_library.h" #include "triton/common/logging.h" #include "triton/common/nvtx.h" #include "tritonserver_apis.h" // For unknown reason, windows will not export the TRITONBACKEND_* // functions declared with dllexport in tritonbackend.h. To get those // functions exported it is (also?) necessary to mark the definitions // in this file with dllexport as well. #if defined(_MSC_VER) #define TRITONAPI_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONAPI_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONAPI_DECLSPEC #endif namespace triton { namespace core { namespace { // Utilities for warmup feature TRITONSERVER_Error* WarmupResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { *buffer = malloc(byte_size); if (*buffer != nullptr) { *actual_memory_type = TRITONSERVER_MEMORY_CPU; *actual_memory_type_id = 0; return nullptr; } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "failed to allocate output buffer for warmup."); } TRITONSERVER_Error* WarmupResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { free(buffer); return nullptr; } ResponseAllocator warmup_allocator = ResponseAllocator( WarmupResponseAlloc, WarmupResponseRelease, nullptr /* start_fn */); void WarmupResponseComplete( TRITONSERVER_InferenceResponse* iresponse, const uint32_t flags, void* userp) { auto res_pair = reinterpret_cast< std::pair, std::vector*>*>(userp); if (iresponse != nullptr) { auto err = TRITONSERVER_InferenceResponseError(iresponse); if (err != nullptr) { // The error vector is shared by all requests in the batch for now static std::mutex res_mtx; { std::lock_guard lk(res_mtx); res_pair->second->emplace_back(TRITONSERVER_ErrorMessage(err)); } TRITONSERVER_ErrorDelete(err); } // Just delete the response, warmup doesn't check for correctness LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(iresponse), "deleting warmup response"); } // Last response if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) != 0) { res_pair->first.set_value(); } } void WarmupRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { if ((flags & TRITONSERVER_REQUEST_RELEASE_ALL) != 0) { // Don't need to release request here, it is managed in WarmupData if (userp != nullptr) { auto warmup_promise = reinterpret_cast*>(userp); warmup_promise->set_value(); } } } // Helper function for creating an instance Status VerifyModelLoadGpuFraction( const std::string& name, TRITONSERVER_InstanceGroupKind kind, int32_t device_id, const triton::common::BackendCmdlineConfigMap& backend_config_map) { size_t free, total; double memory_limit; RETURN_IF_ERROR(GetDeviceMemoryInfo(device_id, &free, &total)); RETURN_IF_ERROR(BackendConfigurationModelLoadGpuFraction( backend_config_map, device_id, &memory_limit)); const size_t allow = total * memory_limit; const size_t used = total - free; if (used > allow) { return Status( Status::Code::UNAVAILABLE, std::string("can not create model '") + name + "': memory limit set for " + TRITONSERVER_InstanceGroupKindString(kind) + " " + std::to_string(device_id) + " has exceeded, model loading is rejected."); } return Status::Success; } } // namespace TritonModelInstance::TritonModelInstance( TritonModel* model, const std::string& name, const Signature& signature, const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id, const std::vector& profile_names, const bool passive, const triton::common::HostPolicyCmdlineConfig& host_policy, const TritonServerMessage& host_policy_message, const std::vector& secondary_devices) : model_(model), name_(name), signature_(signature), kind_(kind), device_id_(device_id), host_policy_(host_policy), host_policy_message_(host_policy_message), profile_names_(profile_names), passive_(passive), secondary_devices_(secondary_devices), state_(nullptr) { #ifdef TRITON_ENABLE_METRICS if (Metrics::Enabled()) { // Use an ID in the metric only for GPU instances. Otherwise use // METRIC_REPORTER_ID_CPU to indicate no device should be reported in the // metric. const int id = (kind_ == TRITONSERVER_INSTANCEGROUPKIND_GPU) ? device_id_ : METRIC_REPORTER_ID_CPU; // Let every metric reporter know if caching is enabled to correctly include // cache miss time into request duration on cache misses. const bool response_cache_enabled = model_->ResponseCacheEnabled() && model_->Server()->ResponseCacheEnabled(); MetricModelReporter::Create( model_->ModelId(), model_->Version(), id, response_cache_enabled, model_->Config().metric_tags(), &reporter_); } #endif // TRITON_ENABLE_METRICS } TritonModelInstance::~TritonModelInstance() { if (triton_backend_thread_.get() != nullptr) { triton_backend_thread_->StopBackendThread(); } model_->Server()->GetRateLimiter()->UnregisterModelInstance(this); // Model finalization is optional... if (model_->Backend()->ModelInstanceFiniFn() != nullptr) { LOG_TRITONSERVER_ERROR( model_->Backend()->ModelInstanceFiniFn()( reinterpret_cast(this)), "failed finalizing model instance"); } } Status TritonModelInstance::CreateInstance( TritonModel* model, const std::string& name, const Signature& signature, TRITONSERVER_InstanceGroupKind kind, int32_t device_id, const std::vector& profile_names, const bool passive, const std::string& host_policy_name, const inference::ModelRateLimiter& rate_limiter_config, const std::vector& secondary_devices, std::shared_ptr* triton_model_instance) { static triton::common::HostPolicyCmdlineConfig empty_host_policy; const triton::common::HostPolicyCmdlineConfig* host_policy = &empty_host_policy; const auto policy_it = model->HostPolicyMap().find(host_policy_name); if (policy_it != model->HostPolicyMap().end()) { host_policy = &policy_it->second; } RETURN_IF_ERROR(SetNumaConfigOnThread(*host_policy)); auto err = ConstructAndInitializeInstance( model, name, signature, kind, device_id, profile_names, passive, host_policy_name, *host_policy, rate_limiter_config, secondary_devices, triton_model_instance); RETURN_IF_ERROR(ResetNumaMemoryPolicy()); RETURN_IF_ERROR(err); // When deploying on GPU, we want to make sure the GPU memory usage // is within allowed range, otherwise, stop the creation to ensure // there is sufficient GPU memory for other use. // We check the usage after loading the instance to better enforcing // the limit. If we check before loading, we may create instance // that occupies the rest of available memory which against the purpose if (kind == TRITONSERVER_INSTANCEGROUPKIND_GPU) { RETURN_IF_ERROR(VerifyModelLoadGpuFraction( name, kind, device_id, model->BackendConfigMap())); } return Status::Success; } Status TritonModelInstance::ConstructAndInitializeInstance( TritonModel* model, const std::string& name, const Signature& signature, const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id, const std::vector& profile_names, const bool passive, const std::string& host_policy_name, const triton::common::HostPolicyCmdlineConfig& host_policy, const inference::ModelRateLimiter& rate_limiter_config, const std::vector& secondary_devices, std::shared_ptr* triton_model_instance) { // Create the JSON representation of the backend configuration. triton::common::TritonJson::Value host_policy_json( triton::common::TritonJson::ValueType::OBJECT); triton::common::TritonJson::Value policy_setting_json( host_policy_json, triton::common::TritonJson::ValueType::OBJECT); for (const auto& pr : host_policy) { RETURN_IF_ERROR(policy_setting_json.AddString(pr.first.c_str(), pr.second)); } RETURN_IF_ERROR(host_policy_json.Add( host_policy_name.c_str(), std::move(policy_setting_json))); TritonServerMessage host_policy_message(host_policy_json); std::unique_ptr local_instance(new TritonModelInstance( model, name, signature, kind, device_id, profile_names, passive, host_policy, host_policy_message, secondary_devices)); TRITONBACKEND_ModelInstance* triton_instance = reinterpret_cast(local_instance.get()); // Instance initialization is optional... We must set set shared // library path to point to the backend directory in case the // backend library attempts to load additional shared libraries. if (model->Backend()->ModelInstanceInitFn() != nullptr) { // We must set set shared library path to point to the backend directory in // case the backend library attempts to load additional shared libraries. // Currently, the set and reset function is effective only on Windows, so // there is no need to set path on non-Windows. // However, parallel model loading will not see any speedup on Windows and // the global lock inside the SharedLibrary is a WAR. // [FIXME] Reduce lock WAR on SharedLibrary (DLIS-4300) #ifdef _WIN32 std::unique_ptr slib; RETURN_IF_ERROR(SharedLibrary::Acquire(&slib)); RETURN_IF_ERROR(slib->SetLibraryDirectory(model->Backend()->Directory())); #endif TRITONSERVER_Error* err = model->Backend()->ModelInstanceInitFn()(triton_instance); #ifdef _WIN32 RETURN_IF_ERROR(slib->ResetLibraryDirectory()); #endif RETURN_IF_TRITONSERVER_ERROR(err); } if (!passive) { RETURN_IF_ERROR(local_instance->GenerateWarmupData()); RETURN_IF_ERROR(model->Server()->GetRateLimiter()->RegisterModelInstance( local_instance.get(), rate_limiter_config)); RETURN_IF_ERROR(local_instance->SetBackendThread( kind, device_id, model->DeviceBlocking())); } triton_model_instance->reset(local_instance.release()); return Status::Success; } Status TritonModelInstance::SetBackendThread( const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id, const bool device_blocking) { if (ShareBackendThread(device_blocking, kind)) { auto device_instances = model_->GetInstancesByDevice(device_id); if (!device_instances.empty()) { LOG_VERBOSE(1) << "Using already started backend thread for " << Name() << " on device " << device_id; triton_backend_thread_ = device_instances[0]->triton_backend_thread_; } } if (triton_backend_thread_.get() == nullptr) { std::unique_ptr local_backend_thread; RETURN_IF_ERROR(TritonBackendThread::CreateBackendThread( Name(), this, 0 /* nice */, device_id, &local_backend_thread)); triton_backend_thread_ = std::move(local_backend_thread); } else { triton_backend_thread_->AddModelInstance(this); } RETURN_IF_ERROR(triton_backend_thread_->InitAndWarmUpModelInstance(this)); return Status::Success; } Status TritonModelInstance::GenerateWarmupData() { warmup_samples_.clear(); for (const auto& warmup_setting : model_->Config().model_warmup()) { if (warmup_setting.batch_size() == 0) { LOG_VERBOSE(1) << "Skipping batch 0 warmup sample '" << warmup_setting.name() << "'"; continue; } LOG_VERBOSE(1) << "Generating warmup sample data for '" << warmup_setting.name() << "'"; // Two passes. First pass to get max byte size for synthetic // data. Second pass to add original inputs and override inputs // for control inputs. int64_t max_zero_byte_size = 0; int64_t max_random_byte_size = 0; for (const auto& input_meta : warmup_setting.inputs()) { auto element_count = triton::common::GetElementCount(input_meta.second.dims()); if (element_count == -1) { return Status( Status::Code::INVALID_ARG, "warmup setting expects all variable-size dimensions are specified " "for input '" + input_meta.first + "'"); } int64_t batch_byte_size = element_count * triton::common::GetDataTypeByteSize(input_meta.second.data_type()); if (batch_byte_size == 0) { batch_byte_size = element_count * sizeof(int32_t); } switch (input_meta.second.input_data_type_case()) { case inference::ModelWarmup_Input::InputDataTypeCase::kZeroData: max_zero_byte_size = std::max(batch_byte_size, max_zero_byte_size); break; case inference::ModelWarmup_Input::InputDataTypeCase::kRandomData: { // Because Triton expects STRING type to be in special format // (prepend 4 bytes to specify string length), so using zero data // for simplicity (4 bytes * element count of zeros). if (input_meta.second.data_type() == inference::DataType::TYPE_STRING) { max_zero_byte_size = std::max(batch_byte_size, max_zero_byte_size); } else { max_random_byte_size = std::max(batch_byte_size, max_random_byte_size); } break; } default: break; } } warmup_samples_.emplace_back(warmup_setting.name(), warmup_setting.count()); auto& warmup_data = warmup_samples_.back(); // Create buffers for synthetic data TRITONSERVER_MemoryType type; int64_t type_id; warmup_data.zero_data_.reset(new AllocatedMemory( max_zero_byte_size, TRITONSERVER_MEMORY_CPU_PINNED /* memory_type */, 0 /* memory_type_id */)); char* zero_buffer = warmup_data.zero_data_->MutableBuffer(&type, &type_id); memset(zero_buffer, 0, max_zero_byte_size); warmup_data.random_data_.reset(new AllocatedMemory( max_random_byte_size, TRITONSERVER_MEMORY_CPU_PINNED /* memory_type */, 0 /* memory_type_id */)); char* random_buffer = warmup_data.random_data_->MutableBuffer(&type, &type_id); for (int64_t offset = 0; offset < max_random_byte_size; offset++) { random_buffer[offset] = rand(); } // Prepare the inference request for the specified sample, not using // in-process C API because the request doesn't go through the same pipeline // (i.e. no normalization / scheduler) so we need to prepare the request to // the state just before calling instance execute function. for (size_t cnt = 0; cnt < warmup_setting.batch_size(); cnt++) { warmup_data.requests_.emplace_back( new InferenceRequest(model_, model_->Version())); auto& lrequest = warmup_data.requests_.back(); // Second pass to prepare original inputs. std::vector> input_sps; for (const auto& input_meta : warmup_setting.inputs()) { auto batch1_element_count = triton::common::GetElementCount(input_meta.second.dims()); auto batch_byte_size = batch1_element_count * triton::common::GetDataTypeByteSize(input_meta.second.data_type()); if (batch_byte_size == 0) { batch_byte_size = batch1_element_count * sizeof(int32_t); } const char* allocated_ptr; switch (input_meta.second.input_data_type_case()) { case inference::ModelWarmup_Input::InputDataTypeCase::kZeroData: allocated_ptr = zero_buffer; break; case inference::ModelWarmup_Input::InputDataTypeCase::kRandomData: { if (input_meta.second.data_type() == inference::DataType::TYPE_STRING) { allocated_ptr = zero_buffer; } else { allocated_ptr = random_buffer; } break; } case inference::ModelWarmup_Input::InputDataTypeCase:: kInputDataFile: { // For data provided from file, we can set buffer in first pass warmup_data.provided_data_.emplace_back(new std::string()); auto input_data = warmup_data.provided_data_.back().get(); RETURN_IF_ERROR(ReadTextFile( JoinPath( {model_->LocalizedModelPath(), kWarmupDataFolder, input_meta.second.input_data_file()}), input_data)); if (input_meta.second.data_type() == inference::DataType::TYPE_STRING) { batch_byte_size = input_data->size(); } else if (((size_t)batch_byte_size) > input_data->size()) { return Status( Status::Code::INVALID_ARG, lrequest->LogRequest() + "warmup setting expects " + std::to_string(batch_byte_size) + " bytes, but the data " "provided from " + input_meta.second.input_data_file() + "only has " + std::to_string(input_data->size()) + " bytes"); } allocated_ptr = input_data->data(); break; } default: return Status( Status::Code::INVALID_ARG, lrequest->LogRequest() + "warmup setting expects input '" + input_meta.first + "' to have input_data_type set"); } const inference::ModelInput* input_config; bool is_original_input = model_->GetInput(input_meta.first, &input_config).IsOk(); InferenceRequest::Input* input = nullptr; std::vector input_meta_shape; // Append batch size only if the model supports batching // and not control inpt. if ((model_->Config().max_batch_size() != 0) && is_original_input) { input_meta_shape.push_back(1); } for (auto d : input_meta.second.dims()) { input_meta_shape.push_back(d); } if (is_original_input) { RETURN_IF_ERROR(lrequest->AddOriginalInput( input_meta.first, input_meta.second.data_type(), input_meta_shape, &input)); } else { input_sps.emplace_back(); RETURN_IF_ERROR(lrequest->AddOverrideInput( input_meta.first, input_meta.second.data_type(), (model_->Config().max_batch_size() != 0 ? 1 : 0), input_meta_shape, &input_sps.back())); input = input_sps.back().get(); } RETURN_IF_ERROR(input->AppendData( allocated_ptr, batch_byte_size, TRITONSERVER_MEMORY_CPU /* memory_type */, 0 /* memory_type_id */)); } RETURN_IF_ERROR(lrequest->PrepareForInference()); // Override inputs must be added after PrepareForInference() is called for (const auto& sp : input_sps) { RETURN_IF_ERROR(lrequest->AddOverrideInput(sp)); } } } return Status::Success; } Status TritonModelInstance::PrepareRequestsForExecution( std::vector>& requests) { for (auto& r : requests) { // Load the input states for the inference request. RETURN_IF_ERROR(r->LoadInputStates()); // Set request state to signify that request is no longer pending. RETURN_IF_ERROR(r->SetState(InferenceRequest::State::EXECUTING)); } return Status::Success; } Status TritonModelInstance::PrepareRequestsOrRespond( std::vector>& requests) { auto status = PrepareRequestsForExecution(requests); // If any errors occurred, respond with error for each request. if (!status.IsOk()) { for (auto& r : requests) { InferenceRequest::RespondIfError( r, status, true /* release_requests */, FailureReason::OTHER); } // Log a single error for batch of requests for better visibility LOG_STATUS_ERROR(status, "Requests failed pre-execution checks"); } return status; } Status TritonModelInstance::Schedule( std::vector>&& requests) { // Prepare requests for execution, respond to requests if any error occur. RETURN_IF_ERROR(PrepareRequestsOrRespond(requests)); // Use a thread local vector to avoid needing to malloc each // time an inference is run. thread_local std::vector triton_requests(1024); triton_requests.clear(); for (auto& r : requests) { triton_requests.push_back( reinterpret_cast(r.release())); } Execute(triton_requests); return Status::Success; } Status TritonModelInstance::Initialize() { RETURN_IF_ERROR(SetNumaConfigOnThread(HostPolicy())); return Status::Success; } Status TritonModelInstance::WarmUp() { // move samples to local variable for scoped cleanup std::vector lwarmup_samples; lwarmup_samples.swap(warmup_samples_); for (auto& sample : lwarmup_samples) { for (size_t iteration = 1; iteration <= sample.count_; ++iteration) { LOG_VERBOSE(1) << "model '" << sample.requests_.back()->ModelName() << "' instance " << Name() << " is running warmup sample '" << sample.sample_name_ << "' for iteration " << iteration; // request/response complete is asynchronous so use promise to wait for // completion. Also collects error message from the responses in a vector. std::vector> request_complete(sample.requests_.size()); std::vector response_errors; std::vector, std::vector*>> response_complete(sample.requests_.size()); std::vector triton_requests; for (size_t i = 0; i < sample.requests_.size(); ++i) { auto& request = sample.requests_[i]; request->SetReleaseCallback( WarmupRequestComplete, &request_complete[i]); response_complete[i].second = &response_errors; request->SetResponseCallback( &warmup_allocator, nullptr, WarmupResponseComplete, &response_complete[i]); // For warmup requests we need to manually set ResponseFactory // since they modify the callback after PrepareForInference has // been called. request->SetResponseFactory(); // Capture timestamp before run to avoid incorrect accumulation from // sequential warmup runs #ifdef TRITON_ENABLE_STATS request->CaptureRequestStartNs(); #endif // TRITON_ENABLE_STATS request->CaptureQueueStartNs(); triton_requests.push_back( reinterpret_cast(request.get())); } Execute(triton_requests); // Wait for warmup sample to complete and check error for (size_t i = 0; i < sample.requests_.size(); ++i) { request_complete[i].get_future().get(); response_complete[i].first.get_future().get(); } if (response_errors.size() != 0) { std::string err_str = "failed to run warmup sample '" + sample.sample_name_ + "': "; for (const auto& error : response_errors) { err_str += (error + "; "); } // End warmup as soon as there is failing sample LOG_VERBOSE(1) << "model '" << sample.requests_.back()->ModelName() << "' instance " << Name() << " failed to run warmup sample '" << sample.sample_name_ << "'"; return Status(Status::Code::INVALID_ARG, err_str); } } } return Status::Success; } void TritonModelInstance::Execute( std::vector& triton_requests) { TRITONBACKEND_ModelInstance* triton_model_instance = reinterpret_cast(this); TritonBackend::TritonModelInstanceExecFn_t inst_exec_fn = model_->Backend()->ModelInstanceExecFn(); // If there is an error then we retain ownership of 'requests' // and must send error responses. TRITONSERVER_Error* err = inst_exec_fn( triton_model_instance, &triton_requests[0], triton_requests.size()); if (err != nullptr) { Status status = Status( TritonCodeToStatusCode(TRITONSERVER_ErrorCode(err)), TRITONSERVER_ErrorMessage(err)); for (TRITONBACKEND_Request* tr : triton_requests) { std::unique_ptr ur( reinterpret_cast(tr)); // NOTE: If a backend both returns an error in // TRITONBACKEND_ModelInstanceExecute and reports an error with // TRITONBACKEND_ModelInstanceReportStatistics, this can result in double // counting of the failure metric for the same request. However, it is // currently not expected for this to be a common case, as the return // value of TRITONBACKEND_ModelInstanceExecute is used to express // ownership of the request rather than success of an inference request. // See tritonbackend.h for more details on this. InferenceRequest::RespondIfError( ur, status, true /* release_requests */, FailureReason::BACKEND); } TRITONSERVER_ErrorDelete(err); } } Status TritonModelInstance::TritonBackendThread::CreateBackendThread( const std::string name, TritonModelInstance* model_instance, const int nice, const int32_t device_id, std::unique_ptr* triton_backend_thread) { TritonBackendThread* raw_triton_backend_thread = new TritonBackendThread(name, model_instance->Model(), nice, device_id); std::unique_ptr runner(raw_triton_backend_thread); runner->AddModelInstance(model_instance); runner->backend_thread_ = std::thread([raw_triton_backend_thread]() { raw_triton_backend_thread->BackendThread(); }); triton_backend_thread->reset(runner.release()); return Status::Success; } void TritonModelInstance::TritonBackendThread::AddModelInstance( TritonModelInstance* model_instance) { model_instances_.push_back(model_instance); } Status TritonModelInstance::TritonBackendThread::InitAndWarmUpModelInstance( TritonModelInstance* model_instance) { // Initialize the instance on the backend thread auto init_payload = model_->Server()->GetRateLimiter()->GetPayload( Payload::Operation::INIT, model_instance); RETURN_IF_ERROR( model_->Server()->GetRateLimiter()->EnqueuePayload(model_, init_payload)); RETURN_IF_ERROR(init_payload->Wait()); // Warm-up the instance on the backend thread auto warmup_payload = model_->Server()->GetRateLimiter()->GetPayload( Payload::Operation::WARM_UP, model_instance); RETURN_IF_ERROR(model_->Server()->GetRateLimiter()->EnqueuePayload( model_, warmup_payload)); RETURN_IF_ERROR(warmup_payload->Wait()); return Status::Success; } TritonModelInstance::TritonBackendThread::TritonBackendThread( const std::string& name, TritonModel* model, const int nice, const int32_t device_id) : name_(name), nice_(nice), device_id_(device_id), model_(model) { } TritonModelInstance::TritonBackendThread::~TritonBackendThread() { StopBackendThread(); } void TritonModelInstance::TritonBackendThread::StopBackendThread() { if (backend_thread_.joinable()) { // Signal the backend thread to exit and then wait for it... auto exit_payload = model_->Server()->GetRateLimiter()->GetPayload( Payload::Operation::EXIT, model_instances_.back()); model_->Server()->GetRateLimiter()->EnqueuePayload(model_, exit_payload); backend_thread_.join(); } } void TritonModelInstance::TritonBackendThread::BackendThread() { #ifndef _WIN32 if (setpriority(PRIO_PROCESS, syscall(SYS_gettid), nice_) == 0) { LOG_VERBOSE(1) << "Starting backend thread for " << name_ << " at nice " << nice_ << " on device " << device_id_ << "..."; } else { LOG_VERBOSE(1) << "Starting backend thread for " << name_ << " at default nice (requested nice " << nice_ << " failed)" << " on device " << device_id_ << "..."; } #else LOG_VERBOSE(1) << "Starting backend thread for " << name_ << " at default nice on device " << device_id_ << "..."; #endif bool should_exit = false; while (!should_exit) { std::shared_ptr payload; model_->Server()->GetRateLimiter()->DequeuePayload( model_instances_, &payload); NVTX_RANGE(nvtx_, "BackendThread " + name_); payload->Execute(&should_exit); model_instances_.push_back(payload->GetInstance()); // Release the payload to the RateLimiter model_->Server()->GetRateLimiter()->PayloadRelease(payload); } LOG_VERBOSE(1) << "Stopping backend thread for " << name_ << "..."; } // Opaque object for the response statistics C-API struct ModelInstanceResponseStatistics { #ifdef TRITON_ENABLE_STATS ModelInstanceResponseStatistics() : model_instance(nullptr), response_factory(nullptr), response_start(0), compute_output_start(0), response_end(0), error(nullptr) { } TritonModelInstance* model_instance; std::shared_ptr* response_factory; uint64_t response_start; uint64_t compute_output_start; uint64_t response_end; TRITONSERVER_Error* error; #endif // TRITON_ENABLE_STATS }; extern "C" { TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceName( TRITONBACKEND_ModelInstance* instance, const char** name) { TritonModelInstance* ti = reinterpret_cast(instance); *name = ti->Name().c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceKind( TRITONBACKEND_ModelInstance* instance, TRITONSERVER_InstanceGroupKind* kind) { TritonModelInstance* ti = reinterpret_cast(instance); *kind = ti->Kind(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceDeviceId( TRITONBACKEND_ModelInstance* instance, int32_t* device_id) { TritonModelInstance* ti = reinterpret_cast(instance); *device_id = ti->DeviceId(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceHostPolicy( TRITONBACKEND_ModelInstance* instance, TRITONSERVER_Message** host_policy) { TritonModelInstance* ti = reinterpret_cast(instance); *host_policy = const_cast( reinterpret_cast(&ti->HostPolicyMessage())); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceProfileCount( TRITONBACKEND_ModelInstance* instance, uint32_t* count) { TritonModelInstance* ti = reinterpret_cast(instance); *count = ti->Profiles().size(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceProfileName( TRITONBACKEND_ModelInstance* instance, const uint32_t index, const char** profile_name) { *profile_name = nullptr; TritonModelInstance* ti = reinterpret_cast(instance); const auto& rprofiles = ti->Profiles(); if (index >= rprofiles.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("out of bounds index ") + std::to_string(index) + ": instance is configured with " + std::to_string(rprofiles.size()) + " profiles") .c_str()); } *profile_name = rprofiles[index].c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceSecondaryDeviceCount( TRITONBACKEND_ModelInstance* instance, uint32_t* count) { TritonModelInstance* ti = reinterpret_cast(instance); *count = ti->SecondaryDevices().size(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceSecondaryDeviceProperties( TRITONBACKEND_ModelInstance* instance, uint32_t index, const char** kind, int64_t* id) { TritonModelInstance* ti = reinterpret_cast(instance); const auto& rsecondarydevices = ti->SecondaryDevices(); if (index >= rsecondarydevices.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("out of bounds index ") + std::to_string(index) + ": instance is configured with " + std::to_string(rsecondarydevices.size()) + " secondary devices") .c_str()); } *kind = rsecondarydevices[index].kind_.c_str(); *id = rsecondarydevices[index].id_; return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceIsPassive( TRITONBACKEND_ModelInstance* instance, bool* is_passive) { TritonModelInstance* ti = reinterpret_cast(instance); *is_passive = ti->IsPassive(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceModel( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Model** model) { TritonModelInstance* ti = reinterpret_cast(instance); *model = reinterpret_cast(ti->Model()); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceState( TRITONBACKEND_ModelInstance* instance, void** state) { TritonModelInstance* ti = reinterpret_cast(instance); *state = ti->State(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceSetState( TRITONBACKEND_ModelInstance* instance, void* state) { TritonModelInstance* ti = reinterpret_cast(instance); ti->SetState(state); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceReportStatistics( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request* request, const bool success, const uint64_t exec_start_ns, const uint64_t compute_start_ns, const uint64_t compute_end_ns, const uint64_t exec_end_ns) { #ifdef TRITON_ENABLE_STATS TritonModelInstance* ti = reinterpret_cast(instance); InferenceRequest* tr = reinterpret_cast(request); tr->ReportStatistics( ti->MetricReporter(), success, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns); #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsNew( TRITONBACKEND_ModelInstanceResponseStatistics** response_statistics) { #ifdef TRITON_ENABLE_STATS *response_statistics = reinterpret_cast( new ModelInstanceResponseStatistics()); #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsDelete( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics) { #ifdef TRITON_ENABLE_STATS delete reinterpret_cast( response_statistics); #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetModelInstance( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, TRITONBACKEND_ModelInstance* model_instance) { #ifdef TRITON_ENABLE_STATS ModelInstanceResponseStatistics* rs = reinterpret_cast(response_statistics); rs->model_instance = reinterpret_cast(model_instance); #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseFactory( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, TRITONBACKEND_ResponseFactory* response_factory) { #ifdef TRITON_ENABLE_STATS ModelInstanceResponseStatistics* rs = reinterpret_cast(response_statistics); rs->response_factory = reinterpret_cast*>( response_factory); ; #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseStart( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, uint64_t response_start) { #ifdef TRITON_ENABLE_STATS ModelInstanceResponseStatistics* rs = reinterpret_cast(response_statistics); rs->response_start = response_start; #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetComputeOutputStart( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, uint64_t compute_output_start) { #ifdef TRITON_ENABLE_STATS ModelInstanceResponseStatistics* rs = reinterpret_cast(response_statistics); rs->compute_output_start = compute_output_start; #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseEnd( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, uint64_t response_end) { #ifdef TRITON_ENABLE_STATS ModelInstanceResponseStatistics* rs = reinterpret_cast(response_statistics); rs->response_end = response_end; #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceResponseStatisticsSetError( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics, TRITONSERVER_Error* error) { #ifdef TRITON_ENABLE_STATS ModelInstanceResponseStatistics* rs = reinterpret_cast(response_statistics); rs->error = error; #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceReportResponseStatistics( TRITONBACKEND_ModelInstanceResponseStatistics* response_statistics) { #ifdef TRITON_ENABLE_STATS ModelInstanceResponseStatistics* rs = reinterpret_cast(response_statistics); InferenceStatsAggregator* sa = rs->model_instance->Model()->MutableStatsAggregator(); std::string key = std::to_string((*rs->response_factory)->GetAndIncrementResponseIndex()); if (rs->error == nullptr) { if (rs->compute_output_start > 0) { RETURN_TRITONSERVER_ERROR_IF_ERROR(sa->UpdateResponseSuccess( key, rs->response_start, rs->compute_output_start, rs->response_end)); } else { RETURN_TRITONSERVER_ERROR_IF_ERROR( sa->UpdateResponseEmpty(key, rs->response_start, rs->response_end)); } } else if ( TRITONSERVER_ErrorCode(rs->error) == TRITONSERVER_ERROR_CANCELLED) { RETURN_TRITONSERVER_ERROR_IF_ERROR( sa->UpdateResponseCancel(key, rs->response_start, rs->response_end)); } else { RETURN_TRITONSERVER_ERROR_IF_ERROR(sa->UpdateResponseFail( key, rs->response_start, rs->compute_output_start, rs->response_end)); } #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceReportBatchStatistics( TRITONBACKEND_ModelInstance* instance, const uint64_t batch_size, const uint64_t exec_start_ns, const uint64_t compute_start_ns, const uint64_t compute_end_ns, const uint64_t exec_end_ns) { #ifdef TRITON_ENABLE_STATS TritonModelInstance* ti = reinterpret_cast(instance); ti->Model()->MutableStatsAggregator()->UpdateInferBatchStats( ti->MetricReporter(), batch_size, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns); #endif // TRITON_ENABLE_STATS return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceReportMemoryUsage( TRITONBACKEND_ModelInstance* instance, TRITONSERVER_BufferAttributes** usage, uint32_t usage_size) { TritonModelInstance* ti = reinterpret_cast(instance); ti->SetMemoryUsage({reinterpret_cast(usage), usage_size}); return nullptr; // success } } // extern C }} // namespace triton::core kytensor-2.49.0.7/core/src/backend_model_instance.h000066400000000000000000000231721527052371200221650ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include "constants.h" #include "memory.h" #include "metric_model_reporter.h" #include "model_config.pb.h" #include "model_config_utils.h" #include "server_message.h" #include "status.h" #include "triton/common/sync_queue.h" namespace triton { namespace core { class TritonModel; class InferenceRequest; // // Represents a model instance. // class TritonModelInstance { public: struct SecondaryDevice { SecondaryDevice(const std::string kind, const int64_t id) : kind_(kind), id_(id) { } const std::string kind_; const int64_t id_; }; class Signature { public: Signature( const inference::ModelInstanceGroup& group_config, int32_t device_id) : group_config_(group_config), device_id_(device_id), hash_(std::hash{}( std::to_string(device_id_) + InstanceConfigSignature(group_config_))) { } bool operator==(const Signature& rhs) const { return device_id_ == rhs.device_id_ && EquivalentInInstanceConfig(group_config_, rhs.group_config_); } bool operator!=(const Signature& rhs) const { return !(*this == rhs); } std::size_t Hash() const { return hash_; } private: const inference::ModelInstanceGroup group_config_; const int32_t device_id_; const std::size_t hash_; }; static Status CreateInstance( TritonModel* model, const std::string& name, const Signature& signature, TRITONSERVER_InstanceGroupKind kind, int32_t device_id, const std::vector& profile_names, const bool passive, const std::string& host_policy_name, const inference::ModelRateLimiter& rate_limiter_config, const std::vector& secondary_devices, std::shared_ptr* triton_model_instance); ~TritonModelInstance(); const std::string& Name() const { return name_; } Signature& GetSignature() { return signature_; } TRITONSERVER_InstanceGroupKind Kind() const { return kind_; } int32_t DeviceId() const { return device_id_; } const triton::common::HostPolicyCmdlineConfig& HostPolicy() const { return host_policy_; } const TritonServerMessage& HostPolicyMessage() const { return host_policy_message_; } bool IsPassive() const { return passive_; } const std::vector& Profiles() const { return profile_names_; } const std::vector& SecondaryDevices() const { return secondary_devices_; } Status Initialize(); Status WarmUp(); Status Schedule(std::vector>&& requests); TritonModel* Model() const { return model_; } void* State() { return state_; } void SetState(void* state) { state_ = state; } MetricModelReporter* MetricReporter() const { return reporter_.get(); } // Directly call from C API, so arguments are in the same style void SetMemoryUsage(boost::span memory_usage) { std::map> lusage; for (const auto& mu : memory_usage) { lusage[mu->MemoryType()][mu->MemoryTypeId()] = mu->ByteSize(); } std::lock_guard lk(usage_mtx_); memory_usage_.swap(lusage); } std::map> MemoryUsage() const { std::lock_guard lk(usage_mtx_); return memory_usage_; } static bool ShareBackendThread( const bool device_blocking, const TRITONSERVER_InstanceGroupKind kind) { return device_blocking && (kind == TRITONSERVER_INSTANCEGROUPKIND_GPU); } private: class TritonBackendThread { public: static Status CreateBackendThread( const std::string name, TritonModelInstance* model, const int nice, const int32_t device_id, std::unique_ptr* triton_backend_thread); void AddModelInstance(TritonModelInstance* model_instance); Status InitAndWarmUpModelInstance(TritonModelInstance* model_instance); void StopBackendThread(); ~TritonBackendThread(); private: TritonBackendThread( const std::string& name, TritonModel* model, const int nice, const int32_t device_id); void BackendThread(); const std::string name_; const int nice_; const int32_t device_id_; TritonModel* model_; std::deque model_instances_; std::thread backend_thread_; std::atomic backend_thread_exit_; }; struct WarmupData { WarmupData(const std::string& sample_name, const size_t count) : sample_name_(sample_name), count_(std::max(count, size_t{1})) { } std::string sample_name_; size_t count_; // Using a batch of requests to satisfy batch size, this provides better // alignment on the batch expected by the model, especially for sequence // model. std::vector> requests_; // Placeholder for input data std::unique_ptr zero_data_; std::unique_ptr random_data_; std::vector> provided_data_; }; DISALLOW_COPY_AND_ASSIGN(TritonModelInstance); TritonModelInstance( TritonModel* model, const std::string& name, const Signature& signature, const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id, const std::vector& profile_names, const bool passive, const triton::common::HostPolicyCmdlineConfig& host_policy, const TritonServerMessage& host_policy_message, const std::vector& secondary_devices); static Status ConstructAndInitializeInstance( TritonModel* model, const std::string& name, const Signature& signature, const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id, const std::vector& profile_names, const bool passive, const std::string& host_policy_name, const triton::common::HostPolicyCmdlineConfig& host_policy, const inference::ModelRateLimiter& rate_limiter_config, const std::vector& secondary_devices, std::shared_ptr* triton_model_instance); Status SetBackendThread( const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id, const bool device_blocking); Status GenerateWarmupData(); Status PrepareRequestsForExecution( std::vector>& requests); Status PrepareRequestsOrRespond( std::vector>& requests); void Execute(std::vector& triton_requests); std::shared_ptr triton_backend_thread_; std::vector warmup_samples_; // The TritonModel object that owns this instance. The instance // holds this as a raw pointer because the lifetime of the model is // guaranteed to be longer than the lifetime of an instance owned by the // model. TritonModel* model_; std::string name_; Signature signature_; // For CPU device_id_ is always 0. For GPU device_id_ indicates the // GPU device to be used by the instance. TRITONSERVER_InstanceGroupKind kind_; const int32_t device_id_; const triton::common::HostPolicyCmdlineConfig host_policy_; TritonServerMessage host_policy_message_; std::vector profile_names_; bool passive_; std::vector secondary_devices_; // Reporter for metrics, or nullptr if no metrics should be reported std::shared_ptr reporter_; // Records of memory used for the model instance std::map> memory_usage_; mutable std::mutex usage_mtx_; // Opaque state associated with this model instance. void* state_; }; }} // namespace triton::core // Interface for triton::core::TritonModelInstance::Signature hash function namespace std { template <> struct hash { std::size_t operator()( const triton::core::TritonModelInstance::Signature& s) const { return s.Hash(); } }; } // namespace std kytensor-2.49.0.7/core/src/buffer_attributes.cc000066400000000000000000000062221527052371200214040ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "buffer_attributes.h" #include #include "constants.h" namespace triton { namespace core { void BufferAttributes::SetByteSize(const size_t& byte_size) { byte_size_ = byte_size; } void BufferAttributes::SetMemoryType(const TRITONSERVER_MemoryType& memory_type) { memory_type_ = memory_type; } void BufferAttributes::SetMemoryTypeId(const int64_t& memory_type_id) { memory_type_id_ = memory_type_id; } void BufferAttributes::SetCudaIpcHandle(void* cuda_ipc_handle) { char* lcuda_ipc_handle = reinterpret_cast(cuda_ipc_handle); cuda_ipc_handle_.clear(); std::copy( lcuda_ipc_handle, lcuda_ipc_handle + CUDA_IPC_STRUCT_SIZE, std::back_inserter(cuda_ipc_handle_)); } void* BufferAttributes::CudaIpcHandle() { if (cuda_ipc_handle_.empty()) { return nullptr; } else { return reinterpret_cast(cuda_ipc_handle_.data()); } } size_t BufferAttributes::ByteSize() const { return byte_size_; } TRITONSERVER_MemoryType BufferAttributes::MemoryType() const { return memory_type_; } int64_t BufferAttributes::MemoryTypeId() const { return memory_type_id_; } BufferAttributes::BufferAttributes( size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, char* cuda_ipc_handle) : byte_size_(byte_size), memory_type_(memory_type), memory_type_id_(memory_type_id) { // cuda ipc handle size cuda_ipc_handle_.reserve(CUDA_IPC_STRUCT_SIZE); if (cuda_ipc_handle != nullptr) { std::copy( cuda_ipc_handle, cuda_ipc_handle + CUDA_IPC_STRUCT_SIZE, std::back_inserter(cuda_ipc_handle_)); } } }} // namespace triton::core kytensor-2.49.0.7/core/src/buffer_attributes.h000066400000000000000000000054001527052371200212430ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "tritonserver_apis.h" #pragma once namespace triton { namespace core { // // A class to hold information about the buffer allocation. // class BufferAttributes { public: BufferAttributes( size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, char cuda_ipc_handle[64]); BufferAttributes() { memory_type_ = TRITONSERVER_MEMORY_CPU; memory_type_id_ = 0; cuda_ipc_handle_.reserve(64); } // Set the buffer byte size void SetByteSize(const size_t& byte_size); // Set the buffer memory_type void SetMemoryType(const TRITONSERVER_MemoryType& memory_type); // Set the buffer memory type id void SetMemoryTypeId(const int64_t& memory_type_id); // Set the cuda ipc handle void SetCudaIpcHandle(void* cuda_ipc_handle); // Get the cuda ipc handle void* CudaIpcHandle(); // Get the byte size size_t ByteSize() const; // Get the memory type TRITONSERVER_MemoryType MemoryType() const; // Get the memory type id int64_t MemoryTypeId() const; private: size_t byte_size_; TRITONSERVER_MemoryType memory_type_; int64_t memory_type_id_; std::vector cuda_ipc_handle_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/cache_entry.cc000066400000000000000000000377741527052371200201710ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "cache_entry.h" #include namespace triton { namespace core { /* CacheEntry */ size_t CacheEntry::BufferCount() { std::unique_lock lk(buffer_mu_); return buffers_.size(); } const std::vector& CacheEntry::Buffers() { std::unique_lock lk(buffer_mu_); return buffers_; } std::vector& CacheEntry::MutableBuffers() { std::unique_lock lk(buffer_mu_); return buffers_; } void CacheEntry::AddBuffer(boost::span byte_span) { void* base = static_cast(byte_span.data()); AddBuffer(base, byte_span.size()); } void CacheEntry::AddBuffer(void* base, size_t byte_size) { std::unique_lock lk(buffer_mu_); buffers_.emplace_back(std::make_pair(base, byte_size)); } CacheEntry::~CacheEntry() { std::unique_lock lk(buffer_mu_); if (!free_buffers_) { return; } for (auto& iter : buffers_) { auto& base = iter.first; if (base) { free(base); base = nullptr; } } } void CacheEntry::AddPlaceholderBuffer(size_t byte_size) { AddBuffer(nullptr, byte_size); } // Set the size of each buffer in the CacheEntry object so the cache knows // how much to allocate for each entry before insertion. Status CacheEntry::SetBufferSizes(std::vector> buffers) { for (const auto buffer : buffers) { AddPlaceholderBuffer(buffer.size()); } return Status::Success; } Status CacheEntry::SetBufferSizes(boost::span responses) { for (const auto response : responses) { RETURN_IF_ERROR(SetBufferSize(response)); } return Status::Success; } Status CacheEntry::SetBufferSize(InferenceResponse* response) { if (!response) { return Status(Status::Code::INTERNAL, "response was nullptr"); } // The packed_response buffer will look like: // [num_outputs, sizeof(output1), output1, ..., sizeof(outputN), outputN] // So the entire packed_response_buffer will be the sum of all the output // sizes and the metadata included to parse them. uint64_t packed_response_byte_size = 0; // 1. First the packed buffer will hold the number of outputs as a uint32_t packed_response_byte_size += sizeof(uint32_t); // These sizes will be used to request allocated buffers from the cache // to copy directly into for (const auto& output : response->Outputs()) { uint64_t packed_output_byte_size = 0; RETURN_IF_ERROR(GetByteSize(output, &packed_output_byte_size)); // 2. Then the packed buffer will hold pairs of (output_size, output_bytes) packed_response_byte_size += sizeof(uint64_t); packed_response_byte_size += packed_output_byte_size; } AddPlaceholderBuffer(packed_response_byte_size); return Status::Success; } Status CacheEntry::SerializeResponses(boost::span responses) { if (buffers_.size() != responses.size()) { return Status( Status::Code::INTERNAL, "Expected number of responses in cache does not match. Expected: " + std::to_string(responses.size()) + ", received: " + std::to_string(buffers_.size())); } for (size_t i = 0; i < responses.size(); i++) { RETURN_IF_ERROR(SerializeResponse(responses[i], buffers_[i])); } return Status::Success; } Status CacheEntry::SerializeResponse(InferenceResponse* response, Buffer& buffer) { if (!response) { return Status(Status::Code::INTERNAL, "response was nullptr"); } // The packed_response buffer will look like: // [num_outputs, sizeof(output1), output1, ..., sizeof(outputN), outputN] size_t position = 0; Byte* base = static_cast(buffer.first); // 1. First the packed buffer will hold the number of outputs as a uint32_t uint32_t num_outputs = response->Outputs().size(); std::memcpy(base, &num_outputs, sizeof(uint32_t)); position += sizeof(uint32_t); for (const auto& output : response->Outputs()) { uint64_t packed_output_byte_size = 0; RETURN_IF_ERROR(SerializeResponseOutput( output, base + position, &packed_output_byte_size)); // 2. Then the packed buffer will hold pairs of (output_size, output_bytes) position += sizeof(packed_output_byte_size); position += packed_output_byte_size; } // Validate serialization fit expected size uint64_t total_buffer_size = buffer.second; if (position != total_buffer_size) { return Status( Status::Code::INTERNAL, "Serialized buffer size does not match. Expected: " + std::to_string(position) + ", received: " + std::to_string(total_buffer_size)); } return Status::Success; } Status CacheEntry::SerializeResponseOutput( const InferenceResponse::Output& output, Byte* buffer, size_t* output_size) { if (!buffer) { return Status(Status::Code::INVALID_ARG, "buffer arg was nullptr"); } // Fetch output buffer details const void* output_base = nullptr; size_t output_byte_size = 0; TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; int64_t memory_type_id = 0; void* userp = nullptr; auto status = output.DataBuffer( &output_base, &output_byte_size, &memory_type, &memory_type_id, &userp); if (!status.IsOk()) { return status; } // DLIS-2673: Add better memory_type support if (memory_type != TRITONSERVER_MEMORY_CPU && memory_type != TRITONSERVER_MEMORY_CPU_PINNED) { return Status( Status::Code::INVALID_ARG, "Only input buffers in CPU memory are allowed in cache currently"); } // Exit early if response buffer from output is invalid if (!output_base) { return Status( Status::Code::INTERNAL, "Response buffer from output was nullptr"); } size_t total_byte_size = 0; // Name std::string name = output.Name(); uint32_t name_byte_size = name.size(); total_byte_size += sizeof(uint32_t); total_byte_size += name_byte_size; // Dtype std::string dtype = triton::common::DataTypeToProtocolString(output.DType()); uint32_t dtype_byte_size = dtype.size(); total_byte_size += sizeof(uint32_t); total_byte_size += dtype_byte_size; // Shape std::vector shape = output.Shape(); uint32_t shape_byte_size = shape.size() * sizeof(int64_t); total_byte_size += sizeof(uint32_t); total_byte_size += shape_byte_size; // Output Buffer: Convert size_t to uint64_t for a fixed-size guarantee uint64_t u64_output_byte_size = static_cast(output_byte_size); total_byte_size += sizeof(uint64_t); total_byte_size += u64_output_byte_size; // Pack everything into provided buffer uint64_t position = 0; // Total serialized output size memcpy(buffer + position, &total_byte_size, sizeof(uint64_t)); position += sizeof(uint64_t); // Name memcpy(buffer + position, &name_byte_size, sizeof(uint32_t)); position += sizeof(uint32_t); memcpy(buffer + position, name.c_str(), name_byte_size); position += name_byte_size; // Dtype memcpy(buffer + position, &dtype_byte_size, sizeof(uint32_t)); position += sizeof(uint32_t); memcpy(buffer + position, dtype.c_str(), dtype_byte_size); position += dtype_byte_size; // Shape memcpy(buffer + position, &shape_byte_size, sizeof(uint32_t)); position += sizeof(uint32_t); memcpy(buffer + position, shape.data(), shape_byte_size); position += shape_byte_size; // Response Output Buffer memcpy(buffer + position, &u64_output_byte_size, sizeof(uint64_t)); position += sizeof(uint64_t); memcpy(buffer + position, output_base, u64_output_byte_size); position += u64_output_byte_size; // Used to increment global position in total response buffer *output_size = total_byte_size; return Status::Success; } Status CacheEntry::DeserializeBuffers(boost::span responses) { if (buffers_.size() != responses.size()) { return Status( Status::Code::INTERNAL, "Expected number of responses in cache does not match. Expected: " + std::to_string(responses.size()) + ", received: " + std::to_string(buffers_.size())); } for (size_t i = 0; i < responses.size(); i++) { RETURN_IF_ERROR(DeserializeBuffer(responses[i], buffers_[i])); } return Status::Success; } Status CacheEntry::DeserializeBuffer(InferenceResponse* response, const Buffer& buffer) { if (!response) { return Status(Status::Code::INTERNAL, "response was nullptr"); } const Byte* base = static_cast(buffer.first); if (!base) { return Status(Status::Code::INTERNAL, "buffer was nullptr"); } uint64_t position = 0; // Number of response outputs serialized in buffer uint32_t num_outputs = 0; std::memcpy(&num_outputs, base, sizeof(num_outputs)); position += sizeof(num_outputs); for (size_t i = 0; i < num_outputs; i++) { // Get size of packed output uint64_t packed_output_size = 0; std::memcpy( &packed_output_size, base + position, sizeof(packed_output_size)); position += sizeof(packed_output_size); // Parse packed output auto cache_output = CacheOutput(); RETURN_IF_ERROR(DeserializeResponseOutput( {base + position, packed_output_size}, &cache_output)); position += packed_output_size; InferenceResponse::Output* response_output = nullptr; RETURN_IF_ERROR(response->AddOutput( cache_output.name_, cache_output.dtype_, cache_output.shape_, &response_output)); if (!response_output) { return Status( Status::Code::INTERNAL, "InferenceResponse::Output pointer as nullptr"); } TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; int64_t memory_type_id = 0; // Allocate buffer for inference response void* output_buffer; RETURN_IF_ERROR(response_output->AllocateDataBuffer( &output_buffer, cache_output.byte_size_, &memory_type, &memory_type_id)); if (memory_type != TRITONSERVER_MEMORY_CPU && memory_type != TRITONSERVER_MEMORY_CPU_PINNED) { return Status( Status::Code::INTERNAL, "Only input buffers in CPU memory are allowed in cache currently"); } if (!output_buffer) { return Status( Status::Code::INTERNAL, "failed to allocate buffer for output '" + cache_output.name_ + "'"); } // COPY: cached output buffer to allocated response output buffer memcpy(output_buffer, cache_output.buffer_, cache_output.byte_size_); } return Status::Success; } Status CacheEntry::GetByteSize( const InferenceResponse::Output& output, uint64_t* packed_output_byte_size) { if (!packed_output_byte_size) { return Status(Status::Code::INVALID_ARG, "byte_size arg was null"); } // Fetch output buffer details const void* output_base = nullptr; size_t output_byte_size = 0; TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; int64_t memory_type_id = 0; void* userp = nullptr; auto status = output.DataBuffer( &output_base, &output_byte_size, &memory_type, &memory_type_id, &userp); if (!status.IsOk()) { return status; } // DLIS-2673: Add better memory_type support if (memory_type != TRITONSERVER_MEMORY_CPU && memory_type != TRITONSERVER_MEMORY_CPU_PINNED) { return Status( Status::Code::INVALID_ARG, "Only input buffers in CPU memory are allowed in cache currently"); } // Exit early if response buffer from output is invalid if (!output_base) { return Status( Status::Code::INTERNAL, "Response buffer from output was nullptr"); } size_t total_byte_size = 0; // Name std::string name = output.Name(); uint32_t name_byte_size = name.size(); total_byte_size += sizeof(uint32_t); total_byte_size += name_byte_size; // Dtype std::string dtype = triton::common::DataTypeToProtocolString(output.DType()); uint32_t dtype_byte_size = dtype.size(); total_byte_size += sizeof(uint32_t); total_byte_size += dtype_byte_size; // Shape std::vector shape = output.Shape(); uint32_t shape_byte_size = shape.size() * sizeof(int64_t); total_byte_size += sizeof(uint32_t); total_byte_size += shape_byte_size; // Output Buffer: Convert size_t to uint64_t for a fixed-size guarantee uint64_t u64_output_byte_size = static_cast(output_byte_size); total_byte_size += sizeof(uint64_t); total_byte_size += u64_output_byte_size; *packed_output_byte_size = total_byte_size; return Status::Success; } Status CacheEntry::DeserializeResponseOutput( boost::span packed_bytes, CacheOutput* output) { if (!output) { return Status(Status::Code::INVALID_ARG, "output arg was nullptr"); } // Name size_t position = 0; uint32_t name_byte_size = 0; memcpy(&name_byte_size, packed_bytes.begin() + position, sizeof(uint32_t)); position += sizeof(uint32_t); auto name_start = packed_bytes.begin() + position; std::string name(name_start, name_start + name_byte_size); position += name_byte_size; // Dtype uint32_t dtype_byte_size = 0; memcpy(&dtype_byte_size, packed_bytes.begin() + position, sizeof(uint32_t)); position += sizeof(uint32_t); auto dtype_start = packed_bytes.begin() + position; std::string dtype(dtype_start, dtype_start + dtype_byte_size); position += dtype_byte_size; // Shape uint32_t shape_byte_size = 0; memcpy(&shape_byte_size, packed_bytes.begin() + position, sizeof(uint32_t)); position += sizeof(uint32_t); std::vector shape(shape_byte_size / sizeof(int64_t), 0); memcpy(shape.data(), packed_bytes.begin() + position, shape_byte_size); position += shape_byte_size; // Output Buffer uint64_t output_byte_size = 0; memcpy(&output_byte_size, packed_bytes.begin() + position, sizeof(uint64_t)); position += sizeof(uint64_t); // NOTE: Reference buffer section of packed bytes directly, DO NOT copy here. // We will copy this buffer into the response object in BufferToResponse, so // the buffer must remain valid until then. They should remain valid until the // CacheEntry is destructed. const void* output_buffer = static_cast(packed_bytes.data() + position); position += output_byte_size; // Verify packed bytes matched expected size before allocating output_buffer if (packed_bytes.begin() + position != packed_bytes.end()) { return Status( Status::Code::INTERNAL, "Unexpected number of bytes received: " + std::to_string(packed_bytes.size()) + ", expected: " + std::to_string(position)); } output->name_ = name; output->dtype_ = triton::common::ProtocolStringToDataType(dtype); output->shape_ = shape; output->byte_size_ = output_byte_size; output->buffer_ = const_cast(output_buffer); return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/cache_entry.h000066400000000000000000000124751527052371200200220ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "infer_response.h" #include "triton/common/logging.h" #include "tritonserver_apis.h" namespace triton { namespace core { struct CacheOutput { // Inference Response output name std::string name_ = ""; // Inference Response output datatype inference::DataType dtype_; // Inference Response output shape std::vector shape_; // Inference Response output buffer // NOTE: Cache Output will only temporarily store pointer, it will managed by // CacheEntry, and will be copied into InferenceResponse object void* buffer_ = nullptr; // Inference Response output buffer size uint64_t byte_size_ = 0; }; // A Buffer is an arbitrary data blob whose type need not be known // by the cache for storage and retrieval. using Buffer = std::pair; // A CacheEntrymay have several Buffers associated with it // ex: one request may have multiple responses class CacheEntry { public: ~CacheEntry(); const std::vector& Buffers(); std::vector& MutableBuffers(); size_t BufferCount(); void AddBuffer(boost::span buffer); void AddBuffer(void* base, size_t byte_size); /* Insert helpers */ Status SerializeResponses(boost::span responses); // Adds a placeholder buffer to this CacheEntry object containing the // necessary size to hold the data we plan to insert into the cache. // - The 'nullptr' base indicates to the cache implementation that we want // the cache to allocate the buffer and overwrite this buffer's base address // with the cache-allocated buffer's base address in-place. // - This lets Triton copy directly into the cache allocated buffer through // the TRITONCACHE_Copy callback. void AddPlaceholderBuffer(size_t byte_size); // Calculates serialized response size to request allocated buffer from cache // and sets entry buffer sizes Status SetBufferSizes(boost::span responses); // Directly sets entry buffer sizes from provided buffers Status SetBufferSizes(std::vector> buffers); /* Lookup helpers */ Status DeserializeBuffers(boost::span responses); // Typically, the cache entry will now own any associated buffers. // However, if a CacheAllocator wants the entry to own the buffers, this // can be used to signal that the entry should free its buffers on destruction void FreeBuffersOnExit() { free_buffers_ = true; } private: // Insert helpers Status SerializeResponse(InferenceResponse* response, Buffer& buffer); Status SerializeResponseOutput( const InferenceResponse::Output& output, Byte* buffer, size_t* output_size); Status SetBufferSize(InferenceResponse* response); // Calculates total byte size required to serialize response output and // returns it in packed_output_byte_size Status GetByteSize( const InferenceResponse::Output& output, uint64_t* packed_output_byte_size); // Lookup helpers Status DeserializeBuffer(InferenceResponse* response, const Buffer& buffer); Status DeserializeResponseOutput( boost::span packed_bytes, CacheOutput* output); // NOTE: performance gain may be possible by removing this mutex and // guaranteeing that no two threads will access/modify an entry // in parallel. This may be guaranteed by documenting that a cache // implementation should not call TRITONCACHE_CacheEntryAddBuffer or // TRITONCACHE_CacheEntryBuffer on the same entry in parallel. // This will remain for simplicity until further profiling is done. std::mutex buffer_mu_; std::vector buffers_; // Free buffers on exit, default is false unless explicitly toggled bool free_buffers_ = false; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/cache_manager.cc000066400000000000000000000402071527052371200204230ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "cache_manager.h" #include #include "cache_entry.h" #include "filesystem/api.h" #include "server_message.h" #include "shared_library.h" #include "triton/common/logging.h" namespace triton { namespace core { std::string TritonCacheLibraryName(const std::string& cache_name) { #ifdef _WIN32 return std::string("tritoncache_") + cache_name + ".dll"; #else return std::string("libtritoncache_") + cache_name + ".so"; #endif } // // TritonCacheAllocator // CacheToResponseAllocator::CacheToResponseAllocator( boost::span responses) { for (const auto& response : responses) { responses_.push_back(response); } } Status CacheToResponseAllocator::Allocate(TRITONCACHE_CacheEntry* entry) { if (!entry) { return Status(Status::Code::INVALID_ARG, "entry is nullptr"); } const auto lentry = reinterpret_cast(entry); // Parse cache buffers into responses RETURN_IF_ERROR(lentry->DeserializeBuffers(responses_)); return Status::Success; } // NOTE: Bytes-related allocators only used for unit testing currently Status CacheToBytesAllocator::Allocate(TRITONCACHE_CacheEntry* entry) { if (!entry) { return Status(Status::Code::INVALID_ARG, "entry is nullptr"); } const auto lentry = reinterpret_cast(entry); auto& buffers = lentry->MutableBuffers(); // FIXME: This allocator could be changed in the future to take a vector of // destination buffers on construction and copy into those instead. Currently // this allocator gives buffer ownership to the CacheEntry object, rather than // only using the entry for metadata/communication like everywhere else. // NOTE: If the same entry object is re-used for multiple lookups and the // entry buffers are not freed between uses, this will leak memory. for (auto& iter : buffers) { auto& base = iter.first; const auto& byte_size = iter.second; void* new_base = malloc(byte_size); std::memcpy(new_base, base, byte_size); base = new_base; } // Signal entry to free these buffers on destruction lentry->FreeBuffersOnExit(); return Status::Success; } BytesToCacheAllocator::BytesToCacheAllocator( std::vector> buffers) { // Span is a read-only view of the underlying buffer, this should only // perform a shallow copy of base pointer and size of each span. buffers_ = buffers; } Status BytesToCacheAllocator::Allocate(TRITONCACHE_CacheEntry* entry) { if (!entry) { return Status(Status::Code::INVALID_ARG, "entry is nullptr"); } const auto lentry = reinterpret_cast(entry); auto& cache_buffers = lentry->MutableBuffers(); if (cache_buffers.size() != buffers_.size()) { return Status( Status::Code::INTERNAL, "Expected number of buffers in cache does not match. Expected: " + std::to_string(buffers_.size()) + ", received: " + std::to_string(cache_buffers.size())); } // Copy from allocator provided buffers into cache-allocated buffers // that were setup in 'entry' by cache implementation. for (size_t i = 0; i < buffers_.size(); i++) { auto cache_buffer = cache_buffers[i].first; auto cache_buffer_size = cache_buffers[i].second; if (buffers_[i].size() != cache_buffer_size) { return Status( Status::Code::INTERNAL, "Expected size of buffer in cache does not match. Expected: " + std::to_string(buffers_[i].size()) + ", received: " + std::to_string(cache_buffer_size)); } std::memcpy(cache_buffer, buffers_[i].data(), cache_buffer_size); } return Status::Success; } ResponseToCacheAllocator::ResponseToCacheAllocator( boost::span responses) { for (const auto& response : responses) { responses_.push_back(response); } } Status ResponseToCacheAllocator::Allocate(TRITONCACHE_CacheEntry* entry) { const auto lentry = reinterpret_cast(entry); RETURN_IF_ERROR(lentry->SerializeResponses(responses_)); return Status::Success; } // // TritonCache // Status TritonCache::Create( const std::string& name, const std::string& libpath, const std::string& cache_config, std::shared_ptr* cache) { LOG_INFO << "Creating TritonCache with name: '" << name << "', libpath: '" << libpath << "', cache_config: '" << cache_config << "'"; auto lcache = std::shared_ptr( new TritonCache(name, libpath, cache_config)); RETURN_IF_ERROR(lcache->LoadCacheLibrary()); RETURN_IF_ERROR(lcache->InitializeCacheImpl()); *cache = std::move(lcache); return Status::Success; } TritonCache::TritonCache( const std::string& name, const std::string& libpath, const std::string& cache_config) : name_(name), libpath_(libpath), cache_config_(cache_config) { ClearHandles(); } TritonCache::~TritonCache() { LOG_VERBOSE(1) << "unloading cache '" << name_ << "'"; if (fini_fn_) { if (cache_impl_) { LOG_VERBOSE(1) << "Calling TRITONCACHE_CacheFinalize from: '" << libpath_ << "'"; LOG_TRITONSERVER_ERROR(fini_fn_(cache_impl_), "failed finalizing cache"); } else { LOG_ERROR << "cache implementation handle is nullptr"; } } else { LOG_ERROR << "cache finalize function is nullptr"; } if (dlhandle_) { std::unique_ptr slib; LOG_STATUS_ERROR(SharedLibrary::Acquire(&slib), "~TritonCache"); LOG_STATUS_ERROR(slib->CloseLibraryHandle(dlhandle_), "~TritonCache"); } ClearHandles(); } void TritonCache::ClearHandles() { dlhandle_ = nullptr; cache_impl_ = nullptr; init_fn_ = nullptr; fini_fn_ = nullptr; lookup_fn_ = nullptr; insert_fn_ = nullptr; } Status TritonCache::LoadCacheLibrary() { LOG_VERBOSE(1) << "Loading cache library: '" << name_ << "' from: '" << libpath_ << "'"; TritonCacheInitFn_t init_fn; TritonCacheFiniFn_t fini_fn; TritonCacheLookupFn_t lookup_fn; TritonCacheInsertFn_t insert_fn; // Load the library and initialize all the entrypoints { std::unique_ptr slib; RETURN_IF_ERROR(SharedLibrary::Acquire(&slib)); RETURN_IF_ERROR(slib->OpenLibraryHandle(libpath_, &dlhandle_)); // Cache initialize and finalize functions, required RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONCACHE_CacheInitialize", false /* optional */, reinterpret_cast(&init_fn))); RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONCACHE_CacheFinalize", false /* optional */, reinterpret_cast(&fini_fn))); RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONCACHE_CacheLookup", false /* optional */, reinterpret_cast(&lookup_fn))); RETURN_IF_ERROR(slib->GetEntrypoint( dlhandle_, "TRITONCACHE_CacheInsert", false /* optional */, reinterpret_cast(&insert_fn))); } init_fn_ = init_fn; fini_fn_ = fini_fn; lookup_fn_ = lookup_fn; insert_fn_ = insert_fn; return Status::Success; } Status TritonCache::InitializeCacheImpl() { if (init_fn_ == nullptr) { return Status(Status::Code::NOT_FOUND, "cache init function is nullptr"); } // Initialize cache implementation RETURN_IF_TRITONSERVER_ERROR(init_fn_(&cache_impl_, cache_config_.c_str())); if (!cache_impl_) { return Status( Status::Code::INTERNAL, "Failed to initialize cache implementation"); } return Status::Success; } Status TritonCache::HashInputBuffers( const InferenceRequest::Input* input, size_t* seed) { // Iterate over each data buffer in input in case of non-contiguous memory for (size_t idx = 0; idx < input->DataBufferCount(); ++idx) { const void* src_buffer; size_t src_byte_size; TRITONSERVER_MemoryType src_memory_type; int64_t src_memory_type_id; RETURN_IF_ERROR(input->DataBuffer( idx, &src_buffer, &src_byte_size, &src_memory_type, &src_memory_type_id)); if (src_memory_type != TRITONSERVER_MEMORY_CPU && src_memory_type != TRITONSERVER_MEMORY_CPU_PINNED) { return Status( Status::Code::INTERNAL, "Only input buffers in CPU memory are allowed in cache currently"); } // Add each byte of input buffer chunk to hash const unsigned char* tmp = static_cast(src_buffer); for (uint64_t byte = 0; byte < src_byte_size; byte++) { boost::hash_combine(*seed, tmp[byte]); } } return Status::Success; } Status TritonCache::HashInputs(const InferenceRequest& request, size_t* seed) { const auto& inputs = request.ImmutableInputs(); // Convert inputs to ordered map for consistency in hashing // inputs sorted by key (input) name std::map ordered_inputs( inputs.begin(), inputs.end()); for (const auto& input : ordered_inputs) { // Add input name to hash boost::hash_combine(*seed, input.second->Name()); // Fetch input buffer for hashing raw data RETURN_IF_ERROR(HashInputBuffers(input.second, seed)); } return Status::Success; } Status TritonCache::Hash(const InferenceRequest& request, std::string* key) { std::size_t seed = 0; // Add request model name to hash boost::hash_combine(seed, request.ModelName()); // Add request model version to hash boost::hash_combine(seed, request.ActualModelVersion()); RETURN_IF_ERROR(HashInputs(request, &seed)); // NOTE: Could prepend model name/version in key for readability/debugging *key = std::to_string(seed); return Status::Success; } Status TritonCache::Insert( CacheEntry* entry, const std::string& key, TRITONCACHE_Allocator* allocator) { LOG_VERBOSE(2) << "Inserting at cache key: " << key; if (insert_fn_ == nullptr) { return Status(Status::Code::INTERNAL, "cache insert function is nullptr"); } if (allocator == nullptr) { return Status(Status::Code::INVALID_ARG, "allocator is nullptr"); } const auto opaque_entry = reinterpret_cast(entry); RETURN_IF_TRITONSERVER_ERROR( insert_fn_(cache_impl_, key.c_str(), opaque_entry, allocator)); return Status::Success; } Status TritonCache::Insert( std::vector> buffers, const std::string& key) { std::unique_ptr entry = std::make_unique(); RETURN_IF_ERROR(entry->SetBufferSizes(buffers)); auto allocator = BytesToCacheAllocator(buffers); auto opaque_allocator = reinterpret_cast(&allocator); return Insert(entry.get(), key, opaque_allocator); } Status TritonCache::Insert( boost::span responses, const std::string& key) { std::unique_ptr entry = std::make_unique(); RETURN_IF_ERROR(entry->SetBufferSizes(responses)); auto allocator = ResponseToCacheAllocator(responses); auto opaque_allocator = reinterpret_cast(&allocator); return Insert(entry.get(), key, opaque_allocator); } Status TritonCache::Insert(InferenceResponse* response, const std::string& key) { if (!response) { return Status(Status::Code::INVALID_ARG, "response is nullptr"); } return Insert({&response, 1}, key); } Status TritonCache::Lookup( const std::string& key, CacheEntry* entry, TRITONCACHE_Allocator* allocator) { LOG_VERBOSE(2) << "Looking up cache key: " << key; if (lookup_fn_ == nullptr) { return Status(Status::Code::INTERNAL, "lookup function is nullptr"); } if (allocator == nullptr) { return Status(Status::Code::INVALID_ARG, "allocator is nullptr"); } auto opaque_entry = reinterpret_cast(entry); RETURN_IF_TRITONSERVER_ERROR( lookup_fn_(cache_impl_, key.c_str(), opaque_entry, allocator)); return Status::Success; } // NOTE: Used for unit testing Status TritonCache::Lookup(const std::string& key, CacheEntry* entry) { // Create allocator to copy directly from cache to byte buffers auto allocator = CacheToBytesAllocator(); auto opaque_allocator = reinterpret_cast(&allocator); return Lookup(key, entry, opaque_allocator); } // NOTE: Multiple responses won't be expected until supporting decoupled // or sequence models. Status TritonCache::Lookup( boost::span responses, const std::string& key) { auto lentry = std::make_unique(); // Create response allocator to copy directly from cache to response buffers auto allocator = CacheToResponseAllocator(responses); auto opaque_allocator = reinterpret_cast(&allocator); RETURN_IF_ERROR(Lookup(key, lentry.get(), opaque_allocator)); return Status::Success; } Status TritonCache::Lookup(InferenceResponse* response, const std::string& key) { if (!response) { return Status(Status::Code::INVALID_ARG, "response is nullptr"); } return Lookup({&response, 1}, key); } // // TritonCacheManager // static std::weak_ptr cache_manager_; static std::mutex mu_; Status TritonCacheManager::Create( std::shared_ptr* manager, std::string cache_dir) { std::lock_guard lock(mu_); // If there is already a manager then we just use it *manager = cache_manager_.lock(); if (*manager != nullptr) { return Status::Success; } if (cache_dir.empty()) { return Status( Status::Code::INVALID_ARG, "cache directory can not be empty"); } LOG_VERBOSE(1) << "Create CacheManager with cache_dir: '" << cache_dir << "'"; manager->reset(new TritonCacheManager(cache_dir)); cache_manager_ = *manager; return Status::Success; } Status TritonCacheManager::CreateCache( const std::string& name, const std::string& cache_config, std::shared_ptr* cache) { std::lock_guard lock(mu_); if (cache_ != nullptr) { return Status( Status::Code::ALREADY_EXISTS, "TritonCacheManager already holds a cache"); } // Get the path to the cache shared library. Search path is global // cache directory. const std::vector search_paths = {JoinPath({cache_dir_, name})}; // Triton will only use a single cache library path for now, // regardless of implementation. std::string cache_libname = TritonCacheLibraryName(name); std::string libpath = ""; for (const auto& path : search_paths) { const auto full_path = JoinPath({path, cache_libname}); bool exists = false; RETURN_IF_ERROR(FileExists(full_path, &exists)); if (exists) { libpath = full_path; break; } } if (libpath.empty()) { return Status( Status::Code::INVALID_ARG, "unable to find '" + cache_libname + "' for cache. Searched: " + cache_dir_); } RETURN_IF_ERROR(TritonCache::Create(name, libpath, cache_config, &cache_)); *cache = cache_; return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/cache_manager.h000066400000000000000000000145121527052371200202650ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include "cache_entry.h" #include "constants.h" #include "infer_request.h" #include "infer_response.h" #include "server_message.h" #include "status.h" #include "triton/common/model_config.h" #include "tritonserver_apis.h" namespace triton { namespace core { // // Custom Allocators to copy directly between Cache buffers <-> Triton buffers // and avoid intermediate copies on Insert/Lookup. // class TritonCacheAllocator { public: virtual Status Allocate(TRITONCACHE_CacheEntry* entry) = 0; }; class CacheToResponseAllocator : TritonCacheAllocator { public: CacheToResponseAllocator(boost::span responses); Status Allocate(TRITONCACHE_CacheEntry* entry); private: std::vector responses_; }; class ResponseToCacheAllocator : TritonCacheAllocator { public: ResponseToCacheAllocator(boost::span responses); Status Allocate(TRITONCACHE_CacheEntry* entry); private: std::vector responses_; }; // NOTE: Bytes-related allocators only used for unit testing currently class CacheToBytesAllocator : TritonCacheAllocator { public: Status Allocate(TRITONCACHE_CacheEntry* entry); }; class BytesToCacheAllocator : TritonCacheAllocator { public: BytesToCacheAllocator(std::vector> buffers); Status Allocate(TRITONCACHE_CacheEntry* entry); private: std::vector> buffers_; }; // // Proxy to a cache shared library. // class TritonCache { public: static Status Create( const std::string& name, const std::string& libpath, const std::string& cache_config, std::shared_ptr* cache); ~TritonCache(); const std::string& Name() const { return name_; } const std::string& CacheConfig() const { return cache_config_; } Status Insert(InferenceResponse* response, const std::string& key); Status Insert( boost::span responses, const std::string& key); Status Insert(std::vector> buffers, const std::string& key); Status Insert( CacheEntry* entry, const std::string& key, TRITONCACHE_Allocator* allocator); Status Lookup(InferenceResponse* response, const std::string& key); Status Lookup( boost::span responses, const std::string& key); Status Lookup(const std::string& key, CacheEntry* entry); Status Lookup( const std::string& key, CacheEntry* entry, TRITONCACHE_Allocator* allocator); // Hashes fields of request and stores it in "key" Status Hash(const InferenceRequest& request, std::string* key); private: TritonCache( const std::string& name, const std::string& libpath, const std::string& cache_config); void ClearHandles(); Status LoadCacheLibrary(); Status InitializeCacheImpl(); // Helper function to hash data buffers used by "input" Status HashInputBuffers(const InferenceRequest::Input* input, size_t* seed); // Helper function to hash each input in "request" Status HashInputs(const InferenceRequest& request, size_t* seed); // The name of the cache. const std::string name_; // Full path to the cache shared library. const std::string libpath_; // Cache configuration as JSON string const std::string cache_config_; // Cache Implementation TRITONCACHE_Cache* cache_impl_; // dlopen / dlsym handles void* dlhandle_; typedef TRITONSERVER_Error* (*TritonCacheInitFn_t)( TRITONCACHE_Cache** cache, const char* config); TritonCacheInitFn_t init_fn_; typedef TRITONSERVER_Error* (*TritonCacheFiniFn_t)(TRITONCACHE_Cache* cache); TritonCacheFiniFn_t fini_fn_; typedef TRITONSERVER_Error* (*TritonCacheLookupFn_t)( TRITONCACHE_Cache* cache, const char* key, TRITONCACHE_CacheEntry* entry, TRITONCACHE_Allocator* allocator); TritonCacheLookupFn_t lookup_fn_; typedef TRITONSERVER_Error* (*TritonCacheInsertFn_t)( TRITONCACHE_Cache* cache, const char* key, TRITONCACHE_CacheEntry* entry, TRITONCACHE_Allocator* allocator); TritonCacheInsertFn_t insert_fn_; }; // // Manage communication with Triton caches and their lifecycle. // class TritonCacheManager { public: static Status Create( std::shared_ptr* manager, std::string cache_dir); Status CreateCache( const std::string& name, const std::string& cache_config, std::shared_ptr* cache); std::shared_ptr Cache() { return cache_; } private: DISALLOW_COPY_AND_ASSIGN(TritonCacheManager); TritonCacheManager(std::string cache_dir) : cache_dir_(cache_dir) {} // Global search path for cache libraries std::string cache_dir_; // This may be a map of caches in the future std::shared_ptr cache_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/constants.h000066400000000000000000000122201527052371200175360ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include namespace triton { namespace core { constexpr char kInferHeaderContentLengthHTTPHeader[] = "Inference-Header-Content-Length"; constexpr char kAcceptEncodingHTTPHeader[] = "Accept-Encoding"; constexpr char kContentEncodingHTTPHeader[] = "Content-Encoding"; constexpr char kContentTypeHeader[] = "Content-Type"; constexpr char kContentLengthHeader[] = "Content-Length"; constexpr char kTensorFlowGraphDefPlatform[] = "tensorflow_graphdef"; constexpr char kTensorFlowSavedModelPlatform[] = "tensorflow_savedmodel"; constexpr char kTensorFlowGraphDefFilename[] = "model.graphdef"; constexpr char kTensorFlowSavedModelFilename[] = "model.savedmodel"; constexpr char kTensorFlowBackend[] = "tensorflow"; constexpr char kTensorRTPlanPlatform[] = "tensorrt_plan"; constexpr char kTensorRTPlanFilename[] = "model.plan"; constexpr char kTensorRTBackend[] = "tensorrt"; constexpr char kOnnxRuntimeOnnxPlatform[] = "onnxruntime_onnx"; constexpr char kOnnxRuntimeOnnxFilename[] = "model.onnx"; constexpr char kOnnxRuntimeBackend[] = "onnxruntime"; constexpr char kOpenVINORuntimeOpenVINOFilename[] = "model.xml"; constexpr char kOpenVINORuntimeBackend[] = "openvino"; constexpr char kPyTorchLibTorchPlatform[] = "pytorch_libtorch"; constexpr char kPyTorchLibTorchFilename[] = "model.pt"; constexpr char kPyTorchBackend[] = "pytorch"; constexpr char kPythonFilename[] = "model.py"; constexpr char kPythonBackend[] = "python"; #ifdef TRITON_ENABLE_ENSEMBLE constexpr char kEnsemblePlatform[] = "ensemble"; #endif // TRITON_ENABLE_ENSEMBLE constexpr char kTensorRTExecutionAccelerator[] = "tensorrt"; constexpr char kOpenVINOExecutionAccelerator[] = "openvino"; constexpr char kCUDAExecutionAccelerator[] = "cuda"; constexpr char kGPUIOExecutionAccelerator[] = "gpu_io"; constexpr char kAutoMixedPrecisionExecutionAccelerator[] = "auto_mixed_precision"; constexpr char kModelConfigPbTxt[] = "config.pbtxt"; constexpr char kPbTxtExtension[] = ".pbtxt"; constexpr char kModelConfigFolder[] = "configs"; constexpr char kMetricsLabelModelNamespace[] = "namespace"; constexpr char kMetricsLabelModelName[] = "model"; constexpr char kMetricsLabelModelVersion[] = "version"; constexpr char kMetricsLabelGpuUuid[] = "gpu_uuid"; constexpr char kWarmupDataFolder[] = "warmup"; constexpr char kInitialStateFolder[] = "initial_state"; // Metric names constexpr char kPendingRequestMetric[] = "inf_pending_request_count"; constexpr uint64_t NANOS_PER_SECOND = 1000000000; constexpr uint64_t NANOS_PER_MILLIS = 1000000; constexpr int MAX_GRPC_MESSAGE_SIZE = INT32_MAX; constexpr uint64_t SEQUENCE_IDLE_DEFAULT_MICROSECONDS = 1000 * 1000; constexpr size_t STRING_CORRELATION_ID_MAX_LENGTH_BYTES = 128; constexpr size_t CUDA_IPC_STRUCT_SIZE = 64; #ifdef TRITON_ENABLE_METRICS // MetricModelReporter expects a device ID for GPUs, but we reuse this device // ID for other metrics as well such as for CPU and Response Cache metrics constexpr int METRIC_REPORTER_ID_CPU = -1; constexpr int METRIC_REPORTER_ID_UTILITY = -2; #endif // Note: This can be replaced with std::byte starting in c++17 using Byte = unsigned char; #define TIMESPEC_TO_NANOS(TS) \ ((TS).tv_sec * triton::core::NANOS_PER_SECOND + (TS).tv_nsec) #define TIMESPEC_TO_MILLIS(TS) \ (TIMESPEC_TO_NANOS(TS) / triton::core::NANOS_PER_MILLIS) #define DISALLOW_MOVE(TypeName) TypeName(Context&& o) = delete; #define DISALLOW_COPY(TypeName) TypeName(const TypeName&) = delete; #define DISALLOW_ASSIGN(TypeName) void operator=(const TypeName&) = delete; #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ DISALLOW_COPY(TypeName) \ DISALLOW_ASSIGN(TypeName) }} // namespace triton::core kytensor-2.49.0.7/core/src/cuda_block_manager.cc000066400000000000000000000122471527052371200214510ustar00rootroot00000000000000// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #include "cuda_block_manager.h" #include "cuda_utils.h" namespace triton { namespace core { std::unique_ptr CudaBlockManager::instance_ = nullptr; void Allocation::Merge(std::unique_ptr&& allocation) { for (auto block : allocation->Blocks()) { blocks_.push_back(block); } allocation->Blocks().clear(); } Status CudaBlockManager::Create(double min_supported_compute_capability) { if (instance_ != nullptr) { return Status( Status::Code::INTERNAL, (std::string("CUDA Block manager has already been created."))); } std::set supported_gpus; RETURN_IF_ERROR( GetSupportedGPUs(&supported_gpus, min_supported_compute_capability)); size_t block_size; RETURN_IF_ERROR(GetAllocationGranularity(block_size)); instance_.reset(new CudaBlockManager()); instance_->block_size_ = block_size; for (auto gpu : supported_gpus) { instance_->free_blocks_[gpu] = std::vector(); } return Status::Success; } Status CudaBlockManager::Allocate( size_t size, std::unique_ptr& allocation, int device_id) { if (instance_ == nullptr) { return Status( Status::Code::INTERNAL, (std::string("CUDA Block manager has not been created."))); } std::lock_guard lock(instance_->mu_); if (instance_->free_blocks_.count(device_id) == 0) { return Status( Status::Code::INTERNAL, (std::string("Invalid device id '") + std::to_string(device_id) + "' to allocate memory from.")); } // Align the byte size to number of blocks. The allocated bytes must be a // multiple of the cudaAllocationGranularity. In the current implementation // the block size is the cuda minimum allocation granularity. size_t num_blocks = (size + instance_->block_size_ - 1) / (instance_->block_size_); size_t free_blocks_size = instance_->free_blocks_[device_id].size(); size_t block_index = 0; for (; block_index < num_blocks && block_index < free_blocks_size; ++block_index) { allocation->AddBlock(instance_->free_blocks_[device_id].back()); instance_->free_blocks_[device_id].pop_back(); } for (; block_index < num_blocks; ++block_index) { CUmemGenericAllocationHandle block{}; CUmemAllocationProp prop = {}; prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; prop.location.id = device_id; RETURN_IF_ERROR(CudaDriverHelper::GetInstance().CuMemCreate( &block, instance_->block_size_, &prop, 0 /* flags */)); allocation->AddBlock(block); } return Status::Success; } Status CudaBlockManager::Free(Allocation* allocation, int device_id) { if (instance_ == nullptr) { return Status( Status::Code::INTERNAL, (std::string("CUDA Block manager has not been created."))); } std::lock_guard lock(instance_->mu_); if (instance_->free_blocks_.count(device_id) == 0) { return Status( Status::Code::INTERNAL, (std::string("Invalid device id '") + std::to_string(device_id) + "' to allocate memory from.")); } auto& free_blocks = instance_->free_blocks_[device_id]; free_blocks.insert( free_blocks.end(), allocation->Blocks().begin(), allocation->Blocks().end()); allocation->Blocks().clear(); return Status::Success; } CudaBlockManager::~CudaBlockManager() { std::lock_guard lock(mu_); for (auto& free_block_pair : free_blocks_) { auto free_blocks = free_block_pair.second; for (auto& free_block : free_blocks) { CudaDriverHelper::GetInstance().CuMemRelease(free_block); } } } }}; // namespace triton::core kytensor-2.49.0.7/core/src/cuda_block_manager.h000066400000000000000000000070261527052371200213120ustar00rootroot00000000000000// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once #include #include #include #include #include #include #include "constants.h" #include "status.h" namespace triton { namespace core { class Allocation; /// /// CudaBlockManager uses CUDA Virtual Memory Management APIs to allow the /// buffers to grow without having to create a new allocation. Currently, /// it is only used with implicit state management. /// class CudaBlockManager { public: static Status Create(double min_supported_compute_capability); static Status Allocate( size_t size, std::unique_ptr& allocation, int device_id); static Status Free(Allocation* allocation, int device_id); static size_t BlockSize() { return instance_->block_size_; } ~CudaBlockManager(); static void Reset() { instance_.reset(); } private: CudaBlockManager(){}; std::unordered_map> free_blocks_; size_t block_size_; std::mutex mu_; static std::unique_ptr instance_; }; /// /// Representing an Allocation object returned from CudaBlockManager. /// class Allocation { public: Allocation(int device_id) : device_id_(device_id) {} /// Add a block to an existing allocation. /// /// \param block The memory block to be added. /// void AddBlock(CUmemGenericAllocationHandle block) { blocks_.push_back(block); } /// Merge one allocation with another one. /// /// \param allocation The other allocation to be merged with. /// void Merge(std::unique_ptr&& allocation); /// Get the list of all the memory blocks corresponding to this /// allocation. /// /// \return The list of all the memory blocks. std::vector& Blocks() { return blocks_; } ~Allocation() { CudaBlockManager::Free(this, device_id_); } private: DISALLOW_COPY_AND_ASSIGN(Allocation); std::vector blocks_; int device_id_; }; }}; // namespace triton::core kytensor-2.49.0.7/core/src/cuda_memory_manager.cc000066400000000000000000000146701527052371200216710ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #include "cuda_memory_manager.h" #include #include #include #include "cuda_utils.h" #include "triton/common/logging.h" namespace { #define RETURN_IF_CNMEM_ERROR(S, MSG) \ do { \ auto status__ = (S); \ if (status__ != CNMEM_STATUS_SUCCESS) { \ return Status( \ Status::Code::INTERNAL, \ (MSG) + ": " + cnmemGetErrorString(status__)); \ } \ } while (false) std::string PointerToString(void* ptr) { std::stringstream ss; ss << ptr; return ss.str(); } } // namespace namespace triton { namespace core { std::unique_ptr CudaMemoryManager::instance_; std::mutex CudaMemoryManager::instance_mu_; CudaMemoryManager::~CudaMemoryManager() { if (has_allocation_) { auto status = cnmemFinalize(); if (status != CNMEM_STATUS_SUCCESS) { LOG_ERROR << "Failed to finalize CUDA memory manager: [" << status << "] " << cnmemGetErrorString(status); } } } void CudaMemoryManager::Reset() { std::lock_guard lock(instance_mu_); instance_.reset(); } Status CudaMemoryManager::Create(const CudaMemoryManager::Options& options) { // Ensure thread-safe creation of CUDA memory pool std::lock_guard lock(instance_mu_); if (instance_ != nullptr) { LOG_WARNING << "New CUDA memory pools could not be created since they " "already exists"; return Status::Success; } std::set supported_gpus; auto status = GetSupportedGPUs( &supported_gpus, options.min_supported_compute_capability_); if (status.IsOk()) { std::vector devices; for (auto gpu : supported_gpus) { const auto it = options.memory_pool_byte_size_.find(gpu); if ((it != options.memory_pool_byte_size_.end()) && (it->second != 0)) { devices.emplace_back(); auto& device = devices.back(); memset(&device, 0, sizeof(device)); device.device = gpu; device.size = it->second; LOG_INFO << "CUDA memory pool is created on device " << device.device << " with size " << device.size; } } if (!devices.empty()) { RETURN_IF_CNMEM_ERROR( cnmemInit(devices.size(), devices.data(), CNMEM_FLAGS_CANNOT_GROW), std::string("Failed to finalize CUDA memory manager")); } else { LOG_INFO << "CUDA memory pool disabled"; } // Use to finalize CNMeM properly when out of scope instance_.reset(new CudaMemoryManager(!devices.empty())); } else { return Status( status.ErrorCode(), "Failed to initialize CUDA memory manager: " + status.Message()); } return Status::Success; } Status CudaMemoryManager::Alloc(void** ptr, uint64_t size, int64_t device_id) { if (instance_ == nullptr) { return Status( Status::Code::UNAVAILABLE, "CudaMemoryManager has not been created"); } else if (!instance_->has_allocation_) { return Status( Status::Code::UNAVAILABLE, "CudaMemoryManager has no preallocated CUDA memory"); } int current_device; RETURN_IF_CUDA_ERR( cudaGetDevice(¤t_device), std::string("Failed to get device")); bool overridden = (current_device != device_id); if (overridden) { RETURN_IF_CUDA_ERR( cudaSetDevice(device_id), std::string("Failed to set device")); } // Defer returning error to make sure the device is recovered auto err = cnmemMalloc(ptr, size, nullptr); if (overridden) { cudaSetDevice(current_device); } RETURN_IF_CNMEM_ERROR( err, std::string("Failed to allocate CUDA memory with byte size ") + std::to_string(size) + " on GPU " + std::to_string(device_id)); return Status::Success; } Status CudaMemoryManager::Free(void* ptr, int64_t device_id) { if (instance_ == nullptr) { return Status( Status::Code::UNAVAILABLE, "CudaMemoryManager has not been created"); } else if (!instance_->has_allocation_) { return Status( Status::Code::UNAVAILABLE, "CudaMemoryManager has no preallocated CUDA memory"); } int current_device; RETURN_IF_CUDA_ERR( cudaGetDevice(¤t_device), std::string("Failed to get device")); bool overridden = (current_device != device_id); if (overridden) { RETURN_IF_CUDA_ERR( cudaSetDevice(device_id), std::string("Failed to set device")); } // Defer returning error to make sure the device is recovered auto err = cnmemFree(ptr, nullptr); if (overridden) { cudaSetDevice(current_device); } RETURN_IF_CNMEM_ERROR( err, std::string("Failed to deallocate CUDA memory at address ") + PointerToString(ptr) + " on GPU " + std::to_string(device_id)); return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/cuda_memory_manager.h000066400000000000000000000067471527052371200215410ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once #include #include #include #include "status.h" namespace triton { namespace core { // This is a singleton class responsible for maintaining CUDA memory pool // used by the inference server. CUDA memory allocations and deallocations // must be requested via functions provided by this class. class CudaMemoryManager { public: // Options to configure CUDA memory manager. struct Options { Options(double cc = 6.0, const std::map& s = {}) : min_supported_compute_capability_(cc), memory_pool_byte_size_(s) { } // The minimum compute capability of the supported devices. double min_supported_compute_capability_; // The size of CUDA memory reserved for the specified devices. // The memory size will be rounded up to align with // the default granularity (512 bytes). // No memory will be reserved for devices that is not listed. std::map memory_pool_byte_size_; }; ~CudaMemoryManager(); // Create the memory manager based on 'options' specified. // Return Status object indicating success or failure. static Status Create(const Options& options); // Provide explicit control on ending the memory manager lifecycle, // CUDA resource must be cleaned up before CUDA context is destroyed. static void Reset(); // Allocate CUDA memory on GPU 'device_id' with // the requested 'size' and return the pointer in 'ptr'. // Return Status object indicating success or failure. static Status Alloc(void** ptr, uint64_t size, int64_t device_id); // Free the memory allocated by the memory manager on 'device_id'. // Return Status object indicating success or failure. static Status Free(void* ptr, int64_t device_id); private: CudaMemoryManager(bool has_allocation) : has_allocation_(has_allocation) {} bool has_allocation_; static std::unique_ptr instance_; static std::mutex instance_mu_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/cuda_utils.cc000066400000000000000000000377071527052371200200350ustar00rootroot00000000000000// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "cuda_utils.h" #include "model_config_utils.h" #include "triton/common/logging.h" #include "triton/common/nvtx.h" #ifdef TRITON_ENABLE_GPU #include #endif namespace triton { namespace core { #ifdef TRITON_ENABLE_GPU void CUDART_CB MemcpyHost(void* args) { auto* copy_params = reinterpret_cast(args); memcpy(copy_params->dst_, copy_params->src_, copy_params->byte_size_); delete copy_params; } #endif // TRITON_ENABLE_GPU Status GetDeviceMemoryInfo(const int device_id, size_t* free, size_t* total) { *free = 0; *total = 0; #ifdef TRITON_ENABLE_GPU // Make sure that correct device is set before creating stream and // then restore the device to what was set by the caller. int current_device; auto cuerr = cudaGetDevice(¤t_device); bool overridden = false; if (cuerr == cudaSuccess) { overridden = (current_device != device_id); if (overridden) { cuerr = cudaSetDevice(device_id); } } if (cuerr == cudaSuccess) { cuerr = cudaMemGetInfo(free, total); } if (overridden) { cudaSetDevice(current_device); } if (cuerr != cudaSuccess) { return Status( Status::Code::INTERNAL, (std::string("unable to get memory info for device ") + std::to_string(device_id) + ": " + cudaGetErrorString(cuerr))); } #endif // TRITON_ENABLE_GPU return Status::Success; } Status EnablePeerAccess(const double min_compute_capability) { #ifdef TRITON_ENABLE_GPU // If we can't enable peer access for one device pair, the best we can // do is skipping it... std::set supported_gpus; bool all_enabled = false; if (GetSupportedGPUs(&supported_gpus, min_compute_capability).IsOk()) { all_enabled = true; int can_access_peer = false; for (const auto& host : supported_gpus) { auto cuerr = cudaSetDevice(host); if (cuerr == cudaSuccess) { for (const auto& peer : supported_gpus) { if (host == peer) { continue; } cuerr = cudaDeviceCanAccessPeer(&can_access_peer, host, peer); if ((cuerr == cudaSuccess) && (can_access_peer == 1)) { cuerr = cudaDeviceEnablePeerAccess(peer, 0); } all_enabled &= ((cuerr == cudaSuccess) && (can_access_peer == 1)); } } } } if (!all_enabled) { return Status( Status::Code::UNSUPPORTED, "failed to enable peer access for some device pairs"); } #endif // TRITON_ENABLE_GPU return Status::Success; } Status CopyBuffer( const std::string& msg, const TRITONSERVER_MemoryType src_memory_type, const int64_t src_memory_type_id, const TRITONSERVER_MemoryType dst_memory_type, const int64_t dst_memory_type_id, const size_t byte_size, const void* src, void* dst, cudaStream_t cuda_stream, bool* cuda_used, bool copy_on_stream) { NVTX_RANGE(nvtx_, "CopyBuffer"); *cuda_used = false; // For CUDA memcpy, all host to host copy will be blocked in respect to the // host, so use memcpy() directly. In this case, need to be careful on whether // the src buffer is valid. if ((src_memory_type != TRITONSERVER_MEMORY_GPU) && (dst_memory_type != TRITONSERVER_MEMORY_GPU)) { #ifdef TRITON_ENABLE_GPU if (copy_on_stream) { auto params = new CopyParams(dst, src, byte_size); cudaLaunchHostFunc( cuda_stream, MemcpyHost, reinterpret_cast(params)); *cuda_used = true; } else { memcpy(dst, src, byte_size); } #else memcpy(dst, src, byte_size); #endif // TRITON_ENABLE_GPU } else { #ifdef TRITON_ENABLE_GPU RETURN_IF_CUDA_ERR( cudaMemcpyAsync(dst, src, byte_size, cudaMemcpyDefault, cuda_stream), msg + ": failed to perform CUDA copy"); *cuda_used = true; #else return Status( Status::Code::INTERNAL, msg + ": try to use CUDA copy while GPU is not supported"); #endif // TRITON_ENABLE_GPU } return Status::Success; } void CopyBufferHandler( const std::string& msg, const TRITONSERVER_MemoryType src_memory_type, const int64_t src_memory_type_id, const TRITONSERVER_MemoryType dst_memory_type, const int64_t dst_memory_type_id, const size_t byte_size, const void* src, void* dst, cudaStream_t cuda_stream, void* response_ptr, triton::common::SyncQueue>* completion_queue) { bool cuda_used = false; Status status = CopyBuffer( msg, src_memory_type, src_memory_type_id, dst_memory_type, dst_memory_type_id, byte_size, src, dst, cuda_stream, &cuda_used); completion_queue->Put(std::make_tuple(status, cuda_used, response_ptr)); } #ifdef TRITON_ENABLE_GPU Status CheckGPUCompatibility(const int gpu_id, const double min_compute_capability) { // Query the compute capability from the device cudaDeviceProp cuprops; cudaError_t cuerr = cudaGetDeviceProperties(&cuprops, gpu_id); if (cuerr != cudaSuccess) { return Status( Status::Code::INTERNAL, "unable to get CUDA device properties for GPU ID" + std::to_string(gpu_id) + ": " + cudaGetErrorString(cuerr)); } double compute_compability = cuprops.major + (cuprops.minor / 10.0); if ((compute_compability > min_compute_capability) || (abs(compute_compability - min_compute_capability) < 0.01)) { return Status::Success; } else { return Status( Status::Code::UNSUPPORTED, "gpu " + std::to_string(gpu_id) + " has compute capability '" + std::to_string(cuprops.major) + "." + std::to_string(cuprops.minor) + "' which is less than the minimum supported of '" + std::to_string(min_compute_capability) + "'"); } } Status GetSupportedGPUs( std::set* supported_gpus, const double min_compute_capability) { // Make sure set is empty before starting supported_gpus->clear(); int device_cnt; cudaError_t cuerr = cudaGetDeviceCount(&device_cnt); if ((cuerr == cudaErrorNoDevice) || (cuerr == cudaErrorInsufficientDriver)) { device_cnt = 0; } else if (cuerr != cudaSuccess) { return Status( Status::Code::INTERNAL, "unable to get number of CUDA devices: " + std::string(cudaGetErrorString(cuerr))); } // populates supported_gpus for (int gpu_id = 0; gpu_id < device_cnt; gpu_id++) { Status status = CheckGPUCompatibility(gpu_id, min_compute_capability); if (status.IsOk()) { supported_gpus->insert(gpu_id); } } return Status::Success; } Status SupportsIntegratedZeroCopy(const int gpu_id, bool* zero_copy_support) { // Query the device to check if integrated cudaDeviceProp cuprops; cudaError_t cuerr = cudaGetDeviceProperties(&cuprops, gpu_id); if (cuerr != cudaSuccess) { return Status( Status::Code::INTERNAL, "unable to get CUDA device properties for GPU ID" + std::to_string(gpu_id) + ": " + cudaGetErrorString(cuerr)); } // Zero-copy supported only on integrated GPU when it can map host memory if (cuprops.integrated && cuprops.canMapHostMemory) { *zero_copy_support = true; } else { *zero_copy_support = false; } return Status::Success; } Status GetAllocationGranularity(size_t& aligned_sz) { CUmemAllocationProp prop = {}; prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; prop.location.id = 0; RETURN_IF_ERROR(CudaDriverHelper::GetInstance().CuMemGetAllocationGranularity( &aligned_sz, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM)); return Status::Success; } CudaDriverHelper::CudaDriverHelper() { dl_open_handle_ = nullptr; #ifndef _WIN32 dl_open_handle_ = dlopen("libcuda.so", RTLD_LAZY); if (dl_open_handle_ != nullptr) { void* cu_mem_create_fn = dlsym(dl_open_handle_, "cuMemCreate"); if (cu_mem_create_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuMemCreate"; dl_open_handle_ = nullptr; return; } *((void**)&cu_mem_create_fn_) = cu_mem_create_fn; void* cu_get_error_string_fn = dlsym(dl_open_handle_, "cuGetErrorString"); if (cu_get_error_string_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuGetErrorString"; dl_open_handle_ = nullptr; return; } *((void**)&cu_get_error_string_fn_) = cu_get_error_string_fn; void* cu_init_fn = dlsym(dl_open_handle_, "cuInit"); if (cu_init_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuInit"; dl_open_handle_ = nullptr; return; } *((void**)&cu_init_fn_) = cu_init_fn; void* cu_mem_set_access_fn = dlsym(dl_open_handle_, "cuMemSetAccess"); if (cu_mem_set_access_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuMemSetAccess"; dl_open_handle_ = nullptr; return; } *((void**)&cu_mem_set_access_fn_) = cu_mem_set_access_fn; void* cu_mem_release_fn = dlsym(dl_open_handle_, "cuMemRelease"); if (cu_mem_release_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuMemRelease"; dl_open_handle_ = nullptr; return; } *((void**)&cu_mem_release_fn_) = cu_mem_release_fn; void* cu_mem_get_allocation_granularity_fn = dlsym(dl_open_handle_, "cuMemGetAllocationGranularity"); if (cu_mem_get_allocation_granularity_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuMemGetAllocationGranularity"; dl_open_handle_ = nullptr; return; } *((void**)&cu_mem_get_allocation_granularity_fn_) = cu_mem_get_allocation_granularity_fn; void* cu_mem_address_free_fn = dlsym(dl_open_handle_, "cuMemAddressFree"); if (cu_mem_address_free_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuMemAddressFree"; dl_open_handle_ = nullptr; return; } *((void**)&cu_mem_address_free_fn_) = cu_mem_address_free_fn; void* cu_mem_unmap_fn = dlsym(dl_open_handle_, "cuMemUnmap"); if (cu_mem_unmap_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuMemUnmap"; dl_open_handle_ = nullptr; return; } *((void**)&cu_mem_unmap_fn_) = cu_mem_unmap_fn; void* cu_mem_address_reserve_fn = dlsym(dl_open_handle_, "cuMemAddressReserve"); if (cu_mem_address_reserve_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuMemAddressReserve"; dl_open_handle_ = nullptr; return; } *((void**)&cu_mem_address_reserve_fn_) = cu_mem_address_reserve_fn; void* cu_mem_map_fn = dlsym(dl_open_handle_, "cuMemMap"); if (cu_mem_map_fn == nullptr) { LOG_WARNING << "Failed to dlsym cuMemMap"; dl_open_handle_ = nullptr; return; } *((void**)&cu_mem_map_fn_) = cu_mem_map_fn; // Initialize the driver API. CUresult cuda_err = (*cu_init_fn_)(0 /* flags */); if (cuda_err != CUDA_SUCCESS) { const char* error_string; (*cu_get_error_string_fn_)(cuda_err, &error_string); error_str_ = std::string("failed to call cuInit: ") + error_string; dlclose(dl_open_handle_); dl_open_handle_ = nullptr; } } #endif } bool CudaDriverHelper::IsAvailable() { return dl_open_handle_ != nullptr; } Status CudaDriverHelper::CuMemGetAllocationGranularity( size_t* aligned_size, const CUmemAllocationProp* prop, CUmemAllocationGranularity_flags flags) { if (!IsAvailable()) { return Status( Status::Code::INTERNAL, "CudaDriverHelper has not been initialized."); } RETURN_IF_CUDA_DRIVER_ERR( cu_mem_get_allocation_granularity_fn_(aligned_size, prop, flags), std::string("failed to call cuMemGetAllocationGranularity")); return Status::Success; } Status CudaDriverHelper::CuMemCreate( CUmemGenericAllocationHandle* handle, size_t allocation_size, CUmemAllocationProp* prop, unsigned long long flags) { if (!IsAvailable()) { return Status( Status::Code::INTERNAL, "CudaDriverHelper has not been initialized."); } RETURN_IF_CUDA_DRIVER_ERR( cu_mem_create_fn_(handle, allocation_size, prop, flags), std::string("failed to call cuMemCreate")); return Status::Success; } Status CudaDriverHelper::CuMemSetAccess( CUdeviceptr ptr, size_t size, const CUmemAccessDesc* desc, size_t count) { if (!IsAvailable()) { return Status( Status::Code::INTERNAL, "CudaDriverHelper has not been initialized."); } RETURN_IF_CUDA_DRIVER_ERR( cu_mem_set_access_fn_(ptr, size, desc, count), std::string("failed to call cuMemSetAccess")); return Status::Success; } Status CudaDriverHelper::CuMemMap( CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags) { if (!IsAvailable()) { return Status( Status::Code::INTERNAL, "CudaDriverHelper has not been initialized."); } RETURN_IF_CUDA_DRIVER_ERR( cu_mem_map_fn_(ptr, size, offset, handle, flags), std::string("failed to call cuMemMap")); return Status::Success; } Status CudaDriverHelper::CuMemRelease(CUmemGenericAllocationHandle handle) { if (!IsAvailable()) { return Status( Status::Code::INTERNAL, "CudaDriverHelper has not been initialized."); } RETURN_IF_CUDA_DRIVER_ERR( cu_mem_release_fn_(handle), std::string("failed to call cuMemRelease")); return Status::Success; } void CudaDriverHelper::CuGetErrorString(const char** error_string, CUresult error) { *error_string = nullptr; if (IsAvailable()) { cu_get_error_string_fn_(error, error_string); } } Status CudaDriverHelper::CuMemAddressFree(CUdeviceptr ptr, size_t size) { if (!IsAvailable()) { return Status( Status::Code::INTERNAL, "CudaDriverHelper has not been initialized."); } RETURN_IF_CUDA_DRIVER_ERR( cu_mem_address_free_fn_(ptr, size), std::string("failed to call cuMemAddressFree")); return Status::Success; } Status CudaDriverHelper::CuMemUnmap(CUdeviceptr ptr, size_t size) { if (!IsAvailable()) { return Status( Status::Code::INTERNAL, "CudaDriverHelper has not been initialized."); } RETURN_IF_CUDA_DRIVER_ERR( cu_mem_unmap_fn_(ptr, size), std::string("failed to call cuMemUnmap")); return Status::Success; } Status CudaDriverHelper::CuMemAddressReserve( CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags) { if (!IsAvailable()) { return Status( Status::Code::INTERNAL, "CudaDriverHelper has not been initialized."); } RETURN_IF_CUDA_DRIVER_ERR( cu_mem_address_reserve_fn_(ptr, size, alignment, addr, flags), std::string("failed to call cuMemAddressReserve")); return Status::Success; } CudaDriverHelper::~CudaDriverHelper() { #ifndef _WIN32 if (dl_open_handle_ != nullptr) { dlclose(dl_open_handle_); } #endif } #endif }} // namespace triton::core kytensor-2.49.0.7/core/src/cuda_utils.h000066400000000000000000000252311527052371200176640ustar00rootroot00000000000000// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifndef _WIN32 #include #endif #include #include "status.h" #include "triton/common/sync_queue.h" #ifdef TRITON_ENABLE_GPU #include #include #endif // TRITON_ENABLE_GPU namespace triton { namespace core { #ifdef TRITON_ENABLE_GPU #define RETURN_IF_CUDA_ERR(X, MSG) \ do { \ cudaError_t err__ = (X); \ if (err__ != cudaSuccess) { \ return Status( \ Status::Code::INTERNAL, (MSG) + ": " + cudaGetErrorString(err__)); \ } \ } while (false) #define RETURN_IF_CUDA_DRIVER_ERR(X, MSG) \ do { \ CUresult cuda_err__ = (X); \ if (cuda_err__ != CUDA_SUCCESS) { \ const char* error_string__; \ CudaDriverHelper::GetInstance().CuGetErrorString( \ &error_string__, cuda_err__); \ return Status(Status::Code::INTERNAL, (MSG) + ": " + error_string__); \ } \ } while (false) #endif // TRITON_ENABLE_GPU #ifndef TRITON_ENABLE_GPU using cudaStream_t = void*; #endif // !TRITON_ENABLE_GPU /// Get the memory info for the specified device. /// \param device_id The device ID. /// \param free Return free memory in bytes. /// \param total Return total memory in bytes. /// \return The error status. A non-OK status means failure to get memory info. Status GetDeviceMemoryInfo(const int device_id, size_t* free, size_t* total); /// Enable peer access for all GPU device pairs /// \param min_compute_capability The minimum support CUDA compute /// capability. /// \return The error status. A non-OK status means not all pairs are enabled Status EnablePeerAccess(const double min_compute_capability); /// Copy buffer from 'src' to 'dst' for given 'byte_size'. The buffer location /// is identified by the memory type and id, and the corresponding copy will be /// initiated. /// \param msg The message to be prepended in error message. /// \param src_memory_type The memory type CPU/GPU of the source. /// \param src_memory_type_id The device id of the source. /// \param dst_memory_type The memory type CPU/GPU of the destination. /// \param dst_memory_type_id The device id of the destination. /// \param byte_size The size in bytes to me copied from source to destination. /// \param src The buffer start address of the source. /// \param dst The buffer start address of the destination. /// \param cuda_stream The stream to be associated with, and 0 can be /// passed for default stream. /// \param cuda_used returns whether a CUDA memory copy is initiated. If true, /// the caller should synchronize on the given 'cuda_stream' to ensure data copy /// is completed. /// \param copy_on_stream whether the memory copies should be performed in cuda /// host functions on the 'cuda_stream'. /// \return The error status. A non-ok status indicates failure to copy the /// buffer. Status CopyBuffer( const std::string& msg, const TRITONSERVER_MemoryType src_memory_type, const int64_t src_memory_type_id, const TRITONSERVER_MemoryType dst_memory_type, const int64_t dst_memory_type_id, const size_t byte_size, const void* src, void* dst, cudaStream_t cuda_stream, bool* cuda_used, bool copy_on_stream = false); #ifdef TRITON_ENABLE_GPU /// Validates the compute capability of the GPU indexed /// \param gpu_id The index of the target GPU. /// \param min_compute_capability The minimum support CUDA compute /// capability. /// \return The error status. A non-OK status means the target GPU is /// not supported. Status CheckGPUCompatibility( const int gpu_id, const double min_compute_capability); /// Obtains a set of gpu ids that is supported by triton. /// \param supported_gpus Returns the set of integers which is /// populated by ids of supported GPUS /// \param min_compute_capability The minimum support CUDA compute /// capability. /// \return The error status. A non-ok status means there were /// errors encountered while querying GPU devices. Status GetSupportedGPUs( std::set* supported_gpus, const double min_compute_capability); /// Checks if the GPU specified is an integrated GPU and supports Zero-copy. /// \param gpu_id The index of the target GPU. /// \param zero_copy_support If true, Zero-copy is supported by this GPU. /// \return The error status. A non-OK status means the target GPU is /// not supported. Status SupportsIntegratedZeroCopy(const int gpu_id, bool* zero_copy_support); /// Set the CUDA context to the specified device ID /// It will rollback to the previous device upon destruction. class ScopedSetDevice { public: ScopedSetDevice(int device) { overriden_ = false; prev_device_ = device; cudaGetDevice(&prev_device_); if (prev_device_ != device) { overriden_ = true; cudaSetDevice(device); } } ~ScopedSetDevice() { if (overriden_) { cudaSetDevice(prev_device_); } } private: int prev_device_; bool overriden_; }; #endif // Helper around CopyBuffer that updates the completion queue with the returned // status and cuda_used flag. void CopyBufferHandler( const std::string& msg, const TRITONSERVER_MemoryType src_memory_type, const int64_t src_memory_type_id, const TRITONSERVER_MemoryType dst_memory_type, const int64_t dst_memory_type_id, const size_t byte_size, const void* src, void* dst, cudaStream_t cuda_stream, void* response_ptr, triton::common::SyncQueue>* completion_queue); struct CopyParams { CopyParams(void* dst, const void* src, const size_t byte_size) : dst_(dst), src_(src), byte_size_(byte_size) { } void* dst_; const void* src_; const size_t byte_size_; }; #ifdef TRITON_ENABLE_GPU /// A singleton for Cuda Driver APIs. In order to continue supporting Triton /// deployments when GPUs are not available we need to use CUDA driver APIs /// through dlopen class CudaDriverHelper { public: static CudaDriverHelper& GetInstance() { static CudaDriverHelper instance; return instance; } private: void* dl_open_handle_ = nullptr; std::string error_str_; CUresult (*cu_mem_create_fn_)( CUmemGenericAllocationHandle*, size_t, CUmemAllocationProp*, unsigned long long) = nullptr; CUresult (*cu_mem_map_fn_)( CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags) = nullptr; CUresult (*cu_mem_set_access_fn_)( CUdeviceptr, size_t, const CUmemAccessDesc*, size_t) = nullptr; CUresult (*cu_get_error_string_fn_)(CUresult, const char**) = nullptr; CUresult (*cu_mem_get_allocation_granularity_fn_)( size_t*, const CUmemAllocationProp*, CUmemAllocationGranularity_flags) = nullptr; CUresult (*cu_mem_release_fn_)(CUmemGenericAllocationHandle) = nullptr; CUresult (*cu_init_fn_)(unsigned int) = nullptr; CUresult (*cu_mem_address_reserve_fn_)( CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags) = nullptr; CUresult (*cu_mem_unmap_fn_)(CUdeviceptr ptr, size_t size) = nullptr; CUresult (*cu_mem_address_free_fn_)(CUdeviceptr ptr, size_t size) = nullptr; CudaDriverHelper(); ~CudaDriverHelper(); public: CudaDriverHelper(CudaDriverHelper const&) = delete; void operator=(CudaDriverHelper const&) = delete; bool IsAvailable(); const std::string& GetErrorString() const { return error_str_; } void ClearErrorString() { return error_str_.clear(); } Status CuMemGetAllocationGranularity( size_t* aligned_size, const CUmemAllocationProp* prop, CUmemAllocationGranularity_flags flags); Status CuMemCreate( CUmemGenericAllocationHandle* block, size_t byte_size, CUmemAllocationProp* prop, unsigned long long flags); Status CuMemSetAccess( CUdeviceptr ptr, size_t size, const CUmemAccessDesc* desc, size_t count); Status CuMemMap( CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags); Status CuMemRelease(CUmemGenericAllocationHandle handle); Status CuMemAddressFree(CUdeviceptr ptr, size_t size); Status CuMemUnmap(CUdeviceptr ptr, size_t size); Status CuMemAddressReserve( CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags); void CuGetErrorString(const char** error_string, CUresult error); }; /// Get the minimum allocation granularity. /// \param aligned_size Returns minimum allocation granularity. /// \return The error status. A non-OK status means there were some errors /// when querying the allocation granularity. Status GetAllocationGranularity(size_t& aligned_sz); #endif }} // namespace triton::core kytensor-2.49.0.7/core/src/dynamic_batch_scheduler.cc000066400000000000000000000756111527052371200225200ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "dynamic_batch_scheduler.h" #ifndef _WIN32 #include #include #include #endif #include "constants.h" #include "server.h" #include "triton/common/logging.h" #include "triton/common/model_config.h" #include "triton/common/nvtx.h" namespace triton { namespace core { bool IsStaleState(Payload::State payload_state) { return ( (payload_state == Payload::State::EXECUTING) || (payload_state == Payload::State::RELEASED)); } void FinishSkippedRequests( std::vector>>&& requests, const Status& response_status, FailureReason reason) { for (auto& queue : requests) { for (auto& request : queue) { InferenceRequest::RespondIfError( request, response_status, true /* release_requests */, reason); } } } void FinishRejectedCancelledRequests( std::vector>>&& rejected_requests, std::vector>>&& cancelled_requests) { const static Status rejected_status = Status(Status::Code::UNAVAILABLE, "Request timeout expired"); const static Status cancelled_status = Status(Status::Code::CANCELLED); FinishSkippedRequests( std::move(rejected_requests), rejected_status, FailureReason::REJECTED); FinishSkippedRequests( std::move(cancelled_requests), cancelled_status, FailureReason::CANCELED); } DynamicBatchScheduler::DynamicBatchScheduler( TritonModel* model, TritonModelInstance* model_instance, const bool dynamic_batching_enabled, const int32_t max_batch_size, const std::unordered_map& enforce_equal_shape_tensors, const bool preserve_ordering, const std::set& preferred_batch_sizes, const uint64_t max_queue_delay_microseconds, const inference::ModelQueuePolicy& default_queue_policy, const uint64_t priority_levels, const ModelQueuePolicyMap& queue_policy_map) : model_(model), model_instance_(model_instance), model_name_(model->Name()), dynamic_batching_enabled_(dynamic_batching_enabled), queue_(default_queue_policy, priority_levels, queue_policy_map), stop_(false), max_batch_size_((size_t)std::max(1, max_batch_size)), preferred_batch_sizes_(preferred_batch_sizes), pending_batch_delay_ns_(max_queue_delay_microseconds * 1000), pending_batch_size_(0), queued_batch_size_(0), next_preferred_batch_size_(0), enforce_equal_shape_tensors_(enforce_equal_shape_tensors), has_optional_input_(false), preserve_ordering_(preserve_ordering) { rate_limiter_ = model_->Server()->GetRateLimiter(); // Both the server and model config should specify // caching enabled for model to utilize response cache. response_cache_enabled_ = model_->ResponseCacheEnabled() && model_->Server()->ResponseCacheEnabled(); max_preferred_batch_size_ = 0; for (const auto size : preferred_batch_sizes_) { max_preferred_batch_size_ = std::max(max_preferred_batch_size_, (size_t)size); } for (const auto& input : model_->Config().input()) { if (input.optional()) { has_optional_input_ = true; break; } } } Status DynamicBatchScheduler::Create( TritonModel* model, TritonModelInstance* model_instance, const int nice, const bool dynamic_batching_enabled, const int32_t max_batch_size, const std::unordered_map& enforce_equal_shape_tensors, const bool preserve_ordering, const std::set& preferred_batch_sizes, const uint64_t max_queue_delay_microseconds, std::unique_ptr* scheduler) { inference::ModelDynamicBatching batcher_config; batcher_config.set_preserve_ordering(preserve_ordering); for (const auto& bs : preferred_batch_sizes) { batcher_config.add_preferred_batch_size(bs); } batcher_config.set_max_queue_delay_microseconds(max_queue_delay_microseconds); return Create( model, model_instance, nice, dynamic_batching_enabled, max_batch_size, enforce_equal_shape_tensors, batcher_config, scheduler); } Status DynamicBatchScheduler::Create( TritonModel* model, TritonModelInstance* model_instance, const int nice, const bool dynamic_batching_enabled, const int32_t max_batch_size, const std::unordered_map& enforce_equal_shape_tensors, const inference::ModelDynamicBatching& batcher_config, std::unique_ptr* scheduler) { std::set preferred_batch_sizes; for (const auto size : batcher_config.preferred_batch_size()) { preferred_batch_sizes.insert(size); } DynamicBatchScheduler* dyna_sched = new DynamicBatchScheduler( model, model_instance, dynamic_batching_enabled, max_batch_size, enforce_equal_shape_tensors, batcher_config.preserve_ordering(), preferred_batch_sizes, batcher_config.max_queue_delay_microseconds(), batcher_config.default_queue_policy(), batcher_config.priority_levels(), batcher_config.priority_queue_policy()); std::unique_ptr sched(dyna_sched); sched->scheduler_thread_exit_.store(false); if (dynamic_batching_enabled) { sched->NewPayload(); sched->scheduler_thread_ = std::thread([dyna_sched, nice]() { dyna_sched->BatcherThread(nice); }); } scheduler->reset(sched.release()); return Status::Success; } DynamicBatchScheduler::~DynamicBatchScheduler() { // Signal the scheduler thread to exit and then wait for it.. scheduler_thread_exit_.store(true); cv_.notify_one(); if (scheduler_thread_.joinable()) { scheduler_thread_.join(); } } Status DynamicBatchScheduler::Enqueue(std::unique_ptr& request) { if (stop_) { return Status( Status::Code::UNAVAILABLE, request->LogRequest() + "Server is stopping, scheduler for model has stopped accepting new " "inference requests"); } // If queue start timestamp hasn't been set, queue timer starts at // the beginning of the queueing and scheduling process. Otherwise, // dynamic batcher is used as component of another batcher and should not // overwrite the queue start timestamp. if (request->QueueStartNs() == 0) { request->CaptureQueueStartNs(); INFER_TRACE_ACTIVITY( request->TraceProxy(), TRITONSERVER_TRACE_QUEUE_START, request->QueueStartNs()); #ifdef TRITON_ENABLE_TRACING request->TraceInputTensors( TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT, "DynamicBatchScheduler Enqueue"); #endif // TRITON_ENABLE_TRACING } // Record time at the beginning of the batcher queueing. In the case of // oldest sequence batcher, this will overwrite the value that was previously // set by sequence batcher, which is okay as by this point, the previous // batcher won't be needing this value and it can be safely reused by // the dynamic batcher. request->CaptureBatcherStartNs(); std::unique_ptr cached_response; if (response_cache_enabled_) { CacheLookUp(request, cached_response); } if (cached_response != nullptr) { // If there was a cache hit then try sending the cached response // and release the request. if (preserve_ordering_) { // In order to preserve the order, the response send must be // delegated. DelegateResponse(request); } // Send cached response and release request InferenceResponse::Send( std::move(cached_response), TRITONSERVER_RESPONSE_COMPLETE_FINAL); InferenceRequest::Release( std::move(request), TRITONSERVER_REQUEST_RELEASE_ALL); return Status::Success; } if (!dynamic_batching_enabled_) { if (preserve_ordering_ || response_cache_enabled_) { DelegateResponse(request); } // If not using dynamic batching, directly enqueue the // request to model for execution auto payload = model_->Server()->GetRateLimiter()->GetPayload( Payload::Operation::INFER_RUN, nullptr /* TritonModelInstance*/); payload->AddRequest(std::move(request)); RETURN_IF_ERROR( model_->Server()->GetRateLimiter()->EnqueuePayload(model_, payload)); } else { bool wake_batcher = true; { std::lock_guard lock(mu_); queued_batch_size_ += std::max(1U, request->BatchSize()); // Assuming no error is returned, this call takes ownership of // 'request' and so we can't use it after this point. RETURN_IF_ERROR(queue_.Enqueue(request->Priority(), request)); // If there are any idle runners and the queued batch size is greater or // equal to next preferred batch size, then wake batcher up to service // this request. We do the actual wake outside of the lock to avoid // having the woken thread immediately block on the lock // Explicitly force non-blocking to prevent waiting for the slot to // be available. wake_batcher = model_->Server()->GetRateLimiter()->PayloadSlotAvailable( model_, model_instance_, queue_.SupportPrefetching(), true /*force_non_blocking*/); // We may wake up runner less often if we don't enforce equal shape // within a batch, otherwise must always wake up runner to check it if (enforce_equal_shape_tensors_.empty()) { std::lock_guard exec_lock(*(curr_payload_->GetExecMutex())); auto payload_state = curr_payload_->GetState(); wake_batcher &= (payload_saturated_ || IsStaleState(payload_state) || (queued_batch_size_ >= next_preferred_batch_size_)); } } if (wake_batcher) { cv_.notify_one(); } } return Status::Success; } void DynamicBatchScheduler::NewPayload() { curr_payload_ = model_->Server()->GetRateLimiter()->GetPayload( Payload::Operation::INFER_RUN, model_instance_); payload_saturated_ = false; CustomBatchInit(); } void DynamicBatchScheduler::BatcherThread(const int nice) { #ifndef _WIN32 if (setpriority(PRIO_PROCESS, syscall(SYS_gettid), nice) == 0) { LOG_VERBOSE(1) << "Starting dynamic-batcher thread for " << model_name_ << " at nice " << nice << "..."; } else { LOG_VERBOSE(1) << "Starting dynamic-batcher thread for " << model_name_ << " at default nice (requested nice " << nice << " failed)..."; } #else LOG_VERBOSE(1) << "Starting dynamic-batcher thread for " << model_name_ << " at default nice..."; #endif // For debugging/testing, delay start of threads until the queue // contains the specified number of entries. size_t delay_cnt = 0; { const char* dstr = getenv("TRITONSERVER_DELAY_SCHEDULER"); if (dstr != nullptr) { delay_cnt = atoi(dstr); LOG_VERBOSE(1) << "Delaying batcher thread for " << model_name_ << " until " << delay_cnt << " queued requests..."; } } const uint64_t default_wait_microseconds = 500 * 1000; while (!scheduler_thread_exit_.load()) { NVTX_RANGE(nvtx_, "DynamicBatcher " + model_name_); std::vector>> rejected_requests, cancelled_requests; uint64_t wait_microseconds = 0; // Hold the lock for as short a time as possible. { std::unique_lock lock(mu_); { std::lock_guard exec_lock(*(curr_payload_->GetExecMutex())); auto payload_state = curr_payload_->GetState(); if (payload_saturated_ || IsStaleState(payload_state)) { NewPayload(); next_preferred_batch_size_ = 0; } } if (delay_cnt > 0) { // Debugging/testing... wait until queue contains 'delay_cnt' // items... wait_microseconds = 10 * 1000; if (queue_.Size() >= delay_cnt) { delay_cnt = 0; } LOG_VERBOSE(1) << "Delaying batcher thread " << model_name_ << " until " << delay_cnt << " queued requests, current total = " << queue_.Size(); } else if (queue_.Empty()) { wait_microseconds = default_wait_microseconds; } else { if (payload_saturated_) { continue; } WaitForPayloadSlotAvailable(&lock, default_wait_microseconds); { std::lock_guard exec_lock( *(curr_payload_->GetExecMutex())); auto payload_state = curr_payload_->GetState(); if (IsStaleState(payload_state)) { continue; } // Use dynamic batching to get request(s) to execute. wait_microseconds = GetDynamicBatch(); // Get requests that are rejected or cancelled from searching dynamic // batch. queue_.ReleaseSkippedRequests( &rejected_requests, &cancelled_requests); // Extract batch only if there is pending batch auto pending_batch_queue_cnt = queue_.PendingBatchCount(); if ((wait_microseconds == 0) && (pending_batch_queue_cnt != 0)) { curr_payload_->ReserveRequests(pending_batch_queue_cnt); for (size_t idx = 0; idx < pending_batch_queue_cnt; ++idx) { std::unique_ptr request; auto status = queue_.Dequeue(&request); if (status.IsOk()) { if (preserve_ordering_ || response_cache_enabled_) { DelegateResponse(request); } curr_payload_->AddRequest(std::move(request)); } else { // The queue is empty which conflicts with pending batch // count. Send the current batch if any and reset related // variables. LOG_ERROR << request->LogRequest() << "Failed to retrieve request from scheduler queue: " << status.Message(); queue_.ResetCursor(); queued_batch_size_ = 0; pending_batch_size_ = 0; break; } } if (curr_payload_->GetState() == Payload::State::UNINITIALIZED) { curr_payload_->SetState(Payload::State::READY); } queued_batch_size_ -= pending_batch_size_; pending_batch_size_ = 0; } } } // If no requests are to be handled, wait for notification or // for the specified timeout before checking the queue again. if (wait_microseconds > 0) { std::chrono::microseconds wait_timeout(wait_microseconds); cv_.wait_for(lock, wait_timeout); } } if (curr_payload_->GetState() == Payload::State::READY) { auto callback = [this]() { cv_.notify_one(); }; curr_payload_->SetCallback(callback); { std::lock_guard exec_lock(*(curr_payload_->GetExecMutex())); CustomBatchFini(); } model_->Server()->GetRateLimiter()->EnqueuePayload(model_, curr_payload_); } // Finish rejected and cancelled requests if any FinishRejectedCancelledRequests( std::move(rejected_requests), std::move(cancelled_requests)); } // end runner loop LOG_VERBOSE(1) << "Stopping dynamic-batcher thread for " << model_name_ << "..."; } void DynamicBatchScheduler::WaitForPayloadSlotAvailable( std::unique_lock* lock, uint64_t wait_microseconds) { // The wait_for_slots conditional can be blocking till the slots are available // for execution. Need to explicitly release the 'mu_' lock to allow the // Enqueue threads above to make progress. lock->unlock(); const std::chrono::microseconds wait_timeout(wait_microseconds); std::mutex slot_mu; std::unique_lock slot_lock(slot_mu); bool slot_available = false; while (!slot_available) { slot_available = cv_.wait_for(slot_lock, wait_timeout, [this]() { return model_->Server()->GetRateLimiter()->PayloadSlotAvailable( model_, model_instance_, queue_.SupportPrefetching(), true /* force_non_blocking */); }); if (!slot_available) { // Reject and release timeout requests from queue. std::vector>> rejected_requests, cancelled_requests; { std::lock_guard lock(mu_); queue_.RejectTimeoutRequests(); queue_.ReleaseSkippedRequests(&rejected_requests, &cancelled_requests); } FinishRejectedCancelledRequests( std::move(rejected_requests), std::move(cancelled_requests)); } } // Recapture the lock. lock->lock(); } uint64_t DynamicBatchScheduler::GetDynamicBatch() { // 'mu_' mutex must be held when this function is called. queue_ // must not be empty. // Examine the new requests. If adding these new requests to the // pending batch allows a preferred batch size then execute it // immediately. Stop examining requests if the maximum preferred // batch size would be exceeded or if the shape of the next request // does not match the shape of the pending batch. bool send_now = false; // If the previous payload was not executed, reset the cursor to the start // of the queue to re-iterate over it and find the ideal batch. if (!queue_.IsCursorValid()) { queue_.ResetCursor(); pending_batch_size_ = 0; if (CustomBatchEnabled()) { CustomBatchFini(); CustomBatchInit(); } } size_t best_preferred_batch_size = 0; queued_batch_size_ -= queue_.ApplyPolicyAtCursor(); // When there is optional input or input shape must be enforced, // the inputs in the requests must be examined for forming a batch const bool check_input = !enforce_equal_shape_tensors_.empty() || has_optional_input_; auto payload_batch_size = curr_payload_->BatchSize(); while (!queue_.CursorEnd()) { const auto batch_size = std::max(1U, queue_.RequestAtCursor()->BatchSize()); // If there is no pending batch, then this request is starting a // new batch. if ((payload_batch_size + queue_.PendingBatchCount()) == 0) { // Get the shape of the new batch that is being started... if (check_input) { if (!curr_payload_->MutableRequiredEqualInputs() ->Initialize( queue_.RequestAtCursor(), enforce_equal_shape_tensors_, has_optional_input_) .IsOk()) { send_now = true; break; } } } else { // There is a pending batch and adding this request would make // the batch size larger than all of the preferred batch sizes, // so mark the cursor at this point. Not sending the pending batch so // that we can examine the queue delay of requests that fits in a batch. if (((payload_batch_size + pending_batch_size_ + batch_size) > max_preferred_batch_size_) && (best_preferred_batch_size == 0)) { best_preferred_batch_size = pending_batch_size_; queue_.MarkCursor(); payload_saturated_ = true; } if ((payload_batch_size + pending_batch_size_ + batch_size) > max_batch_size_) { send_now = true; break; } // There is a pending batch and it has a different shape then // this request, so send the pending batch as it is. if (check_input && !curr_payload_->MutableRequiredEqualInputs()->HasEqualInputs( queue_.RequestAtCursor())) { curr_payload_->MarkSaturated(); send_now = true; break; } } if (CustomBatchEnabled()) { bool should_include = false; CustomBatchIncl(queue_.RequestAtCursor().get(), &should_include); if (!should_include) { curr_payload_->MarkSaturated(); send_now = true; break; } } pending_batch_size_ += batch_size; queue_.AdvanceCursor(); queued_batch_size_ -= queue_.ApplyPolicyAtCursor(); if (preferred_batch_sizes_.find(pending_batch_size_ + payload_batch_size) != preferred_batch_sizes_.end()) { best_preferred_batch_size = pending_batch_size_; queue_.MarkCursor(); } } // Obtain the age of the oldest pending request to compare with the maximum // batch queuing delay uint64_t now_ns = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); uint64_t delay_ns = now_ns - queue_.OldestEnqueueTime(); bool delay_is_exceeded = (pending_batch_delay_ns_ != 0) && (delay_ns >= pending_batch_delay_ns_); // If we found a preferred batch size and the queue delay hasn't // been exceeded, then execute that. if ((best_preferred_batch_size != 0) && !delay_is_exceeded) { if (pending_batch_delay_ns_ == 0) { payload_saturated_ = true; } pending_batch_size_ = best_preferred_batch_size; queue_.SetCursorToMark(); return 0; } // No request in pending batch happens when all queued requests have expired // timeout and the policies are REJECT if (queue_.PendingBatchCount() == 0) { return 0; } // If the delay has been exceeded, or if the current batch can't grow // any larger then just immediately execute whatever is pending. if (send_now || ((payload_batch_size + pending_batch_size_) >= max_preferred_batch_size_)) { payload_saturated_ = true; return 0; } if (delay_is_exceeded || (pending_batch_delay_ns_ == 0)) { return 0; } // Set the next preferred batch size given the pending batch size auto next_preferred_batch_size_it = preferred_batch_sizes_.upper_bound( pending_batch_size_ + payload_batch_size); if (next_preferred_batch_size_it != preferred_batch_sizes_.end()) { next_preferred_batch_size_ = *next_preferred_batch_size_it; } else { next_preferred_batch_size_ = preferred_batch_sizes_.empty() ? 0 : *preferred_batch_sizes_.begin(); } if (next_preferred_batch_size_ != 0) { next_preferred_batch_size_ -= payload_batch_size; } // By this point, we have not seen the pending batch that should be executed // immediately. However, if we have scheduled a payload that can be grown and // not yet in preferred batch size, we should move the pending batch over to // ensure the model instance will pick up largest available batch even if it // is not the preferred batch. if (!payload_saturated_ && (payload_batch_size != 0) && (preferred_batch_sizes_.find(payload_batch_size) == preferred_batch_sizes_.end())) { return 0; } uint64_t wait_ns = pending_batch_delay_ns_ - delay_ns; // Note that taking request timeout into consideration allows us to reset // pending batch as soon as it is invalidated. But the cost is that in edge // case where the timeout will be expired one by one, the thread will be // waken frequently. if (queue_.ClosestTimeout() != 0) { if (now_ns <= queue_.ClosestTimeout()) { wait_ns = std::min(queue_.ClosestTimeout() - now_ns, wait_ns); } else { // A request in pending batch is timed-out, wait for 1 us to force the // thread to reset the pending batch right the way. wait_ns = 1000; } } // Return non-zero wait microseconds to cause this thread to wait // until the queue delay or the closest timeout has expired. // Another thread may be awaken due to incoming request to handle the // pending batch before this thread wakes and that is ok. But if no other // request comes in then this thread will wake and revisit the pending batch // (and at that time will then see the delay has been exceeded and will send // the batch). return wait_ns / 1000; } void DynamicBatchScheduler::DelegateResponse( std::unique_ptr& request) { std::lock_guard lock(completion_queue_mtx_); completion_queue_.emplace_back(); auto queue_slot = &completion_queue_.back(); // Cache plumbing const std::string& key = request->CacheKey(); const bool is_key_set = request->CacheKeyIsSet(); const uint64_t lookup_end_ns = request->CacheLookupEndNs(); const uint64_t lookup_start_ns = request->CacheLookupStartNs(); request->SetResponseDelegator( [this, queue_slot, key, is_key_set, lookup_end_ns, lookup_start_ns]( std::unique_ptr&& response, const uint32_t flags) { if (response_cache_enabled_) { // Logical error, the key should be set if caching is enabled // for this model if (!is_key_set) { LOG_ERROR << "Request cache key was not set correctly."; } // Cache insertion happens here because we need the backend to have // computed the inference response first in the case of cache miss auto cache = model_->Server()->CacheManager()->Cache(); #ifdef TRITON_ENABLE_STATS const uint64_t insert_start_ns = CaptureTimeNs(); #endif // TRITON_ENABLE_STATS auto status = cache->Insert(response.get(), key); #ifdef TRITON_ENABLE_STATS const uint64_t insert_end_ns = CaptureTimeNs(); #endif // TRITON_ENABLE_STATS bool cache_miss = (status.StatusCode() != Status::Code::ALREADY_EXISTS); if (cache_miss) { #ifdef TRITON_ENABLE_STATS uint64_t lookup_ns = lookup_end_ns - lookup_start_ns; // Logical error, this shouldn't happen if (lookup_start_ns > lookup_end_ns) { lookup_ns = 0; LOG_ERROR << "Request lookup duration was not set correctly."; } uint64_t insert_ns = insert_end_ns - insert_start_ns; uint64_t cache_miss_ns = lookup_ns + insert_ns; // Use model_ to update stats directly because request object can be // released by the backend before getting to this callback. model_->MutableStatsAggregator()->UpdateSuccessCacheMiss( model_->MetricReporter().get(), cache_miss_ns); #endif // TRITON_ENABLE_STATS if (!status.IsOk()) { LOG_ERROR << "Failed to insert key [" << key << "] into response cache: " << status.Message(); } } // Otherwise do nothing; we update cache hit statistics on Lookup } if (preserve_ordering_) { { std::lock_guard lock(completion_queue_mtx_); queue_slot->emplace_back(std::move(response), flags); } FinalizeResponses(); } else { InferenceResponse::Send(std::move(response), flags); } }); } void DynamicBatchScheduler::CacheLookUp( std::unique_ptr& request, std::unique_ptr& cached_response) { auto cache = model_->Server()->CacheManager()->Cache(); bool is_lookup_success = CacheLookUpUtil(request, cached_response, cache); if (is_lookup_success) { #ifdef TRITON_ENABLE_STATS // Update model metrics/stats on cache hits // Backends will update metrics as normal on cache misses request->ReportStatisticsCacheHit(model_->MetricReporter().get()); #endif // TRITON_ENABLE_STATS } } void DynamicBatchScheduler::FinalizeResponses() { // Need exclusive access of the function to ensure responses are sent // in order std::lock_guard lock(finalize_mtx_); // Finalize the completed payloads in-order as far as possible std::deque, const uint32_t>> responses; { std::lock_guard queue_lock(completion_queue_mtx_); while (!completion_queue_.empty() && !completion_queue_.front().empty()) { bool response_complete = false; for (auto& response_pair : completion_queue_.front()) { // Assuming FINAL flag is set only in the last response of the request response_complete = ((response_pair.second & TRITONSERVER_RESPONSE_COMPLETE_FINAL) != 0); responses.emplace_back(std::move(response_pair)); } if (response_complete) { completion_queue_.pop_front(); } else { completion_queue_.front().clear(); } } } for (auto& response : responses) { InferenceResponse::Send(std::move(response.first), response.second); } } bool DynamicBatchScheduler::CustomBatchEnabled() const { return model_->ModelBatchInitFn(); } void DynamicBatchScheduler::CustomBatchIncl( InferenceRequest* request, bool* should_include) { if (!CustomBatchEnabled()) return; TRITONSERVER_Error* err = model_->ModelBatchInclFn()( reinterpret_cast(request), *curr_payload_.get()->UserPointerAddr(), should_include); if (err) { LOG_ERROR << "Custom batching include function failed for model " << model_->Name() << ": " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } } void DynamicBatchScheduler::CustomBatchInit() { if (!CustomBatchEnabled()) return; TRITONSERVER_Error* err = model_->ModelBatchInitFn()( *model_->Batcher(), curr_payload_.get()->UserPointerAddr()); if (err != nullptr) { LOG_ERROR << "Custom batching initialization function failed for model " << model_->Name() << ": " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } } void DynamicBatchScheduler::CustomBatchFini() { if (!CustomBatchEnabled() || *curr_payload_.get()->UserPointerAddr() == nullptr) return; TRITONSERVER_Error* err = model_->ModelBatchFiniFn()(*curr_payload_.get()->UserPointerAddr()); *curr_payload_.get()->UserPointerAddr() = nullptr; if (err != nullptr) { LOG_ERROR << "Custom batching finalization function failed for model " << model_->Name() << ": " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } } }} // namespace triton::core kytensor-2.49.0.7/core/src/dynamic_batch_scheduler.h000066400000000000000000000173601527052371200223570ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include #include "backend_model.h" #include "backend_model_instance.h" #include "model_config.pb.h" #include "rate_limiter.h" #include "scheduler.h" #include "scheduler_utils.h" #include "status.h" #include "triton/common/model_config.h" namespace triton { namespace core { // Scheduler that implements dynamic batching. class DynamicBatchScheduler : public Scheduler { public: // Create a scheduler to support a given number of runners and a run // function to call when a request is scheduled. static Status Create( TritonModel* model, TritonModelInstance* model_instance, const int nice, const bool dynamic_batching_enabled, const int32_t max_batch_size, const std::unordered_map& enforce_equal_shape_tensors, const bool preserve_ordering, const std::set& preferred_batch_sizes, const uint64_t max_queue_delay_microseconds, std::unique_ptr* scheduler); // Create a scheduler to support a given number of runners and a run // function to call when a request is scheduled. And the scheduler also // supports different queue policies for different priority levels. static Status Create( TritonModel* model, TritonModelInstance* model_instance, const int nice, const bool dynamic_batching_enabled, const int32_t max_batch_size, const std::unordered_map& enforce_equal_shape_tensors, const inference::ModelDynamicBatching& batcher_config, std::unique_ptr* scheduler); ~DynamicBatchScheduler(); // \see Scheduler::Enqueue() Status Enqueue(std::unique_ptr& request) override; // \see Scheduler::InflightInferenceCount() size_t InflightInferenceCount() override { std::unique_lock lock(mu_); if (curr_payload_ != nullptr) { return queue_.Size() + curr_payload_->RequestCount(); } return queue_.Size(); } // \see Scheduler::Stop() void Stop() override { stop_ = true; } private: DynamicBatchScheduler( TritonModel* model, TritonModelInstance* model_instance, const bool dynamic_batching_enabled, const int32_t max_batch_size, const std::unordered_map& enforce_equal_shape_tensors, const bool preserve_ordering, const std::set& preferred_batch_sizes, const uint64_t max_queue_delay_microseconds, const inference::ModelQueuePolicy& default_queue_policy, const uint64_t priority_levels, const ModelQueuePolicyMap& queue_policy_map); void BatcherThread(const int nice); void NewPayload(); uint64_t GetDynamicBatch(); void DelegateResponse(std::unique_ptr& request); void CacheLookUp( std::unique_ptr& request, std::unique_ptr& cached_response); void FinalizeResponses(); // Block until a payload slot is available on the rate limiter. The 'lock' // should be acquired when calling this function. The 'lock' will be released // when waiting for payload slot and re-acquired before this function returns. // For queued requests under policy REJECT, they will be rejected if timed-out // while waiting for a slot. The timeout will be checked every // 'wait_microseconds'. The 'wait_microseconds' should be non-zero. void WaitForPayloadSlotAvailable( std::unique_lock* lock, uint64_t wait_microseconds); // Custom batching function calls // Returns whether custom batching is enabled. bool CustomBatchEnabled() const; // If custom batching is enabled for this model, see if this request should be // included. void CustomBatchIncl(InferenceRequest* request, bool* should_include); // If custom batching is enabled for this model, initialize the batching // function. void CustomBatchInit(); // If custom batching is enabled for this model, finalizethe batching // function. void CustomBatchFini(); TritonModel* model_; TritonModelInstance* model_instance_; // Name of the model. std::string model_name_; // True if dynamic batching is enabled. const bool dynamic_batching_enabled_; // Map from priority level to queue holding inference requests for the model // represented by this scheduler. If priority queues are not supported by the // scheduler, then priority zero entry is used as the single queue. PriorityQueue queue_; bool stop_; std::thread scheduler_thread_; std::atomic scheduler_thread_exit_; // Mutex and condvar for signaling scheduler thread std::mutex mu_; std::condition_variable cv_; std::shared_ptr rate_limiter_; std::shared_ptr curr_payload_; bool payload_saturated_; size_t max_batch_size_; size_t max_preferred_batch_size_; std::set preferred_batch_sizes_; uint64_t pending_batch_delay_ns_; size_t pending_batch_size_; size_t queued_batch_size_; size_t next_preferred_batch_size_; // The input tensors that require shape checking before being // allowed in a batch. As a map from the tensor name to a bool. If // tensor is in map then its shape must match shape of same tensor // in requests already in the batch. If value is "true" then // additional tensor is treated as a shape tensor and the values // contained in the shape tensor must match same tensor already in // the batch. const std::unordered_map enforce_equal_shape_tensors_; // Store information on whether the model contains optional inputs. bool has_optional_input_; // If true the ordering of responses matches the order of requests // even when there are multiple scheduler threads. const bool preserve_ordering_; // If true, the scheduler will try to retrieve responses from cache. bool response_cache_enabled_; // Per completion-id queues to store the ready responses std::deque< std::vector, uint32_t>>> completion_queue_; // Lock to protect the completion_queues_ std::mutex completion_queue_mtx_; // Preserves the order in which responses are finalized std::mutex finalize_mtx_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/ensemble_scheduler/000077500000000000000000000000001527052371200212045ustar00rootroot00000000000000kytensor-2.49.0.7/core/src/ensemble_scheduler/ensemble_model.cc000066400000000000000000000054761527052371200245010ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ensemble_model.h" #include #include "constants.h" #include "ensemble_scheduler.h" #include "model_config_utils.h" #include "triton/common/logging.h" namespace triton { namespace core { Status EnsembleModel::Create( InferenceServer* server, const std::string& path, const ModelIdentifier& model_id, const int64_t version, const inference::ModelConfig& model_config, const bool is_config_provided, const double min_compute_capability, std::unique_ptr* model) { // Create the ensemble model. std::unique_ptr local_model(new EnsembleModel( min_compute_capability, path, model_id, version, model_config)); RETURN_IF_ERROR(local_model->Init(is_config_provided)); std::unique_ptr scheduler; RETURN_IF_ERROR(EnsembleScheduler::Create( local_model->MutableStatsAggregator(), server, local_model->ModelId(), model_config, &scheduler)); RETURN_IF_ERROR(local_model->SetScheduler(std::move(scheduler))); LOG_VERBOSE(1) << "ensemble model for " << local_model->Name() << std::endl; *model = std::move(local_model); return Status::Success; } std::ostream& operator<<(std::ostream& out, const EnsembleModel& pb) { out << "name=" << pb.Name() << std::endl; return out; } }} // namespace triton::core kytensor-2.49.0.7/core/src/ensemble_scheduler/ensemble_model.h000066400000000000000000000051311527052371200243270ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "model.h" #include "model_config.pb.h" #include "scheduler.h" #include "status.h" namespace triton { namespace core { class InferenceServer; class EnsembleModel : public Model { public: EnsembleModel(EnsembleModel&&) = default; static Status Create( InferenceServer* server, const std::string& path, const ModelIdentifier& model_id, const int64_t version, const inference::ModelConfig& model_config, const bool is_config_provided, const double min_compute_capability, std::unique_ptr* model); private: DISALLOW_COPY_AND_ASSIGN(EnsembleModel); explicit EnsembleModel( const double min_compute_capability, const std::string& model_dir, const ModelIdentifier& model_id, const int64_t version, const inference::ModelConfig& config) : Model(min_compute_capability, model_dir, model_id, version, config) { } friend std::ostream& operator<<(std::ostream&, const EnsembleModel&); }; std::ostream& operator<<(std::ostream& out, const EnsembleModel& pb); }} // namespace triton::core kytensor-2.49.0.7/core/src/ensemble_scheduler/ensemble_scheduler.cc000066400000000000000000001575631527052371200253640ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef TRITON_ENABLE_ENSEMBLE #include "ensemble_scheduler.h" #include #include "cuda_utils.h" #include "metrics.h" #include "model.h" #include "model_config_utils.h" #include "server.h" #include "triton/common/logging.h" namespace triton { namespace core { namespace { class EnsembleContext; using IterationCount = size_t; // Request tracker is passed as 'userp' in RequestRelease function and used // to manage the lifecycle of the ensemble request class RequestTracker { public: explicit RequestTracker( std::unique_ptr&& request, uint64_t compute_start_ns, MetricModelReporter* metric_reporter, InferenceStatsAggregator* stats_aggregator) : inflight_request_counter_(1), request_(std::move(request)), compute_start_ns_(compute_start_ns), metric_reporter_(metric_reporter), stats_aggregator_(stats_aggregator), status_(Status::Success) { } std::unique_ptr& Request() { return request_; } InferenceStatsAggregator* StatsAggregator() { return stats_aggregator_; } MetricModelReporter* MetricReporter() { return metric_reporter_; } InferenceStatsAggregator& ContextStatsAggregator() { return context_stats_aggregator_; } void IncrementCounter() { std::lock_guard lk(mtx_); inflight_request_counter_++; } bool DecrementCounter() { std::lock_guard lk(mtx_); inflight_request_counter_--; if (inflight_request_counter_ == 0) { if (request_ != nullptr) { #ifdef TRITON_ENABLE_STATS const auto& infer_stats = context_stats_aggregator_.ImmutableInferStats(); request_->ReportStatisticsWithDuration( metric_reporter_, status_.IsOk(), compute_start_ns_, infer_stats.compute_input_duration_ns_, infer_stats.compute_infer_duration_ns_, infer_stats.compute_output_duration_ns_); if (status_.IsOk()) { stats_aggregator_->UpdateInferBatchStatsWithDuration( metric_reporter_, std::max(1U, request_->BatchSize()), infer_stats.compute_input_duration_ns_, infer_stats.compute_infer_duration_ns_, infer_stats.compute_output_duration_ns_); } #endif InferenceRequest::Release( std::move(request_), TRITONSERVER_REQUEST_RELEASE_ALL); } } return (inflight_request_counter_ == 0); } void SetStatus(const Status& status) { std::lock_guard lk(mtx_); status_ = status; } private: std::mutex mtx_; uint32_t inflight_request_counter_; std::unique_ptr request_; uint64_t compute_start_ns_; MetricModelReporter* metric_reporter_; InferenceStatsAggregator* stats_aggregator_; InferenceStatsAggregator context_stats_aggregator_; Status status_; }; // Step is used as 'userp' and keeps ensemble context alive // until no more internal requests are inflight. // Step contains metadata, and status for the // internal infer request struct Step { Step( size_t step_idx, const InferenceRequest::SequenceId& correlation_id, uint32_t flags) : correlation_id_(correlation_id), flags_(flags), response_flags_(0), step_idx_(step_idx) { } std::shared_ptr ctx_; std::unique_ptr request_; InferenceRequest::SequenceId correlation_id_; uint32_t flags_; std::mutex output_mtx_; // Different output map to avoid address conflict from different memory types std::unordered_map> cpu_output_map_; std::unordered_map< int64_t, std::unordered_map>> gpu_output_map_; std::set> updated_tensors_; // Storing information conveyed through response callback. // This should be properly consumed by EnsembleContext and cleaned up before // returning from the callback. uint32_t response_flags_; TRITONSERVER_InferenceResponse* response_; size_t step_idx_; }; struct TensorData { struct Metadata { Metadata() = default; Metadata( std::unique_ptr&& data, size_t reference_count) : data_(std::move(data)), remaining_reference_count_(reference_count), parameter_override_(false) { } Metadata( std::unique_ptr&& data, size_t reference_count, const InferenceRequest::SequenceId& correlation_id, uint32_t flags) : data_(std::move(data)), remaining_reference_count_(reference_count), parameter_override_(true), correlation_id_(correlation_id), flags_(flags) { } std::unique_ptr data_; size_t remaining_reference_count_; bool parameter_override_; InferenceRequest::SequenceId correlation_id_; uint32_t flags_; }; TensorData() = default; TensorData(const size_t outgoing_steps_count) : current_iteration_(0), outgoing_steps_count_(outgoing_steps_count), batch_size_(0) { } IterationCount AddTensor(std::unique_ptr&& tensor) { tensor_.emplace( current_iteration_, Metadata(std::move(tensor), outgoing_steps_count_)); return current_iteration_++; } IterationCount AddTensor( std::unique_ptr&& tensor, const InferenceRequest::SequenceId& correlation_id, uint32_t flags) { tensor_.emplace( current_iteration_, Metadata( std::move(tensor), outgoing_steps_count_, correlation_id, flags)); return current_iteration_++; } // Tensors associated with the particular ensemble tensor. // A container is used to handle the decoupled case // where variable number of tensors will be produced. // map 'iteration count' to pair of std::unordered_map tensor_; size_t current_iteration_; size_t outgoing_steps_count_; // Ensemble may be configured to passing tensor between batching model and // non-batching model as long as the full shapes match and storing the batch // size of the generated tensor explicitly for checking and setting proper // shape for the downstream model request. size_t batch_size_; }; // EnsembleContext maintains the state of the ensemble request // // Using static functions to take advantage of shared_ptr, a copy of the // shared_ptr will be made when a step is scheduled and it will go out of // scope after the step's callback is finished. The step's callback will // schedule new steps if available and the last step will finish the ensemble // request. // So we don't have to maintain the context in scheduler as the shared_ptr // will destroy the context for us if there are no "in-flight" steps. class EnsembleContext { public: EnsembleContext( MetricModelReporter* metric_reporter, InferenceStatsAggregator* stats_aggregator, InferenceServer* is, EnsembleInfo* info, std::unique_ptr& request, cudaStream_t stream); // Perform transition on 'context' state given the information of // 'completed_step' static void Proceed( const std::shared_ptr& context, const std::unique_ptr& completed_step = nullptr); private: static TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* allocated_memory_type, int64_t* allocated_memory_type_id); static TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); static TRITONSERVER_Error* OutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); static void RequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp); static void ResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); using StepList = std::vector>; using VersionMap = std::unordered_map>; // Helper function to reshape the given tensor according to the // config shape and batching info and its actual shape and batching info. // Note that 'dims' will be in full shape as opposed to 'config_dims'. // Return the dims after reshape. std::vector ReshapeTensorDims( const triton::common::DimsList& config_dims, const bool config_allow_batching, const size_t tensor_batch_size, const std::vector& dims); // Return the list of step that becomes ready due to tensor update // from 'completed_step' Status PrepareSteps( const std::unique_ptr& completed_step, StepList* steps); // Prepare infer stats and call the inference server's function to process // the infer requests specified in 'steps' static void ScheduleSteps( const std::shared_ptr& context, StepList&& steps); // Helper function that updates ensemble state given 'completed_step' and // returns the list of updated tensors in 'updated_tensors' Status UpdateEnsembleState( const std::unique_ptr& completed_step, std::set>* updated_tensors); // Helper function of 'UpdateEnsembleState' to convert a // TRITONSERVER_InferenceResponse captured in 'completed_step' into // internal variables of this EnsembleContext object. Status ConsumeResponse(const std::unique_ptr& completed_step); // Helper function that returns a list of 'steps' that should be run under // current ensemble state. 'updated_tensors' is used so that we don't need to // iterate all the tensors to determine which step can be run. Status GetNextSteps( const std::set>& updated_tensors, StepList* steps); // Helper function that completes the response of the ensemble request Status FinishEnsemble( std::unique_ptr&& response = nullptr); // Helper function that initialize the 'step' given the info at 'step_idx'. // The 'step' will have proper request / response provider for the model Status InitStep( const size_t step_idx, const IterationCount iteration_count, std::unique_ptr* step); // Helper function that set the output of the ensemble request if it is ready // and valid. Status CheckAndSetEnsembleOutput( const std::set>& updated_tensors, std::unique_ptr* response); void CacheEnsembleTopLevelRequest( std::unique_ptr& response); InferenceServer* is_; EnsembleInfo* info_; // All EnsembleContext will use the same CUDA stream managed by // the ensemble scheduler cudaStream_t stream_; // Mutex to avoid concurrent call on 'PrepareSteps' where ensemble state // are being modified std::mutex mutex_; size_t inflight_step_counter_; // pointer that either points to 'pruned_tensor_to_step_' or to // 'info_->tensor_to_step_' if all ensemble outputs are requested std::unordered_map>* tensor_to_step_; std::unordered_map> pruned_tensor_to_step_; std::unordered_map tensor_data_; // Handle to all models that may be used in the ensemble std::unordered_map handles_; // Request specific information that obtained from ensemble request and // should be applied to all internal requests uint32_t flags_; std::string request_id_; InferenceRequest::SequenceId correlation_id_; uint64_t priority_; uint64_t timeout_; // The parameters of the ensemble-level request. Use this to forward // parameters to composing models. std::deque parameters_; // Objects related to the ensemble infer request Status ensemble_status_; RequestTracker* request_tracker_; // Use in conjunction with 'is_decoupled_' in EnsembleInfo to // better distinguish ensemble ending behavior (see annotation in // FinishEnsemble for details). bool response_sent_{false}; // The allocator that will be used to allocate buffers for the // inference result tensors. std::unique_ptr< TRITONSERVER_ResponseAllocator, decltype(&TRITONSERVER_ResponseAllocatorDelete)> allocator_; }; EnsembleContext::EnsembleContext( MetricModelReporter* metric_reporter, InferenceStatsAggregator* stats_aggregator, InferenceServer* is, EnsembleInfo* info, std::unique_ptr& request, cudaStream_t stream) : is_(is), info_(info), stream_(stream), inflight_step_counter_(0), allocator_(nullptr, TRITONSERVER_ResponseAllocatorDelete) { uint64_t compute_start_ns = 0; INFER_STATS_SET_TIMESTAMP(compute_start_ns); request_tracker_ = new RequestTracker( std::move(request), compute_start_ns, metric_reporter, stats_aggregator); auto& lrequest = request_tracker_->Request(); // Obtain model handles of all models in ensemble request such that // they have the same lifetime as the ensemble request to avoid unloading // while the ensemble is executing. for (const auto& step_info : info_->steps_) { auto it = handles_.find(step_info.model_id_); if (it == handles_.end()) { it = handles_.emplace(std::make_pair(step_info.model_id_, VersionMap())) .first; } auto ver_it = it->second.find(step_info.model_version_); if (ver_it == it->second.end()) { std::shared_ptr model = nullptr; ensemble_status_ = is_->GetModel(step_info.model_id_, step_info.model_version_, &model); if (!ensemble_status_.IsOk()) { break; } it->second.emplace(std::make_pair(step_info.model_version_, model)); } } // Prune ensemble first if not all outputs are requested std::set ignored_tensor; for (const auto& ensemble_output : info_->ensemble_output_shape_) { ignored_tensor.insert(ensemble_output.first); } for (const auto& requested_output : lrequest->ImmutableRequestedOutputs()) { ignored_tensor.erase(requested_output); } if (ignored_tensor.empty()) { tensor_to_step_ = &(info_->tensor_to_step_); } else { pruned_tensor_to_step_ = info_->tensor_to_step_; tensor_to_step_ = &pruned_tensor_to_step_; // Backward traversal std::unordered_map step_requested_output_count; while (!ignored_tensor.empty()) { std::set new_ignored_tensor; for (const auto& output : ignored_tensor) { auto step_idx = info_->tensor_to_prev_step_[output]; auto& step = info_->steps_[step_idx]; auto it = step_requested_output_count.find(step_idx); if (it == step_requested_output_count.end()) { auto output_count = step.output_to_tensor_.size(); it = step_requested_output_count.emplace(step_idx, output_count).first; } // If none of the outputs of the step is requested, // then the step can be pruned if (--it->second == 0) { for (const auto& input : step.input_to_tensor_) { auto& step_set = pruned_tensor_to_step_[input.second]; step_set.erase(step_idx); // If all steps depend on a tensor are pruned, // then the tensor can be ignored. if (step_set.empty()) { new_ignored_tensor.insert(input.second); } } } } ignored_tensor.swap(new_ignored_tensor); } } for (const auto& pair : *tensor_to_step_) { const auto& requested_outputs = lrequest->ImmutableRequestedOutputs(); // For requested outputs, add 1 to outgoing count as the ensemble itself // isn't counted as step. if (requested_outputs.find(pair.first) != requested_outputs.end()) { tensor_data_.emplace(pair.first, TensorData(pair.second.size() + 1)); } else { tensor_data_.emplace(pair.first, TensorData(pair.second.size())); } } if (ensemble_status_.IsOk()) { request_id_ = lrequest->Id(); correlation_id_ = lrequest->CorrelationId(); flags_ = lrequest->Flags(); priority_ = lrequest->Priority(); timeout_ = lrequest->TimeoutMicroseconds(); parameters_ = lrequest->Parameters(); for (const auto& pr : lrequest->ImmutableInputs()) { const InferenceRequest::Input* input = pr.second; auto it = tensor_data_.find(input->Name()); if (it != tensor_data_.end()) { auto& tensor_data = it->second; // Shape() represents reshaped value without batch dimension, // thus need to fill it if necessary. std::unique_ptr tensor; if (lrequest->BatchSize() != 0) { std::vector shape{lrequest->BatchSize()}; shape.insert( shape.end(), input->Shape().begin(), input->Shape().end()); tensor.reset(new InferenceRequest::Input( input->Name(), input->DType(), shape)); } else { tensor.reset(new InferenceRequest::Input( input->Name(), input->DType(), input->Shape())); } tensor->SetData(input->Data()); for (const auto& host_policy_data : input->HostPolicyData()) { tensor->SetData(host_policy_data.first, host_policy_data.second); } tensor_data.AddTensor(std::move(tensor)); tensor_data.batch_size_ = lrequest->BatchSize(); } else { ensemble_status_ = Status( Status::Code::INVALID_ARG, lrequest->LogRequest() + "unexpected input '" + input->Name() + "' in request header that does not map to any ensemble inputs"); } } // Iterate the ensemble optional inputs and add empty tensor data entry // if the input is not provided for (const auto& name : info_->optional_inputs_) { auto it = tensor_data_.find(name); if ((it != tensor_data_.end()) && it->second.tensor_.empty()) { it->second.AddTensor(nullptr); it->second.batch_size_ = lrequest->BatchSize(); } } } TRITONSERVER_ResponseAllocator* allocator; TRITONSERVER_Error* err = TRITONSERVER_ResponseAllocatorNew( &allocator, ResponseAlloc, ResponseRelease, nullptr /* start_fn */); if (err == nullptr) { err = TRITONSERVER_ResponseAllocatorSetQueryFunction( allocator, OutputBufferQuery); } if (err != nullptr) { ensemble_status_ = Status( TritonCodeToStatusCode(TRITONSERVER_ErrorCode(err)), TRITONSERVER_ErrorMessage(err)); TRITONSERVER_ErrorDelete(err); } else { allocator_.reset(allocator); } } TRITONSERVER_Error* EnsembleContext::ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* allocated_memory_type, int64_t* allocated_memory_type_id) { *buffer = nullptr; *buffer_userp = nullptr; auto allocated_buffer = std::make_shared( byte_size, preferred_memory_type, preferred_memory_type_id); auto mutable_buffer = allocated_buffer->MutableBuffer( allocated_memory_type, allocated_memory_type_id); if ((mutable_buffer != nullptr) || (byte_size == 0)) { if (byte_size != 0) { *buffer = static_cast(mutable_buffer); auto step = reinterpret_cast(userp); std::lock_guard lk(step->output_mtx_); if (*allocated_memory_type == TRITONSERVER_MEMORY_GPU) { step->gpu_output_map_[*allocated_memory_type_id].emplace( reinterpret_cast(*buffer), std::move(allocated_buffer)); } else { step->cpu_output_map_.emplace( reinterpret_cast(*buffer), std::move(allocated_buffer)); } } LOG_VERBOSE(1) << "Internal response allocation: " << tensor_name << ", size " << byte_size << ", addr " << *buffer << ", memory type " << *allocated_memory_type << ", type id " << *allocated_memory_type_id; } return nullptr; // Success } TRITONSERVER_Error* EnsembleContext::ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { LOG_VERBOSE(1) << "Internal response release: " << "size " << byte_size << ", addr " << buffer; // Don't do anything when releasing a buffer since ResponseAlloc // passes the ownership of the data to ensemble context. return nullptr; // Success } TRITONSERVER_Error* EnsembleContext::OutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { // Ensemble will always attempt to satisfy any output buffer request return nullptr; // Success } void EnsembleContext::RequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { if ((flags & TRITONSERVER_REQUEST_RELEASE_ALL) != 0) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestDelete(request), "deleting ensemble inference request"); auto request_tracker = reinterpret_cast(userp); if (request_tracker->DecrementCounter()) { delete request_tracker; } } } void EnsembleContext::ResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { auto step_ptr = std::unique_ptr(reinterpret_cast(userp)); step_ptr->response_flags_ = flags; step_ptr->response_ = response; EnsembleContext::Proceed(step_ptr->ctx_, step_ptr); // Expecting more responses if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { step_ptr.release(); } } Status EnsembleContext::ConsumeResponse(const std::unique_ptr& completed_step) { // Scoped deleter for response static auto res_dlt = [](TRITONSERVER_InferenceResponse* res) { if (res != nullptr) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(res), "deleting inference response"); } }; std::unique_ptr managed_response(completed_step->response_, res_dlt); completed_step->response_ = nullptr; auto response = managed_response.get(); auto step_ptr = completed_step.get(); if (response != nullptr) { RETURN_IF_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseError(response)); uint32_t count; bool parameter_override = false; InferenceRequest::SequenceId correlation_id = step_ptr->correlation_id_; uint32_t flags = 0; RETURN_IF_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseParameterCount(response, &count)); for (uint32_t idx = 0; idx < count; idx++) { const char* name; TRITONSERVER_ParameterType type; const void* vvalue; RETURN_IF_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseParameter( response, idx, &name, &type, &vvalue)); if (!strcmp(name, "sequence_id")) { switch (type) { case TRITONSERVER_PARAMETER_INT: correlation_id = InferenceRequest::SequenceId( *reinterpret_cast(vvalue)); parameter_override = true; break; case TRITONSERVER_PARAMETER_STRING: correlation_id = InferenceRequest::SequenceId( std::string(*reinterpret_cast(vvalue))); parameter_override = true; break; default: RETURN_IF_TRITONSERVER_ERROR(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "expected parameter 'sequence_id' to be " "TRITONSERVER_PARAMETER_INT or " "TRITONSERVER_PARAMETER_STRING")); } } else if (!strcmp(name, "sequence_start")) { if (type != TRITONSERVER_PARAMETER_BOOL) { RETURN_IF_TRITONSERVER_ERROR(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "expect parameter 'sequence_start' to be " "TRITONSERVER_PARAMETER_BOOL")); } else { if (*reinterpret_cast(vvalue)) { flags |= TRITONSERVER_REQUEST_FLAG_SEQUENCE_START; } parameter_override = true; } } else if (!strcmp(name, "sequence_end")) { if (type != TRITONSERVER_PARAMETER_BOOL) { RETURN_IF_TRITONSERVER_ERROR(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "expect parameter 'sequence_end' to be " "TRITONSERVER_PARAMETER_BOOL")); } else { if (*reinterpret_cast(vvalue)) { flags |= TRITONSERVER_REQUEST_FLAG_SEQUENCE_END; } parameter_override = true; } } } RETURN_IF_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseOutputCount(response, &count)); // Continuously shrink 'output_to_tensor' to identify missing outputs, // a placeholder tensor will be put for missing output, which will be // used to relax the connectivity constraint if the output happens to be // an optional input for the next model. auto output_to_tensor = info_->steps_[step_ptr->step_idx_].output_to_tensor_; for (uint32_t idx = 0; idx < count; idx++) { const char* name; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; RETURN_IF_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseOutput( response, idx, &name, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); auto it = output_to_tensor.find(name); if (it != output_to_tensor.end()) { std::unique_ptr tensor( new InferenceRequest::Input( it->second, TritonToDataType(datatype), shape, dim_count)); if (byte_size != 0) { std::lock_guard output_lk(step_ptr->output_mtx_); if (memory_type == TRITONSERVER_MEMORY_GPU) { auto& gpu_output_map = step_ptr->gpu_output_map_[memory_type_id]; auto it = gpu_output_map.find(reinterpret_cast(base)); tensor->SetData(std::move(it->second)); gpu_output_map.erase(it); } else { auto it = step_ptr->cpu_output_map_.find( reinterpret_cast(base)); tensor->SetData(std::move(it->second)); step_ptr->cpu_output_map_.erase(it); } } auto& tensor_data = tensor_data_[it->second]; if (parameter_override) { step_ptr->updated_tensors_.emplace( it->second, tensor_data.AddTensor(std::move(tensor), correlation_id, flags)); } else { step_ptr->updated_tensors_.emplace( it->second, tensor_data.AddTensor( std::move(tensor), step_ptr->correlation_id_, step_ptr->flags_)); } output_to_tensor.erase(it); } else { LOG_VERBOSE(1) << "in ensemble, an internal response header specified " "output '" << name << "' that does not map to any ensemble tensors"; } } // Consider not-provided output to be optional inputs of the next model, // and add empty tensor data entry to satisfy connectivity check. for (const auto& ott : output_to_tensor) { auto& tensor_data = tensor_data_[ott.second]; step_ptr->updated_tensors_.emplace( ott.second, tensor_data.AddTensor(nullptr, correlation_id, flags)); } } return Status::Success; } void EnsembleContext::Proceed( const std::shared_ptr& context, const std::unique_ptr& completed_step) { StepList ready_steps; Status status = context->PrepareSteps(completed_step, &ready_steps); if (status.IsOk()) { ScheduleSteps(context, std::move(ready_steps)); } } Status EnsembleContext::PrepareSteps( const std::unique_ptr& completed_step, StepList* ready_steps) { { std::lock_guard lock(mutex_); // Initialization error, ensemble status will be not ok since the beginning if (completed_step == nullptr && !ensemble_status_.IsOk()) { ensemble_status_ = FinishEnsemble(); } if (ensemble_status_.IsOk()) { StepList res; std::set> updated_tensors; ensemble_status_ = UpdateEnsembleState(completed_step, &updated_tensors); if (ensemble_status_.IsOk()) { ensemble_status_ = GetNextSteps(updated_tensors, ready_steps); } // Reaching the end of processing completed step, check if the ensemble // should send response / error. if (!ensemble_status_.IsOk()) { ensemble_status_ = FinishEnsemble(); } else { std::unique_ptr response; if (!updated_tensors.empty()) { ensemble_status_ = CheckAndSetEnsembleOutput(updated_tensors, &response); } ensemble_status_ = FinishEnsemble(std::move(response)); } } return ensemble_status_; } } Status EnsembleContext::UpdateEnsembleState( const std::unique_ptr& completed_step, std::set>* updated_tensors) { updated_tensors->clear(); if (completed_step == nullptr) { for (const auto& tensor_data : tensor_data_) { if (!tensor_data.second.tensor_.empty()) { updated_tensors->emplace(tensor_data.first, 0); } } } else { if (completed_step->response_flags_ & TRITONSERVER_RESPONSE_COMPLETE_FINAL) { inflight_step_counter_--; } RETURN_IF_ERROR(ConsumeResponse(completed_step)); updated_tensors->swap(completed_step->updated_tensors_); } return Status::Success; } Status EnsembleContext::GetNextSteps( const std::set>& updated_tensors, StepList* steps) { steps->clear(); std::set> next_step_idx; // Get steps whose tensors used for input are set for (const auto& updated_tensor : updated_tensors) { const auto& step_idx = (*tensor_to_step_)[updated_tensor.first]; for (const auto& idx : step_idx) { bool ready = true; for (const auto& input_pair : info_->steps_[idx].input_to_tensor_) { auto& tensor = tensor_data_[input_pair.second].tensor_; if (tensor.empty()) { ready = false; break; } else { // Check if other inputs have tensor with corresponding iteration // count if (tensor.find(updated_tensor.second) == tensor.end()) { ready = false; break; } } } if (ready) { next_step_idx.emplace(idx, updated_tensor.second); } } } for (const auto& idx : next_step_idx) { steps->emplace_back(); RETURN_IF_ERROR(InitStep(idx.first, idx.second, &(steps->back()))); } inflight_step_counter_ += steps->size(); return Status::Success; } Status EnsembleContext::InitStep( const size_t step_idx, const IterationCount iteration_count, std::unique_ptr* step) { const auto& istep = info_->steps_[step_idx]; auto& version_map = handles_[istep.model_id_]; auto& model = version_map[istep.model_version_]; const bool allow_batching = (model->Config().max_batch_size() > 0); auto irequest = std::unique_ptr( new InferenceRequest(model, istep.model_version_)); // Store the pointers to tensors used so that we can prune them afterward. // Can't prune the tensor in the input loop below as it may be used by // multiple inputs in the same step. std::map releasing_tensors; // Set inputs in request, prepare input map, // and set overridden parameter if any. auto correlation_id = correlation_id_; auto flags = flags_; bool parameter_set = false; for (const auto& pair : istep.input_to_tensor_) { auto& tensor_data = tensor_data_[pair.second]; auto& tensor = tensor_data.tensor_[iteration_count]; // nullptr if and only if the tensor is optional ensemble input and // not provided in the ensemble request. In such case, we don't add // the input and expect the ensemble pipeline is configured correctly // (the input to the inner model is also optional) if (tensor.data_ != nullptr) { // If the actual shape and config shape agree with each other without // considering batch size, non-batch / batch conversion are not required. const inference::ModelInput* input_config; model->GetInput(pair.first, &input_config); auto shape = ReshapeTensorDims( input_config->dims(), allow_batching, tensor_data.batch_size_, tensor.data_->OriginalShape()); InferenceRequest::Input* input; RETURN_IF_ERROR(irequest->AddOriginalInput( pair.first, tensor.data_->DType(), shape, &input)); RETURN_IF_ERROR(input->SetData(tensor.data_->Data())); for (const auto& host_policy_data : tensor.data_->HostPolicyData()) { RETURN_IF_ERROR( input->SetData(host_policy_data.first, host_policy_data.second)); } } releasing_tensors.emplace(&tensor_data, &tensor.remaining_reference_count_); if (tensor.parameter_override_) { if (parameter_set && ((correlation_id != tensor.correlation_id_) || (flags != tensor.flags_))) { LOG_ERROR << irequest->LogRequest() << "Different set of response parameters are set for '" << istep.model_id_ << "'. Parameter correlation ID " << correlation_id << ", flags " << flags << " is used."; continue; } correlation_id = tensor.correlation_id_; flags = tensor.flags_; parameter_set = true; } } // Prune the tensor if it is not needed by other steps for (auto& releasing_pair : releasing_tensors) { if ((--(*releasing_pair.second)) == 0) { releasing_pair.first->tensor_.erase(iteration_count); } } // Set requested outputs in request header for (const auto& pair : istep.output_to_tensor_) { irequest->AddOriginalRequestedOutput(pair.first); } step->reset(new Step(step_idx, correlation_id, flags)); irequest->SetId(request_id_); irequest->SetCorrelationId(correlation_id); irequest->SetFlags(flags); irequest->SetPriority(priority_); irequest->SetTimeoutMicroseconds(timeout_); irequest->SetParameters(parameters_); #ifdef TRITON_ENABLE_STATS irequest->SetSecondaryStatsAggregator( &request_tracker_->ContextStatsAggregator()); #endif irequest->SetResponseCallback( reinterpret_cast(allocator_.get()), step->get(), ResponseComplete, step->get()); irequest->SetReleaseCallback(RequestComplete, request_tracker_); RETURN_IF_ERROR(irequest->PrepareForInference()); #ifdef TRITON_ENABLE_TRACING auto& parent_trace = request_tracker_->Request()->TraceProxy(); if (parent_trace != nullptr) { irequest->SetTrace(parent_trace->SpawnChildTrace()); irequest->TraceProxy()->SetModelName(irequest->ModelName()); irequest->TraceProxy()->SetRequestId(irequest->Id()); irequest->TraceProxy()->SetModelVersion(irequest->ActualModelVersion()); } #endif // Record the batch size of output in advance as // there is no other way to access it later on. for (const auto& pair : istep.output_to_tensor_) { auto& output_data_ = tensor_data_[pair.second]; output_data_.batch_size_ = irequest->BatchSize(); } (*step)->request_ = std::move(irequest); return Status::Success; } std::vector EnsembleContext::ReshapeTensorDims( const triton::common::DimsList& config_dims, const bool config_allow_batching, const size_t tensor_batch_size, const std::vector& dims) { bool reshaped = false; std::vector res; // Only attempt to reshape if one setting is batchable while the other is not, // the case of two mismatched batchable shapes is not considered. // If the actual shape and config shape agree with each other without // considering batch size, non-batch / batch conversion are not required. if (config_allow_batching != (tensor_batch_size != 0)) { // expect batching but the tensor is generated from nobatching model if (config_allow_batching) { if (triton::common::CompareDimsWithWildcard(config_dims, dims)) { // If 'dims' already matches 'config_dims', prepend with batch size 1 res.push_back(1); res.insert(res.end(), dims.begin(), dims.end()); reshaped = true; } // Otherwise, assuming the tensor is already in the batch expected // by the model and do nothing } else { // Check if the batched tensor can be sent to the non-batching // model as one tensor. If not, strip the batch dimension if // it is batch size 1 if (!triton::common::CompareDimsWithWildcard(config_dims, dims) && (tensor_batch_size == 1)) { res.assign(dims.begin() + 1, dims.end()); reshaped = true; } } } if (!reshaped) { res = dims; } return res; } // Caching function void EnsembleContext::CacheEnsembleTopLevelRequest( std::unique_ptr& response) { const std::string key = request_tracker_->Request()->CacheKey(); const bool is_key_set = request_tracker_->Request()->CacheKeyIsSet(); #ifdef TRITON_ENABLE_STATS const uint64_t lookup_end_ns = request_tracker_->Request()->CacheLookupEndNs(); const uint64_t lookup_start_ns = request_tracker_->Request()->CacheLookupStartNs(); #endif if (!is_key_set) { LOG_ERROR << "Request cache key was not set correctly."; } auto cache = is_->CacheManager()->Cache(); #ifdef TRITON_ENABLE_STATS const uint64_t insert_start_ns = CaptureTimeNs(); #endif auto status = cache->Insert(response.get(), key); if (!status.IsOk()) { LOG_ERROR << "Failed to insert key [" << key << "] into response cache: " << status.Message(); } #ifdef TRITON_ENABLE_STATS const uint64_t insert_end_ns = CaptureTimeNs(); uint64_t lookup_ns = lookup_end_ns - lookup_start_ns; if (lookup_start_ns > lookup_end_ns) { lookup_ns = 0; LOG_ERROR << "Request lookup duration was not set correctly."; } uint64_t insert_ns = insert_end_ns - insert_start_ns; uint64_t cache_miss_ns = lookup_ns + insert_ns; request_tracker_->StatsAggregator()->UpdateSuccessCacheMiss( request_tracker_->MetricReporter(), cache_miss_ns); #endif } Status EnsembleContext::FinishEnsemble(std::unique_ptr&& response) { // Do nothing if the ensemble is finished if (request_tracker_ == nullptr) { return ensemble_status_; } // Add ensemble name to make error message more trackable if (!ensemble_status_.IsOk()) { ensemble_status_ = Status( ensemble_status_.StatusCode(), "in ensemble '" + info_->ensemble_name_ + "', " + ensemble_status_.Message()); } if (ensemble_status_.IsOk()) { auto flags = (inflight_step_counter_ == 0) ? TRITONSERVER_RESPONSE_COMPLETE_FINAL : 0; if (response != nullptr) { // Cache the request if caching is enabled. if (info_->is_cache_enabled_) { CacheEnsembleTopLevelRequest(response); } InferenceResponse::Send(std::move(response), flags); response_sent_ = true; } else if (flags != 0) { // check if any response is sent, if not, decoupled must be specified. // For decoupled model, it may send 0 response. To emulate the same // behavior, ensemble will not consider the below case a deadlock, // instead, it will send only the flag in this case. if ((!info_->is_decoupled_) && (!response_sent_)) { ensemble_status_ = Status( Status::Code::INVALID_ARG, "in ensemble '" + info_->ensemble_name_ + "', " + request_tracker_->Request()->LogRequest() + "unexpected deadlock, at least one output is not set while no " "more " "ensemble steps can be made"); InferenceRequest::RespondIfError( request_tracker_->Request(), ensemble_status_, false /* release_requests */, FailureReason::OTHER); } else { request_tracker_->Request()->ResponseFactory()->SendFlags( TRITONSERVER_RESPONSE_COMPLETE_FINAL); } } } else { if (response != nullptr) { InferenceResponse::SendWithStatus( std::move(response), TRITONSERVER_RESPONSE_COMPLETE_FINAL, ensemble_status_); } else { InferenceRequest::RespondIfError( request_tracker_->Request(), ensemble_status_, false /* release_requests */, FailureReason::OTHER); } } if (inflight_step_counter_ == 0) { // Reach here when the ensemble execution comes to the end, // 'ensemble_status_' at this point is representative. request_tracker_->SetStatus(ensemble_status_); if (request_tracker_->DecrementCounter()) { delete request_tracker_; } request_tracker_ = nullptr; } return ensemble_status_; } Status EnsembleContext::CheckAndSetEnsembleOutput( const std::set>& updated_tensors, std::unique_ptr* response) { IterationCount iteration_count = 0; // Check if updated tensor is one of the ensemble output and if all outputs // have tensor of the same iteration count bool ready = false; auto& lrequest = request_tracker_->Request(); const auto& requested_outputs = lrequest->ImmutableRequestedOutputs(); for (const auto& updated_tensor : updated_tensors) { if (requested_outputs.find(updated_tensor.first) == requested_outputs.end()) { continue; } ready = true; iteration_count = updated_tensor.second; for (const auto& output : requested_outputs) { auto& tensor = tensor_data_[output].tensor_; if (tensor.empty()) { ready = false; break; } // Check if other outputs have tensor with corresponding iteration count else if (tensor.find(iteration_count) == tensor.end()) { ready = false; break; } // check if the output is provided else if (tensor[iteration_count].data_ == nullptr) { ready = false; break; } } } if (!ready) { // Do not create response if the actual response is not ready. return Status::Success; } RETURN_IF_ERROR(lrequest->ResponseFactory()->CreateResponse(response)); bool cuda_async_copy = false; std::map releasing_tensors; for (const auto& output_pair : info_->ensemble_output_shape_) { if (requested_outputs.find(output_pair.first) == requested_outputs.end()) { continue; } // Check if output is ready auto& tensor_data = tensor_data_[output_pair.first]; auto& tensor = tensor_data.tensor_[iteration_count]; auto shape = ReshapeTensorDims( output_pair.second, (lrequest->BatchSize() != 0), tensor_data.batch_size_, tensor.data_->OriginalShape()); InferenceResponse::Output* output; RETURN_IF_ERROR((*response)->AddOutput( output_pair.first, tensor.data_->DType(), shape, &output)); // Use the memory type of the memory block as preferred memory type TRITONSERVER_MemoryType dst_memory_type; int64_t dst_memory_type_id; size_t content_size; tensor.data_->Data()->BufferAt( 0, &content_size, &dst_memory_type, &dst_memory_type_id); void* buffer; RETURN_IF_ERROR(output->AllocateDataBuffer( &buffer, content_size, &dst_memory_type, &dst_memory_type_id)); // Done with this output if 'expected_byte_size' is 0 if (content_size == 0) { continue; } else if (buffer == nullptr) { return Status( Status::Code::INTERNAL, "failed to allocate buffer for output '" + output_pair.first + "'"); } size_t content_offset = 0; size_t content_idx = 0; TRITONSERVER_MemoryType src_memory_type; int64_t src_memory_type_id; const char* content = tensor.data_->Data()->BufferAt( content_idx, &content_size, &src_memory_type, &src_memory_type_id); bool cuda_used = false; while (content != nullptr) { RETURN_IF_ERROR(CopyBuffer( output_pair.first, src_memory_type, src_memory_type_id, dst_memory_type, dst_memory_type_id, content_size, content, ((char*)buffer) + content_offset, stream_, &cuda_used)); cuda_async_copy |= cuda_used; content_offset += content_size; content_idx++; content = tensor.data_->Data()->BufferAt( content_idx, &content_size, &src_memory_type, &src_memory_type_id); } releasing_tensors.emplace(&tensor_data, &tensor.remaining_reference_count_); if (tensor.parameter_override_) { switch (lrequest->CorrelationId().Type()) { case InferenceRequest::SequenceId::DataType::STRING: (*response)->AddParameter( "sequence_id", tensor.correlation_id_.StringValue().c_str()); break; case InferenceRequest::SequenceId::DataType::UINT64: (*response)->AddParameter( "sequence_id", (int64_t)tensor.correlation_id_.UnsignedIntValue()); break; default: (*response)->AddParameter( "sequence_id", (int64_t)tensor.correlation_id_.UnsignedIntValue()); break; } (*response)->AddParameter( "sequence_start", (tensor.flags_ & TRITONSERVER_REQUEST_FLAG_SEQUENCE_START) != 0); (*response)->AddParameter( "sequence_end", (tensor.flags_ & TRITONSERVER_REQUEST_FLAG_SEQUENCE_END) != 0); } } if (cuda_async_copy) { #ifdef TRITON_ENABLE_GPU cudaStreamSynchronize(stream_); #else return Status( Status::Code::INTERNAL, "unexpected CUDA copy flag set while GPU is not supported"); #endif // TRITON_ENABLE_GPU } // Prune the tensor if it is not needed by other steps for (auto& releasing_pair : releasing_tensors) { if ((--(*releasing_pair.second)) == 0) { releasing_pair.first->tensor_.erase(iteration_count); } } return Status::Success; } void EnsembleContext::ScheduleSteps( const std::shared_ptr& context, StepList&& steps) { for (auto& step : steps) { step->ctx_ = context; bool should_schedule = false; // Must release lock before InferAsync to avoid deadlock, as the same thread // will be calling request/response callbacks on cache hits, which will // attempt to acquire the lock already held { std::lock_guard lock(context->mutex_); // Need to check the ensemble_status_ to ensure the FinishEnsemble() // is called only once. if (context->ensemble_status_.IsOk()) { context->request_tracker_->IncrementCounter(); should_schedule = true; } } if (should_schedule) { // If the ensemble request is cancelled, propagate the cancellation to the // next request step. if (context->request_tracker_->Request()->IsCancelled()) { step->request_->Cancel(); } // On a successful call to InferAsync(), the step will be released by // the response callback. When the response callback is invoked, the // step must not own (and release) the request as the request should be // transferred and managed by Triton core. In the case of cache hit, the // request hasn't been transferred and can cause double-free, so moving // the request ownership out of step here to avoid that std::unique_ptr request = std::move(step->request_); auto step_status = context->is_->InferAsync(request); if (step_status.IsOk()) { step.release(); continue; } else { std::lock_guard lock(context->mutex_); context->ensemble_status_ = step_status; } } // Reaching here means the step is not being scheduled, update corresponding // counters and attempt to finish ensemble if it is the last step. std::lock_guard lock(context->mutex_); // The request is not sent to server properly, shouldn't expect its // release function get called. context->request_tracker_->DecrementCounter(); --context->inflight_step_counter_; if (context->inflight_step_counter_ == 0) { context->ensemble_status_ = context->FinishEnsemble(); } } } } // namespace Status EnsembleScheduler::Create( InferenceStatsAggregator* const stats_aggregator, InferenceServer* const server, const ModelIdentifier& model_id, const inference::ModelConfig& config, std::unique_ptr* scheduler) { scheduler->reset( new EnsembleScheduler(stats_aggregator, server, model_id, config)); return Status::Success; } void EnsembleScheduler::CacheLookUp( std::unique_ptr& request, std::unique_ptr& cached_response) { auto cache = is_->CacheManager()->Cache(); bool is_lookup_success = CacheLookUpUtil(request, cached_response, cache); if (is_lookup_success) { #ifdef TRITON_ENABLE_STATS request->ReportStatisticsCacheHit(metric_reporter_.get()); #endif } } Status EnsembleScheduler::Enqueue(std::unique_ptr& request) { // Queue timer starts at the beginning of the queueing and // scheduling process request->CaptureQueueStartNs(); INFER_TRACE_ACTIVITY( request->TraceProxy(), TRITONSERVER_TRACE_QUEUE_START, request->QueueStartNs()); #ifdef TRITON_ENABLE_TRACING request->TraceInputTensors( TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT, "EnsembleScheduler Enqueue"); #endif // TRITON_ENABLE_TRACING std::unique_ptr cached_response; if (info_->is_cache_enabled_) { CacheLookUp(request, cached_response); } if (cached_response != nullptr) { InferenceResponse::Send( std::move(cached_response), TRITONSERVER_RESPONSE_COMPLETE_FINAL); InferenceRequest::Release( std::move(request), TRITONSERVER_REQUEST_RELEASE_ALL); return Status::Success; } // Add additional callback to keep track of in-flight count ++inflight_count_; request->AddInternalReleaseCallback( [this]( std::unique_ptr& request, const uint32_t flags) -> Status { --inflight_count_; return Status::Success; }); // Consider the top-level "ensemble" request executing once passed to a // composing model. Composing model requests will track their own states. RETURN_IF_ERROR(request->SetState(InferenceRequest::State::EXECUTING)); std::shared_ptr context(new EnsembleContext( metric_reporter_.get(), stats_aggregator_, is_, info_.get(), request, stream_)); EnsembleContext::Proceed(context); return Status::Success; } EnsembleScheduler::EnsembleScheduler( InferenceStatsAggregator* const stats_aggregator, InferenceServer* const server, const ModelIdentifier& model_id, const inference::ModelConfig& config) : stats_aggregator_(stats_aggregator), is_(server), stream_(nullptr), inflight_count_(0) { #ifdef TRITON_ENABLE_GPU // create CUDA stream auto cuerr = cudaStreamCreate(&stream_); if (cuerr != cudaSuccess) { stream_ = nullptr; LOG_ERROR << "unable to create stream for " << config.name() << ": " << cudaGetErrorString(cuerr); } #endif // TRITON_ENABLE_GPU #ifdef TRITON_ENABLE_METRICS if (Metrics::Enabled()) { // Ensemble scheduler doesn't currently support response cache at top level. MetricModelReporter::Create( model_id, 1 /* model_version */, METRIC_REPORTER_ID_CPU, false /* response_cache_enabled */, config.metric_tags(), &metric_reporter_); } #endif // TRITON_ENABLE_METRICS // Set 'info_' based on 'config' info_.reset(new EnsembleInfo()); info_->ensemble_name_ = config.name(); // This config field is filled internally for ensemble models info_->is_decoupled_ = config.model_transaction_policy().decoupled(); // field to check if response cache enabled in the ensemble model config. info_->is_cache_enabled_ = config.response_cache().enable() && is_->ResponseCacheEnabled(); for (const auto& input : config.input()) { info_->tensor_to_step_.emplace(input.name(), std::set()); if (input.optional()) { info_->optional_inputs_.emplace(input.name()); } } for (const auto& output : config.output()) { info_->tensor_to_step_.emplace(output.name(), std::set()); if (output.has_reshape()) { info_->ensemble_output_shape_[output.name()] = output.reshape().shape(); } else { info_->ensemble_output_shape_[output.name()] = output.dims(); } } for (const auto& element : config.ensemble_scheduling().step()) { size_t step_idx = info_->steps_.size(); info_->steps_.emplace_back( ModelIdentifier(element.model_namespace(), element.model_name()), element.model_version()); for (const auto& pair : element.input_map()) { auto it = info_->tensor_to_step_.find(pair.second); if (it == info_->tensor_to_step_.end()) { it = info_->tensor_to_step_.emplace(pair.second, std::set()) .first; } it->second.insert(step_idx); info_->steps_[step_idx].input_to_tensor_.emplace( std::make_pair(pair.first, pair.second)); } for (const auto& pair : element.output_map()) { auto it = info_->tensor_to_step_.find(pair.second); if (it == info_->tensor_to_step_.end()) { it = info_->tensor_to_step_.emplace(pair.second, std::set()) .first; } info_->steps_[step_idx].output_to_tensor_.emplace( std::make_pair(pair.first, pair.second)); info_->tensor_to_prev_step_.emplace(pair.second, step_idx); } } } EnsembleScheduler::~EnsembleScheduler() { #ifdef TRITON_ENABLE_GPU if (stream_ != nullptr) { cudaError_t err = cudaStreamDestroy(stream_); if (err != cudaSuccess) { LOG_ERROR << "Failed to destroy cuda stream: " << cudaGetErrorString(err); } } #endif // TRITON_ENABLE_GPU } }} // namespace triton::core #endif // TRITON_ENABLE_ENSEMBLE kytensor-2.49.0.7/core/src/ensemble_scheduler/ensemble_scheduler.h000066400000000000000000000107731527052371200252150ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef TRITON_ENABLE_ENSEMBLE #include #include "metric_model_reporter.h" #include "model.h" #include "model_config.pb.h" #include "model_config_utils.h" #include "scheduler.h" #include "scheduler_utils.h" #include "status.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace core { #ifndef TRITON_ENABLE_GPU using cudaStream_t = void*; #endif // TRITON_ENABLE_GPU class InferenceServer; struct EnsembleInfo { struct StepInfo { StepInfo(const ModelIdentifier& model_id, const int64_t model_version) : model_id_(model_id), model_version_(model_version) { } ModelIdentifier model_id_; int64_t model_version_; std::unordered_map input_to_tensor_; std::unordered_map output_to_tensor_; }; std::string ensemble_name_; bool is_decoupled_; bool is_cache_enabled_; // the ensemble output (re)shape expected by the ensemble std::unordered_map ensemble_output_shape_; // Inputs that is marked optional for the ensemble std::set optional_inputs_; std::vector steps_; // Only include a step if the ensemble tensor is used as input in that step std::unordered_map> tensor_to_step_; // backward path, ensemble tensor to the step that provides its data std::unordered_map tensor_to_prev_step_; }; // Scheduler that implements ensemble scheduling. class EnsembleScheduler : public Scheduler { public: // Create a scheduler to process ensemble requests and // to dispatch requests to models in ensemble internally. static Status Create( InferenceStatsAggregator* const stats_aggregator, InferenceServer* const server, const ModelIdentifier& model_id, const inference::ModelConfig& config, std::unique_ptr* scheduler); ~EnsembleScheduler(); // \see Scheduler::Enqueue() Status Enqueue(std::unique_ptr& request) override; // \see Scheduler::InflightInferenceCount() size_t InflightInferenceCount() override { return inflight_count_; } // \see Scheduler::Stop() void Stop() override {} private: EnsembleScheduler( InferenceStatsAggregator* const stats_aggregator, InferenceServer* const server, const ModelIdentifier& model_id, const inference::ModelConfig& config); void CacheLookUp( std::unique_ptr& request, std::unique_ptr& cached_response); std::shared_ptr metric_reporter_; InferenceStatsAggregator* const stats_aggregator_; InferenceServer* const is_; // Ensemble information that is built from model config std::unique_ptr info_; // The stream used for data transfer. cudaStream_t stream_; std::atomic inflight_count_; }; }} // namespace triton::core #endif // TRITON_ENABLE_ENSEMBLE kytensor-2.49.0.7/core/src/ensemble_scheduler/ensemble_utils.cc000066400000000000000000000345201527052371200245310ustar00rootroot00000000000000// Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef TRITON_ENABLE_ENSEMBLE #include "ensemble_utils.h" #include #include "constants.h" #include "model.h" #include "model_config_utils.h" #include "triton/common/logging.h" namespace triton { namespace core { namespace { /// A basic unit in ensemble graph that records the data type and shape /// of the ensemble tensor and which model they are inferred from. struct TensorNode { TensorNode( const ModelIdentifier& model_id, const bool batching, const inference::DataType& type, const triton::common::DimsList& dims) : model_id_(model_id), type_(type), dims_(dims), is_decoupled_(false), decouple_label_(0), visited_(false) { // Expand dims to full shape, which includes batch dimension if exist if (batching) { full_dims_.Add(-1); } full_dims_.MergeFrom(dims_); } // Constructor for symbolic nodes TensorNode(const ModelIdentifier& model_id) : model_id_(model_id), is_decoupled_(false), decouple_label_(0), visited_(false) { } ModelIdentifier model_id_; inference::DataType type_; triton::common::DimsList dims_; triton::common::DimsList full_dims_; bool is_decoupled_; size_t decouple_label_; bool visited_; std::vector prev_nodes_; std::vector next_nodes_; // A symbolic node to keep track of the decouple label of nodes that // are outputs of the same step. std::shared_ptr sibling_node_; }; /// Validate if the data type and the shape of two TensorNode object are /// consistent. /// \param lhs One of the TensorNode object to be validated. /// \param rhs Another TensorNode object to be validated. /// \param message Extra message included in the front of error message /// if error status is non-OK. /// \return The error status. A non-OK status indicates the TensorNode objects /// are not consistent. Status ValidateTensorConsistency( const TensorNode& lhs, const TensorNode& rhs, const std::string& message) { if (lhs.type_ != rhs.type_) { return Status( Status::Code::INVALID_ARG, message + "inconsistent data type: " + inference::DataType_Name(lhs.type_) + " is inferred from model " + lhs.model_id_.str() + " while " + inference::DataType_Name(rhs.type_) + " is inferred from model " + rhs.model_id_.str()); } // Shapes must match or either one uses variable size shape, if one uses // variable size shape, shape consistency will be checked at runtime. // If dims mismatch, compare again with full dims in case the tensor is // used for both non-batching model and batching model. In that case, it // is acceptable if non-batching model shape is [-1, d_0, d_1, ..., d_n] // while the batching model shape is [d_0, d_1, ..., d_n]. if (!triton::common::CompareDimsWithWildcard(lhs.dims_, rhs.dims_) && !triton::common::CompareDimsWithWildcard( lhs.full_dims_, rhs.full_dims_)) { return Status( Status::Code::INVALID_ARG, message + "inconsistent shape: " + triton::common::DimsListToString(lhs.full_dims_) + " is inferred from model " + lhs.model_id_.str() + " while " + triton::common::DimsListToString(rhs.full_dims_) + " is inferred from model " + rhs.model_id_.str()); } return Status::Success; } Status ValidateTensorMapping( const ModelIdentifier& ensemble_id, const inference::ModelEnsembling::Step& step, const inference::ModelConfig& model_config, std::unordered_map* ensemble_tensors) { ModelIdentifier step_id{step.model_namespace(), step.model_name()}; const bool batching = (model_config.max_batch_size() > 0); // Check all inputs are mapped and no mapping to invalid inputs std::set input_names; for (const auto& model_input : model_config.input()) { input_names.insert(model_input.name()); } for (const auto& input_map : step.input_map()) { if (input_names.find(input_map.first) == input_names.end()) { return Status( Status::Code::INVALID_ARG, "in ensemble " + ensemble_id.str() + ", ensemble tensor " + input_map.second + " is mapping to non-existing input " + input_map.first + " in model " + step_id.str()); } } for (const auto& model_input : model_config.input()) { size_t mapped_cnt = 0; for (const auto& input_map : step.input_map()) { if (model_input.name() == input_map.first) { TensorNode model_tensor( step_id, batching, model_input.data_type(), model_input.dims()); auto it = ensemble_tensors->find(input_map.second); if (it != ensemble_tensors->end()) { RETURN_IF_ERROR(ValidateTensorConsistency( it->second, model_tensor, "in ensemble " + ensemble_id.str() + ", ensemble tensor " + input_map.second + ": ")); } else { ensemble_tensors->emplace( std::make_pair(input_map.second, model_tensor)); } mapped_cnt++; } } if (mapped_cnt == 0) { // Allow the input to be excluded from ensemble if it is optional if (model_input.optional()) { continue; } return Status( Status::Code::INVALID_ARG, "in ensemble " + ensemble_id.str() + ", input " + model_input.name() + " in model " + step_id.str() + " is not mapped to any ensemble tensors"); } else if (mapped_cnt > 1) { return Status( Status::Code::INVALID_ARG, "in ensemble " + ensemble_id.str() + ", input " + model_input.name() + " in model " + step_id.str() + " is mapped to multiple ensemble tensors"); } } // Check no multiple mappings to same ensemble tensor // and no mapping from invalid outputs std::set output_names; for (const auto& model_output : model_config.output()) { output_names.insert(model_output.name()); } for (const auto& output_map : step.output_map()) { if (output_names.find(output_map.first) == output_names.end()) { return Status( Status::Code::INVALID_ARG, "in ensemble " + ensemble_id.str() + ", ensemble tensor " + output_map.second + " is mapped from non-existing output " + output_map.first + " in model " + step.model_name()); } } std::shared_ptr sibling_node(new TensorNode(step_id)); for (const auto& output_map : step.output_map()) { size_t mapped_cnt = 0; for (const auto& model_output : model_config.output()) { if (model_output.name() == output_map.first) { TensorNode model_tensor( step_id, batching, model_output.data_type(), model_output.dims()); auto it = ensemble_tensors->find(output_map.second); if (it != ensemble_tensors->end()) { RETURN_IF_ERROR(ValidateTensorConsistency( it->second, model_tensor, "in ensemble " + ensemble_id.str() + ", ensemble tensor " + output_map.second + ": ")); } else { it = ensemble_tensors ->emplace(std::make_pair(output_map.second, model_tensor)) .first; } it->second.sibling_node_ = sibling_node; mapped_cnt++; } } if (mapped_cnt > 1) { return Status( Status::Code::INVALID_ARG, "in ensemble " + ensemble_id.str() + ", multiple outputs in model " + step_id.str() + " are mapped to the same ensemble tensor " + output_map.second); } } // link ensemble tensors bool is_decoupled = model_config.model_transaction_policy().decoupled(); for (const auto& output_map : step.output_map()) { auto& node = ensemble_tensors->find(output_map.second)->second; node.is_decoupled_ = is_decoupled; for (const auto& input_map : step.input_map()) { auto& prev_node = ensemble_tensors->find(input_map.second)->second; node.prev_nodes_.push_back(&prev_node); prev_node.next_nodes_.push_back(&node); } } return Status::Success; } } // namespace Status ValidateEnsembleConfig( ModelRepositoryManager* model_repository_manager, ModelRepositoryManager::DependencyNode* ensemble) { const auto& ensemble_config = ensemble->model_config_; if (!ensemble_config.has_ensemble_scheduling()) { return Status::Success; } const auto& ensemble_id = ensemble->model_id_; const bool batching = (ensemble_config.max_batch_size() > 0); std::unordered_map ensemble_tensors; for (const auto& input : ensemble_config.input()) { const auto& dims = input.has_reshape() ? input.reshape().shape() : input.dims(); TensorNode input_node(ensemble_id, batching, input.data_type(), dims); ensemble_tensors.emplace(std::make_pair(input.name(), input_node)); } TensorNode sink_node(ensemble_id); for (const auto& output : ensemble_config.output()) { const auto& dims = output.has_reshape() ? output.reshape().shape() : output.dims(); TensorNode output_node(ensemble_id, batching, output.data_type(), dims); auto it = ensemble_tensors.emplace(std::make_pair(output.name(), output_node)) .first; sink_node.prev_nodes_.emplace_back(&(it->second)); it->second.next_nodes_.emplace_back(&sink_node); } for (const auto& step : ensemble_config.ensemble_scheduling().step()) { const ModelIdentifier model_id{step.model_namespace(), step.model_name()}; inference::ModelConfig model_config; for (auto& node : ensemble->upstreams_) { if (model_id == node.first->model_id_) { // Obtain completed config from model instance std::shared_ptr model; RETURN_IF_ERROR( model_repository_manager->GetModel(model_id, -1, &model)); model_config = model->Config(); break; } } // batchable ensemble can include non-batchable models as long as // the expanded shapes are consistent if ((model_config.max_batch_size() != 0) && (model_config.max_batch_size() < ensemble_config.max_batch_size())) { return Status( Status::Code::INVALID_ARG, "ensemble " + ensemble_id.str() + " allows maximum batch size " + std::to_string(ensemble_config.max_batch_size()) + ", but it contains model " + step.model_name() + " which only allows maximum batch size to be " + std::to_string(model_config.max_batch_size())); } RETURN_IF_ERROR(ValidateTensorMapping( ensemble_id, step, model_config, &ensemble_tensors)); } // Visit nodes and validate decoupled workflow if any // check data flow size_t decouple_label = 0; std::deque current_iterators; for (const auto& input : ensemble_config.input()) { auto it = ensemble_tensors.find(input.name()); it->second.visited_ = true; current_iterators.push_back(&(it->second)); } while (!current_iterators.empty()) { auto& current_node = current_iterators.front(); for (auto& next_node : current_node->next_nodes_) { if (next_node->visited_) { continue; } bool next_node_ready = true; for (auto& prev_node : next_node->prev_nodes_) { if (!prev_node->visited_) { next_node_ready = false; break; } } if (next_node_ready) { size_t prev_decouple_label = next_node->prev_nodes_[0]->decouple_label_; for (auto& prev_node : next_node->prev_nodes_) { if (prev_node->decouple_label_ != prev_decouple_label) { return Status( Status::Code::INVALID_ARG, "in ensemble " + ensemble_id.str() + ", step of model '" + next_node->model_id_.str() + "' receives inputs originated from different decoupled " "models"); } } if (next_node->sibling_node_ != nullptr) { if (next_node->sibling_node_->visited_) { next_node->decouple_label_ = next_node->sibling_node_->decouple_label_; } else { next_node->decouple_label_ = next_node->is_decoupled_ ? ++decouple_label : prev_decouple_label; next_node->sibling_node_->decouple_label_ = next_node->decouple_label_; next_node->sibling_node_->visited_ = true; } } else { next_node->decouple_label_ = next_node->is_decoupled_ ? ++decouple_label : prev_decouple_label; } next_node->visited_ = true; current_iterators.push_back(next_node); } } current_iterators.pop_front(); } ensemble->model_config_.mutable_model_transaction_policy()->set_decoupled( decouple_label != 0); return Status::Success; } }} // namespace triton::core #endif // TRITON_ENABLE_ENSEMBLE kytensor-2.49.0.7/core/src/ensemble_scheduler/ensemble_utils.h000066400000000000000000000045021527052371200243700ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef TRITON_ENABLE_ENSEMBLE #include #include #include "model_config.pb.h" #include "model_repository_manager/model_repository_manager.h" #include "status.h" #include "triton/common/model_config.h" namespace triton { namespace core { /// Validate that the ensemble are specified correctly. Assuming that the /// inputs and outputs specified in depending model configurations are accurate. /// \param model_repository_manager The model manager to acquire model config. /// \param ensemble The ensemble to be validated. /// \return The error status. Status ValidateEnsembleConfig( ModelRepositoryManager* model_repository_manager, ModelRepositoryManager::DependencyNode* ensemble); }} // namespace triton::core #endif // TRITON_ENABLE_ENSEMBLE kytensor-2.49.0.7/core/src/filesystem/000077500000000000000000000000001527052371200175405ustar00rootroot00000000000000kytensor-2.49.0.7/core/src/filesystem/api.cc000066400000000000000000000525171527052371200206320ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "api.h" #include "implementations/common.h" // Different file system implementations #include "implementations/local.h" #ifdef TRITON_ENABLE_GCS #include "implementations/gcs.h" #endif // TRITON_ENABLE_GCS #ifdef TRITON_ENABLE_S3 #include "implementations/s3.h" #endif // TRITON_ENABLE_S3 #ifdef TRITON_ENABLE_AZURE_STORAGE #include "implementations/as.h" #endif // TRITON_ENABLE_AZURE_STORAGE #include #include #include #include #include #include namespace triton { namespace core { LocalizedPath::~LocalizedPath() { if (!local_path_.empty()) { bool is_dir = true; IsDirectory(local_path_, &is_dir); LOG_STATUS_ERROR( DeletePath(is_dir ? local_path_ : DirName(local_path_)), "failed to delete localized path"); } } namespace { class FileSystemManager { public: Status GetFileSystem( const std::string& path, std::shared_ptr& file_system); Status GetFileSystem( FileSystemType type, std::shared_ptr& file_system); FileSystemManager(); private: template Status GetFileSystem( const std::string& path, CacheType& cache, std::shared_ptr& file_system); template Status ReturnErrorOrReload( const Status& load_status, const Status& error_status, const std::string& path, CacheType& cache, std::shared_ptr& file_system); Status LoadCredentials(bool flush_cache = false); template static void LoadCredential( triton::common::TritonJson::Value& creds_json, const char* fs_type, CacheType& cache); template static void SortCache( std::vector>>& cache); template static Status GetLongestMatchingNameIndex( const std::vector>>& cache, const std::string& path, size_t& idx); std::shared_ptr local_fs_; std::mutex mu_; // protect concurrent access into variables bool is_cached_; // if name and credential is cached, lazy load file system // cloud credential cache should be sorted in descending name length order // [(name_long, credential, file_system), (name, ...)] #ifdef TRITON_ENABLE_GCS std::vector< std::tuple>> gs_cache_; #endif // TRITON_ENABLE_GCS #ifdef TRITON_ENABLE_S3 std::vector< std::tuple>> s3_cache_; #endif // TRITON_ENABLE_S3 #ifdef TRITON_ENABLE_AZURE_STORAGE std::vector< std::tuple>> as_cache_; #endif // TRITON_ENABLE_AZURE_STORAGE }; FileSystemManager::FileSystemManager() : local_fs_(new LocalFileSystem()), is_cached_(false) { } Status FileSystemManager::GetFileSystem( const std::string& path, std::shared_ptr& file_system) { // Check if this is a GCS path (gs://$BUCKET_NAME) if (!path.empty() && !path.rfind("gs://", 0)) { #ifndef TRITON_ENABLE_GCS return Status( Status::Code::INTERNAL, "gs:// file-system not supported. To enable, build with " "-DTRITON_ENABLE_GCS=ON."); #else return GetFileSystem< std::vector>>, GCSCredential, GCSFileSystem>(path, gs_cache_, file_system); #endif // TRITON_ENABLE_GCS } // Check if this is an S3 path (s3://$BUCKET_NAME) if (!path.empty() && !path.rfind("s3://", 0)) { #ifndef TRITON_ENABLE_S3 return Status( Status::Code::INTERNAL, "s3:// file-system not supported. To enable, build with " "-DTRITON_ENABLE_S3=ON."); #else return GetFileSystem< std::vector>>, S3Credential, S3FileSystem>(path, s3_cache_, file_system); #endif // TRITON_ENABLE_S3 } // Check if this is an Azure Storage path if (!path.empty() && !path.rfind("as://", 0)) { #ifndef TRITON_ENABLE_AZURE_STORAGE return Status( Status::Code::INTERNAL, "as:// file-system not supported. To enable, build with " "-DTRITON_ENABLE_AZURE_STORAGE=ON."); #else return GetFileSystem< std::vector>>, ASCredential, ASFileSystem>(path, as_cache_, file_system); #endif // TRITON_ENABLE_AZURE_STORAGE } // Assume path is for local filesystem file_system = local_fs_; return Status::Success; } Status FileSystemManager::GetFileSystem( FileSystemType type, std::shared_ptr& file_system) { // only LOCAL and GCS are not path-dependent and can be accessed by type switch (type) { case FileSystemType::LOCAL: return GetFileSystem("", file_system); case FileSystemType::GCS: return GetFileSystem("gs://", file_system); case FileSystemType::S3: return Status( Status::Code::UNSUPPORTED, "S3 filesystem cannot be accessed by type"); case FileSystemType::AS: return Status( Status::Code::UNSUPPORTED, "AS filesystem cannot be accessed by type"); default: return Status(Status::Code::UNSUPPORTED, "Unsupported filesystem type"); } } template Status FileSystemManager::GetFileSystem( const std::string& path, CacheType& cache, std::shared_ptr& file_system) { const Status& cred_status = LoadCredentials(); if (cred_status.IsOk() || cred_status.StatusCode() == Status::Code::ALREADY_EXISTS) { // Find credential size_t idx; const Status& match_status = GetLongestMatchingNameIndex(cache, path, idx); if (!match_status.IsOk()) { return ReturnErrorOrReload( cred_status, match_status, path, cache, file_system); } // Find or lazy load file system std::shared_ptr fs = std::get<2>(cache[idx]); if (fs == nullptr) { std::string cred_name = std::get<0>(cache[idx]); CredentialType cred = std::get<1>(cache[idx]); fs = std::make_shared(path, cred); cache[idx] = std::make_tuple(cred_name, cred, fs); } // Check client const Status& client_status = fs->CheckClient(path); if (!client_status.IsOk()) { return ReturnErrorOrReload( cred_status, client_status, path, cache, file_system); } // Return client file_system = fs; return Status::Success; } return cred_status; } template Status FileSystemManager::ReturnErrorOrReload( const Status& load_status, const Status& error_status, const std::string& path, CacheType& cache, std::shared_ptr& file_system) { if (load_status.StatusCode() == Status::Code::ALREADY_EXISTS) { return error_status; } LoadCredentials(true); // flush cache return GetFileSystem( path, cache, file_system); } // return status meaning: // - SUCCESS, "" -> loaded credential from file // - ALREADY_EXISTS, "Cached" -> credential already loaded Status FileSystemManager::LoadCredentials(bool flush_cache) { // prevent concurrent access into class variables std::lock_guard lock(mu_); // check if credential is already cached if (is_cached_ && !flush_cache) { return Status(Status::Code::ALREADY_EXISTS, "Cached"); } const char* file_path_c_str = std::getenv("TRITON_CLOUD_CREDENTIAL_PATH"); if (file_path_c_str != nullptr) { // Load from credential file std::string file_path = std::string(file_path_c_str); LOG_VERBOSE(1) << "Reading cloud credential from " << file_path; triton::common::TritonJson::Value creds_json; std::string cred_file_content; RETURN_IF_ERROR(local_fs_->ReadTextFile(file_path, &cred_file_content)); RETURN_IF_ERROR(creds_json.Parse(cred_file_content)); #ifdef TRITON_ENABLE_GCS // load GCS credentials LoadCredential< std::vector>>, GCSCredential, GCSFileSystem>(creds_json, "gs", gs_cache_); #endif // TRITON_ENABLE_GCS #ifdef TRITON_ENABLE_S3 // load S3 credentials LoadCredential< std::vector>>, S3Credential, S3FileSystem>(creds_json, "s3", s3_cache_); #endif // TRITON_ENABLE_S3 #ifdef TRITON_ENABLE_AZURE_STORAGE // load AS credentials LoadCredential< std::vector>>, ASCredential, ASFileSystem>(creds_json, "as", as_cache_); #endif // TRITON_ENABLE_AZURE_STORAGE } else { // Load from environment variables LOG_VERBOSE(1) << "TRITON_CLOUD_CREDENTIAL_PATH environment variable is " "not set, reading from environment variables"; #ifdef TRITON_ENABLE_GCS // load GCS credentials gs_cache_.clear(); gs_cache_.push_back( std::make_tuple("", GCSCredential(), std::shared_ptr())); #endif // TRITON_ENABLE_GCS #ifdef TRITON_ENABLE_S3 // load S3 credentials s3_cache_.clear(); s3_cache_.push_back( std::make_tuple("", S3Credential(), std::shared_ptr())); #endif // TRITON_ENABLE_S3 #ifdef TRITON_ENABLE_AZURE_STORAGE // load AS credentials as_cache_.clear(); as_cache_.push_back( std::make_tuple("", ASCredential(), std::shared_ptr())); #endif // TRITON_ENABLE_AZURE_STORAGE } is_cached_ = true; return Status::Success; } template void FileSystemManager::LoadCredential( triton::common::TritonJson::Value& creds_json, const char* fs_type, CacheType& cache) { cache.clear(); triton::common::TritonJson::Value creds_fs_json; if (creds_json.Find(fs_type, &creds_fs_json)) { std::vector cred_names; creds_fs_json.Members(&cred_names); for (size_t i = 0; i < cred_names.size(); i++) { std::string cred_name = cred_names[i]; triton::common::TritonJson::Value cred_json; creds_fs_json.Find(cred_name.c_str(), &cred_json); cache.push_back(std::make_tuple( cred_name, CredentialType(cred_json), std::shared_ptr())); } SortCache(cache); } } template void FileSystemManager::SortCache( std::vector>>& cache) { std::sort( cache.begin(), cache.end(), [](std::tuple< std::string, CredentialType, std::shared_ptr> a, std::tuple< std::string, CredentialType, std::shared_ptr> b) { return std::get<0>(a).size() >= std::get<0>(b).size(); }); } template Status FileSystemManager::GetLongestMatchingNameIndex( const std::vector>>& cache, const std::string& path, size_t& idx) { for (size_t i = 0; i < cache.size(); i++) { if (!path.rfind(std::get<0>(cache[i]), 0)) { idx = i; LOG_VERBOSE(1) << "Using credential " + std::get<0>(cache[i]) + " for path " + path; return Status::Success; } } return Status( Status::Code::NOT_FOUND, "Cannot match credential for path " + path); } static FileSystemManager fsm_; } // namespace // FIXME: Windows support '/'? If so, the below doesn't need to change bool IsAbsolutePath(const std::string& path) { return !path.empty() && (path[0] == '/'); } bool IsChildPathEscapingParentPath( const std::string& child_path, const std::string& parent_path) { const std::string absolute_child_path = std::filesystem::weakly_canonical(child_path).string(); const std::string absolute_parent_path = std::filesystem::canonical(parent_path).string(); // Can use starts_with() over rfind() in C++20. bool is_escape = absolute_child_path.rfind(absolute_parent_path, 0) != 0; return is_escape; } std::string JoinPath(std::initializer_list segments) { std::string joined; for (const auto& seg : segments) { if (joined.empty()) { joined = seg; } else if (IsAbsolutePath(seg)) { if (joined[joined.size() - 1] == '/') { joined.append(seg.substr(1)); } else { joined.append(seg); } } else { // !IsAbsolutePath(seg) if (joined[joined.size() - 1] != '/') { joined.append("/"); } joined.append(seg); } } return joined; } std::string BaseName(const std::string& path) { if (path.empty()) { return path; } size_t last = path.size() - 1; while ((last > 0) && (path[last] == '/')) { last -= 1; } if (path[last] == '/') { return std::string(); } const size_t idx = path.find_last_of("/", last); if (idx == std::string::npos) { return path.substr(0, last + 1); } return path.substr(idx + 1, last - idx); } std::string DirName(const std::string& path) { if (path.empty()) { return path; } size_t last = path.size() - 1; while ((last > 0) && (path[last] == '/')) { last -= 1; } if (path[last] == '/') { return std::string("/"); } const size_t idx = path.find_last_of("/", last); if (idx == std::string::npos) { return std::string("."); } if (idx == 0) { return std::string("/"); } return path.substr(0, idx); } Status FileExists(const std::string& path, bool* exists) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->FileExists(path, exists); } Status IsDirectory(const std::string& path, bool* is_dir) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->IsDirectory(path, is_dir); } Status FileModificationTime(const std::string& path, int64_t* mtime_ns) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->FileModificationTime(path, mtime_ns); } Status GetDirectoryContents(const std::string& path, std::set* contents) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->GetDirectoryContents(path, contents); } Status GetDirectorySubdirs(const std::string& path, std::set* subdirs) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->GetDirectorySubdirs(path, subdirs); } Status GetDirectoryFiles( const std::string& path, const bool skip_hidden_files, std::set* files) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); std::set all_files; RETURN_IF_ERROR(fs->GetDirectoryFiles(path, &all_files)); // Remove the hidden files for (auto f : all_files) { if ((f[0] != '.') || (!skip_hidden_files)) { files->insert(f); } } return Status::Success; } Status ReadTextFile(const std::string& path, std::string* contents) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->ReadTextFile(path, contents); } Status ReadTextProto(const std::string& path, google::protobuf::Message* msg) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); std::string contents; RETURN_IF_ERROR(fs->ReadTextFile(path, &contents)); if (!google::protobuf::TextFormat::ParseFromString(contents, msg)) { return Status( Status::Code::INTERNAL, "failed to read text proto from " + path); } return Status::Success; } Status LocalizePath(const std::string& path, std::shared_ptr* localized) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->LocalizePath(path, localized); } Status WriteTextProto(const std::string& path, const google::protobuf::Message& msg) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); std::string prototxt; if (!google::protobuf::TextFormat::PrintToString(msg, &prototxt)) { return Status( Status::Code::INTERNAL, "failed to write text proto to " + path); } return fs->WriteTextFile(path, prototxt); } Status WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->WriteBinaryFile(path, contents, content_len); } Status ReadBinaryProto(const std::string& path, google::protobuf::MessageLite* msg) { std::string msg_str; RETURN_IF_ERROR(ReadTextFile(path, &msg_str)); google::protobuf::io::CodedInputStream coded_stream( reinterpret_cast(msg_str.c_str()), msg_str.size()); coded_stream.SetTotalBytesLimit(INT_MAX); if (!msg->ParseFromCodedStream(&coded_stream)) { return Status( Status::Code::INTERNAL, "Can't parse " + path + " as binary proto"); } return Status::Success; } Status MakeDirectory(const std::string& dir, const bool recursive) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(dir, fs)); return fs->MakeDirectory(dir, recursive); } Status MakeTemporaryDirectory(const FileSystemType type, std::string* temp_dir) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(type, fs)); return fs->MakeTemporaryDirectory(kDefaultMountDirectory, temp_dir); } Status MakeTemporaryDirectory( const FileSystemType type, std::string dir_path, std::string* temp_dir) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(type, fs)); return fs->MakeTemporaryDirectory(dir_path, temp_dir); } Status DeletePath(const std::string& path) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(path, fs)); return fs->DeletePath(path); } Status GetFileSystemType(const std::string& path, FileSystemType* type) { if (path.empty()) { return Status( Status::Code::INVALID_ARG, "Can not infer filesystem type from empty path"); } #ifdef TRITON_ENABLE_GCS // Check if this is a GCS path (gs://$BUCKET_NAME) if (!path.rfind("gs://", 0)) { *type = FileSystemType::GCS; return Status::Success; } #endif // TRITON_ENABLE_GCS #ifdef TRITON_ENABLE_S3 // Check if this is an S3 path (s3://$BUCKET_NAME) if (!path.rfind("s3://", 0)) { *type = FileSystemType::S3; return Status::Success; } #endif // TRITON_ENABLE_S3 #ifdef TRITON_ENABLE_AZURE_STORAGE // Check if this is an Azure Storage path if (!path.rfind("as://", 0)) { *type = FileSystemType::AS; return Status::Success; } #endif // TRITON_ENABLE_AZURE_STORAGE // Assume path is for local filesystem *type = FileSystemType::LOCAL; return Status::Success; } const std::string& FileSystemTypeString(const FileSystemType type) { static const std::string local_str("LOCAL"); static const std::string gcs_str("GCS"); static const std::string s3_str("S3"); static const std::string as_str("AS"); static const std::string unknown_str("UNKNOWN"); switch (type) { case FileSystemType::LOCAL: return local_str; case FileSystemType::GCS: return gcs_str; case FileSystemType::S3: return s3_str; case FileSystemType::AS: return as_str; default: return unknown_str; } } }} // namespace triton::core kytensor-2.49.0.7/core/src/filesystem/api.h000066400000000000000000000220601527052371200204620ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "../status.h" #include "google/protobuf/message.h" namespace triton { namespace core { enum class FileSystemType { LOCAL, GCS, S3, AS }; // This class stores the paths of local temporary files needed for loading // models from Cloud repositories and performs necessary cleanup after the // models are loaded. class LocalizedPath { public: // Create an object for a path that is already local. LocalizedPath(const std::string& original_path) : original_path_(original_path) { } // Create an object for a remote path. Store both the original path and the // temporary local path. LocalizedPath(const std::string& original_path, const std::string& local_path) : original_path_(original_path), local_path_(local_path) { } // Destructor. Remove temporary local storage associated with the object. // If the local path is a directory, delete the directory. // If the local path is a file, delete the directory containing the file. ~LocalizedPath(); // Return the localized path represented by this object. const std::string& Path() const { return (local_path_.empty()) ? original_path_ : local_path_; } // Maintain a vector of LocalizedPath that should be kept available in the // tmp directory for the lifetime of this object // FIXME: Remove when no longer required std::vector> other_localized_path; private: std::string original_path_; std::string local_path_; }; /// Is a path an absolute path? /// \param path The path. /// \return true if absolute path, false if relative path. bool IsAbsolutePath(const std::string& path); /// Check if the child path escapes from its parent path. /// \param child_path The child path. /// \param parent_path The parent path. The path must exist. /// \return true if the child path escapes from its parent path, false if the /// child path is within its parent path. bool IsChildPathEscapingParentPath( const std::string& child_path, const std::string& parent_path); /// Join path segments into a longer path /// \param segments The path segments. /// \return the path formed by joining the segments. std::string JoinPath(std::initializer_list segments); /// Get the basename of a path. /// \param path The path. /// \return the last segment of the path. std::string BaseName(const std::string& path); /// Get the dirname of a path. /// \param path The path. /// \return all but the last segment of the path. std::string DirName(const std::string& path); /// Does a file or directory exist? /// \param path The path to check for existence. /// \param exists Returns true if file/dir exists /// \return Error status if unable to perform the check Status FileExists(const std::string& path, bool* exists); /// Is a path a directory? /// \param path The path to check. /// \param is_dir Returns true if path represents a directory /// \return Error status Status IsDirectory(const std::string& path, bool* is_dir); /// Get file modification time in nanoseconds. /// A file is considered modified in Triton when its binary content has changed /// including the action of replacing it with another file. /// \param path The path. /// \param mtime_ns Returns the file modification time. For some filesystems a /// file/folder may not have a modification time, in that case return 0. /// \return Error status Status FileModificationTime(const std::string& path, int64_t* mtime_ns); /// Get the contents of a directory. /// \param path The directory path. /// \param subdirs Returns the directory contents. /// \return Error status Status GetDirectoryContents( const std::string& path, std::set* contents); /// Get the sub-directories of a path. /// \param path The path. /// \param subdirs Returns the names of the sub-directories. /// \return Error status Status GetDirectorySubdirs( const std::string& path, std::set* subdirs); /// Get the files contained in a directory. /// \param path The directory. /// \param skip_hidden_files Ignores the hidden files in the directory. /// \param files Returns the names of the files. /// \return Error status Status GetDirectoryFiles( const std::string& path, const bool skip_hidden_files, std::set* files); /// Read a text file into a string. /// \param path The path of the file. /// \param contents Returns the contents of the file. /// \return Error status Status ReadTextFile(const std::string& path, std::string* contents); /// Create an object representing a local copy of a path. /// \param path The path of the directory or file. /// \param localized Returns the LocalizedPath object /// representing the local copy of the path. /// \return Error status Status LocalizePath( const std::string& path, std::shared_ptr* localized); /// Write a string to a file. /// \param path The path of the file. /// \param contents The contents to write to the file. /// \return Error status Status WriteTextFile(const std::string& path, const std::string& contents); /// Write binary to a file. /// \param path The path of the file. /// \param contents The contents to write to the file. /// \param content_len The size of the content. /// \return Error status Status WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len); /// Read a prototext file. /// \param path The path of the file. /// \param msg Returns the protobuf message for the file. /// \return Error status Status ReadTextProto(const std::string& path, google::protobuf::Message* msg); /// Write a prototext file. /// \param path The path of the file. /// \param msg The protobuf to write. /// \return Error status Status WriteTextProto( const std::string& path, const google::protobuf::Message& msg); /// Read a binary protobuf file. /// \param path The path of the file. /// \param msg Returns the protobuf message for the file. /// \return Error status Status ReadBinaryProto( const std::string& path, google::protobuf::MessageLite* msg); /// Create a directory of the specified path. /// \param dir The path to the directory. /// \param recursive Whether the parent directories will be created /// if not exist. /// \return Error status if the directory can't be created Status MakeDirectory(const std::string& dir, const bool recursive); /// Create a temporary directory of the specified filesystem type. /// \param type The type of the filesystem. /// \param temp_dir Returns the path to the temporary directory. /// \return Error status Status MakeTemporaryDirectory(const FileSystemType type, std::string* temp_dir); /// Create a temporary directory of the specified filesystem type /// in a provided location. /// \param type The type of the filesystem. /// \param dir_path The specified path. /// \param temp_dir Returns the path to the temporary directory. /// \return Error status Status MakeTemporaryDirectory( const FileSystemType type, std::string dir_path, std::string* temp_dir); /// Delete a path. /// \param path The path to the directory or file. /// \return Error status Status DeletePath(const std::string& path); /// Infer the filesystem type from the given path. /// \param path The path to infer the filesystem type from. /// \param type Returns the filesystem type of the path. /// \return Error status Status GetFileSystemType(const std::string& path, FileSystemType* type); /// Return the string representation of the filesystem type. /// \param type The filesystem type. /// \return The string representation of the type. const std::string& FileSystemTypeString(const FileSystemType type); }} // namespace triton::core kytensor-2.49.0.7/core/src/filesystem/implementations/000077500000000000000000000000001527052371200227505ustar00rootroot00000000000000kytensor-2.49.0.7/core/src/filesystem/implementations/as.h000066400000000000000000000416321527052371200235320ustar00rootroot00000000000000// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "common.h" // [WIP] below needed? #undef LOG_INFO #undef LOG_WARNING namespace triton { namespace core { namespace as = Azure::Storage; namespace asb = Azure::Storage::Blobs; const std::string AS_URL_PATTERN = "as://([^/]+)/([^/?]+)(?:/([^?]*))?(\\?.*)?"; struct ASCredential { std::string account_str_; std::string account_key_; ASCredential(); // from env var ASCredential(triton::common::TritonJson::Value& cred_json); }; ASCredential::ASCredential() { const auto to_str = [](const char* s) -> std::string { return (s != nullptr ? std::string(s) : ""); }; const char* account_str = std::getenv("AZURE_STORAGE_ACCOUNT"); const char* account_key = std::getenv("AZURE_STORAGE_KEY"); account_str_ = to_str(account_str); account_key_ = to_str(account_key); } ASCredential::ASCredential(triton::common::TritonJson::Value& cred_json) { triton::common::TritonJson::Value account_str_json, account_key_json; if (cred_json.Find("account_str", &account_str_json)) account_str_json.AsString(&account_str_); if (cred_json.Find("account_key", &account_key_json)) account_key_json.AsString(&account_key_); } class ASFileSystem : public FileSystem { public: ASFileSystem(const std::string& path, const ASCredential& as_cred); Status CheckClient(); // unify with S3 interface Status CheckClient(const std::string& path) { return CheckClient(); } Status FileExists(const std::string& path, bool* exists) override; Status IsDirectory(const std::string& path, bool* is_dir) override; Status FileModificationTime( const std::string& path, int64_t* mtime_ns) override; Status GetDirectoryContents( const std::string& path, std::set* contents) override; Status GetDirectorySubdirs( const std::string& path, std::set* subdirs) override; Status GetDirectoryFiles( const std::string& path, std::set* files) override; Status ReadTextFile(const std::string& path, std::string* contents) override; Status LocalizePath( const std::string& path, std::shared_ptr* localized) override; Status WriteTextFile( const std::string& path, const std::string& contents) override; Status WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; Status MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: Status ParsePath( const std::string& path, std::string* container, std::string* blob); // 'callback' will be invoked when directory content is received, it may // be invoked multiple times within the same ListDirectory() call if the // result is paged. Status ListDirectory( const std::string& path, const std::string& dir_path, std::function& blobs, const std::vector& blob_prefixes)> callback); Status DownloadFolder( const std::string& container, const std::string& path, const std::string& dest); std::shared_ptr client_; re2::RE2 as_regex_; }; Status ASFileSystem::ParsePath( const std::string& path, std::string* container, std::string* blob) { std::string host_name, query; if (!RE2::FullMatch(path, as_regex_, &host_name, container, blob, &query)) { return Status( Status::Code::INTERNAL, "Invalid azure storage path: " + path); } return Status::Success; } ASFileSystem::ASFileSystem(const std::string& path, const ASCredential& as_cred) : as_regex_(AS_URL_PATTERN) { std::string host_name, container, blob_path, query; if (RE2::FullMatch( path, as_regex_, &host_name, &container, &blob_path, &query)) { size_t pos = host_name.rfind(".blob.core.windows.net"); std::string account_name; if (as_cred.account_str_.empty()) { if (pos != std::string::npos) { account_name = host_name.substr(0, pos); } else { account_name = host_name; } } else { account_name = as_cred.account_str_; } std::string service_url( "https://" + account_name + ".blob.core.windows.net"); if (!as_cred.account_key_.empty()) { // Shared Key auto cred = std::make_shared( account_name, as_cred.account_key_); client_ = std::make_shared(service_url, cred); } else { client_ = std::make_shared(service_url); } } } Status ASFileSystem::CheckClient() { if (client_ == nullptr) { return Status( Status::Code::INTERNAL, "Unable to create Azure filesystem client. Check account credentials."); } return Status::Success; } Status ASFileSystem::FileModificationTime(const std::string& path, int64_t* mtime_ns) { std::string container, blob; RETURN_IF_ERROR(ParsePath(path, &container, &blob)); auto bc = client_->GetBlobContainerClient(container).GetBlobClient(blob); try { auto blobProperty = bc.GetProperties().Value; *mtime_ns = std::chrono::time_point_cast( blobProperty.LastModified) .time_since_epoch() .count(); } catch (as::StorageException& ex) { return Status( Status::Code::INTERNAL, "Unable to get blob property for file at " + path + ":" + ex.what()); } return Status::Success; }; Status ASFileSystem::ListDirectory( const std::string& container, const std::string& dir_path, std::function& blobs, const std::vector& blob_prefixes)> callback) { auto container_client = client_->GetBlobContainerClient(container); auto options = asb::ListBlobsOptions(); // Append a slash to make it easier to list contents std::string full_dir = AppendSlash(dir_path); options.Prefix = full_dir; try { for (auto blobPage = container_client.ListBlobsByHierarchy("/", options); blobPage.HasPage(); blobPage.MoveToNextPage()) { // per-page per-blob RETURN_IF_ERROR(callback(blobPage.Blobs, blobPage.BlobPrefixes)); } } catch (as::StorageException& ex) { return Status( Status::Code::INTERNAL, "Failed to get contents of directory " + dir_path + ":" + ex.what()); } return Status::Success; } Status ASFileSystem::GetDirectoryContents( const std::string& path, std::set* contents) { auto func = [&](const std::vector& blobs, const std::vector& blob_prefixes) { for (const auto& blob_item : blobs) { // Fail-safe check to ensure the item name is not empty if (blob_item.Name.empty()) { return Status( Status::Code::INTERNAL, "Cannot handle item with empty name at " + path); } contents->insert(BaseName(blob_item.Name)); } for (const auto& directory_item : blob_prefixes) { // Fail-safe check to ensure the item name is not empty if (directory_item.empty()) { return Status( Status::Code::INTERNAL, "Cannot handle item with empty name at " + path); } contents->insert(BaseName(directory_item)); } return Status::Success; }; std::string container, dir_path; RETURN_IF_ERROR(ParsePath(path, &container, &dir_path)); return ListDirectory(container, dir_path, func); } Status ASFileSystem::GetDirectorySubdirs( const std::string& path, std::set* subdirs) { auto func = [&](const std::vector& blobs, const std::vector& blob_prefixes) { for (const auto& directory_item : blob_prefixes) { // Fail-safe check to ensure the item name is not empty if (directory_item.empty()) { return Status( Status::Code::INTERNAL, "Cannot handle item with empty name at " + path); } subdirs->insert(BaseName(directory_item)); } return Status::Success; }; std::string container, dir_path; RETURN_IF_ERROR(ParsePath(path, &container, &dir_path)); return ListDirectory(container, dir_path, func); } Status ASFileSystem::GetDirectoryFiles( const std::string& path, std::set* files) { auto func = [&](const std::vector& blobs, const std::vector& blob_prefixes) { for (const auto& blob_item : blobs) { // Fail-safe check to ensure the item name is not empty if (blob_item.Name.empty()) { return Status( Status::Code::INTERNAL, "Cannot handle item with empty name at " + path); } files->insert(BaseName(blob_item.Name)); } return Status::Success; }; std::string container, dir_path; RETURN_IF_ERROR(ParsePath(path, &container, &dir_path)); return ListDirectory(container, dir_path, func); } Status ASFileSystem::IsDirectory(const std::string& path, bool* is_dir) { *is_dir = false; std::string container, blob_path; RETURN_IF_ERROR(ParsePath(path, &container, &blob_path)); auto container_client = client_->GetBlobContainerClient(container); auto options = asb::ListBlobsOptions(); // Append a slash to make it easier to list contents std::string full_dir = AppendSlash(blob_path); options.Prefix = full_dir; try { for (auto blobPage = container_client.ListBlobsByHierarchy("/", options); blobPage.HasPage(); blobPage.MoveToNextPage()) { if ((blobPage.Blobs.size() == 1) && (blobPage.Blobs[0].Name == blob_path)) { // It's a file return Status::Success; } *is_dir = ((blobPage.Blobs.size() > 0) || (blobPage.BlobPrefixes.size() > 0)); break; } } catch (as::StorageException& ex) { return Status( Status::Code::INTERNAL, "Failed to check if directory at " + path + ":" + ex.what()); } return Status::Success; }; Status ASFileSystem::ReadTextFile(const std::string& path, std::string* contents) { std::string container, blob_path; RETURN_IF_ERROR(ParsePath(path, &container, &blob_path)); try { auto res = client_->GetBlobContainerClient(container) .GetBlobClient(blob_path) .Download(); *contents = std::string( (const char*)res.Value.BodyStream->ReadToEnd().data(), res.Value.BlobSize); } catch (as::StorageException& ex) { return Status( Status::Code::INTERNAL, "Failed to read text file at " + path + ":" + ex.what()); } return Status::Success; } Status ASFileSystem::FileExists(const std::string& path, bool* exists) { *exists = false; std::string container, blob; RETURN_IF_ERROR(ParsePath(path, &container, &blob)); auto container_client = client_->GetBlobContainerClient(container); auto options = asb::ListBlobsOptions(); options.Prefix = blob; try { for (auto blobPage = container_client.ListBlobsByHierarchy("/", options); blobPage.HasPage(); blobPage.MoveToNextPage()) { // If any entries are returned from ListBlobs, the file / directory exists *exists = ((blobPage.Blobs.size() > 0) || (blobPage.BlobPrefixes.size() > 0)); break; } } catch (as::StorageException& ex) { return Status( Status::Code::INTERNAL, "Failed to check if file exists at " + path + ":" + ex.what()); } return Status::Success; } Status ASFileSystem::DownloadFolder( const std::string& container, const std::string& path, const std::string& dest) { auto container_client = client_->GetBlobContainerClient(container); auto func = [&](const std::vector& blobs, const std::vector& blob_prefixes) { for (const auto& blob_item : blobs) { const auto& local_path = JoinPath({dest, BaseName(blob_item.Name)}); try { container_client.GetBlobClient(blob_item.Name).DownloadTo(local_path); } catch (as::StorageException& ex) { return Status( Status::Code::INTERNAL, "Failed to download file at " + blob_item.Name + ":" + ex.what()); } } for (const auto& directory_item : blob_prefixes) { const auto& local_path = JoinPath({dest, BaseName(directory_item)}); int status = mkdir( const_cast(local_path.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); if (status == -1) { return Status( Status::Code::INTERNAL, "Failed to create local folder: " + local_path + ", errno:" + strerror(errno)); } RETURN_IF_ERROR(DownloadFolder(container, directory_item, local_path)); } return Status::Success; }; return ListDirectory(container, path, func); } Status ASFileSystem::LocalizePath( const std::string& path, std::shared_ptr* localized) { bool exists; RETURN_IF_ERROR(FileExists(path, &exists)); if (!exists) { return Status( Status::Code::INTERNAL, "directory or file does not exist at " + path); } bool is_dir; RETURN_IF_ERROR(IsDirectory(path, &is_dir)); if (!is_dir) { return Status( Status::Code::UNSUPPORTED, "AS file localization not yet implemented " + path); } // Create a local directory for azure model store. // If ENV variable are not set, creates a temporary directory // under `/tmp` with the format: "folderXXXXXX". // Otherwise, will create a folder under specified directory with the same // format. std::string env_mount_dir = GetEnvironmentVariableOrDefault( "TRITON_AZURE_MOUNT_DIRECTORY", kDefaultMountDirectory); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); localized->reset(new LocalizedPath(path, tmp_folder)); std::string dest(tmp_folder); std::string container, blob; RETURN_IF_ERROR(ParsePath(path, &container, &blob)); return DownloadFolder(container, blob, dest); } Status ASFileSystem::WriteTextFile( const std::string& path, const std::string& contents) { std::string container, blob; RETURN_IF_ERROR(ParsePath(path, &container, &blob)); try { client_->GetBlobContainerClient(container) .GetBlockBlobClient(blob) .UploadFrom((const uint8_t*)contents.data(), contents.size()); } catch (as::StorageException& ex) { return Status( Status::Code::INTERNAL, "Failed to write file to " + path + ":" + ex.what()); } return Status::Success; } Status ASFileSystem::WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) { return Status( Status::Code::UNSUPPORTED, "Write text file operation not yet implemented " + path); } Status ASFileSystem::MakeDirectory(const std::string& dir, const bool recursive) { return Status( Status::Code::UNSUPPORTED, "Make directory operation not yet implemented"); } Status ASFileSystem::MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, "Make temporary directory operation not yet implemented"); } Status ASFileSystem::DeletePath(const std::string& path) { return Status( Status::Code::UNSUPPORTED, "Delete path operation not yet implemented"); } }} // namespace triton::core kytensor-2.49.0.7/core/src/filesystem/implementations/common.h000066400000000000000000000114321527052371200244120ustar00rootroot00000000000000// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "../api.h" #include "triton/common/logging.h" #ifdef _WIN32 // suppress the min and max definitions in Windef.h. #define NOMINMAX #include // _CRT_INTERNAL_NONSTDC_NAMES 1 before including Microsoft provided C Runtime // library to expose declarations without "_" prefix to match POSIX style. #define _CRT_INTERNAL_NONSTDC_NAMES 1 #include #include #else #include #include #endif #ifdef _WIN32 // in Windows doesn't define S_ISDIR macro #if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) #define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) #endif #define F_OK 0 #endif #define TRITONJSON_STATUSTYPE triton::core::Status #define TRITONJSON_STATUSRETURN(M) \ return triton::core::Status(triton::core::Status::Code::INTERNAL, (M)) #define TRITONJSON_STATUSSUCCESS triton::core::Status::Success #include "triton/common/triton_json.h" namespace triton { namespace core { // Default folder for temporary local cache constexpr char kDefaultMountDirectory[] = "/tmp"; // FileSystem interface that all file system implementation should inherit from. // To add new file system support, the implementation should be added and made // visible to FileSystemManager in api.cc class FileSystem { public: virtual Status FileExists(const std::string& path, bool* exists) = 0; virtual Status IsDirectory(const std::string& path, bool* is_dir) = 0; virtual Status FileModificationTime( const std::string& path, int64_t* mtime_ns) = 0; virtual Status GetDirectoryContents( const std::string& path, std::set* contents) = 0; virtual Status GetDirectorySubdirs( const std::string& path, std::set* subdirs) = 0; virtual Status GetDirectoryFiles( const std::string& path, std::set* files) = 0; virtual Status ReadTextFile( const std::string& path, std::string* contents) = 0; virtual Status LocalizePath( const std::string& path, std::shared_ptr* localized) = 0; virtual Status WriteTextFile( const std::string& path, const std::string& contents) = 0; virtual Status WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) = 0; virtual Status MakeDirectory( const std::string& dir, const bool recursive) = 0; virtual Status MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) = 0; virtual Status DeletePath(const std::string& path) = 0; }; // Helper function to take care of lack of trailing slashes std::string AppendSlash(const std::string& name) { if (name.empty() || (name.back() == '/')) { return name; } return (name + "/"); } /// Helper function to get the value of the environment variable, /// or default value if not set. /// /// \param variable_name The name of the environment variable. /// \param default_value The default value. /// \return The environment variable or the default value if not set. std::string GetEnvironmentVariableOrDefault( const std::string& variable_name, const std::string& default_value) { const char* value = getenv(variable_name.c_str()); return value ? value : default_value; } }} // namespace triton::core kytensor-2.49.0.7/core/src/filesystem/implementations/gcs.h000066400000000000000000000373711527052371200237100ustar00rootroot00000000000000// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "common.h" namespace triton { namespace core { namespace gcs = google::cloud::storage; struct GCSCredential { std::string path_; GCSCredential(); // from env var GCSCredential(triton::common::TritonJson::Value& cred_json); }; GCSCredential::GCSCredential() { const char* path = std::getenv("GOOGLE_APPLICATION_CREDENTIALS"); path_ = (path != nullptr ? std::string(path) : ""); } GCSCredential::GCSCredential(triton::common::TritonJson::Value& cred_json) { cred_json.AsString(&path_); } class GCSFileSystem : public FileSystem { public: GCSFileSystem(const GCSCredential& gs_cred); // unify with S3/azure interface GCSFileSystem(const std::string& path, const GCSCredential& gs_cred) : GCSFileSystem(gs_cred) { } Status CheckClient(); // unify with S3 interface Status CheckClient(const std::string& path) { return CheckClient(); } Status FileExists(const std::string& path, bool* exists) override; Status IsDirectory(const std::string& path, bool* is_dir) override; Status FileModificationTime( const std::string& path, int64_t* mtime_ns) override; Status GetDirectoryContents( const std::string& path, std::set* contents) override; Status GetDirectorySubdirs( const std::string& path, std::set* subdirs) override; Status GetDirectoryFiles( const std::string& path, std::set* files) override; Status ReadTextFile(const std::string& path, std::string* contents) override; Status LocalizePath( const std::string& path, std::shared_ptr* localized) override; Status WriteTextFile( const std::string& path, const std::string& contents) override; Status WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; Status MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: Status ParsePath( const std::string& path, std::string* bucket, std::string* object); Status MetaDataExists( const std::string path, bool* exists, google::cloud::StatusOr* metadata); std::unique_ptr client_; }; GCSFileSystem::GCSFileSystem(const GCSCredential& gs_cred) { google::cloud::Options options; auto creds = gcs::oauth2::CreateAuthorizedUserCredentialsFromJsonFilePath( gs_cred.path_); if (creds) { options.set(*creds); // json user credential } else { auto creds = gcs::oauth2::CreateServiceAccountCredentialsFromJsonFilePath( gs_cred.path_); if (creds) { options.set(*creds); // json sa credential } else { auto creds = gcs::oauth2::CreateComputeEngineCredentials(); if (creds->AuthorizationHeader()) { options.set(creds); // metadata service } else { options.set( gcs::oauth2::CreateAnonymousCredentials()); // no credential } } } client_ = std::make_unique(options); } Status GCSFileSystem::CheckClient() { if (!client_) { return Status( Status::Code::INTERNAL, "Unable to create GCS client. Check account credentials."); } return Status::Success; } Status GCSFileSystem::ParsePath( const std::string& path, std::string* bucket, std::string* object) { // Get the bucket name and the object path. Return error if input is malformed int bucket_start = path.find("gs://") + strlen("gs://"); int bucket_end = path.find("/", bucket_start); // If there isn't a second slash, the address has only the bucket if (bucket_end > bucket_start) { *bucket = path.substr(bucket_start, bucket_end - bucket_start); *object = path.substr(bucket_end + 1); } else { *bucket = path.substr(bucket_start); *object = ""; } if (bucket->empty()) { return Status( Status::Code::INTERNAL, "No bucket name found in path: " + path); } return Status::Success; } Status GCSFileSystem::FileExists(const std::string& path, bool* exists) { *exists = false; std::string bucket, object; RETURN_IF_ERROR(ParsePath(path, &bucket, &object)); // Make a request for metadata and check the response google::cloud::StatusOr object_metadata = client_->GetObjectMetadata(bucket, object); if (object_metadata) { *exists = true; return Status::Success; } // GCS doesn't make objects for directories, so it could still be a directory bool is_dir; RETURN_IF_ERROR(IsDirectory(path, &is_dir)); *exists = is_dir; return Status::Success; } Status GCSFileSystem::IsDirectory(const std::string& path, bool* is_dir) { *is_dir = false; std::string bucket, object_path; RETURN_IF_ERROR(ParsePath(path, &bucket, &object_path)); // Check if the bucket exists google::cloud::StatusOr bucket_metadata = client_->GetBucketMetadata(bucket); if (!bucket_metadata) { return Status( Status::Code::INTERNAL, "Could not get MetaData for bucket with name " + bucket + " : " + bucket_metadata.status().message()); } // Root case - bucket exists and object path is empty if (object_path.empty()) { *is_dir = true; return Status::Success; } // Check whether it has children. If at least one child, it is a directory for (auto&& object_metadata : client_->ListObjects(bucket, gcs::Prefix(AppendSlash(object_path)))) { if (object_metadata) { *is_dir = true; break; } } return Status::Success; } Status GCSFileSystem::FileModificationTime(const std::string& path, int64_t* mtime_ns) { // We don't need to worry about the case when this is a directory bool is_dir; RETURN_IF_ERROR(IsDirectory(path, &is_dir)); if (is_dir) { *mtime_ns = 0; return Status::Success; } std::string bucket, object; RETURN_IF_ERROR(ParsePath(path, &bucket, &object)); // Otherwise check the object metadata for update time google::cloud::StatusOr object_metadata = client_->GetObjectMetadata(bucket, object); if (!object_metadata) { return Status( Status::Code::INTERNAL, "Failed to get metadata for " + object + " : " + object_metadata.status().message()); } // Get duration from time point with respect to object clock auto update_time = std::chrono::time_point_cast( object_metadata->updated()) .time_since_epoch() .count(); *mtime_ns = update_time; return Status::Success; } Status GCSFileSystem::GetDirectoryContents( const std::string& path, std::set* contents) { std::string bucket, dir_path; RETURN_IF_ERROR(ParsePath(path, &bucket, &dir_path)); // Append a slash to make it easier to list contents std::string full_dir = AppendSlash(dir_path); // Get objects with prefix equal to full directory path for (auto&& object_metadata : client_->ListObjects(bucket, gcs::Prefix(full_dir))) { if (!object_metadata) { return Status( Status::Code::INTERNAL, "Could not list contents of directory at " + path + " : " + object_metadata.status().message()); } // In the case of empty directories, the directory itself will appear here if (object_metadata->name() == full_dir) { continue; } // We have to make sure that subdirectory contents do not appear here std::string name = object_metadata->name(); int item_start = name.find(full_dir) + full_dir.size(); // GCS response prepends parent directory name int item_end = name.find("/", item_start); // Let set take care of subdirectory contents std::string item = name.substr(item_start, item_end - item_start); contents->insert(item); // Fail-safe check to ensure the item name is not empty if (item.empty()) { return Status( Status::Code::INTERNAL, "Cannot handle item with empty name at " + path); } } return Status::Success; } Status GCSFileSystem::GetDirectorySubdirs( const std::string& path, std::set* subdirs) { RETURN_IF_ERROR(GetDirectoryContents(path, subdirs)); // Erase non-directory entries... for (auto iter = subdirs->begin(); iter != subdirs->end();) { bool is_dir; RETURN_IF_ERROR(IsDirectory(JoinPath({path, *iter}), &is_dir)); if (!is_dir) { iter = subdirs->erase(iter); } else { ++iter; } } return Status::Success; } Status GCSFileSystem::GetDirectoryFiles( const std::string& path, std::set* files) { RETURN_IF_ERROR(GetDirectoryContents(path, files)); // Erase directory entries... for (auto iter = files->begin(); iter != files->end();) { bool is_dir; RETURN_IF_ERROR(IsDirectory(JoinPath({path, *iter}), &is_dir)); if (is_dir) { iter = files->erase(iter); } else { ++iter; } } return Status::Success; } Status GCSFileSystem::ReadTextFile(const std::string& path, std::string* contents) { bool exists; RETURN_IF_ERROR(FileExists(path, &exists)); if (!exists) { return Status(Status::Code::INTERNAL, "File does not exist at " + path); } std::string bucket, object; ParsePath(path, &bucket, &object); gcs::ObjectReadStream stream = client_->ReadObject(bucket, object); if (!stream) { return Status( Status::Code::INTERNAL, "Failed to open object read stream for " + path + " : " + stream.status().message()); } std::string data = ""; char c; while (stream.get(c)) { data += c; } *contents = data; return Status::Success; } Status GCSFileSystem::LocalizePath( const std::string& path, std::shared_ptr* localized) { bool exists; RETURN_IF_ERROR(FileExists(path, &exists)); if (!exists) { return Status( Status::Code::INTERNAL, "directory or file does not exist at " + path); } bool is_dir; RETURN_IF_ERROR(IsDirectory(path, &is_dir)); if (!is_dir) { return Status( Status::Code::UNSUPPORTED, "GCS file localization not yet implemented " + path); } // Create a local directory for GCS model store. std::string env_mount_dir = GetEnvironmentVariableOrDefault( "TRITON_GCS_MOUNT_DIRECTORY", kDefaultMountDirectory); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); localized->reset(new LocalizedPath(path, tmp_folder)); std::set contents, filenames; RETURN_IF_ERROR(GetDirectoryContents(path, &filenames)); for (auto itr = filenames.begin(); itr != filenames.end(); ++itr) { contents.insert(JoinPath({path, *itr})); } while (contents.size() != 0) { std::set tmp_contents = contents; contents.clear(); for (auto iter = tmp_contents.begin(); iter != tmp_contents.end(); ++iter) { bool is_subdir; std::string gcs_fpath = *iter; std::string gcs_removed_path = gcs_fpath.substr(path.size()); std::string local_fpath = JoinPath({(*localized)->Path(), gcs_removed_path}); RETURN_IF_ERROR(IsDirectory(gcs_fpath, &is_subdir)); if (is_subdir) { // Create local mirror of sub-directories #ifdef _WIN32 int status = mkdir(const_cast(local_fpath.c_str())); #else int status = mkdir( const_cast(local_fpath.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); #endif if (status == -1) { return Status( Status::Code::INTERNAL, "Failed to create local folder: " + local_fpath + ", errno:" + strerror(errno)); } // Add sub-directories and deeper files to contents std::set subdir_contents; RETURN_IF_ERROR(GetDirectoryContents(gcs_fpath, &subdir_contents)); for (auto itr = subdir_contents.begin(); itr != subdir_contents.end(); ++itr) { contents.insert(JoinPath({gcs_fpath, *itr})); } } else { // Create local copy of file std::string file_bucket, file_object; RETURN_IF_ERROR(ParsePath(gcs_fpath, &file_bucket, &file_object)); // Send a request to read the object gcs::ObjectReadStream filestream = client_->ReadObject(file_bucket, file_object); if (!filestream) { return Status( Status::Code::INTERNAL, "Failed to get object at " + *iter + " : " + filestream.status().message()); } std::string gcs_removed_path = (*iter).substr(path.size()); std::string local_file_path = JoinPath({(*localized)->Path(), gcs_removed_path}); std::ofstream output_file(local_file_path.c_str(), std::ios::binary); output_file << filestream.rdbuf(); output_file.close(); } } } return Status::Success; } Status GCSFileSystem::WriteTextFile( const std::string& path, const std::string& contents) { return Status( Status::Code::UNSUPPORTED, "Write text file operation not yet implemented " + path); } Status GCSFileSystem::WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) { return Status( Status::Code::UNSUPPORTED, "Write text file operation not yet implemented " + path); } Status GCSFileSystem::MakeDirectory(const std::string& dir, const bool recursive) { return Status( Status::Code::UNSUPPORTED, "Make temporary directory operation not yet implemented"); } Status GCSFileSystem::MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, "Make temporary directory operation not yet implemented"); } Status GCSFileSystem::DeletePath(const std::string& path) { return Status( Status::Code::UNSUPPORTED, "Delete path operation not yet implemented"); } }} // namespace triton::core kytensor-2.49.0.7/core/src/filesystem/implementations/local.h000066400000000000000000000253611527052371200242220ustar00rootroot00000000000000// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include "../../constants.h" #include "common.h" namespace triton { namespace core { class LocalFileSystem : public FileSystem { public: Status FileExists(const std::string& path, bool* exists) override; Status IsDirectory(const std::string& path, bool* is_dir) override; Status FileModificationTime( const std::string& path, int64_t* mtime_ns) override; Status GetDirectoryContents( const std::string& path, std::set* contents) override; Status GetDirectorySubdirs( const std::string& path, std::set* subdirs) override; Status GetDirectoryFiles( const std::string& path, std::set* files) override; Status ReadTextFile(const std::string& path, std::string* contents) override; Status LocalizePath( const std::string& path, std::shared_ptr* localized) override; Status WriteTextFile( const std::string& path, const std::string& contents) override; Status WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; Status MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; }; Status LocalFileSystem::FileExists(const std::string& path, bool* exists) { *exists = (access(path.c_str(), F_OK) == 0); return Status::Success; } Status LocalFileSystem::IsDirectory(const std::string& path, bool* is_dir) { *is_dir = false; struct stat st; if (stat(path.c_str(), &st) != 0) { return Status(Status::Code::INTERNAL, "failed to stat file " + path); } *is_dir = S_ISDIR(st.st_mode); return Status::Success; } Status LocalFileSystem::FileModificationTime( const std::string& path, int64_t* mtime_ns) { struct stat st; if (stat(path.c_str(), &st) != 0) { return Status(Status::Code::INTERNAL, "failed to stat file " + path); } #ifdef _WIN32 // In Windows, st_mtime is in time_t *mtime_ns = std::max(st.st_mtime, st.st_ctime); #else *mtime_ns = std::max(TIMESPEC_TO_NANOS(st.st_mtim), TIMESPEC_TO_NANOS(st.st_ctim)); #endif return Status::Success; } Status LocalFileSystem::GetDirectoryContents( const std::string& path, std::set* contents) { #ifdef _WIN32 WIN32_FIND_DATA entry; // Append "*" to obtain all files under 'path' HANDLE dir = FindFirstFile(JoinPath({path, "*"}).c_str(), &entry); if (dir == INVALID_HANDLE_VALUE) { return Status(Status::Code::INTERNAL, "failed to open directory " + path); } if ((strcmp(entry.cFileName, ".") != 0) && (strcmp(entry.cFileName, "..") != 0)) { contents->insert(entry.cFileName); } while (FindNextFile(dir, &entry)) { if ((strcmp(entry.cFileName, ".") != 0) && (strcmp(entry.cFileName, "..") != 0)) { contents->insert(entry.cFileName); } } FindClose(dir); #else DIR* dir = opendir(path.c_str()); if (dir == nullptr) { return Status(Status::Code::INTERNAL, "failed to open directory " + path); } struct dirent* entry; while ((entry = readdir(dir)) != nullptr) { std::string entryname = entry->d_name; if ((entryname != ".") && (entryname != "..")) { contents->insert(entryname); } } closedir(dir); #endif return Status::Success; } Status LocalFileSystem::GetDirectorySubdirs( const std::string& path, std::set* subdirs) { RETURN_IF_ERROR(GetDirectoryContents(path, subdirs)); // Erase non-directory entries... for (auto iter = subdirs->begin(); iter != subdirs->end();) { bool is_dir; RETURN_IF_ERROR(IsDirectory(JoinPath({path, *iter}), &is_dir)); if (!is_dir) { iter = subdirs->erase(iter); } else { ++iter; } } return Status::Success; } Status LocalFileSystem::GetDirectoryFiles( const std::string& path, std::set* files) { RETURN_IF_ERROR(GetDirectoryContents(path, files)); // Erase directory entries... for (auto iter = files->begin(); iter != files->end();) { bool is_dir; RETURN_IF_ERROR(IsDirectory(JoinPath({path, *iter}), &is_dir)); if (is_dir) { iter = files->erase(iter); } else { ++iter; } } return Status::Success; } Status LocalFileSystem::ReadTextFile(const std::string& path, std::string* contents) { std::ifstream in(path, std::ios::in | std::ios::binary); if (!in) { return Status( Status::Code::INTERNAL, "failed to open text file for read " + path + ": " + strerror(errno)); } in.seekg(0, std::ios::end); contents->resize(in.tellg()); in.seekg(0, std::ios::beg); in.read(&(*contents)[0], contents->size()); in.close(); return Status::Success; } Status LocalFileSystem::LocalizePath( const std::string& path, std::shared_ptr* localized) { // For local file system we don't actually need to download the // directory or file. We use it in place. localized->reset(new LocalizedPath(path)); return Status::Success; } Status LocalFileSystem::WriteTextFile( const std::string& path, const std::string& contents) { std::ofstream out(path, std::ios::out | std::ios::binary); if (!out) { return Status( Status::Code::INTERNAL, "failed to open text file for write " + path + ": " + strerror(errno)); } out.write(&contents[0], contents.size()); out.close(); return Status::Success; } Status LocalFileSystem::WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) { std::ofstream out(path, std::ios::out | std::ios::binary); if (!out) { return Status( Status::Code::INTERNAL, "failed to open binary file for write " + path + ": " + strerror(errno)); } out.write(contents, content_len); return Status::Success; } Status LocalFileSystem::MakeDirectory(const std::string& dir, const bool recursive) { #ifdef _WIN32 if (mkdir(dir.c_str()) == -1) #else if (mkdir(dir.c_str(), S_IRWXU) == -1) #endif { // Only allow the error due to parent directory does not exist // if 'recursive' is requested if ((errno == ENOENT) && (!dir.empty()) && recursive) { RETURN_IF_ERROR(MakeDirectory(DirName(dir), recursive)); // Retry the creation #ifdef _WIN32 if (mkdir(dir.c_str()) == -1) #else if (mkdir(dir.c_str(), S_IRWXU) == -1) #endif { return Status( Status::Code::INTERNAL, "Failed to create directory '" + dir + "', errno:" + strerror(errno)); } } else { return Status( Status::Code::INTERNAL, "Failed to create directory '" + dir + "', errno:" + strerror(errno)); } } return Status::Success; } Status LocalFileSystem::MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) { #ifdef _WIN32 char temp_path[MAX_PATH + 1]; size_t temp_path_length = GetTempPath(MAX_PATH + 1, temp_path); if (temp_path_length == 0) { return Status( Status::Code::INTERNAL, "Failed to get local directory for temporary files"); } // There is no single operation like 'mkdtemp' in Windows, thus generating // unique temporary directory is a process of getting temporary file name, // deleting the file (file creation is side effect fo getting name), creating // corresponding directory, so mutex is used to avoid possible race condition. // However, it doesn't prevent other process on creating temporary file and // thus the race condition may still happen. One possible solution is // to reserve a temporary directory for the process and generate temporary // model directories inside it. static std::mutex mtx; std::lock_guard lk(mtx); // Construct a std::string as filled 'temp_path' is not C string, // and so that we can reuse 'temp_path' to hold the temp file name. std::string temp_path_str(temp_path, temp_path_length); if (GetTempFileName(temp_path_str.c_str(), "folder", 0, temp_path) == 0) { return Status(Status::Code::INTERNAL, "Failed to create local temp folder"); } *temp_dir = temp_path; DeleteFile(temp_dir->c_str()); if (CreateDirectory(temp_dir->c_str(), NULL) == 0) { return Status( Status::Code::INTERNAL, "Failed to create local temp folder: " + *temp_dir); } #else if (dir_path.empty()) { dir_path = kDefaultMountDirectory; } std::string folder_template = JoinPath({dir_path, "folderXXXXXX"}); char* res = mkdtemp(const_cast(folder_template.c_str())); if (res == nullptr) { return Status( Status::Code::INTERNAL, "Failed to create local temp folder: " + folder_template + ", errno:" + strerror(errno)); } *temp_dir = res; #endif return Status::Success; } Status LocalFileSystem::DeletePath(const std::string& path) { bool is_dir = false; RETURN_IF_ERROR(IsDirectory(path, &is_dir)); if (is_dir) { std::set contents; RETURN_IF_ERROR(GetDirectoryContents(path, &contents)); for (const auto& content : contents) { RETURN_IF_ERROR(DeletePath(JoinPath({path, content}))); } rmdir(path.c_str()); } else { remove(path.c_str()); } return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/filesystem/implementations/s3.h000066400000000000000000000655351527052371200234640ustar00rootroot00000000000000// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "common.h" // [FIXME: DLIS-4973] #include #include #include #include #include #include #include #include #include namespace triton { namespace core { namespace s3 = Aws::S3; // Override the default S3 Curl initialization for disabling HTTP/2 on s3. // Remove once s3 fully supports HTTP/2 [FIXME: DLIS-4973]. // Reference: // https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/cpp/example_code/s3/list_buckets_disabling_dns_cache.cpp static const char S3_ALLOCATION_TAG[] = "OverrideDefaultHttpClient"; class S3CurlHttpClient : public Aws::Http::CurlHttpClient { public: explicit S3CurlHttpClient( const Aws::Client::ClientConfiguration& client_config) : Aws::Http::CurlHttpClient(client_config) { } protected: void OverrideOptionsOnConnectionHandle(CURL* connectionHandle) const override { curl_easy_setopt( connectionHandle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); } }; class S3HttpClientFactory : public Aws::Http::HttpClientFactory { std::shared_ptr CreateHttpClient( const Aws::Client::ClientConfiguration& client_config) const override { return Aws::MakeShared(S3_ALLOCATION_TAG, client_config); } std::shared_ptr CreateHttpRequest( const Aws::String& uri, Aws::Http::HttpMethod method, const Aws::IOStreamFactory& stream_factory) const override { return CreateHttpRequest(Aws::Http::URI(uri), method, stream_factory); } std::shared_ptr CreateHttpRequest( const Aws::Http::URI& uri, Aws::Http::HttpMethod method, const Aws::IOStreamFactory& stream_factory) const override { auto req = Aws::MakeShared( S3_ALLOCATION_TAG, uri, method); req->SetResponseStreamFactory(stream_factory); return req; } void InitStaticState() override { S3CurlHttpClient::InitGlobalState(); } void CleanupStaticState() override { S3CurlHttpClient::CleanupGlobalState(); } }; struct S3Credential { std::string secret_key_; std::string key_id_; std::string region_; std::string session_token_; std::string profile_name_; S3Credential(); // from env var S3Credential(triton::common::TritonJson::Value& cred_json); }; S3Credential::S3Credential() { const auto to_str = [](const char* s) -> std::string { return (s != nullptr ? std::string(s) : ""); }; const char* secret_key = std::getenv("AWS_SECRET_ACCESS_KEY"); const char* key_id = std::getenv("AWS_ACCESS_KEY_ID"); const char* region = std::getenv("AWS_DEFAULT_REGION"); const char* session_token = std::getenv("AWS_SESSION_TOKEN"); const char* profile = std::getenv("AWS_PROFILE"); secret_key_ = to_str(secret_key); key_id_ = to_str(key_id); region_ = to_str(region); session_token_ = to_str(session_token); profile_name_ = to_str(profile); } S3Credential::S3Credential(triton::common::TritonJson::Value& cred_json) { triton::common::TritonJson::Value secret_key_json, key_id_json, region_json, session_token_json, profile_json; if (cred_json.Find("secret_key", &secret_key_json)) secret_key_json.AsString(&secret_key_); if (cred_json.Find("key_id", &key_id_json)) key_id_json.AsString(&key_id_); if (cred_json.Find("region", ®ion_json)) region_json.AsString(®ion_); if (cred_json.Find("session_token", &session_token_json)) session_token_json.AsString(&session_token_); if (cred_json.Find("profile", &profile_json)) profile_json.AsString(&profile_name_); } class S3FileSystem : public FileSystem { public: S3FileSystem(const std::string& s3_path, const S3Credential& s3_cred); Status CheckClient(const std::string& s3_path); Status FileExists(const std::string& path, bool* exists) override; Status IsDirectory(const std::string& path, bool* is_dir) override; Status FileModificationTime( const std::string& path, int64_t* mtime_ns) override; Status GetDirectoryContents( const std::string& path, std::set* contents) override; Status GetDirectorySubdirs( const std::string& path, std::set* subdirs) override; Status GetDirectoryFiles( const std::string& path, std::set* files) override; Status ReadTextFile(const std::string& path, std::string* contents) override; Status LocalizePath( const std::string& path, std::shared_ptr* localized) override; Status WriteTextFile( const std::string& path, const std::string& contents) override; Status WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; Status MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: Status ParsePath( const std::string& path, std::string* bucket, std::string* object); Status CleanPath(const std::string& s3_path, std::string* clean_path); std::unique_ptr client_; // init after Aws::InitAPI is called re2::RE2 s3_regex_; }; Status S3FileSystem::ParsePath( const std::string& path, std::string* bucket, std::string* object) { // Cleanup extra slashes std::string clean_path; RETURN_IF_ERROR(CleanPath(path, &clean_path)); // Get the bucket name and the object path. Return error if path is malformed std::string protocol, host_name, host_port; if (!RE2::FullMatch( clean_path, s3_regex_, &protocol, &host_name, &host_port, bucket, object)) { int bucket_start = clean_path.find("s3://") + strlen("s3://"); int bucket_end = clean_path.find("/", bucket_start); // If there isn't a slash, the address has only the bucket if (bucket_end > bucket_start) { *bucket = clean_path.substr(bucket_start, bucket_end - bucket_start); *object = clean_path.substr(bucket_end + 1); } else { *bucket = clean_path.substr(bucket_start); *object = ""; } } else { // Erase leading '/' that is left behind in object name if ((*object)[0] == '/') { object->erase(0, 1); } } if (bucket->empty()) { return Status( Status::Code::INTERNAL, "No bucket name found in path: " + path); } return Status::Success; } Status S3FileSystem::CleanPath(const std::string& s3_path, std::string* clean_path) { // Must handle paths with s3 prefix size_t start = s3_path.find("s3://"); std::string path = ""; if (start != std::string::npos) { path = s3_path.substr(start + strlen("s3://")); *clean_path = "s3://"; } else { path = s3_path; *clean_path = ""; } // Must handle paths with https:// or http:// prefix size_t https_start = path.find("https://"); if (https_start != std::string::npos) { path = path.substr(https_start + strlen("https://")); *clean_path += "https://"; } else { size_t http_start = path.find("http://"); if (http_start != std::string::npos) { path = path.substr(http_start + strlen("http://")); *clean_path += "http://"; } } // Remove trailing slashes size_t rtrim_length = path.find_last_not_of('/'); if (rtrim_length == std::string::npos) { return Status( Status::Code::INVALID_ARG, "Invalid bucket name: '" + path + "'"); } // Remove leading slashes size_t ltrim_length = path.find_first_not_of('/'); if (ltrim_length == std::string::npos) { return Status( Status::Code::INVALID_ARG, "Invalid bucket name: '" + path + "'"); } // Remove extra internal slashes std::string true_path = path.substr(ltrim_length, rtrim_length + 1); std::vector slash_locations; bool previous_slash = false; for (size_t i = 0; i < true_path.size(); i++) { if (true_path[i] == '/') { if (!previous_slash) { *clean_path += true_path[i]; } previous_slash = true; } else { *clean_path += true_path[i]; previous_slash = false; } } return Status::Success; } S3FileSystem::S3FileSystem( const std::string& s3_path, const S3Credential& s3_cred) : s3_regex_( "s3://(http://|https://|)([0-9a-zA-Z\\-.]+):([0-9]+)/" "([0-9a-z.\\-]+)(((/[0-9a-zA-Z.\\-_]+)*)?)") { // init aws api if not already Aws::SDKOptions options; static std::once_flag onceFlag; std::call_once(onceFlag, [&options] { Aws::InitAPI(options); }); // [FIXME: DLIS-4973] Aws::Http::SetHttpClientFactory( Aws::MakeShared(S3_ALLOCATION_TAG)); Aws::Client::ClientConfiguration config; Aws::Auth::AWSCredentials credentials; // check vars for S3 credentials -> aws profile -> default if (!s3_cred.secret_key_.empty() && !s3_cred.key_id_.empty()) { credentials.SetAWSAccessKeyId(s3_cred.key_id_.c_str()); credentials.SetAWSSecretKey(s3_cred.secret_key_.c_str()); if (!s3_cred.session_token_.empty()) { credentials.SetSessionToken(s3_cred.session_token_.c_str()); } config = Aws::Client::ClientConfiguration(); if (!s3_cred.region_.empty()) { config.region = s3_cred.region_.c_str(); } } else if (!s3_cred.profile_name_.empty()) { config = Aws::Client::ClientConfiguration(s3_cred.profile_name_.c_str()); } else { config = Aws::Client::ClientConfiguration("default"); } // Cleanup extra slashes std::string clean_path; LOG_STATUS_ERROR(CleanPath(s3_path, &clean_path), "failed to parse S3 path"); std::string protocol, host_name, host_port, bucket, object; if (RE2::FullMatch( clean_path, s3_regex_, &protocol, &host_name, &host_port, &bucket, &object)) { config.endpointOverride = Aws::String(host_name + ":" + host_port); if (protocol == "https://") { config.scheme = Aws::Http::Scheme::HTTPS; } else { config.scheme = Aws::Http::Scheme::HTTP; } } if (!s3_cred.secret_key_.empty() && !s3_cred.key_id_.empty()) { client_ = std::make_unique( credentials, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, /*useVirtualAdressing*/ false); } else { client_ = std::make_unique( config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, /*useVirtualAdressing*/ false); } } Status S3FileSystem::CheckClient(const std::string& s3_path) { std::string bucket, object_path; RETURN_IF_ERROR(ParsePath(s3_path, &bucket, &object_path)); // check if can connect to the bucket s3::Model::HeadBucketRequest head_request; head_request.WithBucket(bucket.c_str()); auto head_object_outcome = client_->HeadBucket(head_request); if (!head_object_outcome.IsSuccess()) { auto err = head_object_outcome.GetError(); return Status( Status::Code::INTERNAL, "Unable to create S3 filesystem client. Check account credentials. " "Exception: '" + err.GetExceptionName() + "' Message: '" + err.GetMessage() + "'"); } return Status::Success; } Status S3FileSystem::FileExists(const std::string& path, bool* exists) { *exists = false; // S3 doesn't make objects for directories, so it could still be a directory bool is_dir; RETURN_IF_ERROR(IsDirectory(path, &is_dir)); if (is_dir) { *exists = is_dir; return Status::Success; } std::string bucket, object; RETURN_IF_ERROR(ParsePath(path, &bucket, &object)); // Construct request for object metadata s3::Model::HeadObjectRequest head_request; head_request.SetBucket(bucket.c_str()); head_request.SetKey(object.c_str()); auto head_object_outcome = client_->HeadObject(head_request); if (!head_object_outcome.IsSuccess()) { if (head_object_outcome.GetError().GetErrorType() != s3::S3Errors::RESOURCE_NOT_FOUND) { return Status( Status::Code::INTERNAL, "Could not get MetaData for object at " + path + " due to exception: " + head_object_outcome.GetError().GetExceptionName() + ", error message: " + head_object_outcome.GetError().GetMessage()); } } else { *exists = true; } return Status::Success; } Status S3FileSystem::IsDirectory(const std::string& path, bool* is_dir) { *is_dir = false; std::string bucket, object_path; RETURN_IF_ERROR(ParsePath(path, &bucket, &object_path)); // Check if the bucket exists s3::Model::HeadBucketRequest head_request; head_request.WithBucket(bucket.c_str()); auto head_bucket_outcome = client_->HeadBucket(head_request); if (!head_bucket_outcome.IsSuccess()) { return Status( Status::Code::INTERNAL, "Could not get MetaData for bucket with name " + bucket + " due to exception: " + head_bucket_outcome.GetError().GetExceptionName() + ", error message: " + head_bucket_outcome.GetError().GetMessage()); } // Root case - bucket exists and object path is empty if (object_path.empty()) { *is_dir = true; return Status::Success; } // List the objects in the bucket s3::Model::ListObjectsV2Request list_objects_request; list_objects_request.SetBucket(bucket.c_str()); list_objects_request.SetPrefix(AppendSlash(object_path).c_str()); auto list_objects_outcome = client_->ListObjectsV2(list_objects_request); if (list_objects_outcome.IsSuccess()) { *is_dir = !list_objects_outcome.GetResult().GetContents().empty(); } else { return Status( Status::Code::INTERNAL, "Failed to list objects with prefix " + path + " due to exception: " + list_objects_outcome.GetError().GetExceptionName() + ", error message: " + list_objects_outcome.GetError().GetMessage()); } return Status::Success; } Status S3FileSystem::FileModificationTime(const std::string& path, int64_t* mtime_ns) { // We don't need to worry about the case when this is a directory bool is_dir; RETURN_IF_ERROR(IsDirectory(path, &is_dir)); if (is_dir) { *mtime_ns = 0; return Status::Success; } std::string bucket, object; RETURN_IF_ERROR(ParsePath(path, &bucket, &object)); // Send a request for the objects metadata s3::Model::HeadObjectRequest head_request; head_request.SetBucket(bucket.c_str()); head_request.SetKey(object.c_str()); // If request succeeds, copy over the modification time auto head_object_outcome = client_->HeadObject(head_request); if (head_object_outcome.IsSuccess()) { *mtime_ns = head_object_outcome.GetResult().GetLastModified().Millis() * NANOS_PER_MILLIS; } else { return Status( Status::Code::INTERNAL, "Failed to get modification time for object at " + path + " due to exception: " + head_object_outcome.GetError().GetExceptionName() + ", error message: " + head_object_outcome.GetError().GetMessage()); } return Status::Success; } Status S3FileSystem::GetDirectoryContents( const std::string& path, std::set* contents) { // Parse bucket and dir_path std::string bucket, dir_path, full_dir; RETURN_IF_ERROR(ParsePath(path, &bucket, &dir_path)); std::string true_path = "s3://" + bucket + '/' + dir_path; // Capture the full path to facilitate content listing full_dir = AppendSlash(dir_path); // Issue request for objects with prefix s3::Model::ListObjectsV2Request objects_request; objects_request.SetBucket(bucket.c_str()); objects_request.SetPrefix(full_dir.c_str()); bool done_listing = false; while (!done_listing) { auto list_objects_outcome = client_->ListObjectsV2(objects_request); if (!list_objects_outcome.IsSuccess()) { return Status( Status::Code::INTERNAL, "Could not list contents of directory at " + true_path + " due to exception: " + list_objects_outcome.GetError().GetExceptionName() + ", error message: " + list_objects_outcome.GetError().GetMessage()); } const auto& list_objects_result = list_objects_outcome.GetResult(); for (const auto& s3_object : list_objects_result.GetContents()) { // In the case of empty directories, the directory itself will appear // here if (s3_object.GetKey().c_str() == full_dir) { continue; } // We have to make sure that subdirectory contents do not appear here std::string name(s3_object.GetKey().c_str()); int item_start = name.find(full_dir) + full_dir.size(); // S3 response prepends parent directory name int item_end = name.find("/", item_start); // Let set take care of subdirectory contents std::string item = name.substr(item_start, item_end - item_start); contents->insert(item); // Fail-safe check to ensure the item name is not empty if (item.empty()) { return Status( Status::Code::INTERNAL, "Cannot handle item with empty name at " + true_path); } } // If there are more pages to retrieve, set the marker to the next page. if (list_objects_result.GetIsTruncated()) { objects_request.SetContinuationToken( list_objects_result.GetNextContinuationToken()); } else { done_listing = true; } } return Status::Success; } Status S3FileSystem::GetDirectorySubdirs( const std::string& path, std::set* subdirs) { // Parse bucket and dir_path std::string bucket, dir_path; RETURN_IF_ERROR(ParsePath(path, &bucket, &dir_path)); std::string true_path = "s3://" + bucket + '/' + dir_path; RETURN_IF_ERROR(GetDirectoryContents(true_path, subdirs)); // Erase non-directory entries... for (auto iter = subdirs->begin(); iter != subdirs->end();) { bool is_dir; RETURN_IF_ERROR(IsDirectory(JoinPath({true_path, *iter}), &is_dir)); if (!is_dir) { iter = subdirs->erase(iter); } else { ++iter; } } return Status::Success; } Status S3FileSystem::GetDirectoryFiles( const std::string& path, std::set* files) { // Parse bucket and dir_path std::string bucket, dir_path; RETURN_IF_ERROR(ParsePath(path, &bucket, &dir_path)); std::string true_path = "s3://" + bucket + '/' + dir_path; RETURN_IF_ERROR(GetDirectoryContents(true_path, files)); // Erase directory entries... for (auto iter = files->begin(); iter != files->end();) { bool is_dir; RETURN_IF_ERROR(IsDirectory(JoinPath({true_path, *iter}), &is_dir)); if (is_dir) { iter = files->erase(iter); } else { ++iter; } } return Status::Success; } Status S3FileSystem::ReadTextFile(const std::string& path, std::string* contents) { bool exists; RETURN_IF_ERROR(FileExists(path, &exists)); if (!exists) { return Status(Status::Code::INTERNAL, "File does not exist at " + path); } std::string bucket, object; RETURN_IF_ERROR(ParsePath(path, &bucket, &object)); // Send a request for the objects metadata s3::Model::GetObjectRequest object_request; object_request.SetBucket(bucket.c_str()); object_request.SetKey(object.c_str()); auto get_object_outcome = client_->GetObject(object_request); if (get_object_outcome.IsSuccess()) { auto& object_result = get_object_outcome.GetResultWithOwnership().GetBody(); std::string data = ""; char c; while (object_result.get(c)) { data += c; } *contents = data; } else { return Status( Status::Code::INTERNAL, "Failed to get object at " + path + " due to exception: " + get_object_outcome.GetError().GetExceptionName() + ", error message: " + get_object_outcome.GetError().GetMessage()); } return Status::Success; } Status S3FileSystem::LocalizePath( const std::string& path, std::shared_ptr* localized) { // Check if the directory or file exists bool exists; RETURN_IF_ERROR(FileExists(path, &exists)); if (!exists) { return Status( Status::Code::INTERNAL, "directory or file does not exist at " + path); } // Cleanup extra slashes std::string clean_path; RETURN_IF_ERROR(CleanPath(path, &clean_path)); // Remove protocol and host name and port std::string effective_path, protocol, host_name, host_port, bucket, object; if (RE2::FullMatch( clean_path, s3_regex_, &protocol, &host_name, &host_port, &bucket, &object)) { effective_path = "s3://" + bucket + object; } else { effective_path = path; } // Create a local directory for AWS model store. // If ENV variable are not set, creates a temporary directory // under `/tmp` with the format: "folderXXXXXX". // Otherwise, will create a folder under specified directory with the same // format. std::string env_mount_dir = GetEnvironmentVariableOrDefault( "TRITON_AWS_MOUNT_DIRECTORY", kDefaultMountDirectory); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); // Specify contents to be downloaded std::set contents; bool is_dir; RETURN_IF_ERROR(IsDirectory(path, &is_dir)); if (is_dir) { // Set localized path localized->reset(new LocalizedPath(effective_path, tmp_folder)); // Specify the entire directory to be downloaded std::set filenames; RETURN_IF_ERROR(GetDirectoryContents(effective_path, &filenames)); for (auto itr = filenames.begin(); itr != filenames.end(); ++itr) { contents.insert(JoinPath({effective_path, *itr})); } } else { // Set localized path std::string filename = effective_path.substr(effective_path.find_last_of('/') + 1); localized->reset( new LocalizedPath(effective_path, JoinPath({tmp_folder, filename}))); // Specify only the file to be downloaded contents.insert(effective_path); } // Download all specified contents and nested contents while (contents.size() != 0) { std::set tmp_contents = contents; contents.clear(); for (auto iter = tmp_contents.begin(); iter != tmp_contents.end(); ++iter) { std::string s3_fpath = *iter; std::string s3_removed_path = s3_fpath.substr(effective_path.size()); std::string local_fpath = s3_removed_path.empty() ? (*localized)->Path() : JoinPath({(*localized)->Path(), s3_removed_path}); bool is_subdir; RETURN_IF_ERROR(IsDirectory(s3_fpath, &is_subdir)); if (is_subdir) { // Create local mirror of sub-directories #ifdef _WIN32 int status = mkdir(const_cast(local_fpath.c_str())); #else int status = mkdir( const_cast(local_fpath.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); #endif if (status == -1) { return Status( Status::Code::INTERNAL, "Failed to create local folder: " + local_fpath + ", errno:" + strerror(errno)); } // Add sub-directories and deeper files to contents std::set subdir_contents; RETURN_IF_ERROR(GetDirectoryContents(s3_fpath, &subdir_contents)); for (auto itr = subdir_contents.begin(); itr != subdir_contents.end(); ++itr) { contents.insert(JoinPath({s3_fpath, *itr})); } } else { // Create local copy of file std::string file_bucket, file_object; RETURN_IF_ERROR(ParsePath(s3_fpath, &file_bucket, &file_object)); s3::Model::GetObjectRequest object_request; object_request.SetBucket(file_bucket.c_str()); object_request.SetKey(file_object.c_str()); auto get_object_outcome = client_->GetObject(object_request); if (get_object_outcome.IsSuccess()) { auto& retrieved_file = get_object_outcome.GetResultWithOwnership().GetBody(); std::ofstream output_file(local_fpath.c_str(), std::ios::binary); output_file << retrieved_file.rdbuf(); output_file.close(); } else { return Status( Status::Code::INTERNAL, "Failed to get object at " + s3_fpath + " due to exception: " + get_object_outcome.GetError().GetExceptionName() + ", error message: " + get_object_outcome.GetError().GetMessage()); } } } } return Status::Success; } Status S3FileSystem::WriteTextFile( const std::string& path, const std::string& contents) { return Status( Status::Code::UNSUPPORTED, "Write text file operation not yet implemented " + path); } Status S3FileSystem::WriteBinaryFile( const std::string& path, const char* contents, const size_t content_len) { return Status( Status::Code::UNSUPPORTED, "Write text file operation not yet implemented " + path); } Status S3FileSystem::MakeDirectory(const std::string& dir, const bool recursive) { return Status( Status::Code::UNSUPPORTED, "Make directory operation not yet implemented"); } Status S3FileSystem::MakeTemporaryDirectory( std::string dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, "Make temporary directory operation not yet implemented"); } Status S3FileSystem::DeletePath(const std::string& path) { return Status( Status::Code::UNSUPPORTED, "Delete path operation not yet implemented"); } }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_parameter.cc000066400000000000000000000050051527052371200210260ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_parameter.h" namespace triton { namespace core { const void* InferenceParameter::ValuePointer() const { switch (type_) { case TRITONSERVER_PARAMETER_STRING: return reinterpret_cast(value_string_.c_str()); case TRITONSERVER_PARAMETER_INT: return reinterpret_cast(&value_int64_); case TRITONSERVER_PARAMETER_BOOL: return reinterpret_cast(&value_bool_); case TRITONSERVER_PARAMETER_DOUBLE: return reinterpret_cast(&value_double_); case TRITONSERVER_PARAMETER_BYTES: return reinterpret_cast(value_bytes_); default: break; } return nullptr; } std::ostream& operator<<(std::ostream& out, const InferenceParameter& parameter) { out << "[0x" << std::addressof(parameter) << "] " << "name: " << parameter.Name() << ", type: " << TRITONSERVER_ParameterTypeString(parameter.Type()) << ", value: "; return out; } }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_parameter.h000066400000000000000000000077331527052371200207020ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "tritonserver_apis.h" namespace triton { namespace core { // // An inference parameter. // class InferenceParameter { public: InferenceParameter(const char* name, const char* value) : name_(name), type_(TRITONSERVER_PARAMETER_STRING), value_string_(value) { byte_size_ = value_string_.size(); } InferenceParameter(const char* name, const int64_t value) : name_(name), type_(TRITONSERVER_PARAMETER_INT), value_int64_(value), byte_size_(sizeof(int64_t)) { } InferenceParameter(const char* name, const bool value) : name_(name), type_(TRITONSERVER_PARAMETER_BOOL), value_bool_(value), byte_size_(sizeof(bool)) { } InferenceParameter(const char* name, const double value) : name_(name), type_(TRITONSERVER_PARAMETER_DOUBLE), value_double_(value), byte_size_(sizeof(double)) { } InferenceParameter(const char* name, const void* ptr, const uint64_t size) : name_(name), type_(TRITONSERVER_PARAMETER_BYTES), value_bytes_(ptr), byte_size_(size) { } // The name of the parameter. const std::string& Name() const { return name_; } // Data type of the parameter. TRITONSERVER_ParameterType Type() const { return type_; } // Return a pointer to the parameter, or a pointer to the data content // if type_ is TRITONSERVER_PARAMETER_BYTES. This returned pointer must be // cast correctly based on 'type_'. // TRITONSERVER_PARAMETER_STRING -> const char* // TRITONSERVER_PARAMETER_INT -> int64_t* // TRITONSERVER_PARAMETER_BOOL -> bool* // TRITONSERVER_PARAMETER_DOUBLE -> double* // TRITONSERVER_PARAMETER_BYTES -> const void* const void* ValuePointer() const; // Return the data byte size of the parameter. uint64_t ValueByteSize() const { return byte_size_; } // Return the parameter value string, the return value is valid only if // Type() returns TRITONSERVER_PARAMETER_STRING const std::string& ValueString() const { return value_string_; } private: friend std::ostream& operator<<( std::ostream& out, const InferenceParameter& parameter); std::string name_; TRITONSERVER_ParameterType type_; std::string value_string_; int64_t value_int64_; bool value_bool_; double value_double_; const void* value_bytes_; uint64_t byte_size_; }; std::ostream& operator<<( std::ostream& out, const InferenceParameter& parameter); }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_request.cc000066400000000000000000001753661527052371200205600ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_request.h" #include #include #include #include "constants.h" #include "model.h" #include "model_config_utils.h" #include "server.h" #include "triton/common/logging.h" #ifdef TRITON_ENABLE_TRACING #include "cuda_utils.h" #endif // TRITON_ENABLE_TRACING namespace triton { namespace core { namespace { // Utilities for Null request feature. TRITONSERVER_Error* NullResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected allocation for null request, no output should be requested."); } TRITONSERVER_Error* NullResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected release for null request, no output should be requested."); } ResponseAllocator null_allocator = ResponseAllocator( NullResponseAlloc, NullResponseRelease, nullptr /* start_fn */); void NullResponseComplete( TRITONSERVER_InferenceResponse* iresponse, const uint32_t flags, void* userp) { if (iresponse != nullptr) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(iresponse), "deleting null response"); } } void NullRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { if ((flags & TRITONSERVER_REQUEST_RELEASE_ALL) != 0) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestDelete(request), "deleting null request"); } } } // namespace InferenceRequest::InferenceRequest( const std::shared_ptr& model, const int64_t requested_model_version) : InferenceRequest(model.get(), requested_model_version) { model_shared_ = model; } InferenceRequest::InferenceRequest( Model* model, const int64_t requested_model_version) : needs_normalization_(true), model_raw_(model), requested_model_version_(requested_model_version), flags_(0), correlation_id_(0), batch_size_(0), timeout_us_(0), collect_stats_(true), state_(InferenceRequest::State::INITIALIZED), null_request_(false) { SetPriority(0); // Outer-most release callback to ensure a request has been taken, this // callback won't be invoked, if certain flags are set. release_callbacks_.emplace_back( [](std::unique_ptr& request, const uint32_t flags) -> Status { if (flags & TRITONSERVER_REQUEST_RELEASE_RESCHEDULE) { return Status( Status::Code::INVALID_ARG, "Request is released with " "TRITONSERVER_REQUEST_RELEASE_RESCHEDULE, while the model is not " "configured to handle such a flag."); } return Status::Success; }); } Status InferenceRequest::SetState(InferenceRequest::State new_state) { LOG_VERBOSE(1) << LogRequest() << "Setting state from " << state_ << " to " << new_state; // No-op if this is already the current state, or if this is a null request. if (new_state == state_ || null_request_) { return Status::Success; } // Generate error when called rather than copying it into every case below. const auto generate_error = [&]() { std::stringstream ss; ss << LogRequest() << "Invalid request state transition from " << state_ << " to " << new_state; return Status(Status::Code::INTERNAL, ss.str()); }; // Define state transitions switch (state_) { case InferenceRequest::State::INITIALIZED: { if (new_state == InferenceRequest::State::PENDING) { IncrementPendingRequestCount(); } else if (new_state == InferenceRequest::State::RELEASED) { // No-op when moving from initialized to released, just releasing early. } else { return generate_error(); } break; } case InferenceRequest::State::PENDING: { // Request may move from pending to either execution when scheduled to // backend, released early due to some error or failure was encountered // when calling enqueue. if (new_state == InferenceRequest::State::EXECUTING || new_state == InferenceRequest::State::RELEASED || new_state == InferenceRequest::State::FAILED_ENQUEUE) { DecrementPendingRequestCount(); } else { // Unexpected state transition return generate_error(); } break; } case InferenceRequest::State::EXECUTING: { if (new_state != InferenceRequest::State::RELEASED) { return generate_error(); } break; } case InferenceRequest::State::RELEASED: { if (new_state != InferenceRequest::State::INITIALIZED) { // Only transition currently supported after release is to start over // again, such as re-using request objects for multiple inferences. return generate_error(); } break; } case InferenceRequest::State::FAILED_ENQUEUE: { if (new_state != InferenceRequest::State::INITIALIZED) { // Only transition currently supported after failed to enqueue is to // start over again, such as re-using request objects for multiple // inferences. return generate_error(); } break; } } state_ = new_state; return Status::Success; } void InferenceRequest::IncrementPendingRequestCount() { #ifdef TRITON_ENABLE_METRICS // Pending request count should always be 0 or 1 per-request. If a request // increments the count, it should not be incremented again until decremented. auto reporter = model_raw_->MetricReporter(); if (reporter) { reporter->IncrementGauge(kPendingRequestMetric, 1); } #endif // TRITON_ENABLE_METRICS } void InferenceRequest::DecrementPendingRequestCount() { #ifdef TRITON_ENABLE_METRICS // Pending request count should always be 0 or 1 per-request. A request should // not decrement the count unless it has already been incremented. auto reporter = model_raw_->MetricReporter(); if (reporter) { reporter->DecrementGauge(kPendingRequestMetric, 1); } #endif // TRITON_ENABLE_METRICS } const std::string& InferenceRequest::ModelName() const { return model_raw_->Name(); } int64_t InferenceRequest::ActualModelVersion() const { return model_raw_->Version(); } void InferenceRequest::SetPriority(uint64_t p) { if ((p == 0) || (p > model_raw_->MaxPriorityLevel())) { priority_ = model_raw_->DefaultPriorityLevel(); } else { priority_ = p; } } Status InferenceRequest::AddParameter(const char* name, const char* value) { parameters_.emplace_back(name, value); return Status::Success; } Status InferenceRequest::AddParameter(const char* name, const int64_t value) { parameters_.emplace_back(name, value); return Status::Success; } Status InferenceRequest::AddParameter(const char* name, const bool value) { parameters_.emplace_back(name, value); return Status::Success; } Status InferenceRequest::AddParameter(const char* name, const double value) { parameters_.emplace_back(name, value); return Status::Success; } Status InferenceRequest::SetParameters( const std::deque& parameters) { // NOTE: For BYTES parameters, this will shallow copy the pointer for now. parameters_ = parameters; return Status::Success; } #ifdef TRITON_ENABLE_TRACING Status InferenceRequest::TraceInputTensors( TRITONSERVER_InferenceTraceActivity activity, const std::string& msg) { const auto& inputs = this->ImmutableInputs(); TRITONSERVER_MemoryType dst_memory_type = TRITONSERVER_MEMORY_CPU; int64_t dst_memory_type_id = 0; for (const auto& pr : inputs) { InferenceRequest::Input* ti = pr.second; // input data const std::string& name = ti->Name(); TRITONSERVER_DataType datatype = DataTypeToTriton(ti->DType()); uint64_t byte_size = ti->Data()->TotalByteSize(); const int64_t* shape = ti->ShapeWithBatchDim().data(); uint32_t dim_count = ti->ShapeWithBatchDim().size(); uint32_t buffer_count = ti->DataBufferCount(); // chunk buffer Status status; const void* buffer; uint64_t buffer_size; TRITONSERVER_MemoryType src_memory_type; int64_t src_memory_type_id; bool cuda_used; if (buffer_count == 0) { LOG_STATUS_ERROR( status, LogRequest() + TRITONSERVER_InferenceTraceActivityString(activity) + ": " + msg + ": tensor: " + name + ": no buffer chunk"); continue; } if (buffer_count == 1) { status = ti->DataBuffer( 0, &buffer, &buffer_size, &src_memory_type, &src_memory_type_id); if (!status.IsOk()) { LOG_STATUS_ERROR( status, LogRequest() + TRITONSERVER_InferenceTraceActivityString(activity) + ": " + msg + ": tensor: " + name + ": fail to get data buffer: " + status.Message()); return status; } if (buffer_size != byte_size) { LOG_STATUS_ERROR( status, LogRequest() + TRITONSERVER_InferenceTraceActivityString(activity) + ": " + msg + ": tensor: " + name + ": truncated buffer"); continue; } INFER_TRACE_TENSOR_ACTIVITY( this->trace_, activity, name.c_str(), datatype, const_cast(buffer), buffer_size, shape, dim_count, src_memory_type, src_memory_type_id); continue; } // input buffer std::vector in_buffer(byte_size); char* base = in_buffer.data(); size_t offset = 0; for (uint32_t b = 0; b < buffer_count; ++b) { status = ti->DataBuffer( b, &buffer, &buffer_size, &src_memory_type, &src_memory_type_id); if (!status.IsOk()) { LOG_STATUS_ERROR( status, LogRequest() + TRITONSERVER_InferenceTraceActivityString(activity) + ": " + msg + ": tensor: " + name + ": fail to get data buffer: " + status.Message()); return status; } status = CopyBuffer( "InferenceRequest TraceInputTensors", src_memory_type, src_memory_type_id, dst_memory_type, dst_memory_type_id, buffer_size, buffer, base + offset, nullptr, &cuda_used); if (!status.IsOk()) { LOG_STATUS_ERROR( status, LogRequest() + TRITONSERVER_InferenceTraceActivityString(activity) + ": " + msg + ": tensor: " + name + ": fail to copy buffer: " + status.Message()); return status; } offset += buffer_size; } INFER_TRACE_TENSOR_ACTIVITY( this->trace_, activity, name.c_str(), datatype, static_cast(base), byte_size, shape, dim_count, dst_memory_type, dst_memory_type_id); } return Status::Success; } #endif // TRITON_ENABLE_TRACING Status InferenceRequest::OutputBufferProperties( const char* name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { const auto allocator = response_factory_->Allocator(); if ((allocator == nullptr) || (allocator->QueryFn() == nullptr)) { return Status( Status::Code::UNAVAILABLE, (LogRequest() + "Output properties are not available").c_str()); } else { RETURN_IF_TRITONSERVER_ERROR(allocator->QueryFn()( reinterpret_cast( const_cast(allocator)), response_factory_->AllocatorUserp(), name, byte_size, memory_type, memory_type_id)); } return Status::Success; } Status InferenceRequest::Run(std::unique_ptr& request) { RETURN_IF_ERROR(request->SetState(InferenceRequest::State::PENDING)); auto status = request->model_raw_->Enqueue(request); if (!status.IsOk()) { LOG_STATUS_ERROR( request->SetState(InferenceRequest::State::FAILED_ENQUEUE), "Failed to set failed_enqueue state"); } return status; } FailureReason stringToFailureReason(const std::string& error_type) { if (error_type == "REJECTED") { return FailureReason::REJECTED; } if (error_type == "CANCELED") { return FailureReason::CANCELED; } if (error_type == "BACKEND") { return FailureReason::BACKEND; } return FailureReason::OTHER; } void InferenceRequest::RespondIfError( std::unique_ptr& request, const Status& status, const bool release_request, FailureReason reason) { if (status.IsOk()) { return; } // Use the response factory to create a response, set the status, // and send it. If something goes wrong all we can do is log the // error. Because this is sending an error we assume that this is // the last response for the request and so set the FINAL flag. std::unique_ptr response; LOG_STATUS_ERROR( request->response_factory_->CreateResponse(&response), (request->LogRequest() + "failed to create error response").c_str()); LOG_STATUS_ERROR( InferenceResponse::SendWithStatus( std::move(response), TRITONSERVER_RESPONSE_COMPLETE_FINAL, status), (request->LogRequest() + "failed to send error response").c_str()); #ifdef TRITON_ENABLE_STATS request->ReportErrorStatistics( request->model_raw_->MetricReporter().get(), reason); #endif // If releasing the request then invoke the release callback which // gives ownership to the callback. So can't access 'request' after // this point. if (release_request) { InferenceRequest::Release( std::move(request), TRITONSERVER_REQUEST_RELEASE_ALL); } } Status InferenceRequest::Release( std::unique_ptr&& request, const uint32_t release_flags) { // Invoke the release callbacks added internally before releasing the // request to user provided callback. for (auto it = request->release_callbacks_.rbegin(); it != request->release_callbacks_.rend(); it++) { RETURN_IF_ERROR((*it)(request, release_flags)); if (request == nullptr) { return Status::Success; } } #ifdef TRITON_ENABLE_TRACING // If tracing then record request end and release the trace. // This must be before the request callback to ensure the trace // is properly layered, as the request may be nested in an ensemble // and the callback may interact with upper level trace. if (request->trace_ != nullptr) { request->trace_->ReportNow(TRITONSERVER_TRACE_REQUEST_END); request->ReleaseTrace(); } #endif // TRITON_ENABLE_TRACING LOG_STATUS_ERROR( request->SetState(InferenceRequest::State::RELEASED), "Failed to set released state"); void* userp = request->release_userp_; auto& release_fn = request->release_fn_; release_fn( reinterpret_cast(request.release()), release_flags, userp); return Status::Success; } InferenceRequest* InferenceRequest::CopyAsNull(const InferenceRequest& from) { // Create a copy of 'from' request with artificial inputs and no requested // outputs. Maybe more efficient to share inputs and other metadata, // but that binds the Null request with 'from' request's lifecycle. std::unique_ptr lrequest( new InferenceRequest(from.model_raw_, from.requested_model_version_)); lrequest->null_request_ = true; lrequest->needs_normalization_ = false; lrequest->batch_size_ = from.batch_size_; lrequest->collect_stats_ = false; // Three passes: first to construct input for the shape tensors inputs, second // to obtain the max input byte size for allocating a large enough buffer for // all non shape tensor inputs; third to construct the inputs for these // tensors. // First pass for (const auto& input : from.OriginalInputs()) { // Handle only shape tensors in this pass if (!input.second.IsShapeTensor()) { continue; } // Prepare the memory to hold input data size_t byte_size = input.second.Data()->TotalByteSize(); auto mem_type = TRITONSERVER_MEMORY_CPU; int64_t mem_id = 0; std::shared_ptr data = std::make_shared(byte_size, mem_type, mem_id); // Get the source buffer. Assumes shape tensors be in a single buffer on the // CPU const auto& from_data = input.second.Data(); size_t from_data_byte_size; TRITONSERVER_MemoryType from_data_memory_type; int64_t from_data_memory_id; const char* from_data_buffer = from_data->BufferAt( 0 /* idx */, &from_data_byte_size, &from_data_memory_type, &from_data_memory_id); if (from_data_byte_size != byte_size) { LOG_WARNING << lrequest->LogRequest() << "The byte size of shape tensor to be copied does not match"; } // Copy the shape values to the input buffer std::memcpy(data->MutableBuffer(), from_data_buffer, from_data_byte_size); Input* new_input; lrequest->AddOriginalInput( input.first, input.second.DType(), input.second.Shape(), &new_input); // Must normalize shape here... *new_input->MutableShape() = input.second.Shape(); *new_input->MutableShapeWithBatchDim() = input.second.ShapeWithBatchDim(); new_input->SetData(data); } // Second pass size_t max_byte_size = 0; size_t max_str_byte_size = 0; const std::string* max_input_name = nullptr; for (const auto& input : from.OriginalInputs()) { // Skip shape tensors in this pass if (input.second.IsShapeTensor()) { continue; } if (input.second.DType() == inference::DataType::TYPE_STRING) { int64_t element_count = triton::common::GetElementCount(input.second.Shape()); size_t str_byte_size = static_cast(4 * element_count); max_str_byte_size = std::max(str_byte_size, max_str_byte_size); if (str_byte_size > max_byte_size) { max_byte_size = str_byte_size; max_input_name = &(input.first); } } else { if (input.second.Data()->TotalByteSize() >= max_byte_size) { max_byte_size = input.second.Data()->TotalByteSize(); max_input_name = &(input.first); } } } // Third pass // [DLIS-1268] should use one growable static buffer for all null requests auto mem_type = TRITONSERVER_MEMORY_CPU; int64_t mem_id = 0; std::shared_ptr data = std::make_shared(max_byte_size, mem_type, mem_id); auto data_base = data->BufferAt(0, &max_byte_size, &mem_type, &mem_id); // Zero initialization is only required when there is a TYPE_BYTES tensor in // the request. Only set the required number of bytes to zero. if (max_str_byte_size > 0) { std::fill( data->MutableBuffer(), data->MutableBuffer() + max_str_byte_size, 0); } for (const auto& input : from.OriginalInputs()) { // skip shape tensors in this pass if (input.second.IsShapeTensor()) { continue; } Input* new_input; lrequest->AddOriginalInput( input.first, input.second.DType(), input.second.Shape(), &new_input); // Must normalize shape here... *new_input->MutableShape() = input.second.Shape(); *new_input->MutableShapeWithBatchDim() = input.second.ShapeWithBatchDim(); // Note that the input that have max byte size will be responsible for // holding the artificial data, while other inputs will hold a reference to // it with byte size that matches 'from' if (input.first == *max_input_name) { new_input->SetData(data); } else { if (inference::DataType::TYPE_STRING == input.second.DType()) { new_input->AppendData( data_base, triton::common::GetElementCount(input.second.Shape()) * 4, mem_type, mem_id); } else { new_input->AppendData( data_base, input.second.Data()->TotalByteSize(), mem_type, mem_id); } } } // No outputs were requested and thus there should be no allocations. lrequest->SetResponseCallback( &null_allocator, nullptr, NullResponseComplete, nullptr); lrequest->SetReleaseCallback(NullRequestComplete, nullptr); lrequest->SetResponseFactory(); // Must normalize inputs here... for (auto& pr : lrequest->original_inputs_) { lrequest->inputs_.emplace( std::make_pair(pr.second.Name(), std::addressof(pr.second))); } return lrequest.release(); } Status InferenceRequest::MutableOriginalInput( const std::string& name, InferenceRequest::Input** input) { auto itr = original_inputs_.find(name); if (itr == original_inputs_.end()) { return Status( Status::Code::INVALID_ARG, LogRequest() + "input '" + name + "' does not exist in request"); } *input = &(itr->second); return Status::Success; } Status InferenceRequest::ImmutableInput( const std::string& name, const InferenceRequest::Input** input) const { auto itr = inputs_.find(name); if (itr == inputs_.end()) { return Status( Status::Code::INVALID_ARG, LogRequest() + "input '" + name + "' does not exist in request"); } *input = itr->second; return Status::Success; } Status InferenceRequest::AddOriginalInput( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count, InferenceRequest::Input** input) { const auto& pr = original_inputs_.emplace( std::piecewise_construct, std::forward_as_tuple(name), std::forward_as_tuple(name, datatype, shape, dim_count)); if (!pr.second) { return Status( Status::Code::INVALID_ARG, LogRequest() + "input '" + name + "' already exists in request"); } if (input != nullptr) { *input = std::addressof(pr.first->second); } needs_normalization_ = true; return Status::Success; } Status InferenceRequest::AddOriginalInput( const std::string& name, const inference::DataType datatype, const std::vector& shape, InferenceRequest::Input** input) { return AddOriginalInput(name, datatype, &shape[0], shape.size(), input); } Status InferenceRequest::AddRawInput( const std::string& name, InferenceRequest::Input** input) { if (original_inputs_.size() != 0) { return Status( Status::Code::INVALID_ARG, LogRequest() + "raw input '" + name + "' can't be added to request with other inputs"); } const auto& pr = original_inputs_.emplace( std::piecewise_construct, std::forward_as_tuple(name), std::forward_as_tuple()); if (!pr.second) { return Status( Status::Code::INVALID_ARG, LogRequest() + "input '" + name + "' already exists in request"); } if (input != nullptr) { *input = std::addressof(pr.first->second); } raw_input_name_ = name; needs_normalization_ = true; return Status::Success; } Status InferenceRequest::RemoveOriginalInput(const std::string& name) { if (original_inputs_.erase(name) != 1) { return Status( Status::Code::INVALID_ARG, LogRequest() + "input '" + name + "' does not exist in request"); } if (name == raw_input_name_) { raw_input_name_.clear(); } needs_normalization_ = true; return Status::Success; } Status InferenceRequest::RemoveAllOriginalInputs() { original_inputs_.clear(); raw_input_name_.clear(); needs_normalization_ = true; return Status::Success; } Status InferenceRequest::AddOverrideInput( const std::string& name, const inference::DataType datatype, const int64_t batch_size, const std::vector& shape, std::shared_ptr* input) { std::shared_ptr i = std::make_shared(name, datatype, shape); *(i->MutableShape()) = i->OriginalShape(); if (batch_size > 0) { *(i->MutableShapeWithBatchDim()) = {batch_size}; i->MutableShapeWithBatchDim()->insert( i->MutableShapeWithBatchDim()->end(), i->OriginalShape().begin(), i->OriginalShape().end()); } else { *(i->MutableShapeWithBatchDim()) = i->OriginalShape(); } RETURN_IF_ERROR(AddOverrideInput(i)); if (input != nullptr) { *input = std::move(i); } return Status::Success; } Status InferenceRequest::AddOverrideInput( const std::shared_ptr& input) { LOG_VERBOSE(1) << LogRequest() << "adding input override for " << input->Name() << ": " << *this; const auto& pr = override_inputs_.emplace(std::make_pair(input->Name(), input)); if (!pr.second) { pr.first->second = input; } // Add or replace this override in the inputs... const auto res = inputs_.emplace(std::make_pair(input->Name(), input.get())); if (!res.second) { res.first->second = input.get(); } LOG_VERBOSE(1) << LogRequest() << "added input override for " << input->Name() << ": " << *this; return Status::Success; } Status InferenceRequest::AddOriginalRequestedOutput(const std::string& name) { original_requested_outputs_.insert(name); needs_normalization_ = true; return Status::Success; } Status InferenceRequest::LoadInputStates() { // Add the input states to the inference request. if (sequence_states_ != nullptr) { if (sequence_states_->IsNullRequest()) { sequence_states_ = SequenceStates::CopyAsNull(sequence_states_->NullSequenceStates()); } for (auto& input_state_pair : sequence_states_->InputStates()) { auto& input_state = input_state_pair.second; std::shared_ptr input = std::make_shared( input_state->Name(), input_state->DType(), input_state->Shape()); *input->MutableShapeWithBatchDim() = input_state->Shape(); input->SetData(input_state->Data()); AddOverrideInput(input); } } return Status::Success; } Status InferenceRequest::RemoveOriginalRequestedOutput(const std::string& name) { original_requested_outputs_.erase(name); needs_normalization_ = true; return Status::Success; } Status InferenceRequest::RemoveAllOriginalRequestedOutputs() { original_requested_outputs_.clear(); needs_normalization_ = true; return Status::Success; } Status InferenceRequest::PrepareForInference() { // Remove override inputs as those are added during any previous // inference execution. inputs_.clear(); override_inputs_.clear(); SetResponseFactory(); // Renormalize if anything has changed in the inference request in a // way that could impact renormalization. if (needs_normalization_) { RETURN_IF_ERROR(Normalize()); needs_normalization_ = false; } // Initially show the actual inputs to be only the original // inputs. If overrides are added later they will be added to // 'inputs_'. for (auto& pr : original_inputs_) { inputs_.emplace( std::make_pair(pr.second.Name(), std::addressof(pr.second))); } // Clear the timestamps queue_start_ns_ = 0; batcher_start_ns_ = 0; #ifdef TRITON_ENABLE_STATS request_start_ns_ = 0; #endif // TRITON_ENABLE_STATS // Help enforce that PrepareForInference() is called prior to Run(). RETURN_IF_ERROR(SetState(InferenceRequest::State::INITIALIZED)); LOG_VERBOSE(1) << LogRequest() << "prepared: " << *this; return Status::Success; } Status InferenceRequest::Normalize() { const inference::ModelConfig& model_config = model_raw_->Config(); // Fill metadata for raw input if (!raw_input_name_.empty()) { const bool has_multiple_inputs = (original_inputs_.size() != 1) || (model_config.input_size() != 1); if (has_multiple_inputs) { return Status( Status::Code::INVALID_ARG, LogRequest() + "Raw request must only have 1 input (found " + std::to_string(original_inputs_.size()) + ") to be deduced but got " + std::to_string(model_config.input_size()) + " inputs in '" + ModelName() + "' model configuration"); } auto it = original_inputs_.begin(); if (raw_input_name_ != it->first) { return Status( Status::Code::INVALID_ARG, LogRequest() + "Unexpected reference name for raw input '" + raw_input_name_ + "' got '" + it->first + "'"); } const auto& config_input = model_config.input(0); auto& raw_input = it->second; std::vector shape; if (model_config.max_batch_size() != 0) { shape.emplace_back(1); } int64_t dynamic_axis = -1; size_t element_cnt = 1; for (const auto& dim : config_input.dims()) { if (dim == triton::common::WILDCARD_DIM) { if (dynamic_axis != -1) { return Status( Status::Code::INVALID_ARG, LogRequest() + "The shape of the raw input '" + config_input.name() + "' can not be deduced because there are more than one " "variable-sized dimension"); } dynamic_axis = shape.size(); } else { element_cnt *= (size_t)dim; } shape.emplace_back(dim); } if ((config_input.data_type() == inference::DataType::TYPE_STRING)) { const bool has_one_element = (dynamic_axis == -1) && (element_cnt == 1); if (!has_one_element) { return Status( Status::Code::INVALID_ARG, LogRequest() + "For BYTE datatype raw input '" + config_input.name() + "', the " "model must have input shape [1]"); } // In the case of BYTE data type, we will prepend the byte size to follow // the Triton convention. raw_input_size_ = raw_input.Data()->TotalByteSize(); RETURN_IF_ERROR(raw_input.PrependData( &raw_input_size_, sizeof(uint32_t), TRITONSERVER_MEMORY_CPU, 0)); // Limit the BYTE raw input not to have host policy specific input for // simplicity, such case won't happen given the current protocol spec. // Will need to extend Input::PrependData() if needed. if (!raw_input.HostPolicyData().empty()) { return Status( Status::Code::INVALID_ARG, LogRequest() + "Raw input with data associated " "with a host policy setting is not " "currently supported"); } } else if (dynamic_axis != -1) { shape[dynamic_axis] = raw_input.Data()->TotalByteSize() / element_cnt / triton::common::GetDataTypeByteSize(config_input.data_type()); } raw_input.SetMetadata(config_input.name(), config_input.data_type(), shape); } // Initialize the requested outputs to be used during inference. If // original_requested_outputs_ is empty assume all outputs specified // in model config are being requested. requested_outputs_.clear(); if (original_requested_outputs_.size() == 0) { for (const auto& output : model_config.output()) { requested_outputs_.insert(output.name()); } } else { // Validate if the original requested output name exists in the // model configuration. for (const auto& output_name : original_requested_outputs_) { const inference::ModelOutput* output_config; RETURN_IF_ERROR(model_raw_->GetOutput(output_name, &output_config)); } } // Make sure that the request is providing the number of inputs // as is expected by the model. RETURN_IF_ERROR(ValidateRequestInputs()); // Determine the batch size and shape of each input. if (model_config.max_batch_size() == 0) { // Model does not support Triton-style batching so set as // batch-size 0 and leave the tensor shapes as they are. batch_size_ = 0; for (auto& pr : original_inputs_) { auto& input = pr.second; *input.MutableShape() = input.OriginalShape(); const inference::ModelInput* input_config; RETURN_IF_ERROR(model_raw_->GetInput(input.Name(), &input_config)); if (input_config->is_shape_tensor()) { // For a shape tensor, mark that the input is a shape tensor. input.SetIsShapeTensor(); } else if (input_config->is_non_linear_format_io()) { // If a tensor uses a non-linear IO format, indicate that the input uses // a non-linear IO format. input.SetIsNonLinearFormatIo(); } } } else { // Model does support Triton-style batching so each input tensor // must have the same first dimension which is the batch // size. Adjust the shape of the input tensors to remove the batch // dimension. batch_size_ = 0; for (auto& pr : original_inputs_) { auto& input = pr.second; const inference::ModelInput* input_config; RETURN_IF_ERROR(model_raw_->GetInput(input.Name(), &input_config)); // For a shape tensor, keep the tensor's shape as it is and mark // that the input is a shape tensor. if (input_config->is_shape_tensor()) { *input.MutableShape() = input.OriginalShape(); input.SetIsShapeTensor(); continue; } else if (input_config->is_non_linear_format_io()) { // If a tensor uses a non-linear IO format, indicate that the input uses // a non-linear IO format. input.SetIsNonLinearFormatIo(); } if (input.OriginalShape().size() == 0) { return Status( Status::Code::INVALID_ARG, LogRequest() + "input '" + input.Name() + "' has no shape but model requires batch dimension for '" + ModelName() + "'"); } if (batch_size_ == 0) { batch_size_ = input.OriginalShape()[0]; } else if (input.OriginalShape()[0] != batch_size_) { return Status( Status::Code::INVALID_ARG, LogRequest() + "input '" + input.Name() + "' batch size does not match other inputs for '" + ModelName() + "'"); } input.MutableShape()->assign( input.OriginalShape().begin() + 1, input.OriginalShape().end()); } } // Make sure request batch-size doesn't exceed what is supported by // the model. if ((int)batch_size_ > model_config.max_batch_size()) { return Status( Status::Code::INVALID_ARG, LogRequest() + "inference request batch-size must be <= " + std::to_string(model_config.max_batch_size()) + " for '" + ModelName() + "'"); } // Verify that each input shape is valid for the model, make // adjustments for reshapes and find the total tensor size. for (auto& pr : original_inputs_) { const inference::ModelInput* input_config; RETURN_IF_ERROR(model_raw_->GetInput(pr.second.Name(), &input_config)); auto& input_id = pr.first; auto& input = pr.second; auto shape = input.MutableShape(); if (input.DType() != input_config->data_type()) { return Status( Status::Code::INVALID_ARG, LogRequest() + "inference input '" + input_id + "' data-type is '" + std::string( triton::common::DataTypeToProtocolString(input.DType())) + "', but model '" + ModelName() + "' expects '" + std::string(triton::common::DataTypeToProtocolString( input_config->data_type())) + "'"); } // Validate input shape { bool match_config = true; const auto& config_dims = input_config->dims(); const auto& input_dims = *shape; if (config_dims.size() != (int64_t)input_dims.size()) { match_config = false; } else { for (int i = 0; i < config_dims.size(); ++i) { if (input_dims[i] == triton::common::WILDCARD_DIM) { return Status( Status::Code::INVALID_ARG, LogRequest() + "All input dimensions should be specified for input '" + input_id + "' for model '" + ModelName() + "', got " + triton::common::DimsListToString(input.OriginalShape())); } else if ( (config_dims[i] != triton::common::WILDCARD_DIM) && (config_dims[i] != input_dims[i])) { match_config = false; break; } } } if (!match_config) { triton::common::DimsList full_dims; std::string implicit_batch_note = ""; if (model_config.max_batch_size() > 0) { full_dims.Add(triton::common::WILDCARD_DIM); implicit_batch_note = "NOTE: Setting a non-zero max_batch_size in the model config " "requires a batch dimension to be prepended to each input shape. " "If you want to specify the full shape including the batch dim " "in your input dims config, try setting max_batch_size to zero. " "See the model configuration docs for more info on " "max_batch_size."; } for (int i = 0; i < input_config->dims_size(); ++i) { full_dims.Add(input_config->dims(i)); } return Status( Status::Code::INVALID_ARG, LogRequest() + "unexpected shape for input '" + input_id + "' for model '" + ModelName() + "'. Expected " + triton::common::DimsListToString(full_dims) + ", got " + triton::common::DimsListToString(input.OriginalShape()) + ". " + implicit_batch_note); } } // If there is a reshape for this input then adjust them to // match the reshape. As reshape may have variable-size // dimensions, we need to record corresponding value so that we // can set the value correctly for reshape. if (input_config->has_reshape()) { std::deque variable_size_values; for (int64_t idx = 0; idx < input_config->dims_size(); idx++) { if (input_config->dims(idx) == -1) { variable_size_values.push_back((*shape)[idx]); } } shape->clear(); for (const auto& dim : input_config->reshape().shape()) { if (dim == -1) { shape->push_back(variable_size_values.front()); variable_size_values.pop_front(); } else { shape->push_back(dim); } } } // Create shape with batch dimension. // FIXME, should not need this!! if (batch_size_ == 0) { *input.MutableShapeWithBatchDim() = *shape; } else { input.MutableShapeWithBatchDim()->clear(); input.MutableShapeWithBatchDim()->push_back(batch_size_); for (int64_t d : *shape) { input.MutableShapeWithBatchDim()->push_back(d); } } // Matching incoming request's shape and byte size to make sure the // payload contains correct number of elements. // Note: Since we're using normalized input.ShapeWithBatchDim() here, // make sure that all the normalization is before the check. { const auto& data_type = input.DType(); // Non-linear IO format input byte size validation will be handled in the // TensorRT backend. if (!input.IsNonLinearFormatIo()) { TRITONSERVER_MemoryType input_memory_type; // Because Triton expects STRING type to be in special format // (prepend 4 bytes to specify string length), so need to add all the // first 4 bytes for each element to find expected byte size if (data_type == inference::DataType::TYPE_STRING) { RETURN_IF_ERROR( ValidateBytesInputs(input_id, input, &input_memory_type)); // FIXME: Temporarily skips byte size checks for GPU tensors. See // DLIS-6820. } else { // Shape tensor with dynamic batching does not introduce a new // dimension to the tensor but adds an additional value to the 1-D // array. const std::vector& input_dims = input.IsShapeTensor() ? input.OriginalShape() : input.ShapeWithBatchDim(); int64_t expected_byte_size = INT_MAX; expected_byte_size = triton::common::GetByteSize(data_type, input_dims); const size_t& byte_size = input.Data()->TotalByteSize(); if ((byte_size > INT_MAX) || (static_cast(byte_size) != expected_byte_size)) { return Status( Status::Code::INVALID_ARG, LogRequest() + "input byte size mismatch for input '" + input_id + "' for model '" + ModelName() + "'. Expected " + std::to_string(expected_byte_size) + ", got " + std::to_string(byte_size)); } } } } } return Status::Success; } Status InferenceRequest::ValidateRequestInputs() { const inference::ModelConfig& model_config = model_raw_->Config(); if ((original_inputs_.size() > (size_t)model_config.input_size()) || (original_inputs_.size() < model_raw_->RequiredInputCount())) { // If no input is marked as optional, then use exact match error message // for consistency / backward compatibility std::string missing_required_input_string = "["; std::string original_input_string = "["; for (size_t i = 0; i < (size_t)model_config.input_size(); ++i) { const inference::ModelInput& input = model_config.input(i); if ((!input.optional()) && (original_inputs_.find(input.name()) == original_inputs_.end())) { missing_required_input_string = missing_required_input_string + "'" + input.name() + "'" + ","; } } // Removes the extra "," missing_required_input_string.pop_back(); missing_required_input_string = missing_required_input_string + "]"; for (const auto& pair : original_inputs_) { original_input_string = original_input_string + "'" + pair.first + "'" + ","; } // Removes the extra "," original_input_string.pop_back(); original_input_string = original_input_string + "]"; if (original_inputs_.size() == 0) { original_input_string = "[]"; } if ((size_t)model_config.input_size() == model_raw_->RequiredInputCount()) { // This is response ONLY when there are no optional parameters in the // model return Status( Status::Code::INVALID_ARG, LogRequest() + "expected " + std::to_string(model_config.input_size()) + " inputs but got " + std::to_string(original_inputs_.size()) + " inputs for model '" + ModelName() + "'. Got input(s) " + original_input_string + ", but missing required input(s) " + missing_required_input_string + ". Please provide all required input(s)."); } else { return Status( Status::Code::INVALID_ARG, LogRequest() + "expected number of inputs between " + std::to_string(model_raw_->RequiredInputCount()) + " and " + std::to_string(model_config.input_size()) + " but got " + std::to_string(original_inputs_.size()) + " inputs for model '" + ModelName() + "'. Got input(s) " + original_input_string + ", but missing required input(s) " + missing_required_input_string + ". Please provide all required input(s)."); } } return Status::Success; } Status InferenceRequest::ValidateBytesInputs( const std::string& input_id, const Input& input, TRITONSERVER_MemoryType* buffer_memory_type) const { const auto& input_dims = input.ShapeWithBatchDim(); int64_t element_count = triton::common::GetElementCount(input_dims); int64_t element_checked = 0; size_t remaining_element_size = 0; size_t buffer_next_idx = 0; const size_t buffer_count = input.DataBufferCount(); const char* buffer = nullptr; size_t remaining_buffer_size = 0; int64_t buffer_memory_id; // Validate elements until all buffers have been fully processed. while (remaining_buffer_size || buffer_next_idx < buffer_count) { // Get the next buffer if not currently processing one. if (!remaining_buffer_size) { // Reset remaining buffer size and pointers for next buffer. RETURN_IF_ERROR(input.DataBuffer( buffer_next_idx++, (const void**)(&buffer), &remaining_buffer_size, buffer_memory_type, &buffer_memory_id)); if (*buffer_memory_type == TRITONSERVER_MEMORY_GPU) { return Status::Success; } } constexpr size_t kElementSizeIndicator = sizeof(uint32_t); // Get the next element if not currently processing one. if (!remaining_element_size) { // FIXME: Assume the string element's byte size indicator is not spread // across buffer boundaries for simplicity. if (remaining_buffer_size < kElementSizeIndicator) { return Status( Status::Code::INVALID_ARG, LogRequest() + "element byte size indicator exceeds the end of the buffer."); } // Start the next element and reset the remaining element size. remaining_element_size = *(reinterpret_cast(buffer)); element_checked++; // Advance pointer and remainder by the indicator size. buffer += kElementSizeIndicator; remaining_buffer_size -= kElementSizeIndicator; } // If the remaining buffer fits it: consume the rest of the element, proceed // to the next element. if (remaining_buffer_size >= remaining_element_size) { buffer += remaining_element_size; remaining_buffer_size -= remaining_element_size; remaining_element_size = 0; } // Otherwise the remaining element is larger: consume the rest of the // buffer, proceed to the next buffer. else { remaining_element_size -= remaining_buffer_size; remaining_buffer_size = 0; } } // Validate the number of processed buffers exactly match expectations. if (buffer_next_idx != buffer_count) { return Status( Status::Code::INVALID_ARG, LogRequest() + "expected " + std::to_string(buffer_count) + " buffers for inference input '" + input_id + "', got " + std::to_string(buffer_next_idx)); } // Validate the number of processed elements exactly match expectations. if (element_checked != element_count) { return Status( Status::Code::INVALID_ARG, LogRequest() + "expected " + std::to_string(element_count) + " string elements for inference input '" + input_id + "', got " + std::to_string(element_checked)); } return Status::Success; } #ifdef TRITON_ENABLE_STATS void InferenceRequest::ReportErrorStatistics( MetricModelReporter* metric_reporter, FailureReason reason) { INFER_STATS_DECL_TIMESTAMP(request_end_ns); model_raw_->MutableStatsAggregator()->UpdateFailure( metric_reporter, request_start_ns_, request_end_ns, reason); if (secondary_stats_aggregator_ != nullptr) { secondary_stats_aggregator_->UpdateFailure( nullptr /* metric_reporter */, request_start_ns_, request_end_ns, reason); } } void InferenceRequest::ReportStatistics( MetricModelReporter* metric_reporter, bool success, const uint64_t compute_start_ns, const uint64_t compute_input_end_ns, const uint64_t compute_output_start_ns, const uint64_t compute_end_ns) { if (!collect_stats_) { return; } #ifdef TRITON_ENABLE_TRACING if (trace_ != nullptr) { trace_->Report(TRITONSERVER_TRACE_COMPUTE_START, compute_start_ns); trace_->Report(TRITONSERVER_TRACE_COMPUTE_INPUT_END, compute_input_end_ns); trace_->Report( TRITONSERVER_TRACE_COMPUTE_OUTPUT_START, compute_output_start_ns); trace_->Report(TRITONSERVER_TRACE_COMPUTE_END, compute_end_ns); } #endif // TRITON_ENABLE_TRACING INFER_STATS_DECL_TIMESTAMP(request_end_ns); if (success) { model_raw_->MutableStatsAggregator()->UpdateSuccess( metric_reporter, std::max(1U, batch_size_), request_start_ns_, queue_start_ns_, compute_start_ns, compute_input_end_ns, compute_output_start_ns, compute_end_ns, request_end_ns); if (secondary_stats_aggregator_ != nullptr) { secondary_stats_aggregator_->UpdateSuccess( nullptr /* metric_reporter */, std::max(1U, batch_size_), request_start_ns_, queue_start_ns_, compute_start_ns, compute_input_end_ns, compute_output_start_ns, compute_end_ns, request_end_ns); } } else { model_raw_->MutableStatsAggregator()->UpdateFailure( metric_reporter, request_start_ns_, request_end_ns, FailureReason::BACKEND); if (secondary_stats_aggregator_ != nullptr) { secondary_stats_aggregator_->UpdateFailure( nullptr /* metric_reporter */, request_start_ns_, request_end_ns, FailureReason::BACKEND); } } } void InferenceRequest::ReportStatisticsWithDuration( MetricModelReporter* metric_reporter, bool success, const uint64_t compute_start_ns, const uint64_t compute_input_duration_ns, const uint64_t compute_infer_duration_ns, const uint64_t compute_output_duration_ns) { if (!collect_stats_) { return; } INFER_STATS_DECL_TIMESTAMP(request_end_ns); if (success) { model_raw_->MutableStatsAggregator()->UpdateSuccessWithDuration( metric_reporter, std::max(1U, batch_size_), request_start_ns_, queue_start_ns_, compute_start_ns, request_end_ns, compute_input_duration_ns, compute_infer_duration_ns, compute_output_duration_ns); if (secondary_stats_aggregator_ != nullptr) { secondary_stats_aggregator_->UpdateSuccessWithDuration( nullptr /* metric_reporter */, std::max(1U, batch_size_), request_start_ns_, queue_start_ns_, compute_start_ns, request_end_ns, compute_input_duration_ns, compute_infer_duration_ns, compute_output_duration_ns); } } else { model_raw_->MutableStatsAggregator()->UpdateFailure( metric_reporter, request_start_ns_, request_end_ns, FailureReason::OTHER); if (secondary_stats_aggregator_ != nullptr) { secondary_stats_aggregator_->UpdateFailure( nullptr /* metric_reporter */, request_start_ns_, request_end_ns, FailureReason::OTHER); } } } void InferenceRequest::ReportStatisticsCacheHit(MetricModelReporter* metric_reporter) { // Capture end of request time INFER_STATS_DECL_TIMESTAMP(request_end_ns); if (cache_lookup_start_ns_ >= cache_lookup_end_ns_) { LOG_WARNING << LogRequest() << "Cache lookup timestamps were not set correctly. Cache " "lookup duration stats may be incorrect."; } const uint64_t cache_lookup_duration_ns = cache_lookup_end_ns_ - cache_lookup_start_ns_; // Cache hit is always success model_raw_->MutableStatsAggregator()->UpdateSuccessCacheHit( metric_reporter, std::max(1U, batch_size_), request_start_ns_, queue_start_ns_, cache_lookup_start_ns_, request_end_ns, cache_lookup_duration_ns); if (secondary_stats_aggregator_ != nullptr) { secondary_stats_aggregator_->UpdateSuccessCacheHit( nullptr /* metric_reporter */, std::max(1U, batch_size_), request_start_ns_, queue_start_ns_, cache_lookup_start_ns_, request_end_ns, cache_lookup_duration_ns); } } #endif // TRITON_ENABLE_STATS // // Input // InferenceRequest::Input::Input() : tensor_type_(TensorType::TENSOR), data_(new MemoryReference), has_host_policy_specific_data_(false) { } InferenceRequest::Input::Input( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count) : name_(name), datatype_(datatype), original_shape_(shape, shape + dim_count), tensor_type_(TensorType::TENSOR), data_(new MemoryReference), has_host_policy_specific_data_(false) { } InferenceRequest::Input::Input( const std::string& name, const inference::DataType datatype, const std::vector& shape) : name_(name), datatype_(datatype), original_shape_(shape), tensor_type_(TensorType::TENSOR), data_(new MemoryReference), has_host_policy_specific_data_(false) { } void InferenceRequest::Input::SetMetadata( const std::string& name, const inference::DataType& dt, const std::vector& shape) { name_ = name; datatype_ = dt; original_shape_ = shape; } Status InferenceRequest::Input::SetIsShapeTensor() { tensor_type_ = TensorType::SHAPE_TENSOR; return Status::Success; } Status InferenceRequest::Input::SetIsNonLinearFormatIo() { tensor_type_ = TensorType::NON_LINEAR; return Status::Success; } const std::shared_ptr& InferenceRequest::Input::Data(const std::string& host_policy_name) const { auto device_data = host_policy_data_map_.find(host_policy_name); if (device_data == host_policy_data_map_.end()) { // Fall back on default data if there is no data that has been added for // this host policy return data_; } return device_data->second; } Status InferenceRequest::Input::AppendData( const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { if (byte_size > 0) { std::static_pointer_cast(data_)->AddBuffer( static_cast(base), byte_size, memory_type, memory_type_id); } return Status::Success; } Status InferenceRequest::Input::AppendDataWithBufferAttributes( const void* base, BufferAttributes* buffer_attributes) { if (buffer_attributes->ByteSize() > 0) { std::static_pointer_cast(data_)->AddBuffer( static_cast(base), buffer_attributes); } return Status::Success; } Status InferenceRequest::Input::AppendDataWithHostPolicy( const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, const char* host_policy_name) { auto device_data = host_policy_data_map_.find(host_policy_name); has_host_policy_specific_data_ = true; if (device_data == host_policy_data_map_.end()) { auto insert_pair = host_policy_data_map_.insert( std::make_pair(std::string(host_policy_name), new MemoryReference)); device_data = insert_pair.first; } if (byte_size > 0) { std::static_pointer_cast(device_data->second) ->AddBuffer( static_cast(base), byte_size, memory_type, memory_type_id); } return Status::Success; } Status InferenceRequest::Input::PrependData( const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { if (byte_size > 0) { std::static_pointer_cast(data_)->AddBufferFront( static_cast(base), byte_size, memory_type, memory_type_id); } return Status::Success; } Status InferenceRequest::Input::SetData(const std::shared_ptr& data) { if (data_->TotalByteSize() != 0) { return Status( Status::Code::INVALID_ARG, "input '" + name_ + "' already has data, can't overwrite"); } data_ = data; return Status::Success; } Status InferenceRequest::Input::SetData( const std::string& host_policy_name, const std::shared_ptr& data) { if (host_policy_data_map_.find(host_policy_name) != host_policy_data_map_.end()) { return Status( Status::Code::INVALID_ARG, "input '" + name_ + "' already has data for host policy '" + host_policy_name + "', can't overwrite"); } host_policy_data_map_.emplace(host_policy_name, data); return Status::Success; } Status InferenceRequest::Input::RemoveAllData() { data_ = std::make_shared(); host_policy_data_map_.clear(); has_host_policy_specific_data_ = false; return Status::Success; } Status InferenceRequest::Input::DataBuffer( const size_t idx, const void** base, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) const { *base = data_->BufferAt(idx, byte_size, memory_type, memory_type_id); return Status::Success; } Status InferenceRequest::Input::DataBufferAttributes( const size_t idx, const void** base, BufferAttributes** buffer_attributes) const { *base = data_->BufferAt(idx, buffer_attributes); return Status::Success; } Status InferenceRequest::Input::DataBufferForHostPolicy( const size_t idx, const void** base, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id, const std::string& host_policy_name) const { auto device_data = host_policy_data_map_.find(host_policy_name); if (device_data == host_policy_data_map_.end()) { // Return data buffer if there is no host-policy specific buffer available *base = data_->BufferAt(idx, byte_size, memory_type, memory_type_id); } else { *base = device_data->second->BufferAt( idx, byte_size, memory_type, memory_type_id); } return Status::Success; } size_t InferenceRequest::Input::DataBufferCountForHostPolicy( const std::string& host_policy_name) const { auto policy_data = host_policy_data_map_.find(host_policy_name); if (policy_data != host_policy_data_map_.end()) { return policy_data->second->BufferCount(); } return data_->BufferCount(); } InferenceRequest::SequenceId::SequenceId() : sequence_label_(""), sequence_index_(0), id_type_(InferenceRequest::SequenceId::DataType::UINT64) { } InferenceRequest::SequenceId::SequenceId(const std::string& sequence_label) : sequence_label_(sequence_label), sequence_index_(0), id_type_(InferenceRequest::SequenceId::DataType::STRING) { } InferenceRequest::SequenceId::SequenceId(uint64_t sequence_index) : sequence_label_(""), sequence_index_(sequence_index), id_type_(InferenceRequest::SequenceId::DataType::UINT64) { } InferenceRequest::SequenceId& InferenceRequest::SequenceId::operator=(const std::string& rhs) { sequence_label_ = rhs; sequence_index_ = 0; id_type_ = InferenceRequest::SequenceId::DataType::STRING; return *this; } InferenceRequest::SequenceId& InferenceRequest::SequenceId::operator=(const uint64_t rhs) { sequence_label_ = ""; sequence_index_ = rhs; id_type_ = InferenceRequest::SequenceId::DataType::UINT64; return *this; } std::ostream& operator<<(std::ostream& out, const InferenceRequest& request) { out << "[0x" << std::addressof(request) << "] " << "request id: " << request.Id() << ", model: " << request.ModelName() << ", requested version: " << request.RequestedModelVersion() << ", actual version: " << request.ActualModelVersion() << ", flags: 0x" << std::hex << request.Flags() << std::dec << ", correlation id: " << request.CorrelationId() << ", batch size: " << request.BatchSize() << ", priority: " << request.Priority() << ", timeout (us): " << request.TimeoutMicroseconds() << std::endl; out << "original inputs:" << std::endl; for (const auto& itr : request.OriginalInputs()) { out << "[0x" << std::addressof(itr.second) << "] " << itr.second << std::endl; } out << "override inputs:" << std::endl; for (const auto& itr : request.OverrideInputs()) { out << "[0x" << itr.second.get() << "] " << *itr.second << std::endl; } out << "inputs:" << std::endl; for (const auto& itr : request.ImmutableInputs()) { out << "[0x" << itr.second << "] " << *itr.second << std::endl; } out << "original requested outputs:" << std::endl; for (const auto& name : request.OriginalRequestedOutputs()) { out << name << std::endl; } out << "requested outputs:" << std::endl; for (const auto& name : request.ImmutableRequestedOutputs()) { out << name << std::endl; } return out; } std::ostream& operator<<(std::ostream& out, const InferenceRequest::Input& input) { out << "input: " << input.Name() << ", type: " << triton::common::DataTypeToProtocolString(input.DType()) << ", original shape: " << triton::common::DimsListToString(input.OriginalShape()) << ", batch + shape: " << triton::common::DimsListToString(input.ShapeWithBatchDim()) << ", shape: " << triton::common::DimsListToString(input.Shape()); if (input.IsShapeTensor()) { out << ", is_shape_tensor: True"; } return out; } std::ostream& operator<<(std::ostream& out, const InferenceRequest::SequenceId& sequence_id) { switch (sequence_id.Type()) { case InferenceRequest::SequenceId::DataType::STRING: out << sequence_id.StringValue(); break; case InferenceRequest::SequenceId::DataType::UINT64: out << sequence_id.UnsignedIntValue(); break; default: out << sequence_id.UnsignedIntValue(); break; } return out; } std::ostream& operator<<(std::ostream& out, const InferenceRequest::State& state) { switch (state) { case InferenceRequest::State::INITIALIZED: { out << "INITIALIZED"; break; } case InferenceRequest::State::PENDING: { out << "PENDING"; break; } case InferenceRequest::State::EXECUTING: { out << "EXECUTING"; break; } case InferenceRequest::State::RELEASED: { out << "RELEASED"; break; } case InferenceRequest::State::FAILED_ENQUEUE: { out << "FAILED_ENQUEUE"; break; } default: out << "UNKNOWN"; } return out; } bool operator==( const InferenceRequest::SequenceId lhs, const InferenceRequest::SequenceId rhs) { if (lhs.Type() == rhs.Type()) { switch (lhs.Type()) { case InferenceRequest::SequenceId::DataType::STRING: return lhs.StringValue() == rhs.StringValue(); case InferenceRequest::SequenceId::DataType::UINT64: return lhs.UnsignedIntValue() == rhs.UnsignedIntValue(); default: return lhs.UnsignedIntValue() == rhs.UnsignedIntValue(); } } else { return false; } } bool operator!=( const InferenceRequest::SequenceId lhs, const InferenceRequest::SequenceId rhs) { return !(lhs == rhs); } }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_request.h000066400000000000000000001047771527052371200204200ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "buffer_attributes.h" #include "infer_response.h" #include "infer_stats.h" #include "infer_trace.h" #include "memory.h" #include "response_allocator.h" #include "sequence_state.h" #include "status.h" #include "triton/common/logging.h" #include "triton/common/model_config.h" #include "tritonserver_apis.h" namespace triton { namespace core { class Model; class InferenceServer; class MetricModelReporter; // // An inference request. A request can be used multiple times for // inference but before each inference run, PrepareForInference() must // be called to verify and prepare the request. Verification involves // ensuring that any changes made since the last inference are // valid. Preparing involves removing/resetting any state left over // from the previous inference. // class InferenceRequest { public: using InternalReleaseFn = std::function&, const uint32_t)>; /// State for the request object. enum class State { // The request has been initialized, but not yet enqueued. INITIALIZED, // The request is pending execution. PENDING, // The request failed to enqueue. FAILED_ENQUEUE, // The request has been picked up by a backend model instance for execution, // but hasn't been released yet. EXECUTING, // The request has been released. RELEASED }; // Input tensor class Input { public: enum class TensorType { TENSOR, SHAPE_TENSOR, NON_LINEAR }; Input(); Input( const std::string& name, const inference::DataType datatype, const std::vector& shape); Input( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count); // Set the name, data type and original shape of the input tensor. void SetMetadata( const std::string& name, const inference::DataType& dt, const std::vector& shape); // The name of the input tensor. There is no mutable operator for // the name because it is used in a InferenceRequest map and a // mutable method would allow it to get out-of-sync. const std::string& Name() const { return name_; } // Data type of the input tensor. inference::DataType DType() const { return datatype_; } // The original shape of the input tensor. const std::vector& OriginalShape() const { return original_shape_; } // The shape of the input tensor after normalization. This shape // is the original shape modified as required/expected by // inference processing. const std::vector& Shape() const { return shape_; } std::vector* MutableShape() { return &shape_; } // FIXME. Should not need these functions. All shapes kept here // should include the batch dimension instead of breaking the same // into batch + shape. const std::vector& ShapeWithBatchDim() const { return shape_with_batch_dim_; } std::vector* MutableShapeWithBatchDim() { return &shape_with_batch_dim_; } // Return true if host-specific data was added for this input bool HasHostPolicySpecificData() const { return has_host_policy_specific_data_; } // Whether or not the input is a tensorrt shape tensor bool IsShapeTensor() const { return tensor_type_ == TensorType::SHAPE_TENSOR; } // Specifies whether the input uses a non-linear IO format bool IsNonLinearFormatIo() const { return tensor_type_ == TensorType::NON_LINEAR; } // Set the input to be treated as a shape tensor. Status SetIsShapeTensor(); // Set the input uses a non-linear IO format Status SetIsNonLinearFormatIo(); // The data for this input. const std::shared_ptr& Data() const { return data_; } // The data for this input for a specific device const std::shared_ptr& Data( const std::string& host_policy_name) const; // Return all host policy data set for this input const std::map>& HostPolicyData() const { return host_policy_data_map_; } // Set the data for this input. Error if input already has some // data. Status SetData(const std::shared_ptr& data); // Set the data associated with the host policy for this input. // Return error if input already has some data. Status SetData( const std::string& host_policy_name, const std::shared_ptr& data); // Append a new buffer of data to this input. Status AppendData( const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); Status AppendDataWithHostPolicy( const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, const char* host_policy_name); Status AppendDataWithBufferAttributes( const void* base, BufferAttributes* buffer_attributes); // Prepend a new buffer of data to this input. Status PrependData( const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); // Remove all existing data for the input. Status RemoveAllData(); // Get the number of buffers containing the input tensor data. size_t DataBufferCount() const { return data_->BufferCount(); } // Get the number of buffers containing the input tensor data with // host policy. If there are no buffers corresponding to the specific // host policy, the number of buffers in the fallback input data is // returned. size_t DataBufferCountForHostPolicy( const std::string& host_policy_name) const; // Get the 'idx' buffer containing a contiguous chunk of bytes for // the input. Return error is 'idx' refers to a buffer that does // not exist. Return a pointer to the chunk in 'base' and the // size of the chunk in 'byte_size'. 'memory_type' acts as // both input and output. On input 'memory_type' is the buffer // memory type preferred by the function caller. On return // 'memory_type' gives the actual memory type of the chunk pointed // to by 'base'. 'memory_type_id' acts as both input and // output. On input 'memory_type_id' is the buffer memory type id // preferred by the function caller. On return 'memory_type_id' // gives the actual memory type id of the chunk pointed to by // 'base'. Status DataBuffer( const size_t idx, const void** base, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) const; // Get the buffer attributes associated with 'idx' buffer. Status DataBufferAttributes( const size_t idx, const void** base, BufferAttributes** buffer_attributes) const; // Get the 'idx' buffer containing a contiguous chunk of bytes for // the input. Return error is 'idx' refers to a buffer that does // not exist. Return a pointer to the chunk in 'base' and the // size of the chunk in 'byte_size'. 'memory_type' acts as // both input and output. On input 'memory_type' is the buffer // memory type preferred by the function caller. On return // 'memory_type' gives the actual memory type of the chunk pointed // to by 'base'. 'memory_type_id' acts as both input and // output. On input 'memory_type_id' is the buffer memory type id // preferred by the function caller. On return 'memory_type_id' // gives the actual memory type id of the chunk pointed to by // 'base'. Status DataBufferForHostPolicy( const size_t idx, const void** base, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id, const std::string& host_policy_name) const; private: DISALLOW_COPY_AND_ASSIGN(Input); friend std::ostream& operator<<( std::ostream& out, const InferenceRequest::Input& input); std::string name_; inference::DataType datatype_; std::vector original_shape_; std::vector shape_; std::vector shape_with_batch_dim_; TensorType tensor_type_; std::shared_ptr data_; bool has_host_policy_specific_data_; // A map of host policy to input data memory std::map> host_policy_data_map_; }; // Sequence ID can be either a 64 bit integer or a string. // This class implements the SequenceId type class SequenceId { public: enum class DataType { UINT64, STRING }; SequenceId(); SequenceId(const std::string& sequence_label); SequenceId(uint64_t sequence_index); SequenceId& operator=(const SequenceId& rhs) = default; SequenceId& operator=(const std::string& rhs); SequenceId& operator=(const uint64_t rhs); // Functions that help determine exact type of sequence Id DataType Type() const { return id_type_; } bool InSequence() const { return ((sequence_label_ != "") || (sequence_index_ != 0)); } // Get the value of the SequenceId based on the type const std::string& StringValue() const { return sequence_label_; } uint64_t UnsignedIntValue() const { return sequence_index_; } private: friend std::ostream& operator<<( std::ostream& out, const InferenceRequest::SequenceId& correlation_id); friend bool operator==(const SequenceId lhs, const SequenceId rhs); friend bool operator!=(const SequenceId lhs, const SequenceId rhs); std::string sequence_label_; uint64_t sequence_index_; DataType id_type_; }; // InferenceRequest // // The two constructors are identical except one takes model as a // shared pointer and the other as a raw pointer. The shared pointer // version is the primary one and acts to keep the model alive as // long as the request is in flight. The raw pointer version is used // only for cases where the model itself is issuing a request // (e.g. warmup) and no shared pointer version of the model exists // (because we aren't using shared_from_this). InferenceRequest( const std::shared_ptr& model, const int64_t requested_model_version); InferenceRequest(Model* model, const int64_t requested_model_version); const std::string& ModelName() const; int64_t RequestedModelVersion() const { return requested_model_version_; } int64_t ActualModelVersion() const; const std::string& Id() const { return id_; } void SetId(const std::string& i) { id_ = i; } // Return string for logging request ID std::string LogRequest() const { std::string id = Id(); if (id.empty()) { id = ""; } return std::string("[request id: ") + id + "] "; } // Flags for the request, union of TRITONSERVER_RequestFlag. uint32_t Flags() const { return flags_; } void SetFlags(uint32_t f) { flags_ = f; } const SequenceId& CorrelationId() const { return correlation_id_; } void SetCorrelationId(const SequenceId& c) { correlation_id_ = c; } // The batch size of the request, as understood by Triton. A // batch-size of 0 indicates that the model doesn't support batching // in a way that Triton understands. Batch size is not set // explicitly so there is no setter for it. It is set when the // request is normalized. uint32_t BatchSize() const { return batch_size_; } uint64_t Priority() const { return priority_; } void SetPriority(uint64_t p); uint64_t TimeoutMicroseconds() const { return timeout_us_; } void SetTimeoutMicroseconds(uint64_t t) { timeout_us_ = t; } const std::string& CacheKey() const { return cache_key_; } // It is up to the user to update the cache_key_ if modifying any hashable // fields of the request after cache_key_is_set_ has been set to true. void SetCacheKey(const std::string& key) { cache_key_ = key; cache_key_is_set_ = true; } bool CacheKeyIsSet() const { return cache_key_is_set_; } // Define and validate state transitions for request. Status SetState(InferenceRequest::State state); #ifdef TRITON_ENABLE_TRACING const std::shared_ptr& TraceProxy() const { return trace_; } std::shared_ptr* MutableTrace() { return &trace_; } void SetTrace(const std::shared_ptr& trace) { trace_ = trace; response_factory_->SetTrace(trace); } void ReleaseTrace() { trace_ = nullptr; response_factory_->ReleaseTrace(); } Status TraceInputTensors( TRITONSERVER_InferenceTraceActivity activity, const std::string& msg); #endif // TRITON_ENABLE_TRACING // Add a parameter to the request. Status AddParameter(const char* name, const char* value); Status AddParameter(const char* name, const int64_t value); Status AddParameter(const char* name, const bool value); Status AddParameter(const char* name, const double value); Status SetParameters(const std::deque& parameters); const std::deque& Parameters() const { return parameters_; } // The original inputs are the inputs added to the request before // the inference execution (that is before // TRITONSERVER_ServerInferAsync is called). Once execution has // started the original inputs should not be modified until // execution completes (and those modifications will apply to the // next inference execution). Status MutableOriginalInput(const std::string& name, Input** input); std::unordered_map* MutableOriginalInputs() { return &original_inputs_; } const std::unordered_map& OriginalInputs() const { return original_inputs_; } // The override inputs are the inputs added to the request after // inference execution has started (that is after // TRITONSERVER_ServerInferAsync or equivalent is called). During // inference processing, if Triton needs to change an original input // it will add an override instead of changing the original. Triton // will also use an override if it needs to add a new input to the // request. Overrides are recorded as shared_ptr so that the same // override can be used efficiently multiple times or even in // multiple requests simultaneously. Must be careful not to modify // an override input if it is being shared unless you want that // change to be reflected in all requests that hold that override // input. Override inputs within a specific request are not // persisted across inference calls. std::unordered_map>* MutableOverrideInputs() { return &override_inputs_; } const std::unordered_map>& OverrideInputs() const { return override_inputs_; } // Get an input taking into account both original inputs and // overrides. If an override input is available use it, otherwise // use the original input. Accessing inputs via this method is not // valid until after PrepareForInference is called. Status ImmutableInput(const std::string& name, const Input** input) const; const std::unordered_map& ImmutableInputs() const { return inputs_; } // The original requested outputs are the requested outputs added to // the request before the inference execution (that is before // TRITONSERVER_ServerInferAsync is called). Once execution has // started the original requested outputs should not be modified // until execution completes (and those modifications will apply to // the next inference execution). const std::set& OriginalRequestedOutputs() const { return original_requested_outputs_; } // Get the requested outputs that should be used during // inference. Accessing outputs via this method is not valid until // after PrepareForInference is called. const std::set& ImmutableRequestedOutputs() const { return (requested_outputs_.empty()) ? original_requested_outputs_ : requested_outputs_; } // Get the response factory. const std::shared_ptr& ResponseFactory() const { return response_factory_; } // Add an original input to the request. If 'input' is non-null // return a pointer to the newly added input. Status AddOriginalInput( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count, Input** input = nullptr); Status AddOriginalInput( const std::string& name, const inference::DataType datatype, const std::vector& shape, Input** input = nullptr); // Add an original raw input to the request. If 'input' is non-null // return a pointer to the newly added input. Status AddRawInput(const std::string& name, Input** input = nullptr); // Remove a single original input or all inputs. Status RemoveOriginalInput(const std::string& name); Status RemoveAllOriginalInputs(); // Add an override input to the request. If 'input' is non-null // return a pointer to the newly added input. // FIXME passing batch size is special handling for backend API. // For override input, the 'shape' is without batch dimension for // backends that implemented w/o backend API (which need correct // input.Shape()), but backend API uses input.ShapeWithBatchDim(). Status AddOverrideInput( const std::string& name, const inference::DataType datatype, const int64_t batch_size, const std::vector& shape, std::shared_ptr* input = nullptr); // Add an override input to the request. Status AddOverrideInput(const std::shared_ptr& input); // Request an original requested output. Status AddOriginalRequestedOutput(const std::string& name); // Remove a single original requested output or all requested // outputs. Status RemoveOriginalRequestedOutput(const std::string& name); Status RemoveAllOriginalRequestedOutputs(); // Initialize the release callback for the request. Status SetReleaseCallback( TRITONSERVER_InferenceRequestReleaseFn_t release_fn, void* release_userp) { release_fn_ = release_fn; release_userp_ = release_userp; return Status::Success; } // Initialize the response factory arguments that are going to be used with // any responses produced for this request. Status SetResponseCallback( const ResponseAllocator* allocator, void* alloc_userp, TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp) { response_allocator_ = allocator; alloc_userp_ = alloc_userp; response_callback_ = response_fn; response_userp_ = response_userp; return Status::Success; } // Returns the preferred memory type and memory type ID of the output buffer // for the request. 'name' and 'byte_size' are optional and set to nullptr // if not specified, if provided, they give the allocator more information. // 'memory_type' and 'memory_type_id' are also used as input to provide types // preferred by the caller. // Status::Code::UNAVAILABLE will be returned if output properties are not // available. Status OutputBufferProperties( const char* name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); // Add a callback to be invoked on releasing the request object from Triton. // Multiple callbacks can be added by calling this function in order, // and they will be invoked in reversed order. Status AddInternalReleaseCallback(InternalReleaseFn&& callback) { release_callbacks_.emplace_back(std::move(callback)); return Status::Success; } // Add a delegator to be invoked on sending the responses of this request. // The response will be passed to 'delegator' and 'delegator' must call the // InferenceResponse::Send() to send the response. Status SetResponseDelegator( std::function&&, const uint32_t)>&& delegator) { response_delegator_ = std::move(delegator); return response_factory_->SetResponseDelegator(response_delegator_); } Status SetSequenceStates( const std::shared_ptr& sequence_states) { sequence_states_ = sequence_states; return Status::Success; } Status LoadInputStates(); void SetResponseFactory() { response_factory_.reset(new InferenceResponseFactory( model_shared_, id_, response_allocator_, alloc_userp_, response_callback_, response_userp_, response_delegator_)); } const std::shared_ptr& GetSequenceStates() const { return sequence_states_; } // Prepare this request for inference. Status PrepareForInference(); // Run this inference request using the model associated with the // request. If Status::Success is returned then the call has taken // ownership of the request object and so 'request' will be // nullptr. If non-success is returned then the caller still retains // ownership of 'request'. static Status Run(std::unique_ptr& request); // Send an error response for this request. If 'status' is Success // then no response is sent and the request is not released (even if // 'release_request' is true). Because this is sending an error it // is assumed that this is the last response for the request and so // the FINAL flag is set in the response callback. If // 'release_request' is true then the release callback is called for // this request and ownership is given to the callback. Thus, if // 'release_request' is true 'request' is returned as nullptr. static void RespondIfError( std::unique_ptr& request, const Status& status, const bool release_request = false, FailureReason reason = FailureReason::OTHER); // Send an error response to a set of 'requests'. If 'status' is // Success then no responses are sent and the requests are not // released (even if 'release_request' is true). Because this is // sending an error it is assumed that this is the last response for // the requests and so the FINAL flag is set in the response // callbacks. If 'release_request' is true then the release callback // is called for each request, and the request ownership is given to // the callback. Thus, if 'release_request' is true 'requests' is // returned with all nullptrs. static void RespondIfError( std::vector>& requests, const Status& status, const bool release_requests = false, FailureReason reason = FailureReason::OTHER); // Release the request. Call the release callback and transfer // ownership of the request to the callback. On return 'request' is // nullptr. static Status Release( std::unique_ptr&& request, const uint32_t release_flags); // Create a copy of 'from' suitable for use as a "null" request as // required for the direct sequence batcher. The returned copy will // contain only the minimum content required for a null request. // The statistics of the copy will not be collected. static InferenceRequest* CopyAsNull(const InferenceRequest& from); uint64_t QueueStartNs() const { return queue_start_ns_; } uint64_t CaptureQueueStartNs() { queue_start_ns_ = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); return queue_start_ns_; } uint64_t CacheLookupStartNs() const { return cache_lookup_start_ns_; } uint64_t CaptureCacheLookupStartNs() { cache_lookup_start_ns_ = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); return cache_lookup_start_ns_; } uint64_t CacheLookupEndNs() const { return cache_lookup_end_ns_; } uint64_t CaptureCacheLookupEndNs() { cache_lookup_end_ns_ = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); return cache_lookup_end_ns_; } uint64_t BatcherStartNs() const { return batcher_start_ns_; } uint64_t CaptureBatcherStartNs() { batcher_start_ns_ = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); return batcher_start_ns_; } #ifdef TRITON_ENABLE_STATS uint64_t RequestStartNs() const { return request_start_ns_; } uint64_t CaptureRequestStartNs() { request_start_ns_ = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); return request_start_ns_; } // Report the statistics to stats collectors associated with the request. // Duration and timestamps provide two granularities for stats collectors. void ReportStatistics( MetricModelReporter* metric_reporter, bool success, const uint64_t compute_start_ns, const uint64_t compute_input_end_ns, const uint64_t compute_output_start_ns, const uint64_t compute_end_ns); // Report the error statistics to stats collectors associated with the // request. // FIXME: A separate function may not be necessary here, but is being used // cautiously in case of unforeseen issues such as possibly capturing a trace // twice. This should be revisited and better tested to see if the // ReportStatistics function can be used as-is for the newly captured failure // cases. void ReportErrorStatistics( MetricModelReporter* metric_reporter, FailureReason reason); // Report the statistics to stats collectors associated with the request. // Duration and timestamps provide two granularities for stats collectors. void ReportStatisticsWithDuration( MetricModelReporter* metric_reporter, bool success, const uint64_t compute_start_ns, const uint64_t compute_input_duration_ns, const uint64_t compute_infer_duration_ns, const uint64_t compute_output_duration_ns); // Report the statistics to stats collectors associated with the request on // response cache hits. Cache miss stats will be updated through model object // directly because the backend may release the request object. void ReportStatisticsCacheHit(MetricModelReporter* metric_reporter); // Statistics for each request are aggregated into the corresponding // model's statistics. Optionally this function may be used to // add an additional aggregator where statistics are also aggregated. void SetSecondaryStatsAggregator( InferenceStatsAggregator* secondary_stats_aggregator) { secondary_stats_aggregator_ = secondary_stats_aggregator; } #endif // TRITON_ENABLE_STATS // Mark the request as cancelled. Status Cancel() { if (!response_factory_) { return Status( Status::Code::INTERNAL, "It is not possible to cancel an inference request before calling " "TRITONSERVER_InferAsync."); } response_factory_->Cancel(); return Status::Success; } // Check if the request is marked as cancelled. This does not indicate if the // request is actually cancelled. The request is cancelled if and only if it // is responded with a cancelled status. Status IsCancelled(bool* is_cancelled) { if (!response_factory_) { return Status( Status::Code::INTERNAL, "It is not possible to query cancellation status before calling " "TRITONSERVER_InferAsync."); } *is_cancelled = response_factory_->IsCancelled(); return Status::Success; } bool IsCancelled() { bool is_cancelled = false; Status status = IsCancelled(&is_cancelled); if (!status.IsOk()) { LOG_ERROR << status.Message(); } return is_cancelled; } private: DISALLOW_COPY_AND_ASSIGN(InferenceRequest); friend std::ostream& operator<<( std::ostream& out, const InferenceRequest& request); friend std::ostream& operator<<( std::ostream& out, const InferenceRequest::State& state); Status Normalize(); // Helper for validating Inputs Status ValidateRequestInputs(); Status ValidateBytesInputs( const std::string& input_id, const Input& input, TRITONSERVER_MemoryType* buffer_memory_type) const; // Helpers for pending request metrics void IncrementPendingRequestCount(); void DecrementPendingRequestCount(); // Has anything in the request potentially changed in a way that // causes normalization to be required when preparing the request // for inference. bool needs_normalization_; // The model associated with this request. For most requests // model_shared_ will be non-null and will act to keep the model // alive as long as this request is live. In this case model_raw_ // will be the raw pointer from the shared pointer. For cases where // the model itself created the request (like running requests for // warmup), model_shared_ will be nullptr, but model_raw_ will // still be defined. Thus model_raw_ is always defined and should // always to used to access the model. std::shared_ptr model_shared_; Model* model_raw_; // The model version as requested and based on version policy the // specific version that is actually used for inference. int64_t requested_model_version_; int64_t actual_model_version_; std::string id_; uint32_t flags_; SequenceId correlation_id_; uint32_t batch_size_; uint64_t priority_; uint64_t timeout_us_; std::string cache_key_ = ""; // Helper to determine if request was successfully hashed // and cache_key_ field is valid bool cache_key_is_set_ = false; std::unordered_map original_inputs_; std::unordered_map> override_inputs_; std::unordered_map inputs_; std::set original_requested_outputs_; std::string raw_input_name_; uint32_t raw_input_size_; // requested_outputs_ is to be used post-normalization. It will be // empty unless it differs from original_requested_outputs_, so // typically should access it through ImmutableRequestedOutputs. std::set requested_outputs_; // The release function and user pointer for this request. TRITONSERVER_InferenceRequestReleaseFn_t release_fn_; void* release_userp_; // Additional release callbacks invoked before 'release_fn_'. std::vector release_callbacks_; // Delegator to be invoked on sending responses. std::function&&, const uint32_t)> response_delegator_; // The response factory associated with this request. std::shared_ptr response_factory_; // Request timestamps. Queue start is needed for schedulers even // when statistics are not being collected. uint64_t queue_start_ns_; // Cache lookup start/end timestamps. Cache manages its own stats even // when statistics are not being colleceted. uint64_t cache_lookup_start_ns_; uint64_t cache_lookup_end_ns_; // Cache insertion start/end timestamps. Cache manages its own stats even // when statistics are not being colleceted. uint64_t cache_insertion_start_ns_; uint64_t cache_insertion_end_ns_; // Dedicated timestamp for batcher internal which can diverge from // queue start timestamp to provide accurate queue time without affecting // batcher functionalities. uint64_t batcher_start_ns_; // Whether the stats of the request should be collected. bool collect_stats_; // The parameters of the request. Use a deque so that there is no // reallocation. std::deque parameters_; #ifdef TRITON_ENABLE_STATS uint64_t request_start_ns_; InferenceStatsAggregator* secondary_stats_aggregator_ = nullptr; #endif // TRITON_ENABLE_STATS #ifdef TRITON_ENABLE_TRACING // Inference trace associated with this request. std::shared_ptr trace_; #endif // TRITON_ENABLE_TRACING // Sequence I/O states used for implicit state. std::shared_ptr sequence_states_; // The state of the request. std::atomic state_; // Whether this is a null request used for direct sequence batch padding or // not. bool null_request_; // Response factory arguments const ResponseAllocator* response_allocator_; void* response_userp_; void* alloc_userp_; TRITONSERVER_InferenceResponseCompleteFn_t response_callback_; }; std::ostream& operator<<(std::ostream& out, const InferenceRequest& request); std::ostream& operator<<( std::ostream& out, const InferenceRequest::Input& input); std::ostream& operator<<( std::ostream& out, const InferenceRequest::SequenceId& sequence_id); bool operator==( const InferenceRequest::SequenceId lhs, const InferenceRequest::SequenceId rhs); }} // namespace triton::core namespace std { using namespace triton::core; template <> class hash { public: size_t operator()(const InferenceRequest::SequenceId& sequence_id) const { if (sequence_id.Type() == InferenceRequest::SequenceId::DataType::STRING) { return std::hash{}(sequence_id.StringValue()); } return std::hash{}(sequence_id.UnsignedIntValue()); } }; } // namespace std kytensor-2.49.0.7/core/src/infer_response.cc000066400000000000000000000326351527052371200207150ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_response.h" #include "model.h" #include "model_config_utils.h" #include "server.h" #include "triton/common/logging.h" namespace triton { namespace core { // // InferenceResponseFactory // Status InferenceResponseFactory::CreateResponse( std::unique_ptr* response) const { response->reset(new InferenceResponse( model_, id_, allocator_, alloc_userp_, response_fn_, response_userp_, response_delegator_)); #ifdef TRITON_ENABLE_TRACING (*response)->SetTrace(trace_); #endif // TRITON_ENABLE_TRACING return Status::Success; } Status InferenceResponseFactory::SendFlags(const uint32_t flags) const { if (response_delegator_ != nullptr) { std::unique_ptr response( new InferenceResponse(response_fn_, response_userp_)); response_delegator_(std::move(response), flags); } else { void* userp = response_userp_; response_fn_(nullptr /* response */, flags, userp); } return Status::Success; } // // InferenceResponse // InferenceResponse::InferenceResponse( const std::shared_ptr& model, const std::string& id, const ResponseAllocator* allocator, void* alloc_userp, TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp, const std::function< void(std::unique_ptr&&, const uint32_t)>& delegator) : model_(model), id_(id), allocator_(allocator), alloc_userp_(alloc_userp), response_fn_(response_fn), response_userp_(response_userp), response_delegator_(delegator), null_response_(false) { // If the allocator has a start_fn then invoke it. TRITONSERVER_ResponseAllocatorStartFn_t start_fn = allocator_->StartFn(); if (start_fn != nullptr) { LOG_TRITONSERVER_ERROR( start_fn( reinterpret_cast( const_cast(allocator_)), alloc_userp_), "response allocation start failed"); } } InferenceResponse::InferenceResponse( TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp) : response_fn_(response_fn), response_userp_(response_userp), null_response_(true) { } const std::string& InferenceResponse::ModelName() const { static const std::string unknown(""); return (model_ == nullptr) ? unknown : model_->Name(); } int64_t InferenceResponse::ActualModelVersion() const { return (model_ == nullptr) ? -1 : model_->Version(); } Status InferenceResponse::AddParameter(const char* name, const char* value) { parameters_.emplace_back(name, value); return Status::Success; } Status InferenceResponse::AddParameter(const char* name, const int64_t value) { parameters_.emplace_back(name, value); return Status::Success; } Status InferenceResponse::AddParameter(const char* name, const bool value) { parameters_.emplace_back(name, value); return Status::Success; } Status InferenceResponse::AddParameter(const char* name, const double value) { parameters_.emplace_back(name, value); return Status::Success; } Status InferenceResponse::AddOutput( const std::string& name, const inference::DataType datatype, const std::vector& shape, InferenceResponse::Output** output) { outputs_.emplace_back(name, datatype, shape, allocator_, alloc_userp_); LOG_VERBOSE(1) << "add response output: " << outputs_.back(); if (model_ != nullptr) { const inference::ModelOutput* output_config; RETURN_IF_ERROR(model_->GetOutput(name, &output_config)); if (output_config->has_reshape()) { const bool has_batch_dim = (model_->Config().max_batch_size() > 0); outputs_.back().Reshape(has_batch_dim, output_config); } } if (output != nullptr) { *output = std::addressof(outputs_.back()); } return Status::Success; } Status InferenceResponse::AddOutput( const std::string& name, const inference::DataType datatype, std::vector&& shape, InferenceResponse::Output** output) { outputs_.emplace_back( name, datatype, std::move(shape), allocator_, alloc_userp_); LOG_VERBOSE(1) << "add response output: " << outputs_.back(); if (model_ != nullptr) { const inference::ModelOutput* output_config; RETURN_IF_ERROR(model_->GetOutput(name, &output_config)); if (output_config->has_reshape()) { const bool has_batch_dim = (model_->Config().max_batch_size() > 0); outputs_.back().Reshape(has_batch_dim, output_config); } } if (output != nullptr) { *output = std::addressof(outputs_.back()); } return Status::Success; } Status InferenceResponse::ClassificationLabel( const InferenceResponse::Output& output, const uint32_t class_index, const char** label) const { const auto& label_provider = model_->GetLabelProvider(); const std::string& l = label_provider->GetLabel(output.Name(), class_index); if (l.empty()) { *label = nullptr; } else { *label = l.c_str(); } return Status::Success; } Status InferenceResponse::Send( std::unique_ptr&& response, const uint32_t flags) { #ifdef TRITON_ENABLE_TRACING response->TraceOutputTensors( TRITONSERVER_TRACE_TENSOR_BACKEND_OUTPUT, "InferenceResponse Send"); #endif // TRITON_ENABLE_TRACING if (response->response_delegator_ != nullptr) { auto ldelegator = std::move(response->response_delegator_); ldelegator(std::move(response), flags); return Status::Success; } void* userp = response->response_userp_; if (response->null_response_) { response->response_fn_(nullptr /* response */, flags, userp); } else { auto& response_fn = response->response_fn_; response_fn( reinterpret_cast(response.release()), flags, userp); } return Status::Success; } Status InferenceResponse::SendWithStatus( std::unique_ptr&& response, const uint32_t flags, const Status& status) { response->status_ = status; return InferenceResponse::Send(std::move(response), flags); } #ifdef TRITON_ENABLE_TRACING Status InferenceResponse::TraceOutputTensors( TRITONSERVER_InferenceTraceActivity activity, const std::string& msg) { const auto& outputs = this->Outputs(); uint32_t output_count = outputs.size(); for (uint32_t idx = 0; idx < output_count; ++idx) { const Output& output = outputs[idx]; // output data const char* cname = output.Name().c_str(); TRITONSERVER_DataType datatype = DataTypeToTriton(output.DType()); const std::vector& oshape = output.Shape(); const int64_t* shape = &oshape[0]; uint64_t dim_count = oshape.size(); const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; Status status = output.DataBuffer( &base, &byte_size, &memory_type, &memory_type_id, &userp); if (!status.IsOk()) { LOG_STATUS_ERROR( status, std::string(TRITONSERVER_InferenceTraceActivityString(activity)) + ": " + msg + ": fail to get data buffer: " + status.Message()); return status; } INFER_TRACE_TENSOR_ACTIVITY( this->trace_, activity, cname, datatype, base, byte_size, shape, dim_count, memory_type, memory_type_id); } return Status::Success; } #endif // TRITON_ENABLE_TRACING // // InferenceResponse::Output // InferenceResponse::Output::~Output() { Status status = ReleaseDataBuffer(); if (!status.IsOk()) { LOG_ERROR << "failed to release buffer for output '" << name_ << "': " << status.AsString(); } } void InferenceResponse::Output::Reshape( const bool has_batch_dim, const inference::ModelOutput* output_config) { std::deque variable_size_values; const int64_t batch_dim = (has_batch_dim && (shape_.size() > 0)) ? shape_[0] : -1; const size_t batch_dim_offset = (has_batch_dim) ? 1 : 0; const auto& from_shape = output_config->reshape().shape(); const auto& to_shape = output_config->dims(); for (int64_t idx = 0; idx < from_shape.size(); idx++) { if (from_shape[idx] == -1) { variable_size_values.push_back(shape_[idx + batch_dim_offset]); } } shape_.clear(); if (batch_dim >= 0) { shape_.push_back(batch_dim); } for (const auto& dim : to_shape) { if (dim == -1) { shape_.push_back(variable_size_values.front()); variable_size_values.pop_front(); } else { shape_.push_back(dim); } } } Status InferenceResponse::Output::DataBuffer( const void** buffer, size_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id, void** userp) const { *buffer = allocated_buffer_; *buffer_byte_size = buffer_attributes_.ByteSize(); *memory_type = buffer_attributes_.MemoryType(); *memory_type_id = buffer_attributes_.MemoryTypeId(); *userp = allocated_userp_; return Status::Success; } Status InferenceResponse::Output::AllocateDataBuffer( void** buffer, size_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { if (allocated_buffer_ != nullptr) { return Status( Status::Code::ALREADY_EXISTS, "allocated buffer for output '" + name_ + "' already exists"); } TRITONSERVER_MemoryType actual_memory_type = *memory_type; int64_t actual_memory_type_id = *memory_type_id; void* alloc_buffer_userp = nullptr; RETURN_IF_TRITONSERVER_ERROR(allocator_->AllocFn()( reinterpret_cast( const_cast(allocator_)), name_.c_str(), buffer_byte_size, *memory_type, *memory_type_id, alloc_userp_, buffer, &alloc_buffer_userp, &actual_memory_type, &actual_memory_type_id)); // Only call the buffer attributes API if it is set. if (allocator_->BufferAttributesFn() != nullptr) { RETURN_IF_TRITONSERVER_ERROR(allocator_->BufferAttributesFn()( reinterpret_cast( const_cast(allocator_)), name_.c_str(), reinterpret_cast(&buffer_attributes_), alloc_userp_, alloc_buffer_userp)); } allocated_buffer_ = *buffer; buffer_attributes_.SetByteSize(buffer_byte_size); buffer_attributes_.SetMemoryType(actual_memory_type); buffer_attributes_.SetMemoryTypeId(actual_memory_type_id); allocated_userp_ = alloc_buffer_userp; *memory_type = actual_memory_type; *memory_type_id = actual_memory_type_id; return Status::Success; } Status InferenceResponse::Output::ReleaseDataBuffer() { TRITONSERVER_Error* err = nullptr; if (allocated_buffer_ != nullptr) { err = allocator_->ReleaseFn()( reinterpret_cast( const_cast(allocator_)), allocated_buffer_, allocated_userp_, buffer_attributes_.ByteSize(), buffer_attributes_.MemoryType(), buffer_attributes_.MemoryTypeId()); } allocated_buffer_ = nullptr; buffer_attributes_.SetByteSize(0); buffer_attributes_.SetMemoryType(TRITONSERVER_MEMORY_CPU); buffer_attributes_.SetMemoryTypeId(0); allocated_userp_ = nullptr; RETURN_IF_TRITONSERVER_ERROR(err); return Status::Success; } std::ostream& operator<<(std::ostream& out, const InferenceResponse& response) { out << "[0x" << std::addressof(response) << "] " << "response id: " << response.Id() << ", model: " << response.ModelName() << ", actual version: " << response.ActualModelVersion() << std::endl; out << "status:" << response.ResponseStatus().AsString() << std::endl; out << "outputs:" << std::endl; for (const auto& output : response.Outputs()) { out << "[0x" << std::addressof(output) << "] " << output << std::endl; } return out; } std::ostream& operator<<(std::ostream& out, const InferenceResponse::Output& output) { out << "output: " << output.Name() << ", type: " << triton::common::DataTypeToProtocolString(output.DType()) << ", shape: " << triton::common::DimsListToString(output.Shape()); return out; } }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_response.h000066400000000000000000000327041527052371200205540ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "buffer_attributes.h" #include "constants.h" #include "infer_parameter.h" #include "infer_trace.h" #include "response_allocator.h" #include "status.h" #include "triton/common/model_config.h" #include "tritonserver_apis.h" namespace triton { namespace core { class Model; class InferenceResponse; // // An inference response factory. // class InferenceResponseFactory { public: InferenceResponseFactory() = default; InferenceResponseFactory( const std::shared_ptr& model, const std::string& id, const ResponseAllocator* allocator, void* alloc_userp, TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp, const std::function&&, const uint32_t)>& delegator) : model_(model), id_(id), allocator_(allocator), alloc_userp_(alloc_userp), response_fn_(response_fn), response_userp_(response_userp), response_delegator_(delegator), is_cancelled_(false) #ifdef TRITON_ENABLE_STATS , response_stats_index_(0) #endif // TRITON_ENABLE_STATS { } void Cancel() { is_cancelled_ = true; } bool IsCancelled() { return is_cancelled_; } const ResponseAllocator* Allocator() { return allocator_; } void* AllocatorUserp() { return alloc_userp_; } Status SetResponseDelegator( const std::function&&, const uint32_t)>& delegator) { response_delegator_ = delegator; return Status::Success; } // Create a new response. Status CreateResponse(std::unique_ptr* response) const; // Send a "null" response with 'flags'. Status SendFlags(const uint32_t flags) const; #ifdef TRITON_ENABLE_TRACING const std::shared_ptr& Trace() const { return trace_; } void SetTrace(const std::shared_ptr& trace) { trace_ = trace; } void ReleaseTrace() { trace_ = nullptr; } #endif // TRITON_ENABLE_TRACING #ifdef TRITON_ENABLE_STATS // Return the current response statistics index and increment it. uint64_t GetAndIncrementResponseIndex() { return response_stats_index_++; }; #endif // TRITON_ENABLE_STATS private: // The model associated with this factory. For normal // requests/responses this will always be defined and acts to keep // the model loaded as long as this factory is live. It may be // nullptr for cases where the model itself created the request // (like running requests for warmup) and so must protect any uses // to handle the nullptr case. std::shared_ptr model_; // The ID of the corresponding request that should be included in every // response. This is a property that can be optionally provided by the user. std::string id_; // The response allocator and user pointer. The 'allocator_' is a // raw pointer because it is owned by the client, and the client is // responsible for ensuring that the lifetime of the allocator // extends longer that any request or response that depend on the // allocator. const ResponseAllocator* allocator_; void* alloc_userp_; // The response callback function and user pointer. TRITONSERVER_InferenceResponseCompleteFn_t response_fn_; void* response_userp_; // Delegator to be invoked on sending responses. std::function&&, const uint32_t)> response_delegator_; std::atomic is_cancelled_; #ifdef TRITON_ENABLE_TRACING // Inference trace associated with this response. std::shared_ptr trace_; #endif // TRITON_ENABLE_TRACING #ifdef TRITON_ENABLE_STATS // Number of response statistics reported. std::atomic response_stats_index_; #endif // TRITON_ENABLE_STATS }; // // An inference response. // class InferenceResponse { public: // Output tensor class Output { public: Output( const std::string& name, const inference::DataType datatype, const std::vector& shape, const ResponseAllocator* allocator, void* alloc_userp) : name_(name), datatype_(datatype), shape_(shape), allocator_(allocator), alloc_userp_(alloc_userp), allocated_buffer_(nullptr) { } Output( const std::string& name, const inference::DataType datatype, std::vector&& shape, const ResponseAllocator* allocator, void* alloc_userp) : name_(name), datatype_(datatype), shape_(std::move(shape)), allocator_(allocator), alloc_userp_(alloc_userp), allocated_buffer_(nullptr) { } ~Output(); // The name of the output tensor. const std::string& Name() const { return name_; } // Data type of the output tensor. inference::DataType DType() const { return datatype_; } // The shape of the output tensor. const std::vector& Shape() const { return shape_; } BufferAttributes* GetBufferAttributes() { return &buffer_attributes_; } // Reshape the output tensor. This function must only be called // for outputs that have respace specified in the model // configuration. void Reshape( const bool has_batch_dim, const inference::ModelOutput* output_config); // Get information about the buffer allocated for this output // tensor's data. If no buffer is allocated 'buffer' will return // nullptr and the other returned values will be undefined. Status DataBuffer( const void** buffer, size_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id, void** userp) const; // Allocate the buffer that should be used for this output // tensor's data. 'buffer' must return a buffer of size // 'buffer_byte_size'. 'memory_type' acts as both input and // output. On input gives the buffer memory type preferred by the // caller and on return holds the actual memory type of // 'buffer'. 'memory_type_id' acts as both input and output. On // input gives the buffer memory type id preferred by the caller // and returns the actual memory type id of 'buffer'. Only a // single buffer may be allocated for the output at any time, so // multiple calls to AllocateDataBuffer without intervening // ReleaseDataBuffer call will result in an error. Status AllocateDataBuffer( void** buffer, const size_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); // Release the buffer that was previously allocated by // AllocateDataBuffer(). Do nothing if AllocateDataBuffer() has // not been called. Status ReleaseDataBuffer(); private: DISALLOW_COPY_AND_ASSIGN(Output); friend std::ostream& operator<<( std::ostream& out, const InferenceResponse::Output& output); std::string name_; inference::DataType datatype_; std::vector shape_; // The response allocator and user pointer. const ResponseAllocator* allocator_; void* alloc_userp_; // Information about the buffer allocated by // AllocateDataBuffer(). This information is needed by // DataBuffer() and ReleaseDataBuffer(). void* allocated_buffer_; BufferAttributes buffer_attributes_; void* allocated_userp_; }; // InferenceResponse InferenceResponse( const std::shared_ptr& model, const std::string& id, const ResponseAllocator* allocator, void* alloc_userp, TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp, const std::function&&, const uint32_t)>& delegator); // "null" InferenceResponse is a special instance of InferenceResponse which // contains minimal information for calling InferenceResponse::Send, // InferenceResponse::NullResponse. nullptr will be passed as response in // 'response_fn'. InferenceResponse( TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp); const std::string& Id() const { return id_; } const std::string& ModelName() const; int64_t ActualModelVersion() const; const Status& ResponseStatus() const { return status_; } // The response parameters. const std::deque& Parameters() const { return parameters_; } // Add an parameter to the response. Status AddParameter(const char* name, const char* value); Status AddParameter(const char* name, const int64_t value); Status AddParameter(const char* name, const bool value); Status AddParameter(const char* name, const double value); // The response outputs. const std::deque& Outputs() const { return outputs_; } // Add an output to the response. If 'output' is non-null // return a pointer to the newly added output. Status AddOutput( const std::string& name, const inference::DataType datatype, const std::vector& shape, Output** output = nullptr); Status AddOutput( const std::string& name, const inference::DataType datatype, std::vector&& shape, Output** output = nullptr); // Get the classification label associated with an output. Return // 'label' == nullptr if no label. Status ClassificationLabel( const Output& output, const uint32_t class_index, const char** label) const; // Send the response with success status. Calling this function // releases ownership of the response object and gives it to the // callback function. static Status Send( std::unique_ptr&& response, const uint32_t flags); // Send the response with explicit status. Calling this function // releases ownership of the response object and gives it to the // callback function. static Status SendWithStatus( std::unique_ptr&& response, const uint32_t flags, const Status& status); #ifdef TRITON_ENABLE_TRACING const std::shared_ptr& Trace() const { return trace_; } void SetTrace(const std::shared_ptr& trace) { trace_ = trace; } void ReleaseTrace() { trace_ = nullptr; } #endif // TRITON_ENABLE_TRACING private: DISALLOW_COPY_AND_ASSIGN(InferenceResponse); friend std::ostream& operator<<( std::ostream& out, const InferenceResponse& response); #ifdef TRITON_ENABLE_TRACING Status TraceOutputTensors( TRITONSERVER_InferenceTraceActivity activity, const std::string& msg); #endif // TRITON_ENABLE_TRACING // The model associated with this factory. For normal // requests/responses this will always be defined and acts to keep // the model loaded as long as this factory is live. It may be // nullptr for cases where the model itself created the request // (like running requests for warmup) and so must protect any uses // to handle the nullptr case. std::shared_ptr model_; // The ID of the corresponding request that should be included in // every response. std::string id_; // Error status for the response. Status status_; // The parameters of the response. Use a deque so that there is no // reallocation. std::deque parameters_; // The result tensors. Use a deque so that there is no reallocation. std::deque outputs_; // The response allocator and user pointer. const ResponseAllocator* allocator_; void* alloc_userp_; // The response callback function and user pointer. TRITONSERVER_InferenceResponseCompleteFn_t response_fn_; void* response_userp_; // Delegator to be invoked on sending responses. std::function&&, const uint32_t)> response_delegator_; bool null_response_; #ifdef TRITON_ENABLE_TRACING // Inference trace associated with this response. std::shared_ptr trace_; #endif // TRITON_ENABLE_TRACING }; std::ostream& operator<<(std::ostream& out, const InferenceResponse& response); std::ostream& operator<<( std::ostream& out, const InferenceResponse::Output& output); }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_stats.cc000066400000000000000000000401631527052371200202100ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_stats.h" #include #include "metric_model_reporter.h" #include "metrics.h" #include "triton/common/logging.h" namespace triton { namespace core { #ifdef TRITON_ENABLE_STATS // This function converts FailureReason enum values to std::string std::string failureReasonToString(FailureReason reason) { switch (reason) { case FailureReason::REJECTED: return "REJECTED"; case FailureReason::CANCELED: return "CANCELED"; case FailureReason::BACKEND: return "BACKEND"; case FailureReason::OTHER: return "OTHER"; default: return "OTHER"; } } void InferenceStatsAggregator::UpdateFailure( MetricModelReporter* metric_reporter, const uint64_t request_start_ns, const uint64_t request_end_ns, FailureReason reason) { std::lock_guard lock(mu_); infer_stats_.failure_count_++; infer_stats_.failure_duration_ns_ += (request_end_ns - request_start_ns); #ifdef TRITON_ENABLE_METRICS if (metric_reporter != nullptr) { std::string reason_str = failureReasonToString(reason); metric_reporter->IncrementCounter("inf_failure_" + reason_str, 1); } #endif // TRITON_ENABLE_METRICS } void InferenceStatsAggregator::UpdateSuccess( MetricModelReporter* metric_reporter, const size_t batch_size, const uint64_t request_start_ns, const uint64_t queue_start_ns, const uint64_t compute_start_ns, const uint64_t compute_input_end_ns, const uint64_t compute_output_start_ns, const uint64_t compute_end_ns, const uint64_t request_end_ns) { const uint64_t compute_input_duration_ns = compute_input_end_ns - compute_start_ns; const uint64_t compute_infer_duration_ns = compute_output_start_ns - compute_input_end_ns; const uint64_t compute_output_duration_ns = compute_end_ns - compute_output_start_ns; UpdateSuccessWithDuration( metric_reporter, batch_size, request_start_ns, queue_start_ns, compute_start_ns, request_end_ns, compute_input_duration_ns, compute_infer_duration_ns, compute_output_duration_ns); } void InferenceStatsAggregator::UpdateSuccessWithDuration( MetricModelReporter* metric_reporter, const size_t batch_size, const uint64_t request_start_ns, const uint64_t queue_start_ns, const uint64_t compute_start_ns, const uint64_t request_end_ns, const uint64_t compute_input_duration_ns, const uint64_t compute_infer_duration_ns, const uint64_t compute_output_duration_ns) { const uint64_t request_duration_ns = request_end_ns - request_start_ns; const uint64_t queue_duration_ns = compute_start_ns - queue_start_ns; std::lock_guard lock(mu_); inference_count_ += batch_size; infer_stats_.success_count_++; infer_stats_.request_duration_ns_ += request_duration_ns; infer_stats_.queue_duration_ns_ += queue_duration_ns; infer_stats_.compute_input_duration_ns_ += compute_input_duration_ns; infer_stats_.compute_infer_duration_ns_ += compute_infer_duration_ns; infer_stats_.compute_output_duration_ns_ += compute_output_duration_ns; #ifdef TRITON_ENABLE_METRICS if (metric_reporter != nullptr) { metric_reporter->IncrementCounter("inf_success", 1); metric_reporter->IncrementCounter("inf_count", batch_size); // Counter Latencies metric_reporter->IncrementCounter( "request_duration", request_duration_ns / 1000); metric_reporter->IncrementCounter( "queue_duration", queue_duration_ns / 1000); metric_reporter->IncrementCounter( "compute_input_duration", compute_input_duration_ns / 1000); metric_reporter->IncrementCounter( "compute_infer_duration", compute_infer_duration_ns / 1000); metric_reporter->IncrementCounter( "compute_output_duration", compute_output_duration_ns / 1000); // Summary Latencies const auto& reporter_config = metric_reporter->Config(); // FIXME [DLIS-4762]: request summary is disabled when cache is enabled. if (!reporter_config.cache_enabled_) { metric_reporter->ObserveSummary( "request_duration", request_duration_ns / 1000); } metric_reporter->ObserveSummary("queue_duration", queue_duration_ns / 1000); metric_reporter->ObserveSummary( "compute_input_duration", compute_input_duration_ns / 1000); metric_reporter->ObserveSummary( "compute_infer_duration", compute_infer_duration_ns / 1000); metric_reporter->ObserveSummary( "compute_output_duration", compute_output_duration_ns / 1000); } #endif // TRITON_ENABLE_METRICS } // Currently cache hits will not go to the inference backend where metrics // are typically updated, so this method allows us to update relevant metrics // from a metric reporter rather than going through the backend. void InferenceStatsAggregator::UpdateSuccessCacheHit( MetricModelReporter* metric_reporter, const size_t batch_size, const uint64_t request_start_ns, const uint64_t queue_start_ns, const uint64_t cache_lookup_start_ns, const uint64_t request_end_ns, const uint64_t cache_hit_duration_ns) { const uint64_t request_duration_ns = request_end_ns - request_start_ns; const uint64_t queue_duration_ns = cache_lookup_start_ns - queue_start_ns; std::lock_guard lock(mu_); infer_stats_.success_count_++; infer_stats_.request_duration_ns_ += request_duration_ns; infer_stats_.queue_duration_ns_ += queue_duration_ns; infer_stats_.cache_hit_count_++; infer_stats_.cache_hit_duration_ns_ += cache_hit_duration_ns; #ifdef TRITON_ENABLE_METRICS if (metric_reporter != nullptr) { // inf_count not recorded on a cache hit metric_reporter->IncrementCounter("inf_success", 1); // Counter Latencies metric_reporter->IncrementCounter( "request_duration", request_duration_ns / 1000); metric_reporter->IncrementCounter( "queue_duration", queue_duration_ns / 1000); metric_reporter->IncrementCounter("cache_hit_count", 1); metric_reporter->IncrementCounter( "cache_hit_duration", cache_hit_duration_ns / 1000); // Summary Latencies // FIXME [DLIS-4762]: request summary is disabled when cache is enabled. // metric_reporter->ObserveSummary( // "request_duration", request_duration_ns / 1000); metric_reporter->ObserveSummary("queue_duration", queue_duration_ns / 1000); metric_reporter->ObserveSummary( "cache_hit_duration", cache_hit_duration_ns / 1000); } #endif // TRITON_ENABLE_METRICS } // Cache misses will go to the inference backend where metrics are typically // updated, but cache insertion happens after the inference backend finishes. // So we use this method to update cache miss stats and adjust the request // duration to include cache insertion time. void InferenceStatsAggregator::UpdateSuccessCacheMiss( MetricModelReporter* metric_reporter, const uint64_t cache_miss_duration_ns) { std::lock_guard lock(mu_); infer_stats_.request_duration_ns_ += cache_miss_duration_ns; infer_stats_.cache_miss_count_++; infer_stats_.cache_miss_duration_ns_ += cache_miss_duration_ns; #ifdef TRITON_ENABLE_METRICS if (metric_reporter != nullptr) { // Add cache insertion time to request duration since insertion // happens after inference backend sets the request duration, and // cache lookup time was already included before the inference backend // was called metric_reporter->IncrementCounter( "request_duration", cache_miss_duration_ns / 1000); metric_reporter->IncrementCounter("cache_miss_count", 1); metric_reporter->IncrementCounter( "cache_miss_duration", cache_miss_duration_ns / 1000); // FIXME [DLIS-4762]: request summary is disabled when cache is enabled. // Need to account for adding cache miss duration on top of // request_duration from backend within a single observation. // metric_reporter->ObserveSummary( // "request_duration", cache_miss_duration_ns / 1000); metric_reporter->ObserveSummary( "cache_miss_duration", cache_miss_duration_ns / 1000); } #endif // TRITON_ENABLE_METRICS } Status InferenceStatsAggregator::UpdateResponseSuccess( const std::string& key, const uint64_t response_start_ns, const uint64_t compute_output_start_ns, const uint64_t response_end_ns) { if (response_start_ns > compute_output_start_ns) { return Status( Status::Code::INVALID_ARG, "Response start cannot happen after compute output start"); } if (compute_output_start_ns > response_end_ns) { return Status( Status::Code::INVALID_ARG, "Compute output start cannot happen after response end"); } const uint64_t compute_infer_duration_ns = compute_output_start_ns - response_start_ns; const uint64_t compute_output_duration_ns = response_end_ns - compute_output_start_ns; const uint64_t total_duration_ns = response_end_ns - response_start_ns; { std::lock_guard lock(mu_); auto it = response_stats_.find(key); if (it == response_stats_.end()) { it = response_stats_.emplace(key, InferResponseStats()).first; } it->second.compute_infer_count++; it->second.compute_infer_duration_ns += compute_infer_duration_ns; it->second.compute_output_count++; it->second.compute_output_duration_ns += compute_output_duration_ns; it->second.success_count++; it->second.success_duration_ns += total_duration_ns; } return Status::Success; } Status InferenceStatsAggregator::UpdateResponseFail( const std::string& key, const uint64_t response_start_ns, const uint64_t compute_output_start_ns, const uint64_t response_end_ns) { uint64_t compute_infer_duration_ns, compute_output_duration_ns, total_duration_ns; if (compute_output_start_ns > 0) { // output tensors copied if (response_start_ns > compute_output_start_ns) { return Status( Status::Code::INVALID_ARG, "Response start cannot happen after compute output start"); } if (compute_output_start_ns > response_end_ns) { return Status( Status::Code::INVALID_ARG, "Compute output start cannot happen after response end"); } compute_infer_duration_ns = compute_output_start_ns - response_start_ns; compute_output_duration_ns = response_end_ns - compute_output_start_ns; total_duration_ns = response_end_ns - response_start_ns; } else { // no output tensors copied if (response_start_ns > response_end_ns) { return Status( Status::Code::INVALID_ARG, "Response start cannot happen after response end"); } compute_infer_duration_ns = response_end_ns - response_start_ns; compute_output_duration_ns = 0; total_duration_ns = response_end_ns - response_start_ns; } { std::lock_guard lock(mu_); auto it = response_stats_.find(key); if (it == response_stats_.end()) { it = response_stats_.emplace(key, InferResponseStats()).first; } it->second.compute_infer_count++; it->second.compute_infer_duration_ns += compute_infer_duration_ns; if (compute_output_duration_ns > 0) { it->second.compute_output_count++; it->second.compute_output_duration_ns += compute_output_duration_ns; } it->second.fail_count++; it->second.fail_duration_ns += total_duration_ns; } return Status::Success; } Status InferenceStatsAggregator::UpdateResponseEmpty( const std::string& key, const uint64_t response_start_ns, const uint64_t response_end_ns) { if (response_start_ns > response_end_ns) { return Status( Status::Code::INVALID_ARG, "Response start cannot happen after response end"); } const uint64_t compute_infer_duration_ns = response_end_ns - response_start_ns; const uint64_t total_duration_ns = response_end_ns - response_start_ns; { std::lock_guard lock(mu_); auto it = response_stats_.find(key); if (it == response_stats_.end()) { it = response_stats_.emplace(key, InferResponseStats()).first; } it->second.compute_infer_count++; it->second.compute_infer_duration_ns += compute_infer_duration_ns; it->second.empty_response_count++; it->second.empty_response_duration_ns += total_duration_ns; } return Status::Success; } Status InferenceStatsAggregator::UpdateResponseCancel( const std::string& key, const uint64_t response_start_ns, const uint64_t response_end_ns) { if (response_start_ns > response_end_ns) { return Status( Status::Code::INVALID_ARG, "Response start cannot happen after response end"); } const uint64_t total_duration_ns = response_end_ns - response_start_ns; { std::lock_guard lock(mu_); auto it = response_stats_.find(key); if (it == response_stats_.end()) { it = response_stats_.emplace(key, InferResponseStats()).first; } it->second.cancel_count++; it->second.cancel_duration_ns += total_duration_ns; } return Status::Success; } void InferenceStatsAggregator::UpdateInferBatchStats( MetricModelReporter* metric_reporter, const size_t batch_size, const uint64_t compute_start_ns, const uint64_t compute_input_end_ns, const uint64_t compute_output_start_ns, const uint64_t compute_end_ns) { auto compute_input_duration_ns = (compute_input_end_ns - compute_start_ns); auto compute_infer_duration_ns = (compute_output_start_ns - compute_input_end_ns); auto compute_output_duration_ns = (compute_end_ns - compute_output_start_ns); UpdateInferBatchStatsWithDuration( metric_reporter, batch_size, compute_input_duration_ns, compute_infer_duration_ns, compute_output_duration_ns); } void InferenceStatsAggregator::UpdateInferBatchStatsWithDuration( MetricModelReporter* metric_reporter, size_t batch_size, const uint64_t compute_input_duration_ns, const uint64_t compute_infer_duration_ns, const uint64_t compute_output_duration_ns) { uint64_t inference_ms = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); std::lock_guard lock(mu_); if (inference_ms > last_inference_ms_) { last_inference_ms_ = inference_ms; } execution_count_++; auto it = batch_stats_.find(batch_size); if (it == batch_stats_.end()) { it = batch_stats_.emplace(batch_size, InferBatchStats()).first; } it->second.count_++; it->second.compute_input_duration_ns_ += compute_input_duration_ns; it->second.compute_infer_duration_ns_ += compute_infer_duration_ns; it->second.compute_output_duration_ns_ += compute_output_duration_ns; #ifdef TRITON_ENABLE_METRICS if (metric_reporter != nullptr) { metric_reporter->IncrementCounter("inf_exec_count", 1); } #endif // TRITON_ENABLE_METRICS } #endif // TRITON_ENABLE_STATS }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_stats.h000066400000000000000000000220571527052371200200540ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include "constants.h" #include "infer_response.h" #include "status.h" #include "tritonserver_apis.h" namespace triton { namespace core { // Define the FailureReason enum within the triton::core namespace enum class FailureReason { REJECTED, CANCELED, BACKEND, OTHER }; class MetricModelReporter; // // InferenceStatsAggregator // // A statistics aggregator. // class InferenceStatsAggregator { #ifdef TRITON_ENABLE_STATS public: struct InferStats { InferStats() : failure_count_(0), failure_duration_ns_(0), success_count_(0), request_duration_ns_(0), queue_duration_ns_(0), compute_input_duration_ns_(0), compute_infer_duration_ns_(0), compute_output_duration_ns_(0), cache_hit_count_(0), cache_hit_duration_ns_(0), cache_miss_count_(0), cache_miss_duration_ns_(0) { } uint64_t failure_count_; uint64_t failure_duration_ns_; uint64_t success_count_; uint64_t request_duration_ns_; uint64_t queue_duration_ns_; uint64_t compute_input_duration_ns_; uint64_t compute_infer_duration_ns_; uint64_t compute_output_duration_ns_; // Cache hit stats uint64_t cache_hit_count_; uint64_t cache_hit_duration_ns_; // Cache miss stats uint64_t cache_miss_count_; uint64_t cache_miss_duration_ns_; }; struct InferResponseStats { InferResponseStats() : compute_infer_count(0), compute_infer_duration_ns(0), compute_output_count(0), compute_output_duration_ns(0), success_count(0), success_duration_ns(0), fail_count(0), fail_duration_ns(0), empty_response_count(0), empty_response_duration_ns(0), cancel_count(0), cancel_duration_ns(0) { } uint64_t compute_infer_count; uint64_t compute_infer_duration_ns; uint64_t compute_output_count; uint64_t compute_output_duration_ns; uint64_t success_count; uint64_t success_duration_ns; uint64_t fail_count; uint64_t fail_duration_ns; uint64_t empty_response_count; uint64_t empty_response_duration_ns; uint64_t cancel_count; uint64_t cancel_duration_ns; }; struct InferBatchStats { InferBatchStats() : count_(0), compute_input_duration_ns_(0), compute_infer_duration_ns_(0), compute_output_duration_ns_(0) { } uint64_t count_; uint64_t compute_input_duration_ns_; uint64_t compute_infer_duration_ns_; uint64_t compute_output_duration_ns_; }; // Create an aggregator for model statistics InferenceStatsAggregator() : last_inference_ms_(0), inference_count_(0), execution_count_(0) { } uint64_t LastInferenceMs() const { return last_inference_ms_; } uint64_t InferenceCount() const { return inference_count_; } uint64_t ExecutionCount() const { return execution_count_; } const InferStats& ImmutableInferStats() const { return infer_stats_; } const std::map& ImmutableInferResponseStats() const { return response_stats_; } const std::map& ImmutableInferBatchStats() const { return batch_stats_; } // Add durations to Infer stats for a failed inference request. void UpdateFailure( MetricModelReporter* metric_reporter, const uint64_t request_start_ns, const uint64_t request_end_ns, FailureReason reason); // Add durations to infer stats for a successful inference request. void UpdateSuccess( MetricModelReporter* metric_reporter, const size_t batch_size, const uint64_t request_start_ns, const uint64_t queue_start_ns, const uint64_t compute_start_ns, const uint64_t compute_input_end_ns, const uint64_t compute_output_start_ns, const uint64_t compute_end_ns, const uint64_t request_end_ns); // Add durations to infer stats for a successful inference request. void UpdateSuccessWithDuration( MetricModelReporter* metric_reporter, const size_t batch_size, const uint64_t request_start_ns, const uint64_t queue_start_ns, const uint64_t compute_start_ns, const uint64_t request_end_ns, const uint64_t compute_input_duration_ns, const uint64_t compute_infer_duration_ns, const uint64_t compute_output_duration_ns); // Add durations to infer stats for a successful cached response. void UpdateSuccessCacheHit( MetricModelReporter* metric_reporter, const size_t batch_size, const uint64_t request_start_ns, const uint64_t queue_start_ns, const uint64_t cache_lookup_start_ns, const uint64_t request_end_ns, const uint64_t cache_hit_duration_ns); // Add durations to infer stats for a cache miss and update request duration // to account for cache insertion after backend computes the response. void UpdateSuccessCacheMiss( MetricModelReporter* metric_reporter, const uint64_t cache_miss_duration_ns); // Add durations to response stats for a successful response. Status UpdateResponseSuccess( const std::string& key, const uint64_t response_start_ns, const uint64_t compute_output_start_ns, const uint64_t response_end_ns); // Add durations to response stats for a failed response. Status UpdateResponseFail( const std::string& key, const uint64_t response_start_ns, const uint64_t compute_output_start_ns, const uint64_t response_end_ns); // Add durations to response stats for an empty response. Status UpdateResponseEmpty( const std::string& key, const uint64_t response_start_ns, const uint64_t response_end_ns); // Add durations to response stats for a cancellation response. Status UpdateResponseCancel( const std::string& key, const uint64_t response_start_ns, const uint64_t response_end_ns); // Add durations to batch infer stats for a batch execution. // 'success_request_count' is the number of success requests in the // batch that have infer_stats attached. void UpdateInferBatchStats( MetricModelReporter* metric_reporter, const size_t batch_size, const uint64_t compute_start_ns, const uint64_t compute_input_end_ns, const uint64_t compute_output_start_ns, const uint64_t compute_end_ns); // Add durations to batch infer stats for a batch execution. // 'success_request_count' is the number of success requests in the // batch that have infer_stats attached. void UpdateInferBatchStatsWithDuration( MetricModelReporter* metric_reporter, size_t batch_size, const uint64_t compute_input_duration_ns, const uint64_t compute_infer_duration_ns, const uint64_t compute_output_duration_ns); private: std::mutex mu_; uint64_t last_inference_ms_; uint64_t inference_count_; uint64_t execution_count_; InferStats infer_stats_; std::map response_stats_; std::map batch_stats_; #endif // TRITON_ENABLE_STATS }; // // Macros to set infer stats. // #ifdef TRITON_ENABLE_STATS #define INFER_STATS_SET_TIMESTAMP(TS_NS) \ { \ TS_NS = std::chrono::duration_cast( \ std::chrono::steady_clock::now().time_since_epoch()) \ .count(); \ } #define INFER_STATS_DECL_TIMESTAMP(TS_NS) \ uint64_t TS_NS; \ INFER_STATS_SET_TIMESTAMP(TS_NS); #else #define INFER_STATS_DECL_TIMESTAMP(TS_NS) #define INFER_STATS_SET_TIMESTAMP(TS_NS) #endif // TRITON_ENABLE_STATS }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_trace.cc000066400000000000000000000063661527052371200201570ustar00rootroot00000000000000// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_trace.h" #define TRITONJSON_STATUSTYPE triton::core::Status #define TRITONJSON_STATUSRETURN(M) \ return triton::core::Status(triton::core::Status::Code::INTERNAL, (M)) #define TRITONJSON_STATUSSUCCESS triton::core::Status::Success #include "triton/common/logging.h" #include "triton/common/triton_json.h" namespace triton { namespace core { #ifdef TRITON_ENABLE_TRACING // Start the trace id at 1, because id 0 is reserved to indicate no // parent. std::atomic InferenceTrace::next_id_(1); InferenceTrace* InferenceTrace::SpawnChildTrace() { InferenceTrace* trace = new InferenceTrace( level_, id_, activity_fn_, tensor_activity_fn_, release_fn_, userp_); return trace; } void InferenceTrace::Release() { release_fn_(reinterpret_cast(this), userp_); } void InferenceTrace::RecordActivityName( uint64_t timestamp_ns, std::string activity_name) { std::lock_guard lock(mu_); triton::common::TritonJson::Value context_json( triton::common::TritonJson::ValueType::OBJECT); if (!context_.empty()) { Status status = context_json.Parse(context_); if (!status.IsOk()) { LOG_ERROR << "Error parsing trace context"; } } std::string key = std::to_string(timestamp_ns); context_json.SetStringObject(key.c_str(), activity_name); triton::common::TritonJson::WriteBuffer buffer; context_json.Write(&buffer); context_ = buffer.Contents(); } std::shared_ptr InferenceTraceProxy::SpawnChildTrace() { std::shared_ptr strace_proxy = std::make_shared(trace_->SpawnChildTrace()); return strace_proxy; } #endif // TRITON_ENABLE_TRACING }} // namespace triton::core kytensor-2.49.0.7/core/src/infer_trace.h000066400000000000000000000206741527052371200200170ustar00rootroot00000000000000// Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "constants.h" #include "status.h" #include "tritonserver_apis.h" namespace triton { namespace core { #ifdef TRITON_ENABLE_TRACING // // InferenceTrace // // Interface to TRITONSERVER_InferenceTrace to report trace events. // class InferenceTrace { public: InferenceTrace( const TRITONSERVER_InferenceTraceLevel level, const uint64_t parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn, TRITONSERVER_InferenceTraceReleaseFn_t release_fn, void* userp) : level_(level), id_(next_id_++), parent_id_(parent_id), activity_fn_(activity_fn), tensor_activity_fn_(tensor_activity_fn), release_fn_(release_fn), userp_(userp), context_("") { } InferenceTrace* SpawnChildTrace(); int64_t Id() const { return id_; } int64_t ParentId() const { return parent_id_; } const std::string& ModelName() const { return model_name_; } int64_t ModelVersion() const { return model_version_; } const std::string& RequestId() const { return request_id_; } const std::string& Context() const { return context_; } void SetModelName(const std::string& n) { model_name_ = n; } void SetModelVersion(int64_t v) { model_version_ = v; } void SetRequestId(const std::string& request_id) { request_id_ = request_id; } void SetContext(const std::string& context) { context_ = context; } void RecordActivityName(uint64_t timestamp_ns, std::string activity_name); // Report trace activity. void Report( const TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, std::string activity_name = "") { if ((level_ & TRITONSERVER_TRACE_LEVEL_TIMESTAMPS) > 0) { if (activity == TRITONSERVER_TRACE_CUSTOM_ACTIVITY) { RecordActivityName(timestamp_ns, activity_name); } activity_fn_( reinterpret_cast(this), activity, timestamp_ns, userp_); } } // Report trace activity at the current time. void ReportNow( const TRITONSERVER_InferenceTraceActivity activity, std::string activity_name = "") { if ((level_ & TRITONSERVER_TRACE_LEVEL_TIMESTAMPS) > 0) { auto now = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); Report(activity, now, activity_name); } } // Report tensor trace activity. void ReportTensor( const TRITONSERVER_InferenceTraceActivity activity, const char* name, TRITONSERVER_DataType datatype, const void* base, size_t byte_size, const int64_t* shape, uint64_t dim_count, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { if ((level_ & TRITONSERVER_TRACE_LEVEL_TENSORS) > 0) { tensor_activity_fn_( reinterpret_cast(this), activity, name, datatype, base, byte_size, shape, dim_count, memory_type, memory_type_id, userp_); } } // Release the trace. Call the trace release callback. void Release(); private: const TRITONSERVER_InferenceTraceLevel level_; const uint64_t id_; const uint64_t parent_id_; TRITONSERVER_InferenceTraceActivityFn_t activity_fn_; TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn_; TRITONSERVER_InferenceTraceReleaseFn_t release_fn_; void* userp_; std::string model_name_; int64_t model_version_; std::string request_id_; // Maintain next id statically so that trace id is unique even // across traces static std::atomic next_id_; std::string context_; std::mutex mu_; }; // // InferenceTraceProxy // // Object attached as shared_ptr to InferenceRequest and // InferenceResponse(s) being traced as part of a single inference // request. // class InferenceTraceProxy { public: InferenceTraceProxy(InferenceTrace* trace) : trace_(trace) {} ~InferenceTraceProxy() { trace_->Release(); } InferenceTrace* Trace() { return trace_; } int64_t Id() const { return trace_->Id(); } int64_t ParentId() const { return trace_->ParentId(); } const std::string& ModelName() const { return trace_->ModelName(); } const std::string& RequestId() const { return trace_->RequestId(); } int64_t ModelVersion() const { return trace_->ModelVersion(); } const std::string& Context() const { return trace_->Context(); } void SetModelName(const std::string& n) { trace_->SetModelName(n); } void SetRequestId(const std::string& n) { trace_->SetRequestId(n); } void SetModelVersion(int64_t v) { trace_->SetModelVersion(v); } void SetContext(const std::string& context) { trace_->SetContext(context); } void RecordActivityName(uint64_t timestamp_ns, std::string activity_name) { trace_->RecordActivityName(timestamp_ns, activity_name); } void Report( const TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns) { trace_->Report(activity, timestamp_ns); } void ReportNow(const TRITONSERVER_InferenceTraceActivity activity) { trace_->ReportNow(activity); } void ReportTensor( const TRITONSERVER_InferenceTraceActivity activity, const char* name, TRITONSERVER_DataType datatype, const void* base, size_t byte_size, const int64_t* shape, uint64_t dim_count, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { trace_->ReportTensor( activity, name, datatype, base, byte_size, shape, dim_count, memory_type, memory_type_id); } std::shared_ptr SpawnChildTrace(); private: InferenceTrace* trace_; }; #endif // TRITON_ENABLE_TRACING // // Macros to generate trace activity // #ifdef TRITON_ENABLE_TRACING #define INFER_TRACE_ACTIVITY(T, A, TS_NS) \ { \ const auto& trace = (T); \ const auto ts_ns = (TS_NS); \ if (trace != nullptr) { \ trace->Report(A, ts_ns); \ } \ } #define INFER_TRACE_ACTIVITY_NOW(T, A) \ { \ const auto& trace = (T); \ if (trace != nullptr) { \ trace->ReportNow(A); \ } \ } #define INFER_TRACE_TENSOR_ACTIVITY(T, A, N, D, BA, BY, S, DI, MT, MTI) \ { \ const auto& trace = (T); \ if (trace != nullptr) { \ trace->ReportTensor(A, N, D, BA, BY, S, DI, MT, MTI); \ } \ } #else #define INFER_TRACE_ACTIVITY(T, A, TS_NS) #define INFER_TRACE_ACTIVITY_NOW(T, A) #define INFER_TRACE_TENSOR_ACTIVITY(T, A, N, D, BA, BY, S, DI, MT, MTI) #endif // TRITON_ENABLE_TRACING }} // namespace triton::core kytensor-2.49.0.7/core/src/instance_queue.cc000066400000000000000000000115001527052371200206700ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "instance_queue.h" #include "triton/common/logging.h" namespace triton { namespace core { InstanceQueue::InstanceQueue(size_t max_batch_size, uint64_t max_queue_delay_ns) : max_batch_size_(max_batch_size), max_queue_delay_ns_(max_queue_delay_ns), waiting_consumer_count_(0) { } size_t InstanceQueue::Size() { return payload_queue_.size(); } bool InstanceQueue::Empty() { return payload_queue_.empty(); } void InstanceQueue::Enqueue(const std::shared_ptr& payload) { payload_queue_.push_back(payload); } void InstanceQueue::Dequeue( std::shared_ptr* payload, std::vector>* merged_payloads) { // Dequeue frontmost payload and mark it for execution. *payload = payload_queue_.front(); payload_queue_.pop_front(); { std::lock_guard exec_lock(*((*payload)->GetExecMutex())); (*payload)->SetState(Payload::State::EXECUTING); // If the payload isn't saturated and a queue delay is set, attempt to // pop and merge additional payloads from the front of the queue into the // largest batch <= max_batch_size until saturated or queue delay is hit. if ((!payload_queue_.empty()) && (max_queue_delay_ns_ > 0) && (max_batch_size_ > 1) && (!(*payload)->IsSaturated())) { bool continue_merge; do { continue_merge = false; uint64_t now_ns = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); size_t batch_size = (*payload)->BatchSize(); if ((!payload_queue_.empty()) && (!payload_queue_.front()->IsSaturated()) && (now_ns - payload_queue_.front()->BatcherStartNs()) > max_queue_delay_ns_) { std::lock_guard exec_lock( *(payload_queue_.front()->GetExecMutex())); payload_queue_.front()->SetState(Payload::State::EXECUTING); size_t front_batch_size = payload_queue_.front()->BatchSize(); if ((batch_size + front_batch_size) <= max_batch_size_) { const auto& status = (*payload)->MergePayload(payload_queue_.front()); // If a payload is merged, remove it from the queue and mark it as // merged so it can be released and cleaned up. if (status.IsOk()) { merged_payloads->push_back(payload_queue_.front()); payload_queue_.pop_front(); continue_merge = true; } } } } while (continue_merge); } } } void InstanceQueue::IncrementConsumerCount() { { std::lock_guard lock(waiting_consumer_mu_); waiting_consumer_count_++; } waiting_consumer_cv_.notify_one(); } void InstanceQueue::DecrementConsumerCount() { { std::lock_guard lock(waiting_consumer_mu_); waiting_consumer_count_--; } waiting_consumer_cv_.notify_one(); } void InstanceQueue::WaitForConsumer() { std::unique_lock lock(waiting_consumer_mu_); waiting_consumer_cv_.wait( lock, [this]() { return waiting_consumer_count_ > 0; }); } int InstanceQueue::WaitingConsumerCount() { std::lock_guard lock(waiting_consumer_mu_); return waiting_consumer_count_; } }} // namespace triton::core kytensor-2.49.0.7/core/src/instance_queue.h000066400000000000000000000047411527052371200205430ustar00rootroot00000000000000// Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "payload.h" namespace triton { namespace core { // // InstanceQueue // // A queue implementation holding Payloads ready to be scheduled on // model instance. class InstanceQueue { public: explicit InstanceQueue(size_t max_batch_size, uint64_t max_queue_delay_ns); size_t Size(); bool Empty(); void Enqueue(const std::shared_ptr& payload); void Dequeue( std::shared_ptr* payload, std::vector>* merged_payloads); void IncrementConsumerCount(); void DecrementConsumerCount(); void WaitForConsumer(); int WaitingConsumerCount(); private: size_t max_batch_size_; uint64_t max_queue_delay_ns_; std::deque> payload_queue_; std::shared_ptr staged_payload_; std::mutex mu_; int waiting_consumer_count_; std::mutex waiting_consumer_mu_; std::condition_variable waiting_consumer_cv_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/label_provider.cc000066400000000000000000000060321527052371200206550ustar00rootroot00000000000000// Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "label_provider.h" #include #include #include #include "filesystem/api.h" namespace triton { namespace core { const std::string& LabelProvider::GetLabel(const std::string& name, size_t index) const { static const std::string not_found; auto itr = label_map_.find(name); if (itr == label_map_.end()) { return not_found; } if (itr->second.size() <= index) { return not_found; } return itr->second[index]; } Status LabelProvider::AddLabels(const std::string& name, const std::string& filepath) { std::string label_file_contents; RETURN_IF_ERROR(ReadTextFile(filepath, &label_file_contents)); auto p = label_map_.insert(std::make_pair(name, std::vector())); if (!p.second) { return Status( Status::Code::INTERNAL, "multiple label files for '" + name + "'"); } auto itr = p.first; std::istringstream label_file_stream(label_file_contents); std::string line; while (std::getline(label_file_stream, line)) { itr->second.push_back(line); } return Status::Success; } const std::vector& LabelProvider::GetLabels(const std::string& name) { static const std::vector not_found; auto itr = label_map_.find(name); if (itr == label_map_.end()) { return not_found; } return itr->second; } Status LabelProvider::AddLabels( const std::string& name, const std::vector& labels) { label_map_.emplace(name, labels); return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/label_provider.h000066400000000000000000000053661527052371200205300ustar00rootroot00000000000000// Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include "constants.h" #include "status.h" namespace triton { namespace core { // Provides classification labels. class LabelProvider { public: LabelProvider() = default; // Return the label associated with 'name' for a given // 'index'. Return empty string if no label is available. const std::string& GetLabel(const std::string& name, size_t index) const; // Associate with 'name' a set of labels initialized from a given // 'filepath'. Within the file each label is specified on its own // line. The first label (line 0) is the index-0 label, the second // label (line 1) is the index-1 label, etc. Status AddLabels(const std::string& name, const std::string& filepath); // Return the labels associated with 'name'. Return empty vector if no labels // are available. const std::vector& GetLabels(const std::string& name); // Associate with 'name' a set of 'labels' Status AddLabels( const std::string& name, const std::vector& labels); private: DISALLOW_COPY_AND_ASSIGN(LabelProvider); std::unordered_map> label_map_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/libtritonserver.ldscript000066400000000000000000000031711527052371200223610ustar00rootroot00000000000000# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONSERVER_*; TRITONBACKEND_*; TRITONREPOAGENT_*; TRITONCACHE_*; local: *; }; kytensor-2.49.0.7/core/src/memory.cc000066400000000000000000000313661527052371200172040ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "memory.h" #include "pinned_memory_manager.h" #include "triton/common/logging.h" #ifdef TRITON_ENABLE_GPU #include #include "cuda_memory_manager.h" #include "cuda_utils.h" #endif // TRITON_ENABLE_GPU namespace triton { namespace core { // // MemoryReference // MemoryReference::MemoryReference() : Memory() {} const char* MemoryReference::BufferAt( size_t idx, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) const { if (idx >= buffer_.size()) { *byte_size = 0; *memory_type = TRITONSERVER_MEMORY_CPU; *memory_type_id = 0; return nullptr; } *memory_type = buffer_[idx].buffer_attributes_.MemoryType(); *memory_type_id = buffer_[idx].buffer_attributes_.MemoryTypeId(); *byte_size = buffer_[idx].buffer_attributes_.ByteSize(); return buffer_[idx].buffer_; } const char* MemoryReference::BufferAt(size_t idx, BufferAttributes** buffer_attributes) { if (idx >= buffer_.size()) { *buffer_attributes = nullptr; return nullptr; } *buffer_attributes = &(buffer_[idx].buffer_attributes_); return buffer_[idx].buffer_; } size_t MemoryReference::AddBuffer( const char* buffer, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { total_byte_size_ += byte_size; buffer_count_++; buffer_.emplace_back(buffer, byte_size, memory_type, memory_type_id); return buffer_.size() - 1; } size_t MemoryReference::AddBuffer( const char* buffer, BufferAttributes* buffer_attributes) { total_byte_size_ += buffer_attributes->ByteSize(); buffer_count_++; buffer_.emplace_back(buffer, buffer_attributes); return buffer_.size() - 1; } size_t MemoryReference::AddBufferFront( const char* buffer, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { total_byte_size_ += byte_size; buffer_count_++; buffer_.emplace( buffer_.begin(), buffer, byte_size, memory_type, memory_type_id); return buffer_.size() - 1; } // // MutableMemory // MutableMemory::MutableMemory( char* buffer, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) : Memory(), buffer_(buffer), buffer_attributes_( BufferAttributes(byte_size, memory_type, memory_type_id, nullptr)) { total_byte_size_ = byte_size; buffer_count_ = (byte_size == 0) ? 0 : 1; } const char* MutableMemory::BufferAt( size_t idx, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) const { if (idx != 0) { *byte_size = 0; *memory_type = TRITONSERVER_MEMORY_CPU; *memory_type_id = 0; return nullptr; } *byte_size = total_byte_size_; *memory_type = buffer_attributes_.MemoryType(); *memory_type_id = buffer_attributes_.MemoryTypeId(); return buffer_; } const char* MutableMemory::BufferAt(size_t idx, BufferAttributes** buffer_attributes) { if (idx != 0) { *buffer_attributes = nullptr; return nullptr; } *buffer_attributes = &buffer_attributes_; return buffer_; } char* MutableMemory::MutableBuffer( TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { if (memory_type != nullptr) { *memory_type = buffer_attributes_.MemoryType(); } if (memory_type_id != nullptr) { *memory_type_id = buffer_attributes_.MemoryTypeId(); } return buffer_; } Status MutableMemory::SetMemory(unsigned char value) { if (buffer_attributes_.MemoryType() == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU ScopedSetDevice scoped_set_device(buffer_attributes_.MemoryTypeId()); RETURN_IF_CUDA_ERR( cudaMemset(buffer_, value, TotalByteSize()), std::string("failed to set the data to zero.")); #else return Status( Status::Code::INVALID_ARG, "Server is compiled with TRITON_ENABLE_GPU=OFF. It doesn't support " "setting cuda memory to zero."); #endif } else if ( buffer_attributes_.MemoryType() == TRITONSERVER_MEMORY_CPU || buffer_attributes_.MemoryType() == TRITONSERVER_MEMORY_CPU_PINNED) { memset(buffer_, value, TotalByteSize()); } else { return Status(Status::Code::INVALID_ARG, "Unsupported memory type"); } return Status::Success; } // // AllocatedMemory // AllocatedMemory::AllocatedMemory( size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) : MutableMemory(nullptr, byte_size, memory_type, memory_type_id) { if (total_byte_size_ != 0) { // Allocate memory with the following fallback policy: // CUDA memory -> pinned system memory -> non-pinned system memory switch (buffer_attributes_.MemoryType()) { #ifdef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_GPU: { auto status = CudaMemoryManager::Alloc( (void**)&buffer_, total_byte_size_, buffer_attributes_.MemoryTypeId()); if (!status.IsOk()) { static bool warning_logged = false; if (!warning_logged) { LOG_WARNING << status.Message() << ", falling back to pinned system memory"; warning_logged = true; } goto pinned_memory_allocation; } break; } pinned_memory_allocation: #endif // TRITON_ENABLE_GPU default: { TRITONSERVER_MemoryType memory_type = buffer_attributes_.MemoryType(); auto status = PinnedMemoryManager::Alloc( (void**)&buffer_, total_byte_size_, &memory_type, true); buffer_attributes_.SetMemoryType(memory_type); if (!status.IsOk()) { LOG_ERROR << status.Message(); buffer_ = nullptr; } break; } } } total_byte_size_ = (buffer_ == nullptr) ? 0 : total_byte_size_; } AllocatedMemory::~AllocatedMemory() { if (buffer_ != nullptr) { switch (buffer_attributes_.MemoryType()) { case TRITONSERVER_MEMORY_GPU: { #ifdef TRITON_ENABLE_GPU auto status = CudaMemoryManager::Free(buffer_, buffer_attributes_.MemoryTypeId()); if (!status.IsOk()) { LOG_ERROR << status.Message(); } #endif // TRITON_ENABLE_GPU break; } default: { auto status = PinnedMemoryManager::Free(buffer_); if (!status.IsOk()) { LOG_ERROR << status.Message(); buffer_ = nullptr; } break; } } buffer_ = nullptr; } } #ifdef TRITON_ENABLE_GPU GrowableMemory::GrowableMemory( size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, std::unique_ptr&& allocation, size_t virtual_address_size) : MutableMemory(nullptr, byte_size, memory_type, memory_type_id) { allocation_prop_.type = CU_MEM_ALLOCATION_TYPE_PINNED; allocation_prop_.location.type = CU_MEM_LOCATION_TYPE_DEVICE; allocation_prop_.location.id = memory_type_id; access_desc_.location = allocation_prop_.location; access_desc_.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; virtual_address_size_ = virtual_address_size; allocation_ = std::move(allocation); virtual_address_offset_ = 0; } #endif Status GrowableMemory::Create( std::unique_ptr& growable_memory, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, size_t virtual_address_size) { #ifdef TRITON_ENABLE_GPU std::unique_ptr allocation = std::make_unique(memory_type_id); // The virtual address size must be a factor of // cudaMinimumAllocationGranularity virtual_address_size = ((virtual_address_size + CudaBlockManager::BlockSize() - 1) / CudaBlockManager::BlockSize()) * CudaBlockManager::BlockSize(); if (memory_type != TRITONSERVER_MEMORY_GPU) { return Status( Status::Code::INVALID_ARG, std::string("Only TRITONSERVER_MEMORY_GPU is supported for growable " "memory.")); } if (byte_size > virtual_address_size) { return Status( Status::Code::INVALID_ARG, std::string("'byte_size' requested for GrowableMemory cannot be smaller" " than the virtual address size. byte_size: ") + std::to_string(byte_size) + ", virtual_address_size:" + std::to_string(virtual_address_size)); } RETURN_IF_ERROR( CudaBlockManager::Allocate(byte_size, allocation, memory_type_id)); void* buffer; RETURN_IF_ERROR(CudaDriverHelper::GetInstance().CuMemAddressReserve( reinterpret_cast(&buffer), virtual_address_size, 0 /* alignment */, 0 /* start_address */, 0 /* flags */)); growable_memory.reset(new GrowableMemory( CudaBlockManager::BlockSize() * allocation->Blocks().size(), memory_type, memory_type_id, std::move(allocation), virtual_address_size)); growable_memory->buffer_ = reinterpret_cast(buffer); for (auto& block : growable_memory->allocation_->Blocks()) { RETURN_IF_ERROR(growable_memory->Map(block)); } return Status::Success; #else return Status( Status::Code::INTERNAL, "The server was build with TRITON_ENABLE_GPU=OFF but growable memory was " "used."); #endif } #ifdef TRITON_ENABLE_GPU Status GrowableMemory::Map(CUmemGenericAllocationHandle& block) { RETURN_IF_ERROR(CudaDriverHelper::GetInstance().CuMemMap( reinterpret_cast(buffer_) + virtual_address_offset_, CudaBlockManager::BlockSize(), 0UL, block, 0UL /* flags */)); RETURN_IF_ERROR(CudaDriverHelper::GetInstance().CuMemSetAccess( reinterpret_cast(buffer_) + virtual_address_offset_, CudaBlockManager::BlockSize(), &access_desc_, 1ULL /* Mapping size */)); virtual_address_offset_ += CudaBlockManager::BlockSize(); return Status::Success; } #endif Status GrowableMemory::Resize(size_t size) { #ifdef TRITON_ENABLE_GPU if (size > virtual_address_size_) { return Status( Status::Code::INVALID_ARG, std::string( "Failed to resize the GrowableMemory. The requested size is larger" " than the virtual page size. requested size: ") + std::to_string(size) + ", virtual_address_size:" + std::to_string(virtual_address_size_)); } if (size < buffer_attributes_.ByteSize()) { return Status::Success; } else { size_t new_size = size - buffer_attributes_.ByteSize(); std::unique_ptr allocation = std::make_unique(buffer_attributes_.MemoryTypeId()); RETURN_IF_ERROR(CudaBlockManager::Allocate( new_size, allocation, buffer_attributes_.MemoryTypeId())); for (auto& block : allocation->Blocks()) { RETURN_IF_ERROR(Map(block)); } allocation_->Merge(std::move(allocation)); } buffer_attributes_.SetByteSize( allocation_->Blocks().size() * CudaBlockManager::BlockSize()); return Status::Success; #else return Status( Status::Code::INTERNAL, "The server was build with TRITON_ENABLE_GPU=OFF but growable memory was " "used."); #endif } #ifdef TRITON_ENABLE_GPU std::unique_ptr& GrowableMemory::GetAllocation() { return allocation_; } #endif GrowableMemory::~GrowableMemory() { #ifdef TRITON_ENABLE_GPU CudaDriverHelper::GetInstance().CuMemUnmap( reinterpret_cast(buffer_), buffer_attributes_.ByteSize()); CudaDriverHelper::GetInstance().CuMemAddressFree( reinterpret_cast(buffer_), virtual_address_size_); #endif } }} // namespace triton::core kytensor-2.49.0.7/core/src/memory.h000066400000000000000000000166651527052371200170530ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "buffer_attributes.h" #include "constants.h" #include "status.h" #ifdef TRITON_ENABLE_GPU #include "cuda_block_manager.h" #endif namespace triton { namespace core { // // Memory used to access data in inference requests // class Memory { public: // Get the 'idx'-th data block in the buffer. Using index to avoid // maintaining internal state such that one buffer can be shared // across multiple providers. // 'idx' zero base index. Valid indices are continuous. // 'byte_size' returns the byte size of the chunk of bytes. // 'memory_type' returns the memory type of the chunk of bytes. // 'memory_type_id' returns the memory type id of the chunk of bytes. // Return the pointer to the data block. Returns nullptr if 'idx' is // out of range virtual const char* BufferAt( size_t idx, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) const = 0; // Similar to the above BufferAt but with BufferAttributes. virtual const char* BufferAt( size_t idx, BufferAttributes** buffer_attributes) = 0; // Get the number of contiguous buffers composing the memory. size_t BufferCount() const { return buffer_count_; } // Return the total byte size of the data buffer size_t TotalByteSize() const { return total_byte_size_; } protected: Memory() : total_byte_size_(0), buffer_count_(0) {} size_t total_byte_size_; size_t buffer_count_; }; // // MemoryReference // class MemoryReference : public Memory { public: // Create a read-only data buffer as a reference to other data buffer MemoryReference(); //\see Memory::BufferAt() const char* BufferAt( size_t idx, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) const override; const char* BufferAt( size_t idx, BufferAttributes** buffer_attributes) override; // Add a 'buffer' with 'byte_size' as part of this data buffer // Return the index of the buffer size_t AddBuffer( const char* buffer, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); size_t AddBuffer(const char* buffer, BufferAttributes* buffer_attributes); // Add a 'buffer' with 'byte_size' as part of this data buffer in the front // Return the index of the buffer size_t AddBufferFront( const char* buffer, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); private: struct Block { Block( const char* buffer, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) : buffer_(buffer), buffer_attributes_(BufferAttributes( byte_size, memory_type, memory_type_id, nullptr)) { } Block(const char* buffer, BufferAttributes* buffer_attributes) : buffer_(buffer), buffer_attributes_(*buffer_attributes) { } const char* buffer_; BufferAttributes buffer_attributes_; }; std::vector buffer_; }; // // MutableMemory // class MutableMemory : public Memory { public: // Create a mutable data buffer referencing to other data buffer. MutableMemory( char* buffer, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); virtual ~MutableMemory() {} //\see Memory::BufferAt() const char* BufferAt( size_t idx, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) const override; //\see Memory::BufferAt() const char* BufferAt( size_t idx, BufferAttributes** buffer_attributes) override; // Return a pointer to the base address of the mutable buffer. If // non-null 'memory_type' returns the memory type of the chunk of // bytes. If non-null 'memory_type_id' returns the memory type id of // the chunk of bytes. char* MutableBuffer( TRITONSERVER_MemoryType* memory_type = nullptr, int64_t* memory_type_id = nullptr); // Set the memory to the specified value. Status SetMemory(unsigned char value); DISALLOW_COPY_AND_ASSIGN(MutableMemory); protected: MutableMemory() : Memory() {} char* buffer_; BufferAttributes buffer_attributes_; }; // // AllocatedMemory // class AllocatedMemory : public MutableMemory { public: // Create a continuous data buffer with 'byte_size', 'memory_type' and // 'memory_type_id'. Note that the buffer may be created on different memory // type and memory type id if the original request type and id can not be // satisfied, thus the function caller should always check the actual memory // type and memory type id before use. AllocatedMemory( size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); ~AllocatedMemory() override; }; class GrowableMemory : public MutableMemory { public: static Status Create( std::unique_ptr& growable_memory, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, size_t virtual_page_size); ~GrowableMemory() override; Status Resize(size_t size); #ifdef TRITON_ENABLE_GPU std::unique_ptr& GetAllocation(); #endif private: // Create a continuous data buffer with 'byte_size', 'memory_type' and // 'memory_type_id'. Note that the buffer may be created on different memory // type and memory type id if the original request type and id can not be // satisfied, thus the function caller should always check the actual memory // type and memory type id before use. #ifdef TRITON_ENABLE_GPU GrowableMemory( size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, std::unique_ptr&& allocation, size_t virtual_page_size); Status Map(CUmemGenericAllocationHandle& block); std::unique_ptr allocation_; size_t virtual_address_offset_; CUmemAllocationProp allocation_prop_; CUmemAccessDesc access_desc_; #endif size_t virtual_address_size_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/metric_family.cc000066400000000000000000000320661527052371200205160ustar00rootroot00000000000000// Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights // reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef TRITON_ENABLE_METRICS #include "metric_family.h" #include "metrics.h" #include "triton/common/logging.h" namespace triton { namespace core { // // Implementation for TRITONSERVER_MetricFamily. // MetricFamily::MetricFamily( TRITONSERVER_MetricKind kind, const char* name, const char* description) { auto registry = Metrics::GetRegistry(); switch (kind) { case TRITONSERVER_METRIC_KIND_COUNTER: family_ = reinterpret_cast(&prometheus::BuildCounter() .Name(name) .Help(description) .Register(*registry)); break; case TRITONSERVER_METRIC_KIND_GAUGE: family_ = reinterpret_cast(&prometheus::BuildGauge() .Name(name) .Help(description) .Register(*registry)); break; case TRITONSERVER_METRIC_KIND_HISTOGRAM: family_ = reinterpret_cast(&prometheus::BuildHistogram() .Name(name) .Help(description) .Register(*registry)); break; default: throw std::invalid_argument( "Unsupported kind passed to MetricFamily constructor."); } kind_ = kind; } void* MetricFamily::Add( std::map label_map, Metric* metric, const TritonServerMetricArgs* args) { void* prom_metric = nullptr; switch (kind_) { case TRITONSERVER_METRIC_KIND_COUNTER: { if (args != nullptr) { throw std::invalid_argument( "Unexpected args found in counter Metric constructor."); } auto counter_family_ptr = reinterpret_cast*>(family_); auto counter_ptr = &counter_family_ptr->Add(label_map); prom_metric = reinterpret_cast(counter_ptr); break; } case TRITONSERVER_METRIC_KIND_GAUGE: { if (args != nullptr) { throw std::invalid_argument( "Unexpected args found in gauge Metric constructor."); } auto gauge_family_ptr = reinterpret_cast*>(family_); auto gauge_ptr = &gauge_family_ptr->Add(label_map); prom_metric = reinterpret_cast(gauge_ptr); break; } case TRITONSERVER_METRIC_KIND_HISTOGRAM: { if (args == nullptr) { throw std::invalid_argument( "Bucket boundaries not found in Metric args."); } if (args->kind() != TRITONSERVER_METRIC_KIND_HISTOGRAM) { throw std::invalid_argument("Metric args not set to histogram kind."); } auto histogram_family_ptr = reinterpret_cast*>(family_); auto histogram_ptr = &histogram_family_ptr->Add(label_map, args->buckets()); prom_metric = reinterpret_cast(histogram_ptr); break; } default: throw std::invalid_argument( "Unsupported family kind passed to Metric constructor."); } std::lock_guard lk(metric_mtx_); ++prom_metric_ref_cnt_[prom_metric]; child_metrics_.insert(metric); return prom_metric; } void MetricFamily::Remove(void* prom_metric, Metric* metric) { { // Remove reference to dependent Metric object std::lock_guard lk(metric_mtx_); child_metrics_.erase(metric); } if (prom_metric == nullptr) { return; } { std::lock_guard lk(metric_mtx_); const auto it = prom_metric_ref_cnt_.find(prom_metric); if (it != prom_metric_ref_cnt_.end()) { --it->second; if (it->second == 0) { prom_metric_ref_cnt_.erase(it); } else { // Done as it is not the last reference return; } } } switch (kind_) { case TRITONSERVER_METRIC_KIND_COUNTER: { auto counter_family_ptr = reinterpret_cast*>(family_); auto counter_ptr = reinterpret_cast(prom_metric); counter_family_ptr->Remove(counter_ptr); break; } case TRITONSERVER_METRIC_KIND_GAUGE: { auto gauge_family_ptr = reinterpret_cast*>(family_); auto gauge_ptr = reinterpret_cast(prom_metric); gauge_family_ptr->Remove(gauge_ptr); break; } case TRITONSERVER_METRIC_KIND_HISTOGRAM: { auto histogram_family_ptr = reinterpret_cast*>(family_); auto histogram_ptr = reinterpret_cast(prom_metric); histogram_family_ptr->Remove(histogram_ptr); break; } default: // Invalid kind should be caught in constructor LOG_ERROR << "Unsupported kind in Metric destructor."; break; } } void MetricFamily::InvalidateReferences() { std::lock_guard lk(metric_mtx_); for (auto& metric : child_metrics_) { if (metric != nullptr) { metric->Invalidate(); } } child_metrics_.clear(); } MetricFamily::~MetricFamily() { if (NumMetrics() > 0) { LOG_WARNING << "MetricFamily was deleted before its child Metrics, this " "should not happen. Make sure to delete all child Metrics " "before deleting their MetricFamily."; } InvalidateReferences(); // DLIS-4072: Support for removing metric families from registry } // // Implementation for TRITONSERVER_Metric. // Metric::Metric( TRITONSERVER_MetricFamily* family, std::vector labels, const TritonServerMetricArgs* args) { family_ = reinterpret_cast(family); kind_ = family_->Kind(); // Create map of labels from InferenceParameters std::map label_map; for (const auto& param : labels) { if (param->Type() != TRITONSERVER_PARAMETER_STRING) { throw std::invalid_argument( "Parameter [" + param->Name() + "] must have a type of TRITONSERVER_PARAMETER_STRING to be " "added as a label."); } label_map[param->Name()] = std::string(reinterpret_cast(param->ValuePointer())); } metric_ = family_->Add(label_map, this, args); } Metric::~Metric() { if (family_ != nullptr) { family_->Remove(metric_, this); } else { LOG_WARNING << "Corresponding MetricFamily was deleted before this Metric, " "this should not happen. Make sure to delete a Metric " "before deleting its MetricFamily."; } // Catch lifetime management / invalid reference issues Invalidate(); } void Metric::Invalidate() { family_ = nullptr; metric_ = nullptr; } TRITONSERVER_Error* Metric::Value(double* value) { if (metric_ == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Could not get metric value. Metric has been invalidated."); } switch (kind_) { case TRITONSERVER_METRIC_KIND_COUNTER: { auto counter_ptr = reinterpret_cast(metric_); LOG_VERBOSE(1) << "SETTING COUNTER METRIC FROM: " << *value << " to " << counter_ptr->Value(); *value = counter_ptr->Value(); break; } case TRITONSERVER_METRIC_KIND_GAUGE: { auto gauge_ptr = reinterpret_cast(metric_); LOG_VERBOSE(1) << "SETTING GAUGE METRIC FROM: " << *value << " to " << gauge_ptr->Value(); *value = gauge_ptr->Value(); break; } case TRITONSERVER_METRIC_KIND_HISTOGRAM: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "TRITONSERVER_METRIC_KIND_HISTOGRAM does not support Value"); } default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Unsupported TRITONSERVER_MetricKind"); } return nullptr; // Success } TRITONSERVER_Error* Metric::Increment(double value) { if (metric_ == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Could not increment metric value. Metric has been invalidated."); } switch (kind_) { case TRITONSERVER_METRIC_KIND_COUNTER: { if (value < 0.0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "TRITONSERVER_METRIC_KIND_COUNTER can only be incremented " "monotonically by non-negative values."); } auto counter_ptr = reinterpret_cast(metric_); counter_ptr->Increment(value); break; } case TRITONSERVER_METRIC_KIND_GAUGE: { auto gauge_ptr = reinterpret_cast(metric_); // Gauge::Increment works for both positive and negative values as of // prometheus-cpp v1.0 but for now on v0.7 we defer call to // Increment/Decrement based on the sign of value // https://github.com/jupp0r/prometheus-cpp/blob/master/core/src/gauge.cc if (value < 0.0) { gauge_ptr->Decrement(-1.0 * value); } else { gauge_ptr->Increment(value); } break; } case TRITONSERVER_METRIC_KIND_HISTOGRAM: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "TRITONSERVER_METRIC_KIND_HISTOGRAM does not support Increment"); } default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Unsupported TRITONSERVER_MetricKind"); } return nullptr; // Success } TRITONSERVER_Error* Metric::Set(double value) { if (metric_ == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Could not set metric value. Metric has been invalidated."); } switch (kind_) { case TRITONSERVER_METRIC_KIND_COUNTER: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "TRITONSERVER_METRIC_KIND_COUNTER does not support Set"); } case TRITONSERVER_METRIC_KIND_GAUGE: { auto gauge_ptr = reinterpret_cast(metric_); gauge_ptr->Set(value); break; } case TRITONSERVER_METRIC_KIND_HISTOGRAM: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "TRITONSERVER_METRIC_KIND_HISTOGRAM does not support Set"); } default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Unsupported TRITONSERVER_MetricKind"); } return nullptr; // Success } TRITONSERVER_Error* Metric::Observe(double value) { if (metric_ == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Could not set metric value. Metric has been invalidated."); } switch (kind_) { case TRITONSERVER_METRIC_KIND_COUNTER: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "TRITONSERVER_METRIC_KIND_COUNTER does not support Observe"); } case TRITONSERVER_METRIC_KIND_GAUGE: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "TRITONSERVER_METRIC_KIND_GAUGE does not support Observe"); } case TRITONSERVER_METRIC_KIND_HISTOGRAM: { auto histogram_ptr = reinterpret_cast(metric_); histogram_ptr->Observe(value); break; } default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Unsupported TRITONSERVER_MetricKind"); } return nullptr; // Success } }} // namespace triton::core #endif // TRITON_ENABLE_METRICS kytensor-2.49.0.7/core/src/metric_family.h000066400000000000000000000111751527052371200203560ustar00rootroot00000000000000// Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights // reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef TRITON_ENABLE_METRICS #include #include #include #include #include "infer_parameter.h" #include "prometheus/registry.h" #include "tritonserver_apis.h" namespace triton { namespace core { // // TritonServerMetricArgs // // Implementation for TRITONSERVER_MetricArgs. // class TritonServerMetricArgs { public: TritonServerMetricArgs() = default; void* SetHistogramArgs(const double* buckets, uint64_t bucket_count) { kind_ = TRITONSERVER_METRIC_KIND_HISTOGRAM; buckets_ = std::vector(buckets, buckets + bucket_count); return nullptr; } TRITONSERVER_MetricKind kind() const { return kind_; } const std::vector& buckets() const { return buckets_; } private: TRITONSERVER_MetricKind kind_; std::vector buckets_; }; // // Implementation for TRITONSERVER_MetricFamily. // class Metric; class MetricFamily { public: MetricFamily( TRITONSERVER_MetricKind kind, const char* name, const char* description); ~MetricFamily(); void* Family() const { return family_; } TRITONSERVER_MetricKind Kind() const { return kind_; } void* Add( std::map label_map, Metric* metric, const TritonServerMetricArgs* args); void Remove(void* prom_metric, Metric* metric); int NumMetrics() { std::lock_guard lk(metric_mtx_); return child_metrics_.size(); } private: // If a MetricFamily is deleted before its dependent Metric, we want to // invalidate the reference so we don't access invalid memory. void InvalidateReferences(); void* family_; TRITONSERVER_MetricKind kind_; // Synchronize access of related metric objects std::mutex metric_mtx_; // Prometheus returns the existing metric pointer if the metric with the same // set of labels are requested, as a result, different Metric objects may // refer to the same prometheus metric. So we must track the reference count // of the metric and request prometheus to remove it only when all references // are released. std::unordered_map prom_metric_ref_cnt_; // Maintain references to metrics created from this metric family to // invalidate their references if a family is deleted before its metric std::set child_metrics_; }; // // Implementation for TRITONSERVER_Metric. // class Metric { public: Metric( TRITONSERVER_MetricFamily* family, std::vector labels, const TritonServerMetricArgs* args); ~Metric(); MetricFamily* Family() const { return family_; } TRITONSERVER_MetricKind Kind() const { return kind_; } TRITONSERVER_Error* Value(double* value); TRITONSERVER_Error* Increment(double value); TRITONSERVER_Error* Set(double value); TRITONSERVER_Error* Observe(double value); // If a MetricFamily is deleted before its dependent Metric, we want to // invalidate the references so we don't access invalid memory. void Invalidate(); private: void* metric_; MetricFamily* family_; TRITONSERVER_MetricKind kind_; }; }} // namespace triton::core #endif // TRITON_ENABLE_METRICS kytensor-2.49.0.7/core/src/metric_model_reporter.cc000066400000000000000000000326121527052371200222540ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "metric_model_reporter.h" #ifdef TRITON_ENABLE_METRICS #include "constants.h" #include "infer_stats.h" #include "triton/common/logging.h" // Global config group has 'name' of empty string. constexpr char GLOBAL_CONFIG_GROUP[] = ""; namespace triton { namespace core { // // MetricReporterConfig // void MetricReporterConfig::ParseConfig(bool response_cache_enabled) { // Global config only for now in config map auto metrics_config_map = Metrics::ConfigMap(); const auto& metrics_config = metrics_config_map[GLOBAL_CONFIG_GROUP]; // Default behavior is counters for most latency metrics if no types specified for (const auto& pair : metrics_config) { if (pair.first == "counter_latencies" && pair.second == "false") { latency_counters_enabled_ = false; } if (pair.first == "summary_latencies" && pair.second == "true") { latency_summaries_enabled_ = true; } // ex: summary_quantiles="0.5:0.05 0.9:0.01 0.99:0.001" if (pair.first == "summary_quantiles") { const auto& quantiles = ParseQuantiles(pair.second); if (!quantiles.empty()) { quantiles_ = quantiles; } } } // Set flag to signal to stats aggregator if caching is enabled or not cache_enabled_ = response_cache_enabled; } prometheus::Summary::Quantiles MetricReporterConfig::ParseQuantiles(std::string options) { prometheus::Summary::Quantiles qpairs; std::stringstream ss(options); std::string pairStr; while (std::getline(ss, pairStr, ',')) { size_t colonPos = pairStr.find(':'); if (colonPos == std::string::npos) { LOG_ERROR << "Invalid option: [" << pairStr << "]. No ':' delimiter found. Expected format is :"; continue; } try { double quantile = std::stod(pairStr.substr(0, colonPos)); double error = std::stod(pairStr.substr(colonPos + 1)); qpairs.push_back({quantile, error}); } catch (const std::invalid_argument& e) { LOG_ERROR << "Invalid option: [" << pairStr << "]. Error: " << e.what(); continue; } } return qpairs; } // // MetricModelReporter // const std::map MetricModelReporter::failure_reasons_map = { {FailureReason::REJECTED, "REJECTED"}, {FailureReason::CANCELED, "CANCELED"}, {FailureReason::BACKEND, "BACKEND"}, {FailureReason::OTHER, "OTHER"}}; Status MetricModelReporter::Create( const ModelIdentifier& model_id, const int64_t model_version, const int device, bool response_cache_enabled, const triton::common::MetricTagsMap& model_tags, std::shared_ptr* metric_model_reporter) { static std::mutex mtx; static std::unordered_map> reporter_map; std::map labels; GetMetricLabels(&labels, model_id, model_version, device, model_tags); auto hash_labels = Metrics::HashLabels(labels); std::lock_guard lock(mtx); const auto& itr = reporter_map.find(hash_labels); if (itr != reporter_map.end()) { // Found in map. If the weak_ptr is still valid that means that // there are other models using the reporter and we just reuse that // same reporter. If the weak_ptr is not valid then we need to remove // the weak_ptr from the map and create the reporter again. *metric_model_reporter = itr->second.lock(); if (*metric_model_reporter != nullptr) { return Status::Success; } reporter_map.erase(itr); } metric_model_reporter->reset(new MetricModelReporter( model_id, model_version, device, response_cache_enabled, model_tags)); reporter_map.insert({hash_labels, *metric_model_reporter}); return Status::Success; } MetricModelReporter::MetricModelReporter( const ModelIdentifier& model_id, const int64_t model_version, const int device, bool response_cache_enabled, const triton::common::MetricTagsMap& model_tags) { std::map labels; GetMetricLabels(&labels, model_id, model_version, device, model_tags); // Parse metrics config to control metric setup and behavior config_.ParseConfig(response_cache_enabled); // Initialize families and metrics InitializeCounters(labels); InitializeGauges(labels); InitializeSummaries(labels); } MetricModelReporter::~MetricModelReporter() { // Cleanup metrics for each family for (auto& iter : counter_families_) { const auto& name = iter.first; auto family_ptr = iter.second; if (family_ptr) { family_ptr->Remove(counters_[name]); } } for (auto& iter : gauge_families_) { const auto& name = iter.first; auto family_ptr = iter.second; if (family_ptr) { family_ptr->Remove(gauges_[name]); } } for (auto& iter : summary_families_) { const auto& name = iter.first; auto family_ptr = iter.second; if (family_ptr) { family_ptr->Remove(summaries_[name]); } } } void MetricModelReporter::InitializeCounters( const std::map& labels) { // Always setup these counters, regardless of config counter_families_["inf_success"] = &Metrics::FamilyInferenceSuccess(); counter_families_["inf_count"] = &Metrics::FamilyInferenceCount(); counter_families_["inf_exec_count"] = &Metrics::FamilyInferenceExecutionCount(); // Latency metrics will be initialized based on config if (config_.latency_counters_enabled_) { // Request counter_families_["request_duration"] = &Metrics::FamilyInferenceRequestDuration(); counter_families_["queue_duration"] = &Metrics::FamilyInferenceQueueDuration(); // Compute counter_families_["compute_input_duration"] = &Metrics::FamilyInferenceComputeInputDuration(); counter_families_["compute_infer_duration"] = &Metrics::FamilyInferenceComputeInferDuration(); counter_families_["compute_output_duration"] = &Metrics::FamilyInferenceComputeOutputDuration(); // Only create cache metrics if cache is enabled to reduce metric output if (config_.cache_enabled_) { counter_families_["cache_hit_count"] = &Metrics::FamilyCacheHitCount(); counter_families_["cache_miss_count"] = &Metrics::FamilyCacheMissCount(); counter_families_["cache_hit_duration"] = &Metrics::FamilyCacheHitDuration(); counter_families_["cache_miss_duration"] = &Metrics::FamilyCacheMissDuration(); } } // Create metrics for each family for (auto& iter : counter_families_) { const auto& name = iter.first; auto family_ptr = iter.second; if (family_ptr) { counters_[name] = CreateMetric(*family_ptr, labels); } } // Initialize failure metrics with reasons for (const auto& reason_pair : failure_reasons_map) { std::map extended_labels = labels; extended_labels["reason"] = reason_pair.second; counters_["inf_failure_" + reason_pair.second] = CreateMetric( Metrics::FamilyInferenceFailure(), extended_labels); } } void MetricModelReporter::InitializeGauges( const std::map& labels) { // Always setup these inference request metrics, regardless of config gauge_families_[kPendingRequestMetric] = &Metrics::FamilyInferenceQueueSize(); for (auto& iter : gauge_families_) { const auto& name = iter.first; auto family_ptr = iter.second; if (family_ptr) { gauges_[name] = CreateMetric(*family_ptr, labels); } } } void MetricModelReporter::InitializeSummaries( const std::map& labels) { // Latency metrics will be initialized based on config if (config_.latency_summaries_enabled_) { // Request if (!config_.cache_enabled_) { // FIXME: request_duration summary is currently disabled when cache is // enabled to avoid publishing misleading metrics. summary_families_["request_duration"] = &Metrics::FamilyInferenceRequestSummary(); } summary_families_["queue_duration"] = &Metrics::FamilyInferenceQueueSummary(); // Compute summary_families_["compute_input_duration"] = &Metrics::FamilyInferenceComputeInputSummary(); summary_families_["compute_infer_duration"] = &Metrics::FamilyInferenceComputeInferSummary(); summary_families_["compute_output_duration"] = &Metrics::FamilyInferenceComputeOutputSummary(); // Only create cache metrics if cache is enabled to reduce metric output if (config_.cache_enabled_) { // Note that counts and sums are included in summaries summary_families_["cache_hit_duration"] = &Metrics::FamilyCacheHitSummary(); summary_families_["cache_miss_duration"] = &Metrics::FamilyCacheMissSummary(); } } // Create metrics for each family for (auto& iter : summary_families_) { const auto& name = iter.first; auto family_ptr = iter.second; if (family_ptr) { summaries_[name] = CreateMetric( *family_ptr, labels, config_.quantiles_); } } } void MetricModelReporter::GetMetricLabels( std::map* labels, const ModelIdentifier& model_id, const int64_t model_version, const int device, const triton::common::MetricTagsMap& model_tags) { if (!model_id.NamespaceDisabled()) { labels->insert(std::map::value_type( std::string(kMetricsLabelModelNamespace), model_id.namespace_)); } labels->insert(std::map::value_type( std::string(kMetricsLabelModelName), model_id.name_)); labels->insert(std::map::value_type( std::string(kMetricsLabelModelVersion), std::to_string(model_version))); for (const auto& tag : model_tags) { labels->insert(std::map::value_type( "_" + tag.first, tag.second)); } // 'device' can be < 0 to indicate that the GPU is not known. In // that case use a metric that doesn't have the gpu_uuid label. if (device >= 0) { std::string uuid; if (Metrics::UUIDForCudaDevice(device, &uuid)) { labels->insert(std::map::value_type( std::string(kMetricsLabelGpuUuid), uuid)); } } } template T* MetricModelReporter::CreateMetric( prometheus::Family& family, const std::map& labels, Args&&... args) { return &family.Add(labels, args...); } const MetricReporterConfig& MetricModelReporter::Config() { return config_; } void MetricModelReporter::IncrementCounter(const std::string& name, double value) { if (!config_.latency_counters_enabled_) { return; } auto iter = counters_.find(name); if (iter == counters_.end()) { // No counter metric exists with this name return; } auto counter = iter->second; if (!counter) { // Counter is uninitialized/nullptr return; } counter->Increment(value); } prometheus::Gauge* MetricModelReporter::GetGauge(const std::string& name) { auto iter = gauges_.find(name); if (iter == gauges_.end()) { // No gauge metric exists with this name return nullptr; } auto gauge = iter->second; return gauge; } void MetricModelReporter::IncrementGauge(const std::string& name, double value) { auto gauge = GetGauge(name); if (gauge) { gauge->Increment(value); } } void MetricModelReporter::DecrementGauge(const std::string& name, double value) { IncrementGauge(name, -1 * value); } void MetricModelReporter::ObserveSummary(const std::string& name, double value) { if (!config_.latency_summaries_enabled_) { return; } auto iter = summaries_.find(name); if (iter == summaries_.end()) { // No summary metric exists with this name return; } auto summary = iter->second; if (!summary) { // Summary is uninitialized/nullptr return; } summary->Observe(value); } }} // namespace triton::core #endif // TRITON_ENABLE_METRICS kytensor-2.49.0.7/core/src/metric_model_reporter.h000066400000000000000000000133021527052371200221110ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "status.h" #include "triton/common/model_config.h" #ifdef TRITON_ENABLE_METRICS #include "metrics.h" #include "model.h" #include "prometheus/registry.h" #endif // TRITON_ENABLE_METRICS namespace triton { namespace core { #ifdef TRITON_ENABLE_METRICS struct ModelIdentifier; #endif // TRITON_ENABLE_METRICS // // MetricReporterConfig // struct MetricReporterConfig { #ifdef TRITON_ENABLE_METRICS // Parses Metrics::ConfigMap and sets fields if specified void ParseConfig(bool response_cache_enabled); // Parses pairs of quantiles "quantile1:error1, quantile2:error2, ..." // and overwrites quantiles_ field if successful. prometheus::Summary::Quantiles ParseQuantiles(std::string options); // Create and use Counters for per-model latency related metrics bool latency_counters_enabled_ = true; // Create and use Summaries for per-model latency related metrics bool latency_summaries_enabled_ = false; // Quantiles used for any summary metrics. Each pair of values represents // { quantile, error }. For example, {0.90, 0.01} means to compute the // 90th percentile with 1% error on either side, so the approximate 90th // percentile value will be between the 89th and 91st percentiles. prometheus::Summary::Quantiles quantiles_ = { {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.001}, {0.99, 0.001}, {0.999, 0.001}}; // Whether this reporter's model has caching enabled or not. // This helps handle infer_stats aggregation for summaries on cache misses. bool cache_enabled_ = false; #endif // TRITON_ENABLE_METRICS }; // // Interface for a metric reporter for a given version of a model. // class MetricModelReporter { public: #ifdef TRITON_ENABLE_METRICS static Status Create( const triton::core::ModelIdentifier& model_id, const int64_t model_version, const int device, bool response_cache_enabled, const triton::common::MetricTagsMap& model_tags, std::shared_ptr* metric_model_reporter); ~MetricModelReporter(); // Get this reporter's config const MetricReporterConfig& Config(); // Lookup counter metric by name, and increment it by value if it exists. void IncrementCounter(const std::string& name, double value); // Increase gauge by value. void IncrementGauge(const std::string& name, double value); // Decrease gauge by value. void DecrementGauge(const std::string& name, double value); // Lookup summary metric by name, and observe the value if it exists. void ObserveSummary(const std::string& name, double value); static const std::map failure_reasons_map; private: MetricModelReporter( const ModelIdentifier& model_id, const int64_t model_version, const int device, bool response_cache_enabled, const triton::common::MetricTagsMap& model_tags); static void GetMetricLabels( std::map* labels, const ModelIdentifier& model_id, const int64_t model_version, const int device, const triton::common::MetricTagsMap& model_tags); template T* CreateMetric( prometheus::Family& family, const std::map& labels, Args&&... args); void InitializeCounters(const std::map& labels); void InitializeGauges(const std::map& labels); void InitializeSummaries(const std::map& labels); // Lookup gauge metric by name. Return gauge if found, nullptr otherwise. prometheus::Gauge* GetGauge(const std::string& name); // Metric Families std::unordered_map*> counter_families_; std::unordered_map*> gauge_families_; std::unordered_map*> summary_families_; // Metrics std::unordered_map counters_; std::unordered_map gauges_; std::unordered_map summaries_; // Config MetricReporterConfig config_; #endif // TRITON_ENABLE_METRICS }; }} // namespace triton::core kytensor-2.49.0.7/core/src/metrics.cc000066400000000000000000001114511527052371200173340ustar00rootroot00000000000000// Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #ifdef TRITON_ENABLE_METRICS #include "metrics.h" #include #include "constants.h" #include "pinned_memory_manager.h" #include "prometheus/detail/utils.h" #include "triton/common/logging.h" #ifdef TRITON_ENABLE_METRICS_GPU #include #include #include #include #include #endif // TRITON_ENABLE_METRICS_GPU namespace triton { namespace core { Metrics::Metrics() : registry_(std::make_shared()), serializer_(new prometheus::TextSerializer()), inf_success_family_( prometheus::BuildCounter() .Name("nv_inference_request_success") .Help("Number of successful inference requests, all batch sizes") .Register(*registry_)), inf_failure_family_( prometheus::BuildCounter() .Name("nv_inference_request_failure") .Help("Number of failed inference requests, all batch sizes") .Register(*registry_)), inf_count_family_(prometheus::BuildCounter() .Name("nv_inference_count") .Help("Number of inferences performed (does not " "include cached requests)") .Register(*registry_)), inf_count_exec_family_(prometheus::BuildCounter() .Name("nv_inference_exec_count") .Help("Number of model executions performed " "(does not include cached requests)") .Register(*registry_)), inf_request_duration_us_family_( prometheus::BuildCounter() .Name("nv_inference_request_duration_us") .Help("Cumulative inference request duration in microseconds " "(includes cached requests)") .Register(*registry_)), inf_queue_duration_us_family_( prometheus::BuildCounter() .Name("nv_inference_queue_duration_us") .Help("Cumulative inference queuing duration in microseconds " "(includes cached requests)") .Register(*registry_)), inf_compute_input_duration_us_family_( prometheus::BuildCounter() .Name("nv_inference_compute_input_duration_us") .Help("Cumulative compute input duration in microseconds (does " "not include cached requests)") .Register(*registry_)), inf_compute_infer_duration_us_family_( prometheus::BuildCounter() .Name("nv_inference_compute_infer_duration_us") .Help("Cumulative compute inference duration in microseconds " "(does not include cached requests)") .Register(*registry_)), inf_compute_output_duration_us_family_( prometheus::BuildCounter() .Name("nv_inference_compute_output_duration_us") .Help("Cumulative inference compute output duration in " "microseconds (does not include cached requests)") .Register(*registry_)), inf_pending_request_count_family_( prometheus::BuildGauge() .Name("nv_inference_pending_request_count") .Help("Instantaneous number of pending requests awaiting " "execution per-model.") .Register(*registry_)), pinned_memory_pool_total_family_( prometheus::BuildGauge() .Name("nv_pinned_memory_pool_total_bytes") .Help("Pinned memory pool total memory size, in bytes") .Register(*registry_)), pinned_memory_pool_used_family_( prometheus::BuildGauge() .Name("nv_pinned_memory_pool_used_bytes") .Help("Pinned memory pool used memory size, in bytes") .Register(*registry_)), // Per-model cache metric families // NOTE: These are used in infer_stats for perf_analyzer cache_num_hits_model_family_(prometheus::BuildCounter() .Name("nv_cache_num_hits_per_model") .Help("Number of cache hits per model") .Register(*registry_)), cache_hit_duration_us_model_family_( prometheus::BuildCounter() .Name("nv_cache_hit_duration_per_model") .Help("Total cache hit duration per model, in microseconds") .Register(*registry_)), cache_num_misses_model_family_( prometheus::BuildCounter() .Name("nv_cache_num_misses_per_model") .Help("Number of cache misses per model") .Register(*registry_)), cache_miss_duration_us_model_family_( prometheus::BuildCounter() .Name("nv_cache_miss_duration_per_model") .Help("Total cache miss (insert+lookup) duration per model, in " "microseconds") .Register(*registry_)), // Summaries inf_request_summary_us_family_( prometheus::BuildSummary() .Name("nv_inference_request_summary_us") .Help("Summary of inference request duration in microseconds " "(includes cached requests)") .Register(*registry_)), inf_queue_summary_us_family_( prometheus::BuildSummary() .Name("nv_inference_queue_summary_us") .Help("Summary of inference queuing duration in microseconds " "(includes cached requests)") .Register(*registry_)), inf_compute_input_summary_us_family_( prometheus::BuildSummary() .Name("nv_inference_compute_input_summary_us") .Help("Cumulative compute input duration in microseconds (does " "not include cached requests)") .Register(*registry_)), inf_compute_infer_summary_us_family_( prometheus::BuildSummary() .Name("nv_inference_compute_infer_summary_us") .Help("Cumulative compute inference duration in microseconds " "(does not include cached requests)") .Register(*registry_)), inf_compute_output_summary_us_family_( prometheus::BuildSummary() .Name("nv_inference_compute_output_summary_us") .Help("Cumulative inference compute output duration in " "microseconds (does not include cached requests)") .Register(*registry_)), cache_hit_summary_us_model_family_( prometheus::BuildSummary() .Name("nv_cache_hit_summary_per_model") .Help("Summary of cache hit counts/durations per model, in " "microseconds.") .Register(*registry_)), cache_miss_summary_us_model_family_( prometheus::BuildSummary() .Name("nv_cache_miss_summary_per_model") .Help("Summary of cache miss counts/durations per model, in " "microseconds.") .Register(*registry_)), #ifdef TRITON_ENABLE_METRICS_GPU gpu_utilization_family_(prometheus::BuildGauge() .Name("nv_gpu_utilization") .Help("GPU utilization rate [0.0 - 1.0)") .Register(*registry_)), gpu_memory_total_family_(prometheus::BuildGauge() .Name("nv_gpu_memory_total_bytes") .Help("GPU total memory, in bytes") .Register(*registry_)), gpu_memory_used_family_(prometheus::BuildGauge() .Name("nv_gpu_memory_used_bytes") .Help("GPU used memory, in bytes") .Register(*registry_)), gpu_power_usage_family_(prometheus::BuildGauge() .Name("nv_gpu_power_usage") .Help("GPU power usage in watts") .Register(*registry_)), gpu_power_limit_family_(prometheus::BuildGauge() .Name("nv_gpu_power_limit") .Help("GPU power management limit in watts") .Register(*registry_)), gpu_energy_consumption_family_( prometheus::BuildCounter() .Name("nv_energy_consumption") .Help("GPU energy consumption in joules since the Triton Server " "started") .Register(*registry_)), #endif // TRITON_ENABLE_METRICS_GPU #ifdef TRITON_ENABLE_METRICS_CPU cpu_utilization_family_(prometheus::BuildGauge() .Name("nv_cpu_utilization") .Help("CPU utilization rate [0.0 - 1.0]") .Register(*registry_)), cpu_memory_total_family_(prometheus::BuildGauge() .Name("nv_cpu_memory_total_bytes") .Help("CPU total memory (RAM), in bytes") .Register(*registry_)), cpu_memory_used_family_(prometheus::BuildGauge() .Name("nv_cpu_memory_used_bytes") .Help("CPU used memory (RAM), in bytes") .Register(*registry_)), #endif // TRITON_ENABLE_METRICS_CPU metrics_enabled_(false), gpu_metrics_enabled_(false), cpu_metrics_enabled_(false), pinned_memory_metrics_enabled_(false), metrics_interval_ms_(2000) { } static prometheus::detail::LabelHasher label_hasher_; size_t Metrics::HashLabels(const std::map& labels) { return label_hasher_(labels); } Metrics::~Metrics() { // Signal the poll thread to exit and then wait for it... if (poll_thread_ != nullptr) { poll_thread_exit_.store(true); poll_thread_->join(); #ifdef TRITON_ENABLE_METRICS_GPU if (dcgm_metadata_.dcgm_initialized_) { dcgmReturn_t derr; // Group destroy will return an error if groupId invalid or dcgm not // initialized or configured correctly derr = dcgmGroupDestroy( dcgm_metadata_.dcgm_handle_, dcgm_metadata_.groupId_); if (derr != DCGM_ST_OK) { LOG_WARNING << "Unable to destroy DCGM group: " << errorString(derr); } // Stop and shutdown DCGM if (dcgm_metadata_.standalone_) { derr = dcgmDisconnect(dcgm_metadata_.dcgm_handle_); } else { derr = dcgmStopEmbedded(dcgm_metadata_.dcgm_handle_); } if (derr != DCGM_ST_OK) { LOG_WARNING << "Unable to stop DCGM: " << errorString(derr); } derr = dcgmShutdown(); if (derr != DCGM_ST_OK) { LOG_WARNING << "Unable to shutdown DCGM: " << errorString(derr); } } #endif // TRITON_ENABLE_METRICS_GPU } } const MetricsConfigMap& Metrics::ConfigMap() { auto singleton = GetSingleton(); return singleton->config_; } void Metrics::SetConfigMap(MetricsConfigMap cfg) { auto singleton = GetSingleton(); singleton->config_ = cfg; } bool Metrics::Enabled() { auto singleton = GetSingleton(); return singleton->metrics_enabled_; } void Metrics::EnableMetrics() { auto singleton = GetSingleton(); singleton->metrics_enabled_ = true; EnablePinnedMemoryMetrics(); } void Metrics::EnablePinnedMemoryMetrics() { auto singleton = GetSingleton(); if (singleton->pinned_memory_metrics_enabled_) { return; } // Initialize singleton->InitializePinnedMemoryMetrics(); singleton->pinned_memory_metrics_enabled_ = true; } void Metrics::EnableGPUMetrics() { auto singleton = GetSingleton(); // Ensure thread-safe enabling of GPU Metrics std::lock_guard lock(singleton->metrics_enabling_); if (singleton->gpu_metrics_enabled_) { return; } if (std::getenv("TRITON_SERVER_CPU_ONLY") == nullptr) { singleton->InitializeDcgmMetrics(); } singleton->gpu_metrics_enabled_ = true; } void Metrics::EnableCpuMetrics() { auto singleton = GetSingleton(); // Ensure thread-safe enabling of CPU Metrics std::lock_guard lock(singleton->metrics_enabling_); if (singleton->cpu_metrics_enabled_) { return; } singleton->InitializeCpuMetrics(); singleton->cpu_metrics_enabled_ = true; } void Metrics::SetMetricsInterval(uint64_t metrics_interval_ms) { auto singleton = GetSingleton(); singleton->metrics_interval_ms_ = metrics_interval_ms; } void Metrics::StartPollingThreadSingleton() { auto singleton = GetSingleton(); // Ensure thread-safe start of polling thread std::lock_guard lock(singleton->poll_thread_starting_); if (singleton->poll_thread_started_) { return; } // Start thread for polling cpu/gpu metrics singleton->StartPollingThread(); // Toggle flag so this function is only executed once singleton->poll_thread_started_ = true; } bool Metrics::StartPollingThread() { // Nothing to poll if no polling metrics enabled, don't spawn a thread if (!gpu_metrics_enabled_ && !cpu_metrics_enabled_ && !pinned_memory_metrics_enabled_) { LOG_WARNING << "No polling metrics (CPU, GPU, Pinned memory) are " "enabled. Will not " "poll for them."; return false; } poll_thread_exit_.store(false); // Start a separate thread for polling metrics at specified interval poll_thread_.reset(new std::thread([this] { // Thread will update metrics indefinitely until exit flag set while (!poll_thread_exit_.load()) { // Sleep for metric interval std::this_thread::sleep_for( std::chrono::milliseconds(metrics_interval_ms_ / 2)); if (pinned_memory_metrics_enabled_) { PollPinnedMemoryMetrics(); } #ifdef TRITON_ENABLE_METRICS_GPU // Poll DCGM GPU metrics if (gpu_metrics_enabled_ && dcgm_metadata_.available_cuda_gpu_ids_.size() > 0) { PollDcgmMetrics(); } #endif // TRITON_ENABLE_METRICS_GPU #ifdef TRITON_ENABLE_METRICS_CPU if (cpu_metrics_enabled_) { PollCpuMetrics(); } #endif // TRITON_ENABLE_METRICS_CPU } })); return true; } bool Metrics::PollPinnedMemoryMetrics() { uint64_t pinned_memory_byte_size = PinnedMemoryManager::GetTotalPinnedMemoryByteSize(); uint64_t used_pinned_memory_byte_size = PinnedMemoryManager::GetUsedPinnedMemoryByteSize(); pinned_memory_pool_total_->Set(pinned_memory_byte_size); pinned_memory_pool_used_->Set(used_pinned_memory_byte_size); return true; } #ifdef TRITON_ENABLE_METRICS_CPU Status Metrics::ParseCpuInfo(CpuInfo& info) { #ifdef _WIN32 return Status( Status::Code::INTERNAL, "CPU metrics not supported on Windows."); #else std::ifstream ifs("/proc/stat"); if (!ifs.good()) { return Status(Status::Code::INTERNAL, "Failed to open /proc/stat."); } std::string line; // Verify first line is aggregate cpu line std::getline(ifs, line); if (line.rfind("cpu ", 0) == std::string::npos) { return Status( Status::Code::INTERNAL, "Failed to find aggregate CPU info in /proc/stat."); } std::string _; std::istringstream iss(line); // Use _ to skip "cpu" at start of line if (!(iss >> _ >> info)) { return Status( Status::Code::INTERNAL, "Failed to parse aggregate CPU info in /proc/stat."); } return Status::Success; #endif // OS } Status Metrics::ParseMemInfo(MemInfo& info) { #ifdef _WIN32 return Status( Status::Code::INTERNAL, "Memory metrics not supported on Windows."); #else std::ifstream ifs("/proc/meminfo"); if (!ifs.good()) { return Status(Status::Code::INTERNAL, "Failed to open /proc/meminfo."); } std::string line; constexpr uint64_t KB = 1024; while (std::getline(ifs, line)) { std::istringstream iss(line); std::string name; uint64_t value = 0; if (iss >> name >> value) { name.pop_back(); info[name] = value * KB; } else { return Status( Status::Code::INTERNAL, "Encountered error parsing /proc/meminfo."); } } if (info.find("MemTotal") == info.end() || info.find("MemAvailable") == info.end()) { return Status( Status::Code::INTERNAL, "Failed to find desired values in /proc/meminfo."); } if (info["MemAvailable"] > info["MemTotal"]) { return Status( Status::Code::INTERNAL, "Available bytes shouldn't be greater than Total bytes"); } // "Used" memory can be defined in many different ways. While many // older applications consider "used = total - (free + cached)", a more // accurate measure of available memory "MemAvailable" was added, // so we choose "used = total - available" for a more accurate measure. // This may change in the future if not sufficient for most use cases. // See https://stackoverflow.com/a/35019697. info["MemUsed"] = info["MemTotal"] - info["MemAvailable"]; return Status::Success; #endif // OS } double Metrics::CpuUtilization(const CpuInfo& info_new, const CpuInfo& info_old) { // Account for overflow const auto wrap_sub = [](uint64_t a, uint64_t b) { return (a > b) ? (a - b) : 0; }; uint64_t util_diff = wrap_sub(info_new.user, info_old.user) + wrap_sub(info_new.nice, info_old.nice) + wrap_sub(info_new.system, info_old.system) + wrap_sub(info_new.irq, info_old.irq) + wrap_sub(info_new.softirq, info_old.softirq) + wrap_sub(info_new.steal, info_old.steal); uint64_t idle_diff = wrap_sub(info_new.idle, info_old.idle) + wrap_sub(info_new.iowait, info_old.iowait); double util_ratio = static_cast(util_diff) / (util_diff + idle_diff); return util_ratio; } #endif // TRITON_ENABLE_METRICS_CPU bool Metrics::PollCpuMetrics() { #ifndef TRITON_ENABLE_METRICS_CPU return false; #else // CPU Utilization double cpu_util = 0.0; auto cpu_info = CpuInfo(); auto status = ParseCpuInfo(cpu_info); if (status.IsOk()) { cpu_util = CpuUtilization(cpu_info, last_cpu_info_); last_cpu_info_ = cpu_info; } cpu_utilization_->Set(cpu_util); // [0.0, 1.0] // RAM / Memory double mem_total_bytes = 0.0; double mem_used_bytes = 0.0; auto mem_info = MemInfo(); status = ParseMemInfo(mem_info); if (status.IsOk()) { // MemTotal will usually not change over time, but if something // goes wrong when querying memory, we can reflect that by updating. mem_total_bytes = mem_info["MemTotal"]; mem_used_bytes = mem_info["MemUsed"]; } cpu_memory_total_->Set(mem_total_bytes); cpu_memory_used_->Set(mem_used_bytes); return true; #endif // TRITON_ENABLE_METRICS_CPU } bool Metrics::PollDcgmMetrics() { #ifndef TRITON_ENABLE_METRICS_GPU return false; #else if (dcgm_metadata_.available_cuda_gpu_ids_.size() == 0) { LOG_WARNING << "error polling GPU metrics, GPU metrics will not be " << "available: no available gpus to poll"; return false; } dcgmUpdateAllFields(dcgm_metadata_.dcgm_handle_, 1 /* wait for update*/); for (unsigned int didx = 0; didx < dcgm_metadata_.available_cuda_gpu_ids_.size(); ++didx) { uint32_t cuda_id = dcgm_metadata_.available_cuda_gpu_ids_[didx]; if (dcgm_metadata_.cuda_ids_to_dcgm_ids_.count(cuda_id) <= 0) { LOG_WARNING << "Cannot find DCGM id for CUDA id " << cuda_id; continue; } uint32_t dcgm_id = dcgm_metadata_.cuda_ids_to_dcgm_ids_.at(cuda_id); dcgmFieldValue_v1 field_values[dcgm_metadata_.field_count_]; dcgmReturn_t dcgmerr = dcgmGetLatestValuesForFields( dcgm_metadata_.dcgm_handle_, dcgm_id, dcgm_metadata_.fields_.data(), dcgm_metadata_.field_count_, field_values); if (dcgmerr != DCGM_ST_OK) { dcgm_metadata_.power_limit_fail_cnt_[didx]++; dcgm_metadata_.power_usage_fail_cnt_[didx]++; dcgm_metadata_.energy_fail_cnt_[didx]++; dcgm_metadata_.util_fail_cnt_[didx]++; dcgm_metadata_.mem_fail_cnt_[didx]++; LOG_WARNING << "Unable to get field values for GPU ID " << cuda_id << ": " << errorString(dcgmerr); } else { // Power limit if (dcgm_metadata_.power_limit_fail_cnt_[didx] < dcgm_metadata_.fail_threshold_) { double power_limit = field_values[0].value.dbl; if ((field_values[0].status == DCGM_ST_OK) && (!DCGM_FP64_IS_BLANK(power_limit))) { dcgm_metadata_.power_limit_fail_cnt_[didx] = 0; } else { dcgm_metadata_.power_limit_fail_cnt_[didx]++; power_limit = 0; dcgmReturn_t status = dcgmReturn_t(field_values[0].status); LOG_WARNING << "Unable to get power limit for GPU " << cuda_id << ". Status:" << errorString(status) << ", value:" << dcgmValueToErrorMessage(power_limit); } gpu_power_limit_[didx]->Set(power_limit); } // Power usage if (dcgm_metadata_.power_usage_fail_cnt_[didx] < dcgm_metadata_.fail_threshold_) { double power_usage = field_values[1].value.dbl; if ((field_values[1].status == DCGM_ST_OK) && (!DCGM_FP64_IS_BLANK(power_usage))) { dcgm_metadata_.power_usage_fail_cnt_[didx] = 0; } else { dcgm_metadata_.power_usage_fail_cnt_[didx]++; power_usage = 0; dcgmReturn_t status = dcgmReturn_t(field_values[1].status); LOG_WARNING << "Unable to get power usage for GPU " << cuda_id << ". Status:" << errorString(status) << ", value:" << dcgmValueToErrorMessage(power_usage); } gpu_power_usage_[didx]->Set(power_usage); } // Energy Consumption if (dcgm_metadata_.energy_fail_cnt_[didx] < dcgm_metadata_.fail_threshold_) { int64_t energy = field_values[2].value.i64; if ((field_values[2].status == DCGM_ST_OK) && (!DCGM_INT64_IS_BLANK(energy))) { dcgm_metadata_.energy_fail_cnt_[didx] = 0; if (dcgm_metadata_.last_energy_[didx] == 0) { dcgm_metadata_.last_energy_[didx] = energy; } gpu_energy_consumption_[didx]->Increment( (double)(energy - dcgm_metadata_.last_energy_[didx]) * 0.001); dcgm_metadata_.last_energy_[didx] = energy; } else { dcgm_metadata_.energy_fail_cnt_[didx]++; energy = 0; dcgmReturn_t status = dcgmReturn_t(field_values[2].status); LOG_WARNING << "Unable to get energy consumption for " << "GPU " << cuda_id << ". Status:" << errorString(status) << ", value:" << dcgmValueToErrorMessage(energy); } } // Utilization if (dcgm_metadata_.util_fail_cnt_[didx] < dcgm_metadata_.fail_threshold_) { int64_t util = field_values[3].value.i64; if ((field_values[3].status == DCGM_ST_OK) && (!DCGM_INT64_IS_BLANK(util))) { dcgm_metadata_.util_fail_cnt_[didx] = 0; } else { dcgm_metadata_.util_fail_cnt_[didx]++; util = 0; dcgmReturn_t status = dcgmReturn_t(field_values[3].status); LOG_WARNING << "Unable to get GPU utilization for GPU " << cuda_id << ". Status:" << errorString(status) << ", value:" << dcgmValueToErrorMessage(util); } gpu_utilization_[didx]->Set((double)util * 0.01); } // Memory Usage if (dcgm_metadata_.mem_fail_cnt_[didx] < dcgm_metadata_.fail_threshold_) { int64_t memory_used = field_values[4].value.i64; int64_t memory_total = field_values[5].value.i64; if ((field_values[4].status == DCGM_ST_OK) && (!DCGM_INT64_IS_BLANK(memory_used)) && (field_values[5].status == DCGM_ST_OK) && (!DCGM_INT64_IS_BLANK(memory_total))) { dcgm_metadata_.mem_fail_cnt_[didx] = 0; } else { memory_total = 0; memory_used = 0; dcgm_metadata_.mem_fail_cnt_[didx]++; dcgmReturn_t usageStatus = dcgmReturn_t(field_values[4].status); dcgmReturn_t memoryTotaltatus = dcgmReturn_t(field_values[5].status); LOG_WARNING << "Unable to get memory usage for GPU " << cuda_id << ". Memory usage status:" << errorString(usageStatus) << ", value:" << dcgmValueToErrorMessage(memory_used) << ". Memory total status:" << errorString(memoryTotaltatus) << ", value:" << dcgmValueToErrorMessage(memory_total); } gpu_memory_total_[didx]->Set(memory_total * 1024 * 1024); // bytes gpu_memory_used_[didx]->Set(memory_used * 1024 * 1024); // bytes } } } return true; #endif // TRITON_ENABLE_METRICS_GPU } bool Metrics::InitializePinnedMemoryMetrics() { const std::map pinned_memory_labels; pinned_memory_pool_total_ = &pinned_memory_pool_total_family_.Add(pinned_memory_labels); pinned_memory_pool_used_ = &pinned_memory_pool_used_family_.Add(pinned_memory_labels); return true; } bool Metrics::InitializeCpuMetrics() { #ifndef TRITON_ENABLE_METRICS_CPU return false; #else const std::map cpu_labels; cpu_utilization_ = &cpu_utilization_family_.Add(cpu_labels); cpu_memory_total_ = &cpu_memory_total_family_.Add(cpu_labels); cpu_memory_used_ = &cpu_memory_used_family_.Add(cpu_labels); // Get baseline CPU info for future comparisons last_cpu_info_ = CpuInfo(); auto status = ParseCpuInfo(last_cpu_info_); if (!status.IsOk()) { LOG_WARNING << "error initializing CPU metrics, CPU utilization may not " "be available: " << status.Message(); return false; } // Verify memory metrics can be parsed auto mem_info = MemInfo(); status = ParseMemInfo(mem_info); if (!status.IsOk()) { LOG_WARNING << "error initializing CPU metrics, CPU memory metrics may not " "be available: " << status.Message(); return false; } LOG_INFO << "Collecting CPU metrics"; return true; #endif // TRITON_ENABLE_METRICS_CPU } bool Metrics::InitializeDcgmMetrics() { #ifndef TRITON_ENABLE_METRICS_GPU return false; #else dcgmReturn_t dcgmerr = dcgmInit(); if (dcgmerr != DCGM_ST_OK) { LOG_WARNING << "error initializing DCGM, GPU metrics will not be " << "available: " << errorString(dcgmerr); return false; } if (dcgm_metadata_.standalone_) { char hostIpAddress[16] = {0}; std::string ipAddress = "127.0.0.1"; strncpy(hostIpAddress, ipAddress.c_str(), 15); dcgmerr = dcgmConnect(hostIpAddress, &dcgm_metadata_.dcgm_handle_); } else { dcgmerr = dcgmStartEmbedded( DCGM_OPERATION_MODE_MANUAL, &dcgm_metadata_.dcgm_handle_); } if (dcgmerr != DCGM_ST_OK) { LOG_WARNING << "DCGM unable to start: " << errorString(dcgmerr); return false; } else { // Set this flag to signal DCGM cleanup in destructor dcgm_metadata_.dcgm_initialized_ = true; } if (dcgm_metadata_.standalone_) { dcgmerr = dcgmUpdateAllFields(dcgm_metadata_.dcgm_handle_, 1); if (dcgmerr != DCGM_ST_OK) { LOG_WARNING << "DCGM unable to update all fields, GPU metrics will " "not be available: " << errorString(dcgmerr); return false; } } unsigned int dcgm_gpu_ids[DCGM_MAX_NUM_DEVICES]; int dcgm_gpu_count; dcgmerr = dcgmGetAllDevices( dcgm_metadata_.dcgm_handle_, dcgm_gpu_ids, &dcgm_gpu_count); if (dcgmerr != DCGM_ST_OK) { LOG_WARNING << "DCGM unable to get device info and count, GPU " "metrics will not be available: " << errorString(dcgmerr); return false; } // Get PCI Bus ID to DCGM device Id map. // Some devices may have problems using DCGM API and // these devices needs to be ignored. std::map pci_bus_id_to_dcgm_id; std::map > pci_bus_id_to_gpu_labels; std::map pci_bus_id_to_device_name; dcgmDeviceAttributes_t gpu_attributes[DCGM_MAX_NUM_DEVICES]; for (int i = 0; i < dcgm_gpu_count; i++) { gpu_attributes[i].version = dcgmDeviceAttributes_version; dcgmerr = dcgmGetDeviceAttributes( dcgm_metadata_.dcgm_handle_, dcgm_gpu_ids[i], &gpu_attributes[i]); if (dcgmerr != DCGM_ST_OK) { LOG_WARNING << "DCGM unable to get device properties for DCGM device " << dcgm_gpu_ids[i] << ", GPU metrics will not be available for this device: " << errorString(dcgmerr); } else { std::string pciBusId = gpu_attributes[i].identifiers.pciBusId; pci_bus_id_to_dcgm_id[pciBusId] = i; pci_bus_id_to_device_name[pciBusId] = std::string(gpu_attributes[i].identifiers.deviceName); std::map gpu_labels; gpu_labels.insert(std::map::value_type( kMetricsLabelGpuUuid, std::string(gpu_attributes[i].identifiers.uuid))); pci_bus_id_to_gpu_labels[pciBusId] = gpu_labels; } } // Get CUDA-visible PCI Bus Ids and get DCGM metrics for each CUDA-visible GPU int cuda_gpu_count; cudaError_t cudaerr = cudaGetDeviceCount(&cuda_gpu_count); if (cudaerr != cudaSuccess) { LOG_WARNING << "Cannot get CUDA device count, GPU metrics will not be available"; return false; } for (int i = 0; i < cuda_gpu_count; ++i) { std::string pci_bus_id = "0000"; // pad 0's for uniformity char pcibusid_str[64]; cudaerr = cudaDeviceGetPCIBusId(pcibusid_str, sizeof(pcibusid_str) - 1, i); if (cudaerr == cudaSuccess) { pci_bus_id.append(pcibusid_str); if (pci_bus_id_to_dcgm_id.count(pci_bus_id) <= 0) { LOG_INFO << "Skipping GPU:" << i << " since it's not CUDA enabled. This should never happen!"; continue; } // Filter out CUDA visible GPUs from GPUs found by DCGM LOG_INFO << "Collecting metrics for GPU " << i << ": " << pci_bus_id_to_device_name[pci_bus_id]; auto& gpu_labels = pci_bus_id_to_gpu_labels[pci_bus_id]; gpu_utilization_.push_back(&gpu_utilization_family_.Add(gpu_labels)); gpu_memory_total_.push_back(&gpu_memory_total_family_.Add(gpu_labels)); gpu_memory_used_.push_back(&gpu_memory_used_family_.Add(gpu_labels)); gpu_power_usage_.push_back(&gpu_power_usage_family_.Add(gpu_labels)); gpu_power_limit_.push_back(&gpu_power_limit_family_.Add(gpu_labels)); gpu_energy_consumption_.push_back( &gpu_energy_consumption_family_.Add(gpu_labels)); uint32_t dcgm_id = pci_bus_id_to_dcgm_id[pci_bus_id]; dcgm_metadata_.cuda_ids_to_dcgm_ids_[i] = dcgm_id; dcgm_metadata_.available_cuda_gpu_ids_.emplace_back(i); } else { LOG_WARNING << "GPU metrics will not be available for device:" << i; } } // create a gpu group char groupName[] = "dcgm_group"; dcgmerr = dcgmGroupCreate( dcgm_metadata_.dcgm_handle_, DCGM_GROUP_DEFAULT, groupName, &dcgm_metadata_.groupId_); if (dcgmerr != DCGM_ST_OK) { LOG_WARNING << "Cannot make GPU group: " << errorString(dcgmerr); } // Initialize tracking vectors for (unsigned int didx = 0; didx < dcgm_metadata_.available_cuda_gpu_ids_.size(); ++didx) { dcgm_metadata_.power_limit_fail_cnt_.push_back(0); dcgm_metadata_.power_usage_fail_cnt_.push_back(0); dcgm_metadata_.energy_fail_cnt_.push_back(0); dcgm_metadata_.util_fail_cnt_.push_back(0); dcgm_metadata_.mem_fail_cnt_.push_back(0); dcgm_metadata_.last_energy_.push_back(0); } // Number of fields for DCGM to use from fields_ below dcgm_metadata_.field_count_ = 6; unsigned short util_flag = dcgm_metadata_.standalone_ ? DCGM_FI_PROF_GR_ENGINE_ACTIVE : DCGM_FI_DEV_GPU_UTIL; dcgm_metadata_.fields_ = { DCGM_FI_DEV_POWER_MGMT_LIMIT, // power limit, watts DCGM_FI_DEV_POWER_USAGE, // power usage, watts DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, // Total energy consumption, mJ util_flag, // util ratio, 1 = 1% DCGM_FI_DEV_FB_USED, // Frame buffer used, MiB DCGM_FI_DEV_FB_TOTAL, // Frame buffer used, MiB }; char fieldName[] = "field_group"; dcgmFieldGrp_t fieldGroupId; dcgmerr = dcgmFieldGroupCreate( dcgm_metadata_.dcgm_handle_, dcgm_metadata_.field_count_, dcgm_metadata_.fields_.data(), fieldName, &fieldGroupId); if (dcgmerr != DCGM_ST_OK) { LOG_WARNING << "Cannot make field group: " << errorString(dcgmerr); } dcgmerr = dcgmWatchFields( dcgm_metadata_.dcgm_handle_, dcgm_metadata_.groupId_, fieldGroupId, metrics_interval_ms_ * 1000 /*update period, usec*/, 5.0 /*maxKeepAge, sec*/, 5 /*maxKeepSamples*/); if (dcgmerr != DCGM_ST_OK) { LOG_WARNING << "Cannot start watching fields: " << errorString(dcgmerr); return false; } return true; #endif // TRITON_ENABLE_METRICS_GPU } #ifdef TRITON_ENABLE_METRICS_GPU std::string Metrics::dcgmValueToErrorMessage(double val) { if (DCGM_FP64_IS_BLANK(val)) { if (val == DCGM_FP64_BLANK) { return "Not Specified"; } else if (val == DCGM_FP64_NOT_FOUND) { return "Not Found"; } else if (val == DCGM_FP64_NOT_SUPPORTED) { return "Not Supported"; } else if (val == DCGM_FP64_NOT_PERMISSIONED) { return "Insf. Permission"; } else { return "Unknown"; } } else { return std::to_string(val); } } std::string Metrics::dcgmValueToErrorMessage(int64_t val) { if (DCGM_INT64_IS_BLANK(val)) { switch (val) { case DCGM_INT64_BLANK: return "Not Specified"; case DCGM_INT64_NOT_FOUND: return "Not Found"; case DCGM_INT64_NOT_SUPPORTED: return "Not Supported"; case DCGM_INT64_NOT_PERMISSIONED: return "Insf. Permission"; default: return "Unknown"; } } else { return std::to_string(val); } } #endif // TRITON_ENABLE_METRICS_GPU bool Metrics::UUIDForCudaDevice(int cuda_device, std::string* uuid) { // If metrics were not initialized then just silently fail since // with DCGM we can't get the CUDA device (and not worth doing // anyway since metrics aren't being reported). auto singleton = GetSingleton(); if (!singleton->gpu_metrics_enabled_) { return false; } // If GPU metrics is not enabled just silently fail. #ifndef TRITON_ENABLE_METRICS_GPU return false; #else dcgmDeviceAttributes_t gpu_attributes; gpu_attributes.version = dcgmDeviceAttributes_version; dcgmReturn_t dcgmerr = dcgmGetDeviceAttributes( singleton->dcgm_metadata_.dcgm_handle_, cuda_device, &gpu_attributes); if (dcgmerr != DCGM_ST_OK) { LOG_ERROR << "Unable to get device UUID: " << errorString(dcgmerr); return false; } *uuid = gpu_attributes.identifiers.uuid; return true; #endif // TRITON_ENABLE_METRICS_GPU } std::shared_ptr Metrics::GetRegistry() { auto singleton = Metrics::GetSingleton(); return singleton->registry_; } const std::string Metrics::SerializedMetrics() { auto singleton = Metrics::GetSingleton(); return singleton->serializer_->Serialize( singleton->registry_.get()->Collect()); } Metrics* Metrics::GetSingleton() { static Metrics singleton; return &singleton; } }} // namespace triton::core #endif // TRITON_ENABLE_METRICS kytensor-2.49.0.7/core/src/metrics.h000066400000000000000000000335741527052371200172070ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once #ifdef TRITON_ENABLE_METRICS #include #include #include #include #include "prometheus/counter.h" #include "prometheus/gauge.h" #include "prometheus/histogram.h" #include "prometheus/registry.h" #include "prometheus/serializer.h" #include "prometheus/summary.h" #include "prometheus/text_serializer.h" #include "status.h" #ifdef TRITON_ENABLE_METRICS_GPU #include #endif // TRITON_ENABLE_METRICS_GPU namespace triton { namespace core { using MetricsConfig = std::vector>; using MetricsConfigMap = std::unordered_map; #ifdef TRITON_ENABLE_METRICS_CPU using MemInfo = std::unordered_map; // References: // - htop source: https://stackoverflow.com/a/23376195 // - Linux docs: https://www.kernel.org/doc/Documentation/filesystems/proc.txt // guest/guestnice values are counted in user/nice so we skip parsing them struct CpuInfo { uint64_t user = 0; // normal processes executing in user mode uint64_t nice = 0; // niced processes executing in user mode uint64_t system = 0; // processes executing in kernel mode uint64_t idle = 0; // twiddling thumbs uint64_t iowait = 0; // waiting for I/O to complete uint64_t irq = 0; // servicing interrupts uint64_t softirq = 0; // servicing softirqs uint64_t steal = 0; // involuntary wait }; inline std::istream& operator>>(std::istream& is, CpuInfo& info) { is >> info.user >> info.nice >> info.system >> info.idle >> info.iowait >> info.irq >> info.softirq >> info.steal; return is; } #endif // TRITON_ENABLE_METRICS_CPU #ifdef TRITON_ENABLE_METRICS_GPU struct DcgmMetadata { // DCGM handles for initialization and destruction dcgmHandle_t dcgm_handle_ = 0; dcgmGpuGrp_t groupId_ = 0; // DCGM Flags bool standalone_ = false; // DCGM Fields size_t field_count_ = 0; std::vector fields_; // GPU Device Mapping std::map cuda_ids_to_dcgm_ids_; std::vector available_cuda_gpu_ids_; // Stop attempting metrics if they fail multiple consecutive // times for a device. const int fail_threshold_ = 3; // DCGM Failure Tracking std::vector power_limit_fail_cnt_; std::vector power_usage_fail_cnt_; std::vector energy_fail_cnt_; std::vector util_fail_cnt_; std::vector mem_fail_cnt_; // DCGM Energy Tracking std::vector last_energy_; // Track if DCGM handle initialized successfully bool dcgm_initialized_ = false; }; #endif // TRITON_ENABLE_METRICS_GPU class Metrics { public: // Return the hash value of the labels static size_t HashLabels(const std::map& labels); // Are metrics enabled? static bool Enabled(); // Enable reporting of metrics static void EnableMetrics(); // Enable reporting of Pinned memory metrics static void EnablePinnedMemoryMetrics(); // Enable reporting of GPU metrics static void EnableGPUMetrics(); // Enable reporting of CPU metrics static void EnableCpuMetrics(); // Start a thread for polling enabled metrics if any static void StartPollingThreadSingleton(); // Set the time interval in secs at which metrics are collected static void SetMetricsInterval(uint64_t metrics_interval_ms); // Set the config for Metrics to control various options generically static void SetConfigMap(MetricsConfigMap cfg); // Get the config for Metrics static const MetricsConfigMap& ConfigMap(); // Get the prometheus registry static std::shared_ptr GetRegistry(); // Get serialized metrics static const std::string SerializedMetrics(); // Get the UUID for a CUDA device. Return true and initialize 'uuid' // if a UUID is found, return false if a UUID cannot be returned. static bool UUIDForCudaDevice(int cuda_device, std::string* uuid); // Metric family counting successful inference requests static prometheus::Family& FamilyInferenceSuccess() { return GetSingleton()->inf_success_family_; } // Metric family counting failed inference requests static prometheus::Family& FamilyInferenceFailure() { return GetSingleton()->inf_failure_family_; } // Metric family counting inferences performed, where a batch-size // 'n' inference request is counted as 'n' inferences static prometheus::Family& FamilyInferenceCount() { return GetSingleton()->inf_count_family_; } // Metric family counting inferences performed, where a batch-size // 'n' inference request is counted as 'n' inferences static prometheus::Family& FamilyInferenceExecutionCount() { return GetSingleton()->inf_count_exec_family_; } // Metric family of cumulative inference request duration, in // microseconds static prometheus::Family& FamilyInferenceRequestDuration() { return GetSingleton()->inf_request_duration_us_family_; } // Metric family of cumulative inference queuing duration, in // microseconds static prometheus::Family& FamilyInferenceQueueDuration() { return GetSingleton()->inf_queue_duration_us_family_; } // Metric family of cumulative inference compute durations, in // microseconds static prometheus::Family& FamilyInferenceComputeInputDuration() { return GetSingleton()->inf_compute_input_duration_us_family_; } static prometheus::Family& FamilyInferenceComputeInferDuration() { return GetSingleton()->inf_compute_infer_duration_us_family_; } static prometheus::Family& FamilyInferenceComputeOutputDuration() { return GetSingleton()->inf_compute_output_duration_us_family_; } // Metric family of instantaneous inference queue size per model static prometheus::Family& FamilyInferenceQueueSize() { return GetSingleton()->inf_pending_request_count_family_; } // Metric families of per-model response cache metrics // NOTE: These are used in infer_stats for perf_analyzer static prometheus::Family& FamilyCacheHitCount() { return GetSingleton()->cache_num_hits_model_family_; } static prometheus::Family& FamilyCacheHitDuration() { return GetSingleton()->cache_hit_duration_us_model_family_; } static prometheus::Family& FamilyCacheMissCount() { return GetSingleton()->cache_num_misses_model_family_; } static prometheus::Family& FamilyCacheMissDuration() { return GetSingleton()->cache_miss_duration_us_model_family_; } // Summaries static prometheus::Family& FamilyInferenceRequestSummary() { return GetSingleton()->inf_request_summary_us_family_; } static prometheus::Family& FamilyInferenceQueueSummary() { return GetSingleton()->inf_queue_summary_us_family_; } static prometheus::Family& FamilyInferenceComputeInputSummary() { return GetSingleton()->inf_compute_input_summary_us_family_; } static prometheus::Family& FamilyInferenceComputeInferSummary() { return GetSingleton()->inf_compute_infer_summary_us_family_; } static prometheus::Family& FamilyInferenceComputeOutputSummary() { return GetSingleton()->inf_compute_output_summary_us_family_; } static prometheus::Family& FamilyCacheHitSummary() { return GetSingleton()->cache_hit_summary_us_model_family_; } static prometheus::Family& FamilyCacheMissSummary() { return GetSingleton()->cache_miss_summary_us_model_family_; } private: Metrics(); virtual ~Metrics(); static Metrics* GetSingleton(); bool InitializeDcgmMetrics(); bool InitializeCpuMetrics(); bool InitializePinnedMemoryMetrics(); bool StartPollingThread(); bool PollPinnedMemoryMetrics(); bool PollDcgmMetrics(); bool PollCpuMetrics(); std::string dcgmValueToErrorMessage(double val); std::string dcgmValueToErrorMessage(int64_t val); std::shared_ptr registry_; std::unique_ptr serializer_; // DLIS-4761: Refactor into groups of families prometheus::Family& inf_success_family_; prometheus::Family& inf_failure_family_; prometheus::Family& inf_count_family_; prometheus::Family& inf_count_exec_family_; prometheus::Family& inf_request_duration_us_family_; prometheus::Family& inf_queue_duration_us_family_; prometheus::Family& inf_compute_input_duration_us_family_; prometheus::Family& inf_compute_infer_duration_us_family_; prometheus::Family& inf_compute_output_duration_us_family_; prometheus::Family& inf_pending_request_count_family_; prometheus::Family& pinned_memory_pool_total_family_; prometheus::Family& pinned_memory_pool_used_family_; prometheus::Gauge* pinned_memory_pool_total_; prometheus::Gauge* pinned_memory_pool_used_; // Per-model Response Cache metrics // NOTE: Per-model metrics are used in infer_stats for perf_analyzer. Global // cache metrics will be implemented by cache and published through // Metrics C API. prometheus::Family& cache_num_hits_model_family_; prometheus::Family& cache_hit_duration_us_model_family_; prometheus::Family& cache_num_misses_model_family_; prometheus::Family& cache_miss_duration_us_model_family_; // Summaries prometheus::Family& inf_request_summary_us_family_; prometheus::Family& inf_queue_summary_us_family_; prometheus::Family& inf_compute_input_summary_us_family_; prometheus::Family& inf_compute_infer_summary_us_family_; prometheus::Family& inf_compute_output_summary_us_family_; prometheus::Family& cache_hit_summary_us_model_family_; prometheus::Family& cache_miss_summary_us_model_family_; #ifdef TRITON_ENABLE_METRICS_GPU prometheus::Family& gpu_utilization_family_; prometheus::Family& gpu_memory_total_family_; prometheus::Family& gpu_memory_used_family_; prometheus::Family& gpu_power_usage_family_; prometheus::Family& gpu_power_limit_family_; prometheus::Family& gpu_energy_consumption_family_; std::vector gpu_utilization_; std::vector gpu_memory_total_; std::vector gpu_memory_used_; std::vector gpu_power_usage_; std::vector gpu_power_limit_; std::vector gpu_energy_consumption_; DcgmMetadata dcgm_metadata_; #endif // TRITON_ENABLE_METRICS_GPU #ifdef TRITON_ENABLE_METRICS_CPU // Parses "/proc/meminfo" for metrics, currently only supported on Linux. Status ParseMemInfo(MemInfo& info); // Parses "/proc/stat" for metrics, currently only supported on Linux. Status ParseCpuInfo(CpuInfo& info); // Computes CPU utilization between "info_new" and "info_old" values double CpuUtilization(const CpuInfo& info_new, const CpuInfo& info_old); prometheus::Family& cpu_utilization_family_; prometheus::Family& cpu_memory_total_family_; prometheus::Family& cpu_memory_used_family_; prometheus::Gauge* cpu_utilization_; prometheus::Gauge* cpu_memory_total_; prometheus::Gauge* cpu_memory_used_; CpuInfo last_cpu_info_; #endif // TRITON_ENABLE_METRICS_CPU // Thread for polling cpu/gpu metrics periodically std::unique_ptr poll_thread_; std::atomic poll_thread_exit_; bool metrics_enabled_; bool gpu_metrics_enabled_; bool cpu_metrics_enabled_; bool pinned_memory_metrics_enabled_; bool poll_thread_started_; std::mutex metrics_enabling_; std::mutex poll_thread_starting_; uint64_t metrics_interval_ms_; MetricsConfigMap config_; }; }} // namespace triton::core #endif // TRITON_ENABLE_METRICS kytensor-2.49.0.7/core/src/model.cc000066400000000000000000000111511527052371200167620ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "model.h" #include #include #include "constants.h" #include "filesystem/api.h" #include "infer_request.h" #include "model_config_utils.h" #include "triton/common/logging.h" namespace triton { namespace core { Status Model::GetInput( const std::string& name, const inference::ModelInput** input) const { const auto itr = input_map_.find(name); if (itr == input_map_.end()) { return Status( Status::Code::INVALID_ARG, "unexpected inference input '" + name + "' for model '" + Name() + "'"); } *input = &itr->second; return Status::Success; } Status Model::GetOutput( const std::string& name, const inference::ModelOutput** output) const { const auto itr = output_map_.find(name); if (itr == output_map_.end()) { return Status( Status::Code::INVALID_ARG, "unexpected inference output '" + name + "' for model '" + Name() + "'"); } *output = &itr->second; return Status::Success; } Status Model::SetModelConfig(const inference::ModelConfig& config) { config_ = config; set_model_config_ = true; return Status::Success; } Status Model::SetScheduler(std::unique_ptr scheduler) { if (scheduler_ != nullptr) { return Status( Status::Code::INTERNAL, "Attempt to change scheduler not allowed"); } scheduler_ = std::move(scheduler); return Status::Success; } Status Model::Init(const bool is_config_provided) { if (!set_model_config_ && !is_config_provided) { return Status( Status::Code::NOT_FOUND, "model configuration is not provided for model '" + Name() + "'"); } RETURN_IF_ERROR(ValidateModelConfig(config_, min_compute_capability_)); RETURN_IF_ERROR(ValidateModelIOConfig(config_)); // Initialize the input map for (const auto& io : config_.input()) { input_map_.insert(std::make_pair(io.name(), io)); if (!io.optional()) { ++required_input_count_; } } // Initialize the output map and label provider for each output label_provider_ = std::make_shared(); for (const auto& io : config_.output()) { output_map_.insert(std::make_pair(io.name(), io)); if (!io.label_filename().empty()) { const auto label_path = JoinPath({model_dir_, io.label_filename()}); RETURN_IF_ERROR(label_provider_->AddLabels(io.name(), label_path)); } } if (config_.has_dynamic_batching()) { default_priority_level_ = config_.dynamic_batching().default_priority_level(); max_priority_level_ = config_.dynamic_batching().priority_levels(); } else if (config_.has_ensemble_scheduling()) { // For ensemble, allow any priority level to pass through default_priority_level_ = 0; max_priority_level_ = UINT64_MAX; } else { default_priority_level_ = 0; max_priority_level_ = 0; } #ifdef TRITON_ENABLE_METRICS MetricModelReporter::Create( ModelId(), Version(), METRIC_REPORTER_ID_UTILITY, ResponseCacheEnabled(), Config().metric_tags(), &reporter_); #endif // TRITON_ENABLE_METRICS return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/model.h000066400000000000000000000235461527052371200166370ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "infer_stats.h" #include "label_provider.h" #include "metric_model_reporter.h" #include "model_config.pb.h" #include "scheduler.h" #include "status.h" namespace triton { namespace core { struct ModelIdentifier { ModelIdentifier(const std::string& model_namespace, const std::string& name) : namespace_(model_namespace), name_(name) { } ModelIdentifier(const ModelIdentifier&) = default; ModelIdentifier& operator=(const ModelIdentifier&) = default; bool operator<(const ModelIdentifier& rhs) const { if (namespace_ == rhs.namespace_) { return (name_ < rhs.name_); } else { return (namespace_ < rhs.namespace_); } } bool operator==(const ModelIdentifier& rhs) const { return ((namespace_ == rhs.namespace_) && (name_ == rhs.name_)); } bool operator!=(const ModelIdentifier& rhs) const { return ((namespace_ != rhs.namespace_) || (name_ != rhs.name_)); } bool NamespaceDisabled() const { return namespace_.empty(); } friend std::ostream& operator<<(std::ostream& os, const ModelIdentifier& rhs) { // Avoid log differences if namespace is disabled if (rhs.NamespaceDisabled()) { os << rhs.name_; return os; } os << rhs.namespace_ << "::" << rhs.name_; return os; } std::string str() const { // Avoid log differences if namespace is disabled if (NamespaceDisabled()) { return name_; } return (namespace_ + "::" + name_); } // namespace is not a reflection of the model repository although it is // currently implemented to be the same as the repository of the model. std::string namespace_; // name is the name registered to Triton, it is the model directory name // by default and may be overwritten. std::string name_; }; }} // namespace triton::core // define hash function for struct to be used as unordered_map key namespace std { template <> class hash { public: size_t operator()(const triton::core::ModelIdentifier& model_id) const { // trivial hash for multiple entries // https://en.cppreference.com/w/cpp/utility/hash return ( hash()(model_id.namespace_) ^ (hash()(model_id.name_) << 1)); } }; } // namespace std namespace triton { namespace core { class InferenceRequest; // // Interface for models that handle inference requests. // class Model { public: explicit Model( const double min_compute_capability, const std::string& model_dir, const ModelIdentifier& model_id, const int64_t version, const inference::ModelConfig& config) : config_(config), min_compute_capability_(min_compute_capability), model_id_(model_id), version_(version), required_input_count_(0), model_dir_(model_dir), set_model_config_(false) { } virtual ~Model() {} // Get the name of model being served. const std::string& Name() const { return config_.name(); } // Get the identifier of model being served. const ModelIdentifier& ModelId() const { return model_id_; } // Get the version of model being served. int64_t Version() const { return version_; } // Get the configuration of model being served. const inference::ModelConfig& Config() const { return config_; } // Get whether response cache is enabled for this model. bool ResponseCacheEnabled() const { return config_.response_cache().enable(); } // Get the number of required inputs size_t RequiredInputCount() const { return required_input_count_; } // Get the stats collector for the model being served. InferenceStatsAggregator* MutableStatsAggregator() { return &stats_aggregator_; } const InferenceStatsAggregator& StatsAggregator() const { return stats_aggregator_; } void SetMemoryUsage(boost::span memory_usage) { std::map> lusage; for (const auto& mu : memory_usage) { lusage[mu->MemoryType()][mu->MemoryTypeId()] = mu->ByteSize(); } std::lock_guard lk(usage_mtx_); memory_usage_.swap(lusage); } std::vector AccumulatedMemoryUsage() const { auto lusage = AccumulatedInstanceMemoryUsage(); // accumulate model memory usage { std::lock_guard lk(usage_mtx_); for (const auto& mem_type_map : memory_usage_) { const auto& mem_type = mem_type_map.first; for (const auto& mem_id_map : mem_type_map.second) { const auto& mem_id = mem_id_map.first; const auto& byte_size = mem_id_map.second; lusage[mem_type][mem_id] += byte_size; } } } // Convert to buffer attribute std::vector res; for (const auto& mem_type_map : lusage) { const auto& mem_type = mem_type_map.first; for (const auto& mem_id_map : mem_type_map.second) { const auto& mem_id = mem_id_map.first; const auto& byte_size = mem_id_map.second; res.emplace_back(byte_size, mem_type, mem_id, nullptr); } } return res; } // Get the model configuration for a named input. Status GetInput( const std::string& name, const inference::ModelInput** input) const; // Get the model configuration for a named output. Status GetOutput( const std::string& name, const inference::ModelOutput** output) const; // Get a label provider for the model. const std::shared_ptr& GetLabelProvider() const { return label_provider_; } // Initialize the instance for Triton core usage Status Init(const bool is_config_provided); // Enqueue a request for execution. If Status::Success is returned // then the model has taken ownership of the request object and so // 'request' will be nullptr. If non-success is returned then the // caller still retains ownership of 'request'. Status Enqueue(std::unique_ptr& request) { return scheduler_->Enqueue(request); } // Return the number of in-flight inferences. size_t InflightInferenceCount() { return scheduler_->InflightInferenceCount(); } // Stop processing future requests unless they are considered as in-flight. void Stop() { scheduler_->Stop(); } uint64_t DefaultPriorityLevel() const { return default_priority_level_; } uint64_t MaxPriorityLevel() const { return max_priority_level_; } // Returns the model's metric reporter std::shared_ptr MetricReporter() const { return reporter_; } protected: virtual std::map> AccumulatedInstanceMemoryUsage() const { return {}; } // Set the configuration of the model being served. Status SetModelConfig(const inference::ModelConfig& config); // Explicitly set the scheduler to use for inference requests to the // model. The scheduler can only be set once for a model. Status SetScheduler(std::unique_ptr scheduler); // The scheduler to use for this model. std::unique_ptr scheduler_; // Configuration of the model. inference::ModelConfig config_; private: // The minimum supported CUDA compute capability. const double min_compute_capability_; // Identifier for the model. ModelIdentifier model_id_; // Version of the model. int64_t version_; // The stats collector for the model. InferenceStatsAggregator stats_aggregator_; // Records of memory used for loading the model std::map> memory_usage_; mutable std::mutex usage_mtx_; // Label provider for this model. std::shared_ptr label_provider_; size_t required_input_count_; // Map from input name to the model configuration for that input. std::unordered_map input_map_; // Map from output name to the model configuration for that output. std::unordered_map output_map_; // Path to model std::string model_dir_; // The default priority level for the model. uint64_t default_priority_level_; // The largest priority value for the model. uint64_t max_priority_level_; // Whether or not model config has been set. bool set_model_config_; // Reporter for metrics, or nullptr if no metrics should be reported std::shared_ptr reporter_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/model_config.cc000066400000000000000000000243731527052371200203210ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/common/model_config.h" namespace triton { namespace common { bool IsFixedSizeDataType(const inference::DataType dtype) { return dtype != inference::DataType::TYPE_STRING; } size_t GetDataTypeByteSize(const inference::DataType dtype) { switch (dtype) { case inference::DataType::TYPE_BOOL: return 1; case inference::DataType::TYPE_UINT8: return 1; case inference::DataType::TYPE_UINT16: return 2; case inference::DataType::TYPE_UINT32: return 4; case inference::DataType::TYPE_UINT64: return 8; case inference::DataType::TYPE_INT8: return 1; case inference::DataType::TYPE_INT16: return 2; case inference::DataType::TYPE_INT32: return 4; case inference::DataType::TYPE_INT64: return 8; case inference::DataType::TYPE_FP16: return 2; case inference::DataType::TYPE_FP32: return 4; case inference::DataType::TYPE_FP64: return 8; case inference::DataType::TYPE_STRING: return 0; case inference::DataType::TYPE_BF16: return 2; default: break; } return 0; } int64_t GetElementCount(const DimsList& dims) { bool first = true; int64_t cnt = 0; for (auto dim : dims) { if (dim == WILDCARD_DIM) { return -1; } if (first) { cnt = dim; first = false; } else { cnt *= dim; } } return cnt; } int64_t GetElementCount(const std::vector& dims) { bool first = true; int64_t cnt = 0; for (auto dim : dims) { if (dim == WILDCARD_DIM) { return -1; } if (first) { cnt = dim; first = false; } else { cnt *= dim; } } return cnt; } int64_t GetElementCount(const inference::ModelInput& mio) { return GetElementCount(mio.dims()); } int64_t GetElementCount(const inference::ModelOutput& mio) { return GetElementCount(mio.dims()); } int64_t GetByteSize(const inference::DataType& dtype, const DimsList& dims) { size_t dt_size = GetDataTypeByteSize(dtype); if (dt_size == 0) { return -1; } int64_t cnt = GetElementCount(dims); if (cnt == -1) { return -1; } return cnt * dt_size; } int64_t GetByteSize(const inference::DataType& dtype, const std::vector& dims) { size_t dt_size = GetDataTypeByteSize(dtype); if (dt_size == 0) { return -1; } int64_t cnt = GetElementCount(dims); if (cnt == -1) { return -1; } return cnt * dt_size; } int64_t GetByteSize( const int batch_size, const inference::DataType& dtype, const DimsList& dims) { if (dims.size() == 0) { return batch_size * GetDataTypeByteSize(dtype); } int64_t bs = GetByteSize(dtype, dims); if (bs == -1) { return -1; } return std::max(1, batch_size) * bs; } int64_t GetByteSize( const int batch_size, const inference::DataType& dtype, const std::vector& dims) { if (dims.size() == 0) { return batch_size * GetDataTypeByteSize(dtype); } int64_t bs = GetByteSize(dtype, dims); if (bs == -1) { return -1; } return std::max(1, batch_size) * bs; } int64_t GetByteSize(const inference::ModelInput& mio) { return GetByteSize(mio.data_type(), mio.dims()); } int64_t GetByteSize(const inference::ModelOutput& mio) { return GetByteSize(mio.data_type(), mio.dims()); } int GetCpuNiceLevel(const inference::ModelConfig& config) { int nice = SCHEDULER_DEFAULT_NICE; if (config.has_optimization()) { switch (config.optimization().priority()) { case inference::ModelOptimizationPolicy::PRIORITY_MAX: nice = 0; break; case inference::ModelOptimizationPolicy::PRIORITY_MIN: nice = 19; break; default: nice = SCHEDULER_DEFAULT_NICE; break; } } return nice; } bool CompareDims(const DimsList& dims0, const DimsList& dims1) { if (dims0.size() != dims1.size()) { return false; } for (int i = 0; i < dims0.size(); ++i) { if (dims0[i] != dims1[i]) { return false; } } return true; } bool CompareDims( const std::vector& dims0, const std::vector& dims1) { if (dims0.size() != dims1.size()) { return false; } for (size_t i = 0; i < dims0.size(); ++i) { if (dims0[i] != dims1[i]) { return false; } } return true; } bool CompareDimsWithWildcard(const DimsList& dims0, const DimsList& dims1) { if (dims0.size() != dims1.size()) { return false; } for (int i = 0; i < dims0.size(); ++i) { if ((dims0[i] != WILDCARD_DIM) && (dims1[i] != WILDCARD_DIM) && (dims0[i] != dims1[i])) { return false; } } return true; } bool CompareDimsWithWildcard( const DimsList& dims0, const std::vector& dims1) { if (dims0.size() != (int64_t)dims1.size()) { return false; } for (int i = 0; i < dims0.size(); ++i) { if ((dims0[i] != WILDCARD_DIM) && (dims1[i] != WILDCARD_DIM) && (dims0[i] != dims1[i])) { return false; } } return true; } std::string DimsListToString(const DimsList& dims) { bool first = true; std::string str("["); for (const auto& dim : dims) { if (!first) { str += ","; } str += std::to_string(dim); first = false; } str += "]"; return str; } std::string DimsListToString(const std::vector& dims, const int start_idx) { int idx = 0; std::string str("["); for (const auto& dim : dims) { if (idx >= start_idx) { if (idx > start_idx) { str += ","; } str += std::to_string(dim); } idx++; } str += "]"; return str; } const char* DataTypeToProtocolString(const inference::DataType dtype) { switch (dtype) { case inference::DataType::TYPE_BOOL: return "BOOL"; case inference::DataType::TYPE_UINT8: return "UINT8"; case inference::DataType::TYPE_UINT16: return "UINT16"; case inference::DataType::TYPE_UINT32: return "UINT32"; case inference::DataType::TYPE_UINT64: return "UINT64"; case inference::DataType::TYPE_INT8: return "INT8"; case inference::DataType::TYPE_INT16: return "INT16"; case inference::DataType::TYPE_INT32: return "INT32"; case inference::DataType::TYPE_INT64: return "INT64"; case inference::DataType::TYPE_FP16: return "FP16"; case inference::DataType::TYPE_FP32: return "FP32"; case inference::DataType::TYPE_FP64: return "FP64"; case inference::DataType::TYPE_STRING: return "BYTES"; case inference::DataType::TYPE_BF16: return "BF16"; default: break; } return ""; } inference::DataType ProtocolStringToDataType(const std::string& dtype) { return ProtocolStringToDataType(dtype.c_str(), dtype.size()); } inference::DataType ProtocolStringToDataType(const char* dtype, size_t len) { if (len < 4 || len > 6) { return inference::DataType::TYPE_INVALID; } if ((*dtype == 'I') && (len != 6)) { if ((dtype[1] == 'N') && (dtype[2] == 'T')) { if ((dtype[3] == '8') && (len == 4)) { return inference::DataType::TYPE_INT8; } else if ((dtype[3] == '1') && (dtype[4] == '6')) { return inference::DataType::TYPE_INT16; } else if ((dtype[3] == '3') && (dtype[4] == '2')) { return inference::DataType::TYPE_INT32; } else if ((dtype[3] == '6') && (dtype[4] == '4')) { return inference::DataType::TYPE_INT64; } } } else if ((*dtype == 'U') && (len != 4)) { if ((dtype[1] == 'I') && (dtype[2] == 'N') && (dtype[3] == 'T')) { if ((dtype[4] == '8') && (len == 5)) { return inference::DataType::TYPE_UINT8; } else if ((dtype[4] == '1') && (dtype[5] == '6')) { return inference::DataType::TYPE_UINT16; } else if ((dtype[4] == '3') && (dtype[5] == '2')) { return inference::DataType::TYPE_UINT32; } else if ((dtype[4] == '6') && (dtype[5] == '4')) { return inference::DataType::TYPE_UINT64; } } } else if ((*dtype == 'F') && (dtype[1] == 'P') && (len == 4)) { if ((dtype[2] == '1') && (dtype[3] == '6')) { return inference::DataType::TYPE_FP16; } else if ((dtype[2] == '3') && (dtype[3] == '2')) { return inference::DataType::TYPE_FP32; } else if ((dtype[2] == '6') && (dtype[3] == '4')) { return inference::DataType::TYPE_FP64; } } else if (*dtype == 'B') { switch (dtype[1]) { case 'Y': if (!strcmp(dtype + 2, "TES")) { return inference::DataType::TYPE_STRING; } break; case 'O': if (!strcmp(dtype + 2, "OL")) { return inference::DataType::TYPE_BOOL; } break; case 'F': if (!strcmp(dtype + 2, "16")) { return inference::DataType::TYPE_BF16; } break; } } return inference::DataType::TYPE_INVALID; } }} // namespace triton::common kytensor-2.49.0.7/core/src/model_config.h000066400000000000000000000243221527052371200201550ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "model_config.pb.h" namespace triton { namespace common { /// The type for a repeated dims field (used for shape). using DimsList = ::google::protobuf::RepeatedField<::google::protobuf::int64>; /// The type for the metric_tags map. using MetricTagsMap = ::google::protobuf::Map; // Map from a host policy name to map of cmdline // settings for the host policy. using HostPolicyCmdlineConfig = std::map; using HostPolicyCmdlineConfigMap = std::unordered_map; // Map from backend name to list of setting=value pairs of cmdline // settings for the backend. using BackendCmdlineConfig = std::vector>; using BackendCmdlineConfigMap = std::unordered_map; /// The value for a dimension in a shape that indicates that that /// dimension can take on any size. constexpr int WILDCARD_DIM = -1; constexpr int SCHEDULER_DEFAULT_NICE = 5; /// Enumeration for the different platform types. enum Platform { PLATFORM_UNKNOWN = 0, PLATFORM_TENSORRT_PLAN = 1, PLATFORM_TENSORFLOW_GRAPHDEF = 2, PLATFORM_TENSORFLOW_SAVEDMODEL = 3, PLATFORM_ENSEMBLE = 4, PLATFORM_ONNXRUNTIME_ONNX = 5, PLATFORM_PYTORCH_LIBTORCH = 6 }; /// Get the number of elements in a shape. /// \param dims The shape. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const DimsList& dims); /// Get the number of elements in a shape. /// \param dims The shape. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const std::vector& dims); /// Get the number of elements in the shape of a model input. /// \param mio The model input. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const inference::ModelInput& mio); /// Get the number of elements in the shape of a model output. /// \param mio The model output. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const inference::ModelOutput& mio); /// Are values of a datatype fixed-size, or variable-sized. /// \param dtype The data-type. /// \return True if datatype values are fixed-sized, false if /// variable-sized. bool IsFixedSizeDataType(const inference::DataType dtype); /// Get the size of objects of a given datatype in bytes. /// \param dtype The data-type. /// \return The size, in bytes, of objects of the datatype, or 0 if /// size cannot be determine (for example, values of type TYPE_STRING /// have variable length and so size cannot be determine just from the /// type). size_t GetDataTypeByteSize(const inference::DataType dtype); /// Get the size, in bytes, of a tensor based on datatype and /// shape. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize(const inference::DataType& dtype, const DimsList& dims); /// Get the size, in bytes, of a tensor based on datatype and /// shape. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize( const inference::DataType& dtype, const std::vector& dims); /// Get the size, in bytes, of a tensor based on batch-size, datatype /// and shape. A tensor that has empty shape [] and non-zero /// batch-size is sized as a tensor with shape [ batch-size ]. /// \param batch_size The batch-size. May be 0 to indicate no /// batching. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize( const int batch_size, const inference::DataType& dtype, const DimsList& dims); /// Get the size, in bytes, of a tensor based on batch-size, datatype /// and shape. A tensor that has empty shape [] and non-zero /// batch-size is sized as a tensor with shape [ batch-size ]. /// \param batch_size The batch-size. May be 0 to indicate no /// batching. /// \param dtype The data-type. /// \param dims The shape. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize( const int batch_size, const inference::DataType& dtype, const std::vector& dims); /// Get the size, in bytes, of a tensor based on ModelInput. /// \param mio The ModelInput protobuf. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize(const inference::ModelInput& mio); /// Get the size, in bytes, of a tensor based on ModelOutput. /// \param mio The ModelOutput protobuf. /// \return The size, in bytes, of the corresponding tensor, or -1 if /// unable to determine the size. int64_t GetByteSize(const inference::ModelOutput& mio); /// Get the CPU thread nice level associate with a model /// configuration's priority. /// \param config The model configuration. /// \return The nice level. int GetCpuNiceLevel(const inference::ModelConfig& config); /// Compare two model configuration shapes for equality. Wildcard /// dimensions (that is, dimensions with size WILDCARD_DIM) are /// compared literally so that to be equal the two shapes must both /// specify WILDCARD_DIM in the same dimensions. /// \params dims0 The first shape. /// \params dims1 The second shape. /// \return True if the shapes are equal, false if not equal. bool CompareDims(const DimsList& dims0, const DimsList& dims1); /// Compare two model configuration shapes for equality. Wildcard /// dimensions (that is, dimensions with size WILDCARD_DIM) are /// compared literally so that to be equal the two shapes must both /// specify WILDCARD_DIM in the same dimensions. /// \params dims0 The first shape. /// \params dims1 The second shape. /// \return True if the shapes are equal, false if not equal. bool CompareDims( const std::vector& dims0, const std::vector& dims1); /// Compare two model configuration shapes for equality. Wildcard /// dimensions (that is, dimensions with size WILDCARD_DIM) are /// allowed to match with any value. So, a dimension in one shape /// specified as WILDCARD_DIM will always match the same dimension in /// the other shape. /// \params dims0 The first shape. /// \params dims1 The second shape. /// \return True if the shapes are equal, false if not equal. bool CompareDimsWithWildcard(const DimsList& dims0, const DimsList& dims1); /// Compare two model configuration shapes for equality. Wildcard /// dimensions (that is, dimensions with size WILDCARD_DIM) are /// allowed to match with any value. So, a dimension in one shape /// specified as WILDCARD_DIM will always match the same dimension in /// the other shape. /// \params dims0 The first shape. /// \params dims1 The second shape. /// \return True if the shapes are equal, false if not equal. bool CompareDimsWithWildcard( const DimsList& dims0, const std::vector& dims1); /// Convert a DimsList to string representation. /// \param dims The DimsList to be converted. /// \return String representation of the DimsList in pattern /// "[d0,d1,...,dn]" std::string DimsListToString(const DimsList& dims); /// Convert a vector representing a shape to string representation. /// \param dims The vector of dimensions to be converted. /// \return String representation of the vector in pattern /// "[d0,d1,...,dn]" std::string DimsListToString( const std::vector& dims, const int start_idx = 0); /// Get the server protocol string representation of a datatype. /// \param dtype The data type. /// \return The string representation. const char* DataTypeToProtocolString(const inference::DataType dtype); /// Get the datatype corresponding to a server protocol string /// representation of a datatype. /// \param dtype string representation. /// \return The data type. inference::DataType ProtocolStringToDataType(const std::string& dtype); /// Get the datatype corresponding to a server protocol string /// representation of a datatype. /// \param dtype Pointer to string. /// \param len Length of the string. /// \return The data type. inference::DataType ProtocolStringToDataType(const char* dtype, size_t len); }} // namespace triton::common kytensor-2.49.0.7/core/src/model_config.pb.cc000066400000000000000000022623051527052371200207220ustar00rootroot00000000000000// Generated by the protocol buffer compiler. DO NOT EDIT! // source: model_config.proto #include "model_config.pb.h" #include #include #include #include #include #include #include #include // @@protoc_insertion_point(includes) #include PROTOBUF_PRAGMA_INIT_SEG namespace _pb = ::PROTOBUF_NAMESPACE_ID; namespace _pbi = _pb::internal; namespace inference { PROTOBUF_CONSTEXPR ModelRateLimiter_Resource::ModelRateLimiter_Resource( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.global_)*/false , /*decltype(_impl_.count_)*/0u , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelRateLimiter_ResourceDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelRateLimiter_ResourceDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelRateLimiter_ResourceDefaultTypeInternal() {} union { ModelRateLimiter_Resource _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelRateLimiter_ResourceDefaultTypeInternal _ModelRateLimiter_Resource_default_instance_; PROTOBUF_CONSTEXPR ModelRateLimiter::ModelRateLimiter( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.resources_)*/{} , /*decltype(_impl_.priority_)*/0u , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelRateLimiterDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelRateLimiterDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelRateLimiterDefaultTypeInternal() {} union { ModelRateLimiter _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelRateLimiterDefaultTypeInternal _ModelRateLimiter_default_instance_; PROTOBUF_CONSTEXPR ModelInstanceGroup_SecondaryDevice::ModelInstanceGroup_SecondaryDevice( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.device_id_)*/int64_t{0} , /*decltype(_impl_.kind_)*/0 , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelInstanceGroup_SecondaryDeviceDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelInstanceGroup_SecondaryDeviceDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelInstanceGroup_SecondaryDeviceDefaultTypeInternal() {} union { ModelInstanceGroup_SecondaryDevice _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelInstanceGroup_SecondaryDeviceDefaultTypeInternal _ModelInstanceGroup_SecondaryDevice_default_instance_; PROTOBUF_CONSTEXPR ModelInstanceGroup::ModelInstanceGroup( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.gpus_)*/{} , /*decltype(_impl_._gpus_cached_byte_size_)*/{0} , /*decltype(_impl_.profile_)*/{} , /*decltype(_impl_.secondary_devices_)*/{} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.host_policy_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.rate_limiter_)*/nullptr , /*decltype(_impl_.count_)*/0 , /*decltype(_impl_.kind_)*/0 , /*decltype(_impl_.passive_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelInstanceGroupDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelInstanceGroupDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelInstanceGroupDefaultTypeInternal() {} union { ModelInstanceGroup _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelInstanceGroupDefaultTypeInternal _ModelInstanceGroup_default_instance_; PROTOBUF_CONSTEXPR ModelTensorReshape::ModelTensorReshape( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.shape_)*/{} , /*decltype(_impl_._shape_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelTensorReshapeDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelTensorReshapeDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelTensorReshapeDefaultTypeInternal() {} union { ModelTensorReshape _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelTensorReshapeDefaultTypeInternal _ModelTensorReshape_default_instance_; PROTOBUF_CONSTEXPR ModelInput::ModelInput( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.dims_)*/{} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.reshape_)*/nullptr , /*decltype(_impl_.data_type_)*/0 , /*decltype(_impl_.format_)*/0 , /*decltype(_impl_.is_shape_tensor_)*/false , /*decltype(_impl_.allow_ragged_batch_)*/false , /*decltype(_impl_.optional_)*/false , /*decltype(_impl_.is_non_linear_format_io_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelInputDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelInputDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelInputDefaultTypeInternal() {} union { ModelInput _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelInputDefaultTypeInternal _ModelInput_default_instance_; PROTOBUF_CONSTEXPR ModelOutput::ModelOutput( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.dims_)*/{} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.label_filename_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.reshape_)*/nullptr , /*decltype(_impl_.data_type_)*/0 , /*decltype(_impl_.is_shape_tensor_)*/false , /*decltype(_impl_.is_non_linear_format_io_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOutputDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOutputDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOutputDefaultTypeInternal() {} union { ModelOutput _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOutputDefaultTypeInternal _ModelOutput_default_instance_; PROTOBUF_CONSTEXPR BatchInput::BatchInput( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.target_name_)*/{} , /*decltype(_impl_.source_input_)*/{} , /*decltype(_impl_.kind_)*/0 , /*decltype(_impl_.data_type_)*/0 , /*decltype(_impl_._cached_size_)*/{}} {} struct BatchInputDefaultTypeInternal { PROTOBUF_CONSTEXPR BatchInputDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~BatchInputDefaultTypeInternal() {} union { BatchInput _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 BatchInputDefaultTypeInternal _BatchInput_default_instance_; PROTOBUF_CONSTEXPR BatchOutput::BatchOutput( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.target_name_)*/{} , /*decltype(_impl_.source_input_)*/{} , /*decltype(_impl_.kind_)*/0 , /*decltype(_impl_._cached_size_)*/{}} {} struct BatchOutputDefaultTypeInternal { PROTOBUF_CONSTEXPR BatchOutputDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~BatchOutputDefaultTypeInternal() {} union { BatchOutput _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 BatchOutputDefaultTypeInternal _BatchOutput_default_instance_; PROTOBUF_CONSTEXPR ModelVersionPolicy_Latest::ModelVersionPolicy_Latest( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.num_versions_)*/0u , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelVersionPolicy_LatestDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelVersionPolicy_LatestDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelVersionPolicy_LatestDefaultTypeInternal() {} union { ModelVersionPolicy_Latest _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelVersionPolicy_LatestDefaultTypeInternal _ModelVersionPolicy_Latest_default_instance_; PROTOBUF_CONSTEXPR ModelVersionPolicy_All::ModelVersionPolicy_All( ::_pbi::ConstantInitialized) {} struct ModelVersionPolicy_AllDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelVersionPolicy_AllDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelVersionPolicy_AllDefaultTypeInternal() {} union { ModelVersionPolicy_All _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelVersionPolicy_AllDefaultTypeInternal _ModelVersionPolicy_All_default_instance_; PROTOBUF_CONSTEXPR ModelVersionPolicy_Specific::ModelVersionPolicy_Specific( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.versions_)*/{} , /*decltype(_impl_._versions_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelVersionPolicy_SpecificDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelVersionPolicy_SpecificDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelVersionPolicy_SpecificDefaultTypeInternal() {} union { ModelVersionPolicy_Specific _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelVersionPolicy_SpecificDefaultTypeInternal _ModelVersionPolicy_Specific_default_instance_; PROTOBUF_CONSTEXPR ModelVersionPolicy::ModelVersionPolicy( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.policy_choice_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}} {} struct ModelVersionPolicyDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelVersionPolicyDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelVersionPolicyDefaultTypeInternal() {} union { ModelVersionPolicy _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelVersionPolicyDefaultTypeInternal _ModelVersionPolicy_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Graph::ModelOptimizationPolicy_Graph( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.level_)*/0 , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicy_GraphDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_GraphDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_GraphDefaultTypeInternal() {} union { ModelOptimizationPolicy_Graph _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_GraphDefaultTypeInternal _ModelOptimizationPolicy_Graph_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_Shape::ModelOptimizationPolicy_Cuda_GraphSpec_Shape( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.dim_)*/{} , /*decltype(_impl_._dim_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicy_Cuda_GraphSpec_ShapeDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_ShapeDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_Cuda_GraphSpec_ShapeDefaultTypeInternal() {} union { ModelOptimizationPolicy_Cuda_GraphSpec_Shape _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_Cuda_GraphSpec_ShapeDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_Shape_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUseDefaultTypeInternal() {} union { ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUseDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.input_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.batch_size_)*/0 , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicy_Cuda_GraphSpec_LowerBoundDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_LowerBoundDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_Cuda_GraphSpec_LowerBoundDefaultTypeInternal() {} union { ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_Cuda_GraphSpec_LowerBoundDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUseDefaultTypeInternal() {} union { ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUseDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec::ModelOptimizationPolicy_Cuda_GraphSpec( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.input_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.graph_lower_bound_)*/nullptr , /*decltype(_impl_.batch_size_)*/0 , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicy_Cuda_GraphSpecDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpecDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_Cuda_GraphSpecDefaultTypeInternal() {} union { ModelOptimizationPolicy_Cuda_GraphSpec _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_Cuda_GraphSpecDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda::ModelOptimizationPolicy_Cuda( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.graph_spec_)*/{} , /*decltype(_impl_.graphs_)*/false , /*decltype(_impl_.busy_wait_events_)*/false , /*decltype(_impl_.output_copy_stream_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicy_CudaDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_CudaDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_CudaDefaultTypeInternal() {} union { ModelOptimizationPolicy_Cuda _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_CudaDefaultTypeInternal _ModelOptimizationPolicy_Cuda_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUseDefaultTypeInternal() {} union { ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUseDefaultTypeInternal _ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.parameters_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicy_ExecutionAccelerators_AcceleratorDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAccelerators_AcceleratorDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_ExecutionAccelerators_AcceleratorDefaultTypeInternal() {} union { ModelOptimizationPolicy_ExecutionAccelerators_Accelerator _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_ExecutionAccelerators_AcceleratorDefaultTypeInternal _ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAccelerators::ModelOptimizationPolicy_ExecutionAccelerators( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.gpu_execution_accelerator_)*/{} , /*decltype(_impl_.cpu_execution_accelerator_)*/{} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicy_ExecutionAcceleratorsDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAcceleratorsDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_ExecutionAcceleratorsDefaultTypeInternal() {} union { ModelOptimizationPolicy_ExecutionAccelerators _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_ExecutionAcceleratorsDefaultTypeInternal _ModelOptimizationPolicy_ExecutionAccelerators_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy_PinnedMemoryBuffer::ModelOptimizationPolicy_PinnedMemoryBuffer( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.enable_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicy_PinnedMemoryBufferDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicy_PinnedMemoryBufferDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicy_PinnedMemoryBufferDefaultTypeInternal() {} union { ModelOptimizationPolicy_PinnedMemoryBuffer _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicy_PinnedMemoryBufferDefaultTypeInternal _ModelOptimizationPolicy_PinnedMemoryBuffer_default_instance_; PROTOBUF_CONSTEXPR ModelOptimizationPolicy::ModelOptimizationPolicy( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.graph_)*/nullptr , /*decltype(_impl_.cuda_)*/nullptr , /*decltype(_impl_.execution_accelerators_)*/nullptr , /*decltype(_impl_.input_pinned_memory_)*/nullptr , /*decltype(_impl_.output_pinned_memory_)*/nullptr , /*decltype(_impl_.priority_)*/0 , /*decltype(_impl_.gather_kernel_buffer_threshold_)*/0u , /*decltype(_impl_.eager_batching_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOptimizationPolicyDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOptimizationPolicyDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOptimizationPolicyDefaultTypeInternal() {} union { ModelOptimizationPolicy _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOptimizationPolicyDefaultTypeInternal _ModelOptimizationPolicy_default_instance_; PROTOBUF_CONSTEXPR ModelQueuePolicy::ModelQueuePolicy( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.default_timeout_microseconds_)*/uint64_t{0u} , /*decltype(_impl_.timeout_action_)*/0 , /*decltype(_impl_.allow_timeout_override_)*/false , /*decltype(_impl_.max_queue_size_)*/0u , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelQueuePolicyDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelQueuePolicyDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelQueuePolicyDefaultTypeInternal() {} union { ModelQueuePolicy _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelQueuePolicyDefaultTypeInternal _ModelQueuePolicy_default_instance_; PROTOBUF_CONSTEXPR ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUseDefaultTypeInternal() {} union { ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUseDefaultTypeInternal _ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelDynamicBatching::ModelDynamicBatching( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.preferred_batch_size_)*/{} , /*decltype(_impl_._preferred_batch_size_cached_byte_size_)*/{0} , /*decltype(_impl_.priority_queue_policy_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.default_queue_policy_)*/nullptr , /*decltype(_impl_.max_queue_delay_microseconds_)*/uint64_t{0u} , /*decltype(_impl_.priority_levels_)*/uint64_t{0u} , /*decltype(_impl_.default_priority_level_)*/uint64_t{0u} , /*decltype(_impl_.preserve_ordering_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelDynamicBatchingDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelDynamicBatchingDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelDynamicBatchingDefaultTypeInternal() {} union { ModelDynamicBatching _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelDynamicBatchingDefaultTypeInternal _ModelDynamicBatching_default_instance_; PROTOBUF_CONSTEXPR ModelSequenceBatching_Control::ModelSequenceBatching_Control( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.int32_false_true_)*/{} , /*decltype(_impl_._int32_false_true_cached_byte_size_)*/{0} , /*decltype(_impl_.fp32_false_true_)*/{} , /*decltype(_impl_.bool_false_true_)*/{} , /*decltype(_impl_.kind_)*/0 , /*decltype(_impl_.data_type_)*/0 , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelSequenceBatching_ControlDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelSequenceBatching_ControlDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelSequenceBatching_ControlDefaultTypeInternal() {} union { ModelSequenceBatching_Control _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelSequenceBatching_ControlDefaultTypeInternal _ModelSequenceBatching_Control_default_instance_; PROTOBUF_CONSTEXPR ModelSequenceBatching_ControlInput::ModelSequenceBatching_ControlInput( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.control_)*/{} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelSequenceBatching_ControlInputDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelSequenceBatching_ControlInputDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelSequenceBatching_ControlInputDefaultTypeInternal() {} union { ModelSequenceBatching_ControlInput _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelSequenceBatching_ControlInputDefaultTypeInternal _ModelSequenceBatching_ControlInput_default_instance_; PROTOBUF_CONSTEXPR ModelSequenceBatching_InitialState::ModelSequenceBatching_InitialState( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.dims_)*/{} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.data_type_)*/0 , /*decltype(_impl_.state_data_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}} {} struct ModelSequenceBatching_InitialStateDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelSequenceBatching_InitialStateDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelSequenceBatching_InitialStateDefaultTypeInternal() {} union { ModelSequenceBatching_InitialState _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelSequenceBatching_InitialStateDefaultTypeInternal _ModelSequenceBatching_InitialState_default_instance_; PROTOBUF_CONSTEXPR ModelSequenceBatching_State::ModelSequenceBatching_State( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.dims_)*/{} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , /*decltype(_impl_.initial_state_)*/{} , /*decltype(_impl_.input_name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.output_name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.data_type_)*/0 , /*decltype(_impl_.use_same_buffer_for_input_output_)*/false , /*decltype(_impl_.use_growable_memory_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelSequenceBatching_StateDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelSequenceBatching_StateDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelSequenceBatching_StateDefaultTypeInternal() {} union { ModelSequenceBatching_State _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelSequenceBatching_StateDefaultTypeInternal _ModelSequenceBatching_State_default_instance_; PROTOBUF_CONSTEXPR ModelSequenceBatching_StrategyDirect::ModelSequenceBatching_StrategyDirect( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.max_queue_delay_microseconds_)*/uint64_t{0u} , /*decltype(_impl_.minimum_slot_utilization_)*/0 , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelSequenceBatching_StrategyDirectDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelSequenceBatching_StrategyDirectDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelSequenceBatching_StrategyDirectDefaultTypeInternal() {} union { ModelSequenceBatching_StrategyDirect _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelSequenceBatching_StrategyDirectDefaultTypeInternal _ModelSequenceBatching_StrategyDirect_default_instance_; PROTOBUF_CONSTEXPR ModelSequenceBatching_StrategyOldest::ModelSequenceBatching_StrategyOldest( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.preferred_batch_size_)*/{} , /*decltype(_impl_._preferred_batch_size_cached_byte_size_)*/{0} , /*decltype(_impl_.max_candidate_sequences_)*/0 , /*decltype(_impl_.preserve_ordering_)*/false , /*decltype(_impl_.max_queue_delay_microseconds_)*/uint64_t{0u} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelSequenceBatching_StrategyOldestDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelSequenceBatching_StrategyOldestDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelSequenceBatching_StrategyOldestDefaultTypeInternal() {} union { ModelSequenceBatching_StrategyOldest _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelSequenceBatching_StrategyOldestDefaultTypeInternal _ModelSequenceBatching_StrategyOldest_default_instance_; PROTOBUF_CONSTEXPR ModelSequenceBatching::ModelSequenceBatching( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.control_input_)*/{} , /*decltype(_impl_.state_)*/{} , /*decltype(_impl_.max_sequence_idle_microseconds_)*/uint64_t{0u} , /*decltype(_impl_.iterative_sequence_)*/false , /*decltype(_impl_.strategy_choice_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}} {} struct ModelSequenceBatchingDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelSequenceBatchingDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelSequenceBatchingDefaultTypeInternal() {} union { ModelSequenceBatching _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelSequenceBatchingDefaultTypeInternal _ModelSequenceBatching_default_instance_; PROTOBUF_CONSTEXPR ModelEnsembling_Step_InputMapEntry_DoNotUse::ModelEnsembling_Step_InputMapEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelEnsembling_Step_InputMapEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelEnsembling_Step_InputMapEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelEnsembling_Step_InputMapEntry_DoNotUseDefaultTypeInternal() {} union { ModelEnsembling_Step_InputMapEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelEnsembling_Step_InputMapEntry_DoNotUseDefaultTypeInternal _ModelEnsembling_Step_InputMapEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelEnsembling_Step_OutputMapEntry_DoNotUse::ModelEnsembling_Step_OutputMapEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelEnsembling_Step_OutputMapEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelEnsembling_Step_OutputMapEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelEnsembling_Step_OutputMapEntry_DoNotUseDefaultTypeInternal() {} union { ModelEnsembling_Step_OutputMapEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelEnsembling_Step_OutputMapEntry_DoNotUseDefaultTypeInternal _ModelEnsembling_Step_OutputMapEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelEnsembling_Step::ModelEnsembling_Step( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.input_map_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.output_map_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.model_name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.model_namespace_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.model_version_)*/int64_t{0} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelEnsembling_StepDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelEnsembling_StepDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelEnsembling_StepDefaultTypeInternal() {} union { ModelEnsembling_Step _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelEnsembling_StepDefaultTypeInternal _ModelEnsembling_Step_default_instance_; PROTOBUF_CONSTEXPR ModelEnsembling::ModelEnsembling( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.step_)*/{} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelEnsemblingDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelEnsemblingDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelEnsemblingDefaultTypeInternal() {} union { ModelEnsembling _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelEnsemblingDefaultTypeInternal _ModelEnsembling_default_instance_; PROTOBUF_CONSTEXPR ModelParameter::ModelParameter( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.string_value_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelParameterDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelParameterDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelParameterDefaultTypeInternal() {} union { ModelParameter _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelParameterDefaultTypeInternal _ModelParameter_default_instance_; PROTOBUF_CONSTEXPR ModelWarmup_Input::ModelWarmup_Input( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.dims_)*/{} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , /*decltype(_impl_.data_type_)*/0 , /*decltype(_impl_.input_data_type_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}} {} struct ModelWarmup_InputDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelWarmup_InputDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelWarmup_InputDefaultTypeInternal() {} union { ModelWarmup_Input _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelWarmup_InputDefaultTypeInternal _ModelWarmup_Input_default_instance_; PROTOBUF_CONSTEXPR ModelWarmup_InputsEntry_DoNotUse::ModelWarmup_InputsEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelWarmup_InputsEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelWarmup_InputsEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelWarmup_InputsEntry_DoNotUseDefaultTypeInternal() {} union { ModelWarmup_InputsEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelWarmup_InputsEntry_DoNotUseDefaultTypeInternal _ModelWarmup_InputsEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelWarmup::ModelWarmup( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.inputs_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.batch_size_)*/0u , /*decltype(_impl_.count_)*/0u , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelWarmupDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelWarmupDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelWarmupDefaultTypeInternal() {} union { ModelWarmup _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelWarmupDefaultTypeInternal _ModelWarmup_default_instance_; PROTOBUF_CONSTEXPR ModelOperations::ModelOperations( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.op_library_filename_)*/{} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelOperationsDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelOperationsDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelOperationsDefaultTypeInternal() {} union { ModelOperations _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelOperationsDefaultTypeInternal _ModelOperations_default_instance_; PROTOBUF_CONSTEXPR ModelTransactionPolicy::ModelTransactionPolicy( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.decoupled_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelTransactionPolicyDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelTransactionPolicyDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelTransactionPolicyDefaultTypeInternal() {} union { ModelTransactionPolicy _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelTransactionPolicyDefaultTypeInternal _ModelTransactionPolicy_default_instance_; PROTOBUF_CONSTEXPR ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelRepositoryAgents_Agent_ParametersEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelRepositoryAgents_Agent_ParametersEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelRepositoryAgents_Agent_ParametersEntry_DoNotUseDefaultTypeInternal() {} union { ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelRepositoryAgents_Agent_ParametersEntry_DoNotUseDefaultTypeInternal _ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelRepositoryAgents_Agent::ModelRepositoryAgents_Agent( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.parameters_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelRepositoryAgents_AgentDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelRepositoryAgents_AgentDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelRepositoryAgents_AgentDefaultTypeInternal() {} union { ModelRepositoryAgents_Agent _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelRepositoryAgents_AgentDefaultTypeInternal _ModelRepositoryAgents_Agent_default_instance_; PROTOBUF_CONSTEXPR ModelRepositoryAgents::ModelRepositoryAgents( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.agents_)*/{} , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelRepositoryAgentsDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelRepositoryAgentsDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelRepositoryAgentsDefaultTypeInternal() {} union { ModelRepositoryAgents _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelRepositoryAgentsDefaultTypeInternal _ModelRepositoryAgents_default_instance_; PROTOBUF_CONSTEXPR ModelResponseCache::ModelResponseCache( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.enable_)*/false , /*decltype(_impl_._cached_size_)*/{}} {} struct ModelResponseCacheDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelResponseCacheDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelResponseCacheDefaultTypeInternal() {} union { ModelResponseCache _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelResponseCacheDefaultTypeInternal _ModelResponseCache_default_instance_; PROTOBUF_CONSTEXPR ModelConfig_CcModelFilenamesEntry_DoNotUse::ModelConfig_CcModelFilenamesEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelConfig_CcModelFilenamesEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelConfig_CcModelFilenamesEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelConfig_CcModelFilenamesEntry_DoNotUseDefaultTypeInternal() {} union { ModelConfig_CcModelFilenamesEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelConfig_CcModelFilenamesEntry_DoNotUseDefaultTypeInternal _ModelConfig_CcModelFilenamesEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelConfig_MetricTagsEntry_DoNotUse::ModelConfig_MetricTagsEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelConfig_MetricTagsEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelConfig_MetricTagsEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelConfig_MetricTagsEntry_DoNotUseDefaultTypeInternal() {} union { ModelConfig_MetricTagsEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelConfig_MetricTagsEntry_DoNotUseDefaultTypeInternal _ModelConfig_MetricTagsEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelConfig_ParametersEntry_DoNotUse::ModelConfig_ParametersEntry_DoNotUse( ::_pbi::ConstantInitialized) {} struct ModelConfig_ParametersEntry_DoNotUseDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelConfig_ParametersEntry_DoNotUseDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelConfig_ParametersEntry_DoNotUseDefaultTypeInternal() {} union { ModelConfig_ParametersEntry_DoNotUse _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelConfig_ParametersEntry_DoNotUseDefaultTypeInternal _ModelConfig_ParametersEntry_DoNotUse_default_instance_; PROTOBUF_CONSTEXPR ModelConfig::ModelConfig( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.input_)*/{} , /*decltype(_impl_.output_)*/{} , /*decltype(_impl_.instance_group_)*/{} , /*decltype(_impl_.cc_model_filenames_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.metric_tags_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.parameters_)*/{::_pbi::ConstantInitialized()} , /*decltype(_impl_.model_warmup_)*/{} , /*decltype(_impl_.batch_input_)*/{} , /*decltype(_impl_.batch_output_)*/{} , /*decltype(_impl_.name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.platform_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.default_model_filename_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.backend_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.runtime_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.version_policy_)*/nullptr , /*decltype(_impl_.optimization_)*/nullptr , /*decltype(_impl_.model_operations_)*/nullptr , /*decltype(_impl_.model_transaction_policy_)*/nullptr , /*decltype(_impl_.model_repository_agents_)*/nullptr , /*decltype(_impl_.response_cache_)*/nullptr , /*decltype(_impl_.max_batch_size_)*/0 , /*decltype(_impl_.scheduling_choice_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}} {} struct ModelConfigDefaultTypeInternal { PROTOBUF_CONSTEXPR ModelConfigDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} ~ModelConfigDefaultTypeInternal() {} union { ModelConfig _instance; }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ModelConfigDefaultTypeInternal _ModelConfig_default_instance_; } // namespace inference static ::_pb::Metadata file_level_metadata_model_5fconfig_2eproto[53]; static const ::_pb::EnumDescriptor* file_level_enum_descriptors_model_5fconfig_2eproto[9]; static constexpr ::_pb::ServiceDescriptor const** file_level_service_descriptors_model_5fconfig_2eproto = nullptr; const uint32_t TableStruct_model_5fconfig_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelRateLimiter_Resource, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelRateLimiter_Resource, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelRateLimiter_Resource, _impl_.global_), PROTOBUF_FIELD_OFFSET(::inference::ModelRateLimiter_Resource, _impl_.count_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelRateLimiter, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelRateLimiter, _impl_.resources_), PROTOBUF_FIELD_OFFSET(::inference::ModelRateLimiter, _impl_.priority_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup_SecondaryDevice, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup_SecondaryDevice, _impl_.kind_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup_SecondaryDevice, _impl_.device_id_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.kind_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.count_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.rate_limiter_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.gpus_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.secondary_devices_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.profile_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.passive_), PROTOBUF_FIELD_OFFSET(::inference::ModelInstanceGroup, _impl_.host_policy_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelTensorReshape, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelTensorReshape, _impl_.shape_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.data_type_), PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.format_), PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.dims_), PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.reshape_), PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.is_shape_tensor_), PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.allow_ragged_batch_), PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.optional_), PROTOBUF_FIELD_OFFSET(::inference::ModelInput, _impl_.is_non_linear_format_io_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOutput, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOutput, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelOutput, _impl_.data_type_), PROTOBUF_FIELD_OFFSET(::inference::ModelOutput, _impl_.dims_), PROTOBUF_FIELD_OFFSET(::inference::ModelOutput, _impl_.reshape_), PROTOBUF_FIELD_OFFSET(::inference::ModelOutput, _impl_.label_filename_), PROTOBUF_FIELD_OFFSET(::inference::ModelOutput, _impl_.is_shape_tensor_), PROTOBUF_FIELD_OFFSET(::inference::ModelOutput, _impl_.is_non_linear_format_io_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::BatchInput, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::BatchInput, _impl_.kind_), PROTOBUF_FIELD_OFFSET(::inference::BatchInput, _impl_.target_name_), PROTOBUF_FIELD_OFFSET(::inference::BatchInput, _impl_.data_type_), PROTOBUF_FIELD_OFFSET(::inference::BatchInput, _impl_.source_input_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::BatchOutput, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::BatchOutput, _impl_.target_name_), PROTOBUF_FIELD_OFFSET(::inference::BatchOutput, _impl_.kind_), PROTOBUF_FIELD_OFFSET(::inference::BatchOutput, _impl_.source_input_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelVersionPolicy_Latest, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelVersionPolicy_Latest, _impl_.num_versions_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelVersionPolicy_All, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelVersionPolicy_Specific, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelVersionPolicy_Specific, _impl_.versions_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelVersionPolicy, _internal_metadata_), ~0u, // no _extensions_ PROTOBUF_FIELD_OFFSET(::inference::ModelVersionPolicy, _impl_._oneof_case_[0]), ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, PROTOBUF_FIELD_OFFSET(::inference::ModelVersionPolicy, _impl_.policy_choice_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Graph, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Graph, _impl_.level_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape, _impl_.dim_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse, value_), 0, 1, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound, _impl_.batch_size_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound, _impl_.input_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse, value_), 0, 1, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec, _impl_.batch_size_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec, _impl_.input_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda_GraphSpec, _impl_.graph_lower_bound_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda, _impl_.graphs_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda, _impl_.busy_wait_events_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda, _impl_.graph_spec_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_Cuda, _impl_.output_copy_stream_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse, value_), 0, 1, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator, _impl_.parameters_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators, _impl_.gpu_execution_accelerator_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_ExecutionAccelerators, _impl_.cpu_execution_accelerator_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_PinnedMemoryBuffer, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy_PinnedMemoryBuffer, _impl_.enable_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _impl_.graph_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _impl_.priority_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _impl_.cuda_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _impl_.execution_accelerators_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _impl_.input_pinned_memory_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _impl_.output_pinned_memory_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _impl_.gather_kernel_buffer_threshold_), PROTOBUF_FIELD_OFFSET(::inference::ModelOptimizationPolicy, _impl_.eager_batching_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelQueuePolicy, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelQueuePolicy, _impl_.timeout_action_), PROTOBUF_FIELD_OFFSET(::inference::ModelQueuePolicy, _impl_.default_timeout_microseconds_), PROTOBUF_FIELD_OFFSET(::inference::ModelQueuePolicy, _impl_.allow_timeout_override_), PROTOBUF_FIELD_OFFSET(::inference::ModelQueuePolicy, _impl_.max_queue_size_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse, value_), 0, 1, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching, _impl_.preferred_batch_size_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching, _impl_.max_queue_delay_microseconds_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching, _impl_.preserve_ordering_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching, _impl_.priority_levels_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching, _impl_.default_priority_level_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching, _impl_.default_queue_policy_), PROTOBUF_FIELD_OFFSET(::inference::ModelDynamicBatching, _impl_.priority_queue_policy_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_Control, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_Control, _impl_.kind_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_Control, _impl_.int32_false_true_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_Control, _impl_.fp32_false_true_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_Control, _impl_.bool_false_true_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_Control, _impl_.data_type_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_ControlInput, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_ControlInput, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_ControlInput, _impl_.control_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_InitialState, _internal_metadata_), ~0u, // no _extensions_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_InitialState, _impl_._oneof_case_[0]), ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_InitialState, _impl_.data_type_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_InitialState, _impl_.dims_), ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_InitialState, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_InitialState, _impl_.state_data_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_State, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_State, _impl_.input_name_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_State, _impl_.output_name_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_State, _impl_.data_type_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_State, _impl_.dims_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_State, _impl_.initial_state_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_State, _impl_.use_same_buffer_for_input_output_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_State, _impl_.use_growable_memory_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_StrategyDirect, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_StrategyDirect, _impl_.max_queue_delay_microseconds_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_StrategyDirect, _impl_.minimum_slot_utilization_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_StrategyOldest, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_StrategyOldest, _impl_.max_candidate_sequences_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_StrategyOldest, _impl_.preferred_batch_size_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_StrategyOldest, _impl_.max_queue_delay_microseconds_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching_StrategyOldest, _impl_.preserve_ordering_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching, _internal_metadata_), ~0u, // no _extensions_ PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching, _impl_._oneof_case_[0]), ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching, _impl_.max_sequence_idle_microseconds_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching, _impl_.control_input_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching, _impl_.state_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching, _impl_.iterative_sequence_), PROTOBUF_FIELD_OFFSET(::inference::ModelSequenceBatching, _impl_.strategy_choice_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse, value_), 0, 1, PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse, value_), 0, 1, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step, _impl_.model_name_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step, _impl_.model_version_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step, _impl_.input_map_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step, _impl_.output_map_), PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling_Step, _impl_.model_namespace_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelEnsembling, _impl_.step_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelParameter, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelParameter, _impl_.string_value_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_Input, _internal_metadata_), ~0u, // no _extensions_ PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_Input, _impl_._oneof_case_[0]), ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_Input, _impl_.data_type_), PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_Input, _impl_.dims_), ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_Input, _impl_.input_data_type_), PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_InputsEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_InputsEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_InputsEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup_InputsEntry_DoNotUse, value_), 0, 1, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup, _impl_.batch_size_), PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup, _impl_.inputs_), PROTOBUF_FIELD_OFFSET(::inference::ModelWarmup, _impl_.count_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelOperations, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelOperations, _impl_.op_library_filename_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelTransactionPolicy, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelTransactionPolicy, _impl_.decoupled_), PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse, value_), 0, 1, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents_Agent, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents_Agent, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents_Agent, _impl_.parameters_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelRepositoryAgents, _impl_.agents_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelResponseCache, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelResponseCache, _impl_.enable_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse, value_), 0, 1, PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_MetricTagsEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_MetricTagsEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_MetricTagsEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_MetricTagsEntry_DoNotUse, value_), 0, 1, PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_ParametersEntry_DoNotUse, _has_bits_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_ParametersEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_ParametersEntry_DoNotUse, key_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig_ParametersEntry_DoNotUse, value_), 0, 1, ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _internal_metadata_), ~0u, // no _extensions_ PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_._oneof_case_[0]), ~0u, // no _weak_field_map_ ~0u, // no _inlined_string_donated_ PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.name_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.platform_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.backend_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.runtime_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.version_policy_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.max_batch_size_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.input_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.output_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.batch_input_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.batch_output_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.optimization_), ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.instance_group_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.default_model_filename_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.cc_model_filenames_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.metric_tags_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.parameters_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.model_warmup_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.model_operations_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.model_transaction_policy_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.model_repository_agents_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.response_cache_), PROTOBUF_FIELD_OFFSET(::inference::ModelConfig, _impl_.scheduling_choice_), }; static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, -1, sizeof(::inference::ModelRateLimiter_Resource)}, { 9, -1, -1, sizeof(::inference::ModelRateLimiter)}, { 17, -1, -1, sizeof(::inference::ModelInstanceGroup_SecondaryDevice)}, { 25, -1, -1, sizeof(::inference::ModelInstanceGroup)}, { 40, -1, -1, sizeof(::inference::ModelTensorReshape)}, { 47, -1, -1, sizeof(::inference::ModelInput)}, { 62, -1, -1, sizeof(::inference::ModelOutput)}, { 75, -1, -1, sizeof(::inference::BatchInput)}, { 85, -1, -1, sizeof(::inference::BatchOutput)}, { 94, -1, -1, sizeof(::inference::ModelVersionPolicy_Latest)}, { 101, -1, -1, sizeof(::inference::ModelVersionPolicy_All)}, { 107, -1, -1, sizeof(::inference::ModelVersionPolicy_Specific)}, { 114, -1, -1, sizeof(::inference::ModelVersionPolicy)}, { 124, -1, -1, sizeof(::inference::ModelOptimizationPolicy_Graph)}, { 131, -1, -1, sizeof(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape)}, { 138, 146, -1, sizeof(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse)}, { 148, -1, -1, sizeof(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound)}, { 156, 164, -1, sizeof(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse)}, { 166, -1, -1, sizeof(::inference::ModelOptimizationPolicy_Cuda_GraphSpec)}, { 175, -1, -1, sizeof(::inference::ModelOptimizationPolicy_Cuda)}, { 185, 193, -1, sizeof(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse)}, { 195, -1, -1, sizeof(::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator)}, { 203, -1, -1, sizeof(::inference::ModelOptimizationPolicy_ExecutionAccelerators)}, { 211, -1, -1, sizeof(::inference::ModelOptimizationPolicy_PinnedMemoryBuffer)}, { 218, -1, -1, sizeof(::inference::ModelOptimizationPolicy)}, { 232, -1, -1, sizeof(::inference::ModelQueuePolicy)}, { 242, 250, -1, sizeof(::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse)}, { 252, -1, -1, sizeof(::inference::ModelDynamicBatching)}, { 265, -1, -1, sizeof(::inference::ModelSequenceBatching_Control)}, { 276, -1, -1, sizeof(::inference::ModelSequenceBatching_ControlInput)}, { 284, -1, -1, sizeof(::inference::ModelSequenceBatching_InitialState)}, { 296, -1, -1, sizeof(::inference::ModelSequenceBatching_State)}, { 309, -1, -1, sizeof(::inference::ModelSequenceBatching_StrategyDirect)}, { 317, -1, -1, sizeof(::inference::ModelSequenceBatching_StrategyOldest)}, { 327, -1, -1, sizeof(::inference::ModelSequenceBatching)}, { 340, 348, -1, sizeof(::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse)}, { 350, 358, -1, sizeof(::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse)}, { 360, -1, -1, sizeof(::inference::ModelEnsembling_Step)}, { 371, -1, -1, sizeof(::inference::ModelEnsembling)}, { 378, -1, -1, sizeof(::inference::ModelParameter)}, { 385, -1, -1, sizeof(::inference::ModelWarmup_Input)}, { 397, 405, -1, sizeof(::inference::ModelWarmup_InputsEntry_DoNotUse)}, { 407, -1, -1, sizeof(::inference::ModelWarmup)}, { 417, -1, -1, sizeof(::inference::ModelOperations)}, { 424, -1, -1, sizeof(::inference::ModelTransactionPolicy)}, { 431, 439, -1, sizeof(::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse)}, { 441, -1, -1, sizeof(::inference::ModelRepositoryAgents_Agent)}, { 449, -1, -1, sizeof(::inference::ModelRepositoryAgents)}, { 456, -1, -1, sizeof(::inference::ModelResponseCache)}, { 463, 471, -1, sizeof(::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse)}, { 473, 481, -1, sizeof(::inference::ModelConfig_MetricTagsEntry_DoNotUse)}, { 483, 491, -1, sizeof(::inference::ModelConfig_ParametersEntry_DoNotUse)}, { 493, -1, -1, sizeof(::inference::ModelConfig)}, }; static const ::_pb::Message* const file_default_instances[] = { &::inference::_ModelRateLimiter_Resource_default_instance_._instance, &::inference::_ModelRateLimiter_default_instance_._instance, &::inference::_ModelInstanceGroup_SecondaryDevice_default_instance_._instance, &::inference::_ModelInstanceGroup_default_instance_._instance, &::inference::_ModelTensorReshape_default_instance_._instance, &::inference::_ModelInput_default_instance_._instance, &::inference::_ModelOutput_default_instance_._instance, &::inference::_BatchInput_default_instance_._instance, &::inference::_BatchOutput_default_instance_._instance, &::inference::_ModelVersionPolicy_Latest_default_instance_._instance, &::inference::_ModelVersionPolicy_All_default_instance_._instance, &::inference::_ModelVersionPolicy_Specific_default_instance_._instance, &::inference::_ModelVersionPolicy_default_instance_._instance, &::inference::_ModelOptimizationPolicy_Graph_default_instance_._instance, &::inference::_ModelOptimizationPolicy_Cuda_GraphSpec_Shape_default_instance_._instance, &::inference::_ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse_default_instance_._instance, &::inference::_ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_default_instance_._instance, &::inference::_ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse_default_instance_._instance, &::inference::_ModelOptimizationPolicy_Cuda_GraphSpec_default_instance_._instance, &::inference::_ModelOptimizationPolicy_Cuda_default_instance_._instance, &::inference::_ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse_default_instance_._instance, &::inference::_ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_default_instance_._instance, &::inference::_ModelOptimizationPolicy_ExecutionAccelerators_default_instance_._instance, &::inference::_ModelOptimizationPolicy_PinnedMemoryBuffer_default_instance_._instance, &::inference::_ModelOptimizationPolicy_default_instance_._instance, &::inference::_ModelQueuePolicy_default_instance_._instance, &::inference::_ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse_default_instance_._instance, &::inference::_ModelDynamicBatching_default_instance_._instance, &::inference::_ModelSequenceBatching_Control_default_instance_._instance, &::inference::_ModelSequenceBatching_ControlInput_default_instance_._instance, &::inference::_ModelSequenceBatching_InitialState_default_instance_._instance, &::inference::_ModelSequenceBatching_State_default_instance_._instance, &::inference::_ModelSequenceBatching_StrategyDirect_default_instance_._instance, &::inference::_ModelSequenceBatching_StrategyOldest_default_instance_._instance, &::inference::_ModelSequenceBatching_default_instance_._instance, &::inference::_ModelEnsembling_Step_InputMapEntry_DoNotUse_default_instance_._instance, &::inference::_ModelEnsembling_Step_OutputMapEntry_DoNotUse_default_instance_._instance, &::inference::_ModelEnsembling_Step_default_instance_._instance, &::inference::_ModelEnsembling_default_instance_._instance, &::inference::_ModelParameter_default_instance_._instance, &::inference::_ModelWarmup_Input_default_instance_._instance, &::inference::_ModelWarmup_InputsEntry_DoNotUse_default_instance_._instance, &::inference::_ModelWarmup_default_instance_._instance, &::inference::_ModelOperations_default_instance_._instance, &::inference::_ModelTransactionPolicy_default_instance_._instance, &::inference::_ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse_default_instance_._instance, &::inference::_ModelRepositoryAgents_Agent_default_instance_._instance, &::inference::_ModelRepositoryAgents_default_instance_._instance, &::inference::_ModelResponseCache_default_instance_._instance, &::inference::_ModelConfig_CcModelFilenamesEntry_DoNotUse_default_instance_._instance, &::inference::_ModelConfig_MetricTagsEntry_DoNotUse_default_instance_._instance, &::inference::_ModelConfig_ParametersEntry_DoNotUse_default_instance_._instance, &::inference::_ModelConfig_default_instance_._instance, }; const char descriptor_table_protodef_model_5fconfig_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\022model_config.proto\022\tinference\"\226\001\n\020Mode" "lRateLimiter\0227\n\tresources\030\001 \003(\0132$.infere" "nce.ModelRateLimiter.Resource\022\020\n\010priorit" "y\030\002 \001(\r\0327\n\010Resource\022\014\n\004name\030\001 \001(\t\022\016\n\006glo" "bal\030\002 \001(\010\022\r\n\005count\030\003 \001(\r\"\207\004\n\022ModelInstan" "ceGroup\022\014\n\004name\030\001 \001(\t\0220\n\004kind\030\004 \001(\0162\".in" "ference.ModelInstanceGroup.Kind\022\r\n\005count" "\030\002 \001(\005\0221\n\014rate_limiter\030\006 \001(\0132\033.inference" ".ModelRateLimiter\022\014\n\004gpus\030\003 \003(\005\022H\n\021secon" "dary_devices\030\010 \003(\0132-.inference.ModelInst" "anceGroup.SecondaryDevice\022\017\n\007profile\030\005 \003" "(\t\022\017\n\007passive\030\007 \001(\010\022\023\n\013host_policy\030\t \001(\t" "\032\234\001\n\017SecondaryDevice\022O\n\004kind\030\001 \001(\0162A.inf" "erence.ModelInstanceGroup.SecondaryDevic" "e.SecondaryDeviceKind\022\021\n\tdevice_id\030\002 \001(\003" "\"%\n\023SecondaryDeviceKind\022\016\n\nKIND_NVDLA\020\000\"" "A\n\004Kind\022\r\n\tKIND_AUTO\020\000\022\014\n\010KIND_GPU\020\001\022\014\n\010" "KIND_CPU\020\002\022\016\n\nKIND_MODEL\020\003\"#\n\022ModelTenso" "rReshape\022\r\n\005shape\030\001 \003(\003\"\323\002\n\nModelInput\022\014" "\n\004name\030\001 \001(\t\022&\n\tdata_type\030\002 \001(\0162\023.infere" "nce.DataType\022,\n\006format\030\003 \001(\0162\034.inference" ".ModelInput.Format\022\014\n\004dims\030\004 \003(\003\022.\n\007resh" "ape\030\005 \001(\0132\035.inference.ModelTensorReshape" "\022\027\n\017is_shape_tensor\030\006 \001(\010\022\032\n\022allow_ragge" "d_batch\030\007 \001(\010\022\020\n\010optional\030\010 \001(\010\022\037\n\027is_no" "n_linear_format_io\030\t \001(\010\";\n\006Format\022\017\n\013FO" "RMAT_NONE\020\000\022\017\n\013FORMAT_NHWC\020\001\022\017\n\013FORMAT_N" "CHW\020\002\"\323\001\n\013ModelOutput\022\014\n\004name\030\001 \001(\t\022&\n\td" "ata_type\030\002 \001(\0162\023.inference.DataType\022\014\n\004d" "ims\030\003 \003(\003\022.\n\007reshape\030\005 \001(\0132\035.inference.M" "odelTensorReshape\022\026\n\016label_filename\030\004 \001(" "\t\022\027\n\017is_shape_tensor\030\006 \001(\010\022\037\n\027is_non_lin" "ear_format_io\030\007 \001(\010\"\331\002\n\nBatchInput\022(\n\004ki" "nd\030\001 \001(\0162\032.inference.BatchInput.Kind\022\023\n\013" "target_name\030\002 \003(\t\022&\n\tdata_type\030\003 \001(\0162\023.i" "nference.DataType\022\024\n\014source_input\030\004 \003(\t\"" "\315\001\n\004Kind\022\027\n\023BATCH_ELEMENT_COUNT\020\000\022#\n\037BAT" "CH_ACCUMULATED_ELEMENT_COUNT\020\001\022-\n)BATCH_" "ACCUMULATED_ELEMENT_COUNT_WITH_ZERO\020\002\022$\n" " BATCH_MAX_ELEMENT_COUNT_AS_SHAPE\020\003\022\024\n\020B" "ATCH_ITEM_SHAPE\020\004\022\034\n\030BATCH_ITEM_SHAPE_FL" "ATTEN\020\005\"\217\001\n\013BatchOutput\022\023\n\013target_name\030\001" " \003(\t\022)\n\004kind\030\002 \001(\0162\033.inference.BatchOutp" "ut.Kind\022\024\n\014source_input\030\003 \003(\t\"*\n\004Kind\022\"\n" "\036BATCH_SCATTER_WITH_INPUT_SHAPE\020\000\"\220\002\n\022Mo" "delVersionPolicy\0226\n\006latest\030\001 \001(\0132$.infer" "ence.ModelVersionPolicy.LatestH\000\0220\n\003all\030" "\002 \001(\0132!.inference.ModelVersionPolicy.All" "H\000\022:\n\010specific\030\003 \001(\0132&.inference.ModelVe" "rsionPolicy.SpecificH\000\032\036\n\006Latest\022\024\n\014num_" "versions\030\001 \001(\r\032\005\n\003All\032\034\n\010Specific\022\020\n\010ver" "sions\030\001 \003(\003B\017\n\rpolicy_choice\"\375\r\n\027ModelOp" "timizationPolicy\0227\n\005graph\030\001 \001(\0132(.infere" "nce.ModelOptimizationPolicy.Graph\022B\n\010pri" "ority\030\002 \001(\01620.inference.ModelOptimizatio" "nPolicy.ModelPriority\0225\n\004cuda\030\003 \001(\0132\'.in" "ference.ModelOptimizationPolicy.Cuda\022X\n\026" "execution_accelerators\030\004 \001(\01328.inference" ".ModelOptimizationPolicy.ExecutionAccele" "rators\022R\n\023input_pinned_memory\030\005 \001(\01325.in" "ference.ModelOptimizationPolicy.PinnedMe" "moryBuffer\022S\n\024output_pinned_memory\030\006 \001(\013" "25.inference.ModelOptimizationPolicy.Pin" "nedMemoryBuffer\022&\n\036gather_kernel_buffer_" "threshold\030\007 \001(\r\022\026\n\016eager_batching\030\010 \001(\010\032" "\026\n\005Graph\022\r\n\005level\030\001 \001(\005\032\272\005\n\004Cuda\022\016\n\006grap" "hs\030\001 \001(\010\022\030\n\020busy_wait_events\030\002 \001(\010\022E\n\ngr" "aph_spec\030\003 \003(\01321.inference.ModelOptimiza" "tionPolicy.Cuda.GraphSpec\022\032\n\022output_copy" "_stream\030\004 \001(\010\032\244\004\n\tGraphSpec\022\022\n\nbatch_siz" "e\030\001 \001(\005\022K\n\005input\030\002 \003(\0132<.inference.Model" "OptimizationPolicy.Cuda.GraphSpec.InputE" "ntry\022W\n\021graph_lower_bound\030\003 \001(\0132<.infere" "nce.ModelOptimizationPolicy.Cuda.GraphSp" "ec.LowerBound\032\024\n\005Shape\022\013\n\003dim\030\001 \003(\003\032\337\001\n\n" "LowerBound\022\022\n\nbatch_size\030\001 \001(\005\022V\n\005input\030" "\002 \003(\0132G.inference.ModelOptimizationPolic" "y.Cuda.GraphSpec.LowerBound.InputEntry\032e" "\n\nInputEntry\022\013\n\003key\030\001 \001(\t\022F\n\005value\030\002 \001(\013" "27.inference.ModelOptimizationPolicy.Cud" "a.GraphSpec.Shape:\0028\001\032e\n\nInputEntry\022\013\n\003k" "ey\030\001 \001(\t\022F\n\005value\030\002 \001(\01327.inference.Mode" "lOptimizationPolicy.Cuda.GraphSpec.Shape" ":\0028\001\032\244\003\n\025ExecutionAccelerators\022g\n\031gpu_ex" "ecution_accelerator\030\001 \003(\0132D.inference.Mo" "delOptimizationPolicy.ExecutionAccelerat" "ors.Accelerator\022g\n\031cpu_execution_acceler" "ator\030\002 \003(\0132D.inference.ModelOptimization" "Policy.ExecutionAccelerators.Accelerator" "\032\270\001\n\013Accelerator\022\014\n\004name\030\001 \001(\t\022h\n\nparame" "ters\030\002 \003(\0132T.inference.ModelOptimization" "Policy.ExecutionAccelerators.Accelerator" ".ParametersEntry\0321\n\017ParametersEntry\022\013\n\003k" "ey\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\032$\n\022PinnedMem" "oryBuffer\022\016\n\006enable\030\001 \001(\010\"I\n\rModelPriori" "ty\022\024\n\020PRIORITY_DEFAULT\020\000\022\020\n\014PRIORITY_MAX" "\020\001\022\020\n\014PRIORITY_MIN\020\002\"\333\001\n\020ModelQueuePolic" "y\022A\n\016timeout_action\030\001 \001(\0162).inference.Mo" "delQueuePolicy.TimeoutAction\022$\n\034default_" "timeout_microseconds\030\002 \001(\004\022\036\n\026allow_time" "out_override\030\003 \001(\010\022\026\n\016max_queue_size\030\004 \001" "(\r\"&\n\rTimeoutAction\022\n\n\006REJECT\020\000\022\t\n\005DELAY" "\020\001\"\233\003\n\024ModelDynamicBatching\022\034\n\024preferred" "_batch_size\030\001 \003(\005\022$\n\034max_queue_delay_mic" "roseconds\030\002 \001(\004\022\031\n\021preserve_ordering\030\003 \001" "(\010\022\027\n\017priority_levels\030\004 \001(\004\022\036\n\026default_p" "riority_level\030\005 \001(\004\0229\n\024default_queue_pol" "icy\030\006 \001(\0132\033.inference.ModelQueuePolicy\022W" "\n\025priority_queue_policy\030\007 \003(\01328.inferenc" "e.ModelDynamicBatching.PriorityQueuePoli" "cyEntry\032W\n\030PriorityQueuePolicyEntry\022\013\n\003k" "ey\030\001 \001(\004\022*\n\005value\030\002 \001(\0132\033.inference.Mode" "lQueuePolicy:\0028\001\"\356\n\n\025ModelSequenceBatchi" "ng\022A\n\006direct\030\003 \001(\0132/.inference.ModelSequ" "enceBatching.StrategyDirectH\000\022A\n\006oldest\030" "\004 \001(\0132/.inference.ModelSequenceBatching." "StrategyOldestH\000\022&\n\036max_sequence_idle_mi" "croseconds\030\001 \001(\004\022D\n\rcontrol_input\030\002 \003(\0132" "-.inference.ModelSequenceBatching.Contro" "lInput\0225\n\005state\030\005 \003(\0132&.inference.ModelS" "equenceBatching.State\022\032\n\022iterative_seque" "nce\030\006 \001(\010\032\261\002\n\007Control\022;\n\004kind\030\001 \001(\0162-.in" "ference.ModelSequenceBatching.Control.Ki" "nd\022\030\n\020int32_false_true\030\002 \003(\005\022\027\n\017fp32_fal" "se_true\030\003 \003(\002\022\027\n\017bool_false_true\030\005 \003(\010\022&" "\n\tdata_type\030\004 \001(\0162\023.inference.DataType\"u" "\n\004Kind\022\032\n\026CONTROL_SEQUENCE_START\020\000\022\032\n\026CO" "NTROL_SEQUENCE_READY\020\001\022\030\n\024CONTROL_SEQUEN" "CE_END\020\002\022\033\n\027CONTROL_SEQUENCE_CORRID\020\003\032W\n" "\014ControlInput\022\014\n\004name\030\001 \001(\t\0229\n\007control\030\002" " \003(\0132(.inference.ModelSequenceBatching.C" "ontrol\032\212\001\n\014InitialState\022&\n\tdata_type\030\001 \001" "(\0162\023.inference.DataType\022\014\n\004dims\030\002 \003(\003\022\023\n" "\tzero_data\030\003 \001(\010H\000\022\023\n\tdata_file\030\004 \001(\tH\000\022" "\014\n\004name\030\005 \001(\tB\014\n\nstate_data\032\363\001\n\005State\022\022\n" "\ninput_name\030\001 \001(\t\022\023\n\013output_name\030\002 \001(\t\022&" "\n\tdata_type\030\003 \001(\0162\023.inference.DataType\022\014" "\n\004dims\030\004 \003(\003\022D\n\rinitial_state\030\005 \003(\0132-.in" "ference.ModelSequenceBatching.InitialSta" "te\022(\n use_same_buffer_for_input_output\030\006" " \001(\010\022\033\n\023use_growable_memory\030\007 \001(\010\032X\n\016Str" "ategyDirect\022$\n\034max_queue_delay_microseco" "nds\030\001 \001(\004\022 \n\030minimum_slot_utilization\030\002 " "\001(\002\032\220\001\n\016StrategyOldest\022\037\n\027max_candidate_" "sequences\030\001 \001(\005\022\034\n\024preferred_batch_size\030" "\002 \003(\005\022$\n\034max_queue_delay_microseconds\030\003 " "\001(\004\022\031\n\021preserve_ordering\030\004 \001(\010B\021\n\017strate" "gy_choice\"\366\002\n\017ModelEnsembling\022-\n\004step\030\001 " "\003(\0132\037.inference.ModelEnsembling.Step\032\263\002\n" "\004Step\022\022\n\nmodel_name\030\001 \001(\t\022\025\n\rmodel_versi" "on\030\002 \001(\003\022@\n\tinput_map\030\003 \003(\0132-.inference." "ModelEnsembling.Step.InputMapEntry\022B\n\nou" "tput_map\030\004 \003(\0132..inference.ModelEnsembli" "ng.Step.OutputMapEntry\022\027\n\017model_namespac" "e\030\005 \001(\t\032/\n\rInputMapEntry\022\013\n\003key\030\001 \001(\t\022\r\n" "\005value\030\002 \001(\t:\0028\001\0320\n\016OutputMapEntry\022\013\n\003ke" "y\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\"&\n\016ModelParam" "eter\022\024\n\014string_value\030\001 \001(\t\"\331\002\n\013ModelWarm" "up\022\014\n\004name\030\001 \001(\t\022\022\n\nbatch_size\030\002 \001(\r\0222\n\006" "inputs\030\003 \003(\0132\".inference.ModelWarmup.Inp" "utsEntry\022\r\n\005count\030\004 \001(\r\032\227\001\n\005Input\022&\n\tdat" "a_type\030\001 \001(\0162\023.inference.DataType\022\014\n\004dim" "s\030\002 \003(\003\022\023\n\tzero_data\030\003 \001(\010H\000\022\025\n\013random_d" "ata\030\004 \001(\010H\000\022\031\n\017input_data_file\030\005 \001(\tH\000B\021" "\n\017input_data_type\032K\n\013InputsEntry\022\013\n\003key\030" "\001 \001(\t\022+\n\005value\030\002 \001(\0132\034.inference.ModelWa" "rmup.Input:\0028\001\".\n\017ModelOperations\022\033\n\023op_" "library_filename\030\001 \003(\t\"+\n\026ModelTransacti" "onPolicy\022\021\n\tdecoupled\030\001 \001(\010\"\346\001\n\025ModelRep" "ositoryAgents\0226\n\006agents\030\001 \003(\0132&.inferenc" "e.ModelRepositoryAgents.Agent\032\224\001\n\005Agent\022" "\014\n\004name\030\001 \001(\t\022J\n\nparameters\030\002 \003(\01326.infe" "rence.ModelRepositoryAgents.Agent.Parame" "tersEntry\0321\n\017ParametersEntry\022\013\n\003key\030\001 \001(" "\t\022\r\n\005value\030\002 \001(\t:\0028\001\"$\n\022ModelResponseCac" "he\022\016\n\006enable\030\001 \001(\010\"\303\n\n\013ModelConfig\022\014\n\004na" "me\030\001 \001(\t\022\020\n\010platform\030\002 \001(\t\022\017\n\007backend\030\021 " "\001(\t\022\017\n\007runtime\030\031 \001(\t\0225\n\016version_policy\030\003" " \001(\0132\035.inference.ModelVersionPolicy\022\026\n\016m" "ax_batch_size\030\004 \001(\005\022$\n\005input\030\005 \003(\0132\025.inf" "erence.ModelInput\022&\n\006output\030\006 \003(\0132\026.infe" "rence.ModelOutput\022*\n\013batch_input\030\024 \003(\0132\025" ".inference.BatchInput\022,\n\014batch_output\030\025 " "\003(\0132\026.inference.BatchOutput\0228\n\014optimizat" "ion\030\014 \001(\0132\".inference.ModelOptimizationP" "olicy\022;\n\020dynamic_batching\030\013 \001(\0132\037.infere" "nce.ModelDynamicBatchingH\000\022=\n\021sequence_b" "atching\030\r \001(\0132 .inference.ModelSequenceB" "atchingH\000\0229\n\023ensemble_scheduling\030\017 \001(\0132\032" ".inference.ModelEnsemblingH\000\0225\n\016instance" "_group\030\007 \003(\0132\035.inference.ModelInstanceGr" "oup\022\036\n\026default_model_filename\030\010 \001(\t\022H\n\022c" "c_model_filenames\030\t \003(\0132,.inference.Mode" "lConfig.CcModelFilenamesEntry\022;\n\013metric_" "tags\030\n \003(\0132&.inference.ModelConfig.Metri" "cTagsEntry\022:\n\nparameters\030\016 \003(\0132&.inferen" "ce.ModelConfig.ParametersEntry\022,\n\014model_" "warmup\030\020 \003(\0132\026.inference.ModelWarmup\0224\n\020" "model_operations\030\022 \001(\0132\032.inference.Model" "Operations\022C\n\030model_transaction_policy\030\023" " \001(\0132!.inference.ModelTransactionPolicy\022" "A\n\027model_repository_agents\030\027 \001(\0132 .infer" "ence.ModelRepositoryAgents\0225\n\016response_c" "ache\030\030 \001(\0132\035.inference.ModelResponseCach" "e\0327\n\025CcModelFilenamesEntry\022\013\n\003key\030\001 \001(\t\022" "\r\n\005value\030\002 \001(\t:\0028\001\0321\n\017MetricTagsEntry\022\013\n" "\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\032L\n\017Paramet" "ersEntry\022\013\n\003key\030\001 \001(\t\022(\n\005value\030\002 \001(\0132\031.i" "nference.ModelParameter:\0028\001B\023\n\021schedulin" "g_choice*\372\001\n\010DataType\022\020\n\014TYPE_INVALID\020\000\022" "\r\n\tTYPE_BOOL\020\001\022\016\n\nTYPE_UINT8\020\002\022\017\n\013TYPE_U" "INT16\020\003\022\017\n\013TYPE_UINT32\020\004\022\017\n\013TYPE_UINT64\020" "\005\022\r\n\tTYPE_INT8\020\006\022\016\n\nTYPE_INT16\020\007\022\016\n\nTYPE" "_INT32\020\010\022\016\n\nTYPE_INT64\020\t\022\r\n\tTYPE_FP16\020\n\022" "\r\n\tTYPE_FP32\020\013\022\r\n\tTYPE_FP64\020\014\022\017\n\013TYPE_ST" "RING\020\r\022\r\n\tTYPE_BF16\020\016b\006proto3" ; static ::_pbi::once_flag descriptor_table_model_5fconfig_2eproto_once; const ::_pbi::DescriptorTable descriptor_table_model_5fconfig_2eproto = { false, false, 8629, descriptor_table_protodef_model_5fconfig_2eproto, "model_config.proto", &descriptor_table_model_5fconfig_2eproto_once, nullptr, 0, 53, schemas, file_default_instances, TableStruct_model_5fconfig_2eproto::offsets, file_level_metadata_model_5fconfig_2eproto, file_level_enum_descriptors_model_5fconfig_2eproto, file_level_service_descriptors_model_5fconfig_2eproto, }; PROTOBUF_ATTRIBUTE_WEAK const ::_pbi::DescriptorTable* descriptor_table_model_5fconfig_2eproto_getter() { return &descriptor_table_model_5fconfig_2eproto; } // Force running AddDescriptors() at dynamic initialization time. PROTOBUF_ATTRIBUTE_INIT_PRIORITY2 static ::_pbi::AddDescriptorsRunner dynamic_init_dummy_model_5fconfig_2eproto(&descriptor_table_model_5fconfig_2eproto); namespace inference { const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[0]; } bool ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_IsValid(int value) { switch (value) { case 0: return true; default: return false; } } #if (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) constexpr ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind ModelInstanceGroup_SecondaryDevice::KIND_NVDLA; constexpr ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind ModelInstanceGroup_SecondaryDevice::SecondaryDeviceKind_MIN; constexpr ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind ModelInstanceGroup_SecondaryDevice::SecondaryDeviceKind_MAX; constexpr int ModelInstanceGroup_SecondaryDevice::SecondaryDeviceKind_ARRAYSIZE; #endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelInstanceGroup_Kind_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[1]; } bool ModelInstanceGroup_Kind_IsValid(int value) { switch (value) { case 0: case 1: case 2: case 3: return true; default: return false; } } #if (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) constexpr ModelInstanceGroup_Kind ModelInstanceGroup::KIND_AUTO; constexpr ModelInstanceGroup_Kind ModelInstanceGroup::KIND_GPU; constexpr ModelInstanceGroup_Kind ModelInstanceGroup::KIND_CPU; constexpr ModelInstanceGroup_Kind ModelInstanceGroup::KIND_MODEL; constexpr ModelInstanceGroup_Kind ModelInstanceGroup::Kind_MIN; constexpr ModelInstanceGroup_Kind ModelInstanceGroup::Kind_MAX; constexpr int ModelInstanceGroup::Kind_ARRAYSIZE; #endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelInput_Format_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[2]; } bool ModelInput_Format_IsValid(int value) { switch (value) { case 0: case 1: case 2: return true; default: return false; } } #if (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) constexpr ModelInput_Format ModelInput::FORMAT_NONE; constexpr ModelInput_Format ModelInput::FORMAT_NHWC; constexpr ModelInput_Format ModelInput::FORMAT_NCHW; constexpr ModelInput_Format ModelInput::Format_MIN; constexpr ModelInput_Format ModelInput::Format_MAX; constexpr int ModelInput::Format_ARRAYSIZE; #endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* BatchInput_Kind_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[3]; } bool BatchInput_Kind_IsValid(int value) { switch (value) { case 0: case 1: case 2: case 3: case 4: case 5: return true; default: return false; } } #if (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) constexpr BatchInput_Kind BatchInput::BATCH_ELEMENT_COUNT; constexpr BatchInput_Kind BatchInput::BATCH_ACCUMULATED_ELEMENT_COUNT; constexpr BatchInput_Kind BatchInput::BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO; constexpr BatchInput_Kind BatchInput::BATCH_MAX_ELEMENT_COUNT_AS_SHAPE; constexpr BatchInput_Kind BatchInput::BATCH_ITEM_SHAPE; constexpr BatchInput_Kind BatchInput::BATCH_ITEM_SHAPE_FLATTEN; constexpr BatchInput_Kind BatchInput::Kind_MIN; constexpr BatchInput_Kind BatchInput::Kind_MAX; constexpr int BatchInput::Kind_ARRAYSIZE; #endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* BatchOutput_Kind_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[4]; } bool BatchOutput_Kind_IsValid(int value) { switch (value) { case 0: return true; default: return false; } } #if (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) constexpr BatchOutput_Kind BatchOutput::BATCH_SCATTER_WITH_INPUT_SHAPE; constexpr BatchOutput_Kind BatchOutput::Kind_MIN; constexpr BatchOutput_Kind BatchOutput::Kind_MAX; constexpr int BatchOutput::Kind_ARRAYSIZE; #endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelOptimizationPolicy_ModelPriority_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[5]; } bool ModelOptimizationPolicy_ModelPriority_IsValid(int value) { switch (value) { case 0: case 1: case 2: return true; default: return false; } } #if (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) constexpr ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy::PRIORITY_DEFAULT; constexpr ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy::PRIORITY_MAX; constexpr ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy::PRIORITY_MIN; constexpr ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy::ModelPriority_MIN; constexpr ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy::ModelPriority_MAX; constexpr int ModelOptimizationPolicy::ModelPriority_ARRAYSIZE; #endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelQueuePolicy_TimeoutAction_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[6]; } bool ModelQueuePolicy_TimeoutAction_IsValid(int value) { switch (value) { case 0: case 1: return true; default: return false; } } #if (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) constexpr ModelQueuePolicy_TimeoutAction ModelQueuePolicy::REJECT; constexpr ModelQueuePolicy_TimeoutAction ModelQueuePolicy::DELAY; constexpr ModelQueuePolicy_TimeoutAction ModelQueuePolicy::TimeoutAction_MIN; constexpr ModelQueuePolicy_TimeoutAction ModelQueuePolicy::TimeoutAction_MAX; constexpr int ModelQueuePolicy::TimeoutAction_ARRAYSIZE; #endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelSequenceBatching_Control_Kind_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[7]; } bool ModelSequenceBatching_Control_Kind_IsValid(int value) { switch (value) { case 0: case 1: case 2: case 3: return true; default: return false; } } #if (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) constexpr ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control::CONTROL_SEQUENCE_START; constexpr ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control::CONTROL_SEQUENCE_READY; constexpr ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control::CONTROL_SEQUENCE_END; constexpr ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control::CONTROL_SEQUENCE_CORRID; constexpr ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control::Kind_MIN; constexpr ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control::Kind_MAX; constexpr int ModelSequenceBatching_Control::Kind_ARRAYSIZE; #endif // (__cplusplus < 201703) && (!defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912)) const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* DataType_descriptor() { ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_model_5fconfig_2eproto); return file_level_enum_descriptors_model_5fconfig_2eproto[8]; } bool DataType_IsValid(int value) { switch (value) { case 0: case 1: case 2: case 3: case 4: case 5: case 6: case 7: case 8: case 9: case 10: case 11: case 12: case 13: case 14: return true; default: return false; } } // =================================================================== class ModelRateLimiter_Resource::_Internal { public: }; ModelRateLimiter_Resource::ModelRateLimiter_Resource(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelRateLimiter.Resource) } ModelRateLimiter_Resource::ModelRateLimiter_Resource(const ModelRateLimiter_Resource& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelRateLimiter_Resource* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.name_){} , decltype(_impl_.global_){} , decltype(_impl_.count_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } ::memcpy(&_impl_.global_, &from._impl_.global_, static_cast(reinterpret_cast(&_impl_.count_) - reinterpret_cast(&_impl_.global_)) + sizeof(_impl_.count_)); // @@protoc_insertion_point(copy_constructor:inference.ModelRateLimiter.Resource) } inline void ModelRateLimiter_Resource::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.name_){} , decltype(_impl_.global_){false} , decltype(_impl_.count_){0u} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelRateLimiter_Resource::~ModelRateLimiter_Resource() { // @@protoc_insertion_point(destructor:inference.ModelRateLimiter.Resource) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelRateLimiter_Resource::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.name_.Destroy(); } void ModelRateLimiter_Resource::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelRateLimiter_Resource::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelRateLimiter.Resource) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.name_.ClearToEmpty(); ::memset(&_impl_.global_, 0, static_cast( reinterpret_cast(&_impl_.count_) - reinterpret_cast(&_impl_.global_)) + sizeof(_impl_.count_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelRateLimiter_Resource::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelRateLimiter.Resource.name")); } else goto handle_unusual; continue; // bool global = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.global_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // uint32 count = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { _impl_.count_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelRateLimiter_Resource::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelRateLimiter.Resource) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelRateLimiter.Resource.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // bool global = 2; if (this->_internal_global() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(2, this->_internal_global(), target); } // uint32 count = 3; if (this->_internal_count() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt32ToArray(3, this->_internal_count(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelRateLimiter.Resource) return target; } size_t ModelRateLimiter_Resource::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelRateLimiter.Resource) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } // bool global = 2; if (this->_internal_global() != 0) { total_size += 1 + 1; } // uint32 count = 3; if (this->_internal_count() != 0) { total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne(this->_internal_count()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelRateLimiter_Resource::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelRateLimiter_Resource::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelRateLimiter_Resource::GetClassData() const { return &_class_data_; } void ModelRateLimiter_Resource::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelRateLimiter.Resource) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } if (from._internal_global() != 0) { _this->_internal_set_global(from._internal_global()); } if (from._internal_count() != 0) { _this->_internal_set_count(from._internal_count()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelRateLimiter_Resource::CopyFrom(const ModelRateLimiter_Resource& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelRateLimiter.Resource) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelRateLimiter_Resource::IsInitialized() const { return true; } void ModelRateLimiter_Resource::InternalSwap(ModelRateLimiter_Resource* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelRateLimiter_Resource, _impl_.count_) + sizeof(ModelRateLimiter_Resource::_impl_.count_) - PROTOBUF_FIELD_OFFSET(ModelRateLimiter_Resource, _impl_.global_)>( reinterpret_cast(&_impl_.global_), reinterpret_cast(&other->_impl_.global_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelRateLimiter_Resource::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[0]); } // =================================================================== class ModelRateLimiter::_Internal { public: }; ModelRateLimiter::ModelRateLimiter(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelRateLimiter) } ModelRateLimiter::ModelRateLimiter(const ModelRateLimiter& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelRateLimiter* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.resources_){from._impl_.resources_} , decltype(_impl_.priority_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.priority_ = from._impl_.priority_; // @@protoc_insertion_point(copy_constructor:inference.ModelRateLimiter) } inline void ModelRateLimiter::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.resources_){arena} , decltype(_impl_.priority_){0u} , /*decltype(_impl_._cached_size_)*/{} }; } ModelRateLimiter::~ModelRateLimiter() { // @@protoc_insertion_point(destructor:inference.ModelRateLimiter) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelRateLimiter::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.resources_.~RepeatedPtrField(); } void ModelRateLimiter::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelRateLimiter::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelRateLimiter) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.resources_.Clear(); _impl_.priority_ = 0u; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelRateLimiter::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated .inference.ModelRateLimiter.Resource resources = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_resources(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); } else goto handle_unusual; continue; // uint32 priority = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.priority_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelRateLimiter::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelRateLimiter) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated .inference.ModelRateLimiter.Resource resources = 1; for (unsigned i = 0, n = static_cast(this->_internal_resources_size()); i < n; i++) { const auto& repfield = this->_internal_resources(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(1, repfield, repfield.GetCachedSize(), target, stream); } // uint32 priority = 2; if (this->_internal_priority() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt32ToArray(2, this->_internal_priority(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelRateLimiter) return target; } size_t ModelRateLimiter::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelRateLimiter) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated .inference.ModelRateLimiter.Resource resources = 1; total_size += 1UL * this->_internal_resources_size(); for (const auto& msg : this->_impl_.resources_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // uint32 priority = 2; if (this->_internal_priority() != 0) { total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne(this->_internal_priority()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelRateLimiter::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelRateLimiter::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelRateLimiter::GetClassData() const { return &_class_data_; } void ModelRateLimiter::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelRateLimiter) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.resources_.MergeFrom(from._impl_.resources_); if (from._internal_priority() != 0) { _this->_internal_set_priority(from._internal_priority()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelRateLimiter::CopyFrom(const ModelRateLimiter& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelRateLimiter) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelRateLimiter::IsInitialized() const { return true; } void ModelRateLimiter::InternalSwap(ModelRateLimiter* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.resources_.InternalSwap(&other->_impl_.resources_); swap(_impl_.priority_, other->_impl_.priority_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelRateLimiter::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[1]); } // =================================================================== class ModelInstanceGroup_SecondaryDevice::_Internal { public: }; ModelInstanceGroup_SecondaryDevice::ModelInstanceGroup_SecondaryDevice(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelInstanceGroup.SecondaryDevice) } ModelInstanceGroup_SecondaryDevice::ModelInstanceGroup_SecondaryDevice(const ModelInstanceGroup_SecondaryDevice& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelInstanceGroup_SecondaryDevice* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.device_id_){} , decltype(_impl_.kind_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); ::memcpy(&_impl_.device_id_, &from._impl_.device_id_, static_cast(reinterpret_cast(&_impl_.kind_) - reinterpret_cast(&_impl_.device_id_)) + sizeof(_impl_.kind_)); // @@protoc_insertion_point(copy_constructor:inference.ModelInstanceGroup.SecondaryDevice) } inline void ModelInstanceGroup_SecondaryDevice::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.device_id_){int64_t{0}} , decltype(_impl_.kind_){0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelInstanceGroup_SecondaryDevice::~ModelInstanceGroup_SecondaryDevice() { // @@protoc_insertion_point(destructor:inference.ModelInstanceGroup.SecondaryDevice) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelInstanceGroup_SecondaryDevice::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); } void ModelInstanceGroup_SecondaryDevice::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelInstanceGroup_SecondaryDevice::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelInstanceGroup.SecondaryDevice) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; ::memset(&_impl_.device_id_, 0, static_cast( reinterpret_cast(&_impl_.kind_) - reinterpret_cast(&_impl_.device_id_)) + sizeof(_impl_.kind_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelInstanceGroup_SecondaryDevice::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // .inference.ModelInstanceGroup.SecondaryDevice.SecondaryDeviceKind kind = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_kind(static_cast<::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind>(val)); } else goto handle_unusual; continue; // int64 device_id = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.device_id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelInstanceGroup_SecondaryDevice::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelInstanceGroup.SecondaryDevice) uint32_t cached_has_bits = 0; (void) cached_has_bits; // .inference.ModelInstanceGroup.SecondaryDevice.SecondaryDeviceKind kind = 1; if (this->_internal_kind() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 1, this->_internal_kind(), target); } // int64 device_id = 2; if (this->_internal_device_id() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteInt64ToArray(2, this->_internal_device_id(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelInstanceGroup.SecondaryDevice) return target; } size_t ModelInstanceGroup_SecondaryDevice::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelInstanceGroup.SecondaryDevice) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // int64 device_id = 2; if (this->_internal_device_id() != 0) { total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_device_id()); } // .inference.ModelInstanceGroup.SecondaryDevice.SecondaryDeviceKind kind = 1; if (this->_internal_kind() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_kind()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelInstanceGroup_SecondaryDevice::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelInstanceGroup_SecondaryDevice::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelInstanceGroup_SecondaryDevice::GetClassData() const { return &_class_data_; } void ModelInstanceGroup_SecondaryDevice::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelInstanceGroup.SecondaryDevice) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_device_id() != 0) { _this->_internal_set_device_id(from._internal_device_id()); } if (from._internal_kind() != 0) { _this->_internal_set_kind(from._internal_kind()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelInstanceGroup_SecondaryDevice::CopyFrom(const ModelInstanceGroup_SecondaryDevice& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelInstanceGroup.SecondaryDevice) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelInstanceGroup_SecondaryDevice::IsInitialized() const { return true; } void ModelInstanceGroup_SecondaryDevice::InternalSwap(ModelInstanceGroup_SecondaryDevice* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelInstanceGroup_SecondaryDevice, _impl_.kind_) + sizeof(ModelInstanceGroup_SecondaryDevice::_impl_.kind_) - PROTOBUF_FIELD_OFFSET(ModelInstanceGroup_SecondaryDevice, _impl_.device_id_)>( reinterpret_cast(&_impl_.device_id_), reinterpret_cast(&other->_impl_.device_id_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelInstanceGroup_SecondaryDevice::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[2]); } // =================================================================== class ModelInstanceGroup::_Internal { public: static const ::inference::ModelRateLimiter& rate_limiter(const ModelInstanceGroup* msg); }; const ::inference::ModelRateLimiter& ModelInstanceGroup::_Internal::rate_limiter(const ModelInstanceGroup* msg) { return *msg->_impl_.rate_limiter_; } ModelInstanceGroup::ModelInstanceGroup(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelInstanceGroup) } ModelInstanceGroup::ModelInstanceGroup(const ModelInstanceGroup& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelInstanceGroup* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.gpus_){from._impl_.gpus_} , /*decltype(_impl_._gpus_cached_byte_size_)*/{0} , decltype(_impl_.profile_){from._impl_.profile_} , decltype(_impl_.secondary_devices_){from._impl_.secondary_devices_} , decltype(_impl_.name_){} , decltype(_impl_.host_policy_){} , decltype(_impl_.rate_limiter_){nullptr} , decltype(_impl_.count_){} , decltype(_impl_.kind_){} , decltype(_impl_.passive_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } _impl_.host_policy_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.host_policy_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_host_policy().empty()) { _this->_impl_.host_policy_.Set(from._internal_host_policy(), _this->GetArenaForAllocation()); } if (from._internal_has_rate_limiter()) { _this->_impl_.rate_limiter_ = new ::inference::ModelRateLimiter(*from._impl_.rate_limiter_); } ::memcpy(&_impl_.count_, &from._impl_.count_, static_cast(reinterpret_cast(&_impl_.passive_) - reinterpret_cast(&_impl_.count_)) + sizeof(_impl_.passive_)); // @@protoc_insertion_point(copy_constructor:inference.ModelInstanceGroup) } inline void ModelInstanceGroup::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.gpus_){arena} , /*decltype(_impl_._gpus_cached_byte_size_)*/{0} , decltype(_impl_.profile_){arena} , decltype(_impl_.secondary_devices_){arena} , decltype(_impl_.name_){} , decltype(_impl_.host_policy_){} , decltype(_impl_.rate_limiter_){nullptr} , decltype(_impl_.count_){0} , decltype(_impl_.kind_){0} , decltype(_impl_.passive_){false} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.host_policy_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.host_policy_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelInstanceGroup::~ModelInstanceGroup() { // @@protoc_insertion_point(destructor:inference.ModelInstanceGroup) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelInstanceGroup::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.gpus_.~RepeatedField(); _impl_.profile_.~RepeatedPtrField(); _impl_.secondary_devices_.~RepeatedPtrField(); _impl_.name_.Destroy(); _impl_.host_policy_.Destroy(); if (this != internal_default_instance()) delete _impl_.rate_limiter_; } void ModelInstanceGroup::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelInstanceGroup::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelInstanceGroup) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.gpus_.Clear(); _impl_.profile_.Clear(); _impl_.secondary_devices_.Clear(); _impl_.name_.ClearToEmpty(); _impl_.host_policy_.ClearToEmpty(); if (GetArenaForAllocation() == nullptr && _impl_.rate_limiter_ != nullptr) { delete _impl_.rate_limiter_; } _impl_.rate_limiter_ = nullptr; ::memset(&_impl_.count_, 0, static_cast( reinterpret_cast(&_impl_.passive_) - reinterpret_cast(&_impl_.count_)) + sizeof(_impl_.passive_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelInstanceGroup::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelInstanceGroup.name")); } else goto handle_unusual; continue; // int32 count = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.count_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated int32 gpus = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt32Parser(_internal_mutable_gpus(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 24) { _internal_add_gpus(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelInstanceGroup.Kind kind = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_kind(static_cast<::inference::ModelInstanceGroup_Kind>(val)); } else goto handle_unusual; continue; // repeated string profile = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { ptr -= 1; do { ptr += 1; auto str = _internal_add_profile(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelInstanceGroup.profile")); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<42>(ptr)); } else goto handle_unusual; continue; // .inference.ModelRateLimiter rate_limiter = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 50)) { ptr = ctx->ParseMessage(_internal_mutable_rate_limiter(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool passive = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 56)) { _impl_.passive_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated .inference.ModelInstanceGroup.SecondaryDevice secondary_devices = 8; case 8: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 66)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_secondary_devices(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<66>(ptr)); } else goto handle_unusual; continue; // string host_policy = 9; case 9: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 74)) { auto str = _internal_mutable_host_policy(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelInstanceGroup.host_policy")); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelInstanceGroup::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelInstanceGroup) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelInstanceGroup.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // int32 count = 2; if (this->_internal_count() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteInt32ToArray(2, this->_internal_count(), target); } // repeated int32 gpus = 3; { int byte_size = _impl_._gpus_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt32Packed( 3, _internal_gpus(), byte_size, target); } } // .inference.ModelInstanceGroup.Kind kind = 4; if (this->_internal_kind() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 4, this->_internal_kind(), target); } // repeated string profile = 5; for (int i = 0, n = this->_internal_profile_size(); i < n; i++) { const auto& s = this->_internal_profile(i); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( s.data(), static_cast(s.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelInstanceGroup.profile"); target = stream->WriteString(5, s, target); } // .inference.ModelRateLimiter rate_limiter = 6; if (this->_internal_has_rate_limiter()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(6, _Internal::rate_limiter(this), _Internal::rate_limiter(this).GetCachedSize(), target, stream); } // bool passive = 7; if (this->_internal_passive() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(7, this->_internal_passive(), target); } // repeated .inference.ModelInstanceGroup.SecondaryDevice secondary_devices = 8; for (unsigned i = 0, n = static_cast(this->_internal_secondary_devices_size()); i < n; i++) { const auto& repfield = this->_internal_secondary_devices(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(8, repfield, repfield.GetCachedSize(), target, stream); } // string host_policy = 9; if (!this->_internal_host_policy().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_host_policy().data(), static_cast(this->_internal_host_policy().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelInstanceGroup.host_policy"); target = stream->WriteStringMaybeAliased( 9, this->_internal_host_policy(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelInstanceGroup) return target; } size_t ModelInstanceGroup::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelInstanceGroup) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int32 gpus = 3; { size_t data_size = ::_pbi::WireFormatLite:: Int32Size(this->_impl_.gpus_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._gpus_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // repeated string profile = 5; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(_impl_.profile_.size()); for (int i = 0, n = _impl_.profile_.size(); i < n; i++) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( _impl_.profile_.Get(i)); } // repeated .inference.ModelInstanceGroup.SecondaryDevice secondary_devices = 8; total_size += 1UL * this->_internal_secondary_devices_size(); for (const auto& msg : this->_impl_.secondary_devices_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } // string host_policy = 9; if (!this->_internal_host_policy().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_host_policy()); } // .inference.ModelRateLimiter rate_limiter = 6; if (this->_internal_has_rate_limiter()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.rate_limiter_); } // int32 count = 2; if (this->_internal_count() != 0) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne(this->_internal_count()); } // .inference.ModelInstanceGroup.Kind kind = 4; if (this->_internal_kind() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_kind()); } // bool passive = 7; if (this->_internal_passive() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelInstanceGroup::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelInstanceGroup::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelInstanceGroup::GetClassData() const { return &_class_data_; } void ModelInstanceGroup::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelInstanceGroup) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.gpus_.MergeFrom(from._impl_.gpus_); _this->_impl_.profile_.MergeFrom(from._impl_.profile_); _this->_impl_.secondary_devices_.MergeFrom(from._impl_.secondary_devices_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } if (!from._internal_host_policy().empty()) { _this->_internal_set_host_policy(from._internal_host_policy()); } if (from._internal_has_rate_limiter()) { _this->_internal_mutable_rate_limiter()->::inference::ModelRateLimiter::MergeFrom( from._internal_rate_limiter()); } if (from._internal_count() != 0) { _this->_internal_set_count(from._internal_count()); } if (from._internal_kind() != 0) { _this->_internal_set_kind(from._internal_kind()); } if (from._internal_passive() != 0) { _this->_internal_set_passive(from._internal_passive()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelInstanceGroup::CopyFrom(const ModelInstanceGroup& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelInstanceGroup) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelInstanceGroup::IsInitialized() const { return true; } void ModelInstanceGroup::InternalSwap(ModelInstanceGroup* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.gpus_.InternalSwap(&other->_impl_.gpus_); _impl_.profile_.InternalSwap(&other->_impl_.profile_); _impl_.secondary_devices_.InternalSwap(&other->_impl_.secondary_devices_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.host_policy_, lhs_arena, &other->_impl_.host_policy_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelInstanceGroup, _impl_.passive_) + sizeof(ModelInstanceGroup::_impl_.passive_) - PROTOBUF_FIELD_OFFSET(ModelInstanceGroup, _impl_.rate_limiter_)>( reinterpret_cast(&_impl_.rate_limiter_), reinterpret_cast(&other->_impl_.rate_limiter_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelInstanceGroup::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[3]); } // =================================================================== class ModelTensorReshape::_Internal { public: }; ModelTensorReshape::ModelTensorReshape(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelTensorReshape) } ModelTensorReshape::ModelTensorReshape(const ModelTensorReshape& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelTensorReshape* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.shape_){from._impl_.shape_} , /*decltype(_impl_._shape_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:inference.ModelTensorReshape) } inline void ModelTensorReshape::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.shape_){arena} , /*decltype(_impl_._shape_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelTensorReshape::~ModelTensorReshape() { // @@protoc_insertion_point(destructor:inference.ModelTensorReshape) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelTensorReshape::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.shape_.~RepeatedField(); } void ModelTensorReshape::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelTensorReshape::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelTensorReshape) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.shape_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelTensorReshape::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated int64 shape = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(_internal_mutable_shape(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 8) { _internal_add_shape(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelTensorReshape::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelTensorReshape) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated int64 shape = 1; { int byte_size = _impl_._shape_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt64Packed( 1, _internal_shape(), byte_size, target); } } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelTensorReshape) return target; } size_t ModelTensorReshape::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelTensorReshape) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int64 shape = 1; { size_t data_size = ::_pbi::WireFormatLite:: Int64Size(this->_impl_.shape_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._shape_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelTensorReshape::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelTensorReshape::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelTensorReshape::GetClassData() const { return &_class_data_; } void ModelTensorReshape::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelTensorReshape) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.shape_.MergeFrom(from._impl_.shape_); _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelTensorReshape::CopyFrom(const ModelTensorReshape& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelTensorReshape) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelTensorReshape::IsInitialized() const { return true; } void ModelTensorReshape::InternalSwap(ModelTensorReshape* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.shape_.InternalSwap(&other->_impl_.shape_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelTensorReshape::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[4]); } // =================================================================== class ModelInput::_Internal { public: static const ::inference::ModelTensorReshape& reshape(const ModelInput* msg); }; const ::inference::ModelTensorReshape& ModelInput::_Internal::reshape(const ModelInput* msg) { return *msg->_impl_.reshape_; } ModelInput::ModelInput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelInput) } ModelInput::ModelInput(const ModelInput& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelInput* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.dims_){from._impl_.dims_} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.name_){} , decltype(_impl_.reshape_){nullptr} , decltype(_impl_.data_type_){} , decltype(_impl_.format_){} , decltype(_impl_.is_shape_tensor_){} , decltype(_impl_.allow_ragged_batch_){} , decltype(_impl_.optional_){} , decltype(_impl_.is_non_linear_format_io_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } if (from._internal_has_reshape()) { _this->_impl_.reshape_ = new ::inference::ModelTensorReshape(*from._impl_.reshape_); } ::memcpy(&_impl_.data_type_, &from._impl_.data_type_, static_cast(reinterpret_cast(&_impl_.is_non_linear_format_io_) - reinterpret_cast(&_impl_.data_type_)) + sizeof(_impl_.is_non_linear_format_io_)); // @@protoc_insertion_point(copy_constructor:inference.ModelInput) } inline void ModelInput::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.dims_){arena} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.name_){} , decltype(_impl_.reshape_){nullptr} , decltype(_impl_.data_type_){0} , decltype(_impl_.format_){0} , decltype(_impl_.is_shape_tensor_){false} , decltype(_impl_.allow_ragged_batch_){false} , decltype(_impl_.optional_){false} , decltype(_impl_.is_non_linear_format_io_){false} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelInput::~ModelInput() { // @@protoc_insertion_point(destructor:inference.ModelInput) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelInput::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.dims_.~RepeatedField(); _impl_.name_.Destroy(); if (this != internal_default_instance()) delete _impl_.reshape_; } void ModelInput::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelInput::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelInput) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.dims_.Clear(); _impl_.name_.ClearToEmpty(); if (GetArenaForAllocation() == nullptr && _impl_.reshape_ != nullptr) { delete _impl_.reshape_; } _impl_.reshape_ = nullptr; ::memset(&_impl_.data_type_, 0, static_cast( reinterpret_cast(&_impl_.is_non_linear_format_io_) - reinterpret_cast(&_impl_.data_type_)) + sizeof(_impl_.is_non_linear_format_io_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelInput::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelInput.name")); } else goto handle_unusual; continue; // .inference.DataType data_type = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_data_type(static_cast<::inference::DataType>(val)); } else goto handle_unusual; continue; // .inference.ModelInput.Format format = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_format(static_cast<::inference::ModelInput_Format>(val)); } else goto handle_unusual; continue; // repeated int64 dims = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 34)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(_internal_mutable_dims(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 32) { _internal_add_dims(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelTensorReshape reshape = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { ptr = ctx->ParseMessage(_internal_mutable_reshape(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool is_shape_tensor = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 48)) { _impl_.is_shape_tensor_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool allow_ragged_batch = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 56)) { _impl_.allow_ragged_batch_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool optional = 8; case 8: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 64)) { _impl_.optional_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool is_non_linear_format_io = 9; case 9: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 72)) { _impl_.is_non_linear_format_io_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelInput::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelInput) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelInput.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // .inference.DataType data_type = 2; if (this->_internal_data_type() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 2, this->_internal_data_type(), target); } // .inference.ModelInput.Format format = 3; if (this->_internal_format() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 3, this->_internal_format(), target); } // repeated int64 dims = 4; { int byte_size = _impl_._dims_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt64Packed( 4, _internal_dims(), byte_size, target); } } // .inference.ModelTensorReshape reshape = 5; if (this->_internal_has_reshape()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(5, _Internal::reshape(this), _Internal::reshape(this).GetCachedSize(), target, stream); } // bool is_shape_tensor = 6; if (this->_internal_is_shape_tensor() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(6, this->_internal_is_shape_tensor(), target); } // bool allow_ragged_batch = 7; if (this->_internal_allow_ragged_batch() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(7, this->_internal_allow_ragged_batch(), target); } // bool optional = 8; if (this->_internal_optional() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(8, this->_internal_optional(), target); } // bool is_non_linear_format_io = 9; if (this->_internal_is_non_linear_format_io() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(9, this->_internal_is_non_linear_format_io(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelInput) return target; } size_t ModelInput::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelInput) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int64 dims = 4; { size_t data_size = ::_pbi::WireFormatLite:: Int64Size(this->_impl_.dims_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._dims_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } // .inference.ModelTensorReshape reshape = 5; if (this->_internal_has_reshape()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.reshape_); } // .inference.DataType data_type = 2; if (this->_internal_data_type() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_data_type()); } // .inference.ModelInput.Format format = 3; if (this->_internal_format() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_format()); } // bool is_shape_tensor = 6; if (this->_internal_is_shape_tensor() != 0) { total_size += 1 + 1; } // bool allow_ragged_batch = 7; if (this->_internal_allow_ragged_batch() != 0) { total_size += 1 + 1; } // bool optional = 8; if (this->_internal_optional() != 0) { total_size += 1 + 1; } // bool is_non_linear_format_io = 9; if (this->_internal_is_non_linear_format_io() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelInput::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelInput::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelInput::GetClassData() const { return &_class_data_; } void ModelInput::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelInput) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.dims_.MergeFrom(from._impl_.dims_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } if (from._internal_has_reshape()) { _this->_internal_mutable_reshape()->::inference::ModelTensorReshape::MergeFrom( from._internal_reshape()); } if (from._internal_data_type() != 0) { _this->_internal_set_data_type(from._internal_data_type()); } if (from._internal_format() != 0) { _this->_internal_set_format(from._internal_format()); } if (from._internal_is_shape_tensor() != 0) { _this->_internal_set_is_shape_tensor(from._internal_is_shape_tensor()); } if (from._internal_allow_ragged_batch() != 0) { _this->_internal_set_allow_ragged_batch(from._internal_allow_ragged_batch()); } if (from._internal_optional() != 0) { _this->_internal_set_optional(from._internal_optional()); } if (from._internal_is_non_linear_format_io() != 0) { _this->_internal_set_is_non_linear_format_io(from._internal_is_non_linear_format_io()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelInput::CopyFrom(const ModelInput& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelInput) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelInput::IsInitialized() const { return true; } void ModelInput::InternalSwap(ModelInput* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.dims_.InternalSwap(&other->_impl_.dims_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelInput, _impl_.is_non_linear_format_io_) + sizeof(ModelInput::_impl_.is_non_linear_format_io_) - PROTOBUF_FIELD_OFFSET(ModelInput, _impl_.reshape_)>( reinterpret_cast(&_impl_.reshape_), reinterpret_cast(&other->_impl_.reshape_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelInput::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[5]); } // =================================================================== class ModelOutput::_Internal { public: static const ::inference::ModelTensorReshape& reshape(const ModelOutput* msg); }; const ::inference::ModelTensorReshape& ModelOutput::_Internal::reshape(const ModelOutput* msg) { return *msg->_impl_.reshape_; } ModelOutput::ModelOutput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelOutput) } ModelOutput::ModelOutput(const ModelOutput& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOutput* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.dims_){from._impl_.dims_} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.name_){} , decltype(_impl_.label_filename_){} , decltype(_impl_.reshape_){nullptr} , decltype(_impl_.data_type_){} , decltype(_impl_.is_shape_tensor_){} , decltype(_impl_.is_non_linear_format_io_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } _impl_.label_filename_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.label_filename_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_label_filename().empty()) { _this->_impl_.label_filename_.Set(from._internal_label_filename(), _this->GetArenaForAllocation()); } if (from._internal_has_reshape()) { _this->_impl_.reshape_ = new ::inference::ModelTensorReshape(*from._impl_.reshape_); } ::memcpy(&_impl_.data_type_, &from._impl_.data_type_, static_cast(reinterpret_cast(&_impl_.is_non_linear_format_io_) - reinterpret_cast(&_impl_.data_type_)) + sizeof(_impl_.is_non_linear_format_io_)); // @@protoc_insertion_point(copy_constructor:inference.ModelOutput) } inline void ModelOutput::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.dims_){arena} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.name_){} , decltype(_impl_.label_filename_){} , decltype(_impl_.reshape_){nullptr} , decltype(_impl_.data_type_){0} , decltype(_impl_.is_shape_tensor_){false} , decltype(_impl_.is_non_linear_format_io_){false} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.label_filename_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.label_filename_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelOutput::~ModelOutput() { // @@protoc_insertion_point(destructor:inference.ModelOutput) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelOutput::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.dims_.~RepeatedField(); _impl_.name_.Destroy(); _impl_.label_filename_.Destroy(); if (this != internal_default_instance()) delete _impl_.reshape_; } void ModelOutput::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOutput::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOutput) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.dims_.Clear(); _impl_.name_.ClearToEmpty(); _impl_.label_filename_.ClearToEmpty(); if (GetArenaForAllocation() == nullptr && _impl_.reshape_ != nullptr) { delete _impl_.reshape_; } _impl_.reshape_ = nullptr; ::memset(&_impl_.data_type_, 0, static_cast( reinterpret_cast(&_impl_.is_non_linear_format_io_) - reinterpret_cast(&_impl_.data_type_)) + sizeof(_impl_.is_non_linear_format_io_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOutput::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelOutput.name")); } else goto handle_unusual; continue; // .inference.DataType data_type = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_data_type(static_cast<::inference::DataType>(val)); } else goto handle_unusual; continue; // repeated int64 dims = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(_internal_mutable_dims(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 24) { _internal_add_dims(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // string label_filename = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 34)) { auto str = _internal_mutable_label_filename(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelOutput.label_filename")); } else goto handle_unusual; continue; // .inference.ModelTensorReshape reshape = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { ptr = ctx->ParseMessage(_internal_mutable_reshape(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool is_shape_tensor = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 48)) { _impl_.is_shape_tensor_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool is_non_linear_format_io = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 56)) { _impl_.is_non_linear_format_io_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOutput::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOutput) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelOutput.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // .inference.DataType data_type = 2; if (this->_internal_data_type() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 2, this->_internal_data_type(), target); } // repeated int64 dims = 3; { int byte_size = _impl_._dims_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt64Packed( 3, _internal_dims(), byte_size, target); } } // string label_filename = 4; if (!this->_internal_label_filename().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_label_filename().data(), static_cast(this->_internal_label_filename().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelOutput.label_filename"); target = stream->WriteStringMaybeAliased( 4, this->_internal_label_filename(), target); } // .inference.ModelTensorReshape reshape = 5; if (this->_internal_has_reshape()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(5, _Internal::reshape(this), _Internal::reshape(this).GetCachedSize(), target, stream); } // bool is_shape_tensor = 6; if (this->_internal_is_shape_tensor() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(6, this->_internal_is_shape_tensor(), target); } // bool is_non_linear_format_io = 7; if (this->_internal_is_non_linear_format_io() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(7, this->_internal_is_non_linear_format_io(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOutput) return target; } size_t ModelOutput::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOutput) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int64 dims = 3; { size_t data_size = ::_pbi::WireFormatLite:: Int64Size(this->_impl_.dims_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._dims_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } // string label_filename = 4; if (!this->_internal_label_filename().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_label_filename()); } // .inference.ModelTensorReshape reshape = 5; if (this->_internal_has_reshape()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.reshape_); } // .inference.DataType data_type = 2; if (this->_internal_data_type() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_data_type()); } // bool is_shape_tensor = 6; if (this->_internal_is_shape_tensor() != 0) { total_size += 1 + 1; } // bool is_non_linear_format_io = 7; if (this->_internal_is_non_linear_format_io() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOutput::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOutput::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOutput::GetClassData() const { return &_class_data_; } void ModelOutput::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOutput) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.dims_.MergeFrom(from._impl_.dims_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } if (!from._internal_label_filename().empty()) { _this->_internal_set_label_filename(from._internal_label_filename()); } if (from._internal_has_reshape()) { _this->_internal_mutable_reshape()->::inference::ModelTensorReshape::MergeFrom( from._internal_reshape()); } if (from._internal_data_type() != 0) { _this->_internal_set_data_type(from._internal_data_type()); } if (from._internal_is_shape_tensor() != 0) { _this->_internal_set_is_shape_tensor(from._internal_is_shape_tensor()); } if (from._internal_is_non_linear_format_io() != 0) { _this->_internal_set_is_non_linear_format_io(from._internal_is_non_linear_format_io()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOutput::CopyFrom(const ModelOutput& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOutput) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOutput::IsInitialized() const { return true; } void ModelOutput::InternalSwap(ModelOutput* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.dims_.InternalSwap(&other->_impl_.dims_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.label_filename_, lhs_arena, &other->_impl_.label_filename_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelOutput, _impl_.is_non_linear_format_io_) + sizeof(ModelOutput::_impl_.is_non_linear_format_io_) - PROTOBUF_FIELD_OFFSET(ModelOutput, _impl_.reshape_)>( reinterpret_cast(&_impl_.reshape_), reinterpret_cast(&other->_impl_.reshape_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOutput::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[6]); } // =================================================================== class BatchInput::_Internal { public: }; BatchInput::BatchInput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.BatchInput) } BatchInput::BatchInput(const BatchInput& from) : ::PROTOBUF_NAMESPACE_ID::Message() { BatchInput* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.target_name_){from._impl_.target_name_} , decltype(_impl_.source_input_){from._impl_.source_input_} , decltype(_impl_.kind_){} , decltype(_impl_.data_type_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); ::memcpy(&_impl_.kind_, &from._impl_.kind_, static_cast(reinterpret_cast(&_impl_.data_type_) - reinterpret_cast(&_impl_.kind_)) + sizeof(_impl_.data_type_)); // @@protoc_insertion_point(copy_constructor:inference.BatchInput) } inline void BatchInput::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.target_name_){arena} , decltype(_impl_.source_input_){arena} , decltype(_impl_.kind_){0} , decltype(_impl_.data_type_){0} , /*decltype(_impl_._cached_size_)*/{} }; } BatchInput::~BatchInput() { // @@protoc_insertion_point(destructor:inference.BatchInput) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void BatchInput::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.target_name_.~RepeatedPtrField(); _impl_.source_input_.~RepeatedPtrField(); } void BatchInput::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void BatchInput::Clear() { // @@protoc_insertion_point(message_clear_start:inference.BatchInput) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.target_name_.Clear(); _impl_.source_input_.Clear(); ::memset(&_impl_.kind_, 0, static_cast( reinterpret_cast(&_impl_.data_type_) - reinterpret_cast(&_impl_.kind_)) + sizeof(_impl_.data_type_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* BatchInput::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // .inference.BatchInput.Kind kind = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_kind(static_cast<::inference::BatchInput_Kind>(val)); } else goto handle_unusual; continue; // repeated string target_name = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr -= 1; do { ptr += 1; auto str = _internal_add_target_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.BatchInput.target_name")); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); } else goto handle_unusual; continue; // .inference.DataType data_type = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_data_type(static_cast<::inference::DataType>(val)); } else goto handle_unusual; continue; // repeated string source_input = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 34)) { ptr -= 1; do { ptr += 1; auto str = _internal_add_source_input(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.BatchInput.source_input")); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<34>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* BatchInput::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.BatchInput) uint32_t cached_has_bits = 0; (void) cached_has_bits; // .inference.BatchInput.Kind kind = 1; if (this->_internal_kind() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 1, this->_internal_kind(), target); } // repeated string target_name = 2; for (int i = 0, n = this->_internal_target_name_size(); i < n; i++) { const auto& s = this->_internal_target_name(i); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( s.data(), static_cast(s.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.BatchInput.target_name"); target = stream->WriteString(2, s, target); } // .inference.DataType data_type = 3; if (this->_internal_data_type() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 3, this->_internal_data_type(), target); } // repeated string source_input = 4; for (int i = 0, n = this->_internal_source_input_size(); i < n; i++) { const auto& s = this->_internal_source_input(i); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( s.data(), static_cast(s.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.BatchInput.source_input"); target = stream->WriteString(4, s, target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.BatchInput) return target; } size_t BatchInput::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.BatchInput) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated string target_name = 2; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(_impl_.target_name_.size()); for (int i = 0, n = _impl_.target_name_.size(); i < n; i++) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( _impl_.target_name_.Get(i)); } // repeated string source_input = 4; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(_impl_.source_input_.size()); for (int i = 0, n = _impl_.source_input_.size(); i < n; i++) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( _impl_.source_input_.Get(i)); } // .inference.BatchInput.Kind kind = 1; if (this->_internal_kind() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_kind()); } // .inference.DataType data_type = 3; if (this->_internal_data_type() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_data_type()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData BatchInput::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, BatchInput::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*BatchInput::GetClassData() const { return &_class_data_; } void BatchInput::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.BatchInput) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.target_name_.MergeFrom(from._impl_.target_name_); _this->_impl_.source_input_.MergeFrom(from._impl_.source_input_); if (from._internal_kind() != 0) { _this->_internal_set_kind(from._internal_kind()); } if (from._internal_data_type() != 0) { _this->_internal_set_data_type(from._internal_data_type()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void BatchInput::CopyFrom(const BatchInput& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.BatchInput) if (&from == this) return; Clear(); MergeFrom(from); } bool BatchInput::IsInitialized() const { return true; } void BatchInput::InternalSwap(BatchInput* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.target_name_.InternalSwap(&other->_impl_.target_name_); _impl_.source_input_.InternalSwap(&other->_impl_.source_input_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(BatchInput, _impl_.data_type_) + sizeof(BatchInput::_impl_.data_type_) - PROTOBUF_FIELD_OFFSET(BatchInput, _impl_.kind_)>( reinterpret_cast(&_impl_.kind_), reinterpret_cast(&other->_impl_.kind_)); } ::PROTOBUF_NAMESPACE_ID::Metadata BatchInput::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[7]); } // =================================================================== class BatchOutput::_Internal { public: }; BatchOutput::BatchOutput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.BatchOutput) } BatchOutput::BatchOutput(const BatchOutput& from) : ::PROTOBUF_NAMESPACE_ID::Message() { BatchOutput* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.target_name_){from._impl_.target_name_} , decltype(_impl_.source_input_){from._impl_.source_input_} , decltype(_impl_.kind_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.kind_ = from._impl_.kind_; // @@protoc_insertion_point(copy_constructor:inference.BatchOutput) } inline void BatchOutput::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.target_name_){arena} , decltype(_impl_.source_input_){arena} , decltype(_impl_.kind_){0} , /*decltype(_impl_._cached_size_)*/{} }; } BatchOutput::~BatchOutput() { // @@protoc_insertion_point(destructor:inference.BatchOutput) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void BatchOutput::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.target_name_.~RepeatedPtrField(); _impl_.source_input_.~RepeatedPtrField(); } void BatchOutput::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void BatchOutput::Clear() { // @@protoc_insertion_point(message_clear_start:inference.BatchOutput) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.target_name_.Clear(); _impl_.source_input_.Clear(); _impl_.kind_ = 0; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* BatchOutput::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated string target_name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr -= 1; do { ptr += 1; auto str = _internal_add_target_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.BatchOutput.target_name")); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); } else goto handle_unusual; continue; // .inference.BatchOutput.Kind kind = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_kind(static_cast<::inference::BatchOutput_Kind>(val)); } else goto handle_unusual; continue; // repeated string source_input = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr -= 1; do { ptr += 1; auto str = _internal_add_source_input(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.BatchOutput.source_input")); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<26>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* BatchOutput::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.BatchOutput) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated string target_name = 1; for (int i = 0, n = this->_internal_target_name_size(); i < n; i++) { const auto& s = this->_internal_target_name(i); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( s.data(), static_cast(s.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.BatchOutput.target_name"); target = stream->WriteString(1, s, target); } // .inference.BatchOutput.Kind kind = 2; if (this->_internal_kind() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 2, this->_internal_kind(), target); } // repeated string source_input = 3; for (int i = 0, n = this->_internal_source_input_size(); i < n; i++) { const auto& s = this->_internal_source_input(i); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( s.data(), static_cast(s.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.BatchOutput.source_input"); target = stream->WriteString(3, s, target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.BatchOutput) return target; } size_t BatchOutput::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.BatchOutput) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated string target_name = 1; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(_impl_.target_name_.size()); for (int i = 0, n = _impl_.target_name_.size(); i < n; i++) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( _impl_.target_name_.Get(i)); } // repeated string source_input = 3; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(_impl_.source_input_.size()); for (int i = 0, n = _impl_.source_input_.size(); i < n; i++) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( _impl_.source_input_.Get(i)); } // .inference.BatchOutput.Kind kind = 2; if (this->_internal_kind() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_kind()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData BatchOutput::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, BatchOutput::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*BatchOutput::GetClassData() const { return &_class_data_; } void BatchOutput::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.BatchOutput) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.target_name_.MergeFrom(from._impl_.target_name_); _this->_impl_.source_input_.MergeFrom(from._impl_.source_input_); if (from._internal_kind() != 0) { _this->_internal_set_kind(from._internal_kind()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void BatchOutput::CopyFrom(const BatchOutput& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.BatchOutput) if (&from == this) return; Clear(); MergeFrom(from); } bool BatchOutput::IsInitialized() const { return true; } void BatchOutput::InternalSwap(BatchOutput* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.target_name_.InternalSwap(&other->_impl_.target_name_); _impl_.source_input_.InternalSwap(&other->_impl_.source_input_); swap(_impl_.kind_, other->_impl_.kind_); } ::PROTOBUF_NAMESPACE_ID::Metadata BatchOutput::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[8]); } // =================================================================== class ModelVersionPolicy_Latest::_Internal { public: }; ModelVersionPolicy_Latest::ModelVersionPolicy_Latest(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelVersionPolicy.Latest) } ModelVersionPolicy_Latest::ModelVersionPolicy_Latest(const ModelVersionPolicy_Latest& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelVersionPolicy_Latest* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.num_versions_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.num_versions_ = from._impl_.num_versions_; // @@protoc_insertion_point(copy_constructor:inference.ModelVersionPolicy.Latest) } inline void ModelVersionPolicy_Latest::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.num_versions_){0u} , /*decltype(_impl_._cached_size_)*/{} }; } ModelVersionPolicy_Latest::~ModelVersionPolicy_Latest() { // @@protoc_insertion_point(destructor:inference.ModelVersionPolicy.Latest) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelVersionPolicy_Latest::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); } void ModelVersionPolicy_Latest::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelVersionPolicy_Latest::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelVersionPolicy.Latest) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.num_versions_ = 0u; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelVersionPolicy_Latest::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // uint32 num_versions = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.num_versions_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelVersionPolicy_Latest::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelVersionPolicy.Latest) uint32_t cached_has_bits = 0; (void) cached_has_bits; // uint32 num_versions = 1; if (this->_internal_num_versions() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt32ToArray(1, this->_internal_num_versions(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelVersionPolicy.Latest) return target; } size_t ModelVersionPolicy_Latest::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelVersionPolicy.Latest) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // uint32 num_versions = 1; if (this->_internal_num_versions() != 0) { total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne(this->_internal_num_versions()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelVersionPolicy_Latest::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelVersionPolicy_Latest::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelVersionPolicy_Latest::GetClassData() const { return &_class_data_; } void ModelVersionPolicy_Latest::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelVersionPolicy.Latest) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_num_versions() != 0) { _this->_internal_set_num_versions(from._internal_num_versions()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelVersionPolicy_Latest::CopyFrom(const ModelVersionPolicy_Latest& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelVersionPolicy.Latest) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelVersionPolicy_Latest::IsInitialized() const { return true; } void ModelVersionPolicy_Latest::InternalSwap(ModelVersionPolicy_Latest* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_.num_versions_, other->_impl_.num_versions_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelVersionPolicy_Latest::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[9]); } // =================================================================== class ModelVersionPolicy_All::_Internal { public: }; ModelVersionPolicy_All::ModelVersionPolicy_All(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase(arena, is_message_owned) { // @@protoc_insertion_point(arena_constructor:inference.ModelVersionPolicy.All) } ModelVersionPolicy_All::ModelVersionPolicy_All(const ModelVersionPolicy_All& from) : ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase() { ModelVersionPolicy_All* const _this = this; (void)_this; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:inference.ModelVersionPolicy.All) } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelVersionPolicy_All::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase::CopyImpl, ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase::MergeImpl, }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelVersionPolicy_All::GetClassData() const { return &_class_data_; } ::PROTOBUF_NAMESPACE_ID::Metadata ModelVersionPolicy_All::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[10]); } // =================================================================== class ModelVersionPolicy_Specific::_Internal { public: }; ModelVersionPolicy_Specific::ModelVersionPolicy_Specific(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelVersionPolicy.Specific) } ModelVersionPolicy_Specific::ModelVersionPolicy_Specific(const ModelVersionPolicy_Specific& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelVersionPolicy_Specific* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.versions_){from._impl_.versions_} , /*decltype(_impl_._versions_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:inference.ModelVersionPolicy.Specific) } inline void ModelVersionPolicy_Specific::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.versions_){arena} , /*decltype(_impl_._versions_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelVersionPolicy_Specific::~ModelVersionPolicy_Specific() { // @@protoc_insertion_point(destructor:inference.ModelVersionPolicy.Specific) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelVersionPolicy_Specific::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.versions_.~RepeatedField(); } void ModelVersionPolicy_Specific::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelVersionPolicy_Specific::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelVersionPolicy.Specific) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.versions_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelVersionPolicy_Specific::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated int64 versions = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(_internal_mutable_versions(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 8) { _internal_add_versions(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelVersionPolicy_Specific::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelVersionPolicy.Specific) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated int64 versions = 1; { int byte_size = _impl_._versions_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt64Packed( 1, _internal_versions(), byte_size, target); } } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelVersionPolicy.Specific) return target; } size_t ModelVersionPolicy_Specific::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelVersionPolicy.Specific) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int64 versions = 1; { size_t data_size = ::_pbi::WireFormatLite:: Int64Size(this->_impl_.versions_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._versions_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelVersionPolicy_Specific::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelVersionPolicy_Specific::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelVersionPolicy_Specific::GetClassData() const { return &_class_data_; } void ModelVersionPolicy_Specific::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelVersionPolicy.Specific) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.versions_.MergeFrom(from._impl_.versions_); _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelVersionPolicy_Specific::CopyFrom(const ModelVersionPolicy_Specific& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelVersionPolicy.Specific) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelVersionPolicy_Specific::IsInitialized() const { return true; } void ModelVersionPolicy_Specific::InternalSwap(ModelVersionPolicy_Specific* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.versions_.InternalSwap(&other->_impl_.versions_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelVersionPolicy_Specific::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[11]); } // =================================================================== class ModelVersionPolicy::_Internal { public: static const ::inference::ModelVersionPolicy_Latest& latest(const ModelVersionPolicy* msg); static const ::inference::ModelVersionPolicy_All& all(const ModelVersionPolicy* msg); static const ::inference::ModelVersionPolicy_Specific& specific(const ModelVersionPolicy* msg); }; const ::inference::ModelVersionPolicy_Latest& ModelVersionPolicy::_Internal::latest(const ModelVersionPolicy* msg) { return *msg->_impl_.policy_choice_.latest_; } const ::inference::ModelVersionPolicy_All& ModelVersionPolicy::_Internal::all(const ModelVersionPolicy* msg) { return *msg->_impl_.policy_choice_.all_; } const ::inference::ModelVersionPolicy_Specific& ModelVersionPolicy::_Internal::specific(const ModelVersionPolicy* msg) { return *msg->_impl_.policy_choice_.specific_; } void ModelVersionPolicy::set_allocated_latest(::inference::ModelVersionPolicy_Latest* latest) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); clear_policy_choice(); if (latest) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(latest); if (message_arena != submessage_arena) { latest = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, latest, submessage_arena); } set_has_latest(); _impl_.policy_choice_.latest_ = latest; } // @@protoc_insertion_point(field_set_allocated:inference.ModelVersionPolicy.latest) } void ModelVersionPolicy::set_allocated_all(::inference::ModelVersionPolicy_All* all) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); clear_policy_choice(); if (all) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(all); if (message_arena != submessage_arena) { all = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, all, submessage_arena); } set_has_all(); _impl_.policy_choice_.all_ = all; } // @@protoc_insertion_point(field_set_allocated:inference.ModelVersionPolicy.all) } void ModelVersionPolicy::set_allocated_specific(::inference::ModelVersionPolicy_Specific* specific) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); clear_policy_choice(); if (specific) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(specific); if (message_arena != submessage_arena) { specific = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, specific, submessage_arena); } set_has_specific(); _impl_.policy_choice_.specific_ = specific; } // @@protoc_insertion_point(field_set_allocated:inference.ModelVersionPolicy.specific) } ModelVersionPolicy::ModelVersionPolicy(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelVersionPolicy) } ModelVersionPolicy::ModelVersionPolicy(const ModelVersionPolicy& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelVersionPolicy* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.policy_choice_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); clear_has_policy_choice(); switch (from.policy_choice_case()) { case kLatest: { _this->_internal_mutable_latest()->::inference::ModelVersionPolicy_Latest::MergeFrom( from._internal_latest()); break; } case kAll: { _this->_internal_mutable_all()->::inference::ModelVersionPolicy_All::MergeFrom( from._internal_all()); break; } case kSpecific: { _this->_internal_mutable_specific()->::inference::ModelVersionPolicy_Specific::MergeFrom( from._internal_specific()); break; } case POLICY_CHOICE_NOT_SET: { break; } } // @@protoc_insertion_point(copy_constructor:inference.ModelVersionPolicy) } inline void ModelVersionPolicy::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.policy_choice_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{} }; clear_has_policy_choice(); } ModelVersionPolicy::~ModelVersionPolicy() { // @@protoc_insertion_point(destructor:inference.ModelVersionPolicy) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelVersionPolicy::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); if (has_policy_choice()) { clear_policy_choice(); } } void ModelVersionPolicy::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelVersionPolicy::clear_policy_choice() { // @@protoc_insertion_point(one_of_clear_start:inference.ModelVersionPolicy) switch (policy_choice_case()) { case kLatest: { if (GetArenaForAllocation() == nullptr) { delete _impl_.policy_choice_.latest_; } break; } case kAll: { if (GetArenaForAllocation() == nullptr) { delete _impl_.policy_choice_.all_; } break; } case kSpecific: { if (GetArenaForAllocation() == nullptr) { delete _impl_.policy_choice_.specific_; } break; } case POLICY_CHOICE_NOT_SET: { break; } } _impl_._oneof_case_[0] = POLICY_CHOICE_NOT_SET; } void ModelVersionPolicy::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelVersionPolicy) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; clear_policy_choice(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelVersionPolicy::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // .inference.ModelVersionPolicy.Latest latest = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr = ctx->ParseMessage(_internal_mutable_latest(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelVersionPolicy.All all = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr = ctx->ParseMessage(_internal_mutable_all(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelVersionPolicy.Specific specific = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr = ctx->ParseMessage(_internal_mutable_specific(), ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelVersionPolicy::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelVersionPolicy) uint32_t cached_has_bits = 0; (void) cached_has_bits; // .inference.ModelVersionPolicy.Latest latest = 1; if (_internal_has_latest()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(1, _Internal::latest(this), _Internal::latest(this).GetCachedSize(), target, stream); } // .inference.ModelVersionPolicy.All all = 2; if (_internal_has_all()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(2, _Internal::all(this), _Internal::all(this).GetCachedSize(), target, stream); } // .inference.ModelVersionPolicy.Specific specific = 3; if (_internal_has_specific()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(3, _Internal::specific(this), _Internal::specific(this).GetCachedSize(), target, stream); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelVersionPolicy) return target; } size_t ModelVersionPolicy::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelVersionPolicy) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; switch (policy_choice_case()) { // .inference.ModelVersionPolicy.Latest latest = 1; case kLatest: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.policy_choice_.latest_); break; } // .inference.ModelVersionPolicy.All all = 2; case kAll: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.policy_choice_.all_); break; } // .inference.ModelVersionPolicy.Specific specific = 3; case kSpecific: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.policy_choice_.specific_); break; } case POLICY_CHOICE_NOT_SET: { break; } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelVersionPolicy::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelVersionPolicy::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelVersionPolicy::GetClassData() const { return &_class_data_; } void ModelVersionPolicy::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelVersionPolicy) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; switch (from.policy_choice_case()) { case kLatest: { _this->_internal_mutable_latest()->::inference::ModelVersionPolicy_Latest::MergeFrom( from._internal_latest()); break; } case kAll: { _this->_internal_mutable_all()->::inference::ModelVersionPolicy_All::MergeFrom( from._internal_all()); break; } case kSpecific: { _this->_internal_mutable_specific()->::inference::ModelVersionPolicy_Specific::MergeFrom( from._internal_specific()); break; } case POLICY_CHOICE_NOT_SET: { break; } } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelVersionPolicy::CopyFrom(const ModelVersionPolicy& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelVersionPolicy) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelVersionPolicy::IsInitialized() const { return true; } void ModelVersionPolicy::InternalSwap(ModelVersionPolicy* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_.policy_choice_, other->_impl_.policy_choice_); swap(_impl_._oneof_case_[0], other->_impl_._oneof_case_[0]); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelVersionPolicy::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[12]); } // =================================================================== class ModelOptimizationPolicy_Graph::_Internal { public: }; ModelOptimizationPolicy_Graph::ModelOptimizationPolicy_Graph(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy.Graph) } ModelOptimizationPolicy_Graph::ModelOptimizationPolicy_Graph(const ModelOptimizationPolicy_Graph& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy_Graph* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.level_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.level_ = from._impl_.level_; // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy.Graph) } inline void ModelOptimizationPolicy_Graph::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.level_){0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOptimizationPolicy_Graph::~ModelOptimizationPolicy_Graph() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy.Graph) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelOptimizationPolicy_Graph::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); } void ModelOptimizationPolicy_Graph::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy_Graph::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy.Graph) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.level_ = 0; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy_Graph::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // int32 level = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.level_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy_Graph::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy.Graph) uint32_t cached_has_bits = 0; (void) cached_has_bits; // int32 level = 1; if (this->_internal_level() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteInt32ToArray(1, this->_internal_level(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy.Graph) return target; } size_t ModelOptimizationPolicy_Graph::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy.Graph) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // int32 level = 1; if (this->_internal_level() != 0) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne(this->_internal_level()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy_Graph::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy_Graph::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy_Graph::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy_Graph::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy.Graph) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_level() != 0) { _this->_internal_set_level(from._internal_level()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy_Graph::CopyFrom(const ModelOptimizationPolicy_Graph& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy.Graph) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy_Graph::IsInitialized() const { return true; } void ModelOptimizationPolicy_Graph::InternalSwap(ModelOptimizationPolicy_Graph* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_.level_, other->_impl_.level_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_Graph::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[13]); } // =================================================================== class ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_Internal { public: }; ModelOptimizationPolicy_Cuda_GraphSpec_Shape::ModelOptimizationPolicy_Cuda_GraphSpec_Shape(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) } ModelOptimizationPolicy_Cuda_GraphSpec_Shape::ModelOptimizationPolicy_Cuda_GraphSpec_Shape(const ModelOptimizationPolicy_Cuda_GraphSpec_Shape& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy_Cuda_GraphSpec_Shape* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.dim_){from._impl_.dim_} , /*decltype(_impl_._dim_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) } inline void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.dim_){arena} , /*decltype(_impl_._dim_cached_byte_size_)*/{0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOptimizationPolicy_Cuda_GraphSpec_Shape::~ModelOptimizationPolicy_Cuda_GraphSpec_Shape() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.dim_.~RepeatedField(); } void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.dim_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated int64 dim = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(_internal_mutable_dim(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 8) { _internal_add_dim(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated int64 dim = 1; { int byte_size = _impl_._dim_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt64Packed( 1, _internal_dim(), byte_size, target); } } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) return target; } size_t ModelOptimizationPolicy_Cuda_GraphSpec_Shape::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int64 dim = 1; { size_t data_size = ::_pbi::WireFormatLite:: Int64Size(this->_impl_.dim_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._dim_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy_Cuda_GraphSpec_Shape::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy_Cuda_GraphSpec_Shape::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.dim_.MergeFrom(from._impl_.dim_); _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::CopyFrom(const ModelOptimizationPolicy_Cuda_GraphSpec_Shape& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy_Cuda_GraphSpec_Shape::IsInitialized() const { return true; } void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::InternalSwap(ModelOptimizationPolicy_Cuda_GraphSpec_Shape* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.dim_.InternalSwap(&other->_impl_.dim_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_Cuda_GraphSpec_Shape::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[14]); } // =================================================================== ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse() {} ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse::MergeFrom(const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[15]); } // =================================================================== class ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_Internal { public: }; ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); if (arena != nullptr && !is_message_owned) { arena->OwnCustomDestructor(this, &ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::ArenaDtor); } // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) } ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound(const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* const _this = this; (void)_this; new (&_impl_) Impl_{ /*decltype(_impl_.input_)*/{} , decltype(_impl_.batch_size_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.input_.MergeFrom(from._impl_.input_); _this->_impl_.batch_size_ = from._impl_.batch_size_; // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) } inline void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ /*decltype(_impl_.input_)*/{::_pbi::ArenaInitialized(), arena} , decltype(_impl_.batch_size_){0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::~ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; ArenaDtor(this); return; } SharedDtor(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.input_.Destruct(); _impl_.input_.~MapField(); } void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::ArenaDtor(void* object) { ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* _this = reinterpret_cast< ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* >(object); _this->_impl_.input_.Destruct(); } void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.input_.Clear(); _impl_.batch_size_ = 0; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // int32 batch_size = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.batch_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // map input = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.input_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) uint32_t cached_has_bits = 0; (void) cached_has_bits; // int32 batch_size = 1; if (this->_internal_batch_size() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteInt32ToArray(1, this->_internal_batch_size(), target); } // map input = 2; if (!this->_internal_input().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_input(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound.InputEntry.key"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(2, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(2, entry.first, entry.second, target, stream); check_utf8(entry); } } } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) return target; } size_t ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // map input = 2; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_input_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >::const_iterator it = this->_internal_input().begin(); it != this->_internal_input().end(); ++it) { total_size += ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // int32 batch_size = 1; if (this->_internal_batch_size() != 0) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne(this->_internal_batch_size()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.input_.MergeFrom(from._impl_.input_); if (from._internal_batch_size() != 0) { _this->_internal_set_batch_size(from._internal_batch_size()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::CopyFrom(const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::IsInitialized() const { return true; } void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::InternalSwap(ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.input_.InternalSwap(&other->_impl_.input_); swap(_impl_.batch_size_, other->_impl_.batch_size_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[16]); } // =================================================================== ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse() {} ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse::MergeFrom(const ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[17]); } // =================================================================== class ModelOptimizationPolicy_Cuda_GraphSpec::_Internal { public: static const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& graph_lower_bound(const ModelOptimizationPolicy_Cuda_GraphSpec* msg); }; const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& ModelOptimizationPolicy_Cuda_GraphSpec::_Internal::graph_lower_bound(const ModelOptimizationPolicy_Cuda_GraphSpec* msg) { return *msg->_impl_.graph_lower_bound_; } ModelOptimizationPolicy_Cuda_GraphSpec::ModelOptimizationPolicy_Cuda_GraphSpec(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); if (arena != nullptr && !is_message_owned) { arena->OwnCustomDestructor(this, &ModelOptimizationPolicy_Cuda_GraphSpec::ArenaDtor); } // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec) } ModelOptimizationPolicy_Cuda_GraphSpec::ModelOptimizationPolicy_Cuda_GraphSpec(const ModelOptimizationPolicy_Cuda_GraphSpec& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy_Cuda_GraphSpec* const _this = this; (void)_this; new (&_impl_) Impl_{ /*decltype(_impl_.input_)*/{} , decltype(_impl_.graph_lower_bound_){nullptr} , decltype(_impl_.batch_size_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.input_.MergeFrom(from._impl_.input_); if (from._internal_has_graph_lower_bound()) { _this->_impl_.graph_lower_bound_ = new ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound(*from._impl_.graph_lower_bound_); } _this->_impl_.batch_size_ = from._impl_.batch_size_; // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec) } inline void ModelOptimizationPolicy_Cuda_GraphSpec::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ /*decltype(_impl_.input_)*/{::_pbi::ArenaInitialized(), arena} , decltype(_impl_.graph_lower_bound_){nullptr} , decltype(_impl_.batch_size_){0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOptimizationPolicy_Cuda_GraphSpec::~ModelOptimizationPolicy_Cuda_GraphSpec() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy.Cuda.GraphSpec) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; ArenaDtor(this); return; } SharedDtor(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.input_.Destruct(); _impl_.input_.~MapField(); if (this != internal_default_instance()) delete _impl_.graph_lower_bound_; } void ModelOptimizationPolicy_Cuda_GraphSpec::ArenaDtor(void* object) { ModelOptimizationPolicy_Cuda_GraphSpec* _this = reinterpret_cast< ModelOptimizationPolicy_Cuda_GraphSpec* >(object); _this->_impl_.input_.Destruct(); } void ModelOptimizationPolicy_Cuda_GraphSpec::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy_Cuda_GraphSpec::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.input_.Clear(); if (GetArenaForAllocation() == nullptr && _impl_.graph_lower_bound_ != nullptr) { delete _impl_.graph_lower_bound_; } _impl_.graph_lower_bound_ = nullptr; _impl_.batch_size_ = 0; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy_Cuda_GraphSpec::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // int32 batch_size = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.batch_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // map input = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.input_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); } else goto handle_unusual; continue; // .inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound graph_lower_bound = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr = ctx->ParseMessage(_internal_mutable_graph_lower_bound(), ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy_Cuda_GraphSpec::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec) uint32_t cached_has_bits = 0; (void) cached_has_bits; // int32 batch_size = 1; if (this->_internal_batch_size() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteInt32ToArray(1, this->_internal_batch_size(), target); } // map input = 2; if (!this->_internal_input().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_input(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelOptimizationPolicy.Cuda.GraphSpec.InputEntry.key"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(2, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(2, entry.first, entry.second, target, stream); check_utf8(entry); } } } // .inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound graph_lower_bound = 3; if (this->_internal_has_graph_lower_bound()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(3, _Internal::graph_lower_bound(this), _Internal::graph_lower_bound(this).GetCachedSize(), target, stream); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy.Cuda.GraphSpec) return target; } size_t ModelOptimizationPolicy_Cuda_GraphSpec::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // map input = 2; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_input_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >::const_iterator it = this->_internal_input().begin(); it != this->_internal_input().end(); ++it) { total_size += ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // .inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound graph_lower_bound = 3; if (this->_internal_has_graph_lower_bound()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.graph_lower_bound_); } // int32 batch_size = 1; if (this->_internal_batch_size() != 0) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne(this->_internal_batch_size()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy_Cuda_GraphSpec::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy_Cuda_GraphSpec::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy_Cuda_GraphSpec::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy_Cuda_GraphSpec::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.input_.MergeFrom(from._impl_.input_); if (from._internal_has_graph_lower_bound()) { _this->_internal_mutable_graph_lower_bound()->::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::MergeFrom( from._internal_graph_lower_bound()); } if (from._internal_batch_size() != 0) { _this->_internal_set_batch_size(from._internal_batch_size()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy_Cuda_GraphSpec::CopyFrom(const ModelOptimizationPolicy_Cuda_GraphSpec& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy.Cuda.GraphSpec) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy_Cuda_GraphSpec::IsInitialized() const { return true; } void ModelOptimizationPolicy_Cuda_GraphSpec::InternalSwap(ModelOptimizationPolicy_Cuda_GraphSpec* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.input_.InternalSwap(&other->_impl_.input_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelOptimizationPolicy_Cuda_GraphSpec, _impl_.batch_size_) + sizeof(ModelOptimizationPolicy_Cuda_GraphSpec::_impl_.batch_size_) - PROTOBUF_FIELD_OFFSET(ModelOptimizationPolicy_Cuda_GraphSpec, _impl_.graph_lower_bound_)>( reinterpret_cast(&_impl_.graph_lower_bound_), reinterpret_cast(&other->_impl_.graph_lower_bound_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_Cuda_GraphSpec::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[18]); } // =================================================================== class ModelOptimizationPolicy_Cuda::_Internal { public: }; ModelOptimizationPolicy_Cuda::ModelOptimizationPolicy_Cuda(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy.Cuda) } ModelOptimizationPolicy_Cuda::ModelOptimizationPolicy_Cuda(const ModelOptimizationPolicy_Cuda& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy_Cuda* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.graph_spec_){from._impl_.graph_spec_} , decltype(_impl_.graphs_){} , decltype(_impl_.busy_wait_events_){} , decltype(_impl_.output_copy_stream_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); ::memcpy(&_impl_.graphs_, &from._impl_.graphs_, static_cast(reinterpret_cast(&_impl_.output_copy_stream_) - reinterpret_cast(&_impl_.graphs_)) + sizeof(_impl_.output_copy_stream_)); // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy.Cuda) } inline void ModelOptimizationPolicy_Cuda::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.graph_spec_){arena} , decltype(_impl_.graphs_){false} , decltype(_impl_.busy_wait_events_){false} , decltype(_impl_.output_copy_stream_){false} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOptimizationPolicy_Cuda::~ModelOptimizationPolicy_Cuda() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy.Cuda) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelOptimizationPolicy_Cuda::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.graph_spec_.~RepeatedPtrField(); } void ModelOptimizationPolicy_Cuda::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy_Cuda::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy.Cuda) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.graph_spec_.Clear(); ::memset(&_impl_.graphs_, 0, static_cast( reinterpret_cast(&_impl_.output_copy_stream_) - reinterpret_cast(&_impl_.graphs_)) + sizeof(_impl_.output_copy_stream_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy_Cuda::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // bool graphs = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.graphs_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool busy_wait_events = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.busy_wait_events_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated .inference.ModelOptimizationPolicy.Cuda.GraphSpec graph_spec = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_graph_spec(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<26>(ptr)); } else goto handle_unusual; continue; // bool output_copy_stream = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { _impl_.output_copy_stream_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy_Cuda::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy.Cuda) uint32_t cached_has_bits = 0; (void) cached_has_bits; // bool graphs = 1; if (this->_internal_graphs() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(1, this->_internal_graphs(), target); } // bool busy_wait_events = 2; if (this->_internal_busy_wait_events() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(2, this->_internal_busy_wait_events(), target); } // repeated .inference.ModelOptimizationPolicy.Cuda.GraphSpec graph_spec = 3; for (unsigned i = 0, n = static_cast(this->_internal_graph_spec_size()); i < n; i++) { const auto& repfield = this->_internal_graph_spec(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(3, repfield, repfield.GetCachedSize(), target, stream); } // bool output_copy_stream = 4; if (this->_internal_output_copy_stream() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(4, this->_internal_output_copy_stream(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy.Cuda) return target; } size_t ModelOptimizationPolicy_Cuda::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy.Cuda) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated .inference.ModelOptimizationPolicy.Cuda.GraphSpec graph_spec = 3; total_size += 1UL * this->_internal_graph_spec_size(); for (const auto& msg : this->_impl_.graph_spec_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // bool graphs = 1; if (this->_internal_graphs() != 0) { total_size += 1 + 1; } // bool busy_wait_events = 2; if (this->_internal_busy_wait_events() != 0) { total_size += 1 + 1; } // bool output_copy_stream = 4; if (this->_internal_output_copy_stream() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy_Cuda::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy_Cuda::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy_Cuda::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy_Cuda::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy.Cuda) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.graph_spec_.MergeFrom(from._impl_.graph_spec_); if (from._internal_graphs() != 0) { _this->_internal_set_graphs(from._internal_graphs()); } if (from._internal_busy_wait_events() != 0) { _this->_internal_set_busy_wait_events(from._internal_busy_wait_events()); } if (from._internal_output_copy_stream() != 0) { _this->_internal_set_output_copy_stream(from._internal_output_copy_stream()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy_Cuda::CopyFrom(const ModelOptimizationPolicy_Cuda& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy.Cuda) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy_Cuda::IsInitialized() const { return true; } void ModelOptimizationPolicy_Cuda::InternalSwap(ModelOptimizationPolicy_Cuda* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.graph_spec_.InternalSwap(&other->_impl_.graph_spec_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelOptimizationPolicy_Cuda, _impl_.output_copy_stream_) + sizeof(ModelOptimizationPolicy_Cuda::_impl_.output_copy_stream_) - PROTOBUF_FIELD_OFFSET(ModelOptimizationPolicy_Cuda, _impl_.graphs_)>( reinterpret_cast(&_impl_.graphs_), reinterpret_cast(&other->_impl_.graphs_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_Cuda::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[19]); } // =================================================================== ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse() {} ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse::MergeFrom(const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[20]); } // =================================================================== class ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_Internal { public: }; ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); if (arena != nullptr && !is_message_owned) { arena->OwnCustomDestructor(this, &ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::ArenaDtor); } // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) } ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator(const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* const _this = this; (void)_this; new (&_impl_) Impl_{ /*decltype(_impl_.parameters_)*/{} , decltype(_impl_.name_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.parameters_.MergeFrom(from._impl_.parameters_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) } inline void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ /*decltype(_impl_.parameters_)*/{::_pbi::ArenaInitialized(), arena} , decltype(_impl_.name_){} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::~ModelOptimizationPolicy_ExecutionAccelerators_Accelerator() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; ArenaDtor(this); return; } SharedDtor(); } inline void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.parameters_.Destruct(); _impl_.parameters_.~MapField(); _impl_.name_.Destroy(); } void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::ArenaDtor(void* object) { ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* _this = reinterpret_cast< ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* >(object); _this->_impl_.parameters_.Destruct(); } void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.parameters_.Clear(); _impl_.name_.ClearToEmpty(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.name")); } else goto handle_unusual; continue; // map parameters = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.parameters_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // map parameters = 2; if (!this->_internal_parameters().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_parameters(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.ParametersEntry.key"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.second.data(), static_cast(entry.second.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.ParametersEntry.value"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(2, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(2, entry.first, entry.second, target, stream); check_utf8(entry); } } } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) return target; } size_t ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // map parameters = 2; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_parameters_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator it = this->_internal_parameters().begin(); it != this->_internal_parameters().end(); ++it) { total_size += ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.parameters_.MergeFrom(from._impl_.parameters_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::CopyFrom(const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::IsInitialized() const { return true; } void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::InternalSwap(ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.parameters_.InternalSwap(&other->_impl_.parameters_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[21]); } // =================================================================== class ModelOptimizationPolicy_ExecutionAccelerators::_Internal { public: }; ModelOptimizationPolicy_ExecutionAccelerators::ModelOptimizationPolicy_ExecutionAccelerators(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy.ExecutionAccelerators) } ModelOptimizationPolicy_ExecutionAccelerators::ModelOptimizationPolicy_ExecutionAccelerators(const ModelOptimizationPolicy_ExecutionAccelerators& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy_ExecutionAccelerators* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.gpu_execution_accelerator_){from._impl_.gpu_execution_accelerator_} , decltype(_impl_.cpu_execution_accelerator_){from._impl_.cpu_execution_accelerator_} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy.ExecutionAccelerators) } inline void ModelOptimizationPolicy_ExecutionAccelerators::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.gpu_execution_accelerator_){arena} , decltype(_impl_.cpu_execution_accelerator_){arena} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOptimizationPolicy_ExecutionAccelerators::~ModelOptimizationPolicy_ExecutionAccelerators() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy.ExecutionAccelerators) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelOptimizationPolicy_ExecutionAccelerators::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.gpu_execution_accelerator_.~RepeatedPtrField(); _impl_.cpu_execution_accelerator_.~RepeatedPtrField(); } void ModelOptimizationPolicy_ExecutionAccelerators::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy_ExecutionAccelerators::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy.ExecutionAccelerators) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.gpu_execution_accelerator_.Clear(); _impl_.cpu_execution_accelerator_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy_ExecutionAccelerators::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator gpu_execution_accelerator = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_gpu_execution_accelerator(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); } else goto handle_unusual; continue; // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator cpu_execution_accelerator = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_cpu_execution_accelerator(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy_ExecutionAccelerators::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy.ExecutionAccelerators) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator gpu_execution_accelerator = 1; for (unsigned i = 0, n = static_cast(this->_internal_gpu_execution_accelerator_size()); i < n; i++) { const auto& repfield = this->_internal_gpu_execution_accelerator(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(1, repfield, repfield.GetCachedSize(), target, stream); } // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator cpu_execution_accelerator = 2; for (unsigned i = 0, n = static_cast(this->_internal_cpu_execution_accelerator_size()); i < n; i++) { const auto& repfield = this->_internal_cpu_execution_accelerator(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(2, repfield, repfield.GetCachedSize(), target, stream); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy.ExecutionAccelerators) return target; } size_t ModelOptimizationPolicy_ExecutionAccelerators::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy.ExecutionAccelerators) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator gpu_execution_accelerator = 1; total_size += 1UL * this->_internal_gpu_execution_accelerator_size(); for (const auto& msg : this->_impl_.gpu_execution_accelerator_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator cpu_execution_accelerator = 2; total_size += 1UL * this->_internal_cpu_execution_accelerator_size(); for (const auto& msg : this->_impl_.cpu_execution_accelerator_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy_ExecutionAccelerators::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy_ExecutionAccelerators::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy_ExecutionAccelerators::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy_ExecutionAccelerators::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy.ExecutionAccelerators) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.gpu_execution_accelerator_.MergeFrom(from._impl_.gpu_execution_accelerator_); _this->_impl_.cpu_execution_accelerator_.MergeFrom(from._impl_.cpu_execution_accelerator_); _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy_ExecutionAccelerators::CopyFrom(const ModelOptimizationPolicy_ExecutionAccelerators& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy.ExecutionAccelerators) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy_ExecutionAccelerators::IsInitialized() const { return true; } void ModelOptimizationPolicy_ExecutionAccelerators::InternalSwap(ModelOptimizationPolicy_ExecutionAccelerators* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.gpu_execution_accelerator_.InternalSwap(&other->_impl_.gpu_execution_accelerator_); _impl_.cpu_execution_accelerator_.InternalSwap(&other->_impl_.cpu_execution_accelerator_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_ExecutionAccelerators::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[22]); } // =================================================================== class ModelOptimizationPolicy_PinnedMemoryBuffer::_Internal { public: }; ModelOptimizationPolicy_PinnedMemoryBuffer::ModelOptimizationPolicy_PinnedMemoryBuffer(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) } ModelOptimizationPolicy_PinnedMemoryBuffer::ModelOptimizationPolicy_PinnedMemoryBuffer(const ModelOptimizationPolicy_PinnedMemoryBuffer& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy_PinnedMemoryBuffer* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.enable_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.enable_ = from._impl_.enable_; // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) } inline void ModelOptimizationPolicy_PinnedMemoryBuffer::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.enable_){false} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOptimizationPolicy_PinnedMemoryBuffer::~ModelOptimizationPolicy_PinnedMemoryBuffer() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelOptimizationPolicy_PinnedMemoryBuffer::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); } void ModelOptimizationPolicy_PinnedMemoryBuffer::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy_PinnedMemoryBuffer::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.enable_ = false; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy_PinnedMemoryBuffer::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // bool enable = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.enable_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy_PinnedMemoryBuffer::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) uint32_t cached_has_bits = 0; (void) cached_has_bits; // bool enable = 1; if (this->_internal_enable() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(1, this->_internal_enable(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) return target; } size_t ModelOptimizationPolicy_PinnedMemoryBuffer::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // bool enable = 1; if (this->_internal_enable() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy_PinnedMemoryBuffer::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy_PinnedMemoryBuffer::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy_PinnedMemoryBuffer::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy_PinnedMemoryBuffer::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_enable() != 0) { _this->_internal_set_enable(from._internal_enable()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy_PinnedMemoryBuffer::CopyFrom(const ModelOptimizationPolicy_PinnedMemoryBuffer& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy_PinnedMemoryBuffer::IsInitialized() const { return true; } void ModelOptimizationPolicy_PinnedMemoryBuffer::InternalSwap(ModelOptimizationPolicy_PinnedMemoryBuffer* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_.enable_, other->_impl_.enable_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy_PinnedMemoryBuffer::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[23]); } // =================================================================== class ModelOptimizationPolicy::_Internal { public: static const ::inference::ModelOptimizationPolicy_Graph& graph(const ModelOptimizationPolicy* msg); static const ::inference::ModelOptimizationPolicy_Cuda& cuda(const ModelOptimizationPolicy* msg); static const ::inference::ModelOptimizationPolicy_ExecutionAccelerators& execution_accelerators(const ModelOptimizationPolicy* msg); static const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& input_pinned_memory(const ModelOptimizationPolicy* msg); static const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& output_pinned_memory(const ModelOptimizationPolicy* msg); }; const ::inference::ModelOptimizationPolicy_Graph& ModelOptimizationPolicy::_Internal::graph(const ModelOptimizationPolicy* msg) { return *msg->_impl_.graph_; } const ::inference::ModelOptimizationPolicy_Cuda& ModelOptimizationPolicy::_Internal::cuda(const ModelOptimizationPolicy* msg) { return *msg->_impl_.cuda_; } const ::inference::ModelOptimizationPolicy_ExecutionAccelerators& ModelOptimizationPolicy::_Internal::execution_accelerators(const ModelOptimizationPolicy* msg) { return *msg->_impl_.execution_accelerators_; } const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& ModelOptimizationPolicy::_Internal::input_pinned_memory(const ModelOptimizationPolicy* msg) { return *msg->_impl_.input_pinned_memory_; } const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& ModelOptimizationPolicy::_Internal::output_pinned_memory(const ModelOptimizationPolicy* msg) { return *msg->_impl_.output_pinned_memory_; } ModelOptimizationPolicy::ModelOptimizationPolicy(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelOptimizationPolicy) } ModelOptimizationPolicy::ModelOptimizationPolicy(const ModelOptimizationPolicy& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOptimizationPolicy* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.graph_){nullptr} , decltype(_impl_.cuda_){nullptr} , decltype(_impl_.execution_accelerators_){nullptr} , decltype(_impl_.input_pinned_memory_){nullptr} , decltype(_impl_.output_pinned_memory_){nullptr} , decltype(_impl_.priority_){} , decltype(_impl_.gather_kernel_buffer_threshold_){} , decltype(_impl_.eager_batching_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); if (from._internal_has_graph()) { _this->_impl_.graph_ = new ::inference::ModelOptimizationPolicy_Graph(*from._impl_.graph_); } if (from._internal_has_cuda()) { _this->_impl_.cuda_ = new ::inference::ModelOptimizationPolicy_Cuda(*from._impl_.cuda_); } if (from._internal_has_execution_accelerators()) { _this->_impl_.execution_accelerators_ = new ::inference::ModelOptimizationPolicy_ExecutionAccelerators(*from._impl_.execution_accelerators_); } if (from._internal_has_input_pinned_memory()) { _this->_impl_.input_pinned_memory_ = new ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer(*from._impl_.input_pinned_memory_); } if (from._internal_has_output_pinned_memory()) { _this->_impl_.output_pinned_memory_ = new ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer(*from._impl_.output_pinned_memory_); } ::memcpy(&_impl_.priority_, &from._impl_.priority_, static_cast(reinterpret_cast(&_impl_.eager_batching_) - reinterpret_cast(&_impl_.priority_)) + sizeof(_impl_.eager_batching_)); // @@protoc_insertion_point(copy_constructor:inference.ModelOptimizationPolicy) } inline void ModelOptimizationPolicy::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.graph_){nullptr} , decltype(_impl_.cuda_){nullptr} , decltype(_impl_.execution_accelerators_){nullptr} , decltype(_impl_.input_pinned_memory_){nullptr} , decltype(_impl_.output_pinned_memory_){nullptr} , decltype(_impl_.priority_){0} , decltype(_impl_.gather_kernel_buffer_threshold_){0u} , decltype(_impl_.eager_batching_){false} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOptimizationPolicy::~ModelOptimizationPolicy() { // @@protoc_insertion_point(destructor:inference.ModelOptimizationPolicy) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelOptimizationPolicy::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); if (this != internal_default_instance()) delete _impl_.graph_; if (this != internal_default_instance()) delete _impl_.cuda_; if (this != internal_default_instance()) delete _impl_.execution_accelerators_; if (this != internal_default_instance()) delete _impl_.input_pinned_memory_; if (this != internal_default_instance()) delete _impl_.output_pinned_memory_; } void ModelOptimizationPolicy::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOptimizationPolicy::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOptimizationPolicy) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; if (GetArenaForAllocation() == nullptr && _impl_.graph_ != nullptr) { delete _impl_.graph_; } _impl_.graph_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.cuda_ != nullptr) { delete _impl_.cuda_; } _impl_.cuda_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.execution_accelerators_ != nullptr) { delete _impl_.execution_accelerators_; } _impl_.execution_accelerators_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.input_pinned_memory_ != nullptr) { delete _impl_.input_pinned_memory_; } _impl_.input_pinned_memory_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.output_pinned_memory_ != nullptr) { delete _impl_.output_pinned_memory_; } _impl_.output_pinned_memory_ = nullptr; ::memset(&_impl_.priority_, 0, static_cast( reinterpret_cast(&_impl_.eager_batching_) - reinterpret_cast(&_impl_.priority_)) + sizeof(_impl_.eager_batching_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOptimizationPolicy::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // .inference.ModelOptimizationPolicy.Graph graph = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr = ctx->ParseMessage(_internal_mutable_graph(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelOptimizationPolicy.ModelPriority priority = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_priority(static_cast<::inference::ModelOptimizationPolicy_ModelPriority>(val)); } else goto handle_unusual; continue; // .inference.ModelOptimizationPolicy.Cuda cuda = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr = ctx->ParseMessage(_internal_mutable_cuda(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelOptimizationPolicy.ExecutionAccelerators execution_accelerators = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 34)) { ptr = ctx->ParseMessage(_internal_mutable_execution_accelerators(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer input_pinned_memory = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { ptr = ctx->ParseMessage(_internal_mutable_input_pinned_memory(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer output_pinned_memory = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 50)) { ptr = ctx->ParseMessage(_internal_mutable_output_pinned_memory(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // uint32 gather_kernel_buffer_threshold = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 56)) { _impl_.gather_kernel_buffer_threshold_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool eager_batching = 8; case 8: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 64)) { _impl_.eager_batching_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOptimizationPolicy::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOptimizationPolicy) uint32_t cached_has_bits = 0; (void) cached_has_bits; // .inference.ModelOptimizationPolicy.Graph graph = 1; if (this->_internal_has_graph()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(1, _Internal::graph(this), _Internal::graph(this).GetCachedSize(), target, stream); } // .inference.ModelOptimizationPolicy.ModelPriority priority = 2; if (this->_internal_priority() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 2, this->_internal_priority(), target); } // .inference.ModelOptimizationPolicy.Cuda cuda = 3; if (this->_internal_has_cuda()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(3, _Internal::cuda(this), _Internal::cuda(this).GetCachedSize(), target, stream); } // .inference.ModelOptimizationPolicy.ExecutionAccelerators execution_accelerators = 4; if (this->_internal_has_execution_accelerators()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(4, _Internal::execution_accelerators(this), _Internal::execution_accelerators(this).GetCachedSize(), target, stream); } // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer input_pinned_memory = 5; if (this->_internal_has_input_pinned_memory()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(5, _Internal::input_pinned_memory(this), _Internal::input_pinned_memory(this).GetCachedSize(), target, stream); } // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer output_pinned_memory = 6; if (this->_internal_has_output_pinned_memory()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(6, _Internal::output_pinned_memory(this), _Internal::output_pinned_memory(this).GetCachedSize(), target, stream); } // uint32 gather_kernel_buffer_threshold = 7; if (this->_internal_gather_kernel_buffer_threshold() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt32ToArray(7, this->_internal_gather_kernel_buffer_threshold(), target); } // bool eager_batching = 8; if (this->_internal_eager_batching() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(8, this->_internal_eager_batching(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOptimizationPolicy) return target; } size_t ModelOptimizationPolicy::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOptimizationPolicy) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // .inference.ModelOptimizationPolicy.Graph graph = 1; if (this->_internal_has_graph()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.graph_); } // .inference.ModelOptimizationPolicy.Cuda cuda = 3; if (this->_internal_has_cuda()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.cuda_); } // .inference.ModelOptimizationPolicy.ExecutionAccelerators execution_accelerators = 4; if (this->_internal_has_execution_accelerators()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.execution_accelerators_); } // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer input_pinned_memory = 5; if (this->_internal_has_input_pinned_memory()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.input_pinned_memory_); } // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer output_pinned_memory = 6; if (this->_internal_has_output_pinned_memory()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.output_pinned_memory_); } // .inference.ModelOptimizationPolicy.ModelPriority priority = 2; if (this->_internal_priority() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_priority()); } // uint32 gather_kernel_buffer_threshold = 7; if (this->_internal_gather_kernel_buffer_threshold() != 0) { total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne(this->_internal_gather_kernel_buffer_threshold()); } // bool eager_batching = 8; if (this->_internal_eager_batching() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOptimizationPolicy::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOptimizationPolicy::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOptimizationPolicy::GetClassData() const { return &_class_data_; } void ModelOptimizationPolicy::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOptimizationPolicy) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_has_graph()) { _this->_internal_mutable_graph()->::inference::ModelOptimizationPolicy_Graph::MergeFrom( from._internal_graph()); } if (from._internal_has_cuda()) { _this->_internal_mutable_cuda()->::inference::ModelOptimizationPolicy_Cuda::MergeFrom( from._internal_cuda()); } if (from._internal_has_execution_accelerators()) { _this->_internal_mutable_execution_accelerators()->::inference::ModelOptimizationPolicy_ExecutionAccelerators::MergeFrom( from._internal_execution_accelerators()); } if (from._internal_has_input_pinned_memory()) { _this->_internal_mutable_input_pinned_memory()->::inference::ModelOptimizationPolicy_PinnedMemoryBuffer::MergeFrom( from._internal_input_pinned_memory()); } if (from._internal_has_output_pinned_memory()) { _this->_internal_mutable_output_pinned_memory()->::inference::ModelOptimizationPolicy_PinnedMemoryBuffer::MergeFrom( from._internal_output_pinned_memory()); } if (from._internal_priority() != 0) { _this->_internal_set_priority(from._internal_priority()); } if (from._internal_gather_kernel_buffer_threshold() != 0) { _this->_internal_set_gather_kernel_buffer_threshold(from._internal_gather_kernel_buffer_threshold()); } if (from._internal_eager_batching() != 0) { _this->_internal_set_eager_batching(from._internal_eager_batching()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOptimizationPolicy::CopyFrom(const ModelOptimizationPolicy& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOptimizationPolicy) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOptimizationPolicy::IsInitialized() const { return true; } void ModelOptimizationPolicy::InternalSwap(ModelOptimizationPolicy* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelOptimizationPolicy, _impl_.eager_batching_) + sizeof(ModelOptimizationPolicy::_impl_.eager_batching_) - PROTOBUF_FIELD_OFFSET(ModelOptimizationPolicy, _impl_.graph_)>( reinterpret_cast(&_impl_.graph_), reinterpret_cast(&other->_impl_.graph_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOptimizationPolicy::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[24]); } // =================================================================== class ModelQueuePolicy::_Internal { public: }; ModelQueuePolicy::ModelQueuePolicy(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelQueuePolicy) } ModelQueuePolicy::ModelQueuePolicy(const ModelQueuePolicy& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelQueuePolicy* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.default_timeout_microseconds_){} , decltype(_impl_.timeout_action_){} , decltype(_impl_.allow_timeout_override_){} , decltype(_impl_.max_queue_size_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); ::memcpy(&_impl_.default_timeout_microseconds_, &from._impl_.default_timeout_microseconds_, static_cast(reinterpret_cast(&_impl_.max_queue_size_) - reinterpret_cast(&_impl_.default_timeout_microseconds_)) + sizeof(_impl_.max_queue_size_)); // @@protoc_insertion_point(copy_constructor:inference.ModelQueuePolicy) } inline void ModelQueuePolicy::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.default_timeout_microseconds_){uint64_t{0u}} , decltype(_impl_.timeout_action_){0} , decltype(_impl_.allow_timeout_override_){false} , decltype(_impl_.max_queue_size_){0u} , /*decltype(_impl_._cached_size_)*/{} }; } ModelQueuePolicy::~ModelQueuePolicy() { // @@protoc_insertion_point(destructor:inference.ModelQueuePolicy) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelQueuePolicy::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); } void ModelQueuePolicy::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelQueuePolicy::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelQueuePolicy) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; ::memset(&_impl_.default_timeout_microseconds_, 0, static_cast( reinterpret_cast(&_impl_.max_queue_size_) - reinterpret_cast(&_impl_.default_timeout_microseconds_)) + sizeof(_impl_.max_queue_size_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelQueuePolicy::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // .inference.ModelQueuePolicy.TimeoutAction timeout_action = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_timeout_action(static_cast<::inference::ModelQueuePolicy_TimeoutAction>(val)); } else goto handle_unusual; continue; // uint64 default_timeout_microseconds = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.default_timeout_microseconds_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool allow_timeout_override = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { _impl_.allow_timeout_override_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // uint32 max_queue_size = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { _impl_.max_queue_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelQueuePolicy::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelQueuePolicy) uint32_t cached_has_bits = 0; (void) cached_has_bits; // .inference.ModelQueuePolicy.TimeoutAction timeout_action = 1; if (this->_internal_timeout_action() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 1, this->_internal_timeout_action(), target); } // uint64 default_timeout_microseconds = 2; if (this->_internal_default_timeout_microseconds() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt64ToArray(2, this->_internal_default_timeout_microseconds(), target); } // bool allow_timeout_override = 3; if (this->_internal_allow_timeout_override() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(3, this->_internal_allow_timeout_override(), target); } // uint32 max_queue_size = 4; if (this->_internal_max_queue_size() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt32ToArray(4, this->_internal_max_queue_size(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelQueuePolicy) return target; } size_t ModelQueuePolicy::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelQueuePolicy) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // uint64 default_timeout_microseconds = 2; if (this->_internal_default_timeout_microseconds() != 0) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_default_timeout_microseconds()); } // .inference.ModelQueuePolicy.TimeoutAction timeout_action = 1; if (this->_internal_timeout_action() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_timeout_action()); } // bool allow_timeout_override = 3; if (this->_internal_allow_timeout_override() != 0) { total_size += 1 + 1; } // uint32 max_queue_size = 4; if (this->_internal_max_queue_size() != 0) { total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne(this->_internal_max_queue_size()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelQueuePolicy::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelQueuePolicy::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelQueuePolicy::GetClassData() const { return &_class_data_; } void ModelQueuePolicy::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelQueuePolicy) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_default_timeout_microseconds() != 0) { _this->_internal_set_default_timeout_microseconds(from._internal_default_timeout_microseconds()); } if (from._internal_timeout_action() != 0) { _this->_internal_set_timeout_action(from._internal_timeout_action()); } if (from._internal_allow_timeout_override() != 0) { _this->_internal_set_allow_timeout_override(from._internal_allow_timeout_override()); } if (from._internal_max_queue_size() != 0) { _this->_internal_set_max_queue_size(from._internal_max_queue_size()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelQueuePolicy::CopyFrom(const ModelQueuePolicy& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelQueuePolicy) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelQueuePolicy::IsInitialized() const { return true; } void ModelQueuePolicy::InternalSwap(ModelQueuePolicy* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelQueuePolicy, _impl_.max_queue_size_) + sizeof(ModelQueuePolicy::_impl_.max_queue_size_) - PROTOBUF_FIELD_OFFSET(ModelQueuePolicy, _impl_.default_timeout_microseconds_)>( reinterpret_cast(&_impl_.default_timeout_microseconds_), reinterpret_cast(&other->_impl_.default_timeout_microseconds_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelQueuePolicy::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[25]); } // =================================================================== ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse() {} ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse::MergeFrom(const ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[26]); } // =================================================================== class ModelDynamicBatching::_Internal { public: static const ::inference::ModelQueuePolicy& default_queue_policy(const ModelDynamicBatching* msg); }; const ::inference::ModelQueuePolicy& ModelDynamicBatching::_Internal::default_queue_policy(const ModelDynamicBatching* msg) { return *msg->_impl_.default_queue_policy_; } ModelDynamicBatching::ModelDynamicBatching(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); if (arena != nullptr && !is_message_owned) { arena->OwnCustomDestructor(this, &ModelDynamicBatching::ArenaDtor); } // @@protoc_insertion_point(arena_constructor:inference.ModelDynamicBatching) } ModelDynamicBatching::ModelDynamicBatching(const ModelDynamicBatching& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelDynamicBatching* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.preferred_batch_size_){from._impl_.preferred_batch_size_} , /*decltype(_impl_._preferred_batch_size_cached_byte_size_)*/{0} , /*decltype(_impl_.priority_queue_policy_)*/{} , decltype(_impl_.default_queue_policy_){nullptr} , decltype(_impl_.max_queue_delay_microseconds_){} , decltype(_impl_.priority_levels_){} , decltype(_impl_.default_priority_level_){} , decltype(_impl_.preserve_ordering_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.priority_queue_policy_.MergeFrom(from._impl_.priority_queue_policy_); if (from._internal_has_default_queue_policy()) { _this->_impl_.default_queue_policy_ = new ::inference::ModelQueuePolicy(*from._impl_.default_queue_policy_); } ::memcpy(&_impl_.max_queue_delay_microseconds_, &from._impl_.max_queue_delay_microseconds_, static_cast(reinterpret_cast(&_impl_.preserve_ordering_) - reinterpret_cast(&_impl_.max_queue_delay_microseconds_)) + sizeof(_impl_.preserve_ordering_)); // @@protoc_insertion_point(copy_constructor:inference.ModelDynamicBatching) } inline void ModelDynamicBatching::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.preferred_batch_size_){arena} , /*decltype(_impl_._preferred_batch_size_cached_byte_size_)*/{0} , /*decltype(_impl_.priority_queue_policy_)*/{::_pbi::ArenaInitialized(), arena} , decltype(_impl_.default_queue_policy_){nullptr} , decltype(_impl_.max_queue_delay_microseconds_){uint64_t{0u}} , decltype(_impl_.priority_levels_){uint64_t{0u}} , decltype(_impl_.default_priority_level_){uint64_t{0u}} , decltype(_impl_.preserve_ordering_){false} , /*decltype(_impl_._cached_size_)*/{} }; } ModelDynamicBatching::~ModelDynamicBatching() { // @@protoc_insertion_point(destructor:inference.ModelDynamicBatching) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; ArenaDtor(this); return; } SharedDtor(); } inline void ModelDynamicBatching::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.preferred_batch_size_.~RepeatedField(); _impl_.priority_queue_policy_.Destruct(); _impl_.priority_queue_policy_.~MapField(); if (this != internal_default_instance()) delete _impl_.default_queue_policy_; } void ModelDynamicBatching::ArenaDtor(void* object) { ModelDynamicBatching* _this = reinterpret_cast< ModelDynamicBatching* >(object); _this->_impl_.priority_queue_policy_.Destruct(); } void ModelDynamicBatching::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelDynamicBatching::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelDynamicBatching) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.preferred_batch_size_.Clear(); _impl_.priority_queue_policy_.Clear(); if (GetArenaForAllocation() == nullptr && _impl_.default_queue_policy_ != nullptr) { delete _impl_.default_queue_policy_; } _impl_.default_queue_policy_ = nullptr; ::memset(&_impl_.max_queue_delay_microseconds_, 0, static_cast( reinterpret_cast(&_impl_.preserve_ordering_) - reinterpret_cast(&_impl_.max_queue_delay_microseconds_)) + sizeof(_impl_.preserve_ordering_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelDynamicBatching::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated int32 preferred_batch_size = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt32Parser(_internal_mutable_preferred_batch_size(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 8) { _internal_add_preferred_batch_size(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // uint64 max_queue_delay_microseconds = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.max_queue_delay_microseconds_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool preserve_ordering = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { _impl_.preserve_ordering_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // uint64 priority_levels = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { _impl_.priority_levels_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // uint64 default_priority_level = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 40)) { _impl_.default_priority_level_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelQueuePolicy default_queue_policy = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 50)) { ptr = ctx->ParseMessage(_internal_mutable_default_queue_policy(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // map priority_queue_policy = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 58)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.priority_queue_policy_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<58>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelDynamicBatching::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelDynamicBatching) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated int32 preferred_batch_size = 1; { int byte_size = _impl_._preferred_batch_size_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt32Packed( 1, _internal_preferred_batch_size(), byte_size, target); } } // uint64 max_queue_delay_microseconds = 2; if (this->_internal_max_queue_delay_microseconds() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt64ToArray(2, this->_internal_max_queue_delay_microseconds(), target); } // bool preserve_ordering = 3; if (this->_internal_preserve_ordering() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(3, this->_internal_preserve_ordering(), target); } // uint64 priority_levels = 4; if (this->_internal_priority_levels() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt64ToArray(4, this->_internal_priority_levels(), target); } // uint64 default_priority_level = 5; if (this->_internal_default_priority_level() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt64ToArray(5, this->_internal_default_priority_level(), target); } // .inference.ModelQueuePolicy default_queue_policy = 6; if (this->_internal_has_default_queue_policy()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(6, _Internal::default_queue_policy(this), _Internal::default_queue_policy(this).GetCachedSize(), target, stream); } // map priority_queue_policy = 7; if (!this->_internal_priority_queue_policy().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_priority_queue_policy(); if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterFlat(map_field)) { target = WireHelper::InternalSerialize(7, entry.first, entry.second, target, stream); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(7, entry.first, entry.second, target, stream); } } } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelDynamicBatching) return target; } size_t ModelDynamicBatching::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelDynamicBatching) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int32 preferred_batch_size = 1; { size_t data_size = ::_pbi::WireFormatLite:: Int32Size(this->_impl_.preferred_batch_size_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._preferred_batch_size_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // map priority_queue_policy = 7; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_priority_queue_policy_size()); for (::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >::const_iterator it = this->_internal_priority_queue_policy().begin(); it != this->_internal_priority_queue_policy().end(); ++it) { total_size += ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // .inference.ModelQueuePolicy default_queue_policy = 6; if (this->_internal_has_default_queue_policy()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.default_queue_policy_); } // uint64 max_queue_delay_microseconds = 2; if (this->_internal_max_queue_delay_microseconds() != 0) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_max_queue_delay_microseconds()); } // uint64 priority_levels = 4; if (this->_internal_priority_levels() != 0) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_priority_levels()); } // uint64 default_priority_level = 5; if (this->_internal_default_priority_level() != 0) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_default_priority_level()); } // bool preserve_ordering = 3; if (this->_internal_preserve_ordering() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelDynamicBatching::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelDynamicBatching::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelDynamicBatching::GetClassData() const { return &_class_data_; } void ModelDynamicBatching::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelDynamicBatching) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.preferred_batch_size_.MergeFrom(from._impl_.preferred_batch_size_); _this->_impl_.priority_queue_policy_.MergeFrom(from._impl_.priority_queue_policy_); if (from._internal_has_default_queue_policy()) { _this->_internal_mutable_default_queue_policy()->::inference::ModelQueuePolicy::MergeFrom( from._internal_default_queue_policy()); } if (from._internal_max_queue_delay_microseconds() != 0) { _this->_internal_set_max_queue_delay_microseconds(from._internal_max_queue_delay_microseconds()); } if (from._internal_priority_levels() != 0) { _this->_internal_set_priority_levels(from._internal_priority_levels()); } if (from._internal_default_priority_level() != 0) { _this->_internal_set_default_priority_level(from._internal_default_priority_level()); } if (from._internal_preserve_ordering() != 0) { _this->_internal_set_preserve_ordering(from._internal_preserve_ordering()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelDynamicBatching::CopyFrom(const ModelDynamicBatching& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelDynamicBatching) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelDynamicBatching::IsInitialized() const { return true; } void ModelDynamicBatching::InternalSwap(ModelDynamicBatching* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.preferred_batch_size_.InternalSwap(&other->_impl_.preferred_batch_size_); _impl_.priority_queue_policy_.InternalSwap(&other->_impl_.priority_queue_policy_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelDynamicBatching, _impl_.preserve_ordering_) + sizeof(ModelDynamicBatching::_impl_.preserve_ordering_) - PROTOBUF_FIELD_OFFSET(ModelDynamicBatching, _impl_.default_queue_policy_)>( reinterpret_cast(&_impl_.default_queue_policy_), reinterpret_cast(&other->_impl_.default_queue_policy_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelDynamicBatching::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[27]); } // =================================================================== class ModelSequenceBatching_Control::_Internal { public: }; ModelSequenceBatching_Control::ModelSequenceBatching_Control(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelSequenceBatching.Control) } ModelSequenceBatching_Control::ModelSequenceBatching_Control(const ModelSequenceBatching_Control& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelSequenceBatching_Control* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.int32_false_true_){from._impl_.int32_false_true_} , /*decltype(_impl_._int32_false_true_cached_byte_size_)*/{0} , decltype(_impl_.fp32_false_true_){from._impl_.fp32_false_true_} , decltype(_impl_.bool_false_true_){from._impl_.bool_false_true_} , decltype(_impl_.kind_){} , decltype(_impl_.data_type_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); ::memcpy(&_impl_.kind_, &from._impl_.kind_, static_cast(reinterpret_cast(&_impl_.data_type_) - reinterpret_cast(&_impl_.kind_)) + sizeof(_impl_.data_type_)); // @@protoc_insertion_point(copy_constructor:inference.ModelSequenceBatching.Control) } inline void ModelSequenceBatching_Control::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.int32_false_true_){arena} , /*decltype(_impl_._int32_false_true_cached_byte_size_)*/{0} , decltype(_impl_.fp32_false_true_){arena} , decltype(_impl_.bool_false_true_){arena} , decltype(_impl_.kind_){0} , decltype(_impl_.data_type_){0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelSequenceBatching_Control::~ModelSequenceBatching_Control() { // @@protoc_insertion_point(destructor:inference.ModelSequenceBatching.Control) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelSequenceBatching_Control::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.int32_false_true_.~RepeatedField(); _impl_.fp32_false_true_.~RepeatedField(); _impl_.bool_false_true_.~RepeatedField(); } void ModelSequenceBatching_Control::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelSequenceBatching_Control::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelSequenceBatching.Control) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.int32_false_true_.Clear(); _impl_.fp32_false_true_.Clear(); _impl_.bool_false_true_.Clear(); ::memset(&_impl_.kind_, 0, static_cast( reinterpret_cast(&_impl_.data_type_) - reinterpret_cast(&_impl_.kind_)) + sizeof(_impl_.data_type_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelSequenceBatching_Control::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // .inference.ModelSequenceBatching.Control.Kind kind = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_kind(static_cast<::inference::ModelSequenceBatching_Control_Kind>(val)); } else goto handle_unusual; continue; // repeated int32 int32_false_true = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt32Parser(_internal_mutable_int32_false_true(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 16) { _internal_add_int32_false_true(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // repeated float fp32_false_true = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedFloatParser(_internal_mutable_fp32_false_true(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 29) { _internal_add_fp32_false_true(::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr)); ptr += sizeof(float); } else goto handle_unusual; continue; // .inference.DataType data_type = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_data_type(static_cast<::inference::DataType>(val)); } else goto handle_unusual; continue; // repeated bool bool_false_true = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedBoolParser(_internal_mutable_bool_false_true(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 40) { _internal_add_bool_false_true(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelSequenceBatching_Control::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelSequenceBatching.Control) uint32_t cached_has_bits = 0; (void) cached_has_bits; // .inference.ModelSequenceBatching.Control.Kind kind = 1; if (this->_internal_kind() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 1, this->_internal_kind(), target); } // repeated int32 int32_false_true = 2; { int byte_size = _impl_._int32_false_true_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt32Packed( 2, _internal_int32_false_true(), byte_size, target); } } // repeated float fp32_false_true = 3; if (this->_internal_fp32_false_true_size() > 0) { target = stream->WriteFixedPacked(3, _internal_fp32_false_true(), target); } // .inference.DataType data_type = 4; if (this->_internal_data_type() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 4, this->_internal_data_type(), target); } // repeated bool bool_false_true = 5; if (this->_internal_bool_false_true_size() > 0) { target = stream->WriteFixedPacked(5, _internal_bool_false_true(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelSequenceBatching.Control) return target; } size_t ModelSequenceBatching_Control::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelSequenceBatching.Control) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int32 int32_false_true = 2; { size_t data_size = ::_pbi::WireFormatLite:: Int32Size(this->_impl_.int32_false_true_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._int32_false_true_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // repeated float fp32_false_true = 3; { unsigned int count = static_cast(this->_internal_fp32_false_true_size()); size_t data_size = 4UL * count; if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } total_size += data_size; } // repeated bool bool_false_true = 5; { unsigned int count = static_cast(this->_internal_bool_false_true_size()); size_t data_size = 1UL * count; if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } total_size += data_size; } // .inference.ModelSequenceBatching.Control.Kind kind = 1; if (this->_internal_kind() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_kind()); } // .inference.DataType data_type = 4; if (this->_internal_data_type() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_data_type()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelSequenceBatching_Control::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelSequenceBatching_Control::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelSequenceBatching_Control::GetClassData() const { return &_class_data_; } void ModelSequenceBatching_Control::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelSequenceBatching.Control) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.int32_false_true_.MergeFrom(from._impl_.int32_false_true_); _this->_impl_.fp32_false_true_.MergeFrom(from._impl_.fp32_false_true_); _this->_impl_.bool_false_true_.MergeFrom(from._impl_.bool_false_true_); if (from._internal_kind() != 0) { _this->_internal_set_kind(from._internal_kind()); } if (from._internal_data_type() != 0) { _this->_internal_set_data_type(from._internal_data_type()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelSequenceBatching_Control::CopyFrom(const ModelSequenceBatching_Control& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelSequenceBatching.Control) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelSequenceBatching_Control::IsInitialized() const { return true; } void ModelSequenceBatching_Control::InternalSwap(ModelSequenceBatching_Control* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.int32_false_true_.InternalSwap(&other->_impl_.int32_false_true_); _impl_.fp32_false_true_.InternalSwap(&other->_impl_.fp32_false_true_); _impl_.bool_false_true_.InternalSwap(&other->_impl_.bool_false_true_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelSequenceBatching_Control, _impl_.data_type_) + sizeof(ModelSequenceBatching_Control::_impl_.data_type_) - PROTOBUF_FIELD_OFFSET(ModelSequenceBatching_Control, _impl_.kind_)>( reinterpret_cast(&_impl_.kind_), reinterpret_cast(&other->_impl_.kind_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelSequenceBatching_Control::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[28]); } // =================================================================== class ModelSequenceBatching_ControlInput::_Internal { public: }; ModelSequenceBatching_ControlInput::ModelSequenceBatching_ControlInput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelSequenceBatching.ControlInput) } ModelSequenceBatching_ControlInput::ModelSequenceBatching_ControlInput(const ModelSequenceBatching_ControlInput& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelSequenceBatching_ControlInput* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.control_){from._impl_.control_} , decltype(_impl_.name_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } // @@protoc_insertion_point(copy_constructor:inference.ModelSequenceBatching.ControlInput) } inline void ModelSequenceBatching_ControlInput::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.control_){arena} , decltype(_impl_.name_){} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelSequenceBatching_ControlInput::~ModelSequenceBatching_ControlInput() { // @@protoc_insertion_point(destructor:inference.ModelSequenceBatching.ControlInput) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelSequenceBatching_ControlInput::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.control_.~RepeatedPtrField(); _impl_.name_.Destroy(); } void ModelSequenceBatching_ControlInput::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelSequenceBatching_ControlInput::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelSequenceBatching.ControlInput) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.control_.Clear(); _impl_.name_.ClearToEmpty(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelSequenceBatching_ControlInput::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelSequenceBatching.ControlInput.name")); } else goto handle_unusual; continue; // repeated .inference.ModelSequenceBatching.Control control = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_control(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelSequenceBatching_ControlInput::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelSequenceBatching.ControlInput) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelSequenceBatching.ControlInput.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // repeated .inference.ModelSequenceBatching.Control control = 2; for (unsigned i = 0, n = static_cast(this->_internal_control_size()); i < n; i++) { const auto& repfield = this->_internal_control(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(2, repfield, repfield.GetCachedSize(), target, stream); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelSequenceBatching.ControlInput) return target; } size_t ModelSequenceBatching_ControlInput::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelSequenceBatching.ControlInput) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated .inference.ModelSequenceBatching.Control control = 2; total_size += 1UL * this->_internal_control_size(); for (const auto& msg : this->_impl_.control_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelSequenceBatching_ControlInput::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelSequenceBatching_ControlInput::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelSequenceBatching_ControlInput::GetClassData() const { return &_class_data_; } void ModelSequenceBatching_ControlInput::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelSequenceBatching.ControlInput) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.control_.MergeFrom(from._impl_.control_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelSequenceBatching_ControlInput::CopyFrom(const ModelSequenceBatching_ControlInput& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelSequenceBatching.ControlInput) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelSequenceBatching_ControlInput::IsInitialized() const { return true; } void ModelSequenceBatching_ControlInput::InternalSwap(ModelSequenceBatching_ControlInput* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.control_.InternalSwap(&other->_impl_.control_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelSequenceBatching_ControlInput::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[29]); } // =================================================================== class ModelSequenceBatching_InitialState::_Internal { public: }; ModelSequenceBatching_InitialState::ModelSequenceBatching_InitialState(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelSequenceBatching.InitialState) } ModelSequenceBatching_InitialState::ModelSequenceBatching_InitialState(const ModelSequenceBatching_InitialState& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelSequenceBatching_InitialState* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.dims_){from._impl_.dims_} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.name_){} , decltype(_impl_.data_type_){} , decltype(_impl_.state_data_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } _this->_impl_.data_type_ = from._impl_.data_type_; clear_has_state_data(); switch (from.state_data_case()) { case kZeroData: { _this->_internal_set_zero_data(from._internal_zero_data()); break; } case kDataFile: { _this->_internal_set_data_file(from._internal_data_file()); break; } case STATE_DATA_NOT_SET: { break; } } // @@protoc_insertion_point(copy_constructor:inference.ModelSequenceBatching.InitialState) } inline void ModelSequenceBatching_InitialState::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.dims_){arena} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.name_){} , decltype(_impl_.data_type_){0} , decltype(_impl_.state_data_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING clear_has_state_data(); } ModelSequenceBatching_InitialState::~ModelSequenceBatching_InitialState() { // @@protoc_insertion_point(destructor:inference.ModelSequenceBatching.InitialState) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelSequenceBatching_InitialState::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.dims_.~RepeatedField(); _impl_.name_.Destroy(); if (has_state_data()) { clear_state_data(); } } void ModelSequenceBatching_InitialState::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelSequenceBatching_InitialState::clear_state_data() { // @@protoc_insertion_point(one_of_clear_start:inference.ModelSequenceBatching.InitialState) switch (state_data_case()) { case kZeroData: { // No need to clear break; } case kDataFile: { _impl_.state_data_.data_file_.Destroy(); break; } case STATE_DATA_NOT_SET: { break; } } _impl_._oneof_case_[0] = STATE_DATA_NOT_SET; } void ModelSequenceBatching_InitialState::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelSequenceBatching.InitialState) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.dims_.Clear(); _impl_.name_.ClearToEmpty(); _impl_.data_type_ = 0; clear_state_data(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelSequenceBatching_InitialState::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // .inference.DataType data_type = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_data_type(static_cast<::inference::DataType>(val)); } else goto handle_unusual; continue; // repeated int64 dims = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(_internal_mutable_dims(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 16) { _internal_add_dims(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // bool zero_data = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { _internal_set_zero_data(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // string data_file = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 34)) { auto str = _internal_mutable_data_file(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelSequenceBatching.InitialState.data_file")); } else goto handle_unusual; continue; // string name = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelSequenceBatching.InitialState.name")); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelSequenceBatching_InitialState::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelSequenceBatching.InitialState) uint32_t cached_has_bits = 0; (void) cached_has_bits; // .inference.DataType data_type = 1; if (this->_internal_data_type() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 1, this->_internal_data_type(), target); } // repeated int64 dims = 2; { int byte_size = _impl_._dims_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt64Packed( 2, _internal_dims(), byte_size, target); } } // bool zero_data = 3; if (_internal_has_zero_data()) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(3, this->_internal_zero_data(), target); } // string data_file = 4; if (_internal_has_data_file()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_data_file().data(), static_cast(this->_internal_data_file().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelSequenceBatching.InitialState.data_file"); target = stream->WriteStringMaybeAliased( 4, this->_internal_data_file(), target); } // string name = 5; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelSequenceBatching.InitialState.name"); target = stream->WriteStringMaybeAliased( 5, this->_internal_name(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelSequenceBatching.InitialState) return target; } size_t ModelSequenceBatching_InitialState::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelSequenceBatching.InitialState) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int64 dims = 2; { size_t data_size = ::_pbi::WireFormatLite:: Int64Size(this->_impl_.dims_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._dims_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // string name = 5; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } // .inference.DataType data_type = 1; if (this->_internal_data_type() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_data_type()); } switch (state_data_case()) { // bool zero_data = 3; case kZeroData: { total_size += 1 + 1; break; } // string data_file = 4; case kDataFile: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_data_file()); break; } case STATE_DATA_NOT_SET: { break; } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelSequenceBatching_InitialState::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelSequenceBatching_InitialState::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelSequenceBatching_InitialState::GetClassData() const { return &_class_data_; } void ModelSequenceBatching_InitialState::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelSequenceBatching.InitialState) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.dims_.MergeFrom(from._impl_.dims_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } if (from._internal_data_type() != 0) { _this->_internal_set_data_type(from._internal_data_type()); } switch (from.state_data_case()) { case kZeroData: { _this->_internal_set_zero_data(from._internal_zero_data()); break; } case kDataFile: { _this->_internal_set_data_file(from._internal_data_file()); break; } case STATE_DATA_NOT_SET: { break; } } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelSequenceBatching_InitialState::CopyFrom(const ModelSequenceBatching_InitialState& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelSequenceBatching.InitialState) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelSequenceBatching_InitialState::IsInitialized() const { return true; } void ModelSequenceBatching_InitialState::InternalSwap(ModelSequenceBatching_InitialState* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.dims_.InternalSwap(&other->_impl_.dims_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); swap(_impl_.data_type_, other->_impl_.data_type_); swap(_impl_.state_data_, other->_impl_.state_data_); swap(_impl_._oneof_case_[0], other->_impl_._oneof_case_[0]); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelSequenceBatching_InitialState::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[30]); } // =================================================================== class ModelSequenceBatching_State::_Internal { public: }; ModelSequenceBatching_State::ModelSequenceBatching_State(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelSequenceBatching.State) } ModelSequenceBatching_State::ModelSequenceBatching_State(const ModelSequenceBatching_State& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelSequenceBatching_State* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.dims_){from._impl_.dims_} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.initial_state_){from._impl_.initial_state_} , decltype(_impl_.input_name_){} , decltype(_impl_.output_name_){} , decltype(_impl_.data_type_){} , decltype(_impl_.use_same_buffer_for_input_output_){} , decltype(_impl_.use_growable_memory_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.input_name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.input_name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_input_name().empty()) { _this->_impl_.input_name_.Set(from._internal_input_name(), _this->GetArenaForAllocation()); } _impl_.output_name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.output_name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_output_name().empty()) { _this->_impl_.output_name_.Set(from._internal_output_name(), _this->GetArenaForAllocation()); } ::memcpy(&_impl_.data_type_, &from._impl_.data_type_, static_cast(reinterpret_cast(&_impl_.use_growable_memory_) - reinterpret_cast(&_impl_.data_type_)) + sizeof(_impl_.use_growable_memory_)); // @@protoc_insertion_point(copy_constructor:inference.ModelSequenceBatching.State) } inline void ModelSequenceBatching_State::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.dims_){arena} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.initial_state_){arena} , decltype(_impl_.input_name_){} , decltype(_impl_.output_name_){} , decltype(_impl_.data_type_){0} , decltype(_impl_.use_same_buffer_for_input_output_){false} , decltype(_impl_.use_growable_memory_){false} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.input_name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.input_name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.output_name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.output_name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelSequenceBatching_State::~ModelSequenceBatching_State() { // @@protoc_insertion_point(destructor:inference.ModelSequenceBatching.State) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelSequenceBatching_State::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.dims_.~RepeatedField(); _impl_.initial_state_.~RepeatedPtrField(); _impl_.input_name_.Destroy(); _impl_.output_name_.Destroy(); } void ModelSequenceBatching_State::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelSequenceBatching_State::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelSequenceBatching.State) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.dims_.Clear(); _impl_.initial_state_.Clear(); _impl_.input_name_.ClearToEmpty(); _impl_.output_name_.ClearToEmpty(); ::memset(&_impl_.data_type_, 0, static_cast( reinterpret_cast(&_impl_.use_growable_memory_) - reinterpret_cast(&_impl_.data_type_)) + sizeof(_impl_.use_growable_memory_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelSequenceBatching_State::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string input_name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_input_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelSequenceBatching.State.input_name")); } else goto handle_unusual; continue; // string output_name = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { auto str = _internal_mutable_output_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelSequenceBatching.State.output_name")); } else goto handle_unusual; continue; // .inference.DataType data_type = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_data_type(static_cast<::inference::DataType>(val)); } else goto handle_unusual; continue; // repeated int64 dims = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 34)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(_internal_mutable_dims(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 32) { _internal_add_dims(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // repeated .inference.ModelSequenceBatching.InitialState initial_state = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_initial_state(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<42>(ptr)); } else goto handle_unusual; continue; // bool use_same_buffer_for_input_output = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 48)) { _impl_.use_same_buffer_for_input_output_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool use_growable_memory = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 56)) { _impl_.use_growable_memory_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelSequenceBatching_State::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelSequenceBatching.State) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string input_name = 1; if (!this->_internal_input_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_input_name().data(), static_cast(this->_internal_input_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelSequenceBatching.State.input_name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_input_name(), target); } // string output_name = 2; if (!this->_internal_output_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_output_name().data(), static_cast(this->_internal_output_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelSequenceBatching.State.output_name"); target = stream->WriteStringMaybeAliased( 2, this->_internal_output_name(), target); } // .inference.DataType data_type = 3; if (this->_internal_data_type() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 3, this->_internal_data_type(), target); } // repeated int64 dims = 4; { int byte_size = _impl_._dims_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt64Packed( 4, _internal_dims(), byte_size, target); } } // repeated .inference.ModelSequenceBatching.InitialState initial_state = 5; for (unsigned i = 0, n = static_cast(this->_internal_initial_state_size()); i < n; i++) { const auto& repfield = this->_internal_initial_state(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(5, repfield, repfield.GetCachedSize(), target, stream); } // bool use_same_buffer_for_input_output = 6; if (this->_internal_use_same_buffer_for_input_output() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(6, this->_internal_use_same_buffer_for_input_output(), target); } // bool use_growable_memory = 7; if (this->_internal_use_growable_memory() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(7, this->_internal_use_growable_memory(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelSequenceBatching.State) return target; } size_t ModelSequenceBatching_State::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelSequenceBatching.State) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int64 dims = 4; { size_t data_size = ::_pbi::WireFormatLite:: Int64Size(this->_impl_.dims_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._dims_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // repeated .inference.ModelSequenceBatching.InitialState initial_state = 5; total_size += 1UL * this->_internal_initial_state_size(); for (const auto& msg : this->_impl_.initial_state_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // string input_name = 1; if (!this->_internal_input_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_input_name()); } // string output_name = 2; if (!this->_internal_output_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_output_name()); } // .inference.DataType data_type = 3; if (this->_internal_data_type() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_data_type()); } // bool use_same_buffer_for_input_output = 6; if (this->_internal_use_same_buffer_for_input_output() != 0) { total_size += 1 + 1; } // bool use_growable_memory = 7; if (this->_internal_use_growable_memory() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelSequenceBatching_State::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelSequenceBatching_State::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelSequenceBatching_State::GetClassData() const { return &_class_data_; } void ModelSequenceBatching_State::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelSequenceBatching.State) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.dims_.MergeFrom(from._impl_.dims_); _this->_impl_.initial_state_.MergeFrom(from._impl_.initial_state_); if (!from._internal_input_name().empty()) { _this->_internal_set_input_name(from._internal_input_name()); } if (!from._internal_output_name().empty()) { _this->_internal_set_output_name(from._internal_output_name()); } if (from._internal_data_type() != 0) { _this->_internal_set_data_type(from._internal_data_type()); } if (from._internal_use_same_buffer_for_input_output() != 0) { _this->_internal_set_use_same_buffer_for_input_output(from._internal_use_same_buffer_for_input_output()); } if (from._internal_use_growable_memory() != 0) { _this->_internal_set_use_growable_memory(from._internal_use_growable_memory()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelSequenceBatching_State::CopyFrom(const ModelSequenceBatching_State& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelSequenceBatching.State) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelSequenceBatching_State::IsInitialized() const { return true; } void ModelSequenceBatching_State::InternalSwap(ModelSequenceBatching_State* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.dims_.InternalSwap(&other->_impl_.dims_); _impl_.initial_state_.InternalSwap(&other->_impl_.initial_state_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.input_name_, lhs_arena, &other->_impl_.input_name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.output_name_, lhs_arena, &other->_impl_.output_name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelSequenceBatching_State, _impl_.use_growable_memory_) + sizeof(ModelSequenceBatching_State::_impl_.use_growable_memory_) - PROTOBUF_FIELD_OFFSET(ModelSequenceBatching_State, _impl_.data_type_)>( reinterpret_cast(&_impl_.data_type_), reinterpret_cast(&other->_impl_.data_type_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelSequenceBatching_State::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[31]); } // =================================================================== class ModelSequenceBatching_StrategyDirect::_Internal { public: }; ModelSequenceBatching_StrategyDirect::ModelSequenceBatching_StrategyDirect(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelSequenceBatching.StrategyDirect) } ModelSequenceBatching_StrategyDirect::ModelSequenceBatching_StrategyDirect(const ModelSequenceBatching_StrategyDirect& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelSequenceBatching_StrategyDirect* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.max_queue_delay_microseconds_){} , decltype(_impl_.minimum_slot_utilization_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); ::memcpy(&_impl_.max_queue_delay_microseconds_, &from._impl_.max_queue_delay_microseconds_, static_cast(reinterpret_cast(&_impl_.minimum_slot_utilization_) - reinterpret_cast(&_impl_.max_queue_delay_microseconds_)) + sizeof(_impl_.minimum_slot_utilization_)); // @@protoc_insertion_point(copy_constructor:inference.ModelSequenceBatching.StrategyDirect) } inline void ModelSequenceBatching_StrategyDirect::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.max_queue_delay_microseconds_){uint64_t{0u}} , decltype(_impl_.minimum_slot_utilization_){0} , /*decltype(_impl_._cached_size_)*/{} }; } ModelSequenceBatching_StrategyDirect::~ModelSequenceBatching_StrategyDirect() { // @@protoc_insertion_point(destructor:inference.ModelSequenceBatching.StrategyDirect) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelSequenceBatching_StrategyDirect::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); } void ModelSequenceBatching_StrategyDirect::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelSequenceBatching_StrategyDirect::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelSequenceBatching.StrategyDirect) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; ::memset(&_impl_.max_queue_delay_microseconds_, 0, static_cast( reinterpret_cast(&_impl_.minimum_slot_utilization_) - reinterpret_cast(&_impl_.max_queue_delay_microseconds_)) + sizeof(_impl_.minimum_slot_utilization_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelSequenceBatching_StrategyDirect::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // uint64 max_queue_delay_microseconds = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.max_queue_delay_microseconds_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // float minimum_slot_utilization = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 21)) { _impl_.minimum_slot_utilization_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); ptr += sizeof(float); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelSequenceBatching_StrategyDirect::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelSequenceBatching.StrategyDirect) uint32_t cached_has_bits = 0; (void) cached_has_bits; // uint64 max_queue_delay_microseconds = 1; if (this->_internal_max_queue_delay_microseconds() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt64ToArray(1, this->_internal_max_queue_delay_microseconds(), target); } // float minimum_slot_utilization = 2; static_assert(sizeof(uint32_t) == sizeof(float), "Code assumes uint32_t and float are the same size."); float tmp_minimum_slot_utilization = this->_internal_minimum_slot_utilization(); uint32_t raw_minimum_slot_utilization; memcpy(&raw_minimum_slot_utilization, &tmp_minimum_slot_utilization, sizeof(tmp_minimum_slot_utilization)); if (raw_minimum_slot_utilization != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteFloatToArray(2, this->_internal_minimum_slot_utilization(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelSequenceBatching.StrategyDirect) return target; } size_t ModelSequenceBatching_StrategyDirect::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelSequenceBatching.StrategyDirect) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // uint64 max_queue_delay_microseconds = 1; if (this->_internal_max_queue_delay_microseconds() != 0) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_max_queue_delay_microseconds()); } // float minimum_slot_utilization = 2; static_assert(sizeof(uint32_t) == sizeof(float), "Code assumes uint32_t and float are the same size."); float tmp_minimum_slot_utilization = this->_internal_minimum_slot_utilization(); uint32_t raw_minimum_slot_utilization; memcpy(&raw_minimum_slot_utilization, &tmp_minimum_slot_utilization, sizeof(tmp_minimum_slot_utilization)); if (raw_minimum_slot_utilization != 0) { total_size += 1 + 4; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelSequenceBatching_StrategyDirect::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelSequenceBatching_StrategyDirect::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelSequenceBatching_StrategyDirect::GetClassData() const { return &_class_data_; } void ModelSequenceBatching_StrategyDirect::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelSequenceBatching.StrategyDirect) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_max_queue_delay_microseconds() != 0) { _this->_internal_set_max_queue_delay_microseconds(from._internal_max_queue_delay_microseconds()); } static_assert(sizeof(uint32_t) == sizeof(float), "Code assumes uint32_t and float are the same size."); float tmp_minimum_slot_utilization = from._internal_minimum_slot_utilization(); uint32_t raw_minimum_slot_utilization; memcpy(&raw_minimum_slot_utilization, &tmp_minimum_slot_utilization, sizeof(tmp_minimum_slot_utilization)); if (raw_minimum_slot_utilization != 0) { _this->_internal_set_minimum_slot_utilization(from._internal_minimum_slot_utilization()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelSequenceBatching_StrategyDirect::CopyFrom(const ModelSequenceBatching_StrategyDirect& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelSequenceBatching.StrategyDirect) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelSequenceBatching_StrategyDirect::IsInitialized() const { return true; } void ModelSequenceBatching_StrategyDirect::InternalSwap(ModelSequenceBatching_StrategyDirect* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelSequenceBatching_StrategyDirect, _impl_.minimum_slot_utilization_) + sizeof(ModelSequenceBatching_StrategyDirect::_impl_.minimum_slot_utilization_) - PROTOBUF_FIELD_OFFSET(ModelSequenceBatching_StrategyDirect, _impl_.max_queue_delay_microseconds_)>( reinterpret_cast(&_impl_.max_queue_delay_microseconds_), reinterpret_cast(&other->_impl_.max_queue_delay_microseconds_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelSequenceBatching_StrategyDirect::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[32]); } // =================================================================== class ModelSequenceBatching_StrategyOldest::_Internal { public: }; ModelSequenceBatching_StrategyOldest::ModelSequenceBatching_StrategyOldest(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelSequenceBatching.StrategyOldest) } ModelSequenceBatching_StrategyOldest::ModelSequenceBatching_StrategyOldest(const ModelSequenceBatching_StrategyOldest& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelSequenceBatching_StrategyOldest* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.preferred_batch_size_){from._impl_.preferred_batch_size_} , /*decltype(_impl_._preferred_batch_size_cached_byte_size_)*/{0} , decltype(_impl_.max_candidate_sequences_){} , decltype(_impl_.preserve_ordering_){} , decltype(_impl_.max_queue_delay_microseconds_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); ::memcpy(&_impl_.max_candidate_sequences_, &from._impl_.max_candidate_sequences_, static_cast(reinterpret_cast(&_impl_.max_queue_delay_microseconds_) - reinterpret_cast(&_impl_.max_candidate_sequences_)) + sizeof(_impl_.max_queue_delay_microseconds_)); // @@protoc_insertion_point(copy_constructor:inference.ModelSequenceBatching.StrategyOldest) } inline void ModelSequenceBatching_StrategyOldest::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.preferred_batch_size_){arena} , /*decltype(_impl_._preferred_batch_size_cached_byte_size_)*/{0} , decltype(_impl_.max_candidate_sequences_){0} , decltype(_impl_.preserve_ordering_){false} , decltype(_impl_.max_queue_delay_microseconds_){uint64_t{0u}} , /*decltype(_impl_._cached_size_)*/{} }; } ModelSequenceBatching_StrategyOldest::~ModelSequenceBatching_StrategyOldest() { // @@protoc_insertion_point(destructor:inference.ModelSequenceBatching.StrategyOldest) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelSequenceBatching_StrategyOldest::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.preferred_batch_size_.~RepeatedField(); } void ModelSequenceBatching_StrategyOldest::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelSequenceBatching_StrategyOldest::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelSequenceBatching.StrategyOldest) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.preferred_batch_size_.Clear(); ::memset(&_impl_.max_candidate_sequences_, 0, static_cast( reinterpret_cast(&_impl_.max_queue_delay_microseconds_) - reinterpret_cast(&_impl_.max_candidate_sequences_)) + sizeof(_impl_.max_queue_delay_microseconds_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelSequenceBatching_StrategyOldest::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // int32 max_candidate_sequences = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.max_candidate_sequences_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated int32 preferred_batch_size = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt32Parser(_internal_mutable_preferred_batch_size(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 16) { _internal_add_preferred_batch_size(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // uint64 max_queue_delay_microseconds = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { _impl_.max_queue_delay_microseconds_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // bool preserve_ordering = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { _impl_.preserve_ordering_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelSequenceBatching_StrategyOldest::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelSequenceBatching.StrategyOldest) uint32_t cached_has_bits = 0; (void) cached_has_bits; // int32 max_candidate_sequences = 1; if (this->_internal_max_candidate_sequences() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteInt32ToArray(1, this->_internal_max_candidate_sequences(), target); } // repeated int32 preferred_batch_size = 2; { int byte_size = _impl_._preferred_batch_size_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt32Packed( 2, _internal_preferred_batch_size(), byte_size, target); } } // uint64 max_queue_delay_microseconds = 3; if (this->_internal_max_queue_delay_microseconds() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt64ToArray(3, this->_internal_max_queue_delay_microseconds(), target); } // bool preserve_ordering = 4; if (this->_internal_preserve_ordering() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(4, this->_internal_preserve_ordering(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelSequenceBatching.StrategyOldest) return target; } size_t ModelSequenceBatching_StrategyOldest::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelSequenceBatching.StrategyOldest) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int32 preferred_batch_size = 2; { size_t data_size = ::_pbi::WireFormatLite:: Int32Size(this->_impl_.preferred_batch_size_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._preferred_batch_size_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // int32 max_candidate_sequences = 1; if (this->_internal_max_candidate_sequences() != 0) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne(this->_internal_max_candidate_sequences()); } // bool preserve_ordering = 4; if (this->_internal_preserve_ordering() != 0) { total_size += 1 + 1; } // uint64 max_queue_delay_microseconds = 3; if (this->_internal_max_queue_delay_microseconds() != 0) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_max_queue_delay_microseconds()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelSequenceBatching_StrategyOldest::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelSequenceBatching_StrategyOldest::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelSequenceBatching_StrategyOldest::GetClassData() const { return &_class_data_; } void ModelSequenceBatching_StrategyOldest::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelSequenceBatching.StrategyOldest) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.preferred_batch_size_.MergeFrom(from._impl_.preferred_batch_size_); if (from._internal_max_candidate_sequences() != 0) { _this->_internal_set_max_candidate_sequences(from._internal_max_candidate_sequences()); } if (from._internal_preserve_ordering() != 0) { _this->_internal_set_preserve_ordering(from._internal_preserve_ordering()); } if (from._internal_max_queue_delay_microseconds() != 0) { _this->_internal_set_max_queue_delay_microseconds(from._internal_max_queue_delay_microseconds()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelSequenceBatching_StrategyOldest::CopyFrom(const ModelSequenceBatching_StrategyOldest& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelSequenceBatching.StrategyOldest) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelSequenceBatching_StrategyOldest::IsInitialized() const { return true; } void ModelSequenceBatching_StrategyOldest::InternalSwap(ModelSequenceBatching_StrategyOldest* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.preferred_batch_size_.InternalSwap(&other->_impl_.preferred_batch_size_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelSequenceBatching_StrategyOldest, _impl_.max_queue_delay_microseconds_) + sizeof(ModelSequenceBatching_StrategyOldest::_impl_.max_queue_delay_microseconds_) - PROTOBUF_FIELD_OFFSET(ModelSequenceBatching_StrategyOldest, _impl_.max_candidate_sequences_)>( reinterpret_cast(&_impl_.max_candidate_sequences_), reinterpret_cast(&other->_impl_.max_candidate_sequences_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelSequenceBatching_StrategyOldest::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[33]); } // =================================================================== class ModelSequenceBatching::_Internal { public: static const ::inference::ModelSequenceBatching_StrategyDirect& direct(const ModelSequenceBatching* msg); static const ::inference::ModelSequenceBatching_StrategyOldest& oldest(const ModelSequenceBatching* msg); }; const ::inference::ModelSequenceBatching_StrategyDirect& ModelSequenceBatching::_Internal::direct(const ModelSequenceBatching* msg) { return *msg->_impl_.strategy_choice_.direct_; } const ::inference::ModelSequenceBatching_StrategyOldest& ModelSequenceBatching::_Internal::oldest(const ModelSequenceBatching* msg) { return *msg->_impl_.strategy_choice_.oldest_; } void ModelSequenceBatching::set_allocated_direct(::inference::ModelSequenceBatching_StrategyDirect* direct) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); clear_strategy_choice(); if (direct) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(direct); if (message_arena != submessage_arena) { direct = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, direct, submessage_arena); } set_has_direct(); _impl_.strategy_choice_.direct_ = direct; } // @@protoc_insertion_point(field_set_allocated:inference.ModelSequenceBatching.direct) } void ModelSequenceBatching::set_allocated_oldest(::inference::ModelSequenceBatching_StrategyOldest* oldest) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); clear_strategy_choice(); if (oldest) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(oldest); if (message_arena != submessage_arena) { oldest = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, oldest, submessage_arena); } set_has_oldest(); _impl_.strategy_choice_.oldest_ = oldest; } // @@protoc_insertion_point(field_set_allocated:inference.ModelSequenceBatching.oldest) } ModelSequenceBatching::ModelSequenceBatching(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelSequenceBatching) } ModelSequenceBatching::ModelSequenceBatching(const ModelSequenceBatching& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelSequenceBatching* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.control_input_){from._impl_.control_input_} , decltype(_impl_.state_){from._impl_.state_} , decltype(_impl_.max_sequence_idle_microseconds_){} , decltype(_impl_.iterative_sequence_){} , decltype(_impl_.strategy_choice_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); ::memcpy(&_impl_.max_sequence_idle_microseconds_, &from._impl_.max_sequence_idle_microseconds_, static_cast(reinterpret_cast(&_impl_.iterative_sequence_) - reinterpret_cast(&_impl_.max_sequence_idle_microseconds_)) + sizeof(_impl_.iterative_sequence_)); clear_has_strategy_choice(); switch (from.strategy_choice_case()) { case kDirect: { _this->_internal_mutable_direct()->::inference::ModelSequenceBatching_StrategyDirect::MergeFrom( from._internal_direct()); break; } case kOldest: { _this->_internal_mutable_oldest()->::inference::ModelSequenceBatching_StrategyOldest::MergeFrom( from._internal_oldest()); break; } case STRATEGY_CHOICE_NOT_SET: { break; } } // @@protoc_insertion_point(copy_constructor:inference.ModelSequenceBatching) } inline void ModelSequenceBatching::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.control_input_){arena} , decltype(_impl_.state_){arena} , decltype(_impl_.max_sequence_idle_microseconds_){uint64_t{0u}} , decltype(_impl_.iterative_sequence_){false} , decltype(_impl_.strategy_choice_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{} }; clear_has_strategy_choice(); } ModelSequenceBatching::~ModelSequenceBatching() { // @@protoc_insertion_point(destructor:inference.ModelSequenceBatching) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelSequenceBatching::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.control_input_.~RepeatedPtrField(); _impl_.state_.~RepeatedPtrField(); if (has_strategy_choice()) { clear_strategy_choice(); } } void ModelSequenceBatching::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelSequenceBatching::clear_strategy_choice() { // @@protoc_insertion_point(one_of_clear_start:inference.ModelSequenceBatching) switch (strategy_choice_case()) { case kDirect: { if (GetArenaForAllocation() == nullptr) { delete _impl_.strategy_choice_.direct_; } break; } case kOldest: { if (GetArenaForAllocation() == nullptr) { delete _impl_.strategy_choice_.oldest_; } break; } case STRATEGY_CHOICE_NOT_SET: { break; } } _impl_._oneof_case_[0] = STRATEGY_CHOICE_NOT_SET; } void ModelSequenceBatching::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelSequenceBatching) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.control_input_.Clear(); _impl_.state_.Clear(); ::memset(&_impl_.max_sequence_idle_microseconds_, 0, static_cast( reinterpret_cast(&_impl_.iterative_sequence_) - reinterpret_cast(&_impl_.max_sequence_idle_microseconds_)) + sizeof(_impl_.iterative_sequence_)); clear_strategy_choice(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelSequenceBatching::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // uint64 max_sequence_idle_microseconds = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.max_sequence_idle_microseconds_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated .inference.ModelSequenceBatching.ControlInput control_input = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_control_input(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); } else goto handle_unusual; continue; // .inference.ModelSequenceBatching.StrategyDirect direct = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr = ctx->ParseMessage(_internal_mutable_direct(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelSequenceBatching.StrategyOldest oldest = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 34)) { ptr = ctx->ParseMessage(_internal_mutable_oldest(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated .inference.ModelSequenceBatching.State state = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_state(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<42>(ptr)); } else goto handle_unusual; continue; // bool iterative_sequence = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 48)) { _impl_.iterative_sequence_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelSequenceBatching::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelSequenceBatching) uint32_t cached_has_bits = 0; (void) cached_has_bits; // uint64 max_sequence_idle_microseconds = 1; if (this->_internal_max_sequence_idle_microseconds() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt64ToArray(1, this->_internal_max_sequence_idle_microseconds(), target); } // repeated .inference.ModelSequenceBatching.ControlInput control_input = 2; for (unsigned i = 0, n = static_cast(this->_internal_control_input_size()); i < n; i++) { const auto& repfield = this->_internal_control_input(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(2, repfield, repfield.GetCachedSize(), target, stream); } // .inference.ModelSequenceBatching.StrategyDirect direct = 3; if (_internal_has_direct()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(3, _Internal::direct(this), _Internal::direct(this).GetCachedSize(), target, stream); } // .inference.ModelSequenceBatching.StrategyOldest oldest = 4; if (_internal_has_oldest()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(4, _Internal::oldest(this), _Internal::oldest(this).GetCachedSize(), target, stream); } // repeated .inference.ModelSequenceBatching.State state = 5; for (unsigned i = 0, n = static_cast(this->_internal_state_size()); i < n; i++) { const auto& repfield = this->_internal_state(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(5, repfield, repfield.GetCachedSize(), target, stream); } // bool iterative_sequence = 6; if (this->_internal_iterative_sequence() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(6, this->_internal_iterative_sequence(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelSequenceBatching) return target; } size_t ModelSequenceBatching::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelSequenceBatching) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated .inference.ModelSequenceBatching.ControlInput control_input = 2; total_size += 1UL * this->_internal_control_input_size(); for (const auto& msg : this->_impl_.control_input_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // repeated .inference.ModelSequenceBatching.State state = 5; total_size += 1UL * this->_internal_state_size(); for (const auto& msg : this->_impl_.state_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // uint64 max_sequence_idle_microseconds = 1; if (this->_internal_max_sequence_idle_microseconds() != 0) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_max_sequence_idle_microseconds()); } // bool iterative_sequence = 6; if (this->_internal_iterative_sequence() != 0) { total_size += 1 + 1; } switch (strategy_choice_case()) { // .inference.ModelSequenceBatching.StrategyDirect direct = 3; case kDirect: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.strategy_choice_.direct_); break; } // .inference.ModelSequenceBatching.StrategyOldest oldest = 4; case kOldest: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.strategy_choice_.oldest_); break; } case STRATEGY_CHOICE_NOT_SET: { break; } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelSequenceBatching::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelSequenceBatching::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelSequenceBatching::GetClassData() const { return &_class_data_; } void ModelSequenceBatching::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelSequenceBatching) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.control_input_.MergeFrom(from._impl_.control_input_); _this->_impl_.state_.MergeFrom(from._impl_.state_); if (from._internal_max_sequence_idle_microseconds() != 0) { _this->_internal_set_max_sequence_idle_microseconds(from._internal_max_sequence_idle_microseconds()); } if (from._internal_iterative_sequence() != 0) { _this->_internal_set_iterative_sequence(from._internal_iterative_sequence()); } switch (from.strategy_choice_case()) { case kDirect: { _this->_internal_mutable_direct()->::inference::ModelSequenceBatching_StrategyDirect::MergeFrom( from._internal_direct()); break; } case kOldest: { _this->_internal_mutable_oldest()->::inference::ModelSequenceBatching_StrategyOldest::MergeFrom( from._internal_oldest()); break; } case STRATEGY_CHOICE_NOT_SET: { break; } } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelSequenceBatching::CopyFrom(const ModelSequenceBatching& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelSequenceBatching) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelSequenceBatching::IsInitialized() const { return true; } void ModelSequenceBatching::InternalSwap(ModelSequenceBatching* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.control_input_.InternalSwap(&other->_impl_.control_input_); _impl_.state_.InternalSwap(&other->_impl_.state_); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelSequenceBatching, _impl_.iterative_sequence_) + sizeof(ModelSequenceBatching::_impl_.iterative_sequence_) - PROTOBUF_FIELD_OFFSET(ModelSequenceBatching, _impl_.max_sequence_idle_microseconds_)>( reinterpret_cast(&_impl_.max_sequence_idle_microseconds_), reinterpret_cast(&other->_impl_.max_sequence_idle_microseconds_)); swap(_impl_.strategy_choice_, other->_impl_.strategy_choice_); swap(_impl_._oneof_case_[0], other->_impl_._oneof_case_[0]); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelSequenceBatching::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[34]); } // =================================================================== ModelEnsembling_Step_InputMapEntry_DoNotUse::ModelEnsembling_Step_InputMapEntry_DoNotUse() {} ModelEnsembling_Step_InputMapEntry_DoNotUse::ModelEnsembling_Step_InputMapEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelEnsembling_Step_InputMapEntry_DoNotUse::MergeFrom(const ModelEnsembling_Step_InputMapEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelEnsembling_Step_InputMapEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[35]); } // =================================================================== ModelEnsembling_Step_OutputMapEntry_DoNotUse::ModelEnsembling_Step_OutputMapEntry_DoNotUse() {} ModelEnsembling_Step_OutputMapEntry_DoNotUse::ModelEnsembling_Step_OutputMapEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelEnsembling_Step_OutputMapEntry_DoNotUse::MergeFrom(const ModelEnsembling_Step_OutputMapEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelEnsembling_Step_OutputMapEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[36]); } // =================================================================== class ModelEnsembling_Step::_Internal { public: }; ModelEnsembling_Step::ModelEnsembling_Step(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); if (arena != nullptr && !is_message_owned) { arena->OwnCustomDestructor(this, &ModelEnsembling_Step::ArenaDtor); } // @@protoc_insertion_point(arena_constructor:inference.ModelEnsembling.Step) } ModelEnsembling_Step::ModelEnsembling_Step(const ModelEnsembling_Step& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelEnsembling_Step* const _this = this; (void)_this; new (&_impl_) Impl_{ /*decltype(_impl_.input_map_)*/{} , /*decltype(_impl_.output_map_)*/{} , decltype(_impl_.model_name_){} , decltype(_impl_.model_namespace_){} , decltype(_impl_.model_version_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.input_map_.MergeFrom(from._impl_.input_map_); _this->_impl_.output_map_.MergeFrom(from._impl_.output_map_); _impl_.model_name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.model_name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_model_name().empty()) { _this->_impl_.model_name_.Set(from._internal_model_name(), _this->GetArenaForAllocation()); } _impl_.model_namespace_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.model_namespace_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_model_namespace().empty()) { _this->_impl_.model_namespace_.Set(from._internal_model_namespace(), _this->GetArenaForAllocation()); } _this->_impl_.model_version_ = from._impl_.model_version_; // @@protoc_insertion_point(copy_constructor:inference.ModelEnsembling.Step) } inline void ModelEnsembling_Step::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ /*decltype(_impl_.input_map_)*/{::_pbi::ArenaInitialized(), arena} , /*decltype(_impl_.output_map_)*/{::_pbi::ArenaInitialized(), arena} , decltype(_impl_.model_name_){} , decltype(_impl_.model_namespace_){} , decltype(_impl_.model_version_){int64_t{0}} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.model_name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.model_name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.model_namespace_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.model_namespace_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelEnsembling_Step::~ModelEnsembling_Step() { // @@protoc_insertion_point(destructor:inference.ModelEnsembling.Step) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; ArenaDtor(this); return; } SharedDtor(); } inline void ModelEnsembling_Step::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.input_map_.Destruct(); _impl_.input_map_.~MapField(); _impl_.output_map_.Destruct(); _impl_.output_map_.~MapField(); _impl_.model_name_.Destroy(); _impl_.model_namespace_.Destroy(); } void ModelEnsembling_Step::ArenaDtor(void* object) { ModelEnsembling_Step* _this = reinterpret_cast< ModelEnsembling_Step* >(object); _this->_impl_.input_map_.Destruct(); _this->_impl_.output_map_.Destruct(); } void ModelEnsembling_Step::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelEnsembling_Step::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelEnsembling.Step) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.input_map_.Clear(); _impl_.output_map_.Clear(); _impl_.model_name_.ClearToEmpty(); _impl_.model_namespace_.ClearToEmpty(); _impl_.model_version_ = int64_t{0}; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelEnsembling_Step::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string model_name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_model_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelEnsembling.Step.model_name")); } else goto handle_unusual; continue; // int64 model_version = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.model_version_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // map input_map = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.input_map_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<26>(ptr)); } else goto handle_unusual; continue; // map output_map = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 34)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.output_map_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<34>(ptr)); } else goto handle_unusual; continue; // string model_namespace = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { auto str = _internal_mutable_model_namespace(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelEnsembling.Step.model_namespace")); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelEnsembling_Step::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelEnsembling.Step) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string model_name = 1; if (!this->_internal_model_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_model_name().data(), static_cast(this->_internal_model_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelEnsembling.Step.model_name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_model_name(), target); } // int64 model_version = 2; if (this->_internal_model_version() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteInt64ToArray(2, this->_internal_model_version(), target); } // map input_map = 3; if (!this->_internal_input_map().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelEnsembling_Step_InputMapEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_input_map(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelEnsembling.Step.InputMapEntry.key"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.second.data(), static_cast(entry.second.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelEnsembling.Step.InputMapEntry.value"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(3, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(3, entry.first, entry.second, target, stream); check_utf8(entry); } } } // map output_map = 4; if (!this->_internal_output_map().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelEnsembling_Step_OutputMapEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_output_map(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelEnsembling.Step.OutputMapEntry.key"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.second.data(), static_cast(entry.second.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelEnsembling.Step.OutputMapEntry.value"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(4, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(4, entry.first, entry.second, target, stream); check_utf8(entry); } } } // string model_namespace = 5; if (!this->_internal_model_namespace().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_model_namespace().data(), static_cast(this->_internal_model_namespace().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelEnsembling.Step.model_namespace"); target = stream->WriteStringMaybeAliased( 5, this->_internal_model_namespace(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelEnsembling.Step) return target; } size_t ModelEnsembling_Step::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelEnsembling.Step) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // map input_map = 3; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_input_map_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator it = this->_internal_input_map().begin(); it != this->_internal_input_map().end(); ++it) { total_size += ModelEnsembling_Step_InputMapEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // map output_map = 4; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_output_map_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator it = this->_internal_output_map().begin(); it != this->_internal_output_map().end(); ++it) { total_size += ModelEnsembling_Step_OutputMapEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // string model_name = 1; if (!this->_internal_model_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_model_name()); } // string model_namespace = 5; if (!this->_internal_model_namespace().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_model_namespace()); } // int64 model_version = 2; if (this->_internal_model_version() != 0) { total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_model_version()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelEnsembling_Step::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelEnsembling_Step::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelEnsembling_Step::GetClassData() const { return &_class_data_; } void ModelEnsembling_Step::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelEnsembling.Step) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.input_map_.MergeFrom(from._impl_.input_map_); _this->_impl_.output_map_.MergeFrom(from._impl_.output_map_); if (!from._internal_model_name().empty()) { _this->_internal_set_model_name(from._internal_model_name()); } if (!from._internal_model_namespace().empty()) { _this->_internal_set_model_namespace(from._internal_model_namespace()); } if (from._internal_model_version() != 0) { _this->_internal_set_model_version(from._internal_model_version()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelEnsembling_Step::CopyFrom(const ModelEnsembling_Step& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelEnsembling.Step) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelEnsembling_Step::IsInitialized() const { return true; } void ModelEnsembling_Step::InternalSwap(ModelEnsembling_Step* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.input_map_.InternalSwap(&other->_impl_.input_map_); _impl_.output_map_.InternalSwap(&other->_impl_.output_map_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.model_name_, lhs_arena, &other->_impl_.model_name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.model_namespace_, lhs_arena, &other->_impl_.model_namespace_, rhs_arena ); swap(_impl_.model_version_, other->_impl_.model_version_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelEnsembling_Step::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[37]); } // =================================================================== class ModelEnsembling::_Internal { public: }; ModelEnsembling::ModelEnsembling(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelEnsembling) } ModelEnsembling::ModelEnsembling(const ModelEnsembling& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelEnsembling* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.step_){from._impl_.step_} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:inference.ModelEnsembling) } inline void ModelEnsembling::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.step_){arena} , /*decltype(_impl_._cached_size_)*/{} }; } ModelEnsembling::~ModelEnsembling() { // @@protoc_insertion_point(destructor:inference.ModelEnsembling) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelEnsembling::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.step_.~RepeatedPtrField(); } void ModelEnsembling::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelEnsembling::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelEnsembling) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.step_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelEnsembling::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated .inference.ModelEnsembling.Step step = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_step(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelEnsembling::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelEnsembling) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated .inference.ModelEnsembling.Step step = 1; for (unsigned i = 0, n = static_cast(this->_internal_step_size()); i < n; i++) { const auto& repfield = this->_internal_step(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(1, repfield, repfield.GetCachedSize(), target, stream); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelEnsembling) return target; } size_t ModelEnsembling::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelEnsembling) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated .inference.ModelEnsembling.Step step = 1; total_size += 1UL * this->_internal_step_size(); for (const auto& msg : this->_impl_.step_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelEnsembling::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelEnsembling::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelEnsembling::GetClassData() const { return &_class_data_; } void ModelEnsembling::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelEnsembling) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.step_.MergeFrom(from._impl_.step_); _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelEnsembling::CopyFrom(const ModelEnsembling& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelEnsembling) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelEnsembling::IsInitialized() const { return true; } void ModelEnsembling::InternalSwap(ModelEnsembling* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.step_.InternalSwap(&other->_impl_.step_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelEnsembling::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[38]); } // =================================================================== class ModelParameter::_Internal { public: }; ModelParameter::ModelParameter(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelParameter) } ModelParameter::ModelParameter(const ModelParameter& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelParameter* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.string_value_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _impl_.string_value_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.string_value_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_string_value().empty()) { _this->_impl_.string_value_.Set(from._internal_string_value(), _this->GetArenaForAllocation()); } // @@protoc_insertion_point(copy_constructor:inference.ModelParameter) } inline void ModelParameter::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.string_value_){} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.string_value_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.string_value_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelParameter::~ModelParameter() { // @@protoc_insertion_point(destructor:inference.ModelParameter) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelParameter::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.string_value_.Destroy(); } void ModelParameter::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelParameter::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelParameter) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.string_value_.ClearToEmpty(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelParameter::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string string_value = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_string_value(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelParameter.string_value")); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelParameter::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelParameter) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string string_value = 1; if (!this->_internal_string_value().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_string_value().data(), static_cast(this->_internal_string_value().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelParameter.string_value"); target = stream->WriteStringMaybeAliased( 1, this->_internal_string_value(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelParameter) return target; } size_t ModelParameter::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelParameter) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // string string_value = 1; if (!this->_internal_string_value().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_string_value()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelParameter::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelParameter::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelParameter::GetClassData() const { return &_class_data_; } void ModelParameter::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelParameter) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (!from._internal_string_value().empty()) { _this->_internal_set_string_value(from._internal_string_value()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelParameter::CopyFrom(const ModelParameter& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelParameter) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelParameter::IsInitialized() const { return true; } void ModelParameter::InternalSwap(ModelParameter* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.string_value_, lhs_arena, &other->_impl_.string_value_, rhs_arena ); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelParameter::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[39]); } // =================================================================== class ModelWarmup_Input::_Internal { public: }; ModelWarmup_Input::ModelWarmup_Input(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelWarmup.Input) } ModelWarmup_Input::ModelWarmup_Input(const ModelWarmup_Input& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelWarmup_Input* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.dims_){from._impl_.dims_} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.data_type_){} , decltype(_impl_.input_data_type_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.data_type_ = from._impl_.data_type_; clear_has_input_data_type(); switch (from.input_data_type_case()) { case kZeroData: { _this->_internal_set_zero_data(from._internal_zero_data()); break; } case kRandomData: { _this->_internal_set_random_data(from._internal_random_data()); break; } case kInputDataFile: { _this->_internal_set_input_data_file(from._internal_input_data_file()); break; } case INPUT_DATA_TYPE_NOT_SET: { break; } } // @@protoc_insertion_point(copy_constructor:inference.ModelWarmup.Input) } inline void ModelWarmup_Input::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.dims_){arena} , /*decltype(_impl_._dims_cached_byte_size_)*/{0} , decltype(_impl_.data_type_){0} , decltype(_impl_.input_data_type_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{} }; clear_has_input_data_type(); } ModelWarmup_Input::~ModelWarmup_Input() { // @@protoc_insertion_point(destructor:inference.ModelWarmup.Input) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelWarmup_Input::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.dims_.~RepeatedField(); if (has_input_data_type()) { clear_input_data_type(); } } void ModelWarmup_Input::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelWarmup_Input::clear_input_data_type() { // @@protoc_insertion_point(one_of_clear_start:inference.ModelWarmup.Input) switch (input_data_type_case()) { case kZeroData: { // No need to clear break; } case kRandomData: { // No need to clear break; } case kInputDataFile: { _impl_.input_data_type_.input_data_file_.Destroy(); break; } case INPUT_DATA_TYPE_NOT_SET: { break; } } _impl_._oneof_case_[0] = INPUT_DATA_TYPE_NOT_SET; } void ModelWarmup_Input::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelWarmup.Input) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.dims_.Clear(); _impl_.data_type_ = 0; clear_input_data_type(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelWarmup_Input::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // .inference.DataType data_type = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); _internal_set_data_type(static_cast<::inference::DataType>(val)); } else goto handle_unusual; continue; // repeated int64 dims = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(_internal_mutable_dims(), ptr, ctx); CHK_(ptr); } else if (static_cast(tag) == 16) { _internal_add_dims(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // bool zero_data = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { _internal_set_zero_data(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // bool random_data = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { _internal_set_random_data(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr)); CHK_(ptr); } else goto handle_unusual; continue; // string input_data_file = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { auto str = _internal_mutable_input_data_file(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelWarmup.Input.input_data_file")); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelWarmup_Input::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelWarmup.Input) uint32_t cached_has_bits = 0; (void) cached_has_bits; // .inference.DataType data_type = 1; if (this->_internal_data_type() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 1, this->_internal_data_type(), target); } // repeated int64 dims = 2; { int byte_size = _impl_._dims_cached_byte_size_.load(std::memory_order_relaxed); if (byte_size > 0) { target = stream->WriteInt64Packed( 2, _internal_dims(), byte_size, target); } } // bool zero_data = 3; if (_internal_has_zero_data()) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(3, this->_internal_zero_data(), target); } // bool random_data = 4; if (_internal_has_random_data()) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(4, this->_internal_random_data(), target); } // string input_data_file = 5; if (_internal_has_input_data_file()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_input_data_file().data(), static_cast(this->_internal_input_data_file().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelWarmup.Input.input_data_file"); target = stream->WriteStringMaybeAliased( 5, this->_internal_input_data_file(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelWarmup.Input) return target; } size_t ModelWarmup_Input::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelWarmup.Input) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated int64 dims = 2; { size_t data_size = ::_pbi::WireFormatLite:: Int64Size(this->_impl_.dims_); if (data_size > 0) { total_size += 1 + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); _impl_._dims_cached_byte_size_.store(cached_size, std::memory_order_relaxed); total_size += data_size; } // .inference.DataType data_type = 1; if (this->_internal_data_type() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_data_type()); } switch (input_data_type_case()) { // bool zero_data = 3; case kZeroData: { total_size += 1 + 1; break; } // bool random_data = 4; case kRandomData: { total_size += 1 + 1; break; } // string input_data_file = 5; case kInputDataFile: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_input_data_file()); break; } case INPUT_DATA_TYPE_NOT_SET: { break; } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelWarmup_Input::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelWarmup_Input::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelWarmup_Input::GetClassData() const { return &_class_data_; } void ModelWarmup_Input::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelWarmup.Input) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.dims_.MergeFrom(from._impl_.dims_); if (from._internal_data_type() != 0) { _this->_internal_set_data_type(from._internal_data_type()); } switch (from.input_data_type_case()) { case kZeroData: { _this->_internal_set_zero_data(from._internal_zero_data()); break; } case kRandomData: { _this->_internal_set_random_data(from._internal_random_data()); break; } case kInputDataFile: { _this->_internal_set_input_data_file(from._internal_input_data_file()); break; } case INPUT_DATA_TYPE_NOT_SET: { break; } } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelWarmup_Input::CopyFrom(const ModelWarmup_Input& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelWarmup.Input) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelWarmup_Input::IsInitialized() const { return true; } void ModelWarmup_Input::InternalSwap(ModelWarmup_Input* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.dims_.InternalSwap(&other->_impl_.dims_); swap(_impl_.data_type_, other->_impl_.data_type_); swap(_impl_.input_data_type_, other->_impl_.input_data_type_); swap(_impl_._oneof_case_[0], other->_impl_._oneof_case_[0]); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelWarmup_Input::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[40]); } // =================================================================== ModelWarmup_InputsEntry_DoNotUse::ModelWarmup_InputsEntry_DoNotUse() {} ModelWarmup_InputsEntry_DoNotUse::ModelWarmup_InputsEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelWarmup_InputsEntry_DoNotUse::MergeFrom(const ModelWarmup_InputsEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelWarmup_InputsEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[41]); } // =================================================================== class ModelWarmup::_Internal { public: }; ModelWarmup::ModelWarmup(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); if (arena != nullptr && !is_message_owned) { arena->OwnCustomDestructor(this, &ModelWarmup::ArenaDtor); } // @@protoc_insertion_point(arena_constructor:inference.ModelWarmup) } ModelWarmup::ModelWarmup(const ModelWarmup& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelWarmup* const _this = this; (void)_this; new (&_impl_) Impl_{ /*decltype(_impl_.inputs_)*/{} , decltype(_impl_.name_){} , decltype(_impl_.batch_size_){} , decltype(_impl_.count_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.inputs_.MergeFrom(from._impl_.inputs_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } ::memcpy(&_impl_.batch_size_, &from._impl_.batch_size_, static_cast(reinterpret_cast(&_impl_.count_) - reinterpret_cast(&_impl_.batch_size_)) + sizeof(_impl_.count_)); // @@protoc_insertion_point(copy_constructor:inference.ModelWarmup) } inline void ModelWarmup::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ /*decltype(_impl_.inputs_)*/{::_pbi::ArenaInitialized(), arena} , decltype(_impl_.name_){} , decltype(_impl_.batch_size_){0u} , decltype(_impl_.count_){0u} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelWarmup::~ModelWarmup() { // @@protoc_insertion_point(destructor:inference.ModelWarmup) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; ArenaDtor(this); return; } SharedDtor(); } inline void ModelWarmup::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.inputs_.Destruct(); _impl_.inputs_.~MapField(); _impl_.name_.Destroy(); } void ModelWarmup::ArenaDtor(void* object) { ModelWarmup* _this = reinterpret_cast< ModelWarmup* >(object); _this->_impl_.inputs_.Destruct(); } void ModelWarmup::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelWarmup::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelWarmup) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.inputs_.Clear(); _impl_.name_.ClearToEmpty(); ::memset(&_impl_.batch_size_, 0, static_cast( reinterpret_cast(&_impl_.count_) - reinterpret_cast(&_impl_.batch_size_)) + sizeof(_impl_.count_)); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelWarmup::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelWarmup.name")); } else goto handle_unusual; continue; // uint32 batch_size = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 16)) { _impl_.batch_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // map inputs = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.inputs_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<26>(ptr)); } else goto handle_unusual; continue; // uint32 count = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { _impl_.count_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelWarmup::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelWarmup) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelWarmup.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // uint32 batch_size = 2; if (this->_internal_batch_size() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt32ToArray(2, this->_internal_batch_size(), target); } // map inputs = 3; if (!this->_internal_inputs().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelWarmup_InputsEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_inputs(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelWarmup.InputsEntry.key"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(3, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(3, entry.first, entry.second, target, stream); check_utf8(entry); } } } // uint32 count = 4; if (this->_internal_count() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteUInt32ToArray(4, this->_internal_count(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelWarmup) return target; } size_t ModelWarmup::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelWarmup) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // map inputs = 3; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_inputs_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >::const_iterator it = this->_internal_inputs().begin(); it != this->_internal_inputs().end(); ++it) { total_size += ModelWarmup_InputsEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } // uint32 batch_size = 2; if (this->_internal_batch_size() != 0) { total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne(this->_internal_batch_size()); } // uint32 count = 4; if (this->_internal_count() != 0) { total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne(this->_internal_count()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelWarmup::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelWarmup::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelWarmup::GetClassData() const { return &_class_data_; } void ModelWarmup::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelWarmup) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.inputs_.MergeFrom(from._impl_.inputs_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } if (from._internal_batch_size() != 0) { _this->_internal_set_batch_size(from._internal_batch_size()); } if (from._internal_count() != 0) { _this->_internal_set_count(from._internal_count()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelWarmup::CopyFrom(const ModelWarmup& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelWarmup) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelWarmup::IsInitialized() const { return true; } void ModelWarmup::InternalSwap(ModelWarmup* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.inputs_.InternalSwap(&other->_impl_.inputs_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelWarmup, _impl_.count_) + sizeof(ModelWarmup::_impl_.count_) - PROTOBUF_FIELD_OFFSET(ModelWarmup, _impl_.batch_size_)>( reinterpret_cast(&_impl_.batch_size_), reinterpret_cast(&other->_impl_.batch_size_)); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelWarmup::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[42]); } // =================================================================== class ModelOperations::_Internal { public: }; ModelOperations::ModelOperations(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelOperations) } ModelOperations::ModelOperations(const ModelOperations& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelOperations* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.op_library_filename_){from._impl_.op_library_filename_} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:inference.ModelOperations) } inline void ModelOperations::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.op_library_filename_){arena} , /*decltype(_impl_._cached_size_)*/{} }; } ModelOperations::~ModelOperations() { // @@protoc_insertion_point(destructor:inference.ModelOperations) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelOperations::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.op_library_filename_.~RepeatedPtrField(); } void ModelOperations::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelOperations::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelOperations) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.op_library_filename_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelOperations::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated string op_library_filename = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr -= 1; do { ptr += 1; auto str = _internal_add_op_library_filename(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelOperations.op_library_filename")); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelOperations::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelOperations) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated string op_library_filename = 1; for (int i = 0, n = this->_internal_op_library_filename_size(); i < n; i++) { const auto& s = this->_internal_op_library_filename(i); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( s.data(), static_cast(s.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelOperations.op_library_filename"); target = stream->WriteString(1, s, target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelOperations) return target; } size_t ModelOperations::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelOperations) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated string op_library_filename = 1; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(_impl_.op_library_filename_.size()); for (int i = 0, n = _impl_.op_library_filename_.size(); i < n; i++) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( _impl_.op_library_filename_.Get(i)); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelOperations::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelOperations::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelOperations::GetClassData() const { return &_class_data_; } void ModelOperations::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelOperations) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.op_library_filename_.MergeFrom(from._impl_.op_library_filename_); _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelOperations::CopyFrom(const ModelOperations& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelOperations) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelOperations::IsInitialized() const { return true; } void ModelOperations::InternalSwap(ModelOperations* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.op_library_filename_.InternalSwap(&other->_impl_.op_library_filename_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelOperations::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[43]); } // =================================================================== class ModelTransactionPolicy::_Internal { public: }; ModelTransactionPolicy::ModelTransactionPolicy(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelTransactionPolicy) } ModelTransactionPolicy::ModelTransactionPolicy(const ModelTransactionPolicy& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelTransactionPolicy* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.decoupled_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.decoupled_ = from._impl_.decoupled_; // @@protoc_insertion_point(copy_constructor:inference.ModelTransactionPolicy) } inline void ModelTransactionPolicy::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.decoupled_){false} , /*decltype(_impl_._cached_size_)*/{} }; } ModelTransactionPolicy::~ModelTransactionPolicy() { // @@protoc_insertion_point(destructor:inference.ModelTransactionPolicy) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelTransactionPolicy::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); } void ModelTransactionPolicy::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelTransactionPolicy::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelTransactionPolicy) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.decoupled_ = false; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelTransactionPolicy::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // bool decoupled = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.decoupled_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelTransactionPolicy::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelTransactionPolicy) uint32_t cached_has_bits = 0; (void) cached_has_bits; // bool decoupled = 1; if (this->_internal_decoupled() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(1, this->_internal_decoupled(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelTransactionPolicy) return target; } size_t ModelTransactionPolicy::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelTransactionPolicy) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // bool decoupled = 1; if (this->_internal_decoupled() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelTransactionPolicy::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelTransactionPolicy::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelTransactionPolicy::GetClassData() const { return &_class_data_; } void ModelTransactionPolicy::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelTransactionPolicy) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_decoupled() != 0) { _this->_internal_set_decoupled(from._internal_decoupled()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelTransactionPolicy::CopyFrom(const ModelTransactionPolicy& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelTransactionPolicy) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelTransactionPolicy::IsInitialized() const { return true; } void ModelTransactionPolicy::InternalSwap(ModelTransactionPolicy* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_.decoupled_, other->_impl_.decoupled_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelTransactionPolicy::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[44]); } // =================================================================== ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse() {} ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse::MergeFrom(const ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[45]); } // =================================================================== class ModelRepositoryAgents_Agent::_Internal { public: }; ModelRepositoryAgents_Agent::ModelRepositoryAgents_Agent(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); if (arena != nullptr && !is_message_owned) { arena->OwnCustomDestructor(this, &ModelRepositoryAgents_Agent::ArenaDtor); } // @@protoc_insertion_point(arena_constructor:inference.ModelRepositoryAgents.Agent) } ModelRepositoryAgents_Agent::ModelRepositoryAgents_Agent(const ModelRepositoryAgents_Agent& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelRepositoryAgents_Agent* const _this = this; (void)_this; new (&_impl_) Impl_{ /*decltype(_impl_.parameters_)*/{} , decltype(_impl_.name_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.parameters_.MergeFrom(from._impl_.parameters_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } // @@protoc_insertion_point(copy_constructor:inference.ModelRepositoryAgents.Agent) } inline void ModelRepositoryAgents_Agent::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ /*decltype(_impl_.parameters_)*/{::_pbi::ArenaInitialized(), arena} , decltype(_impl_.name_){} , /*decltype(_impl_._cached_size_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING } ModelRepositoryAgents_Agent::~ModelRepositoryAgents_Agent() { // @@protoc_insertion_point(destructor:inference.ModelRepositoryAgents.Agent) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; ArenaDtor(this); return; } SharedDtor(); } inline void ModelRepositoryAgents_Agent::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.parameters_.Destruct(); _impl_.parameters_.~MapField(); _impl_.name_.Destroy(); } void ModelRepositoryAgents_Agent::ArenaDtor(void* object) { ModelRepositoryAgents_Agent* _this = reinterpret_cast< ModelRepositoryAgents_Agent* >(object); _this->_impl_.parameters_.Destruct(); } void ModelRepositoryAgents_Agent::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelRepositoryAgents_Agent::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelRepositoryAgents.Agent) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.parameters_.Clear(); _impl_.name_.ClearToEmpty(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelRepositoryAgents_Agent::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelRepositoryAgents.Agent.name")); } else goto handle_unusual; continue; // map parameters = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.parameters_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelRepositoryAgents_Agent::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelRepositoryAgents.Agent) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelRepositoryAgents.Agent.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // map parameters = 2; if (!this->_internal_parameters().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_parameters(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelRepositoryAgents.Agent.ParametersEntry.key"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.second.data(), static_cast(entry.second.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelRepositoryAgents.Agent.ParametersEntry.value"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(2, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(2, entry.first, entry.second, target, stream); check_utf8(entry); } } } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelRepositoryAgents.Agent) return target; } size_t ModelRepositoryAgents_Agent::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelRepositoryAgents.Agent) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // map parameters = 2; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_parameters_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator it = this->_internal_parameters().begin(); it != this->_internal_parameters().end(); ++it) { total_size += ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelRepositoryAgents_Agent::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelRepositoryAgents_Agent::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelRepositoryAgents_Agent::GetClassData() const { return &_class_data_; } void ModelRepositoryAgents_Agent::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelRepositoryAgents.Agent) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.parameters_.MergeFrom(from._impl_.parameters_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelRepositoryAgents_Agent::CopyFrom(const ModelRepositoryAgents_Agent& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelRepositoryAgents.Agent) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelRepositoryAgents_Agent::IsInitialized() const { return true; } void ModelRepositoryAgents_Agent::InternalSwap(ModelRepositoryAgents_Agent* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.parameters_.InternalSwap(&other->_impl_.parameters_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelRepositoryAgents_Agent::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[46]); } // =================================================================== class ModelRepositoryAgents::_Internal { public: }; ModelRepositoryAgents::ModelRepositoryAgents(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelRepositoryAgents) } ModelRepositoryAgents::ModelRepositoryAgents(const ModelRepositoryAgents& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelRepositoryAgents* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.agents_){from._impl_.agents_} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); // @@protoc_insertion_point(copy_constructor:inference.ModelRepositoryAgents) } inline void ModelRepositoryAgents::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.agents_){arena} , /*decltype(_impl_._cached_size_)*/{} }; } ModelRepositoryAgents::~ModelRepositoryAgents() { // @@protoc_insertion_point(destructor:inference.ModelRepositoryAgents) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelRepositoryAgents::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.agents_.~RepeatedPtrField(); } void ModelRepositoryAgents::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelRepositoryAgents::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelRepositoryAgents) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.agents_.Clear(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelRepositoryAgents::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // repeated .inference.ModelRepositoryAgents.Agent agents = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_agents(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelRepositoryAgents::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelRepositoryAgents) uint32_t cached_has_bits = 0; (void) cached_has_bits; // repeated .inference.ModelRepositoryAgents.Agent agents = 1; for (unsigned i = 0, n = static_cast(this->_internal_agents_size()); i < n; i++) { const auto& repfield = this->_internal_agents(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(1, repfield, repfield.GetCachedSize(), target, stream); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelRepositoryAgents) return target; } size_t ModelRepositoryAgents::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelRepositoryAgents) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated .inference.ModelRepositoryAgents.Agent agents = 1; total_size += 1UL * this->_internal_agents_size(); for (const auto& msg : this->_impl_.agents_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelRepositoryAgents::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelRepositoryAgents::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelRepositoryAgents::GetClassData() const { return &_class_data_; } void ModelRepositoryAgents::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelRepositoryAgents) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.agents_.MergeFrom(from._impl_.agents_); _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelRepositoryAgents::CopyFrom(const ModelRepositoryAgents& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelRepositoryAgents) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelRepositoryAgents::IsInitialized() const { return true; } void ModelRepositoryAgents::InternalSwap(ModelRepositoryAgents* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.agents_.InternalSwap(&other->_impl_.agents_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelRepositoryAgents::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[47]); } // =================================================================== class ModelResponseCache::_Internal { public: }; ModelResponseCache::ModelResponseCache(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); // @@protoc_insertion_point(arena_constructor:inference.ModelResponseCache) } ModelResponseCache::ModelResponseCache(const ModelResponseCache& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelResponseCache* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.enable_){} , /*decltype(_impl_._cached_size_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.enable_ = from._impl_.enable_; // @@protoc_insertion_point(copy_constructor:inference.ModelResponseCache) } inline void ModelResponseCache::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.enable_){false} , /*decltype(_impl_._cached_size_)*/{} }; } ModelResponseCache::~ModelResponseCache() { // @@protoc_insertion_point(destructor:inference.ModelResponseCache) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; return; } SharedDtor(); } inline void ModelResponseCache::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); } void ModelResponseCache::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelResponseCache::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelResponseCache) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.enable_ = false; _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelResponseCache::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // bool enable = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { _impl_.enable_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelResponseCache::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelResponseCache) uint32_t cached_has_bits = 0; (void) cached_has_bits; // bool enable = 1; if (this->_internal_enable() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray(1, this->_internal_enable(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelResponseCache) return target; } size_t ModelResponseCache::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelResponseCache) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // bool enable = 1; if (this->_internal_enable() != 0) { total_size += 1 + 1; } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelResponseCache::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelResponseCache::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelResponseCache::GetClassData() const { return &_class_data_; } void ModelResponseCache::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelResponseCache) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; if (from._internal_enable() != 0) { _this->_internal_set_enable(from._internal_enable()); } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelResponseCache::CopyFrom(const ModelResponseCache& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelResponseCache) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelResponseCache::IsInitialized() const { return true; } void ModelResponseCache::InternalSwap(ModelResponseCache* other) { using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_.enable_, other->_impl_.enable_); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelResponseCache::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[48]); } // =================================================================== ModelConfig_CcModelFilenamesEntry_DoNotUse::ModelConfig_CcModelFilenamesEntry_DoNotUse() {} ModelConfig_CcModelFilenamesEntry_DoNotUse::ModelConfig_CcModelFilenamesEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelConfig_CcModelFilenamesEntry_DoNotUse::MergeFrom(const ModelConfig_CcModelFilenamesEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelConfig_CcModelFilenamesEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[49]); } // =================================================================== ModelConfig_MetricTagsEntry_DoNotUse::ModelConfig_MetricTagsEntry_DoNotUse() {} ModelConfig_MetricTagsEntry_DoNotUse::ModelConfig_MetricTagsEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelConfig_MetricTagsEntry_DoNotUse::MergeFrom(const ModelConfig_MetricTagsEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelConfig_MetricTagsEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[50]); } // =================================================================== ModelConfig_ParametersEntry_DoNotUse::ModelConfig_ParametersEntry_DoNotUse() {} ModelConfig_ParametersEntry_DoNotUse::ModelConfig_ParametersEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) : SuperType(arena) {} void ModelConfig_ParametersEntry_DoNotUse::MergeFrom(const ModelConfig_ParametersEntry_DoNotUse& other) { MergeFromInternal(other); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelConfig_ParametersEntry_DoNotUse::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[51]); } // =================================================================== class ModelConfig::_Internal { public: static const ::inference::ModelVersionPolicy& version_policy(const ModelConfig* msg); static const ::inference::ModelOptimizationPolicy& optimization(const ModelConfig* msg); static const ::inference::ModelDynamicBatching& dynamic_batching(const ModelConfig* msg); static const ::inference::ModelSequenceBatching& sequence_batching(const ModelConfig* msg); static const ::inference::ModelEnsembling& ensemble_scheduling(const ModelConfig* msg); static const ::inference::ModelOperations& model_operations(const ModelConfig* msg); static const ::inference::ModelTransactionPolicy& model_transaction_policy(const ModelConfig* msg); static const ::inference::ModelRepositoryAgents& model_repository_agents(const ModelConfig* msg); static const ::inference::ModelResponseCache& response_cache(const ModelConfig* msg); }; const ::inference::ModelVersionPolicy& ModelConfig::_Internal::version_policy(const ModelConfig* msg) { return *msg->_impl_.version_policy_; } const ::inference::ModelOptimizationPolicy& ModelConfig::_Internal::optimization(const ModelConfig* msg) { return *msg->_impl_.optimization_; } const ::inference::ModelDynamicBatching& ModelConfig::_Internal::dynamic_batching(const ModelConfig* msg) { return *msg->_impl_.scheduling_choice_.dynamic_batching_; } const ::inference::ModelSequenceBatching& ModelConfig::_Internal::sequence_batching(const ModelConfig* msg) { return *msg->_impl_.scheduling_choice_.sequence_batching_; } const ::inference::ModelEnsembling& ModelConfig::_Internal::ensemble_scheduling(const ModelConfig* msg) { return *msg->_impl_.scheduling_choice_.ensemble_scheduling_; } const ::inference::ModelOperations& ModelConfig::_Internal::model_operations(const ModelConfig* msg) { return *msg->_impl_.model_operations_; } const ::inference::ModelTransactionPolicy& ModelConfig::_Internal::model_transaction_policy(const ModelConfig* msg) { return *msg->_impl_.model_transaction_policy_; } const ::inference::ModelRepositoryAgents& ModelConfig::_Internal::model_repository_agents(const ModelConfig* msg) { return *msg->_impl_.model_repository_agents_; } const ::inference::ModelResponseCache& ModelConfig::_Internal::response_cache(const ModelConfig* msg) { return *msg->_impl_.response_cache_; } void ModelConfig::set_allocated_dynamic_batching(::inference::ModelDynamicBatching* dynamic_batching) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); clear_scheduling_choice(); if (dynamic_batching) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(dynamic_batching); if (message_arena != submessage_arena) { dynamic_batching = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, dynamic_batching, submessage_arena); } set_has_dynamic_batching(); _impl_.scheduling_choice_.dynamic_batching_ = dynamic_batching; } // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.dynamic_batching) } void ModelConfig::set_allocated_sequence_batching(::inference::ModelSequenceBatching* sequence_batching) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); clear_scheduling_choice(); if (sequence_batching) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(sequence_batching); if (message_arena != submessage_arena) { sequence_batching = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, sequence_batching, submessage_arena); } set_has_sequence_batching(); _impl_.scheduling_choice_.sequence_batching_ = sequence_batching; } // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.sequence_batching) } void ModelConfig::set_allocated_ensemble_scheduling(::inference::ModelEnsembling* ensemble_scheduling) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); clear_scheduling_choice(); if (ensemble_scheduling) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(ensemble_scheduling); if (message_arena != submessage_arena) { ensemble_scheduling = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, ensemble_scheduling, submessage_arena); } set_has_ensemble_scheduling(); _impl_.scheduling_choice_.ensemble_scheduling_ = ensemble_scheduling; } // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.ensemble_scheduling) } ModelConfig::ModelConfig(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned) : ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) { SharedCtor(arena, is_message_owned); if (arena != nullptr && !is_message_owned) { arena->OwnCustomDestructor(this, &ModelConfig::ArenaDtor); } // @@protoc_insertion_point(arena_constructor:inference.ModelConfig) } ModelConfig::ModelConfig(const ModelConfig& from) : ::PROTOBUF_NAMESPACE_ID::Message() { ModelConfig* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.input_){from._impl_.input_} , decltype(_impl_.output_){from._impl_.output_} , decltype(_impl_.instance_group_){from._impl_.instance_group_} , /*decltype(_impl_.cc_model_filenames_)*/{} , /*decltype(_impl_.metric_tags_)*/{} , /*decltype(_impl_.parameters_)*/{} , decltype(_impl_.model_warmup_){from._impl_.model_warmup_} , decltype(_impl_.batch_input_){from._impl_.batch_input_} , decltype(_impl_.batch_output_){from._impl_.batch_output_} , decltype(_impl_.name_){} , decltype(_impl_.platform_){} , decltype(_impl_.default_model_filename_){} , decltype(_impl_.backend_){} , decltype(_impl_.runtime_){} , decltype(_impl_.version_policy_){nullptr} , decltype(_impl_.optimization_){nullptr} , decltype(_impl_.model_operations_){nullptr} , decltype(_impl_.model_transaction_policy_){nullptr} , decltype(_impl_.model_repository_agents_){nullptr} , decltype(_impl_.response_cache_){nullptr} , decltype(_impl_.max_batch_size_){} , decltype(_impl_.scheduling_choice_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}}; _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_impl_.cc_model_filenames_.MergeFrom(from._impl_.cc_model_filenames_); _this->_impl_.metric_tags_.MergeFrom(from._impl_.metric_tags_); _this->_impl_.parameters_.MergeFrom(from._impl_.parameters_); _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_name().empty()) { _this->_impl_.name_.Set(from._internal_name(), _this->GetArenaForAllocation()); } _impl_.platform_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.platform_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_platform().empty()) { _this->_impl_.platform_.Set(from._internal_platform(), _this->GetArenaForAllocation()); } _impl_.default_model_filename_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.default_model_filename_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_default_model_filename().empty()) { _this->_impl_.default_model_filename_.Set(from._internal_default_model_filename(), _this->GetArenaForAllocation()); } _impl_.backend_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.backend_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_backend().empty()) { _this->_impl_.backend_.Set(from._internal_backend(), _this->GetArenaForAllocation()); } _impl_.runtime_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.runtime_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING if (!from._internal_runtime().empty()) { _this->_impl_.runtime_.Set(from._internal_runtime(), _this->GetArenaForAllocation()); } if (from._internal_has_version_policy()) { _this->_impl_.version_policy_ = new ::inference::ModelVersionPolicy(*from._impl_.version_policy_); } if (from._internal_has_optimization()) { _this->_impl_.optimization_ = new ::inference::ModelOptimizationPolicy(*from._impl_.optimization_); } if (from._internal_has_model_operations()) { _this->_impl_.model_operations_ = new ::inference::ModelOperations(*from._impl_.model_operations_); } if (from._internal_has_model_transaction_policy()) { _this->_impl_.model_transaction_policy_ = new ::inference::ModelTransactionPolicy(*from._impl_.model_transaction_policy_); } if (from._internal_has_model_repository_agents()) { _this->_impl_.model_repository_agents_ = new ::inference::ModelRepositoryAgents(*from._impl_.model_repository_agents_); } if (from._internal_has_response_cache()) { _this->_impl_.response_cache_ = new ::inference::ModelResponseCache(*from._impl_.response_cache_); } _this->_impl_.max_batch_size_ = from._impl_.max_batch_size_; clear_has_scheduling_choice(); switch (from.scheduling_choice_case()) { case kDynamicBatching: { _this->_internal_mutable_dynamic_batching()->::inference::ModelDynamicBatching::MergeFrom( from._internal_dynamic_batching()); break; } case kSequenceBatching: { _this->_internal_mutable_sequence_batching()->::inference::ModelSequenceBatching::MergeFrom( from._internal_sequence_batching()); break; } case kEnsembleScheduling: { _this->_internal_mutable_ensemble_scheduling()->::inference::ModelEnsembling::MergeFrom( from._internal_ensemble_scheduling()); break; } case SCHEDULING_CHOICE_NOT_SET: { break; } } // @@protoc_insertion_point(copy_constructor:inference.ModelConfig) } inline void ModelConfig::SharedCtor( ::_pb::Arena* arena, bool is_message_owned) { (void)arena; (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.input_){arena} , decltype(_impl_.output_){arena} , decltype(_impl_.instance_group_){arena} , /*decltype(_impl_.cc_model_filenames_)*/{::_pbi::ArenaInitialized(), arena} , /*decltype(_impl_.metric_tags_)*/{::_pbi::ArenaInitialized(), arena} , /*decltype(_impl_.parameters_)*/{::_pbi::ArenaInitialized(), arena} , decltype(_impl_.model_warmup_){arena} , decltype(_impl_.batch_input_){arena} , decltype(_impl_.batch_output_){arena} , decltype(_impl_.name_){} , decltype(_impl_.platform_){} , decltype(_impl_.default_model_filename_){} , decltype(_impl_.backend_){} , decltype(_impl_.runtime_){} , decltype(_impl_.version_policy_){nullptr} , decltype(_impl_.optimization_){nullptr} , decltype(_impl_.model_operations_){nullptr} , decltype(_impl_.model_transaction_policy_){nullptr} , decltype(_impl_.model_repository_agents_){nullptr} , decltype(_impl_.response_cache_){nullptr} , decltype(_impl_.max_batch_size_){0} , decltype(_impl_.scheduling_choice_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{} }; _impl_.name_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.name_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.platform_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.platform_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.default_model_filename_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.default_model_filename_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.backend_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.backend_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.runtime_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING _impl_.runtime_.Set("", GetArenaForAllocation()); #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING clear_has_scheduling_choice(); } ModelConfig::~ModelConfig() { // @@protoc_insertion_point(destructor:inference.ModelConfig) if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) { (void)arena; ArenaDtor(this); return; } SharedDtor(); } inline void ModelConfig::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.input_.~RepeatedPtrField(); _impl_.output_.~RepeatedPtrField(); _impl_.instance_group_.~RepeatedPtrField(); _impl_.cc_model_filenames_.Destruct(); _impl_.cc_model_filenames_.~MapField(); _impl_.metric_tags_.Destruct(); _impl_.metric_tags_.~MapField(); _impl_.parameters_.Destruct(); _impl_.parameters_.~MapField(); _impl_.model_warmup_.~RepeatedPtrField(); _impl_.batch_input_.~RepeatedPtrField(); _impl_.batch_output_.~RepeatedPtrField(); _impl_.name_.Destroy(); _impl_.platform_.Destroy(); _impl_.default_model_filename_.Destroy(); _impl_.backend_.Destroy(); _impl_.runtime_.Destroy(); if (this != internal_default_instance()) delete _impl_.version_policy_; if (this != internal_default_instance()) delete _impl_.optimization_; if (this != internal_default_instance()) delete _impl_.model_operations_; if (this != internal_default_instance()) delete _impl_.model_transaction_policy_; if (this != internal_default_instance()) delete _impl_.model_repository_agents_; if (this != internal_default_instance()) delete _impl_.response_cache_; if (has_scheduling_choice()) { clear_scheduling_choice(); } } void ModelConfig::ArenaDtor(void* object) { ModelConfig* _this = reinterpret_cast< ModelConfig* >(object); _this->_impl_.cc_model_filenames_.Destruct(); _this->_impl_.metric_tags_.Destruct(); _this->_impl_.parameters_.Destruct(); } void ModelConfig::SetCachedSize(int size) const { _impl_._cached_size_.Set(size); } void ModelConfig::clear_scheduling_choice() { // @@protoc_insertion_point(one_of_clear_start:inference.ModelConfig) switch (scheduling_choice_case()) { case kDynamicBatching: { if (GetArenaForAllocation() == nullptr) { delete _impl_.scheduling_choice_.dynamic_batching_; } break; } case kSequenceBatching: { if (GetArenaForAllocation() == nullptr) { delete _impl_.scheduling_choice_.sequence_batching_; } break; } case kEnsembleScheduling: { if (GetArenaForAllocation() == nullptr) { delete _impl_.scheduling_choice_.ensemble_scheduling_; } break; } case SCHEDULING_CHOICE_NOT_SET: { break; } } _impl_._oneof_case_[0] = SCHEDULING_CHOICE_NOT_SET; } void ModelConfig::Clear() { // @@protoc_insertion_point(message_clear_start:inference.ModelConfig) uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; _impl_.input_.Clear(); _impl_.output_.Clear(); _impl_.instance_group_.Clear(); _impl_.cc_model_filenames_.Clear(); _impl_.metric_tags_.Clear(); _impl_.parameters_.Clear(); _impl_.model_warmup_.Clear(); _impl_.batch_input_.Clear(); _impl_.batch_output_.Clear(); _impl_.name_.ClearToEmpty(); _impl_.platform_.ClearToEmpty(); _impl_.default_model_filename_.ClearToEmpty(); _impl_.backend_.ClearToEmpty(); _impl_.runtime_.ClearToEmpty(); if (GetArenaForAllocation() == nullptr && _impl_.version_policy_ != nullptr) { delete _impl_.version_policy_; } _impl_.version_policy_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.optimization_ != nullptr) { delete _impl_.optimization_; } _impl_.optimization_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.model_operations_ != nullptr) { delete _impl_.model_operations_; } _impl_.model_operations_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.model_transaction_policy_ != nullptr) { delete _impl_.model_transaction_policy_; } _impl_.model_transaction_policy_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.model_repository_agents_ != nullptr) { delete _impl_.model_repository_agents_; } _impl_.model_repository_agents_ = nullptr; if (GetArenaForAllocation() == nullptr && _impl_.response_cache_ != nullptr) { delete _impl_.response_cache_; } _impl_.response_cache_ = nullptr; _impl_.max_batch_size_ = 0; clear_scheduling_choice(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } const char* ModelConfig::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { // string name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { auto str = _internal_mutable_name(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelConfig.name")); } else goto handle_unusual; continue; // string platform = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { auto str = _internal_mutable_platform(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelConfig.platform")); } else goto handle_unusual; continue; // .inference.ModelVersionPolicy version_policy = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { ptr = ctx->ParseMessage(_internal_mutable_version_policy(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // int32 max_batch_size = 4; case 4: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 32)) { _impl_.max_batch_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated .inference.ModelInput input = 5; case 5: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 42)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_input(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<42>(ptr)); } else goto handle_unusual; continue; // repeated .inference.ModelOutput output = 6; case 6: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 50)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_output(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<50>(ptr)); } else goto handle_unusual; continue; // repeated .inference.ModelInstanceGroup instance_group = 7; case 7: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 58)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(_internal_add_instance_group(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<58>(ptr)); } else goto handle_unusual; continue; // string default_model_filename = 8; case 8: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 66)) { auto str = _internal_mutable_default_model_filename(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelConfig.default_model_filename")); } else goto handle_unusual; continue; // map cc_model_filenames = 9; case 9: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 74)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.cc_model_filenames_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<74>(ptr)); } else goto handle_unusual; continue; // map metric_tags = 10; case 10: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 82)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.metric_tags_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<82>(ptr)); } else goto handle_unusual; continue; // .inference.ModelDynamicBatching dynamic_batching = 11; case 11: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 90)) { ptr = ctx->ParseMessage(_internal_mutable_dynamic_batching(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelOptimizationPolicy optimization = 12; case 12: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 98)) { ptr = ctx->ParseMessage(_internal_mutable_optimization(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelSequenceBatching sequence_batching = 13; case 13: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 106)) { ptr = ctx->ParseMessage(_internal_mutable_sequence_batching(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // map parameters = 14; case 14: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 114)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(&_impl_.parameters_, ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<114>(ptr)); } else goto handle_unusual; continue; // .inference.ModelEnsembling ensemble_scheduling = 15; case 15: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 122)) { ptr = ctx->ParseMessage(_internal_mutable_ensemble_scheduling(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated .inference.ModelWarmup model_warmup = 16; case 16: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 130)) { ptr -= 2; do { ptr += 2; ptr = ctx->ParseMessage(_internal_add_model_warmup(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<130>(ptr)); } else goto handle_unusual; continue; // string backend = 17; case 17: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 138)) { auto str = _internal_mutable_backend(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelConfig.backend")); } else goto handle_unusual; continue; // .inference.ModelOperations model_operations = 18; case 18: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 146)) { ptr = ctx->ParseMessage(_internal_mutable_model_operations(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelTransactionPolicy model_transaction_policy = 19; case 19: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 154)) { ptr = ctx->ParseMessage(_internal_mutable_model_transaction_policy(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // repeated .inference.BatchInput batch_input = 20; case 20: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 162)) { ptr -= 2; do { ptr += 2; ptr = ctx->ParseMessage(_internal_add_batch_input(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<162>(ptr)); } else goto handle_unusual; continue; // repeated .inference.BatchOutput batch_output = 21; case 21: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 170)) { ptr -= 2; do { ptr += 2; ptr = ctx->ParseMessage(_internal_add_batch_output(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<170>(ptr)); } else goto handle_unusual; continue; // .inference.ModelRepositoryAgents model_repository_agents = 23; case 23: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 186)) { ptr = ctx->ParseMessage(_internal_mutable_model_repository_agents(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // .inference.ModelResponseCache response_cache = 24; case 24: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 194)) { ptr = ctx->ParseMessage(_internal_mutable_response_cache(), ptr); CHK_(ptr); } else goto handle_unusual; continue; // string runtime = 25; case 25: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 202)) { auto str = _internal_mutable_runtime(); ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); CHK_(ptr); CHK_(::_pbi::VerifyUTF8(str, "inference.ModelConfig.runtime")); } else goto handle_unusual; continue; default: goto handle_unusual; } // switch handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { CHK_(ptr); ctx->SetLastTag(tag); goto message_done; } ptr = UnknownFieldParse( tag, _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), ptr, ctx); CHK_(ptr != nullptr); } // while message_done: return ptr; failure: ptr = nullptr; goto message_done; #undef CHK_ } uint8_t* ModelConfig::_InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { // @@protoc_insertion_point(serialize_to_array_start:inference.ModelConfig) uint32_t cached_has_bits = 0; (void) cached_has_bits; // string name = 1; if (!this->_internal_name().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_name().data(), static_cast(this->_internal_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.name"); target = stream->WriteStringMaybeAliased( 1, this->_internal_name(), target); } // string platform = 2; if (!this->_internal_platform().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_platform().data(), static_cast(this->_internal_platform().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.platform"); target = stream->WriteStringMaybeAliased( 2, this->_internal_platform(), target); } // .inference.ModelVersionPolicy version_policy = 3; if (this->_internal_has_version_policy()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(3, _Internal::version_policy(this), _Internal::version_policy(this).GetCachedSize(), target, stream); } // int32 max_batch_size = 4; if (this->_internal_max_batch_size() != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteInt32ToArray(4, this->_internal_max_batch_size(), target); } // repeated .inference.ModelInput input = 5; for (unsigned i = 0, n = static_cast(this->_internal_input_size()); i < n; i++) { const auto& repfield = this->_internal_input(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(5, repfield, repfield.GetCachedSize(), target, stream); } // repeated .inference.ModelOutput output = 6; for (unsigned i = 0, n = static_cast(this->_internal_output_size()); i < n; i++) { const auto& repfield = this->_internal_output(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(6, repfield, repfield.GetCachedSize(), target, stream); } // repeated .inference.ModelInstanceGroup instance_group = 7; for (unsigned i = 0, n = static_cast(this->_internal_instance_group_size()); i < n; i++) { const auto& repfield = this->_internal_instance_group(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(7, repfield, repfield.GetCachedSize(), target, stream); } // string default_model_filename = 8; if (!this->_internal_default_model_filename().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_default_model_filename().data(), static_cast(this->_internal_default_model_filename().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.default_model_filename"); target = stream->WriteStringMaybeAliased( 8, this->_internal_default_model_filename(), target); } // map cc_model_filenames = 9; if (!this->_internal_cc_model_filenames().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelConfig_CcModelFilenamesEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_cc_model_filenames(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.CcModelFilenamesEntry.key"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.second.data(), static_cast(entry.second.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.CcModelFilenamesEntry.value"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(9, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(9, entry.first, entry.second, target, stream); check_utf8(entry); } } } // map metric_tags = 10; if (!this->_internal_metric_tags().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelConfig_MetricTagsEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_metric_tags(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.MetricTagsEntry.key"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.second.data(), static_cast(entry.second.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.MetricTagsEntry.value"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(10, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(10, entry.first, entry.second, target, stream); check_utf8(entry); } } } // .inference.ModelDynamicBatching dynamic_batching = 11; if (_internal_has_dynamic_batching()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(11, _Internal::dynamic_batching(this), _Internal::dynamic_batching(this).GetCachedSize(), target, stream); } // .inference.ModelOptimizationPolicy optimization = 12; if (this->_internal_has_optimization()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(12, _Internal::optimization(this), _Internal::optimization(this).GetCachedSize(), target, stream); } // .inference.ModelSequenceBatching sequence_batching = 13; if (_internal_has_sequence_batching()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(13, _Internal::sequence_batching(this), _Internal::sequence_batching(this).GetCachedSize(), target, stream); } // map parameters = 14; if (!this->_internal_parameters().empty()) { using MapType = ::_pb::Map; using WireHelper = ModelConfig_ParametersEntry_DoNotUse::Funcs; const auto& map_field = this->_internal_parameters(); auto check_utf8 = [](const MapType::value_type& entry) { (void)entry; ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( entry.first.data(), static_cast(entry.first.length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.ParametersEntry.key"); }; if (stream->IsSerializationDeterministic() && map_field.size() > 1) { for (const auto& entry : ::_pbi::MapSorterPtr(map_field)) { target = WireHelper::InternalSerialize(14, entry.first, entry.second, target, stream); check_utf8(entry); } } else { for (const auto& entry : map_field) { target = WireHelper::InternalSerialize(14, entry.first, entry.second, target, stream); check_utf8(entry); } } } // .inference.ModelEnsembling ensemble_scheduling = 15; if (_internal_has_ensemble_scheduling()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(15, _Internal::ensemble_scheduling(this), _Internal::ensemble_scheduling(this).GetCachedSize(), target, stream); } // repeated .inference.ModelWarmup model_warmup = 16; for (unsigned i = 0, n = static_cast(this->_internal_model_warmup_size()); i < n; i++) { const auto& repfield = this->_internal_model_warmup(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(16, repfield, repfield.GetCachedSize(), target, stream); } // string backend = 17; if (!this->_internal_backend().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_backend().data(), static_cast(this->_internal_backend().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.backend"); target = stream->WriteStringMaybeAliased( 17, this->_internal_backend(), target); } // .inference.ModelOperations model_operations = 18; if (this->_internal_has_model_operations()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(18, _Internal::model_operations(this), _Internal::model_operations(this).GetCachedSize(), target, stream); } // .inference.ModelTransactionPolicy model_transaction_policy = 19; if (this->_internal_has_model_transaction_policy()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(19, _Internal::model_transaction_policy(this), _Internal::model_transaction_policy(this).GetCachedSize(), target, stream); } // repeated .inference.BatchInput batch_input = 20; for (unsigned i = 0, n = static_cast(this->_internal_batch_input_size()); i < n; i++) { const auto& repfield = this->_internal_batch_input(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(20, repfield, repfield.GetCachedSize(), target, stream); } // repeated .inference.BatchOutput batch_output = 21; for (unsigned i = 0, n = static_cast(this->_internal_batch_output_size()); i < n; i++) { const auto& repfield = this->_internal_batch_output(i); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(21, repfield, repfield.GetCachedSize(), target, stream); } // .inference.ModelRepositoryAgents model_repository_agents = 23; if (this->_internal_has_model_repository_agents()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(23, _Internal::model_repository_agents(this), _Internal::model_repository_agents(this).GetCachedSize(), target, stream); } // .inference.ModelResponseCache response_cache = 24; if (this->_internal_has_response_cache()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessage(24, _Internal::response_cache(this), _Internal::response_cache(this).GetCachedSize(), target, stream); } // string runtime = 25; if (!this->_internal_runtime().empty()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->_internal_runtime().data(), static_cast(this->_internal_runtime().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "inference.ModelConfig.runtime"); target = stream->WriteStringMaybeAliased( 25, this->_internal_runtime(), target); } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); } // @@protoc_insertion_point(serialize_to_array_end:inference.ModelConfig) return target; } size_t ModelConfig::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:inference.ModelConfig) size_t total_size = 0; uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; // repeated .inference.ModelInput input = 5; total_size += 1UL * this->_internal_input_size(); for (const auto& msg : this->_impl_.input_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // repeated .inference.ModelOutput output = 6; total_size += 1UL * this->_internal_output_size(); for (const auto& msg : this->_impl_.output_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // repeated .inference.ModelInstanceGroup instance_group = 7; total_size += 1UL * this->_internal_instance_group_size(); for (const auto& msg : this->_impl_.instance_group_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // map cc_model_filenames = 9; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_cc_model_filenames_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator it = this->_internal_cc_model_filenames().begin(); it != this->_internal_cc_model_filenames().end(); ++it) { total_size += ModelConfig_CcModelFilenamesEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // map metric_tags = 10; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_metric_tags_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator it = this->_internal_metric_tags().begin(); it != this->_internal_metric_tags().end(); ++it) { total_size += ModelConfig_MetricTagsEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // map parameters = 14; total_size += 1 * ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_parameters_size()); for (::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >::const_iterator it = this->_internal_parameters().begin(); it != this->_internal_parameters().end(); ++it) { total_size += ModelConfig_ParametersEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } // repeated .inference.ModelWarmup model_warmup = 16; total_size += 2UL * this->_internal_model_warmup_size(); for (const auto& msg : this->_impl_.model_warmup_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // repeated .inference.BatchInput batch_input = 20; total_size += 2UL * this->_internal_batch_input_size(); for (const auto& msg : this->_impl_.batch_input_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // repeated .inference.BatchOutput batch_output = 21; total_size += 2UL * this->_internal_batch_output_size(); for (const auto& msg : this->_impl_.batch_output_) { total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); } // string name = 1; if (!this->_internal_name().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_name()); } // string platform = 2; if (!this->_internal_platform().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_platform()); } // string default_model_filename = 8; if (!this->_internal_default_model_filename().empty()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_default_model_filename()); } // string backend = 17; if (!this->_internal_backend().empty()) { total_size += 2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_backend()); } // string runtime = 25; if (!this->_internal_runtime().empty()) { total_size += 2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_runtime()); } // .inference.ModelVersionPolicy version_policy = 3; if (this->_internal_has_version_policy()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.version_policy_); } // .inference.ModelOptimizationPolicy optimization = 12; if (this->_internal_has_optimization()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.optimization_); } // .inference.ModelOperations model_operations = 18; if (this->_internal_has_model_operations()) { total_size += 2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.model_operations_); } // .inference.ModelTransactionPolicy model_transaction_policy = 19; if (this->_internal_has_model_transaction_policy()) { total_size += 2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.model_transaction_policy_); } // .inference.ModelRepositoryAgents model_repository_agents = 23; if (this->_internal_has_model_repository_agents()) { total_size += 2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.model_repository_agents_); } // .inference.ModelResponseCache response_cache = 24; if (this->_internal_has_response_cache()) { total_size += 2 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.response_cache_); } // int32 max_batch_size = 4; if (this->_internal_max_batch_size() != 0) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne(this->_internal_max_batch_size()); } switch (scheduling_choice_case()) { // .inference.ModelDynamicBatching dynamic_batching = 11; case kDynamicBatching: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.scheduling_choice_.dynamic_batching_); break; } // .inference.ModelSequenceBatching sequence_batching = 13; case kSequenceBatching: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.scheduling_choice_.sequence_batching_); break; } // .inference.ModelEnsembling ensemble_scheduling = 15; case kEnsembleScheduling: { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( *_impl_.scheduling_choice_.ensemble_scheduling_); break; } case SCHEDULING_CHOICE_NOT_SET: { break; } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ModelConfig::_class_data_ = { ::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck, ModelConfig::MergeImpl }; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ModelConfig::GetClassData() const { return &_class_data_; } void ModelConfig::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) { auto* const _this = static_cast(&to_msg); auto& from = static_cast(from_msg); // @@protoc_insertion_point(class_specific_merge_from_start:inference.ModelConfig) GOOGLE_DCHECK_NE(&from, _this); uint32_t cached_has_bits = 0; (void) cached_has_bits; _this->_impl_.input_.MergeFrom(from._impl_.input_); _this->_impl_.output_.MergeFrom(from._impl_.output_); _this->_impl_.instance_group_.MergeFrom(from._impl_.instance_group_); _this->_impl_.cc_model_filenames_.MergeFrom(from._impl_.cc_model_filenames_); _this->_impl_.metric_tags_.MergeFrom(from._impl_.metric_tags_); _this->_impl_.parameters_.MergeFrom(from._impl_.parameters_); _this->_impl_.model_warmup_.MergeFrom(from._impl_.model_warmup_); _this->_impl_.batch_input_.MergeFrom(from._impl_.batch_input_); _this->_impl_.batch_output_.MergeFrom(from._impl_.batch_output_); if (!from._internal_name().empty()) { _this->_internal_set_name(from._internal_name()); } if (!from._internal_platform().empty()) { _this->_internal_set_platform(from._internal_platform()); } if (!from._internal_default_model_filename().empty()) { _this->_internal_set_default_model_filename(from._internal_default_model_filename()); } if (!from._internal_backend().empty()) { _this->_internal_set_backend(from._internal_backend()); } if (!from._internal_runtime().empty()) { _this->_internal_set_runtime(from._internal_runtime()); } if (from._internal_has_version_policy()) { _this->_internal_mutable_version_policy()->::inference::ModelVersionPolicy::MergeFrom( from._internal_version_policy()); } if (from._internal_has_optimization()) { _this->_internal_mutable_optimization()->::inference::ModelOptimizationPolicy::MergeFrom( from._internal_optimization()); } if (from._internal_has_model_operations()) { _this->_internal_mutable_model_operations()->::inference::ModelOperations::MergeFrom( from._internal_model_operations()); } if (from._internal_has_model_transaction_policy()) { _this->_internal_mutable_model_transaction_policy()->::inference::ModelTransactionPolicy::MergeFrom( from._internal_model_transaction_policy()); } if (from._internal_has_model_repository_agents()) { _this->_internal_mutable_model_repository_agents()->::inference::ModelRepositoryAgents::MergeFrom( from._internal_model_repository_agents()); } if (from._internal_has_response_cache()) { _this->_internal_mutable_response_cache()->::inference::ModelResponseCache::MergeFrom( from._internal_response_cache()); } if (from._internal_max_batch_size() != 0) { _this->_internal_set_max_batch_size(from._internal_max_batch_size()); } switch (from.scheduling_choice_case()) { case kDynamicBatching: { _this->_internal_mutable_dynamic_batching()->::inference::ModelDynamicBatching::MergeFrom( from._internal_dynamic_batching()); break; } case kSequenceBatching: { _this->_internal_mutable_sequence_batching()->::inference::ModelSequenceBatching::MergeFrom( from._internal_sequence_batching()); break; } case kEnsembleScheduling: { _this->_internal_mutable_ensemble_scheduling()->::inference::ModelEnsembling::MergeFrom( from._internal_ensemble_scheduling()); break; } case SCHEDULING_CHOICE_NOT_SET: { break; } } _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); } void ModelConfig::CopyFrom(const ModelConfig& from) { // @@protoc_insertion_point(class_specific_copy_from_start:inference.ModelConfig) if (&from == this) return; Clear(); MergeFrom(from); } bool ModelConfig::IsInitialized() const { return true; } void ModelConfig::InternalSwap(ModelConfig* other) { using std::swap; auto* lhs_arena = GetArenaForAllocation(); auto* rhs_arena = other->GetArenaForAllocation(); _internal_metadata_.InternalSwap(&other->_internal_metadata_); _impl_.input_.InternalSwap(&other->_impl_.input_); _impl_.output_.InternalSwap(&other->_impl_.output_); _impl_.instance_group_.InternalSwap(&other->_impl_.instance_group_); _impl_.cc_model_filenames_.InternalSwap(&other->_impl_.cc_model_filenames_); _impl_.metric_tags_.InternalSwap(&other->_impl_.metric_tags_); _impl_.parameters_.InternalSwap(&other->_impl_.parameters_); _impl_.model_warmup_.InternalSwap(&other->_impl_.model_warmup_); _impl_.batch_input_.InternalSwap(&other->_impl_.batch_input_); _impl_.batch_output_.InternalSwap(&other->_impl_.batch_output_); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.name_, lhs_arena, &other->_impl_.name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.platform_, lhs_arena, &other->_impl_.platform_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.default_model_filename_, lhs_arena, &other->_impl_.default_model_filename_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.backend_, lhs_arena, &other->_impl_.backend_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( &_impl_.runtime_, lhs_arena, &other->_impl_.runtime_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(ModelConfig, _impl_.max_batch_size_) + sizeof(ModelConfig::_impl_.max_batch_size_) - PROTOBUF_FIELD_OFFSET(ModelConfig, _impl_.version_policy_)>( reinterpret_cast(&_impl_.version_policy_), reinterpret_cast(&other->_impl_.version_policy_)); swap(_impl_.scheduling_choice_, other->_impl_.scheduling_choice_); swap(_impl_._oneof_case_[0], other->_impl_._oneof_case_[0]); } ::PROTOBUF_NAMESPACE_ID::Metadata ModelConfig::GetMetadata() const { return ::_pbi::AssignDescriptors( &descriptor_table_model_5fconfig_2eproto_getter, &descriptor_table_model_5fconfig_2eproto_once, file_level_metadata_model_5fconfig_2eproto[52]); } // @@protoc_insertion_point(namespace_scope) } // namespace inference PROTOBUF_NAMESPACE_OPEN template<> PROTOBUF_NOINLINE ::inference::ModelRateLimiter_Resource* Arena::CreateMaybeMessage< ::inference::ModelRateLimiter_Resource >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelRateLimiter_Resource >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelRateLimiter* Arena::CreateMaybeMessage< ::inference::ModelRateLimiter >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelRateLimiter >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelInstanceGroup_SecondaryDevice* Arena::CreateMaybeMessage< ::inference::ModelInstanceGroup_SecondaryDevice >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelInstanceGroup_SecondaryDevice >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelInstanceGroup* Arena::CreateMaybeMessage< ::inference::ModelInstanceGroup >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelInstanceGroup >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelTensorReshape* Arena::CreateMaybeMessage< ::inference::ModelTensorReshape >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelTensorReshape >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelInput* Arena::CreateMaybeMessage< ::inference::ModelInput >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelInput >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOutput* Arena::CreateMaybeMessage< ::inference::ModelOutput >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOutput >(arena); } template<> PROTOBUF_NOINLINE ::inference::BatchInput* Arena::CreateMaybeMessage< ::inference::BatchInput >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::BatchInput >(arena); } template<> PROTOBUF_NOINLINE ::inference::BatchOutput* Arena::CreateMaybeMessage< ::inference::BatchOutput >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::BatchOutput >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelVersionPolicy_Latest* Arena::CreateMaybeMessage< ::inference::ModelVersionPolicy_Latest >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelVersionPolicy_Latest >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelVersionPolicy_All* Arena::CreateMaybeMessage< ::inference::ModelVersionPolicy_All >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelVersionPolicy_All >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelVersionPolicy_Specific* Arena::CreateMaybeMessage< ::inference::ModelVersionPolicy_Specific >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelVersionPolicy_Specific >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelVersionPolicy* Arena::CreateMaybeMessage< ::inference::ModelVersionPolicy >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelVersionPolicy >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_Graph* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_Graph >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_Graph >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_Cuda* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_Cuda >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_Cuda >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_ExecutionAccelerators* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_ExecutionAccelerators >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_ExecutionAccelerators >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOptimizationPolicy* Arena::CreateMaybeMessage< ::inference::ModelOptimizationPolicy >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOptimizationPolicy >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelQueuePolicy* Arena::CreateMaybeMessage< ::inference::ModelQueuePolicy >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelQueuePolicy >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelDynamicBatching* Arena::CreateMaybeMessage< ::inference::ModelDynamicBatching >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelDynamicBatching >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelSequenceBatching_Control* Arena::CreateMaybeMessage< ::inference::ModelSequenceBatching_Control >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelSequenceBatching_Control >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelSequenceBatching_ControlInput* Arena::CreateMaybeMessage< ::inference::ModelSequenceBatching_ControlInput >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelSequenceBatching_ControlInput >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelSequenceBatching_InitialState* Arena::CreateMaybeMessage< ::inference::ModelSequenceBatching_InitialState >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelSequenceBatching_InitialState >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelSequenceBatching_State* Arena::CreateMaybeMessage< ::inference::ModelSequenceBatching_State >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelSequenceBatching_State >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelSequenceBatching_StrategyDirect* Arena::CreateMaybeMessage< ::inference::ModelSequenceBatching_StrategyDirect >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelSequenceBatching_StrategyDirect >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelSequenceBatching_StrategyOldest* Arena::CreateMaybeMessage< ::inference::ModelSequenceBatching_StrategyOldest >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelSequenceBatching_StrategyOldest >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelSequenceBatching* Arena::CreateMaybeMessage< ::inference::ModelSequenceBatching >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelSequenceBatching >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelEnsembling_Step* Arena::CreateMaybeMessage< ::inference::ModelEnsembling_Step >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelEnsembling_Step >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelEnsembling* Arena::CreateMaybeMessage< ::inference::ModelEnsembling >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelEnsembling >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelParameter* Arena::CreateMaybeMessage< ::inference::ModelParameter >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelParameter >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelWarmup_Input* Arena::CreateMaybeMessage< ::inference::ModelWarmup_Input >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelWarmup_Input >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelWarmup_InputsEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelWarmup_InputsEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelWarmup_InputsEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelWarmup* Arena::CreateMaybeMessage< ::inference::ModelWarmup >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelWarmup >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelOperations* Arena::CreateMaybeMessage< ::inference::ModelOperations >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelOperations >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelTransactionPolicy* Arena::CreateMaybeMessage< ::inference::ModelTransactionPolicy >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelTransactionPolicy >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelRepositoryAgents_Agent* Arena::CreateMaybeMessage< ::inference::ModelRepositoryAgents_Agent >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelRepositoryAgents_Agent >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelRepositoryAgents* Arena::CreateMaybeMessage< ::inference::ModelRepositoryAgents >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelRepositoryAgents >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelResponseCache* Arena::CreateMaybeMessage< ::inference::ModelResponseCache >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelResponseCache >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelConfig_MetricTagsEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelConfig_MetricTagsEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelConfig_MetricTagsEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelConfig_ParametersEntry_DoNotUse* Arena::CreateMaybeMessage< ::inference::ModelConfig_ParametersEntry_DoNotUse >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelConfig_ParametersEntry_DoNotUse >(arena); } template<> PROTOBUF_NOINLINE ::inference::ModelConfig* Arena::CreateMaybeMessage< ::inference::ModelConfig >(Arena* arena) { return Arena::CreateMessageInternal< ::inference::ModelConfig >(arena); } PROTOBUF_NAMESPACE_CLOSE // @@protoc_insertion_point(global_scope) #include kytensor-2.49.0.7/core/src/model_config.pb.h000066400000000000000000025000371527052371200205610ustar00rootroot00000000000000// Generated by the protocol buffer compiler. DO NOT EDIT! // source: model_config.proto #ifndef GOOGLE_PROTOBUF_INCLUDED_model_5fconfig_2eproto #define GOOGLE_PROTOBUF_INCLUDED_model_5fconfig_2eproto #include #include #include #if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif #if 3021012 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. #endif #include #include #include #include #include #include #include #include #include #include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export #include #include #include #include // @@protoc_insertion_point(includes) #include #define PROTOBUF_INTERNAL_EXPORT_model_5fconfig_2eproto PROTOBUF_NAMESPACE_OPEN namespace internal { class AnyMetadata; } // namespace internal PROTOBUF_NAMESPACE_CLOSE // Internal implementation detail -- do not use these members. struct TableStruct_model_5fconfig_2eproto { static const uint32_t offsets[]; }; extern const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_model_5fconfig_2eproto; namespace inference { class BatchInput; struct BatchInputDefaultTypeInternal; extern BatchInputDefaultTypeInternal _BatchInput_default_instance_; class BatchOutput; struct BatchOutputDefaultTypeInternal; extern BatchOutputDefaultTypeInternal _BatchOutput_default_instance_; class ModelConfig; struct ModelConfigDefaultTypeInternal; extern ModelConfigDefaultTypeInternal _ModelConfig_default_instance_; class ModelConfig_CcModelFilenamesEntry_DoNotUse; struct ModelConfig_CcModelFilenamesEntry_DoNotUseDefaultTypeInternal; extern ModelConfig_CcModelFilenamesEntry_DoNotUseDefaultTypeInternal _ModelConfig_CcModelFilenamesEntry_DoNotUse_default_instance_; class ModelConfig_MetricTagsEntry_DoNotUse; struct ModelConfig_MetricTagsEntry_DoNotUseDefaultTypeInternal; extern ModelConfig_MetricTagsEntry_DoNotUseDefaultTypeInternal _ModelConfig_MetricTagsEntry_DoNotUse_default_instance_; class ModelConfig_ParametersEntry_DoNotUse; struct ModelConfig_ParametersEntry_DoNotUseDefaultTypeInternal; extern ModelConfig_ParametersEntry_DoNotUseDefaultTypeInternal _ModelConfig_ParametersEntry_DoNotUse_default_instance_; class ModelDynamicBatching; struct ModelDynamicBatchingDefaultTypeInternal; extern ModelDynamicBatchingDefaultTypeInternal _ModelDynamicBatching_default_instance_; class ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse; struct ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUseDefaultTypeInternal; extern ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUseDefaultTypeInternal _ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse_default_instance_; class ModelEnsembling; struct ModelEnsemblingDefaultTypeInternal; extern ModelEnsemblingDefaultTypeInternal _ModelEnsembling_default_instance_; class ModelEnsembling_Step; struct ModelEnsembling_StepDefaultTypeInternal; extern ModelEnsembling_StepDefaultTypeInternal _ModelEnsembling_Step_default_instance_; class ModelEnsembling_Step_InputMapEntry_DoNotUse; struct ModelEnsembling_Step_InputMapEntry_DoNotUseDefaultTypeInternal; extern ModelEnsembling_Step_InputMapEntry_DoNotUseDefaultTypeInternal _ModelEnsembling_Step_InputMapEntry_DoNotUse_default_instance_; class ModelEnsembling_Step_OutputMapEntry_DoNotUse; struct ModelEnsembling_Step_OutputMapEntry_DoNotUseDefaultTypeInternal; extern ModelEnsembling_Step_OutputMapEntry_DoNotUseDefaultTypeInternal _ModelEnsembling_Step_OutputMapEntry_DoNotUse_default_instance_; class ModelInput; struct ModelInputDefaultTypeInternal; extern ModelInputDefaultTypeInternal _ModelInput_default_instance_; class ModelInstanceGroup; struct ModelInstanceGroupDefaultTypeInternal; extern ModelInstanceGroupDefaultTypeInternal _ModelInstanceGroup_default_instance_; class ModelInstanceGroup_SecondaryDevice; struct ModelInstanceGroup_SecondaryDeviceDefaultTypeInternal; extern ModelInstanceGroup_SecondaryDeviceDefaultTypeInternal _ModelInstanceGroup_SecondaryDevice_default_instance_; class ModelOperations; struct ModelOperationsDefaultTypeInternal; extern ModelOperationsDefaultTypeInternal _ModelOperations_default_instance_; class ModelOptimizationPolicy; struct ModelOptimizationPolicyDefaultTypeInternal; extern ModelOptimizationPolicyDefaultTypeInternal _ModelOptimizationPolicy_default_instance_; class ModelOptimizationPolicy_Cuda; struct ModelOptimizationPolicy_CudaDefaultTypeInternal; extern ModelOptimizationPolicy_CudaDefaultTypeInternal _ModelOptimizationPolicy_Cuda_default_instance_; class ModelOptimizationPolicy_Cuda_GraphSpec; struct ModelOptimizationPolicy_Cuda_GraphSpecDefaultTypeInternal; extern ModelOptimizationPolicy_Cuda_GraphSpecDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_default_instance_; class ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse; struct ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUseDefaultTypeInternal; extern ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUseDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse_default_instance_; class ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound; struct ModelOptimizationPolicy_Cuda_GraphSpec_LowerBoundDefaultTypeInternal; extern ModelOptimizationPolicy_Cuda_GraphSpec_LowerBoundDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_default_instance_; class ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse; struct ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUseDefaultTypeInternal; extern ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUseDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse_default_instance_; class ModelOptimizationPolicy_Cuda_GraphSpec_Shape; struct ModelOptimizationPolicy_Cuda_GraphSpec_ShapeDefaultTypeInternal; extern ModelOptimizationPolicy_Cuda_GraphSpec_ShapeDefaultTypeInternal _ModelOptimizationPolicy_Cuda_GraphSpec_Shape_default_instance_; class ModelOptimizationPolicy_ExecutionAccelerators; struct ModelOptimizationPolicy_ExecutionAcceleratorsDefaultTypeInternal; extern ModelOptimizationPolicy_ExecutionAcceleratorsDefaultTypeInternal _ModelOptimizationPolicy_ExecutionAccelerators_default_instance_; class ModelOptimizationPolicy_ExecutionAccelerators_Accelerator; struct ModelOptimizationPolicy_ExecutionAccelerators_AcceleratorDefaultTypeInternal; extern ModelOptimizationPolicy_ExecutionAccelerators_AcceleratorDefaultTypeInternal _ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_default_instance_; class ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse; struct ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUseDefaultTypeInternal; extern ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUseDefaultTypeInternal _ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse_default_instance_; class ModelOptimizationPolicy_Graph; struct ModelOptimizationPolicy_GraphDefaultTypeInternal; extern ModelOptimizationPolicy_GraphDefaultTypeInternal _ModelOptimizationPolicy_Graph_default_instance_; class ModelOptimizationPolicy_PinnedMemoryBuffer; struct ModelOptimizationPolicy_PinnedMemoryBufferDefaultTypeInternal; extern ModelOptimizationPolicy_PinnedMemoryBufferDefaultTypeInternal _ModelOptimizationPolicy_PinnedMemoryBuffer_default_instance_; class ModelOutput; struct ModelOutputDefaultTypeInternal; extern ModelOutputDefaultTypeInternal _ModelOutput_default_instance_; class ModelParameter; struct ModelParameterDefaultTypeInternal; extern ModelParameterDefaultTypeInternal _ModelParameter_default_instance_; class ModelQueuePolicy; struct ModelQueuePolicyDefaultTypeInternal; extern ModelQueuePolicyDefaultTypeInternal _ModelQueuePolicy_default_instance_; class ModelRateLimiter; struct ModelRateLimiterDefaultTypeInternal; extern ModelRateLimiterDefaultTypeInternal _ModelRateLimiter_default_instance_; class ModelRateLimiter_Resource; struct ModelRateLimiter_ResourceDefaultTypeInternal; extern ModelRateLimiter_ResourceDefaultTypeInternal _ModelRateLimiter_Resource_default_instance_; class ModelRepositoryAgents; struct ModelRepositoryAgentsDefaultTypeInternal; extern ModelRepositoryAgentsDefaultTypeInternal _ModelRepositoryAgents_default_instance_; class ModelRepositoryAgents_Agent; struct ModelRepositoryAgents_AgentDefaultTypeInternal; extern ModelRepositoryAgents_AgentDefaultTypeInternal _ModelRepositoryAgents_Agent_default_instance_; class ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse; struct ModelRepositoryAgents_Agent_ParametersEntry_DoNotUseDefaultTypeInternal; extern ModelRepositoryAgents_Agent_ParametersEntry_DoNotUseDefaultTypeInternal _ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse_default_instance_; class ModelResponseCache; struct ModelResponseCacheDefaultTypeInternal; extern ModelResponseCacheDefaultTypeInternal _ModelResponseCache_default_instance_; class ModelSequenceBatching; struct ModelSequenceBatchingDefaultTypeInternal; extern ModelSequenceBatchingDefaultTypeInternal _ModelSequenceBatching_default_instance_; class ModelSequenceBatching_Control; struct ModelSequenceBatching_ControlDefaultTypeInternal; extern ModelSequenceBatching_ControlDefaultTypeInternal _ModelSequenceBatching_Control_default_instance_; class ModelSequenceBatching_ControlInput; struct ModelSequenceBatching_ControlInputDefaultTypeInternal; extern ModelSequenceBatching_ControlInputDefaultTypeInternal _ModelSequenceBatching_ControlInput_default_instance_; class ModelSequenceBatching_InitialState; struct ModelSequenceBatching_InitialStateDefaultTypeInternal; extern ModelSequenceBatching_InitialStateDefaultTypeInternal _ModelSequenceBatching_InitialState_default_instance_; class ModelSequenceBatching_State; struct ModelSequenceBatching_StateDefaultTypeInternal; extern ModelSequenceBatching_StateDefaultTypeInternal _ModelSequenceBatching_State_default_instance_; class ModelSequenceBatching_StrategyDirect; struct ModelSequenceBatching_StrategyDirectDefaultTypeInternal; extern ModelSequenceBatching_StrategyDirectDefaultTypeInternal _ModelSequenceBatching_StrategyDirect_default_instance_; class ModelSequenceBatching_StrategyOldest; struct ModelSequenceBatching_StrategyOldestDefaultTypeInternal; extern ModelSequenceBatching_StrategyOldestDefaultTypeInternal _ModelSequenceBatching_StrategyOldest_default_instance_; class ModelTensorReshape; struct ModelTensorReshapeDefaultTypeInternal; extern ModelTensorReshapeDefaultTypeInternal _ModelTensorReshape_default_instance_; class ModelTransactionPolicy; struct ModelTransactionPolicyDefaultTypeInternal; extern ModelTransactionPolicyDefaultTypeInternal _ModelTransactionPolicy_default_instance_; class ModelVersionPolicy; struct ModelVersionPolicyDefaultTypeInternal; extern ModelVersionPolicyDefaultTypeInternal _ModelVersionPolicy_default_instance_; class ModelVersionPolicy_All; struct ModelVersionPolicy_AllDefaultTypeInternal; extern ModelVersionPolicy_AllDefaultTypeInternal _ModelVersionPolicy_All_default_instance_; class ModelVersionPolicy_Latest; struct ModelVersionPolicy_LatestDefaultTypeInternal; extern ModelVersionPolicy_LatestDefaultTypeInternal _ModelVersionPolicy_Latest_default_instance_; class ModelVersionPolicy_Specific; struct ModelVersionPolicy_SpecificDefaultTypeInternal; extern ModelVersionPolicy_SpecificDefaultTypeInternal _ModelVersionPolicy_Specific_default_instance_; class ModelWarmup; struct ModelWarmupDefaultTypeInternal; extern ModelWarmupDefaultTypeInternal _ModelWarmup_default_instance_; class ModelWarmup_Input; struct ModelWarmup_InputDefaultTypeInternal; extern ModelWarmup_InputDefaultTypeInternal _ModelWarmup_Input_default_instance_; class ModelWarmup_InputsEntry_DoNotUse; struct ModelWarmup_InputsEntry_DoNotUseDefaultTypeInternal; extern ModelWarmup_InputsEntry_DoNotUseDefaultTypeInternal _ModelWarmup_InputsEntry_DoNotUse_default_instance_; } // namespace inference PROTOBUF_NAMESPACE_OPEN template<> ::inference::BatchInput* Arena::CreateMaybeMessage<::inference::BatchInput>(Arena*); template<> ::inference::BatchOutput* Arena::CreateMaybeMessage<::inference::BatchOutput>(Arena*); template<> ::inference::ModelConfig* Arena::CreateMaybeMessage<::inference::ModelConfig>(Arena*); template<> ::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelConfig_CcModelFilenamesEntry_DoNotUse>(Arena*); template<> ::inference::ModelConfig_MetricTagsEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelConfig_MetricTagsEntry_DoNotUse>(Arena*); template<> ::inference::ModelConfig_ParametersEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelConfig_ParametersEntry_DoNotUse>(Arena*); template<> ::inference::ModelDynamicBatching* Arena::CreateMaybeMessage<::inference::ModelDynamicBatching>(Arena*); template<> ::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse>(Arena*); template<> ::inference::ModelEnsembling* Arena::CreateMaybeMessage<::inference::ModelEnsembling>(Arena*); template<> ::inference::ModelEnsembling_Step* Arena::CreateMaybeMessage<::inference::ModelEnsembling_Step>(Arena*); template<> ::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelEnsembling_Step_InputMapEntry_DoNotUse>(Arena*); template<> ::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelEnsembling_Step_OutputMapEntry_DoNotUse>(Arena*); template<> ::inference::ModelInput* Arena::CreateMaybeMessage<::inference::ModelInput>(Arena*); template<> ::inference::ModelInstanceGroup* Arena::CreateMaybeMessage<::inference::ModelInstanceGroup>(Arena*); template<> ::inference::ModelInstanceGroup_SecondaryDevice* Arena::CreateMaybeMessage<::inference::ModelInstanceGroup_SecondaryDevice>(Arena*); template<> ::inference::ModelOperations* Arena::CreateMaybeMessage<::inference::ModelOperations>(Arena*); template<> ::inference::ModelOptimizationPolicy* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy>(Arena*); template<> ::inference::ModelOptimizationPolicy_Cuda* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_Cuda>(Arena*); template<> ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_Cuda_GraphSpec>(Arena*); template<> ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse>(Arena*); template<> ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound>(Arena*); template<> ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse>(Arena*); template<> ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape>(Arena*); template<> ::inference::ModelOptimizationPolicy_ExecutionAccelerators* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_ExecutionAccelerators>(Arena*); template<> ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator>(Arena*); template<> ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse>(Arena*); template<> ::inference::ModelOptimizationPolicy_Graph* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_Graph>(Arena*); template<> ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* Arena::CreateMaybeMessage<::inference::ModelOptimizationPolicy_PinnedMemoryBuffer>(Arena*); template<> ::inference::ModelOutput* Arena::CreateMaybeMessage<::inference::ModelOutput>(Arena*); template<> ::inference::ModelParameter* Arena::CreateMaybeMessage<::inference::ModelParameter>(Arena*); template<> ::inference::ModelQueuePolicy* Arena::CreateMaybeMessage<::inference::ModelQueuePolicy>(Arena*); template<> ::inference::ModelRateLimiter* Arena::CreateMaybeMessage<::inference::ModelRateLimiter>(Arena*); template<> ::inference::ModelRateLimiter_Resource* Arena::CreateMaybeMessage<::inference::ModelRateLimiter_Resource>(Arena*); template<> ::inference::ModelRepositoryAgents* Arena::CreateMaybeMessage<::inference::ModelRepositoryAgents>(Arena*); template<> ::inference::ModelRepositoryAgents_Agent* Arena::CreateMaybeMessage<::inference::ModelRepositoryAgents_Agent>(Arena*); template<> ::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse>(Arena*); template<> ::inference::ModelResponseCache* Arena::CreateMaybeMessage<::inference::ModelResponseCache>(Arena*); template<> ::inference::ModelSequenceBatching* Arena::CreateMaybeMessage<::inference::ModelSequenceBatching>(Arena*); template<> ::inference::ModelSequenceBatching_Control* Arena::CreateMaybeMessage<::inference::ModelSequenceBatching_Control>(Arena*); template<> ::inference::ModelSequenceBatching_ControlInput* Arena::CreateMaybeMessage<::inference::ModelSequenceBatching_ControlInput>(Arena*); template<> ::inference::ModelSequenceBatching_InitialState* Arena::CreateMaybeMessage<::inference::ModelSequenceBatching_InitialState>(Arena*); template<> ::inference::ModelSequenceBatching_State* Arena::CreateMaybeMessage<::inference::ModelSequenceBatching_State>(Arena*); template<> ::inference::ModelSequenceBatching_StrategyDirect* Arena::CreateMaybeMessage<::inference::ModelSequenceBatching_StrategyDirect>(Arena*); template<> ::inference::ModelSequenceBatching_StrategyOldest* Arena::CreateMaybeMessage<::inference::ModelSequenceBatching_StrategyOldest>(Arena*); template<> ::inference::ModelTensorReshape* Arena::CreateMaybeMessage<::inference::ModelTensorReshape>(Arena*); template<> ::inference::ModelTransactionPolicy* Arena::CreateMaybeMessage<::inference::ModelTransactionPolicy>(Arena*); template<> ::inference::ModelVersionPolicy* Arena::CreateMaybeMessage<::inference::ModelVersionPolicy>(Arena*); template<> ::inference::ModelVersionPolicy_All* Arena::CreateMaybeMessage<::inference::ModelVersionPolicy_All>(Arena*); template<> ::inference::ModelVersionPolicy_Latest* Arena::CreateMaybeMessage<::inference::ModelVersionPolicy_Latest>(Arena*); template<> ::inference::ModelVersionPolicy_Specific* Arena::CreateMaybeMessage<::inference::ModelVersionPolicy_Specific>(Arena*); template<> ::inference::ModelWarmup* Arena::CreateMaybeMessage<::inference::ModelWarmup>(Arena*); template<> ::inference::ModelWarmup_Input* Arena::CreateMaybeMessage<::inference::ModelWarmup_Input>(Arena*); template<> ::inference::ModelWarmup_InputsEntry_DoNotUse* Arena::CreateMaybeMessage<::inference::ModelWarmup_InputsEntry_DoNotUse>(Arena*); PROTOBUF_NAMESPACE_CLOSE namespace inference { enum ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind : int { ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_KIND_NVDLA = 0, ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_IsValid(int value); constexpr ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_SecondaryDeviceKind_MIN = ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_KIND_NVDLA; constexpr ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_SecondaryDeviceKind_MAX = ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_KIND_NVDLA; constexpr int ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_SecondaryDeviceKind_ARRAYSIZE = ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_SecondaryDeviceKind_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_descriptor(); template inline const std::string& ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_descriptor(), enum_t_value); } inline bool ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_descriptor(), name, value); } enum ModelInstanceGroup_Kind : int { ModelInstanceGroup_Kind_KIND_AUTO = 0, ModelInstanceGroup_Kind_KIND_GPU = 1, ModelInstanceGroup_Kind_KIND_CPU = 2, ModelInstanceGroup_Kind_KIND_MODEL = 3, ModelInstanceGroup_Kind_ModelInstanceGroup_Kind_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), ModelInstanceGroup_Kind_ModelInstanceGroup_Kind_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool ModelInstanceGroup_Kind_IsValid(int value); constexpr ModelInstanceGroup_Kind ModelInstanceGroup_Kind_Kind_MIN = ModelInstanceGroup_Kind_KIND_AUTO; constexpr ModelInstanceGroup_Kind ModelInstanceGroup_Kind_Kind_MAX = ModelInstanceGroup_Kind_KIND_MODEL; constexpr int ModelInstanceGroup_Kind_Kind_ARRAYSIZE = ModelInstanceGroup_Kind_Kind_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelInstanceGroup_Kind_descriptor(); template inline const std::string& ModelInstanceGroup_Kind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function ModelInstanceGroup_Kind_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( ModelInstanceGroup_Kind_descriptor(), enum_t_value); } inline bool ModelInstanceGroup_Kind_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, ModelInstanceGroup_Kind* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( ModelInstanceGroup_Kind_descriptor(), name, value); } enum ModelInput_Format : int { ModelInput_Format_FORMAT_NONE = 0, ModelInput_Format_FORMAT_NHWC = 1, ModelInput_Format_FORMAT_NCHW = 2, ModelInput_Format_ModelInput_Format_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), ModelInput_Format_ModelInput_Format_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool ModelInput_Format_IsValid(int value); constexpr ModelInput_Format ModelInput_Format_Format_MIN = ModelInput_Format_FORMAT_NONE; constexpr ModelInput_Format ModelInput_Format_Format_MAX = ModelInput_Format_FORMAT_NCHW; constexpr int ModelInput_Format_Format_ARRAYSIZE = ModelInput_Format_Format_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelInput_Format_descriptor(); template inline const std::string& ModelInput_Format_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function ModelInput_Format_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( ModelInput_Format_descriptor(), enum_t_value); } inline bool ModelInput_Format_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, ModelInput_Format* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( ModelInput_Format_descriptor(), name, value); } enum BatchInput_Kind : int { BatchInput_Kind_BATCH_ELEMENT_COUNT = 0, BatchInput_Kind_BATCH_ACCUMULATED_ELEMENT_COUNT = 1, BatchInput_Kind_BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO = 2, BatchInput_Kind_BATCH_MAX_ELEMENT_COUNT_AS_SHAPE = 3, BatchInput_Kind_BATCH_ITEM_SHAPE = 4, BatchInput_Kind_BATCH_ITEM_SHAPE_FLATTEN = 5, BatchInput_Kind_BatchInput_Kind_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), BatchInput_Kind_BatchInput_Kind_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool BatchInput_Kind_IsValid(int value); constexpr BatchInput_Kind BatchInput_Kind_Kind_MIN = BatchInput_Kind_BATCH_ELEMENT_COUNT; constexpr BatchInput_Kind BatchInput_Kind_Kind_MAX = BatchInput_Kind_BATCH_ITEM_SHAPE_FLATTEN; constexpr int BatchInput_Kind_Kind_ARRAYSIZE = BatchInput_Kind_Kind_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* BatchInput_Kind_descriptor(); template inline const std::string& BatchInput_Kind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function BatchInput_Kind_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( BatchInput_Kind_descriptor(), enum_t_value); } inline bool BatchInput_Kind_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, BatchInput_Kind* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( BatchInput_Kind_descriptor(), name, value); } enum BatchOutput_Kind : int { BatchOutput_Kind_BATCH_SCATTER_WITH_INPUT_SHAPE = 0, BatchOutput_Kind_BatchOutput_Kind_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), BatchOutput_Kind_BatchOutput_Kind_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool BatchOutput_Kind_IsValid(int value); constexpr BatchOutput_Kind BatchOutput_Kind_Kind_MIN = BatchOutput_Kind_BATCH_SCATTER_WITH_INPUT_SHAPE; constexpr BatchOutput_Kind BatchOutput_Kind_Kind_MAX = BatchOutput_Kind_BATCH_SCATTER_WITH_INPUT_SHAPE; constexpr int BatchOutput_Kind_Kind_ARRAYSIZE = BatchOutput_Kind_Kind_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* BatchOutput_Kind_descriptor(); template inline const std::string& BatchOutput_Kind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function BatchOutput_Kind_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( BatchOutput_Kind_descriptor(), enum_t_value); } inline bool BatchOutput_Kind_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, BatchOutput_Kind* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( BatchOutput_Kind_descriptor(), name, value); } enum ModelOptimizationPolicy_ModelPriority : int { ModelOptimizationPolicy_ModelPriority_PRIORITY_DEFAULT = 0, ModelOptimizationPolicy_ModelPriority_PRIORITY_MAX = 1, ModelOptimizationPolicy_ModelPriority_PRIORITY_MIN = 2, ModelOptimizationPolicy_ModelPriority_ModelOptimizationPolicy_ModelPriority_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), ModelOptimizationPolicy_ModelPriority_ModelOptimizationPolicy_ModelPriority_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool ModelOptimizationPolicy_ModelPriority_IsValid(int value); constexpr ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy_ModelPriority_ModelPriority_MIN = ModelOptimizationPolicy_ModelPriority_PRIORITY_DEFAULT; constexpr ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy_ModelPriority_ModelPriority_MAX = ModelOptimizationPolicy_ModelPriority_PRIORITY_MIN; constexpr int ModelOptimizationPolicy_ModelPriority_ModelPriority_ARRAYSIZE = ModelOptimizationPolicy_ModelPriority_ModelPriority_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelOptimizationPolicy_ModelPriority_descriptor(); template inline const std::string& ModelOptimizationPolicy_ModelPriority_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function ModelOptimizationPolicy_ModelPriority_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( ModelOptimizationPolicy_ModelPriority_descriptor(), enum_t_value); } inline bool ModelOptimizationPolicy_ModelPriority_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, ModelOptimizationPolicy_ModelPriority* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( ModelOptimizationPolicy_ModelPriority_descriptor(), name, value); } enum ModelQueuePolicy_TimeoutAction : int { ModelQueuePolicy_TimeoutAction_REJECT = 0, ModelQueuePolicy_TimeoutAction_DELAY = 1, ModelQueuePolicy_TimeoutAction_ModelQueuePolicy_TimeoutAction_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), ModelQueuePolicy_TimeoutAction_ModelQueuePolicy_TimeoutAction_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool ModelQueuePolicy_TimeoutAction_IsValid(int value); constexpr ModelQueuePolicy_TimeoutAction ModelQueuePolicy_TimeoutAction_TimeoutAction_MIN = ModelQueuePolicy_TimeoutAction_REJECT; constexpr ModelQueuePolicy_TimeoutAction ModelQueuePolicy_TimeoutAction_TimeoutAction_MAX = ModelQueuePolicy_TimeoutAction_DELAY; constexpr int ModelQueuePolicy_TimeoutAction_TimeoutAction_ARRAYSIZE = ModelQueuePolicy_TimeoutAction_TimeoutAction_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelQueuePolicy_TimeoutAction_descriptor(); template inline const std::string& ModelQueuePolicy_TimeoutAction_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function ModelQueuePolicy_TimeoutAction_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( ModelQueuePolicy_TimeoutAction_descriptor(), enum_t_value); } inline bool ModelQueuePolicy_TimeoutAction_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, ModelQueuePolicy_TimeoutAction* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( ModelQueuePolicy_TimeoutAction_descriptor(), name, value); } enum ModelSequenceBatching_Control_Kind : int { ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_START = 0, ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_READY = 1, ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_END = 2, ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_CORRID = 3, ModelSequenceBatching_Control_Kind_ModelSequenceBatching_Control_Kind_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), ModelSequenceBatching_Control_Kind_ModelSequenceBatching_Control_Kind_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool ModelSequenceBatching_Control_Kind_IsValid(int value); constexpr ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control_Kind_Kind_MIN = ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_START; constexpr ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control_Kind_Kind_MAX = ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_CORRID; constexpr int ModelSequenceBatching_Control_Kind_Kind_ARRAYSIZE = ModelSequenceBatching_Control_Kind_Kind_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelSequenceBatching_Control_Kind_descriptor(); template inline const std::string& ModelSequenceBatching_Control_Kind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function ModelSequenceBatching_Control_Kind_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( ModelSequenceBatching_Control_Kind_descriptor(), enum_t_value); } inline bool ModelSequenceBatching_Control_Kind_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, ModelSequenceBatching_Control_Kind* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( ModelSequenceBatching_Control_Kind_descriptor(), name, value); } enum DataType : int { TYPE_INVALID = 0, TYPE_BOOL = 1, TYPE_UINT8 = 2, TYPE_UINT16 = 3, TYPE_UINT32 = 4, TYPE_UINT64 = 5, TYPE_INT8 = 6, TYPE_INT16 = 7, TYPE_INT32 = 8, TYPE_INT64 = 9, TYPE_FP16 = 10, TYPE_FP32 = 11, TYPE_FP64 = 12, TYPE_STRING = 13, TYPE_BF16 = 14, DataType_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits::min(), DataType_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits::max() }; bool DataType_IsValid(int value); constexpr DataType DataType_MIN = TYPE_INVALID; constexpr DataType DataType_MAX = TYPE_BF16; constexpr int DataType_ARRAYSIZE = DataType_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* DataType_descriptor(); template inline const std::string& DataType_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function DataType_Name."); return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum( DataType_descriptor(), enum_t_value); } inline bool DataType_Parse( ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, DataType* value) { return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum( DataType_descriptor(), name, value); } // =================================================================== class ModelRateLimiter_Resource final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelRateLimiter.Resource) */ { public: inline ModelRateLimiter_Resource() : ModelRateLimiter_Resource(nullptr) {} ~ModelRateLimiter_Resource() override; explicit PROTOBUF_CONSTEXPR ModelRateLimiter_Resource(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelRateLimiter_Resource(const ModelRateLimiter_Resource& from); ModelRateLimiter_Resource(ModelRateLimiter_Resource&& from) noexcept : ModelRateLimiter_Resource() { *this = ::std::move(from); } inline ModelRateLimiter_Resource& operator=(const ModelRateLimiter_Resource& from) { CopyFrom(from); return *this; } inline ModelRateLimiter_Resource& operator=(ModelRateLimiter_Resource&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelRateLimiter_Resource& default_instance() { return *internal_default_instance(); } static inline const ModelRateLimiter_Resource* internal_default_instance() { return reinterpret_cast( &_ModelRateLimiter_Resource_default_instance_); } static constexpr int kIndexInFileMessages = 0; friend void swap(ModelRateLimiter_Resource& a, ModelRateLimiter_Resource& b) { a.Swap(&b); } inline void Swap(ModelRateLimiter_Resource* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelRateLimiter_Resource* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelRateLimiter_Resource* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelRateLimiter_Resource& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelRateLimiter_Resource& from) { ModelRateLimiter_Resource::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelRateLimiter_Resource* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelRateLimiter.Resource"; } protected: explicit ModelRateLimiter_Resource(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kNameFieldNumber = 1, kGlobalFieldNumber = 2, kCountFieldNumber = 3, }; // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // bool global = 2; void clear_global(); bool global() const; void set_global(bool value); private: bool _internal_global() const; void _internal_set_global(bool value); public: // uint32 count = 3; void clear_count(); uint32_t count() const; void set_count(uint32_t value); private: uint32_t _internal_count() const; void _internal_set_count(uint32_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelRateLimiter.Resource) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; bool global_; uint32_t count_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelRateLimiter final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelRateLimiter) */ { public: inline ModelRateLimiter() : ModelRateLimiter(nullptr) {} ~ModelRateLimiter() override; explicit PROTOBUF_CONSTEXPR ModelRateLimiter(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelRateLimiter(const ModelRateLimiter& from); ModelRateLimiter(ModelRateLimiter&& from) noexcept : ModelRateLimiter() { *this = ::std::move(from); } inline ModelRateLimiter& operator=(const ModelRateLimiter& from) { CopyFrom(from); return *this; } inline ModelRateLimiter& operator=(ModelRateLimiter&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelRateLimiter& default_instance() { return *internal_default_instance(); } static inline const ModelRateLimiter* internal_default_instance() { return reinterpret_cast( &_ModelRateLimiter_default_instance_); } static constexpr int kIndexInFileMessages = 1; friend void swap(ModelRateLimiter& a, ModelRateLimiter& b) { a.Swap(&b); } inline void Swap(ModelRateLimiter* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelRateLimiter* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelRateLimiter* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelRateLimiter& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelRateLimiter& from) { ModelRateLimiter::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelRateLimiter* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelRateLimiter"; } protected: explicit ModelRateLimiter(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelRateLimiter_Resource Resource; // accessors ------------------------------------------------------- enum : int { kResourcesFieldNumber = 1, kPriorityFieldNumber = 2, }; // repeated .inference.ModelRateLimiter.Resource resources = 1; int resources_size() const; private: int _internal_resources_size() const; public: void clear_resources(); ::inference::ModelRateLimiter_Resource* mutable_resources(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRateLimiter_Resource >* mutable_resources(); private: const ::inference::ModelRateLimiter_Resource& _internal_resources(int index) const; ::inference::ModelRateLimiter_Resource* _internal_add_resources(); public: const ::inference::ModelRateLimiter_Resource& resources(int index) const; ::inference::ModelRateLimiter_Resource* add_resources(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRateLimiter_Resource >& resources() const; // uint32 priority = 2; void clear_priority(); uint32_t priority() const; void set_priority(uint32_t value); private: uint32_t _internal_priority() const; void _internal_set_priority(uint32_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelRateLimiter) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRateLimiter_Resource > resources_; uint32_t priority_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelInstanceGroup_SecondaryDevice final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelInstanceGroup.SecondaryDevice) */ { public: inline ModelInstanceGroup_SecondaryDevice() : ModelInstanceGroup_SecondaryDevice(nullptr) {} ~ModelInstanceGroup_SecondaryDevice() override; explicit PROTOBUF_CONSTEXPR ModelInstanceGroup_SecondaryDevice(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelInstanceGroup_SecondaryDevice(const ModelInstanceGroup_SecondaryDevice& from); ModelInstanceGroup_SecondaryDevice(ModelInstanceGroup_SecondaryDevice&& from) noexcept : ModelInstanceGroup_SecondaryDevice() { *this = ::std::move(from); } inline ModelInstanceGroup_SecondaryDevice& operator=(const ModelInstanceGroup_SecondaryDevice& from) { CopyFrom(from); return *this; } inline ModelInstanceGroup_SecondaryDevice& operator=(ModelInstanceGroup_SecondaryDevice&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelInstanceGroup_SecondaryDevice& default_instance() { return *internal_default_instance(); } static inline const ModelInstanceGroup_SecondaryDevice* internal_default_instance() { return reinterpret_cast( &_ModelInstanceGroup_SecondaryDevice_default_instance_); } static constexpr int kIndexInFileMessages = 2; friend void swap(ModelInstanceGroup_SecondaryDevice& a, ModelInstanceGroup_SecondaryDevice& b) { a.Swap(&b); } inline void Swap(ModelInstanceGroup_SecondaryDevice* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelInstanceGroup_SecondaryDevice* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelInstanceGroup_SecondaryDevice* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelInstanceGroup_SecondaryDevice& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelInstanceGroup_SecondaryDevice& from) { ModelInstanceGroup_SecondaryDevice::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelInstanceGroup_SecondaryDevice* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelInstanceGroup.SecondaryDevice"; } protected: explicit ModelInstanceGroup_SecondaryDevice(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind SecondaryDeviceKind; static constexpr SecondaryDeviceKind KIND_NVDLA = ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_KIND_NVDLA; static inline bool SecondaryDeviceKind_IsValid(int value) { return ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_IsValid(value); } static constexpr SecondaryDeviceKind SecondaryDeviceKind_MIN = ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_SecondaryDeviceKind_MIN; static constexpr SecondaryDeviceKind SecondaryDeviceKind_MAX = ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_SecondaryDeviceKind_MAX; static constexpr int SecondaryDeviceKind_ARRAYSIZE = ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_SecondaryDeviceKind_ARRAYSIZE; static inline const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* SecondaryDeviceKind_descriptor() { return ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_descriptor(); } template static inline const std::string& SecondaryDeviceKind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function SecondaryDeviceKind_Name."); return ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_Name(enum_t_value); } static inline bool SecondaryDeviceKind_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name, SecondaryDeviceKind* value) { return ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_Parse(name, value); } // accessors ------------------------------------------------------- enum : int { kDeviceIdFieldNumber = 2, kKindFieldNumber = 1, }; // int64 device_id = 2; void clear_device_id(); int64_t device_id() const; void set_device_id(int64_t value); private: int64_t _internal_device_id() const; void _internal_set_device_id(int64_t value); public: // .inference.ModelInstanceGroup.SecondaryDevice.SecondaryDeviceKind kind = 1; void clear_kind(); ::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind kind() const; void set_kind(::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind value); private: ::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind _internal_kind() const; void _internal_set_kind(::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind value); public: // @@protoc_insertion_point(class_scope:inference.ModelInstanceGroup.SecondaryDevice) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { int64_t device_id_; int kind_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelInstanceGroup final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelInstanceGroup) */ { public: inline ModelInstanceGroup() : ModelInstanceGroup(nullptr) {} ~ModelInstanceGroup() override; explicit PROTOBUF_CONSTEXPR ModelInstanceGroup(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelInstanceGroup(const ModelInstanceGroup& from); ModelInstanceGroup(ModelInstanceGroup&& from) noexcept : ModelInstanceGroup() { *this = ::std::move(from); } inline ModelInstanceGroup& operator=(const ModelInstanceGroup& from) { CopyFrom(from); return *this; } inline ModelInstanceGroup& operator=(ModelInstanceGroup&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelInstanceGroup& default_instance() { return *internal_default_instance(); } static inline const ModelInstanceGroup* internal_default_instance() { return reinterpret_cast( &_ModelInstanceGroup_default_instance_); } static constexpr int kIndexInFileMessages = 3; friend void swap(ModelInstanceGroup& a, ModelInstanceGroup& b) { a.Swap(&b); } inline void Swap(ModelInstanceGroup* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelInstanceGroup* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelInstanceGroup* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelInstanceGroup& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelInstanceGroup& from) { ModelInstanceGroup::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelInstanceGroup* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelInstanceGroup"; } protected: explicit ModelInstanceGroup(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelInstanceGroup_SecondaryDevice SecondaryDevice; typedef ModelInstanceGroup_Kind Kind; static constexpr Kind KIND_AUTO = ModelInstanceGroup_Kind_KIND_AUTO; static constexpr Kind KIND_GPU = ModelInstanceGroup_Kind_KIND_GPU; static constexpr Kind KIND_CPU = ModelInstanceGroup_Kind_KIND_CPU; static constexpr Kind KIND_MODEL = ModelInstanceGroup_Kind_KIND_MODEL; static inline bool Kind_IsValid(int value) { return ModelInstanceGroup_Kind_IsValid(value); } static constexpr Kind Kind_MIN = ModelInstanceGroup_Kind_Kind_MIN; static constexpr Kind Kind_MAX = ModelInstanceGroup_Kind_Kind_MAX; static constexpr int Kind_ARRAYSIZE = ModelInstanceGroup_Kind_Kind_ARRAYSIZE; static inline const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* Kind_descriptor() { return ModelInstanceGroup_Kind_descriptor(); } template static inline const std::string& Kind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function Kind_Name."); return ModelInstanceGroup_Kind_Name(enum_t_value); } static inline bool Kind_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name, Kind* value) { return ModelInstanceGroup_Kind_Parse(name, value); } // accessors ------------------------------------------------------- enum : int { kGpusFieldNumber = 3, kProfileFieldNumber = 5, kSecondaryDevicesFieldNumber = 8, kNameFieldNumber = 1, kHostPolicyFieldNumber = 9, kRateLimiterFieldNumber = 6, kCountFieldNumber = 2, kKindFieldNumber = 4, kPassiveFieldNumber = 7, }; // repeated int32 gpus = 3; int gpus_size() const; private: int _internal_gpus_size() const; public: void clear_gpus(); private: int32_t _internal_gpus(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& _internal_gpus() const; void _internal_add_gpus(int32_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* _internal_mutable_gpus(); public: int32_t gpus(int index) const; void set_gpus(int index, int32_t value); void add_gpus(int32_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& gpus() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* mutable_gpus(); // repeated string profile = 5; int profile_size() const; private: int _internal_profile_size() const; public: void clear_profile(); const std::string& profile(int index) const; std::string* mutable_profile(int index); void set_profile(int index, const std::string& value); void set_profile(int index, std::string&& value); void set_profile(int index, const char* value); void set_profile(int index, const char* value, size_t size); std::string* add_profile(); void add_profile(const std::string& value); void add_profile(std::string&& value); void add_profile(const char* value); void add_profile(const char* value, size_t size); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& profile() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_profile(); private: const std::string& _internal_profile(int index) const; std::string* _internal_add_profile(); public: // repeated .inference.ModelInstanceGroup.SecondaryDevice secondary_devices = 8; int secondary_devices_size() const; private: int _internal_secondary_devices_size() const; public: void clear_secondary_devices(); ::inference::ModelInstanceGroup_SecondaryDevice* mutable_secondary_devices(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup_SecondaryDevice >* mutable_secondary_devices(); private: const ::inference::ModelInstanceGroup_SecondaryDevice& _internal_secondary_devices(int index) const; ::inference::ModelInstanceGroup_SecondaryDevice* _internal_add_secondary_devices(); public: const ::inference::ModelInstanceGroup_SecondaryDevice& secondary_devices(int index) const; ::inference::ModelInstanceGroup_SecondaryDevice* add_secondary_devices(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup_SecondaryDevice >& secondary_devices() const; // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // string host_policy = 9; void clear_host_policy(); const std::string& host_policy() const; template void set_host_policy(ArgT0&& arg0, ArgT... args); std::string* mutable_host_policy(); PROTOBUF_NODISCARD std::string* release_host_policy(); void set_allocated_host_policy(std::string* host_policy); private: const std::string& _internal_host_policy() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_host_policy(const std::string& value); std::string* _internal_mutable_host_policy(); public: // .inference.ModelRateLimiter rate_limiter = 6; bool has_rate_limiter() const; private: bool _internal_has_rate_limiter() const; public: void clear_rate_limiter(); const ::inference::ModelRateLimiter& rate_limiter() const; PROTOBUF_NODISCARD ::inference::ModelRateLimiter* release_rate_limiter(); ::inference::ModelRateLimiter* mutable_rate_limiter(); void set_allocated_rate_limiter(::inference::ModelRateLimiter* rate_limiter); private: const ::inference::ModelRateLimiter& _internal_rate_limiter() const; ::inference::ModelRateLimiter* _internal_mutable_rate_limiter(); public: void unsafe_arena_set_allocated_rate_limiter( ::inference::ModelRateLimiter* rate_limiter); ::inference::ModelRateLimiter* unsafe_arena_release_rate_limiter(); // int32 count = 2; void clear_count(); int32_t count() const; void set_count(int32_t value); private: int32_t _internal_count() const; void _internal_set_count(int32_t value); public: // .inference.ModelInstanceGroup.Kind kind = 4; void clear_kind(); ::inference::ModelInstanceGroup_Kind kind() const; void set_kind(::inference::ModelInstanceGroup_Kind value); private: ::inference::ModelInstanceGroup_Kind _internal_kind() const; void _internal_set_kind(::inference::ModelInstanceGroup_Kind value); public: // bool passive = 7; void clear_passive(); bool passive() const; void set_passive(bool value); private: bool _internal_passive() const; void _internal_set_passive(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelInstanceGroup) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > gpus_; mutable std::atomic _gpus_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField profile_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup_SecondaryDevice > secondary_devices_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr host_policy_; ::inference::ModelRateLimiter* rate_limiter_; int32_t count_; int kind_; bool passive_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelTensorReshape final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelTensorReshape) */ { public: inline ModelTensorReshape() : ModelTensorReshape(nullptr) {} ~ModelTensorReshape() override; explicit PROTOBUF_CONSTEXPR ModelTensorReshape(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelTensorReshape(const ModelTensorReshape& from); ModelTensorReshape(ModelTensorReshape&& from) noexcept : ModelTensorReshape() { *this = ::std::move(from); } inline ModelTensorReshape& operator=(const ModelTensorReshape& from) { CopyFrom(from); return *this; } inline ModelTensorReshape& operator=(ModelTensorReshape&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelTensorReshape& default_instance() { return *internal_default_instance(); } static inline const ModelTensorReshape* internal_default_instance() { return reinterpret_cast( &_ModelTensorReshape_default_instance_); } static constexpr int kIndexInFileMessages = 4; friend void swap(ModelTensorReshape& a, ModelTensorReshape& b) { a.Swap(&b); } inline void Swap(ModelTensorReshape* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelTensorReshape* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelTensorReshape* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelTensorReshape& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelTensorReshape& from) { ModelTensorReshape::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelTensorReshape* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelTensorReshape"; } protected: explicit ModelTensorReshape(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kShapeFieldNumber = 1, }; // repeated int64 shape = 1; int shape_size() const; private: int _internal_shape_size() const; public: void clear_shape(); private: int64_t _internal_shape(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& _internal_shape() const; void _internal_add_shape(int64_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* _internal_mutable_shape(); public: int64_t shape(int index) const; void set_shape(int index, int64_t value); void add_shape(int64_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& shape() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* mutable_shape(); // @@protoc_insertion_point(class_scope:inference.ModelTensorReshape) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > shape_; mutable std::atomic _shape_cached_byte_size_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelInput final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelInput) */ { public: inline ModelInput() : ModelInput(nullptr) {} ~ModelInput() override; explicit PROTOBUF_CONSTEXPR ModelInput(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelInput(const ModelInput& from); ModelInput(ModelInput&& from) noexcept : ModelInput() { *this = ::std::move(from); } inline ModelInput& operator=(const ModelInput& from) { CopyFrom(from); return *this; } inline ModelInput& operator=(ModelInput&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelInput& default_instance() { return *internal_default_instance(); } static inline const ModelInput* internal_default_instance() { return reinterpret_cast( &_ModelInput_default_instance_); } static constexpr int kIndexInFileMessages = 5; friend void swap(ModelInput& a, ModelInput& b) { a.Swap(&b); } inline void Swap(ModelInput* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelInput* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelInput* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelInput& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelInput& from) { ModelInput::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelInput* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelInput"; } protected: explicit ModelInput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelInput_Format Format; static constexpr Format FORMAT_NONE = ModelInput_Format_FORMAT_NONE; static constexpr Format FORMAT_NHWC = ModelInput_Format_FORMAT_NHWC; static constexpr Format FORMAT_NCHW = ModelInput_Format_FORMAT_NCHW; static inline bool Format_IsValid(int value) { return ModelInput_Format_IsValid(value); } static constexpr Format Format_MIN = ModelInput_Format_Format_MIN; static constexpr Format Format_MAX = ModelInput_Format_Format_MAX; static constexpr int Format_ARRAYSIZE = ModelInput_Format_Format_ARRAYSIZE; static inline const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* Format_descriptor() { return ModelInput_Format_descriptor(); } template static inline const std::string& Format_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function Format_Name."); return ModelInput_Format_Name(enum_t_value); } static inline bool Format_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name, Format* value) { return ModelInput_Format_Parse(name, value); } // accessors ------------------------------------------------------- enum : int { kDimsFieldNumber = 4, kNameFieldNumber = 1, kReshapeFieldNumber = 5, kDataTypeFieldNumber = 2, kFormatFieldNumber = 3, kIsShapeTensorFieldNumber = 6, kAllowRaggedBatchFieldNumber = 7, kOptionalFieldNumber = 8, kIsNonLinearFormatIoFieldNumber = 9, }; // repeated int64 dims = 4; int dims_size() const; private: int _internal_dims_size() const; public: void clear_dims(); private: int64_t _internal_dims(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& _internal_dims() const; void _internal_add_dims(int64_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* _internal_mutable_dims(); public: int64_t dims(int index) const; void set_dims(int index, int64_t value); void add_dims(int64_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& dims() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* mutable_dims(); // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // .inference.ModelTensorReshape reshape = 5; bool has_reshape() const; private: bool _internal_has_reshape() const; public: void clear_reshape(); const ::inference::ModelTensorReshape& reshape() const; PROTOBUF_NODISCARD ::inference::ModelTensorReshape* release_reshape(); ::inference::ModelTensorReshape* mutable_reshape(); void set_allocated_reshape(::inference::ModelTensorReshape* reshape); private: const ::inference::ModelTensorReshape& _internal_reshape() const; ::inference::ModelTensorReshape* _internal_mutable_reshape(); public: void unsafe_arena_set_allocated_reshape( ::inference::ModelTensorReshape* reshape); ::inference::ModelTensorReshape* unsafe_arena_release_reshape(); // .inference.DataType data_type = 2; void clear_data_type(); ::inference::DataType data_type() const; void set_data_type(::inference::DataType value); private: ::inference::DataType _internal_data_type() const; void _internal_set_data_type(::inference::DataType value); public: // .inference.ModelInput.Format format = 3; void clear_format(); ::inference::ModelInput_Format format() const; void set_format(::inference::ModelInput_Format value); private: ::inference::ModelInput_Format _internal_format() const; void _internal_set_format(::inference::ModelInput_Format value); public: // bool is_shape_tensor = 6; void clear_is_shape_tensor(); bool is_shape_tensor() const; void set_is_shape_tensor(bool value); private: bool _internal_is_shape_tensor() const; void _internal_set_is_shape_tensor(bool value); public: // bool allow_ragged_batch = 7; void clear_allow_ragged_batch(); bool allow_ragged_batch() const; void set_allow_ragged_batch(bool value); private: bool _internal_allow_ragged_batch() const; void _internal_set_allow_ragged_batch(bool value); public: // bool optional = 8; void clear_optional(); bool optional() const; void set_optional(bool value); private: bool _internal_optional() const; void _internal_set_optional(bool value); public: // bool is_non_linear_format_io = 9; void clear_is_non_linear_format_io(); bool is_non_linear_format_io() const; void set_is_non_linear_format_io(bool value); private: bool _internal_is_non_linear_format_io() const; void _internal_set_is_non_linear_format_io(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelInput) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > dims_; mutable std::atomic _dims_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; ::inference::ModelTensorReshape* reshape_; int data_type_; int format_; bool is_shape_tensor_; bool allow_ragged_batch_; bool optional_; bool is_non_linear_format_io_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOutput final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOutput) */ { public: inline ModelOutput() : ModelOutput(nullptr) {} ~ModelOutput() override; explicit PROTOBUF_CONSTEXPR ModelOutput(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOutput(const ModelOutput& from); ModelOutput(ModelOutput&& from) noexcept : ModelOutput() { *this = ::std::move(from); } inline ModelOutput& operator=(const ModelOutput& from) { CopyFrom(from); return *this; } inline ModelOutput& operator=(ModelOutput&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOutput& default_instance() { return *internal_default_instance(); } static inline const ModelOutput* internal_default_instance() { return reinterpret_cast( &_ModelOutput_default_instance_); } static constexpr int kIndexInFileMessages = 6; friend void swap(ModelOutput& a, ModelOutput& b) { a.Swap(&b); } inline void Swap(ModelOutput* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOutput* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOutput* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOutput& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOutput& from) { ModelOutput::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOutput* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOutput"; } protected: explicit ModelOutput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kDimsFieldNumber = 3, kNameFieldNumber = 1, kLabelFilenameFieldNumber = 4, kReshapeFieldNumber = 5, kDataTypeFieldNumber = 2, kIsShapeTensorFieldNumber = 6, kIsNonLinearFormatIoFieldNumber = 7, }; // repeated int64 dims = 3; int dims_size() const; private: int _internal_dims_size() const; public: void clear_dims(); private: int64_t _internal_dims(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& _internal_dims() const; void _internal_add_dims(int64_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* _internal_mutable_dims(); public: int64_t dims(int index) const; void set_dims(int index, int64_t value); void add_dims(int64_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& dims() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* mutable_dims(); // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // string label_filename = 4; void clear_label_filename(); const std::string& label_filename() const; template void set_label_filename(ArgT0&& arg0, ArgT... args); std::string* mutable_label_filename(); PROTOBUF_NODISCARD std::string* release_label_filename(); void set_allocated_label_filename(std::string* label_filename); private: const std::string& _internal_label_filename() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_label_filename(const std::string& value); std::string* _internal_mutable_label_filename(); public: // .inference.ModelTensorReshape reshape = 5; bool has_reshape() const; private: bool _internal_has_reshape() const; public: void clear_reshape(); const ::inference::ModelTensorReshape& reshape() const; PROTOBUF_NODISCARD ::inference::ModelTensorReshape* release_reshape(); ::inference::ModelTensorReshape* mutable_reshape(); void set_allocated_reshape(::inference::ModelTensorReshape* reshape); private: const ::inference::ModelTensorReshape& _internal_reshape() const; ::inference::ModelTensorReshape* _internal_mutable_reshape(); public: void unsafe_arena_set_allocated_reshape( ::inference::ModelTensorReshape* reshape); ::inference::ModelTensorReshape* unsafe_arena_release_reshape(); // .inference.DataType data_type = 2; void clear_data_type(); ::inference::DataType data_type() const; void set_data_type(::inference::DataType value); private: ::inference::DataType _internal_data_type() const; void _internal_set_data_type(::inference::DataType value); public: // bool is_shape_tensor = 6; void clear_is_shape_tensor(); bool is_shape_tensor() const; void set_is_shape_tensor(bool value); private: bool _internal_is_shape_tensor() const; void _internal_set_is_shape_tensor(bool value); public: // bool is_non_linear_format_io = 7; void clear_is_non_linear_format_io(); bool is_non_linear_format_io() const; void set_is_non_linear_format_io(bool value); private: bool _internal_is_non_linear_format_io() const; void _internal_set_is_non_linear_format_io(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelOutput) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > dims_; mutable std::atomic _dims_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr label_filename_; ::inference::ModelTensorReshape* reshape_; int data_type_; bool is_shape_tensor_; bool is_non_linear_format_io_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class BatchInput final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.BatchInput) */ { public: inline BatchInput() : BatchInput(nullptr) {} ~BatchInput() override; explicit PROTOBUF_CONSTEXPR BatchInput(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); BatchInput(const BatchInput& from); BatchInput(BatchInput&& from) noexcept : BatchInput() { *this = ::std::move(from); } inline BatchInput& operator=(const BatchInput& from) { CopyFrom(from); return *this; } inline BatchInput& operator=(BatchInput&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const BatchInput& default_instance() { return *internal_default_instance(); } static inline const BatchInput* internal_default_instance() { return reinterpret_cast( &_BatchInput_default_instance_); } static constexpr int kIndexInFileMessages = 7; friend void swap(BatchInput& a, BatchInput& b) { a.Swap(&b); } inline void Swap(BatchInput* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(BatchInput* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- BatchInput* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const BatchInput& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const BatchInput& from) { BatchInput::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(BatchInput* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.BatchInput"; } protected: explicit BatchInput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef BatchInput_Kind Kind; static constexpr Kind BATCH_ELEMENT_COUNT = BatchInput_Kind_BATCH_ELEMENT_COUNT; static constexpr Kind BATCH_ACCUMULATED_ELEMENT_COUNT = BatchInput_Kind_BATCH_ACCUMULATED_ELEMENT_COUNT; static constexpr Kind BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO = BatchInput_Kind_BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO; static constexpr Kind BATCH_MAX_ELEMENT_COUNT_AS_SHAPE = BatchInput_Kind_BATCH_MAX_ELEMENT_COUNT_AS_SHAPE; static constexpr Kind BATCH_ITEM_SHAPE = BatchInput_Kind_BATCH_ITEM_SHAPE; static constexpr Kind BATCH_ITEM_SHAPE_FLATTEN = BatchInput_Kind_BATCH_ITEM_SHAPE_FLATTEN; static inline bool Kind_IsValid(int value) { return BatchInput_Kind_IsValid(value); } static constexpr Kind Kind_MIN = BatchInput_Kind_Kind_MIN; static constexpr Kind Kind_MAX = BatchInput_Kind_Kind_MAX; static constexpr int Kind_ARRAYSIZE = BatchInput_Kind_Kind_ARRAYSIZE; static inline const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* Kind_descriptor() { return BatchInput_Kind_descriptor(); } template static inline const std::string& Kind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function Kind_Name."); return BatchInput_Kind_Name(enum_t_value); } static inline bool Kind_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name, Kind* value) { return BatchInput_Kind_Parse(name, value); } // accessors ------------------------------------------------------- enum : int { kTargetNameFieldNumber = 2, kSourceInputFieldNumber = 4, kKindFieldNumber = 1, kDataTypeFieldNumber = 3, }; // repeated string target_name = 2; int target_name_size() const; private: int _internal_target_name_size() const; public: void clear_target_name(); const std::string& target_name(int index) const; std::string* mutable_target_name(int index); void set_target_name(int index, const std::string& value); void set_target_name(int index, std::string&& value); void set_target_name(int index, const char* value); void set_target_name(int index, const char* value, size_t size); std::string* add_target_name(); void add_target_name(const std::string& value); void add_target_name(std::string&& value); void add_target_name(const char* value); void add_target_name(const char* value, size_t size); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& target_name() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_target_name(); private: const std::string& _internal_target_name(int index) const; std::string* _internal_add_target_name(); public: // repeated string source_input = 4; int source_input_size() const; private: int _internal_source_input_size() const; public: void clear_source_input(); const std::string& source_input(int index) const; std::string* mutable_source_input(int index); void set_source_input(int index, const std::string& value); void set_source_input(int index, std::string&& value); void set_source_input(int index, const char* value); void set_source_input(int index, const char* value, size_t size); std::string* add_source_input(); void add_source_input(const std::string& value); void add_source_input(std::string&& value); void add_source_input(const char* value); void add_source_input(const char* value, size_t size); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& source_input() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_source_input(); private: const std::string& _internal_source_input(int index) const; std::string* _internal_add_source_input(); public: // .inference.BatchInput.Kind kind = 1; void clear_kind(); ::inference::BatchInput_Kind kind() const; void set_kind(::inference::BatchInput_Kind value); private: ::inference::BatchInput_Kind _internal_kind() const; void _internal_set_kind(::inference::BatchInput_Kind value); public: // .inference.DataType data_type = 3; void clear_data_type(); ::inference::DataType data_type() const; void set_data_type(::inference::DataType value); private: ::inference::DataType _internal_data_type() const; void _internal_set_data_type(::inference::DataType value); public: // @@protoc_insertion_point(class_scope:inference.BatchInput) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField target_name_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField source_input_; int kind_; int data_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class BatchOutput final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.BatchOutput) */ { public: inline BatchOutput() : BatchOutput(nullptr) {} ~BatchOutput() override; explicit PROTOBUF_CONSTEXPR BatchOutput(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); BatchOutput(const BatchOutput& from); BatchOutput(BatchOutput&& from) noexcept : BatchOutput() { *this = ::std::move(from); } inline BatchOutput& operator=(const BatchOutput& from) { CopyFrom(from); return *this; } inline BatchOutput& operator=(BatchOutput&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const BatchOutput& default_instance() { return *internal_default_instance(); } static inline const BatchOutput* internal_default_instance() { return reinterpret_cast( &_BatchOutput_default_instance_); } static constexpr int kIndexInFileMessages = 8; friend void swap(BatchOutput& a, BatchOutput& b) { a.Swap(&b); } inline void Swap(BatchOutput* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(BatchOutput* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- BatchOutput* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const BatchOutput& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const BatchOutput& from) { BatchOutput::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(BatchOutput* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.BatchOutput"; } protected: explicit BatchOutput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef BatchOutput_Kind Kind; static constexpr Kind BATCH_SCATTER_WITH_INPUT_SHAPE = BatchOutput_Kind_BATCH_SCATTER_WITH_INPUT_SHAPE; static inline bool Kind_IsValid(int value) { return BatchOutput_Kind_IsValid(value); } static constexpr Kind Kind_MIN = BatchOutput_Kind_Kind_MIN; static constexpr Kind Kind_MAX = BatchOutput_Kind_Kind_MAX; static constexpr int Kind_ARRAYSIZE = BatchOutput_Kind_Kind_ARRAYSIZE; static inline const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* Kind_descriptor() { return BatchOutput_Kind_descriptor(); } template static inline const std::string& Kind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function Kind_Name."); return BatchOutput_Kind_Name(enum_t_value); } static inline bool Kind_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name, Kind* value) { return BatchOutput_Kind_Parse(name, value); } // accessors ------------------------------------------------------- enum : int { kTargetNameFieldNumber = 1, kSourceInputFieldNumber = 3, kKindFieldNumber = 2, }; // repeated string target_name = 1; int target_name_size() const; private: int _internal_target_name_size() const; public: void clear_target_name(); const std::string& target_name(int index) const; std::string* mutable_target_name(int index); void set_target_name(int index, const std::string& value); void set_target_name(int index, std::string&& value); void set_target_name(int index, const char* value); void set_target_name(int index, const char* value, size_t size); std::string* add_target_name(); void add_target_name(const std::string& value); void add_target_name(std::string&& value); void add_target_name(const char* value); void add_target_name(const char* value, size_t size); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& target_name() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_target_name(); private: const std::string& _internal_target_name(int index) const; std::string* _internal_add_target_name(); public: // repeated string source_input = 3; int source_input_size() const; private: int _internal_source_input_size() const; public: void clear_source_input(); const std::string& source_input(int index) const; std::string* mutable_source_input(int index); void set_source_input(int index, const std::string& value); void set_source_input(int index, std::string&& value); void set_source_input(int index, const char* value); void set_source_input(int index, const char* value, size_t size); std::string* add_source_input(); void add_source_input(const std::string& value); void add_source_input(std::string&& value); void add_source_input(const char* value); void add_source_input(const char* value, size_t size); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& source_input() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_source_input(); private: const std::string& _internal_source_input(int index) const; std::string* _internal_add_source_input(); public: // .inference.BatchOutput.Kind kind = 2; void clear_kind(); ::inference::BatchOutput_Kind kind() const; void set_kind(::inference::BatchOutput_Kind value); private: ::inference::BatchOutput_Kind _internal_kind() const; void _internal_set_kind(::inference::BatchOutput_Kind value); public: // @@protoc_insertion_point(class_scope:inference.BatchOutput) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField target_name_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField source_input_; int kind_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelVersionPolicy_Latest final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelVersionPolicy.Latest) */ { public: inline ModelVersionPolicy_Latest() : ModelVersionPolicy_Latest(nullptr) {} ~ModelVersionPolicy_Latest() override; explicit PROTOBUF_CONSTEXPR ModelVersionPolicy_Latest(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelVersionPolicy_Latest(const ModelVersionPolicy_Latest& from); ModelVersionPolicy_Latest(ModelVersionPolicy_Latest&& from) noexcept : ModelVersionPolicy_Latest() { *this = ::std::move(from); } inline ModelVersionPolicy_Latest& operator=(const ModelVersionPolicy_Latest& from) { CopyFrom(from); return *this; } inline ModelVersionPolicy_Latest& operator=(ModelVersionPolicy_Latest&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelVersionPolicy_Latest& default_instance() { return *internal_default_instance(); } static inline const ModelVersionPolicy_Latest* internal_default_instance() { return reinterpret_cast( &_ModelVersionPolicy_Latest_default_instance_); } static constexpr int kIndexInFileMessages = 9; friend void swap(ModelVersionPolicy_Latest& a, ModelVersionPolicy_Latest& b) { a.Swap(&b); } inline void Swap(ModelVersionPolicy_Latest* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelVersionPolicy_Latest* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelVersionPolicy_Latest* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelVersionPolicy_Latest& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelVersionPolicy_Latest& from) { ModelVersionPolicy_Latest::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelVersionPolicy_Latest* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelVersionPolicy.Latest"; } protected: explicit ModelVersionPolicy_Latest(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kNumVersionsFieldNumber = 1, }; // uint32 num_versions = 1; void clear_num_versions(); uint32_t num_versions() const; void set_num_versions(uint32_t value); private: uint32_t _internal_num_versions() const; void _internal_set_num_versions(uint32_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelVersionPolicy.Latest) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { uint32_t num_versions_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelVersionPolicy_All final : public ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase /* @@protoc_insertion_point(class_definition:inference.ModelVersionPolicy.All) */ { public: inline ModelVersionPolicy_All() : ModelVersionPolicy_All(nullptr) {} explicit PROTOBUF_CONSTEXPR ModelVersionPolicy_All(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelVersionPolicy_All(const ModelVersionPolicy_All& from); ModelVersionPolicy_All(ModelVersionPolicy_All&& from) noexcept : ModelVersionPolicy_All() { *this = ::std::move(from); } inline ModelVersionPolicy_All& operator=(const ModelVersionPolicy_All& from) { CopyFrom(from); return *this; } inline ModelVersionPolicy_All& operator=(ModelVersionPolicy_All&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelVersionPolicy_All& default_instance() { return *internal_default_instance(); } static inline const ModelVersionPolicy_All* internal_default_instance() { return reinterpret_cast( &_ModelVersionPolicy_All_default_instance_); } static constexpr int kIndexInFileMessages = 10; friend void swap(ModelVersionPolicy_All& a, ModelVersionPolicy_All& b) { a.Swap(&b); } inline void Swap(ModelVersionPolicy_All* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelVersionPolicy_All* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelVersionPolicy_All* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase::CopyFrom; inline void CopyFrom(const ModelVersionPolicy_All& from) { ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase::CopyImpl(*this, from); } using ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase::MergeFrom; void MergeFrom(const ModelVersionPolicy_All& from) { ::PROTOBUF_NAMESPACE_ID::internal::ZeroFieldsBase::MergeImpl(*this, from); } public: private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelVersionPolicy.All"; } protected: explicit ModelVersionPolicy_All(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- // @@protoc_insertion_point(class_scope:inference.ModelVersionPolicy.All) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelVersionPolicy_Specific final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelVersionPolicy.Specific) */ { public: inline ModelVersionPolicy_Specific() : ModelVersionPolicy_Specific(nullptr) {} ~ModelVersionPolicy_Specific() override; explicit PROTOBUF_CONSTEXPR ModelVersionPolicy_Specific(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelVersionPolicy_Specific(const ModelVersionPolicy_Specific& from); ModelVersionPolicy_Specific(ModelVersionPolicy_Specific&& from) noexcept : ModelVersionPolicy_Specific() { *this = ::std::move(from); } inline ModelVersionPolicy_Specific& operator=(const ModelVersionPolicy_Specific& from) { CopyFrom(from); return *this; } inline ModelVersionPolicy_Specific& operator=(ModelVersionPolicy_Specific&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelVersionPolicy_Specific& default_instance() { return *internal_default_instance(); } static inline const ModelVersionPolicy_Specific* internal_default_instance() { return reinterpret_cast( &_ModelVersionPolicy_Specific_default_instance_); } static constexpr int kIndexInFileMessages = 11; friend void swap(ModelVersionPolicy_Specific& a, ModelVersionPolicy_Specific& b) { a.Swap(&b); } inline void Swap(ModelVersionPolicy_Specific* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelVersionPolicy_Specific* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelVersionPolicy_Specific* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelVersionPolicy_Specific& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelVersionPolicy_Specific& from) { ModelVersionPolicy_Specific::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelVersionPolicy_Specific* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelVersionPolicy.Specific"; } protected: explicit ModelVersionPolicy_Specific(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kVersionsFieldNumber = 1, }; // repeated int64 versions = 1; int versions_size() const; private: int _internal_versions_size() const; public: void clear_versions(); private: int64_t _internal_versions(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& _internal_versions() const; void _internal_add_versions(int64_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* _internal_mutable_versions(); public: int64_t versions(int index) const; void set_versions(int index, int64_t value); void add_versions(int64_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& versions() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* mutable_versions(); // @@protoc_insertion_point(class_scope:inference.ModelVersionPolicy.Specific) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > versions_; mutable std::atomic _versions_cached_byte_size_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelVersionPolicy final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelVersionPolicy) */ { public: inline ModelVersionPolicy() : ModelVersionPolicy(nullptr) {} ~ModelVersionPolicy() override; explicit PROTOBUF_CONSTEXPR ModelVersionPolicy(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelVersionPolicy(const ModelVersionPolicy& from); ModelVersionPolicy(ModelVersionPolicy&& from) noexcept : ModelVersionPolicy() { *this = ::std::move(from); } inline ModelVersionPolicy& operator=(const ModelVersionPolicy& from) { CopyFrom(from); return *this; } inline ModelVersionPolicy& operator=(ModelVersionPolicy&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelVersionPolicy& default_instance() { return *internal_default_instance(); } enum PolicyChoiceCase { kLatest = 1, kAll = 2, kSpecific = 3, POLICY_CHOICE_NOT_SET = 0, }; static inline const ModelVersionPolicy* internal_default_instance() { return reinterpret_cast( &_ModelVersionPolicy_default_instance_); } static constexpr int kIndexInFileMessages = 12; friend void swap(ModelVersionPolicy& a, ModelVersionPolicy& b) { a.Swap(&b); } inline void Swap(ModelVersionPolicy* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelVersionPolicy* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelVersionPolicy* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelVersionPolicy& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelVersionPolicy& from) { ModelVersionPolicy::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelVersionPolicy* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelVersionPolicy"; } protected: explicit ModelVersionPolicy(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelVersionPolicy_Latest Latest; typedef ModelVersionPolicy_All All; typedef ModelVersionPolicy_Specific Specific; // accessors ------------------------------------------------------- enum : int { kLatestFieldNumber = 1, kAllFieldNumber = 2, kSpecificFieldNumber = 3, }; // .inference.ModelVersionPolicy.Latest latest = 1; bool has_latest() const; private: bool _internal_has_latest() const; public: void clear_latest(); const ::inference::ModelVersionPolicy_Latest& latest() const; PROTOBUF_NODISCARD ::inference::ModelVersionPolicy_Latest* release_latest(); ::inference::ModelVersionPolicy_Latest* mutable_latest(); void set_allocated_latest(::inference::ModelVersionPolicy_Latest* latest); private: const ::inference::ModelVersionPolicy_Latest& _internal_latest() const; ::inference::ModelVersionPolicy_Latest* _internal_mutable_latest(); public: void unsafe_arena_set_allocated_latest( ::inference::ModelVersionPolicy_Latest* latest); ::inference::ModelVersionPolicy_Latest* unsafe_arena_release_latest(); // .inference.ModelVersionPolicy.All all = 2; bool has_all() const; private: bool _internal_has_all() const; public: void clear_all(); const ::inference::ModelVersionPolicy_All& all() const; PROTOBUF_NODISCARD ::inference::ModelVersionPolicy_All* release_all(); ::inference::ModelVersionPolicy_All* mutable_all(); void set_allocated_all(::inference::ModelVersionPolicy_All* all); private: const ::inference::ModelVersionPolicy_All& _internal_all() const; ::inference::ModelVersionPolicy_All* _internal_mutable_all(); public: void unsafe_arena_set_allocated_all( ::inference::ModelVersionPolicy_All* all); ::inference::ModelVersionPolicy_All* unsafe_arena_release_all(); // .inference.ModelVersionPolicy.Specific specific = 3; bool has_specific() const; private: bool _internal_has_specific() const; public: void clear_specific(); const ::inference::ModelVersionPolicy_Specific& specific() const; PROTOBUF_NODISCARD ::inference::ModelVersionPolicy_Specific* release_specific(); ::inference::ModelVersionPolicy_Specific* mutable_specific(); void set_allocated_specific(::inference::ModelVersionPolicy_Specific* specific); private: const ::inference::ModelVersionPolicy_Specific& _internal_specific() const; ::inference::ModelVersionPolicy_Specific* _internal_mutable_specific(); public: void unsafe_arena_set_allocated_specific( ::inference::ModelVersionPolicy_Specific* specific); ::inference::ModelVersionPolicy_Specific* unsafe_arena_release_specific(); void clear_policy_choice(); PolicyChoiceCase policy_choice_case() const; // @@protoc_insertion_point(class_scope:inference.ModelVersionPolicy) private: class _Internal; void set_has_latest(); void set_has_all(); void set_has_specific(); inline bool has_policy_choice() const; inline void clear_has_policy_choice(); template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { union PolicyChoiceUnion { constexpr PolicyChoiceUnion() : _constinit_{} {} ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized _constinit_; ::inference::ModelVersionPolicy_Latest* latest_; ::inference::ModelVersionPolicy_All* all_; ::inference::ModelVersionPolicy_Specific* specific_; } policy_choice_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; uint32_t _oneof_case_[1]; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_Graph final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy.Graph) */ { public: inline ModelOptimizationPolicy_Graph() : ModelOptimizationPolicy_Graph(nullptr) {} ~ModelOptimizationPolicy_Graph() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Graph(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy_Graph(const ModelOptimizationPolicy_Graph& from); ModelOptimizationPolicy_Graph(ModelOptimizationPolicy_Graph&& from) noexcept : ModelOptimizationPolicy_Graph() { *this = ::std::move(from); } inline ModelOptimizationPolicy_Graph& operator=(const ModelOptimizationPolicy_Graph& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy_Graph& operator=(ModelOptimizationPolicy_Graph&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy_Graph& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy_Graph* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_Graph_default_instance_); } static constexpr int kIndexInFileMessages = 13; friend void swap(ModelOptimizationPolicy_Graph& a, ModelOptimizationPolicy_Graph& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy_Graph* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy_Graph* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy_Graph* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy_Graph& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy_Graph& from) { ModelOptimizationPolicy_Graph::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy_Graph* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy.Graph"; } protected: explicit ModelOptimizationPolicy_Graph(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kLevelFieldNumber = 1, }; // int32 level = 1; void clear_level(); int32_t level() const; void set_level(int32_t value); private: int32_t _internal_level() const; void _internal_set_level(int32_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy.Graph) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { int32_t level_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_Cuda_GraphSpec_Shape final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) */ { public: inline ModelOptimizationPolicy_Cuda_GraphSpec_Shape() : ModelOptimizationPolicy_Cuda_GraphSpec_Shape(nullptr) {} ~ModelOptimizationPolicy_Cuda_GraphSpec_Shape() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_Shape(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy_Cuda_GraphSpec_Shape(const ModelOptimizationPolicy_Cuda_GraphSpec_Shape& from); ModelOptimizationPolicy_Cuda_GraphSpec_Shape(ModelOptimizationPolicy_Cuda_GraphSpec_Shape&& from) noexcept : ModelOptimizationPolicy_Cuda_GraphSpec_Shape() { *this = ::std::move(from); } inline ModelOptimizationPolicy_Cuda_GraphSpec_Shape& operator=(const ModelOptimizationPolicy_Cuda_GraphSpec_Shape& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy_Cuda_GraphSpec_Shape& operator=(ModelOptimizationPolicy_Cuda_GraphSpec_Shape&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy_Cuda_GraphSpec_Shape& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy_Cuda_GraphSpec_Shape* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_Cuda_GraphSpec_Shape_default_instance_); } static constexpr int kIndexInFileMessages = 14; friend void swap(ModelOptimizationPolicy_Cuda_GraphSpec_Shape& a, ModelOptimizationPolicy_Cuda_GraphSpec_Shape& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy_Cuda_GraphSpec_Shape* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy_Cuda_GraphSpec_Shape* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy_Cuda_GraphSpec_Shape* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy_Cuda_GraphSpec_Shape& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy_Cuda_GraphSpec_Shape& from) { ModelOptimizationPolicy_Cuda_GraphSpec_Shape::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy_Cuda_GraphSpec_Shape* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape"; } protected: explicit ModelOptimizationPolicy_Cuda_GraphSpec_Shape(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kDimFieldNumber = 1, }; // repeated int64 dim = 1; int dim_size() const; private: int _internal_dim_size() const; public: void clear_dim(); private: int64_t _internal_dim(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& _internal_dim() const; void _internal_add_dim(int64_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* _internal_mutable_dim(); public: int64_t dim(int index) const; void set_dim(int index, int64_t value); void add_dim(int64_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& dim() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* mutable_dim(); // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > dim_; mutable std::atomic _dim_cached_byte_size_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse& other); static const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound.InputEntry.key"); } static bool ValidateValue(void*) { return true; } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) */ { public: inline ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound() : ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound(nullptr) {} ~ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound(const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& from); ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound(ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound&& from) noexcept : ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound() { *this = ::std::move(from); } inline ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& operator=(const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& operator=(ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_default_instance_); } static constexpr int kIndexInFileMessages = 16; friend void swap(ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& a, ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& from) { ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound"; } protected: explicit ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); private: static void ArenaDtor(void* object); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kInputFieldNumber = 2, kBatchSizeFieldNumber = 1, }; // map input = 2; int input_size() const; private: int _internal_input_size() const; public: void clear_input(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >& _internal_input() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >* _internal_mutable_input(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >& input() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >* mutable_input(); // int32 batch_size = 1; void clear_batch_size(); int32_t batch_size() const; void set_batch_size(int32_t value); private: int32_t _internal_batch_size() const; void _internal_set_batch_size(int32_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_InputEntry_DoNotUse, std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_MESSAGE> input_; int32_t batch_size_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse& other); static const ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelOptimizationPolicy.Cuda.GraphSpec.InputEntry.key"); } static bool ValidateValue(void*) { return true; } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_Cuda_GraphSpec final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy.Cuda.GraphSpec) */ { public: inline ModelOptimizationPolicy_Cuda_GraphSpec() : ModelOptimizationPolicy_Cuda_GraphSpec(nullptr) {} ~ModelOptimizationPolicy_Cuda_GraphSpec() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda_GraphSpec(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy_Cuda_GraphSpec(const ModelOptimizationPolicy_Cuda_GraphSpec& from); ModelOptimizationPolicy_Cuda_GraphSpec(ModelOptimizationPolicy_Cuda_GraphSpec&& from) noexcept : ModelOptimizationPolicy_Cuda_GraphSpec() { *this = ::std::move(from); } inline ModelOptimizationPolicy_Cuda_GraphSpec& operator=(const ModelOptimizationPolicy_Cuda_GraphSpec& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy_Cuda_GraphSpec& operator=(ModelOptimizationPolicy_Cuda_GraphSpec&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy_Cuda_GraphSpec& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy_Cuda_GraphSpec* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_Cuda_GraphSpec_default_instance_); } static constexpr int kIndexInFileMessages = 18; friend void swap(ModelOptimizationPolicy_Cuda_GraphSpec& a, ModelOptimizationPolicy_Cuda_GraphSpec& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy_Cuda_GraphSpec* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy_Cuda_GraphSpec* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy_Cuda_GraphSpec* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy_Cuda_GraphSpec& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy_Cuda_GraphSpec& from) { ModelOptimizationPolicy_Cuda_GraphSpec::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy_Cuda_GraphSpec* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy.Cuda.GraphSpec"; } protected: explicit ModelOptimizationPolicy_Cuda_GraphSpec(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); private: static void ArenaDtor(void* object); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelOptimizationPolicy_Cuda_GraphSpec_Shape Shape; typedef ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound LowerBound; // accessors ------------------------------------------------------- enum : int { kInputFieldNumber = 2, kGraphLowerBoundFieldNumber = 3, kBatchSizeFieldNumber = 1, }; // map input = 2; int input_size() const; private: int _internal_input_size() const; public: void clear_input(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >& _internal_input() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >* _internal_mutable_input(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >& input() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >* mutable_input(); // .inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound graph_lower_bound = 3; bool has_graph_lower_bound() const; private: bool _internal_has_graph_lower_bound() const; public: void clear_graph_lower_bound(); const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& graph_lower_bound() const; PROTOBUF_NODISCARD ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* release_graph_lower_bound(); ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* mutable_graph_lower_bound(); void set_allocated_graph_lower_bound(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* graph_lower_bound); private: const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& _internal_graph_lower_bound() const; ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* _internal_mutable_graph_lower_bound(); public: void unsafe_arena_set_allocated_graph_lower_bound( ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* graph_lower_bound); ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* unsafe_arena_release_graph_lower_bound(); // int32 batch_size = 1; void clear_batch_size(); int32_t batch_size() const; void set_batch_size(int32_t value); private: int32_t _internal_batch_size() const; void _internal_set_batch_size(int32_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy.Cuda.GraphSpec) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelOptimizationPolicy_Cuda_GraphSpec_InputEntry_DoNotUse, std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_MESSAGE> input_; ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* graph_lower_bound_; int32_t batch_size_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_Cuda final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy.Cuda) */ { public: inline ModelOptimizationPolicy_Cuda() : ModelOptimizationPolicy_Cuda(nullptr) {} ~ModelOptimizationPolicy_Cuda() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_Cuda(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy_Cuda(const ModelOptimizationPolicy_Cuda& from); ModelOptimizationPolicy_Cuda(ModelOptimizationPolicy_Cuda&& from) noexcept : ModelOptimizationPolicy_Cuda() { *this = ::std::move(from); } inline ModelOptimizationPolicy_Cuda& operator=(const ModelOptimizationPolicy_Cuda& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy_Cuda& operator=(ModelOptimizationPolicy_Cuda&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy_Cuda& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy_Cuda* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_Cuda_default_instance_); } static constexpr int kIndexInFileMessages = 19; friend void swap(ModelOptimizationPolicy_Cuda& a, ModelOptimizationPolicy_Cuda& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy_Cuda* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy_Cuda* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy_Cuda* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy_Cuda& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy_Cuda& from) { ModelOptimizationPolicy_Cuda::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy_Cuda* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy.Cuda"; } protected: explicit ModelOptimizationPolicy_Cuda(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelOptimizationPolicy_Cuda_GraphSpec GraphSpec; // accessors ------------------------------------------------------- enum : int { kGraphSpecFieldNumber = 3, kGraphsFieldNumber = 1, kBusyWaitEventsFieldNumber = 2, kOutputCopyStreamFieldNumber = 4, }; // repeated .inference.ModelOptimizationPolicy.Cuda.GraphSpec graph_spec = 3; int graph_spec_size() const; private: int _internal_graph_spec_size() const; public: void clear_graph_spec(); ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* mutable_graph_spec(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec >* mutable_graph_spec(); private: const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec& _internal_graph_spec(int index) const; ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* _internal_add_graph_spec(); public: const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec& graph_spec(int index) const; ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* add_graph_spec(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec >& graph_spec() const; // bool graphs = 1; void clear_graphs(); bool graphs() const; void set_graphs(bool value); private: bool _internal_graphs() const; void _internal_set_graphs(bool value); public: // bool busy_wait_events = 2; void clear_busy_wait_events(); bool busy_wait_events() const; void set_busy_wait_events(bool value); private: bool _internal_busy_wait_events() const; void _internal_set_busy_wait_events(bool value); public: // bool output_copy_stream = 4; void clear_output_copy_stream(); bool output_copy_stream() const; void set_output_copy_stream(bool value); private: bool _internal_output_copy_stream() const; void _internal_set_output_copy_stream(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy.Cuda) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec > graph_spec_; bool graphs_; bool busy_wait_events_; bool output_copy_stream_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse& other); static const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.ParametersEntry.key"); } static bool ValidateValue(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.ParametersEntry.value"); } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_ExecutionAccelerators_Accelerator final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) */ { public: inline ModelOptimizationPolicy_ExecutionAccelerators_Accelerator() : ModelOptimizationPolicy_ExecutionAccelerators_Accelerator(nullptr) {} ~ModelOptimizationPolicy_ExecutionAccelerators_Accelerator() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAccelerators_Accelerator(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy_ExecutionAccelerators_Accelerator(const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& from); ModelOptimizationPolicy_ExecutionAccelerators_Accelerator(ModelOptimizationPolicy_ExecutionAccelerators_Accelerator&& from) noexcept : ModelOptimizationPolicy_ExecutionAccelerators_Accelerator() { *this = ::std::move(from); } inline ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& operator=(const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& operator=(ModelOptimizationPolicy_ExecutionAccelerators_Accelerator&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_default_instance_); } static constexpr int kIndexInFileMessages = 21; friend void swap(ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& a, ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& from) { ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator"; } protected: explicit ModelOptimizationPolicy_ExecutionAccelerators_Accelerator(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); private: static void ArenaDtor(void* object); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kParametersFieldNumber = 2, kNameFieldNumber = 1, }; // map parameters = 2; int parameters_size() const; private: int _internal_parameters_size() const; public: void clear_parameters(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& _internal_parameters() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* _internal_mutable_parameters(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& parameters() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* mutable_parameters(); // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelOptimizationPolicy_ExecutionAccelerators_Accelerator_ParametersEntry_DoNotUse, std::string, std::string, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> parameters_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_ExecutionAccelerators final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy.ExecutionAccelerators) */ { public: inline ModelOptimizationPolicy_ExecutionAccelerators() : ModelOptimizationPolicy_ExecutionAccelerators(nullptr) {} ~ModelOptimizationPolicy_ExecutionAccelerators() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_ExecutionAccelerators(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy_ExecutionAccelerators(const ModelOptimizationPolicy_ExecutionAccelerators& from); ModelOptimizationPolicy_ExecutionAccelerators(ModelOptimizationPolicy_ExecutionAccelerators&& from) noexcept : ModelOptimizationPolicy_ExecutionAccelerators() { *this = ::std::move(from); } inline ModelOptimizationPolicy_ExecutionAccelerators& operator=(const ModelOptimizationPolicy_ExecutionAccelerators& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy_ExecutionAccelerators& operator=(ModelOptimizationPolicy_ExecutionAccelerators&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy_ExecutionAccelerators& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy_ExecutionAccelerators* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_ExecutionAccelerators_default_instance_); } static constexpr int kIndexInFileMessages = 22; friend void swap(ModelOptimizationPolicy_ExecutionAccelerators& a, ModelOptimizationPolicy_ExecutionAccelerators& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy_ExecutionAccelerators* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy_ExecutionAccelerators* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy_ExecutionAccelerators* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy_ExecutionAccelerators& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy_ExecutionAccelerators& from) { ModelOptimizationPolicy_ExecutionAccelerators::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy_ExecutionAccelerators* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy.ExecutionAccelerators"; } protected: explicit ModelOptimizationPolicy_ExecutionAccelerators(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelOptimizationPolicy_ExecutionAccelerators_Accelerator Accelerator; // accessors ------------------------------------------------------- enum : int { kGpuExecutionAcceleratorFieldNumber = 1, kCpuExecutionAcceleratorFieldNumber = 2, }; // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator gpu_execution_accelerator = 1; int gpu_execution_accelerator_size() const; private: int _internal_gpu_execution_accelerator_size() const; public: void clear_gpu_execution_accelerator(); ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* mutable_gpu_execution_accelerator(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >* mutable_gpu_execution_accelerator(); private: const ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& _internal_gpu_execution_accelerator(int index) const; ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* _internal_add_gpu_execution_accelerator(); public: const ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& gpu_execution_accelerator(int index) const; ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* add_gpu_execution_accelerator(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >& gpu_execution_accelerator() const; // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator cpu_execution_accelerator = 2; int cpu_execution_accelerator_size() const; private: int _internal_cpu_execution_accelerator_size() const; public: void clear_cpu_execution_accelerator(); ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* mutable_cpu_execution_accelerator(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >* mutable_cpu_execution_accelerator(); private: const ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& _internal_cpu_execution_accelerator(int index) const; ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* _internal_add_cpu_execution_accelerator(); public: const ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& cpu_execution_accelerator(int index) const; ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* add_cpu_execution_accelerator(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >& cpu_execution_accelerator() const; // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy.ExecutionAccelerators) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator > gpu_execution_accelerator_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator > cpu_execution_accelerator_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy_PinnedMemoryBuffer final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) */ { public: inline ModelOptimizationPolicy_PinnedMemoryBuffer() : ModelOptimizationPolicy_PinnedMemoryBuffer(nullptr) {} ~ModelOptimizationPolicy_PinnedMemoryBuffer() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy_PinnedMemoryBuffer(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy_PinnedMemoryBuffer(const ModelOptimizationPolicy_PinnedMemoryBuffer& from); ModelOptimizationPolicy_PinnedMemoryBuffer(ModelOptimizationPolicy_PinnedMemoryBuffer&& from) noexcept : ModelOptimizationPolicy_PinnedMemoryBuffer() { *this = ::std::move(from); } inline ModelOptimizationPolicy_PinnedMemoryBuffer& operator=(const ModelOptimizationPolicy_PinnedMemoryBuffer& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy_PinnedMemoryBuffer& operator=(ModelOptimizationPolicy_PinnedMemoryBuffer&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy_PinnedMemoryBuffer& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy_PinnedMemoryBuffer* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_PinnedMemoryBuffer_default_instance_); } static constexpr int kIndexInFileMessages = 23; friend void swap(ModelOptimizationPolicy_PinnedMemoryBuffer& a, ModelOptimizationPolicy_PinnedMemoryBuffer& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy_PinnedMemoryBuffer* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy_PinnedMemoryBuffer* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy_PinnedMemoryBuffer* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy_PinnedMemoryBuffer& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy_PinnedMemoryBuffer& from) { ModelOptimizationPolicy_PinnedMemoryBuffer::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy_PinnedMemoryBuffer* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy.PinnedMemoryBuffer"; } protected: explicit ModelOptimizationPolicy_PinnedMemoryBuffer(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kEnableFieldNumber = 1, }; // bool enable = 1; void clear_enable(); bool enable() const; void set_enable(bool value); private: bool _internal_enable() const; void _internal_set_enable(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy.PinnedMemoryBuffer) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { bool enable_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOptimizationPolicy final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOptimizationPolicy) */ { public: inline ModelOptimizationPolicy() : ModelOptimizationPolicy(nullptr) {} ~ModelOptimizationPolicy() override; explicit PROTOBUF_CONSTEXPR ModelOptimizationPolicy(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOptimizationPolicy(const ModelOptimizationPolicy& from); ModelOptimizationPolicy(ModelOptimizationPolicy&& from) noexcept : ModelOptimizationPolicy() { *this = ::std::move(from); } inline ModelOptimizationPolicy& operator=(const ModelOptimizationPolicy& from) { CopyFrom(from); return *this; } inline ModelOptimizationPolicy& operator=(ModelOptimizationPolicy&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOptimizationPolicy& default_instance() { return *internal_default_instance(); } static inline const ModelOptimizationPolicy* internal_default_instance() { return reinterpret_cast( &_ModelOptimizationPolicy_default_instance_); } static constexpr int kIndexInFileMessages = 24; friend void swap(ModelOptimizationPolicy& a, ModelOptimizationPolicy& b) { a.Swap(&b); } inline void Swap(ModelOptimizationPolicy* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOptimizationPolicy* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOptimizationPolicy* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOptimizationPolicy& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOptimizationPolicy& from) { ModelOptimizationPolicy::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOptimizationPolicy* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOptimizationPolicy"; } protected: explicit ModelOptimizationPolicy(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelOptimizationPolicy_Graph Graph; typedef ModelOptimizationPolicy_Cuda Cuda; typedef ModelOptimizationPolicy_ExecutionAccelerators ExecutionAccelerators; typedef ModelOptimizationPolicy_PinnedMemoryBuffer PinnedMemoryBuffer; typedef ModelOptimizationPolicy_ModelPriority ModelPriority; static constexpr ModelPriority PRIORITY_DEFAULT = ModelOptimizationPolicy_ModelPriority_PRIORITY_DEFAULT; static constexpr ModelPriority PRIORITY_MAX = ModelOptimizationPolicy_ModelPriority_PRIORITY_MAX; static constexpr ModelPriority PRIORITY_MIN = ModelOptimizationPolicy_ModelPriority_PRIORITY_MIN; static inline bool ModelPriority_IsValid(int value) { return ModelOptimizationPolicy_ModelPriority_IsValid(value); } static constexpr ModelPriority ModelPriority_MIN = ModelOptimizationPolicy_ModelPriority_ModelPriority_MIN; static constexpr ModelPriority ModelPriority_MAX = ModelOptimizationPolicy_ModelPriority_ModelPriority_MAX; static constexpr int ModelPriority_ARRAYSIZE = ModelOptimizationPolicy_ModelPriority_ModelPriority_ARRAYSIZE; static inline const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ModelPriority_descriptor() { return ModelOptimizationPolicy_ModelPriority_descriptor(); } template static inline const std::string& ModelPriority_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function ModelPriority_Name."); return ModelOptimizationPolicy_ModelPriority_Name(enum_t_value); } static inline bool ModelPriority_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name, ModelPriority* value) { return ModelOptimizationPolicy_ModelPriority_Parse(name, value); } // accessors ------------------------------------------------------- enum : int { kGraphFieldNumber = 1, kCudaFieldNumber = 3, kExecutionAcceleratorsFieldNumber = 4, kInputPinnedMemoryFieldNumber = 5, kOutputPinnedMemoryFieldNumber = 6, kPriorityFieldNumber = 2, kGatherKernelBufferThresholdFieldNumber = 7, kEagerBatchingFieldNumber = 8, }; // .inference.ModelOptimizationPolicy.Graph graph = 1; bool has_graph() const; private: bool _internal_has_graph() const; public: void clear_graph(); const ::inference::ModelOptimizationPolicy_Graph& graph() const; PROTOBUF_NODISCARD ::inference::ModelOptimizationPolicy_Graph* release_graph(); ::inference::ModelOptimizationPolicy_Graph* mutable_graph(); void set_allocated_graph(::inference::ModelOptimizationPolicy_Graph* graph); private: const ::inference::ModelOptimizationPolicy_Graph& _internal_graph() const; ::inference::ModelOptimizationPolicy_Graph* _internal_mutable_graph(); public: void unsafe_arena_set_allocated_graph( ::inference::ModelOptimizationPolicy_Graph* graph); ::inference::ModelOptimizationPolicy_Graph* unsafe_arena_release_graph(); // .inference.ModelOptimizationPolicy.Cuda cuda = 3; bool has_cuda() const; private: bool _internal_has_cuda() const; public: void clear_cuda(); const ::inference::ModelOptimizationPolicy_Cuda& cuda() const; PROTOBUF_NODISCARD ::inference::ModelOptimizationPolicy_Cuda* release_cuda(); ::inference::ModelOptimizationPolicy_Cuda* mutable_cuda(); void set_allocated_cuda(::inference::ModelOptimizationPolicy_Cuda* cuda); private: const ::inference::ModelOptimizationPolicy_Cuda& _internal_cuda() const; ::inference::ModelOptimizationPolicy_Cuda* _internal_mutable_cuda(); public: void unsafe_arena_set_allocated_cuda( ::inference::ModelOptimizationPolicy_Cuda* cuda); ::inference::ModelOptimizationPolicy_Cuda* unsafe_arena_release_cuda(); // .inference.ModelOptimizationPolicy.ExecutionAccelerators execution_accelerators = 4; bool has_execution_accelerators() const; private: bool _internal_has_execution_accelerators() const; public: void clear_execution_accelerators(); const ::inference::ModelOptimizationPolicy_ExecutionAccelerators& execution_accelerators() const; PROTOBUF_NODISCARD ::inference::ModelOptimizationPolicy_ExecutionAccelerators* release_execution_accelerators(); ::inference::ModelOptimizationPolicy_ExecutionAccelerators* mutable_execution_accelerators(); void set_allocated_execution_accelerators(::inference::ModelOptimizationPolicy_ExecutionAccelerators* execution_accelerators); private: const ::inference::ModelOptimizationPolicy_ExecutionAccelerators& _internal_execution_accelerators() const; ::inference::ModelOptimizationPolicy_ExecutionAccelerators* _internal_mutable_execution_accelerators(); public: void unsafe_arena_set_allocated_execution_accelerators( ::inference::ModelOptimizationPolicy_ExecutionAccelerators* execution_accelerators); ::inference::ModelOptimizationPolicy_ExecutionAccelerators* unsafe_arena_release_execution_accelerators(); // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer input_pinned_memory = 5; bool has_input_pinned_memory() const; private: bool _internal_has_input_pinned_memory() const; public: void clear_input_pinned_memory(); const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& input_pinned_memory() const; PROTOBUF_NODISCARD ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* release_input_pinned_memory(); ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* mutable_input_pinned_memory(); void set_allocated_input_pinned_memory(::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* input_pinned_memory); private: const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& _internal_input_pinned_memory() const; ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* _internal_mutable_input_pinned_memory(); public: void unsafe_arena_set_allocated_input_pinned_memory( ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* input_pinned_memory); ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* unsafe_arena_release_input_pinned_memory(); // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer output_pinned_memory = 6; bool has_output_pinned_memory() const; private: bool _internal_has_output_pinned_memory() const; public: void clear_output_pinned_memory(); const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& output_pinned_memory() const; PROTOBUF_NODISCARD ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* release_output_pinned_memory(); ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* mutable_output_pinned_memory(); void set_allocated_output_pinned_memory(::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* output_pinned_memory); private: const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& _internal_output_pinned_memory() const; ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* _internal_mutable_output_pinned_memory(); public: void unsafe_arena_set_allocated_output_pinned_memory( ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* output_pinned_memory); ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* unsafe_arena_release_output_pinned_memory(); // .inference.ModelOptimizationPolicy.ModelPriority priority = 2; void clear_priority(); ::inference::ModelOptimizationPolicy_ModelPriority priority() const; void set_priority(::inference::ModelOptimizationPolicy_ModelPriority value); private: ::inference::ModelOptimizationPolicy_ModelPriority _internal_priority() const; void _internal_set_priority(::inference::ModelOptimizationPolicy_ModelPriority value); public: // uint32 gather_kernel_buffer_threshold = 7; void clear_gather_kernel_buffer_threshold(); uint32_t gather_kernel_buffer_threshold() const; void set_gather_kernel_buffer_threshold(uint32_t value); private: uint32_t _internal_gather_kernel_buffer_threshold() const; void _internal_set_gather_kernel_buffer_threshold(uint32_t value); public: // bool eager_batching = 8; void clear_eager_batching(); bool eager_batching() const; void set_eager_batching(bool value); private: bool _internal_eager_batching() const; void _internal_set_eager_batching(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelOptimizationPolicy) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::inference::ModelOptimizationPolicy_Graph* graph_; ::inference::ModelOptimizationPolicy_Cuda* cuda_; ::inference::ModelOptimizationPolicy_ExecutionAccelerators* execution_accelerators_; ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* input_pinned_memory_; ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* output_pinned_memory_; int priority_; uint32_t gather_kernel_buffer_threshold_; bool eager_batching_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelQueuePolicy final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelQueuePolicy) */ { public: inline ModelQueuePolicy() : ModelQueuePolicy(nullptr) {} ~ModelQueuePolicy() override; explicit PROTOBUF_CONSTEXPR ModelQueuePolicy(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelQueuePolicy(const ModelQueuePolicy& from); ModelQueuePolicy(ModelQueuePolicy&& from) noexcept : ModelQueuePolicy() { *this = ::std::move(from); } inline ModelQueuePolicy& operator=(const ModelQueuePolicy& from) { CopyFrom(from); return *this; } inline ModelQueuePolicy& operator=(ModelQueuePolicy&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelQueuePolicy& default_instance() { return *internal_default_instance(); } static inline const ModelQueuePolicy* internal_default_instance() { return reinterpret_cast( &_ModelQueuePolicy_default_instance_); } static constexpr int kIndexInFileMessages = 25; friend void swap(ModelQueuePolicy& a, ModelQueuePolicy& b) { a.Swap(&b); } inline void Swap(ModelQueuePolicy* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelQueuePolicy* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelQueuePolicy* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelQueuePolicy& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelQueuePolicy& from) { ModelQueuePolicy::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelQueuePolicy* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelQueuePolicy"; } protected: explicit ModelQueuePolicy(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelQueuePolicy_TimeoutAction TimeoutAction; static constexpr TimeoutAction REJECT = ModelQueuePolicy_TimeoutAction_REJECT; static constexpr TimeoutAction DELAY = ModelQueuePolicy_TimeoutAction_DELAY; static inline bool TimeoutAction_IsValid(int value) { return ModelQueuePolicy_TimeoutAction_IsValid(value); } static constexpr TimeoutAction TimeoutAction_MIN = ModelQueuePolicy_TimeoutAction_TimeoutAction_MIN; static constexpr TimeoutAction TimeoutAction_MAX = ModelQueuePolicy_TimeoutAction_TimeoutAction_MAX; static constexpr int TimeoutAction_ARRAYSIZE = ModelQueuePolicy_TimeoutAction_TimeoutAction_ARRAYSIZE; static inline const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* TimeoutAction_descriptor() { return ModelQueuePolicy_TimeoutAction_descriptor(); } template static inline const std::string& TimeoutAction_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function TimeoutAction_Name."); return ModelQueuePolicy_TimeoutAction_Name(enum_t_value); } static inline bool TimeoutAction_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name, TimeoutAction* value) { return ModelQueuePolicy_TimeoutAction_Parse(name, value); } // accessors ------------------------------------------------------- enum : int { kDefaultTimeoutMicrosecondsFieldNumber = 2, kTimeoutActionFieldNumber = 1, kAllowTimeoutOverrideFieldNumber = 3, kMaxQueueSizeFieldNumber = 4, }; // uint64 default_timeout_microseconds = 2; void clear_default_timeout_microseconds(); uint64_t default_timeout_microseconds() const; void set_default_timeout_microseconds(uint64_t value); private: uint64_t _internal_default_timeout_microseconds() const; void _internal_set_default_timeout_microseconds(uint64_t value); public: // .inference.ModelQueuePolicy.TimeoutAction timeout_action = 1; void clear_timeout_action(); ::inference::ModelQueuePolicy_TimeoutAction timeout_action() const; void set_timeout_action(::inference::ModelQueuePolicy_TimeoutAction value); private: ::inference::ModelQueuePolicy_TimeoutAction _internal_timeout_action() const; void _internal_set_timeout_action(::inference::ModelQueuePolicy_TimeoutAction value); public: // bool allow_timeout_override = 3; void clear_allow_timeout_override(); bool allow_timeout_override() const; void set_allow_timeout_override(bool value); private: bool _internal_allow_timeout_override() const; void _internal_set_allow_timeout_override(bool value); public: // uint32 max_queue_size = 4; void clear_max_queue_size(); uint32_t max_queue_size() const; void set_max_queue_size(uint32_t value); private: uint32_t _internal_max_queue_size() const; void _internal_set_max_queue_size(uint32_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelQueuePolicy) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { uint64_t default_timeout_microseconds_; int timeout_action_; bool allow_timeout_override_; uint32_t max_queue_size_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse& other); static const ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse_default_instance_); } static bool ValidateKey(void*) { return true; } static bool ValidateValue(void*) { return true; } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelDynamicBatching final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelDynamicBatching) */ { public: inline ModelDynamicBatching() : ModelDynamicBatching(nullptr) {} ~ModelDynamicBatching() override; explicit PROTOBUF_CONSTEXPR ModelDynamicBatching(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelDynamicBatching(const ModelDynamicBatching& from); ModelDynamicBatching(ModelDynamicBatching&& from) noexcept : ModelDynamicBatching() { *this = ::std::move(from); } inline ModelDynamicBatching& operator=(const ModelDynamicBatching& from) { CopyFrom(from); return *this; } inline ModelDynamicBatching& operator=(ModelDynamicBatching&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelDynamicBatching& default_instance() { return *internal_default_instance(); } static inline const ModelDynamicBatching* internal_default_instance() { return reinterpret_cast( &_ModelDynamicBatching_default_instance_); } static constexpr int kIndexInFileMessages = 27; friend void swap(ModelDynamicBatching& a, ModelDynamicBatching& b) { a.Swap(&b); } inline void Swap(ModelDynamicBatching* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelDynamicBatching* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelDynamicBatching* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelDynamicBatching& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelDynamicBatching& from) { ModelDynamicBatching::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelDynamicBatching* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelDynamicBatching"; } protected: explicit ModelDynamicBatching(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); private: static void ArenaDtor(void* object); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kPreferredBatchSizeFieldNumber = 1, kPriorityQueuePolicyFieldNumber = 7, kDefaultQueuePolicyFieldNumber = 6, kMaxQueueDelayMicrosecondsFieldNumber = 2, kPriorityLevelsFieldNumber = 4, kDefaultPriorityLevelFieldNumber = 5, kPreserveOrderingFieldNumber = 3, }; // repeated int32 preferred_batch_size = 1; int preferred_batch_size_size() const; private: int _internal_preferred_batch_size_size() const; public: void clear_preferred_batch_size(); private: int32_t _internal_preferred_batch_size(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& _internal_preferred_batch_size() const; void _internal_add_preferred_batch_size(int32_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* _internal_mutable_preferred_batch_size(); public: int32_t preferred_batch_size(int index) const; void set_preferred_batch_size(int index, int32_t value); void add_preferred_batch_size(int32_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& preferred_batch_size() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* mutable_preferred_batch_size(); // map priority_queue_policy = 7; int priority_queue_policy_size() const; private: int _internal_priority_queue_policy_size() const; public: void clear_priority_queue_policy(); private: const ::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >& _internal_priority_queue_policy() const; ::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >* _internal_mutable_priority_queue_policy(); public: const ::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >& priority_queue_policy() const; ::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >* mutable_priority_queue_policy(); // .inference.ModelQueuePolicy default_queue_policy = 6; bool has_default_queue_policy() const; private: bool _internal_has_default_queue_policy() const; public: void clear_default_queue_policy(); const ::inference::ModelQueuePolicy& default_queue_policy() const; PROTOBUF_NODISCARD ::inference::ModelQueuePolicy* release_default_queue_policy(); ::inference::ModelQueuePolicy* mutable_default_queue_policy(); void set_allocated_default_queue_policy(::inference::ModelQueuePolicy* default_queue_policy); private: const ::inference::ModelQueuePolicy& _internal_default_queue_policy() const; ::inference::ModelQueuePolicy* _internal_mutable_default_queue_policy(); public: void unsafe_arena_set_allocated_default_queue_policy( ::inference::ModelQueuePolicy* default_queue_policy); ::inference::ModelQueuePolicy* unsafe_arena_release_default_queue_policy(); // uint64 max_queue_delay_microseconds = 2; void clear_max_queue_delay_microseconds(); uint64_t max_queue_delay_microseconds() const; void set_max_queue_delay_microseconds(uint64_t value); private: uint64_t _internal_max_queue_delay_microseconds() const; void _internal_set_max_queue_delay_microseconds(uint64_t value); public: // uint64 priority_levels = 4; void clear_priority_levels(); uint64_t priority_levels() const; void set_priority_levels(uint64_t value); private: uint64_t _internal_priority_levels() const; void _internal_set_priority_levels(uint64_t value); public: // uint64 default_priority_level = 5; void clear_default_priority_level(); uint64_t default_priority_level() const; void set_default_priority_level(uint64_t value); private: uint64_t _internal_default_priority_level() const; void _internal_set_default_priority_level(uint64_t value); public: // bool preserve_ordering = 3; void clear_preserve_ordering(); bool preserve_ordering() const; void set_preserve_ordering(bool value); private: bool _internal_preserve_ordering() const; void _internal_set_preserve_ordering(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelDynamicBatching) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > preferred_batch_size_; mutable std::atomic _preferred_batch_size_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelDynamicBatching_PriorityQueuePolicyEntry_DoNotUse, uint64_t, ::inference::ModelQueuePolicy, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_UINT64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_MESSAGE> priority_queue_policy_; ::inference::ModelQueuePolicy* default_queue_policy_; uint64_t max_queue_delay_microseconds_; uint64_t priority_levels_; uint64_t default_priority_level_; bool preserve_ordering_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelSequenceBatching_Control final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelSequenceBatching.Control) */ { public: inline ModelSequenceBatching_Control() : ModelSequenceBatching_Control(nullptr) {} ~ModelSequenceBatching_Control() override; explicit PROTOBUF_CONSTEXPR ModelSequenceBatching_Control(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelSequenceBatching_Control(const ModelSequenceBatching_Control& from); ModelSequenceBatching_Control(ModelSequenceBatching_Control&& from) noexcept : ModelSequenceBatching_Control() { *this = ::std::move(from); } inline ModelSequenceBatching_Control& operator=(const ModelSequenceBatching_Control& from) { CopyFrom(from); return *this; } inline ModelSequenceBatching_Control& operator=(ModelSequenceBatching_Control&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelSequenceBatching_Control& default_instance() { return *internal_default_instance(); } static inline const ModelSequenceBatching_Control* internal_default_instance() { return reinterpret_cast( &_ModelSequenceBatching_Control_default_instance_); } static constexpr int kIndexInFileMessages = 28; friend void swap(ModelSequenceBatching_Control& a, ModelSequenceBatching_Control& b) { a.Swap(&b); } inline void Swap(ModelSequenceBatching_Control* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelSequenceBatching_Control* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelSequenceBatching_Control* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelSequenceBatching_Control& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelSequenceBatching_Control& from) { ModelSequenceBatching_Control::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelSequenceBatching_Control* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelSequenceBatching.Control"; } protected: explicit ModelSequenceBatching_Control(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelSequenceBatching_Control_Kind Kind; static constexpr Kind CONTROL_SEQUENCE_START = ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_START; static constexpr Kind CONTROL_SEQUENCE_READY = ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_READY; static constexpr Kind CONTROL_SEQUENCE_END = ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_END; static constexpr Kind CONTROL_SEQUENCE_CORRID = ModelSequenceBatching_Control_Kind_CONTROL_SEQUENCE_CORRID; static inline bool Kind_IsValid(int value) { return ModelSequenceBatching_Control_Kind_IsValid(value); } static constexpr Kind Kind_MIN = ModelSequenceBatching_Control_Kind_Kind_MIN; static constexpr Kind Kind_MAX = ModelSequenceBatching_Control_Kind_Kind_MAX; static constexpr int Kind_ARRAYSIZE = ModelSequenceBatching_Control_Kind_Kind_ARRAYSIZE; static inline const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* Kind_descriptor() { return ModelSequenceBatching_Control_Kind_descriptor(); } template static inline const std::string& Kind_Name(T enum_t_value) { static_assert(::std::is_same::value || ::std::is_integral::value, "Incorrect type passed to function Kind_Name."); return ModelSequenceBatching_Control_Kind_Name(enum_t_value); } static inline bool Kind_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name, Kind* value) { return ModelSequenceBatching_Control_Kind_Parse(name, value); } // accessors ------------------------------------------------------- enum : int { kInt32FalseTrueFieldNumber = 2, kFp32FalseTrueFieldNumber = 3, kBoolFalseTrueFieldNumber = 5, kKindFieldNumber = 1, kDataTypeFieldNumber = 4, }; // repeated int32 int32_false_true = 2; int int32_false_true_size() const; private: int _internal_int32_false_true_size() const; public: void clear_int32_false_true(); private: int32_t _internal_int32_false_true(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& _internal_int32_false_true() const; void _internal_add_int32_false_true(int32_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* _internal_mutable_int32_false_true(); public: int32_t int32_false_true(int index) const; void set_int32_false_true(int index, int32_t value); void add_int32_false_true(int32_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& int32_false_true() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* mutable_int32_false_true(); // repeated float fp32_false_true = 3; int fp32_false_true_size() const; private: int _internal_fp32_false_true_size() const; public: void clear_fp32_false_true(); private: float _internal_fp32_false_true(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& _internal_fp32_false_true() const; void _internal_add_fp32_false_true(float value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* _internal_mutable_fp32_false_true(); public: float fp32_false_true(int index) const; void set_fp32_false_true(int index, float value); void add_fp32_false_true(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& fp32_false_true() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* mutable_fp32_false_true(); // repeated bool bool_false_true = 5; int bool_false_true_size() const; private: int _internal_bool_false_true_size() const; public: void clear_bool_false_true(); private: bool _internal_bool_false_true(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool >& _internal_bool_false_true() const; void _internal_add_bool_false_true(bool value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool >* _internal_mutable_bool_false_true(); public: bool bool_false_true(int index) const; void set_bool_false_true(int index, bool value); void add_bool_false_true(bool value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool >& bool_false_true() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool >* mutable_bool_false_true(); // .inference.ModelSequenceBatching.Control.Kind kind = 1; void clear_kind(); ::inference::ModelSequenceBatching_Control_Kind kind() const; void set_kind(::inference::ModelSequenceBatching_Control_Kind value); private: ::inference::ModelSequenceBatching_Control_Kind _internal_kind() const; void _internal_set_kind(::inference::ModelSequenceBatching_Control_Kind value); public: // .inference.DataType data_type = 4; void clear_data_type(); ::inference::DataType data_type() const; void set_data_type(::inference::DataType value); private: ::inference::DataType _internal_data_type() const; void _internal_set_data_type(::inference::DataType value); public: // @@protoc_insertion_point(class_scope:inference.ModelSequenceBatching.Control) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > int32_false_true_; mutable std::atomic _int32_false_true_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float > fp32_false_true_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool > bool_false_true_; int kind_; int data_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelSequenceBatching_ControlInput final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelSequenceBatching.ControlInput) */ { public: inline ModelSequenceBatching_ControlInput() : ModelSequenceBatching_ControlInput(nullptr) {} ~ModelSequenceBatching_ControlInput() override; explicit PROTOBUF_CONSTEXPR ModelSequenceBatching_ControlInput(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelSequenceBatching_ControlInput(const ModelSequenceBatching_ControlInput& from); ModelSequenceBatching_ControlInput(ModelSequenceBatching_ControlInput&& from) noexcept : ModelSequenceBatching_ControlInput() { *this = ::std::move(from); } inline ModelSequenceBatching_ControlInput& operator=(const ModelSequenceBatching_ControlInput& from) { CopyFrom(from); return *this; } inline ModelSequenceBatching_ControlInput& operator=(ModelSequenceBatching_ControlInput&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelSequenceBatching_ControlInput& default_instance() { return *internal_default_instance(); } static inline const ModelSequenceBatching_ControlInput* internal_default_instance() { return reinterpret_cast( &_ModelSequenceBatching_ControlInput_default_instance_); } static constexpr int kIndexInFileMessages = 29; friend void swap(ModelSequenceBatching_ControlInput& a, ModelSequenceBatching_ControlInput& b) { a.Swap(&b); } inline void Swap(ModelSequenceBatching_ControlInput* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelSequenceBatching_ControlInput* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelSequenceBatching_ControlInput* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelSequenceBatching_ControlInput& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelSequenceBatching_ControlInput& from) { ModelSequenceBatching_ControlInput::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelSequenceBatching_ControlInput* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelSequenceBatching.ControlInput"; } protected: explicit ModelSequenceBatching_ControlInput(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kControlFieldNumber = 2, kNameFieldNumber = 1, }; // repeated .inference.ModelSequenceBatching.Control control = 2; int control_size() const; private: int _internal_control_size() const; public: void clear_control(); ::inference::ModelSequenceBatching_Control* mutable_control(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_Control >* mutable_control(); private: const ::inference::ModelSequenceBatching_Control& _internal_control(int index) const; ::inference::ModelSequenceBatching_Control* _internal_add_control(); public: const ::inference::ModelSequenceBatching_Control& control(int index) const; ::inference::ModelSequenceBatching_Control* add_control(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_Control >& control() const; // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // @@protoc_insertion_point(class_scope:inference.ModelSequenceBatching.ControlInput) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_Control > control_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelSequenceBatching_InitialState final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelSequenceBatching.InitialState) */ { public: inline ModelSequenceBatching_InitialState() : ModelSequenceBatching_InitialState(nullptr) {} ~ModelSequenceBatching_InitialState() override; explicit PROTOBUF_CONSTEXPR ModelSequenceBatching_InitialState(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelSequenceBatching_InitialState(const ModelSequenceBatching_InitialState& from); ModelSequenceBatching_InitialState(ModelSequenceBatching_InitialState&& from) noexcept : ModelSequenceBatching_InitialState() { *this = ::std::move(from); } inline ModelSequenceBatching_InitialState& operator=(const ModelSequenceBatching_InitialState& from) { CopyFrom(from); return *this; } inline ModelSequenceBatching_InitialState& operator=(ModelSequenceBatching_InitialState&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelSequenceBatching_InitialState& default_instance() { return *internal_default_instance(); } enum StateDataCase { kZeroData = 3, kDataFile = 4, STATE_DATA_NOT_SET = 0, }; static inline const ModelSequenceBatching_InitialState* internal_default_instance() { return reinterpret_cast( &_ModelSequenceBatching_InitialState_default_instance_); } static constexpr int kIndexInFileMessages = 30; friend void swap(ModelSequenceBatching_InitialState& a, ModelSequenceBatching_InitialState& b) { a.Swap(&b); } inline void Swap(ModelSequenceBatching_InitialState* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelSequenceBatching_InitialState* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelSequenceBatching_InitialState* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelSequenceBatching_InitialState& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelSequenceBatching_InitialState& from) { ModelSequenceBatching_InitialState::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelSequenceBatching_InitialState* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelSequenceBatching.InitialState"; } protected: explicit ModelSequenceBatching_InitialState(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kDimsFieldNumber = 2, kNameFieldNumber = 5, kDataTypeFieldNumber = 1, kZeroDataFieldNumber = 3, kDataFileFieldNumber = 4, }; // repeated int64 dims = 2; int dims_size() const; private: int _internal_dims_size() const; public: void clear_dims(); private: int64_t _internal_dims(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& _internal_dims() const; void _internal_add_dims(int64_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* _internal_mutable_dims(); public: int64_t dims(int index) const; void set_dims(int index, int64_t value); void add_dims(int64_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& dims() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* mutable_dims(); // string name = 5; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // .inference.DataType data_type = 1; void clear_data_type(); ::inference::DataType data_type() const; void set_data_type(::inference::DataType value); private: ::inference::DataType _internal_data_type() const; void _internal_set_data_type(::inference::DataType value); public: // bool zero_data = 3; bool has_zero_data() const; private: bool _internal_has_zero_data() const; public: void clear_zero_data(); bool zero_data() const; void set_zero_data(bool value); private: bool _internal_zero_data() const; void _internal_set_zero_data(bool value); public: // string data_file = 4; bool has_data_file() const; private: bool _internal_has_data_file() const; public: void clear_data_file(); const std::string& data_file() const; template void set_data_file(ArgT0&& arg0, ArgT... args); std::string* mutable_data_file(); PROTOBUF_NODISCARD std::string* release_data_file(); void set_allocated_data_file(std::string* data_file); private: const std::string& _internal_data_file() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_data_file(const std::string& value); std::string* _internal_mutable_data_file(); public: void clear_state_data(); StateDataCase state_data_case() const; // @@protoc_insertion_point(class_scope:inference.ModelSequenceBatching.InitialState) private: class _Internal; void set_has_zero_data(); void set_has_data_file(); inline bool has_state_data() const; inline void clear_has_state_data(); template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > dims_; mutable std::atomic _dims_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; int data_type_; union StateDataUnion { constexpr StateDataUnion() : _constinit_{} {} ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized _constinit_; bool zero_data_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr data_file_; } state_data_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; uint32_t _oneof_case_[1]; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelSequenceBatching_State final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelSequenceBatching.State) */ { public: inline ModelSequenceBatching_State() : ModelSequenceBatching_State(nullptr) {} ~ModelSequenceBatching_State() override; explicit PROTOBUF_CONSTEXPR ModelSequenceBatching_State(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelSequenceBatching_State(const ModelSequenceBatching_State& from); ModelSequenceBatching_State(ModelSequenceBatching_State&& from) noexcept : ModelSequenceBatching_State() { *this = ::std::move(from); } inline ModelSequenceBatching_State& operator=(const ModelSequenceBatching_State& from) { CopyFrom(from); return *this; } inline ModelSequenceBatching_State& operator=(ModelSequenceBatching_State&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelSequenceBatching_State& default_instance() { return *internal_default_instance(); } static inline const ModelSequenceBatching_State* internal_default_instance() { return reinterpret_cast( &_ModelSequenceBatching_State_default_instance_); } static constexpr int kIndexInFileMessages = 31; friend void swap(ModelSequenceBatching_State& a, ModelSequenceBatching_State& b) { a.Swap(&b); } inline void Swap(ModelSequenceBatching_State* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelSequenceBatching_State* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelSequenceBatching_State* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelSequenceBatching_State& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelSequenceBatching_State& from) { ModelSequenceBatching_State::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelSequenceBatching_State* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelSequenceBatching.State"; } protected: explicit ModelSequenceBatching_State(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kDimsFieldNumber = 4, kInitialStateFieldNumber = 5, kInputNameFieldNumber = 1, kOutputNameFieldNumber = 2, kDataTypeFieldNumber = 3, kUseSameBufferForInputOutputFieldNumber = 6, kUseGrowableMemoryFieldNumber = 7, }; // repeated int64 dims = 4; int dims_size() const; private: int _internal_dims_size() const; public: void clear_dims(); private: int64_t _internal_dims(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& _internal_dims() const; void _internal_add_dims(int64_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* _internal_mutable_dims(); public: int64_t dims(int index) const; void set_dims(int index, int64_t value); void add_dims(int64_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& dims() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* mutable_dims(); // repeated .inference.ModelSequenceBatching.InitialState initial_state = 5; int initial_state_size() const; private: int _internal_initial_state_size() const; public: void clear_initial_state(); ::inference::ModelSequenceBatching_InitialState* mutable_initial_state(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_InitialState >* mutable_initial_state(); private: const ::inference::ModelSequenceBatching_InitialState& _internal_initial_state(int index) const; ::inference::ModelSequenceBatching_InitialState* _internal_add_initial_state(); public: const ::inference::ModelSequenceBatching_InitialState& initial_state(int index) const; ::inference::ModelSequenceBatching_InitialState* add_initial_state(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_InitialState >& initial_state() const; // string input_name = 1; void clear_input_name(); const std::string& input_name() const; template void set_input_name(ArgT0&& arg0, ArgT... args); std::string* mutable_input_name(); PROTOBUF_NODISCARD std::string* release_input_name(); void set_allocated_input_name(std::string* input_name); private: const std::string& _internal_input_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_input_name(const std::string& value); std::string* _internal_mutable_input_name(); public: // string output_name = 2; void clear_output_name(); const std::string& output_name() const; template void set_output_name(ArgT0&& arg0, ArgT... args); std::string* mutable_output_name(); PROTOBUF_NODISCARD std::string* release_output_name(); void set_allocated_output_name(std::string* output_name); private: const std::string& _internal_output_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_output_name(const std::string& value); std::string* _internal_mutable_output_name(); public: // .inference.DataType data_type = 3; void clear_data_type(); ::inference::DataType data_type() const; void set_data_type(::inference::DataType value); private: ::inference::DataType _internal_data_type() const; void _internal_set_data_type(::inference::DataType value); public: // bool use_same_buffer_for_input_output = 6; void clear_use_same_buffer_for_input_output(); bool use_same_buffer_for_input_output() const; void set_use_same_buffer_for_input_output(bool value); private: bool _internal_use_same_buffer_for_input_output() const; void _internal_set_use_same_buffer_for_input_output(bool value); public: // bool use_growable_memory = 7; void clear_use_growable_memory(); bool use_growable_memory() const; void set_use_growable_memory(bool value); private: bool _internal_use_growable_memory() const; void _internal_set_use_growable_memory(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelSequenceBatching.State) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > dims_; mutable std::atomic _dims_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_InitialState > initial_state_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr input_name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr output_name_; int data_type_; bool use_same_buffer_for_input_output_; bool use_growable_memory_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelSequenceBatching_StrategyDirect final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelSequenceBatching.StrategyDirect) */ { public: inline ModelSequenceBatching_StrategyDirect() : ModelSequenceBatching_StrategyDirect(nullptr) {} ~ModelSequenceBatching_StrategyDirect() override; explicit PROTOBUF_CONSTEXPR ModelSequenceBatching_StrategyDirect(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelSequenceBatching_StrategyDirect(const ModelSequenceBatching_StrategyDirect& from); ModelSequenceBatching_StrategyDirect(ModelSequenceBatching_StrategyDirect&& from) noexcept : ModelSequenceBatching_StrategyDirect() { *this = ::std::move(from); } inline ModelSequenceBatching_StrategyDirect& operator=(const ModelSequenceBatching_StrategyDirect& from) { CopyFrom(from); return *this; } inline ModelSequenceBatching_StrategyDirect& operator=(ModelSequenceBatching_StrategyDirect&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelSequenceBatching_StrategyDirect& default_instance() { return *internal_default_instance(); } static inline const ModelSequenceBatching_StrategyDirect* internal_default_instance() { return reinterpret_cast( &_ModelSequenceBatching_StrategyDirect_default_instance_); } static constexpr int kIndexInFileMessages = 32; friend void swap(ModelSequenceBatching_StrategyDirect& a, ModelSequenceBatching_StrategyDirect& b) { a.Swap(&b); } inline void Swap(ModelSequenceBatching_StrategyDirect* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelSequenceBatching_StrategyDirect* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelSequenceBatching_StrategyDirect* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelSequenceBatching_StrategyDirect& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelSequenceBatching_StrategyDirect& from) { ModelSequenceBatching_StrategyDirect::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelSequenceBatching_StrategyDirect* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelSequenceBatching.StrategyDirect"; } protected: explicit ModelSequenceBatching_StrategyDirect(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kMaxQueueDelayMicrosecondsFieldNumber = 1, kMinimumSlotUtilizationFieldNumber = 2, }; // uint64 max_queue_delay_microseconds = 1; void clear_max_queue_delay_microseconds(); uint64_t max_queue_delay_microseconds() const; void set_max_queue_delay_microseconds(uint64_t value); private: uint64_t _internal_max_queue_delay_microseconds() const; void _internal_set_max_queue_delay_microseconds(uint64_t value); public: // float minimum_slot_utilization = 2; void clear_minimum_slot_utilization(); float minimum_slot_utilization() const; void set_minimum_slot_utilization(float value); private: float _internal_minimum_slot_utilization() const; void _internal_set_minimum_slot_utilization(float value); public: // @@protoc_insertion_point(class_scope:inference.ModelSequenceBatching.StrategyDirect) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { uint64_t max_queue_delay_microseconds_; float minimum_slot_utilization_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelSequenceBatching_StrategyOldest final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelSequenceBatching.StrategyOldest) */ { public: inline ModelSequenceBatching_StrategyOldest() : ModelSequenceBatching_StrategyOldest(nullptr) {} ~ModelSequenceBatching_StrategyOldest() override; explicit PROTOBUF_CONSTEXPR ModelSequenceBatching_StrategyOldest(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelSequenceBatching_StrategyOldest(const ModelSequenceBatching_StrategyOldest& from); ModelSequenceBatching_StrategyOldest(ModelSequenceBatching_StrategyOldest&& from) noexcept : ModelSequenceBatching_StrategyOldest() { *this = ::std::move(from); } inline ModelSequenceBatching_StrategyOldest& operator=(const ModelSequenceBatching_StrategyOldest& from) { CopyFrom(from); return *this; } inline ModelSequenceBatching_StrategyOldest& operator=(ModelSequenceBatching_StrategyOldest&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelSequenceBatching_StrategyOldest& default_instance() { return *internal_default_instance(); } static inline const ModelSequenceBatching_StrategyOldest* internal_default_instance() { return reinterpret_cast( &_ModelSequenceBatching_StrategyOldest_default_instance_); } static constexpr int kIndexInFileMessages = 33; friend void swap(ModelSequenceBatching_StrategyOldest& a, ModelSequenceBatching_StrategyOldest& b) { a.Swap(&b); } inline void Swap(ModelSequenceBatching_StrategyOldest* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelSequenceBatching_StrategyOldest* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelSequenceBatching_StrategyOldest* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelSequenceBatching_StrategyOldest& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelSequenceBatching_StrategyOldest& from) { ModelSequenceBatching_StrategyOldest::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelSequenceBatching_StrategyOldest* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelSequenceBatching.StrategyOldest"; } protected: explicit ModelSequenceBatching_StrategyOldest(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kPreferredBatchSizeFieldNumber = 2, kMaxCandidateSequencesFieldNumber = 1, kPreserveOrderingFieldNumber = 4, kMaxQueueDelayMicrosecondsFieldNumber = 3, }; // repeated int32 preferred_batch_size = 2; int preferred_batch_size_size() const; private: int _internal_preferred_batch_size_size() const; public: void clear_preferred_batch_size(); private: int32_t _internal_preferred_batch_size(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& _internal_preferred_batch_size() const; void _internal_add_preferred_batch_size(int32_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* _internal_mutable_preferred_batch_size(); public: int32_t preferred_batch_size(int index) const; void set_preferred_batch_size(int index, int32_t value); void add_preferred_batch_size(int32_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& preferred_batch_size() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* mutable_preferred_batch_size(); // int32 max_candidate_sequences = 1; void clear_max_candidate_sequences(); int32_t max_candidate_sequences() const; void set_max_candidate_sequences(int32_t value); private: int32_t _internal_max_candidate_sequences() const; void _internal_set_max_candidate_sequences(int32_t value); public: // bool preserve_ordering = 4; void clear_preserve_ordering(); bool preserve_ordering() const; void set_preserve_ordering(bool value); private: bool _internal_preserve_ordering() const; void _internal_set_preserve_ordering(bool value); public: // uint64 max_queue_delay_microseconds = 3; void clear_max_queue_delay_microseconds(); uint64_t max_queue_delay_microseconds() const; void set_max_queue_delay_microseconds(uint64_t value); private: uint64_t _internal_max_queue_delay_microseconds() const; void _internal_set_max_queue_delay_microseconds(uint64_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelSequenceBatching.StrategyOldest) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > preferred_batch_size_; mutable std::atomic _preferred_batch_size_cached_byte_size_; int32_t max_candidate_sequences_; bool preserve_ordering_; uint64_t max_queue_delay_microseconds_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelSequenceBatching final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelSequenceBatching) */ { public: inline ModelSequenceBatching() : ModelSequenceBatching(nullptr) {} ~ModelSequenceBatching() override; explicit PROTOBUF_CONSTEXPR ModelSequenceBatching(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelSequenceBatching(const ModelSequenceBatching& from); ModelSequenceBatching(ModelSequenceBatching&& from) noexcept : ModelSequenceBatching() { *this = ::std::move(from); } inline ModelSequenceBatching& operator=(const ModelSequenceBatching& from) { CopyFrom(from); return *this; } inline ModelSequenceBatching& operator=(ModelSequenceBatching&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelSequenceBatching& default_instance() { return *internal_default_instance(); } enum StrategyChoiceCase { kDirect = 3, kOldest = 4, STRATEGY_CHOICE_NOT_SET = 0, }; static inline const ModelSequenceBatching* internal_default_instance() { return reinterpret_cast( &_ModelSequenceBatching_default_instance_); } static constexpr int kIndexInFileMessages = 34; friend void swap(ModelSequenceBatching& a, ModelSequenceBatching& b) { a.Swap(&b); } inline void Swap(ModelSequenceBatching* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelSequenceBatching* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelSequenceBatching* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelSequenceBatching& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelSequenceBatching& from) { ModelSequenceBatching::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelSequenceBatching* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelSequenceBatching"; } protected: explicit ModelSequenceBatching(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelSequenceBatching_Control Control; typedef ModelSequenceBatching_ControlInput ControlInput; typedef ModelSequenceBatching_InitialState InitialState; typedef ModelSequenceBatching_State State; typedef ModelSequenceBatching_StrategyDirect StrategyDirect; typedef ModelSequenceBatching_StrategyOldest StrategyOldest; // accessors ------------------------------------------------------- enum : int { kControlInputFieldNumber = 2, kStateFieldNumber = 5, kMaxSequenceIdleMicrosecondsFieldNumber = 1, kIterativeSequenceFieldNumber = 6, kDirectFieldNumber = 3, kOldestFieldNumber = 4, }; // repeated .inference.ModelSequenceBatching.ControlInput control_input = 2; int control_input_size() const; private: int _internal_control_input_size() const; public: void clear_control_input(); ::inference::ModelSequenceBatching_ControlInput* mutable_control_input(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_ControlInput >* mutable_control_input(); private: const ::inference::ModelSequenceBatching_ControlInput& _internal_control_input(int index) const; ::inference::ModelSequenceBatching_ControlInput* _internal_add_control_input(); public: const ::inference::ModelSequenceBatching_ControlInput& control_input(int index) const; ::inference::ModelSequenceBatching_ControlInput* add_control_input(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_ControlInput >& control_input() const; // repeated .inference.ModelSequenceBatching.State state = 5; int state_size() const; private: int _internal_state_size() const; public: void clear_state(); ::inference::ModelSequenceBatching_State* mutable_state(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_State >* mutable_state(); private: const ::inference::ModelSequenceBatching_State& _internal_state(int index) const; ::inference::ModelSequenceBatching_State* _internal_add_state(); public: const ::inference::ModelSequenceBatching_State& state(int index) const; ::inference::ModelSequenceBatching_State* add_state(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_State >& state() const; // uint64 max_sequence_idle_microseconds = 1; void clear_max_sequence_idle_microseconds(); uint64_t max_sequence_idle_microseconds() const; void set_max_sequence_idle_microseconds(uint64_t value); private: uint64_t _internal_max_sequence_idle_microseconds() const; void _internal_set_max_sequence_idle_microseconds(uint64_t value); public: // bool iterative_sequence = 6; void clear_iterative_sequence(); bool iterative_sequence() const; void set_iterative_sequence(bool value); private: bool _internal_iterative_sequence() const; void _internal_set_iterative_sequence(bool value); public: // .inference.ModelSequenceBatching.StrategyDirect direct = 3; bool has_direct() const; private: bool _internal_has_direct() const; public: void clear_direct(); const ::inference::ModelSequenceBatching_StrategyDirect& direct() const; PROTOBUF_NODISCARD ::inference::ModelSequenceBatching_StrategyDirect* release_direct(); ::inference::ModelSequenceBatching_StrategyDirect* mutable_direct(); void set_allocated_direct(::inference::ModelSequenceBatching_StrategyDirect* direct); private: const ::inference::ModelSequenceBatching_StrategyDirect& _internal_direct() const; ::inference::ModelSequenceBatching_StrategyDirect* _internal_mutable_direct(); public: void unsafe_arena_set_allocated_direct( ::inference::ModelSequenceBatching_StrategyDirect* direct); ::inference::ModelSequenceBatching_StrategyDirect* unsafe_arena_release_direct(); // .inference.ModelSequenceBatching.StrategyOldest oldest = 4; bool has_oldest() const; private: bool _internal_has_oldest() const; public: void clear_oldest(); const ::inference::ModelSequenceBatching_StrategyOldest& oldest() const; PROTOBUF_NODISCARD ::inference::ModelSequenceBatching_StrategyOldest* release_oldest(); ::inference::ModelSequenceBatching_StrategyOldest* mutable_oldest(); void set_allocated_oldest(::inference::ModelSequenceBatching_StrategyOldest* oldest); private: const ::inference::ModelSequenceBatching_StrategyOldest& _internal_oldest() const; ::inference::ModelSequenceBatching_StrategyOldest* _internal_mutable_oldest(); public: void unsafe_arena_set_allocated_oldest( ::inference::ModelSequenceBatching_StrategyOldest* oldest); ::inference::ModelSequenceBatching_StrategyOldest* unsafe_arena_release_oldest(); void clear_strategy_choice(); StrategyChoiceCase strategy_choice_case() const; // @@protoc_insertion_point(class_scope:inference.ModelSequenceBatching) private: class _Internal; void set_has_direct(); void set_has_oldest(); inline bool has_strategy_choice() const; inline void clear_has_strategy_choice(); template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_ControlInput > control_input_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_State > state_; uint64_t max_sequence_idle_microseconds_; bool iterative_sequence_; union StrategyChoiceUnion { constexpr StrategyChoiceUnion() : _constinit_{} {} ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized _constinit_; ::inference::ModelSequenceBatching_StrategyDirect* direct_; ::inference::ModelSequenceBatching_StrategyOldest* oldest_; } strategy_choice_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; uint32_t _oneof_case_[1]; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelEnsembling_Step_InputMapEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelEnsembling_Step_InputMapEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelEnsembling_Step_InputMapEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelEnsembling_Step_InputMapEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelEnsembling_Step_InputMapEntry_DoNotUse& other); static const ModelEnsembling_Step_InputMapEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelEnsembling_Step_InputMapEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelEnsembling.Step.InputMapEntry.key"); } static bool ValidateValue(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelEnsembling.Step.InputMapEntry.value"); } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelEnsembling_Step_OutputMapEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelEnsembling_Step_OutputMapEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelEnsembling_Step_OutputMapEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelEnsembling_Step_OutputMapEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelEnsembling_Step_OutputMapEntry_DoNotUse& other); static const ModelEnsembling_Step_OutputMapEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelEnsembling_Step_OutputMapEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelEnsembling.Step.OutputMapEntry.key"); } static bool ValidateValue(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelEnsembling.Step.OutputMapEntry.value"); } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelEnsembling_Step final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelEnsembling.Step) */ { public: inline ModelEnsembling_Step() : ModelEnsembling_Step(nullptr) {} ~ModelEnsembling_Step() override; explicit PROTOBUF_CONSTEXPR ModelEnsembling_Step(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelEnsembling_Step(const ModelEnsembling_Step& from); ModelEnsembling_Step(ModelEnsembling_Step&& from) noexcept : ModelEnsembling_Step() { *this = ::std::move(from); } inline ModelEnsembling_Step& operator=(const ModelEnsembling_Step& from) { CopyFrom(from); return *this; } inline ModelEnsembling_Step& operator=(ModelEnsembling_Step&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelEnsembling_Step& default_instance() { return *internal_default_instance(); } static inline const ModelEnsembling_Step* internal_default_instance() { return reinterpret_cast( &_ModelEnsembling_Step_default_instance_); } static constexpr int kIndexInFileMessages = 37; friend void swap(ModelEnsembling_Step& a, ModelEnsembling_Step& b) { a.Swap(&b); } inline void Swap(ModelEnsembling_Step* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelEnsembling_Step* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelEnsembling_Step* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelEnsembling_Step& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelEnsembling_Step& from) { ModelEnsembling_Step::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelEnsembling_Step* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelEnsembling.Step"; } protected: explicit ModelEnsembling_Step(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); private: static void ArenaDtor(void* object); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kInputMapFieldNumber = 3, kOutputMapFieldNumber = 4, kModelNameFieldNumber = 1, kModelNamespaceFieldNumber = 5, kModelVersionFieldNumber = 2, }; // map input_map = 3; int input_map_size() const; private: int _internal_input_map_size() const; public: void clear_input_map(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& _internal_input_map() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* _internal_mutable_input_map(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& input_map() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* mutable_input_map(); // map output_map = 4; int output_map_size() const; private: int _internal_output_map_size() const; public: void clear_output_map(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& _internal_output_map() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* _internal_mutable_output_map(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& output_map() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* mutable_output_map(); // string model_name = 1; void clear_model_name(); const std::string& model_name() const; template void set_model_name(ArgT0&& arg0, ArgT... args); std::string* mutable_model_name(); PROTOBUF_NODISCARD std::string* release_model_name(); void set_allocated_model_name(std::string* model_name); private: const std::string& _internal_model_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_model_name(const std::string& value); std::string* _internal_mutable_model_name(); public: // string model_namespace = 5; void clear_model_namespace(); const std::string& model_namespace() const; template void set_model_namespace(ArgT0&& arg0, ArgT... args); std::string* mutable_model_namespace(); PROTOBUF_NODISCARD std::string* release_model_namespace(); void set_allocated_model_namespace(std::string* model_namespace); private: const std::string& _internal_model_namespace() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_model_namespace(const std::string& value); std::string* _internal_mutable_model_namespace(); public: // int64 model_version = 2; void clear_model_version(); int64_t model_version() const; void set_model_version(int64_t value); private: int64_t _internal_model_version() const; void _internal_set_model_version(int64_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelEnsembling.Step) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelEnsembling_Step_InputMapEntry_DoNotUse, std::string, std::string, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> input_map_; ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelEnsembling_Step_OutputMapEntry_DoNotUse, std::string, std::string, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> output_map_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr model_name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr model_namespace_; int64_t model_version_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelEnsembling final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelEnsembling) */ { public: inline ModelEnsembling() : ModelEnsembling(nullptr) {} ~ModelEnsembling() override; explicit PROTOBUF_CONSTEXPR ModelEnsembling(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelEnsembling(const ModelEnsembling& from); ModelEnsembling(ModelEnsembling&& from) noexcept : ModelEnsembling() { *this = ::std::move(from); } inline ModelEnsembling& operator=(const ModelEnsembling& from) { CopyFrom(from); return *this; } inline ModelEnsembling& operator=(ModelEnsembling&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelEnsembling& default_instance() { return *internal_default_instance(); } static inline const ModelEnsembling* internal_default_instance() { return reinterpret_cast( &_ModelEnsembling_default_instance_); } static constexpr int kIndexInFileMessages = 38; friend void swap(ModelEnsembling& a, ModelEnsembling& b) { a.Swap(&b); } inline void Swap(ModelEnsembling* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelEnsembling* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelEnsembling* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelEnsembling& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelEnsembling& from) { ModelEnsembling::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelEnsembling* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelEnsembling"; } protected: explicit ModelEnsembling(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelEnsembling_Step Step; // accessors ------------------------------------------------------- enum : int { kStepFieldNumber = 1, }; // repeated .inference.ModelEnsembling.Step step = 1; int step_size() const; private: int _internal_step_size() const; public: void clear_step(); ::inference::ModelEnsembling_Step* mutable_step(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelEnsembling_Step >* mutable_step(); private: const ::inference::ModelEnsembling_Step& _internal_step(int index) const; ::inference::ModelEnsembling_Step* _internal_add_step(); public: const ::inference::ModelEnsembling_Step& step(int index) const; ::inference::ModelEnsembling_Step* add_step(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelEnsembling_Step >& step() const; // @@protoc_insertion_point(class_scope:inference.ModelEnsembling) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelEnsembling_Step > step_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelParameter final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelParameter) */ { public: inline ModelParameter() : ModelParameter(nullptr) {} ~ModelParameter() override; explicit PROTOBUF_CONSTEXPR ModelParameter(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelParameter(const ModelParameter& from); ModelParameter(ModelParameter&& from) noexcept : ModelParameter() { *this = ::std::move(from); } inline ModelParameter& operator=(const ModelParameter& from) { CopyFrom(from); return *this; } inline ModelParameter& operator=(ModelParameter&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelParameter& default_instance() { return *internal_default_instance(); } static inline const ModelParameter* internal_default_instance() { return reinterpret_cast( &_ModelParameter_default_instance_); } static constexpr int kIndexInFileMessages = 39; friend void swap(ModelParameter& a, ModelParameter& b) { a.Swap(&b); } inline void Swap(ModelParameter* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelParameter* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelParameter* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelParameter& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelParameter& from) { ModelParameter::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelParameter* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelParameter"; } protected: explicit ModelParameter(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kStringValueFieldNumber = 1, }; // string string_value = 1; void clear_string_value(); const std::string& string_value() const; template void set_string_value(ArgT0&& arg0, ArgT... args); std::string* mutable_string_value(); PROTOBUF_NODISCARD std::string* release_string_value(); void set_allocated_string_value(std::string* string_value); private: const std::string& _internal_string_value() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_string_value(const std::string& value); std::string* _internal_mutable_string_value(); public: // @@protoc_insertion_point(class_scope:inference.ModelParameter) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr string_value_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelWarmup_Input final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelWarmup.Input) */ { public: inline ModelWarmup_Input() : ModelWarmup_Input(nullptr) {} ~ModelWarmup_Input() override; explicit PROTOBUF_CONSTEXPR ModelWarmup_Input(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelWarmup_Input(const ModelWarmup_Input& from); ModelWarmup_Input(ModelWarmup_Input&& from) noexcept : ModelWarmup_Input() { *this = ::std::move(from); } inline ModelWarmup_Input& operator=(const ModelWarmup_Input& from) { CopyFrom(from); return *this; } inline ModelWarmup_Input& operator=(ModelWarmup_Input&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelWarmup_Input& default_instance() { return *internal_default_instance(); } enum InputDataTypeCase { kZeroData = 3, kRandomData = 4, kInputDataFile = 5, INPUT_DATA_TYPE_NOT_SET = 0, }; static inline const ModelWarmup_Input* internal_default_instance() { return reinterpret_cast( &_ModelWarmup_Input_default_instance_); } static constexpr int kIndexInFileMessages = 40; friend void swap(ModelWarmup_Input& a, ModelWarmup_Input& b) { a.Swap(&b); } inline void Swap(ModelWarmup_Input* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelWarmup_Input* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelWarmup_Input* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelWarmup_Input& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelWarmup_Input& from) { ModelWarmup_Input::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelWarmup_Input* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelWarmup.Input"; } protected: explicit ModelWarmup_Input(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kDimsFieldNumber = 2, kDataTypeFieldNumber = 1, kZeroDataFieldNumber = 3, kRandomDataFieldNumber = 4, kInputDataFileFieldNumber = 5, }; // repeated int64 dims = 2; int dims_size() const; private: int _internal_dims_size() const; public: void clear_dims(); private: int64_t _internal_dims(int index) const; const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& _internal_dims() const; void _internal_add_dims(int64_t value); ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* _internal_mutable_dims(); public: int64_t dims(int index) const; void set_dims(int index, int64_t value); void add_dims(int64_t value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& dims() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* mutable_dims(); // .inference.DataType data_type = 1; void clear_data_type(); ::inference::DataType data_type() const; void set_data_type(::inference::DataType value); private: ::inference::DataType _internal_data_type() const; void _internal_set_data_type(::inference::DataType value); public: // bool zero_data = 3; bool has_zero_data() const; private: bool _internal_has_zero_data() const; public: void clear_zero_data(); bool zero_data() const; void set_zero_data(bool value); private: bool _internal_zero_data() const; void _internal_set_zero_data(bool value); public: // bool random_data = 4; bool has_random_data() const; private: bool _internal_has_random_data() const; public: void clear_random_data(); bool random_data() const; void set_random_data(bool value); private: bool _internal_random_data() const; void _internal_set_random_data(bool value); public: // string input_data_file = 5; bool has_input_data_file() const; private: bool _internal_has_input_data_file() const; public: void clear_input_data_file(); const std::string& input_data_file() const; template void set_input_data_file(ArgT0&& arg0, ArgT... args); std::string* mutable_input_data_file(); PROTOBUF_NODISCARD std::string* release_input_data_file(); void set_allocated_input_data_file(std::string* input_data_file); private: const std::string& _internal_input_data_file() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_input_data_file(const std::string& value); std::string* _internal_mutable_input_data_file(); public: void clear_input_data_type(); InputDataTypeCase input_data_type_case() const; // @@protoc_insertion_point(class_scope:inference.ModelWarmup.Input) private: class _Internal; void set_has_zero_data(); void set_has_random_data(); void set_has_input_data_file(); inline bool has_input_data_type() const; inline void clear_has_input_data_type(); template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > dims_; mutable std::atomic _dims_cached_byte_size_; int data_type_; union InputDataTypeUnion { constexpr InputDataTypeUnion() : _constinit_{} {} ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized _constinit_; bool zero_data_; bool random_data_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr input_data_file_; } input_data_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; uint32_t _oneof_case_[1]; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelWarmup_InputsEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelWarmup_InputsEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelWarmup_InputsEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelWarmup_InputsEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelWarmup_InputsEntry_DoNotUse& other); static const ModelWarmup_InputsEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelWarmup_InputsEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelWarmup.InputsEntry.key"); } static bool ValidateValue(void*) { return true; } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelWarmup final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelWarmup) */ { public: inline ModelWarmup() : ModelWarmup(nullptr) {} ~ModelWarmup() override; explicit PROTOBUF_CONSTEXPR ModelWarmup(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelWarmup(const ModelWarmup& from); ModelWarmup(ModelWarmup&& from) noexcept : ModelWarmup() { *this = ::std::move(from); } inline ModelWarmup& operator=(const ModelWarmup& from) { CopyFrom(from); return *this; } inline ModelWarmup& operator=(ModelWarmup&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelWarmup& default_instance() { return *internal_default_instance(); } static inline const ModelWarmup* internal_default_instance() { return reinterpret_cast( &_ModelWarmup_default_instance_); } static constexpr int kIndexInFileMessages = 42; friend void swap(ModelWarmup& a, ModelWarmup& b) { a.Swap(&b); } inline void Swap(ModelWarmup* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelWarmup* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelWarmup* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelWarmup& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelWarmup& from) { ModelWarmup::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelWarmup* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelWarmup"; } protected: explicit ModelWarmup(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); private: static void ArenaDtor(void* object); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelWarmup_Input Input; // accessors ------------------------------------------------------- enum : int { kInputsFieldNumber = 3, kNameFieldNumber = 1, kBatchSizeFieldNumber = 2, kCountFieldNumber = 4, }; // map inputs = 3; int inputs_size() const; private: int _internal_inputs_size() const; public: void clear_inputs(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >& _internal_inputs() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >* _internal_mutable_inputs(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >& inputs() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >* mutable_inputs(); // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // uint32 batch_size = 2; void clear_batch_size(); uint32_t batch_size() const; void set_batch_size(uint32_t value); private: uint32_t _internal_batch_size() const; void _internal_set_batch_size(uint32_t value); public: // uint32 count = 4; void clear_count(); uint32_t count() const; void set_count(uint32_t value); private: uint32_t _internal_count() const; void _internal_set_count(uint32_t value); public: // @@protoc_insertion_point(class_scope:inference.ModelWarmup) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelWarmup_InputsEntry_DoNotUse, std::string, ::inference::ModelWarmup_Input, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_MESSAGE> inputs_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; uint32_t batch_size_; uint32_t count_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelOperations final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelOperations) */ { public: inline ModelOperations() : ModelOperations(nullptr) {} ~ModelOperations() override; explicit PROTOBUF_CONSTEXPR ModelOperations(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelOperations(const ModelOperations& from); ModelOperations(ModelOperations&& from) noexcept : ModelOperations() { *this = ::std::move(from); } inline ModelOperations& operator=(const ModelOperations& from) { CopyFrom(from); return *this; } inline ModelOperations& operator=(ModelOperations&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelOperations& default_instance() { return *internal_default_instance(); } static inline const ModelOperations* internal_default_instance() { return reinterpret_cast( &_ModelOperations_default_instance_); } static constexpr int kIndexInFileMessages = 43; friend void swap(ModelOperations& a, ModelOperations& b) { a.Swap(&b); } inline void Swap(ModelOperations* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelOperations* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelOperations* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelOperations& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelOperations& from) { ModelOperations::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelOperations* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelOperations"; } protected: explicit ModelOperations(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kOpLibraryFilenameFieldNumber = 1, }; // repeated string op_library_filename = 1; int op_library_filename_size() const; private: int _internal_op_library_filename_size() const; public: void clear_op_library_filename(); const std::string& op_library_filename(int index) const; std::string* mutable_op_library_filename(int index); void set_op_library_filename(int index, const std::string& value); void set_op_library_filename(int index, std::string&& value); void set_op_library_filename(int index, const char* value); void set_op_library_filename(int index, const char* value, size_t size); std::string* add_op_library_filename(); void add_op_library_filename(const std::string& value); void add_op_library_filename(std::string&& value); void add_op_library_filename(const char* value); void add_op_library_filename(const char* value, size_t size); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& op_library_filename() const; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_op_library_filename(); private: const std::string& _internal_op_library_filename(int index) const; std::string* _internal_add_op_library_filename(); public: // @@protoc_insertion_point(class_scope:inference.ModelOperations) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField op_library_filename_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelTransactionPolicy final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelTransactionPolicy) */ { public: inline ModelTransactionPolicy() : ModelTransactionPolicy(nullptr) {} ~ModelTransactionPolicy() override; explicit PROTOBUF_CONSTEXPR ModelTransactionPolicy(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelTransactionPolicy(const ModelTransactionPolicy& from); ModelTransactionPolicy(ModelTransactionPolicy&& from) noexcept : ModelTransactionPolicy() { *this = ::std::move(from); } inline ModelTransactionPolicy& operator=(const ModelTransactionPolicy& from) { CopyFrom(from); return *this; } inline ModelTransactionPolicy& operator=(ModelTransactionPolicy&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelTransactionPolicy& default_instance() { return *internal_default_instance(); } static inline const ModelTransactionPolicy* internal_default_instance() { return reinterpret_cast( &_ModelTransactionPolicy_default_instance_); } static constexpr int kIndexInFileMessages = 44; friend void swap(ModelTransactionPolicy& a, ModelTransactionPolicy& b) { a.Swap(&b); } inline void Swap(ModelTransactionPolicy* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelTransactionPolicy* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelTransactionPolicy* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelTransactionPolicy& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelTransactionPolicy& from) { ModelTransactionPolicy::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelTransactionPolicy* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelTransactionPolicy"; } protected: explicit ModelTransactionPolicy(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kDecoupledFieldNumber = 1, }; // bool decoupled = 1; void clear_decoupled(); bool decoupled() const; void set_decoupled(bool value); private: bool _internal_decoupled() const; void _internal_set_decoupled(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelTransactionPolicy) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { bool decoupled_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse& other); static const ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelRepositoryAgents.Agent.ParametersEntry.key"); } static bool ValidateValue(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelRepositoryAgents.Agent.ParametersEntry.value"); } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelRepositoryAgents_Agent final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelRepositoryAgents.Agent) */ { public: inline ModelRepositoryAgents_Agent() : ModelRepositoryAgents_Agent(nullptr) {} ~ModelRepositoryAgents_Agent() override; explicit PROTOBUF_CONSTEXPR ModelRepositoryAgents_Agent(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelRepositoryAgents_Agent(const ModelRepositoryAgents_Agent& from); ModelRepositoryAgents_Agent(ModelRepositoryAgents_Agent&& from) noexcept : ModelRepositoryAgents_Agent() { *this = ::std::move(from); } inline ModelRepositoryAgents_Agent& operator=(const ModelRepositoryAgents_Agent& from) { CopyFrom(from); return *this; } inline ModelRepositoryAgents_Agent& operator=(ModelRepositoryAgents_Agent&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelRepositoryAgents_Agent& default_instance() { return *internal_default_instance(); } static inline const ModelRepositoryAgents_Agent* internal_default_instance() { return reinterpret_cast( &_ModelRepositoryAgents_Agent_default_instance_); } static constexpr int kIndexInFileMessages = 46; friend void swap(ModelRepositoryAgents_Agent& a, ModelRepositoryAgents_Agent& b) { a.Swap(&b); } inline void Swap(ModelRepositoryAgents_Agent* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelRepositoryAgents_Agent* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelRepositoryAgents_Agent* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelRepositoryAgents_Agent& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelRepositoryAgents_Agent& from) { ModelRepositoryAgents_Agent::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelRepositoryAgents_Agent* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelRepositoryAgents.Agent"; } protected: explicit ModelRepositoryAgents_Agent(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); private: static void ArenaDtor(void* object); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kParametersFieldNumber = 2, kNameFieldNumber = 1, }; // map parameters = 2; int parameters_size() const; private: int _internal_parameters_size() const; public: void clear_parameters(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& _internal_parameters() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* _internal_mutable_parameters(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& parameters() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* mutable_parameters(); // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // @@protoc_insertion_point(class_scope:inference.ModelRepositoryAgents.Agent) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelRepositoryAgents_Agent_ParametersEntry_DoNotUse, std::string, std::string, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> parameters_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelRepositoryAgents final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelRepositoryAgents) */ { public: inline ModelRepositoryAgents() : ModelRepositoryAgents(nullptr) {} ~ModelRepositoryAgents() override; explicit PROTOBUF_CONSTEXPR ModelRepositoryAgents(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelRepositoryAgents(const ModelRepositoryAgents& from); ModelRepositoryAgents(ModelRepositoryAgents&& from) noexcept : ModelRepositoryAgents() { *this = ::std::move(from); } inline ModelRepositoryAgents& operator=(const ModelRepositoryAgents& from) { CopyFrom(from); return *this; } inline ModelRepositoryAgents& operator=(ModelRepositoryAgents&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelRepositoryAgents& default_instance() { return *internal_default_instance(); } static inline const ModelRepositoryAgents* internal_default_instance() { return reinterpret_cast( &_ModelRepositoryAgents_default_instance_); } static constexpr int kIndexInFileMessages = 47; friend void swap(ModelRepositoryAgents& a, ModelRepositoryAgents& b) { a.Swap(&b); } inline void Swap(ModelRepositoryAgents* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelRepositoryAgents* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelRepositoryAgents* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelRepositoryAgents& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelRepositoryAgents& from) { ModelRepositoryAgents::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelRepositoryAgents* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelRepositoryAgents"; } protected: explicit ModelRepositoryAgents(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- typedef ModelRepositoryAgents_Agent Agent; // accessors ------------------------------------------------------- enum : int { kAgentsFieldNumber = 1, }; // repeated .inference.ModelRepositoryAgents.Agent agents = 1; int agents_size() const; private: int _internal_agents_size() const; public: void clear_agents(); ::inference::ModelRepositoryAgents_Agent* mutable_agents(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRepositoryAgents_Agent >* mutable_agents(); private: const ::inference::ModelRepositoryAgents_Agent& _internal_agents(int index) const; ::inference::ModelRepositoryAgents_Agent* _internal_add_agents(); public: const ::inference::ModelRepositoryAgents_Agent& agents(int index) const; ::inference::ModelRepositoryAgents_Agent* add_agents(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRepositoryAgents_Agent >& agents() const; // @@protoc_insertion_point(class_scope:inference.ModelRepositoryAgents) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRepositoryAgents_Agent > agents_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelResponseCache final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelResponseCache) */ { public: inline ModelResponseCache() : ModelResponseCache(nullptr) {} ~ModelResponseCache() override; explicit PROTOBUF_CONSTEXPR ModelResponseCache(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelResponseCache(const ModelResponseCache& from); ModelResponseCache(ModelResponseCache&& from) noexcept : ModelResponseCache() { *this = ::std::move(from); } inline ModelResponseCache& operator=(const ModelResponseCache& from) { CopyFrom(from); return *this; } inline ModelResponseCache& operator=(ModelResponseCache&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelResponseCache& default_instance() { return *internal_default_instance(); } static inline const ModelResponseCache* internal_default_instance() { return reinterpret_cast( &_ModelResponseCache_default_instance_); } static constexpr int kIndexInFileMessages = 48; friend void swap(ModelResponseCache& a, ModelResponseCache& b) { a.Swap(&b); } inline void Swap(ModelResponseCache* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelResponseCache* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelResponseCache* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelResponseCache& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelResponseCache& from) { ModelResponseCache::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelResponseCache* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelResponseCache"; } protected: explicit ModelResponseCache(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kEnableFieldNumber = 1, }; // bool enable = 1; void clear_enable(); bool enable() const; void set_enable(bool value); private: bool _internal_enable() const; void _internal_set_enable(bool value); public: // @@protoc_insertion_point(class_scope:inference.ModelResponseCache) private: class _Internal; template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { bool enable_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelConfig_CcModelFilenamesEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelConfig_CcModelFilenamesEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelConfig_CcModelFilenamesEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelConfig_CcModelFilenamesEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelConfig_CcModelFilenamesEntry_DoNotUse& other); static const ModelConfig_CcModelFilenamesEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelConfig_CcModelFilenamesEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelConfig.CcModelFilenamesEntry.key"); } static bool ValidateValue(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelConfig.CcModelFilenamesEntry.value"); } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelConfig_MetricTagsEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelConfig_MetricTagsEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelConfig_MetricTagsEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelConfig_MetricTagsEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelConfig_MetricTagsEntry_DoNotUse& other); static const ModelConfig_MetricTagsEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelConfig_MetricTagsEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelConfig.MetricTagsEntry.key"); } static bool ValidateValue(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelConfig.MetricTagsEntry.value"); } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelConfig_ParametersEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { public: typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; ModelConfig_ParametersEntry_DoNotUse(); explicit PROTOBUF_CONSTEXPR ModelConfig_ParametersEntry_DoNotUse( ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); explicit ModelConfig_ParametersEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); void MergeFrom(const ModelConfig_ParametersEntry_DoNotUse& other); static const ModelConfig_ParametersEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_ModelConfig_ParametersEntry_DoNotUse_default_instance_); } static bool ValidateKey(std::string* s) { return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "inference.ModelConfig.ParametersEntry.key"); } static bool ValidateValue(void*) { return true; } using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; friend struct ::TableStruct_model_5fconfig_2eproto; }; // ------------------------------------------------------------------- class ModelConfig final : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:inference.ModelConfig) */ { public: inline ModelConfig() : ModelConfig(nullptr) {} ~ModelConfig() override; explicit PROTOBUF_CONSTEXPR ModelConfig(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); ModelConfig(const ModelConfig& from); ModelConfig(ModelConfig&& from) noexcept : ModelConfig() { *this = ::std::move(from); } inline ModelConfig& operator=(const ModelConfig& from) { CopyFrom(from); return *this; } inline ModelConfig& operator=(ModelConfig&& from) noexcept { if (this == &from) return *this; if (GetOwningArena() == from.GetOwningArena() #ifdef PROTOBUF_FORCE_COPY_IN_MOVE && GetOwningArena() != nullptr #endif // !PROTOBUF_FORCE_COPY_IN_MOVE ) { InternalSwap(&from); } else { CopyFrom(from); } return *this; } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { return GetDescriptor(); } static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { return default_instance().GetMetadata().descriptor; } static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return default_instance().GetMetadata().reflection; } static const ModelConfig& default_instance() { return *internal_default_instance(); } enum SchedulingChoiceCase { kDynamicBatching = 11, kSequenceBatching = 13, kEnsembleScheduling = 15, SCHEDULING_CHOICE_NOT_SET = 0, }; static inline const ModelConfig* internal_default_instance() { return reinterpret_cast( &_ModelConfig_default_instance_); } static constexpr int kIndexInFileMessages = 52; friend void swap(ModelConfig& a, ModelConfig& b) { a.Swap(&b); } inline void Swap(ModelConfig* other) { if (other == this) return; #ifdef PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() != nullptr && GetOwningArena() == other->GetOwningArena()) { #else // PROTOBUF_FORCE_COPY_IN_SWAP if (GetOwningArena() == other->GetOwningArena()) { #endif // !PROTOBUF_FORCE_COPY_IN_SWAP InternalSwap(other); } else { ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); } } void UnsafeArenaSwap(ModelConfig* other) { if (other == this) return; GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); InternalSwap(other); } // implements Message ---------------------------------------------- ModelConfig* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { return CreateMaybeMessage(arena); } using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom; void CopyFrom(const ModelConfig& from); using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom; void MergeFrom( const ModelConfig& from) { ModelConfig::MergeImpl(*this, from); } private: static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg); public: PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; size_t ByteSizeLong() const final; const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; uint8_t* _InternalSerialize( uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; int GetCachedSize() const final { return _impl_._cached_size_.Get(); } private: void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); void SharedDtor(); void SetCachedSize(int size) const final; void InternalSwap(ModelConfig* other); private: friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { return "inference.ModelConfig"; } protected: explicit ModelConfig(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned = false); private: static void ArenaDtor(void* object); public: static const ClassData _class_data_; const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final; ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; // nested types ---------------------------------------------------- // accessors ------------------------------------------------------- enum : int { kInputFieldNumber = 5, kOutputFieldNumber = 6, kInstanceGroupFieldNumber = 7, kCcModelFilenamesFieldNumber = 9, kMetricTagsFieldNumber = 10, kParametersFieldNumber = 14, kModelWarmupFieldNumber = 16, kBatchInputFieldNumber = 20, kBatchOutputFieldNumber = 21, kNameFieldNumber = 1, kPlatformFieldNumber = 2, kDefaultModelFilenameFieldNumber = 8, kBackendFieldNumber = 17, kRuntimeFieldNumber = 25, kVersionPolicyFieldNumber = 3, kOptimizationFieldNumber = 12, kModelOperationsFieldNumber = 18, kModelTransactionPolicyFieldNumber = 19, kModelRepositoryAgentsFieldNumber = 23, kResponseCacheFieldNumber = 24, kMaxBatchSizeFieldNumber = 4, kDynamicBatchingFieldNumber = 11, kSequenceBatchingFieldNumber = 13, kEnsembleSchedulingFieldNumber = 15, }; // repeated .inference.ModelInput input = 5; int input_size() const; private: int _internal_input_size() const; public: void clear_input(); ::inference::ModelInput* mutable_input(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInput >* mutable_input(); private: const ::inference::ModelInput& _internal_input(int index) const; ::inference::ModelInput* _internal_add_input(); public: const ::inference::ModelInput& input(int index) const; ::inference::ModelInput* add_input(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInput >& input() const; // repeated .inference.ModelOutput output = 6; int output_size() const; private: int _internal_output_size() const; public: void clear_output(); ::inference::ModelOutput* mutable_output(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOutput >* mutable_output(); private: const ::inference::ModelOutput& _internal_output(int index) const; ::inference::ModelOutput* _internal_add_output(); public: const ::inference::ModelOutput& output(int index) const; ::inference::ModelOutput* add_output(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOutput >& output() const; // repeated .inference.ModelInstanceGroup instance_group = 7; int instance_group_size() const; private: int _internal_instance_group_size() const; public: void clear_instance_group(); ::inference::ModelInstanceGroup* mutable_instance_group(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup >* mutable_instance_group(); private: const ::inference::ModelInstanceGroup& _internal_instance_group(int index) const; ::inference::ModelInstanceGroup* _internal_add_instance_group(); public: const ::inference::ModelInstanceGroup& instance_group(int index) const; ::inference::ModelInstanceGroup* add_instance_group(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup >& instance_group() const; // map cc_model_filenames = 9; int cc_model_filenames_size() const; private: int _internal_cc_model_filenames_size() const; public: void clear_cc_model_filenames(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& _internal_cc_model_filenames() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* _internal_mutable_cc_model_filenames(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& cc_model_filenames() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* mutable_cc_model_filenames(); // map metric_tags = 10; int metric_tags_size() const; private: int _internal_metric_tags_size() const; public: void clear_metric_tags(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& _internal_metric_tags() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* _internal_mutable_metric_tags(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& metric_tags() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* mutable_metric_tags(); // map parameters = 14; int parameters_size() const; private: int _internal_parameters_size() const; public: void clear_parameters(); private: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >& _internal_parameters() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >* _internal_mutable_parameters(); public: const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >& parameters() const; ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >* mutable_parameters(); // repeated .inference.ModelWarmup model_warmup = 16; int model_warmup_size() const; private: int _internal_model_warmup_size() const; public: void clear_model_warmup(); ::inference::ModelWarmup* mutable_model_warmup(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelWarmup >* mutable_model_warmup(); private: const ::inference::ModelWarmup& _internal_model_warmup(int index) const; ::inference::ModelWarmup* _internal_add_model_warmup(); public: const ::inference::ModelWarmup& model_warmup(int index) const; ::inference::ModelWarmup* add_model_warmup(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelWarmup >& model_warmup() const; // repeated .inference.BatchInput batch_input = 20; int batch_input_size() const; private: int _internal_batch_input_size() const; public: void clear_batch_input(); ::inference::BatchInput* mutable_batch_input(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchInput >* mutable_batch_input(); private: const ::inference::BatchInput& _internal_batch_input(int index) const; ::inference::BatchInput* _internal_add_batch_input(); public: const ::inference::BatchInput& batch_input(int index) const; ::inference::BatchInput* add_batch_input(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchInput >& batch_input() const; // repeated .inference.BatchOutput batch_output = 21; int batch_output_size() const; private: int _internal_batch_output_size() const; public: void clear_batch_output(); ::inference::BatchOutput* mutable_batch_output(int index); ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchOutput >* mutable_batch_output(); private: const ::inference::BatchOutput& _internal_batch_output(int index) const; ::inference::BatchOutput* _internal_add_batch_output(); public: const ::inference::BatchOutput& batch_output(int index) const; ::inference::BatchOutput* add_batch_output(); const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchOutput >& batch_output() const; // string name = 1; void clear_name(); const std::string& name() const; template void set_name(ArgT0&& arg0, ArgT... args); std::string* mutable_name(); PROTOBUF_NODISCARD std::string* release_name(); void set_allocated_name(std::string* name); private: const std::string& _internal_name() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_name(const std::string& value); std::string* _internal_mutable_name(); public: // string platform = 2; void clear_platform(); const std::string& platform() const; template void set_platform(ArgT0&& arg0, ArgT... args); std::string* mutable_platform(); PROTOBUF_NODISCARD std::string* release_platform(); void set_allocated_platform(std::string* platform); private: const std::string& _internal_platform() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_platform(const std::string& value); std::string* _internal_mutable_platform(); public: // string default_model_filename = 8; void clear_default_model_filename(); const std::string& default_model_filename() const; template void set_default_model_filename(ArgT0&& arg0, ArgT... args); std::string* mutable_default_model_filename(); PROTOBUF_NODISCARD std::string* release_default_model_filename(); void set_allocated_default_model_filename(std::string* default_model_filename); private: const std::string& _internal_default_model_filename() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_default_model_filename(const std::string& value); std::string* _internal_mutable_default_model_filename(); public: // string backend = 17; void clear_backend(); const std::string& backend() const; template void set_backend(ArgT0&& arg0, ArgT... args); std::string* mutable_backend(); PROTOBUF_NODISCARD std::string* release_backend(); void set_allocated_backend(std::string* backend); private: const std::string& _internal_backend() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_backend(const std::string& value); std::string* _internal_mutable_backend(); public: // string runtime = 25; void clear_runtime(); const std::string& runtime() const; template void set_runtime(ArgT0&& arg0, ArgT... args); std::string* mutable_runtime(); PROTOBUF_NODISCARD std::string* release_runtime(); void set_allocated_runtime(std::string* runtime); private: const std::string& _internal_runtime() const; inline PROTOBUF_ALWAYS_INLINE void _internal_set_runtime(const std::string& value); std::string* _internal_mutable_runtime(); public: // .inference.ModelVersionPolicy version_policy = 3; bool has_version_policy() const; private: bool _internal_has_version_policy() const; public: void clear_version_policy(); const ::inference::ModelVersionPolicy& version_policy() const; PROTOBUF_NODISCARD ::inference::ModelVersionPolicy* release_version_policy(); ::inference::ModelVersionPolicy* mutable_version_policy(); void set_allocated_version_policy(::inference::ModelVersionPolicy* version_policy); private: const ::inference::ModelVersionPolicy& _internal_version_policy() const; ::inference::ModelVersionPolicy* _internal_mutable_version_policy(); public: void unsafe_arena_set_allocated_version_policy( ::inference::ModelVersionPolicy* version_policy); ::inference::ModelVersionPolicy* unsafe_arena_release_version_policy(); // .inference.ModelOptimizationPolicy optimization = 12; bool has_optimization() const; private: bool _internal_has_optimization() const; public: void clear_optimization(); const ::inference::ModelOptimizationPolicy& optimization() const; PROTOBUF_NODISCARD ::inference::ModelOptimizationPolicy* release_optimization(); ::inference::ModelOptimizationPolicy* mutable_optimization(); void set_allocated_optimization(::inference::ModelOptimizationPolicy* optimization); private: const ::inference::ModelOptimizationPolicy& _internal_optimization() const; ::inference::ModelOptimizationPolicy* _internal_mutable_optimization(); public: void unsafe_arena_set_allocated_optimization( ::inference::ModelOptimizationPolicy* optimization); ::inference::ModelOptimizationPolicy* unsafe_arena_release_optimization(); // .inference.ModelOperations model_operations = 18; bool has_model_operations() const; private: bool _internal_has_model_operations() const; public: void clear_model_operations(); const ::inference::ModelOperations& model_operations() const; PROTOBUF_NODISCARD ::inference::ModelOperations* release_model_operations(); ::inference::ModelOperations* mutable_model_operations(); void set_allocated_model_operations(::inference::ModelOperations* model_operations); private: const ::inference::ModelOperations& _internal_model_operations() const; ::inference::ModelOperations* _internal_mutable_model_operations(); public: void unsafe_arena_set_allocated_model_operations( ::inference::ModelOperations* model_operations); ::inference::ModelOperations* unsafe_arena_release_model_operations(); // .inference.ModelTransactionPolicy model_transaction_policy = 19; bool has_model_transaction_policy() const; private: bool _internal_has_model_transaction_policy() const; public: void clear_model_transaction_policy(); const ::inference::ModelTransactionPolicy& model_transaction_policy() const; PROTOBUF_NODISCARD ::inference::ModelTransactionPolicy* release_model_transaction_policy(); ::inference::ModelTransactionPolicy* mutable_model_transaction_policy(); void set_allocated_model_transaction_policy(::inference::ModelTransactionPolicy* model_transaction_policy); private: const ::inference::ModelTransactionPolicy& _internal_model_transaction_policy() const; ::inference::ModelTransactionPolicy* _internal_mutable_model_transaction_policy(); public: void unsafe_arena_set_allocated_model_transaction_policy( ::inference::ModelTransactionPolicy* model_transaction_policy); ::inference::ModelTransactionPolicy* unsafe_arena_release_model_transaction_policy(); // .inference.ModelRepositoryAgents model_repository_agents = 23; bool has_model_repository_agents() const; private: bool _internal_has_model_repository_agents() const; public: void clear_model_repository_agents(); const ::inference::ModelRepositoryAgents& model_repository_agents() const; PROTOBUF_NODISCARD ::inference::ModelRepositoryAgents* release_model_repository_agents(); ::inference::ModelRepositoryAgents* mutable_model_repository_agents(); void set_allocated_model_repository_agents(::inference::ModelRepositoryAgents* model_repository_agents); private: const ::inference::ModelRepositoryAgents& _internal_model_repository_agents() const; ::inference::ModelRepositoryAgents* _internal_mutable_model_repository_agents(); public: void unsafe_arena_set_allocated_model_repository_agents( ::inference::ModelRepositoryAgents* model_repository_agents); ::inference::ModelRepositoryAgents* unsafe_arena_release_model_repository_agents(); // .inference.ModelResponseCache response_cache = 24; bool has_response_cache() const; private: bool _internal_has_response_cache() const; public: void clear_response_cache(); const ::inference::ModelResponseCache& response_cache() const; PROTOBUF_NODISCARD ::inference::ModelResponseCache* release_response_cache(); ::inference::ModelResponseCache* mutable_response_cache(); void set_allocated_response_cache(::inference::ModelResponseCache* response_cache); private: const ::inference::ModelResponseCache& _internal_response_cache() const; ::inference::ModelResponseCache* _internal_mutable_response_cache(); public: void unsafe_arena_set_allocated_response_cache( ::inference::ModelResponseCache* response_cache); ::inference::ModelResponseCache* unsafe_arena_release_response_cache(); // int32 max_batch_size = 4; void clear_max_batch_size(); int32_t max_batch_size() const; void set_max_batch_size(int32_t value); private: int32_t _internal_max_batch_size() const; void _internal_set_max_batch_size(int32_t value); public: // .inference.ModelDynamicBatching dynamic_batching = 11; bool has_dynamic_batching() const; private: bool _internal_has_dynamic_batching() const; public: void clear_dynamic_batching(); const ::inference::ModelDynamicBatching& dynamic_batching() const; PROTOBUF_NODISCARD ::inference::ModelDynamicBatching* release_dynamic_batching(); ::inference::ModelDynamicBatching* mutable_dynamic_batching(); void set_allocated_dynamic_batching(::inference::ModelDynamicBatching* dynamic_batching); private: const ::inference::ModelDynamicBatching& _internal_dynamic_batching() const; ::inference::ModelDynamicBatching* _internal_mutable_dynamic_batching(); public: void unsafe_arena_set_allocated_dynamic_batching( ::inference::ModelDynamicBatching* dynamic_batching); ::inference::ModelDynamicBatching* unsafe_arena_release_dynamic_batching(); // .inference.ModelSequenceBatching sequence_batching = 13; bool has_sequence_batching() const; private: bool _internal_has_sequence_batching() const; public: void clear_sequence_batching(); const ::inference::ModelSequenceBatching& sequence_batching() const; PROTOBUF_NODISCARD ::inference::ModelSequenceBatching* release_sequence_batching(); ::inference::ModelSequenceBatching* mutable_sequence_batching(); void set_allocated_sequence_batching(::inference::ModelSequenceBatching* sequence_batching); private: const ::inference::ModelSequenceBatching& _internal_sequence_batching() const; ::inference::ModelSequenceBatching* _internal_mutable_sequence_batching(); public: void unsafe_arena_set_allocated_sequence_batching( ::inference::ModelSequenceBatching* sequence_batching); ::inference::ModelSequenceBatching* unsafe_arena_release_sequence_batching(); // .inference.ModelEnsembling ensemble_scheduling = 15; bool has_ensemble_scheduling() const; private: bool _internal_has_ensemble_scheduling() const; public: void clear_ensemble_scheduling(); const ::inference::ModelEnsembling& ensemble_scheduling() const; PROTOBUF_NODISCARD ::inference::ModelEnsembling* release_ensemble_scheduling(); ::inference::ModelEnsembling* mutable_ensemble_scheduling(); void set_allocated_ensemble_scheduling(::inference::ModelEnsembling* ensemble_scheduling); private: const ::inference::ModelEnsembling& _internal_ensemble_scheduling() const; ::inference::ModelEnsembling* _internal_mutable_ensemble_scheduling(); public: void unsafe_arena_set_allocated_ensemble_scheduling( ::inference::ModelEnsembling* ensemble_scheduling); ::inference::ModelEnsembling* unsafe_arena_release_ensemble_scheduling(); void clear_scheduling_choice(); SchedulingChoiceCase scheduling_choice_case() const; // @@protoc_insertion_point(class_scope:inference.ModelConfig) private: class _Internal; void set_has_dynamic_batching(); void set_has_sequence_batching(); void set_has_ensemble_scheduling(); inline bool has_scheduling_choice() const; inline void clear_has_scheduling_choice(); template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInput > input_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOutput > output_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup > instance_group_; ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelConfig_CcModelFilenamesEntry_DoNotUse, std::string, std::string, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> cc_model_filenames_; ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelConfig_MetricTagsEntry_DoNotUse, std::string, std::string, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> metric_tags_; ::PROTOBUF_NAMESPACE_ID::internal::MapField< ModelConfig_ParametersEntry_DoNotUse, std::string, ::inference::ModelParameter, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_MESSAGE> parameters_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelWarmup > model_warmup_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchInput > batch_input_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchOutput > batch_output_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr platform_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr default_model_filename_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr backend_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr runtime_; ::inference::ModelVersionPolicy* version_policy_; ::inference::ModelOptimizationPolicy* optimization_; ::inference::ModelOperations* model_operations_; ::inference::ModelTransactionPolicy* model_transaction_policy_; ::inference::ModelRepositoryAgents* model_repository_agents_; ::inference::ModelResponseCache* response_cache_; int32_t max_batch_size_; union SchedulingChoiceUnion { constexpr SchedulingChoiceUnion() : _constinit_{} {} ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized _constinit_; ::inference::ModelDynamicBatching* dynamic_batching_; ::inference::ModelSequenceBatching* sequence_batching_; ::inference::ModelEnsembling* ensemble_scheduling_; } scheduling_choice_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; uint32_t _oneof_case_[1]; }; union { Impl_ _impl_; }; friend struct ::TableStruct_model_5fconfig_2eproto; }; // =================================================================== // =================================================================== #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif // __GNUC__ // ModelRateLimiter_Resource // string name = 1; inline void ModelRateLimiter_Resource::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelRateLimiter_Resource::name() const { // @@protoc_insertion_point(field_get:inference.ModelRateLimiter.Resource.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelRateLimiter_Resource::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelRateLimiter.Resource.name) } inline std::string* ModelRateLimiter_Resource::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelRateLimiter.Resource.name) return _s; } inline const std::string& ModelRateLimiter_Resource::_internal_name() const { return _impl_.name_.Get(); } inline void ModelRateLimiter_Resource::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelRateLimiter_Resource::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelRateLimiter_Resource::release_name() { // @@protoc_insertion_point(field_release:inference.ModelRateLimiter.Resource.name) return _impl_.name_.Release(); } inline void ModelRateLimiter_Resource::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelRateLimiter.Resource.name) } // bool global = 2; inline void ModelRateLimiter_Resource::clear_global() { _impl_.global_ = false; } inline bool ModelRateLimiter_Resource::_internal_global() const { return _impl_.global_; } inline bool ModelRateLimiter_Resource::global() const { // @@protoc_insertion_point(field_get:inference.ModelRateLimiter.Resource.global) return _internal_global(); } inline void ModelRateLimiter_Resource::_internal_set_global(bool value) { _impl_.global_ = value; } inline void ModelRateLimiter_Resource::set_global(bool value) { _internal_set_global(value); // @@protoc_insertion_point(field_set:inference.ModelRateLimiter.Resource.global) } // uint32 count = 3; inline void ModelRateLimiter_Resource::clear_count() { _impl_.count_ = 0u; } inline uint32_t ModelRateLimiter_Resource::_internal_count() const { return _impl_.count_; } inline uint32_t ModelRateLimiter_Resource::count() const { // @@protoc_insertion_point(field_get:inference.ModelRateLimiter.Resource.count) return _internal_count(); } inline void ModelRateLimiter_Resource::_internal_set_count(uint32_t value) { _impl_.count_ = value; } inline void ModelRateLimiter_Resource::set_count(uint32_t value) { _internal_set_count(value); // @@protoc_insertion_point(field_set:inference.ModelRateLimiter.Resource.count) } // ------------------------------------------------------------------- // ModelRateLimiter // repeated .inference.ModelRateLimiter.Resource resources = 1; inline int ModelRateLimiter::_internal_resources_size() const { return _impl_.resources_.size(); } inline int ModelRateLimiter::resources_size() const { return _internal_resources_size(); } inline void ModelRateLimiter::clear_resources() { _impl_.resources_.Clear(); } inline ::inference::ModelRateLimiter_Resource* ModelRateLimiter::mutable_resources(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelRateLimiter.resources) return _impl_.resources_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRateLimiter_Resource >* ModelRateLimiter::mutable_resources() { // @@protoc_insertion_point(field_mutable_list:inference.ModelRateLimiter.resources) return &_impl_.resources_; } inline const ::inference::ModelRateLimiter_Resource& ModelRateLimiter::_internal_resources(int index) const { return _impl_.resources_.Get(index); } inline const ::inference::ModelRateLimiter_Resource& ModelRateLimiter::resources(int index) const { // @@protoc_insertion_point(field_get:inference.ModelRateLimiter.resources) return _internal_resources(index); } inline ::inference::ModelRateLimiter_Resource* ModelRateLimiter::_internal_add_resources() { return _impl_.resources_.Add(); } inline ::inference::ModelRateLimiter_Resource* ModelRateLimiter::add_resources() { ::inference::ModelRateLimiter_Resource* _add = _internal_add_resources(); // @@protoc_insertion_point(field_add:inference.ModelRateLimiter.resources) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRateLimiter_Resource >& ModelRateLimiter::resources() const { // @@protoc_insertion_point(field_list:inference.ModelRateLimiter.resources) return _impl_.resources_; } // uint32 priority = 2; inline void ModelRateLimiter::clear_priority() { _impl_.priority_ = 0u; } inline uint32_t ModelRateLimiter::_internal_priority() const { return _impl_.priority_; } inline uint32_t ModelRateLimiter::priority() const { // @@protoc_insertion_point(field_get:inference.ModelRateLimiter.priority) return _internal_priority(); } inline void ModelRateLimiter::_internal_set_priority(uint32_t value) { _impl_.priority_ = value; } inline void ModelRateLimiter::set_priority(uint32_t value) { _internal_set_priority(value); // @@protoc_insertion_point(field_set:inference.ModelRateLimiter.priority) } // ------------------------------------------------------------------- // ModelInstanceGroup_SecondaryDevice // .inference.ModelInstanceGroup.SecondaryDevice.SecondaryDeviceKind kind = 1; inline void ModelInstanceGroup_SecondaryDevice::clear_kind() { _impl_.kind_ = 0; } inline ::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind ModelInstanceGroup_SecondaryDevice::_internal_kind() const { return static_cast< ::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind >(_impl_.kind_); } inline ::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind ModelInstanceGroup_SecondaryDevice::kind() const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.SecondaryDevice.kind) return _internal_kind(); } inline void ModelInstanceGroup_SecondaryDevice::_internal_set_kind(::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind value) { _impl_.kind_ = value; } inline void ModelInstanceGroup_SecondaryDevice::set_kind(::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind value) { _internal_set_kind(value); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.SecondaryDevice.kind) } // int64 device_id = 2; inline void ModelInstanceGroup_SecondaryDevice::clear_device_id() { _impl_.device_id_ = int64_t{0}; } inline int64_t ModelInstanceGroup_SecondaryDevice::_internal_device_id() const { return _impl_.device_id_; } inline int64_t ModelInstanceGroup_SecondaryDevice::device_id() const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.SecondaryDevice.device_id) return _internal_device_id(); } inline void ModelInstanceGroup_SecondaryDevice::_internal_set_device_id(int64_t value) { _impl_.device_id_ = value; } inline void ModelInstanceGroup_SecondaryDevice::set_device_id(int64_t value) { _internal_set_device_id(value); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.SecondaryDevice.device_id) } // ------------------------------------------------------------------- // ModelInstanceGroup // string name = 1; inline void ModelInstanceGroup::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelInstanceGroup::name() const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelInstanceGroup::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.name) } inline std::string* ModelInstanceGroup::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelInstanceGroup.name) return _s; } inline const std::string& ModelInstanceGroup::_internal_name() const { return _impl_.name_.Get(); } inline void ModelInstanceGroup::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelInstanceGroup::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelInstanceGroup::release_name() { // @@protoc_insertion_point(field_release:inference.ModelInstanceGroup.name) return _impl_.name_.Release(); } inline void ModelInstanceGroup::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelInstanceGroup.name) } // .inference.ModelInstanceGroup.Kind kind = 4; inline void ModelInstanceGroup::clear_kind() { _impl_.kind_ = 0; } inline ::inference::ModelInstanceGroup_Kind ModelInstanceGroup::_internal_kind() const { return static_cast< ::inference::ModelInstanceGroup_Kind >(_impl_.kind_); } inline ::inference::ModelInstanceGroup_Kind ModelInstanceGroup::kind() const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.kind) return _internal_kind(); } inline void ModelInstanceGroup::_internal_set_kind(::inference::ModelInstanceGroup_Kind value) { _impl_.kind_ = value; } inline void ModelInstanceGroup::set_kind(::inference::ModelInstanceGroup_Kind value) { _internal_set_kind(value); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.kind) } // int32 count = 2; inline void ModelInstanceGroup::clear_count() { _impl_.count_ = 0; } inline int32_t ModelInstanceGroup::_internal_count() const { return _impl_.count_; } inline int32_t ModelInstanceGroup::count() const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.count) return _internal_count(); } inline void ModelInstanceGroup::_internal_set_count(int32_t value) { _impl_.count_ = value; } inline void ModelInstanceGroup::set_count(int32_t value) { _internal_set_count(value); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.count) } // .inference.ModelRateLimiter rate_limiter = 6; inline bool ModelInstanceGroup::_internal_has_rate_limiter() const { return this != internal_default_instance() && _impl_.rate_limiter_ != nullptr; } inline bool ModelInstanceGroup::has_rate_limiter() const { return _internal_has_rate_limiter(); } inline void ModelInstanceGroup::clear_rate_limiter() { if (GetArenaForAllocation() == nullptr && _impl_.rate_limiter_ != nullptr) { delete _impl_.rate_limiter_; } _impl_.rate_limiter_ = nullptr; } inline const ::inference::ModelRateLimiter& ModelInstanceGroup::_internal_rate_limiter() const { const ::inference::ModelRateLimiter* p = _impl_.rate_limiter_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelRateLimiter_default_instance_); } inline const ::inference::ModelRateLimiter& ModelInstanceGroup::rate_limiter() const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.rate_limiter) return _internal_rate_limiter(); } inline void ModelInstanceGroup::unsafe_arena_set_allocated_rate_limiter( ::inference::ModelRateLimiter* rate_limiter) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.rate_limiter_); } _impl_.rate_limiter_ = rate_limiter; if (rate_limiter) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelInstanceGroup.rate_limiter) } inline ::inference::ModelRateLimiter* ModelInstanceGroup::release_rate_limiter() { ::inference::ModelRateLimiter* temp = _impl_.rate_limiter_; _impl_.rate_limiter_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelRateLimiter* ModelInstanceGroup::unsafe_arena_release_rate_limiter() { // @@protoc_insertion_point(field_release:inference.ModelInstanceGroup.rate_limiter) ::inference::ModelRateLimiter* temp = _impl_.rate_limiter_; _impl_.rate_limiter_ = nullptr; return temp; } inline ::inference::ModelRateLimiter* ModelInstanceGroup::_internal_mutable_rate_limiter() { if (_impl_.rate_limiter_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelRateLimiter>(GetArenaForAllocation()); _impl_.rate_limiter_ = p; } return _impl_.rate_limiter_; } inline ::inference::ModelRateLimiter* ModelInstanceGroup::mutable_rate_limiter() { ::inference::ModelRateLimiter* _msg = _internal_mutable_rate_limiter(); // @@protoc_insertion_point(field_mutable:inference.ModelInstanceGroup.rate_limiter) return _msg; } inline void ModelInstanceGroup::set_allocated_rate_limiter(::inference::ModelRateLimiter* rate_limiter) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.rate_limiter_; } if (rate_limiter) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(rate_limiter); if (message_arena != submessage_arena) { rate_limiter = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, rate_limiter, submessage_arena); } } else { } _impl_.rate_limiter_ = rate_limiter; // @@protoc_insertion_point(field_set_allocated:inference.ModelInstanceGroup.rate_limiter) } // repeated int32 gpus = 3; inline int ModelInstanceGroup::_internal_gpus_size() const { return _impl_.gpus_.size(); } inline int ModelInstanceGroup::gpus_size() const { return _internal_gpus_size(); } inline void ModelInstanceGroup::clear_gpus() { _impl_.gpus_.Clear(); } inline int32_t ModelInstanceGroup::_internal_gpus(int index) const { return _impl_.gpus_.Get(index); } inline int32_t ModelInstanceGroup::gpus(int index) const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.gpus) return _internal_gpus(index); } inline void ModelInstanceGroup::set_gpus(int index, int32_t value) { _impl_.gpus_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.gpus) } inline void ModelInstanceGroup::_internal_add_gpus(int32_t value) { _impl_.gpus_.Add(value); } inline void ModelInstanceGroup::add_gpus(int32_t value) { _internal_add_gpus(value); // @@protoc_insertion_point(field_add:inference.ModelInstanceGroup.gpus) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& ModelInstanceGroup::_internal_gpus() const { return _impl_.gpus_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& ModelInstanceGroup::gpus() const { // @@protoc_insertion_point(field_list:inference.ModelInstanceGroup.gpus) return _internal_gpus(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* ModelInstanceGroup::_internal_mutable_gpus() { return &_impl_.gpus_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* ModelInstanceGroup::mutable_gpus() { // @@protoc_insertion_point(field_mutable_list:inference.ModelInstanceGroup.gpus) return _internal_mutable_gpus(); } // repeated .inference.ModelInstanceGroup.SecondaryDevice secondary_devices = 8; inline int ModelInstanceGroup::_internal_secondary_devices_size() const { return _impl_.secondary_devices_.size(); } inline int ModelInstanceGroup::secondary_devices_size() const { return _internal_secondary_devices_size(); } inline void ModelInstanceGroup::clear_secondary_devices() { _impl_.secondary_devices_.Clear(); } inline ::inference::ModelInstanceGroup_SecondaryDevice* ModelInstanceGroup::mutable_secondary_devices(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelInstanceGroup.secondary_devices) return _impl_.secondary_devices_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup_SecondaryDevice >* ModelInstanceGroup::mutable_secondary_devices() { // @@protoc_insertion_point(field_mutable_list:inference.ModelInstanceGroup.secondary_devices) return &_impl_.secondary_devices_; } inline const ::inference::ModelInstanceGroup_SecondaryDevice& ModelInstanceGroup::_internal_secondary_devices(int index) const { return _impl_.secondary_devices_.Get(index); } inline const ::inference::ModelInstanceGroup_SecondaryDevice& ModelInstanceGroup::secondary_devices(int index) const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.secondary_devices) return _internal_secondary_devices(index); } inline ::inference::ModelInstanceGroup_SecondaryDevice* ModelInstanceGroup::_internal_add_secondary_devices() { return _impl_.secondary_devices_.Add(); } inline ::inference::ModelInstanceGroup_SecondaryDevice* ModelInstanceGroup::add_secondary_devices() { ::inference::ModelInstanceGroup_SecondaryDevice* _add = _internal_add_secondary_devices(); // @@protoc_insertion_point(field_add:inference.ModelInstanceGroup.secondary_devices) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup_SecondaryDevice >& ModelInstanceGroup::secondary_devices() const { // @@protoc_insertion_point(field_list:inference.ModelInstanceGroup.secondary_devices) return _impl_.secondary_devices_; } // repeated string profile = 5; inline int ModelInstanceGroup::_internal_profile_size() const { return _impl_.profile_.size(); } inline int ModelInstanceGroup::profile_size() const { return _internal_profile_size(); } inline void ModelInstanceGroup::clear_profile() { _impl_.profile_.Clear(); } inline std::string* ModelInstanceGroup::add_profile() { std::string* _s = _internal_add_profile(); // @@protoc_insertion_point(field_add_mutable:inference.ModelInstanceGroup.profile) return _s; } inline const std::string& ModelInstanceGroup::_internal_profile(int index) const { return _impl_.profile_.Get(index); } inline const std::string& ModelInstanceGroup::profile(int index) const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.profile) return _internal_profile(index); } inline std::string* ModelInstanceGroup::mutable_profile(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelInstanceGroup.profile) return _impl_.profile_.Mutable(index); } inline void ModelInstanceGroup::set_profile(int index, const std::string& value) { _impl_.profile_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.profile) } inline void ModelInstanceGroup::set_profile(int index, std::string&& value) { _impl_.profile_.Mutable(index)->assign(std::move(value)); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.profile) } inline void ModelInstanceGroup::set_profile(int index, const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.profile_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:inference.ModelInstanceGroup.profile) } inline void ModelInstanceGroup::set_profile(int index, const char* value, size_t size) { _impl_.profile_.Mutable(index)->assign( reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:inference.ModelInstanceGroup.profile) } inline std::string* ModelInstanceGroup::_internal_add_profile() { return _impl_.profile_.Add(); } inline void ModelInstanceGroup::add_profile(const std::string& value) { _impl_.profile_.Add()->assign(value); // @@protoc_insertion_point(field_add:inference.ModelInstanceGroup.profile) } inline void ModelInstanceGroup::add_profile(std::string&& value) { _impl_.profile_.Add(std::move(value)); // @@protoc_insertion_point(field_add:inference.ModelInstanceGroup.profile) } inline void ModelInstanceGroup::add_profile(const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.profile_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:inference.ModelInstanceGroup.profile) } inline void ModelInstanceGroup::add_profile(const char* value, size_t size) { _impl_.profile_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:inference.ModelInstanceGroup.profile) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& ModelInstanceGroup::profile() const { // @@protoc_insertion_point(field_list:inference.ModelInstanceGroup.profile) return _impl_.profile_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* ModelInstanceGroup::mutable_profile() { // @@protoc_insertion_point(field_mutable_list:inference.ModelInstanceGroup.profile) return &_impl_.profile_; } // bool passive = 7; inline void ModelInstanceGroup::clear_passive() { _impl_.passive_ = false; } inline bool ModelInstanceGroup::_internal_passive() const { return _impl_.passive_; } inline bool ModelInstanceGroup::passive() const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.passive) return _internal_passive(); } inline void ModelInstanceGroup::_internal_set_passive(bool value) { _impl_.passive_ = value; } inline void ModelInstanceGroup::set_passive(bool value) { _internal_set_passive(value); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.passive) } // string host_policy = 9; inline void ModelInstanceGroup::clear_host_policy() { _impl_.host_policy_.ClearToEmpty(); } inline const std::string& ModelInstanceGroup::host_policy() const { // @@protoc_insertion_point(field_get:inference.ModelInstanceGroup.host_policy) return _internal_host_policy(); } template inline PROTOBUF_ALWAYS_INLINE void ModelInstanceGroup::set_host_policy(ArgT0&& arg0, ArgT... args) { _impl_.host_policy_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelInstanceGroup.host_policy) } inline std::string* ModelInstanceGroup::mutable_host_policy() { std::string* _s = _internal_mutable_host_policy(); // @@protoc_insertion_point(field_mutable:inference.ModelInstanceGroup.host_policy) return _s; } inline const std::string& ModelInstanceGroup::_internal_host_policy() const { return _impl_.host_policy_.Get(); } inline void ModelInstanceGroup::_internal_set_host_policy(const std::string& value) { _impl_.host_policy_.Set(value, GetArenaForAllocation()); } inline std::string* ModelInstanceGroup::_internal_mutable_host_policy() { return _impl_.host_policy_.Mutable(GetArenaForAllocation()); } inline std::string* ModelInstanceGroup::release_host_policy() { // @@protoc_insertion_point(field_release:inference.ModelInstanceGroup.host_policy) return _impl_.host_policy_.Release(); } inline void ModelInstanceGroup::set_allocated_host_policy(std::string* host_policy) { if (host_policy != nullptr) { } else { } _impl_.host_policy_.SetAllocated(host_policy, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.host_policy_.IsDefault()) { _impl_.host_policy_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelInstanceGroup.host_policy) } // ------------------------------------------------------------------- // ModelTensorReshape // repeated int64 shape = 1; inline int ModelTensorReshape::_internal_shape_size() const { return _impl_.shape_.size(); } inline int ModelTensorReshape::shape_size() const { return _internal_shape_size(); } inline void ModelTensorReshape::clear_shape() { _impl_.shape_.Clear(); } inline int64_t ModelTensorReshape::_internal_shape(int index) const { return _impl_.shape_.Get(index); } inline int64_t ModelTensorReshape::shape(int index) const { // @@protoc_insertion_point(field_get:inference.ModelTensorReshape.shape) return _internal_shape(index); } inline void ModelTensorReshape::set_shape(int index, int64_t value) { _impl_.shape_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelTensorReshape.shape) } inline void ModelTensorReshape::_internal_add_shape(int64_t value) { _impl_.shape_.Add(value); } inline void ModelTensorReshape::add_shape(int64_t value) { _internal_add_shape(value); // @@protoc_insertion_point(field_add:inference.ModelTensorReshape.shape) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelTensorReshape::_internal_shape() const { return _impl_.shape_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelTensorReshape::shape() const { // @@protoc_insertion_point(field_list:inference.ModelTensorReshape.shape) return _internal_shape(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelTensorReshape::_internal_mutable_shape() { return &_impl_.shape_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelTensorReshape::mutable_shape() { // @@protoc_insertion_point(field_mutable_list:inference.ModelTensorReshape.shape) return _internal_mutable_shape(); } // ------------------------------------------------------------------- // ModelInput // string name = 1; inline void ModelInput::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelInput::name() const { // @@protoc_insertion_point(field_get:inference.ModelInput.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelInput::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelInput.name) } inline std::string* ModelInput::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelInput.name) return _s; } inline const std::string& ModelInput::_internal_name() const { return _impl_.name_.Get(); } inline void ModelInput::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelInput::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelInput::release_name() { // @@protoc_insertion_point(field_release:inference.ModelInput.name) return _impl_.name_.Release(); } inline void ModelInput::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelInput.name) } // .inference.DataType data_type = 2; inline void ModelInput::clear_data_type() { _impl_.data_type_ = 0; } inline ::inference::DataType ModelInput::_internal_data_type() const { return static_cast< ::inference::DataType >(_impl_.data_type_); } inline ::inference::DataType ModelInput::data_type() const { // @@protoc_insertion_point(field_get:inference.ModelInput.data_type) return _internal_data_type(); } inline void ModelInput::_internal_set_data_type(::inference::DataType value) { _impl_.data_type_ = value; } inline void ModelInput::set_data_type(::inference::DataType value) { _internal_set_data_type(value); // @@protoc_insertion_point(field_set:inference.ModelInput.data_type) } // .inference.ModelInput.Format format = 3; inline void ModelInput::clear_format() { _impl_.format_ = 0; } inline ::inference::ModelInput_Format ModelInput::_internal_format() const { return static_cast< ::inference::ModelInput_Format >(_impl_.format_); } inline ::inference::ModelInput_Format ModelInput::format() const { // @@protoc_insertion_point(field_get:inference.ModelInput.format) return _internal_format(); } inline void ModelInput::_internal_set_format(::inference::ModelInput_Format value) { _impl_.format_ = value; } inline void ModelInput::set_format(::inference::ModelInput_Format value) { _internal_set_format(value); // @@protoc_insertion_point(field_set:inference.ModelInput.format) } // repeated int64 dims = 4; inline int ModelInput::_internal_dims_size() const { return _impl_.dims_.size(); } inline int ModelInput::dims_size() const { return _internal_dims_size(); } inline void ModelInput::clear_dims() { _impl_.dims_.Clear(); } inline int64_t ModelInput::_internal_dims(int index) const { return _impl_.dims_.Get(index); } inline int64_t ModelInput::dims(int index) const { // @@protoc_insertion_point(field_get:inference.ModelInput.dims) return _internal_dims(index); } inline void ModelInput::set_dims(int index, int64_t value) { _impl_.dims_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelInput.dims) } inline void ModelInput::_internal_add_dims(int64_t value) { _impl_.dims_.Add(value); } inline void ModelInput::add_dims(int64_t value) { _internal_add_dims(value); // @@protoc_insertion_point(field_add:inference.ModelInput.dims) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelInput::_internal_dims() const { return _impl_.dims_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelInput::dims() const { // @@protoc_insertion_point(field_list:inference.ModelInput.dims) return _internal_dims(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelInput::_internal_mutable_dims() { return &_impl_.dims_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelInput::mutable_dims() { // @@protoc_insertion_point(field_mutable_list:inference.ModelInput.dims) return _internal_mutable_dims(); } // .inference.ModelTensorReshape reshape = 5; inline bool ModelInput::_internal_has_reshape() const { return this != internal_default_instance() && _impl_.reshape_ != nullptr; } inline bool ModelInput::has_reshape() const { return _internal_has_reshape(); } inline void ModelInput::clear_reshape() { if (GetArenaForAllocation() == nullptr && _impl_.reshape_ != nullptr) { delete _impl_.reshape_; } _impl_.reshape_ = nullptr; } inline const ::inference::ModelTensorReshape& ModelInput::_internal_reshape() const { const ::inference::ModelTensorReshape* p = _impl_.reshape_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelTensorReshape_default_instance_); } inline const ::inference::ModelTensorReshape& ModelInput::reshape() const { // @@protoc_insertion_point(field_get:inference.ModelInput.reshape) return _internal_reshape(); } inline void ModelInput::unsafe_arena_set_allocated_reshape( ::inference::ModelTensorReshape* reshape) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.reshape_); } _impl_.reshape_ = reshape; if (reshape) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelInput.reshape) } inline ::inference::ModelTensorReshape* ModelInput::release_reshape() { ::inference::ModelTensorReshape* temp = _impl_.reshape_; _impl_.reshape_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelTensorReshape* ModelInput::unsafe_arena_release_reshape() { // @@protoc_insertion_point(field_release:inference.ModelInput.reshape) ::inference::ModelTensorReshape* temp = _impl_.reshape_; _impl_.reshape_ = nullptr; return temp; } inline ::inference::ModelTensorReshape* ModelInput::_internal_mutable_reshape() { if (_impl_.reshape_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelTensorReshape>(GetArenaForAllocation()); _impl_.reshape_ = p; } return _impl_.reshape_; } inline ::inference::ModelTensorReshape* ModelInput::mutable_reshape() { ::inference::ModelTensorReshape* _msg = _internal_mutable_reshape(); // @@protoc_insertion_point(field_mutable:inference.ModelInput.reshape) return _msg; } inline void ModelInput::set_allocated_reshape(::inference::ModelTensorReshape* reshape) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.reshape_; } if (reshape) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(reshape); if (message_arena != submessage_arena) { reshape = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, reshape, submessage_arena); } } else { } _impl_.reshape_ = reshape; // @@protoc_insertion_point(field_set_allocated:inference.ModelInput.reshape) } // bool is_shape_tensor = 6; inline void ModelInput::clear_is_shape_tensor() { _impl_.is_shape_tensor_ = false; } inline bool ModelInput::_internal_is_shape_tensor() const { return _impl_.is_shape_tensor_; } inline bool ModelInput::is_shape_tensor() const { // @@protoc_insertion_point(field_get:inference.ModelInput.is_shape_tensor) return _internal_is_shape_tensor(); } inline void ModelInput::_internal_set_is_shape_tensor(bool value) { _impl_.is_shape_tensor_ = value; } inline void ModelInput::set_is_shape_tensor(bool value) { _internal_set_is_shape_tensor(value); // @@protoc_insertion_point(field_set:inference.ModelInput.is_shape_tensor) } // bool allow_ragged_batch = 7; inline void ModelInput::clear_allow_ragged_batch() { _impl_.allow_ragged_batch_ = false; } inline bool ModelInput::_internal_allow_ragged_batch() const { return _impl_.allow_ragged_batch_; } inline bool ModelInput::allow_ragged_batch() const { // @@protoc_insertion_point(field_get:inference.ModelInput.allow_ragged_batch) return _internal_allow_ragged_batch(); } inline void ModelInput::_internal_set_allow_ragged_batch(bool value) { _impl_.allow_ragged_batch_ = value; } inline void ModelInput::set_allow_ragged_batch(bool value) { _internal_set_allow_ragged_batch(value); // @@protoc_insertion_point(field_set:inference.ModelInput.allow_ragged_batch) } // bool optional = 8; inline void ModelInput::clear_optional() { _impl_.optional_ = false; } inline bool ModelInput::_internal_optional() const { return _impl_.optional_; } inline bool ModelInput::optional() const { // @@protoc_insertion_point(field_get:inference.ModelInput.optional) return _internal_optional(); } inline void ModelInput::_internal_set_optional(bool value) { _impl_.optional_ = value; } inline void ModelInput::set_optional(bool value) { _internal_set_optional(value); // @@protoc_insertion_point(field_set:inference.ModelInput.optional) } // bool is_non_linear_format_io = 9; inline void ModelInput::clear_is_non_linear_format_io() { _impl_.is_non_linear_format_io_ = false; } inline bool ModelInput::_internal_is_non_linear_format_io() const { return _impl_.is_non_linear_format_io_; } inline bool ModelInput::is_non_linear_format_io() const { // @@protoc_insertion_point(field_get:inference.ModelInput.is_non_linear_format_io) return _internal_is_non_linear_format_io(); } inline void ModelInput::_internal_set_is_non_linear_format_io(bool value) { _impl_.is_non_linear_format_io_ = value; } inline void ModelInput::set_is_non_linear_format_io(bool value) { _internal_set_is_non_linear_format_io(value); // @@protoc_insertion_point(field_set:inference.ModelInput.is_non_linear_format_io) } // ------------------------------------------------------------------- // ModelOutput // string name = 1; inline void ModelOutput::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelOutput::name() const { // @@protoc_insertion_point(field_get:inference.ModelOutput.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelOutput::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelOutput.name) } inline std::string* ModelOutput::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelOutput.name) return _s; } inline const std::string& ModelOutput::_internal_name() const { return _impl_.name_.Get(); } inline void ModelOutput::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelOutput::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelOutput::release_name() { // @@protoc_insertion_point(field_release:inference.ModelOutput.name) return _impl_.name_.Release(); } inline void ModelOutput::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelOutput.name) } // .inference.DataType data_type = 2; inline void ModelOutput::clear_data_type() { _impl_.data_type_ = 0; } inline ::inference::DataType ModelOutput::_internal_data_type() const { return static_cast< ::inference::DataType >(_impl_.data_type_); } inline ::inference::DataType ModelOutput::data_type() const { // @@protoc_insertion_point(field_get:inference.ModelOutput.data_type) return _internal_data_type(); } inline void ModelOutput::_internal_set_data_type(::inference::DataType value) { _impl_.data_type_ = value; } inline void ModelOutput::set_data_type(::inference::DataType value) { _internal_set_data_type(value); // @@protoc_insertion_point(field_set:inference.ModelOutput.data_type) } // repeated int64 dims = 3; inline int ModelOutput::_internal_dims_size() const { return _impl_.dims_.size(); } inline int ModelOutput::dims_size() const { return _internal_dims_size(); } inline void ModelOutput::clear_dims() { _impl_.dims_.Clear(); } inline int64_t ModelOutput::_internal_dims(int index) const { return _impl_.dims_.Get(index); } inline int64_t ModelOutput::dims(int index) const { // @@protoc_insertion_point(field_get:inference.ModelOutput.dims) return _internal_dims(index); } inline void ModelOutput::set_dims(int index, int64_t value) { _impl_.dims_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelOutput.dims) } inline void ModelOutput::_internal_add_dims(int64_t value) { _impl_.dims_.Add(value); } inline void ModelOutput::add_dims(int64_t value) { _internal_add_dims(value); // @@protoc_insertion_point(field_add:inference.ModelOutput.dims) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelOutput::_internal_dims() const { return _impl_.dims_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelOutput::dims() const { // @@protoc_insertion_point(field_list:inference.ModelOutput.dims) return _internal_dims(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelOutput::_internal_mutable_dims() { return &_impl_.dims_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelOutput::mutable_dims() { // @@protoc_insertion_point(field_mutable_list:inference.ModelOutput.dims) return _internal_mutable_dims(); } // .inference.ModelTensorReshape reshape = 5; inline bool ModelOutput::_internal_has_reshape() const { return this != internal_default_instance() && _impl_.reshape_ != nullptr; } inline bool ModelOutput::has_reshape() const { return _internal_has_reshape(); } inline void ModelOutput::clear_reshape() { if (GetArenaForAllocation() == nullptr && _impl_.reshape_ != nullptr) { delete _impl_.reshape_; } _impl_.reshape_ = nullptr; } inline const ::inference::ModelTensorReshape& ModelOutput::_internal_reshape() const { const ::inference::ModelTensorReshape* p = _impl_.reshape_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelTensorReshape_default_instance_); } inline const ::inference::ModelTensorReshape& ModelOutput::reshape() const { // @@protoc_insertion_point(field_get:inference.ModelOutput.reshape) return _internal_reshape(); } inline void ModelOutput::unsafe_arena_set_allocated_reshape( ::inference::ModelTensorReshape* reshape) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.reshape_); } _impl_.reshape_ = reshape; if (reshape) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelOutput.reshape) } inline ::inference::ModelTensorReshape* ModelOutput::release_reshape() { ::inference::ModelTensorReshape* temp = _impl_.reshape_; _impl_.reshape_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelTensorReshape* ModelOutput::unsafe_arena_release_reshape() { // @@protoc_insertion_point(field_release:inference.ModelOutput.reshape) ::inference::ModelTensorReshape* temp = _impl_.reshape_; _impl_.reshape_ = nullptr; return temp; } inline ::inference::ModelTensorReshape* ModelOutput::_internal_mutable_reshape() { if (_impl_.reshape_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelTensorReshape>(GetArenaForAllocation()); _impl_.reshape_ = p; } return _impl_.reshape_; } inline ::inference::ModelTensorReshape* ModelOutput::mutable_reshape() { ::inference::ModelTensorReshape* _msg = _internal_mutable_reshape(); // @@protoc_insertion_point(field_mutable:inference.ModelOutput.reshape) return _msg; } inline void ModelOutput::set_allocated_reshape(::inference::ModelTensorReshape* reshape) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.reshape_; } if (reshape) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(reshape); if (message_arena != submessage_arena) { reshape = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, reshape, submessage_arena); } } else { } _impl_.reshape_ = reshape; // @@protoc_insertion_point(field_set_allocated:inference.ModelOutput.reshape) } // string label_filename = 4; inline void ModelOutput::clear_label_filename() { _impl_.label_filename_.ClearToEmpty(); } inline const std::string& ModelOutput::label_filename() const { // @@protoc_insertion_point(field_get:inference.ModelOutput.label_filename) return _internal_label_filename(); } template inline PROTOBUF_ALWAYS_INLINE void ModelOutput::set_label_filename(ArgT0&& arg0, ArgT... args) { _impl_.label_filename_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelOutput.label_filename) } inline std::string* ModelOutput::mutable_label_filename() { std::string* _s = _internal_mutable_label_filename(); // @@protoc_insertion_point(field_mutable:inference.ModelOutput.label_filename) return _s; } inline const std::string& ModelOutput::_internal_label_filename() const { return _impl_.label_filename_.Get(); } inline void ModelOutput::_internal_set_label_filename(const std::string& value) { _impl_.label_filename_.Set(value, GetArenaForAllocation()); } inline std::string* ModelOutput::_internal_mutable_label_filename() { return _impl_.label_filename_.Mutable(GetArenaForAllocation()); } inline std::string* ModelOutput::release_label_filename() { // @@protoc_insertion_point(field_release:inference.ModelOutput.label_filename) return _impl_.label_filename_.Release(); } inline void ModelOutput::set_allocated_label_filename(std::string* label_filename) { if (label_filename != nullptr) { } else { } _impl_.label_filename_.SetAllocated(label_filename, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.label_filename_.IsDefault()) { _impl_.label_filename_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelOutput.label_filename) } // bool is_shape_tensor = 6; inline void ModelOutput::clear_is_shape_tensor() { _impl_.is_shape_tensor_ = false; } inline bool ModelOutput::_internal_is_shape_tensor() const { return _impl_.is_shape_tensor_; } inline bool ModelOutput::is_shape_tensor() const { // @@protoc_insertion_point(field_get:inference.ModelOutput.is_shape_tensor) return _internal_is_shape_tensor(); } inline void ModelOutput::_internal_set_is_shape_tensor(bool value) { _impl_.is_shape_tensor_ = value; } inline void ModelOutput::set_is_shape_tensor(bool value) { _internal_set_is_shape_tensor(value); // @@protoc_insertion_point(field_set:inference.ModelOutput.is_shape_tensor) } // bool is_non_linear_format_io = 7; inline void ModelOutput::clear_is_non_linear_format_io() { _impl_.is_non_linear_format_io_ = false; } inline bool ModelOutput::_internal_is_non_linear_format_io() const { return _impl_.is_non_linear_format_io_; } inline bool ModelOutput::is_non_linear_format_io() const { // @@protoc_insertion_point(field_get:inference.ModelOutput.is_non_linear_format_io) return _internal_is_non_linear_format_io(); } inline void ModelOutput::_internal_set_is_non_linear_format_io(bool value) { _impl_.is_non_linear_format_io_ = value; } inline void ModelOutput::set_is_non_linear_format_io(bool value) { _internal_set_is_non_linear_format_io(value); // @@protoc_insertion_point(field_set:inference.ModelOutput.is_non_linear_format_io) } // ------------------------------------------------------------------- // BatchInput // .inference.BatchInput.Kind kind = 1; inline void BatchInput::clear_kind() { _impl_.kind_ = 0; } inline ::inference::BatchInput_Kind BatchInput::_internal_kind() const { return static_cast< ::inference::BatchInput_Kind >(_impl_.kind_); } inline ::inference::BatchInput_Kind BatchInput::kind() const { // @@protoc_insertion_point(field_get:inference.BatchInput.kind) return _internal_kind(); } inline void BatchInput::_internal_set_kind(::inference::BatchInput_Kind value) { _impl_.kind_ = value; } inline void BatchInput::set_kind(::inference::BatchInput_Kind value) { _internal_set_kind(value); // @@protoc_insertion_point(field_set:inference.BatchInput.kind) } // repeated string target_name = 2; inline int BatchInput::_internal_target_name_size() const { return _impl_.target_name_.size(); } inline int BatchInput::target_name_size() const { return _internal_target_name_size(); } inline void BatchInput::clear_target_name() { _impl_.target_name_.Clear(); } inline std::string* BatchInput::add_target_name() { std::string* _s = _internal_add_target_name(); // @@protoc_insertion_point(field_add_mutable:inference.BatchInput.target_name) return _s; } inline const std::string& BatchInput::_internal_target_name(int index) const { return _impl_.target_name_.Get(index); } inline const std::string& BatchInput::target_name(int index) const { // @@protoc_insertion_point(field_get:inference.BatchInput.target_name) return _internal_target_name(index); } inline std::string* BatchInput::mutable_target_name(int index) { // @@protoc_insertion_point(field_mutable:inference.BatchInput.target_name) return _impl_.target_name_.Mutable(index); } inline void BatchInput::set_target_name(int index, const std::string& value) { _impl_.target_name_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set:inference.BatchInput.target_name) } inline void BatchInput::set_target_name(int index, std::string&& value) { _impl_.target_name_.Mutable(index)->assign(std::move(value)); // @@protoc_insertion_point(field_set:inference.BatchInput.target_name) } inline void BatchInput::set_target_name(int index, const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.target_name_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:inference.BatchInput.target_name) } inline void BatchInput::set_target_name(int index, const char* value, size_t size) { _impl_.target_name_.Mutable(index)->assign( reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:inference.BatchInput.target_name) } inline std::string* BatchInput::_internal_add_target_name() { return _impl_.target_name_.Add(); } inline void BatchInput::add_target_name(const std::string& value) { _impl_.target_name_.Add()->assign(value); // @@protoc_insertion_point(field_add:inference.BatchInput.target_name) } inline void BatchInput::add_target_name(std::string&& value) { _impl_.target_name_.Add(std::move(value)); // @@protoc_insertion_point(field_add:inference.BatchInput.target_name) } inline void BatchInput::add_target_name(const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.target_name_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:inference.BatchInput.target_name) } inline void BatchInput::add_target_name(const char* value, size_t size) { _impl_.target_name_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:inference.BatchInput.target_name) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& BatchInput::target_name() const { // @@protoc_insertion_point(field_list:inference.BatchInput.target_name) return _impl_.target_name_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* BatchInput::mutable_target_name() { // @@protoc_insertion_point(field_mutable_list:inference.BatchInput.target_name) return &_impl_.target_name_; } // .inference.DataType data_type = 3; inline void BatchInput::clear_data_type() { _impl_.data_type_ = 0; } inline ::inference::DataType BatchInput::_internal_data_type() const { return static_cast< ::inference::DataType >(_impl_.data_type_); } inline ::inference::DataType BatchInput::data_type() const { // @@protoc_insertion_point(field_get:inference.BatchInput.data_type) return _internal_data_type(); } inline void BatchInput::_internal_set_data_type(::inference::DataType value) { _impl_.data_type_ = value; } inline void BatchInput::set_data_type(::inference::DataType value) { _internal_set_data_type(value); // @@protoc_insertion_point(field_set:inference.BatchInput.data_type) } // repeated string source_input = 4; inline int BatchInput::_internal_source_input_size() const { return _impl_.source_input_.size(); } inline int BatchInput::source_input_size() const { return _internal_source_input_size(); } inline void BatchInput::clear_source_input() { _impl_.source_input_.Clear(); } inline std::string* BatchInput::add_source_input() { std::string* _s = _internal_add_source_input(); // @@protoc_insertion_point(field_add_mutable:inference.BatchInput.source_input) return _s; } inline const std::string& BatchInput::_internal_source_input(int index) const { return _impl_.source_input_.Get(index); } inline const std::string& BatchInput::source_input(int index) const { // @@protoc_insertion_point(field_get:inference.BatchInput.source_input) return _internal_source_input(index); } inline std::string* BatchInput::mutable_source_input(int index) { // @@protoc_insertion_point(field_mutable:inference.BatchInput.source_input) return _impl_.source_input_.Mutable(index); } inline void BatchInput::set_source_input(int index, const std::string& value) { _impl_.source_input_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set:inference.BatchInput.source_input) } inline void BatchInput::set_source_input(int index, std::string&& value) { _impl_.source_input_.Mutable(index)->assign(std::move(value)); // @@protoc_insertion_point(field_set:inference.BatchInput.source_input) } inline void BatchInput::set_source_input(int index, const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.source_input_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:inference.BatchInput.source_input) } inline void BatchInput::set_source_input(int index, const char* value, size_t size) { _impl_.source_input_.Mutable(index)->assign( reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:inference.BatchInput.source_input) } inline std::string* BatchInput::_internal_add_source_input() { return _impl_.source_input_.Add(); } inline void BatchInput::add_source_input(const std::string& value) { _impl_.source_input_.Add()->assign(value); // @@protoc_insertion_point(field_add:inference.BatchInput.source_input) } inline void BatchInput::add_source_input(std::string&& value) { _impl_.source_input_.Add(std::move(value)); // @@protoc_insertion_point(field_add:inference.BatchInput.source_input) } inline void BatchInput::add_source_input(const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.source_input_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:inference.BatchInput.source_input) } inline void BatchInput::add_source_input(const char* value, size_t size) { _impl_.source_input_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:inference.BatchInput.source_input) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& BatchInput::source_input() const { // @@protoc_insertion_point(field_list:inference.BatchInput.source_input) return _impl_.source_input_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* BatchInput::mutable_source_input() { // @@protoc_insertion_point(field_mutable_list:inference.BatchInput.source_input) return &_impl_.source_input_; } // ------------------------------------------------------------------- // BatchOutput // repeated string target_name = 1; inline int BatchOutput::_internal_target_name_size() const { return _impl_.target_name_.size(); } inline int BatchOutput::target_name_size() const { return _internal_target_name_size(); } inline void BatchOutput::clear_target_name() { _impl_.target_name_.Clear(); } inline std::string* BatchOutput::add_target_name() { std::string* _s = _internal_add_target_name(); // @@protoc_insertion_point(field_add_mutable:inference.BatchOutput.target_name) return _s; } inline const std::string& BatchOutput::_internal_target_name(int index) const { return _impl_.target_name_.Get(index); } inline const std::string& BatchOutput::target_name(int index) const { // @@protoc_insertion_point(field_get:inference.BatchOutput.target_name) return _internal_target_name(index); } inline std::string* BatchOutput::mutable_target_name(int index) { // @@protoc_insertion_point(field_mutable:inference.BatchOutput.target_name) return _impl_.target_name_.Mutable(index); } inline void BatchOutput::set_target_name(int index, const std::string& value) { _impl_.target_name_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set:inference.BatchOutput.target_name) } inline void BatchOutput::set_target_name(int index, std::string&& value) { _impl_.target_name_.Mutable(index)->assign(std::move(value)); // @@protoc_insertion_point(field_set:inference.BatchOutput.target_name) } inline void BatchOutput::set_target_name(int index, const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.target_name_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:inference.BatchOutput.target_name) } inline void BatchOutput::set_target_name(int index, const char* value, size_t size) { _impl_.target_name_.Mutable(index)->assign( reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:inference.BatchOutput.target_name) } inline std::string* BatchOutput::_internal_add_target_name() { return _impl_.target_name_.Add(); } inline void BatchOutput::add_target_name(const std::string& value) { _impl_.target_name_.Add()->assign(value); // @@protoc_insertion_point(field_add:inference.BatchOutput.target_name) } inline void BatchOutput::add_target_name(std::string&& value) { _impl_.target_name_.Add(std::move(value)); // @@protoc_insertion_point(field_add:inference.BatchOutput.target_name) } inline void BatchOutput::add_target_name(const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.target_name_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:inference.BatchOutput.target_name) } inline void BatchOutput::add_target_name(const char* value, size_t size) { _impl_.target_name_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:inference.BatchOutput.target_name) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& BatchOutput::target_name() const { // @@protoc_insertion_point(field_list:inference.BatchOutput.target_name) return _impl_.target_name_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* BatchOutput::mutable_target_name() { // @@protoc_insertion_point(field_mutable_list:inference.BatchOutput.target_name) return &_impl_.target_name_; } // .inference.BatchOutput.Kind kind = 2; inline void BatchOutput::clear_kind() { _impl_.kind_ = 0; } inline ::inference::BatchOutput_Kind BatchOutput::_internal_kind() const { return static_cast< ::inference::BatchOutput_Kind >(_impl_.kind_); } inline ::inference::BatchOutput_Kind BatchOutput::kind() const { // @@protoc_insertion_point(field_get:inference.BatchOutput.kind) return _internal_kind(); } inline void BatchOutput::_internal_set_kind(::inference::BatchOutput_Kind value) { _impl_.kind_ = value; } inline void BatchOutput::set_kind(::inference::BatchOutput_Kind value) { _internal_set_kind(value); // @@protoc_insertion_point(field_set:inference.BatchOutput.kind) } // repeated string source_input = 3; inline int BatchOutput::_internal_source_input_size() const { return _impl_.source_input_.size(); } inline int BatchOutput::source_input_size() const { return _internal_source_input_size(); } inline void BatchOutput::clear_source_input() { _impl_.source_input_.Clear(); } inline std::string* BatchOutput::add_source_input() { std::string* _s = _internal_add_source_input(); // @@protoc_insertion_point(field_add_mutable:inference.BatchOutput.source_input) return _s; } inline const std::string& BatchOutput::_internal_source_input(int index) const { return _impl_.source_input_.Get(index); } inline const std::string& BatchOutput::source_input(int index) const { // @@protoc_insertion_point(field_get:inference.BatchOutput.source_input) return _internal_source_input(index); } inline std::string* BatchOutput::mutable_source_input(int index) { // @@protoc_insertion_point(field_mutable:inference.BatchOutput.source_input) return _impl_.source_input_.Mutable(index); } inline void BatchOutput::set_source_input(int index, const std::string& value) { _impl_.source_input_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set:inference.BatchOutput.source_input) } inline void BatchOutput::set_source_input(int index, std::string&& value) { _impl_.source_input_.Mutable(index)->assign(std::move(value)); // @@protoc_insertion_point(field_set:inference.BatchOutput.source_input) } inline void BatchOutput::set_source_input(int index, const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.source_input_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:inference.BatchOutput.source_input) } inline void BatchOutput::set_source_input(int index, const char* value, size_t size) { _impl_.source_input_.Mutable(index)->assign( reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:inference.BatchOutput.source_input) } inline std::string* BatchOutput::_internal_add_source_input() { return _impl_.source_input_.Add(); } inline void BatchOutput::add_source_input(const std::string& value) { _impl_.source_input_.Add()->assign(value); // @@protoc_insertion_point(field_add:inference.BatchOutput.source_input) } inline void BatchOutput::add_source_input(std::string&& value) { _impl_.source_input_.Add(std::move(value)); // @@protoc_insertion_point(field_add:inference.BatchOutput.source_input) } inline void BatchOutput::add_source_input(const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.source_input_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:inference.BatchOutput.source_input) } inline void BatchOutput::add_source_input(const char* value, size_t size) { _impl_.source_input_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:inference.BatchOutput.source_input) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& BatchOutput::source_input() const { // @@protoc_insertion_point(field_list:inference.BatchOutput.source_input) return _impl_.source_input_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* BatchOutput::mutable_source_input() { // @@protoc_insertion_point(field_mutable_list:inference.BatchOutput.source_input) return &_impl_.source_input_; } // ------------------------------------------------------------------- // ModelVersionPolicy_Latest // uint32 num_versions = 1; inline void ModelVersionPolicy_Latest::clear_num_versions() { _impl_.num_versions_ = 0u; } inline uint32_t ModelVersionPolicy_Latest::_internal_num_versions() const { return _impl_.num_versions_; } inline uint32_t ModelVersionPolicy_Latest::num_versions() const { // @@protoc_insertion_point(field_get:inference.ModelVersionPolicy.Latest.num_versions) return _internal_num_versions(); } inline void ModelVersionPolicy_Latest::_internal_set_num_versions(uint32_t value) { _impl_.num_versions_ = value; } inline void ModelVersionPolicy_Latest::set_num_versions(uint32_t value) { _internal_set_num_versions(value); // @@protoc_insertion_point(field_set:inference.ModelVersionPolicy.Latest.num_versions) } // ------------------------------------------------------------------- // ModelVersionPolicy_All // ------------------------------------------------------------------- // ModelVersionPolicy_Specific // repeated int64 versions = 1; inline int ModelVersionPolicy_Specific::_internal_versions_size() const { return _impl_.versions_.size(); } inline int ModelVersionPolicy_Specific::versions_size() const { return _internal_versions_size(); } inline void ModelVersionPolicy_Specific::clear_versions() { _impl_.versions_.Clear(); } inline int64_t ModelVersionPolicy_Specific::_internal_versions(int index) const { return _impl_.versions_.Get(index); } inline int64_t ModelVersionPolicy_Specific::versions(int index) const { // @@protoc_insertion_point(field_get:inference.ModelVersionPolicy.Specific.versions) return _internal_versions(index); } inline void ModelVersionPolicy_Specific::set_versions(int index, int64_t value) { _impl_.versions_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelVersionPolicy.Specific.versions) } inline void ModelVersionPolicy_Specific::_internal_add_versions(int64_t value) { _impl_.versions_.Add(value); } inline void ModelVersionPolicy_Specific::add_versions(int64_t value) { _internal_add_versions(value); // @@protoc_insertion_point(field_add:inference.ModelVersionPolicy.Specific.versions) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelVersionPolicy_Specific::_internal_versions() const { return _impl_.versions_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelVersionPolicy_Specific::versions() const { // @@protoc_insertion_point(field_list:inference.ModelVersionPolicy.Specific.versions) return _internal_versions(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelVersionPolicy_Specific::_internal_mutable_versions() { return &_impl_.versions_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelVersionPolicy_Specific::mutable_versions() { // @@protoc_insertion_point(field_mutable_list:inference.ModelVersionPolicy.Specific.versions) return _internal_mutable_versions(); } // ------------------------------------------------------------------- // ModelVersionPolicy // .inference.ModelVersionPolicy.Latest latest = 1; inline bool ModelVersionPolicy::_internal_has_latest() const { return policy_choice_case() == kLatest; } inline bool ModelVersionPolicy::has_latest() const { return _internal_has_latest(); } inline void ModelVersionPolicy::set_has_latest() { _impl_._oneof_case_[0] = kLatest; } inline void ModelVersionPolicy::clear_latest() { if (_internal_has_latest()) { if (GetArenaForAllocation() == nullptr) { delete _impl_.policy_choice_.latest_; } clear_has_policy_choice(); } } inline ::inference::ModelVersionPolicy_Latest* ModelVersionPolicy::release_latest() { // @@protoc_insertion_point(field_release:inference.ModelVersionPolicy.latest) if (_internal_has_latest()) { clear_has_policy_choice(); ::inference::ModelVersionPolicy_Latest* temp = _impl_.policy_choice_.latest_; if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } _impl_.policy_choice_.latest_ = nullptr; return temp; } else { return nullptr; } } inline const ::inference::ModelVersionPolicy_Latest& ModelVersionPolicy::_internal_latest() const { return _internal_has_latest() ? *_impl_.policy_choice_.latest_ : reinterpret_cast< ::inference::ModelVersionPolicy_Latest&>(::inference::_ModelVersionPolicy_Latest_default_instance_); } inline const ::inference::ModelVersionPolicy_Latest& ModelVersionPolicy::latest() const { // @@protoc_insertion_point(field_get:inference.ModelVersionPolicy.latest) return _internal_latest(); } inline ::inference::ModelVersionPolicy_Latest* ModelVersionPolicy::unsafe_arena_release_latest() { // @@protoc_insertion_point(field_unsafe_arena_release:inference.ModelVersionPolicy.latest) if (_internal_has_latest()) { clear_has_policy_choice(); ::inference::ModelVersionPolicy_Latest* temp = _impl_.policy_choice_.latest_; _impl_.policy_choice_.latest_ = nullptr; return temp; } else { return nullptr; } } inline void ModelVersionPolicy::unsafe_arena_set_allocated_latest(::inference::ModelVersionPolicy_Latest* latest) { clear_policy_choice(); if (latest) { set_has_latest(); _impl_.policy_choice_.latest_ = latest; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelVersionPolicy.latest) } inline ::inference::ModelVersionPolicy_Latest* ModelVersionPolicy::_internal_mutable_latest() { if (!_internal_has_latest()) { clear_policy_choice(); set_has_latest(); _impl_.policy_choice_.latest_ = CreateMaybeMessage< ::inference::ModelVersionPolicy_Latest >(GetArenaForAllocation()); } return _impl_.policy_choice_.latest_; } inline ::inference::ModelVersionPolicy_Latest* ModelVersionPolicy::mutable_latest() { ::inference::ModelVersionPolicy_Latest* _msg = _internal_mutable_latest(); // @@protoc_insertion_point(field_mutable:inference.ModelVersionPolicy.latest) return _msg; } // .inference.ModelVersionPolicy.All all = 2; inline bool ModelVersionPolicy::_internal_has_all() const { return policy_choice_case() == kAll; } inline bool ModelVersionPolicy::has_all() const { return _internal_has_all(); } inline void ModelVersionPolicy::set_has_all() { _impl_._oneof_case_[0] = kAll; } inline void ModelVersionPolicy::clear_all() { if (_internal_has_all()) { if (GetArenaForAllocation() == nullptr) { delete _impl_.policy_choice_.all_; } clear_has_policy_choice(); } } inline ::inference::ModelVersionPolicy_All* ModelVersionPolicy::release_all() { // @@protoc_insertion_point(field_release:inference.ModelVersionPolicy.all) if (_internal_has_all()) { clear_has_policy_choice(); ::inference::ModelVersionPolicy_All* temp = _impl_.policy_choice_.all_; if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } _impl_.policy_choice_.all_ = nullptr; return temp; } else { return nullptr; } } inline const ::inference::ModelVersionPolicy_All& ModelVersionPolicy::_internal_all() const { return _internal_has_all() ? *_impl_.policy_choice_.all_ : reinterpret_cast< ::inference::ModelVersionPolicy_All&>(::inference::_ModelVersionPolicy_All_default_instance_); } inline const ::inference::ModelVersionPolicy_All& ModelVersionPolicy::all() const { // @@protoc_insertion_point(field_get:inference.ModelVersionPolicy.all) return _internal_all(); } inline ::inference::ModelVersionPolicy_All* ModelVersionPolicy::unsafe_arena_release_all() { // @@protoc_insertion_point(field_unsafe_arena_release:inference.ModelVersionPolicy.all) if (_internal_has_all()) { clear_has_policy_choice(); ::inference::ModelVersionPolicy_All* temp = _impl_.policy_choice_.all_; _impl_.policy_choice_.all_ = nullptr; return temp; } else { return nullptr; } } inline void ModelVersionPolicy::unsafe_arena_set_allocated_all(::inference::ModelVersionPolicy_All* all) { clear_policy_choice(); if (all) { set_has_all(); _impl_.policy_choice_.all_ = all; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelVersionPolicy.all) } inline ::inference::ModelVersionPolicy_All* ModelVersionPolicy::_internal_mutable_all() { if (!_internal_has_all()) { clear_policy_choice(); set_has_all(); _impl_.policy_choice_.all_ = CreateMaybeMessage< ::inference::ModelVersionPolicy_All >(GetArenaForAllocation()); } return _impl_.policy_choice_.all_; } inline ::inference::ModelVersionPolicy_All* ModelVersionPolicy::mutable_all() { ::inference::ModelVersionPolicy_All* _msg = _internal_mutable_all(); // @@protoc_insertion_point(field_mutable:inference.ModelVersionPolicy.all) return _msg; } // .inference.ModelVersionPolicy.Specific specific = 3; inline bool ModelVersionPolicy::_internal_has_specific() const { return policy_choice_case() == kSpecific; } inline bool ModelVersionPolicy::has_specific() const { return _internal_has_specific(); } inline void ModelVersionPolicy::set_has_specific() { _impl_._oneof_case_[0] = kSpecific; } inline void ModelVersionPolicy::clear_specific() { if (_internal_has_specific()) { if (GetArenaForAllocation() == nullptr) { delete _impl_.policy_choice_.specific_; } clear_has_policy_choice(); } } inline ::inference::ModelVersionPolicy_Specific* ModelVersionPolicy::release_specific() { // @@protoc_insertion_point(field_release:inference.ModelVersionPolicy.specific) if (_internal_has_specific()) { clear_has_policy_choice(); ::inference::ModelVersionPolicy_Specific* temp = _impl_.policy_choice_.specific_; if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } _impl_.policy_choice_.specific_ = nullptr; return temp; } else { return nullptr; } } inline const ::inference::ModelVersionPolicy_Specific& ModelVersionPolicy::_internal_specific() const { return _internal_has_specific() ? *_impl_.policy_choice_.specific_ : reinterpret_cast< ::inference::ModelVersionPolicy_Specific&>(::inference::_ModelVersionPolicy_Specific_default_instance_); } inline const ::inference::ModelVersionPolicy_Specific& ModelVersionPolicy::specific() const { // @@protoc_insertion_point(field_get:inference.ModelVersionPolicy.specific) return _internal_specific(); } inline ::inference::ModelVersionPolicy_Specific* ModelVersionPolicy::unsafe_arena_release_specific() { // @@protoc_insertion_point(field_unsafe_arena_release:inference.ModelVersionPolicy.specific) if (_internal_has_specific()) { clear_has_policy_choice(); ::inference::ModelVersionPolicy_Specific* temp = _impl_.policy_choice_.specific_; _impl_.policy_choice_.specific_ = nullptr; return temp; } else { return nullptr; } } inline void ModelVersionPolicy::unsafe_arena_set_allocated_specific(::inference::ModelVersionPolicy_Specific* specific) { clear_policy_choice(); if (specific) { set_has_specific(); _impl_.policy_choice_.specific_ = specific; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelVersionPolicy.specific) } inline ::inference::ModelVersionPolicy_Specific* ModelVersionPolicy::_internal_mutable_specific() { if (!_internal_has_specific()) { clear_policy_choice(); set_has_specific(); _impl_.policy_choice_.specific_ = CreateMaybeMessage< ::inference::ModelVersionPolicy_Specific >(GetArenaForAllocation()); } return _impl_.policy_choice_.specific_; } inline ::inference::ModelVersionPolicy_Specific* ModelVersionPolicy::mutable_specific() { ::inference::ModelVersionPolicy_Specific* _msg = _internal_mutable_specific(); // @@protoc_insertion_point(field_mutable:inference.ModelVersionPolicy.specific) return _msg; } inline bool ModelVersionPolicy::has_policy_choice() const { return policy_choice_case() != POLICY_CHOICE_NOT_SET; } inline void ModelVersionPolicy::clear_has_policy_choice() { _impl_._oneof_case_[0] = POLICY_CHOICE_NOT_SET; } inline ModelVersionPolicy::PolicyChoiceCase ModelVersionPolicy::policy_choice_case() const { return ModelVersionPolicy::PolicyChoiceCase(_impl_._oneof_case_[0]); } // ------------------------------------------------------------------- // ModelOptimizationPolicy_Graph // int32 level = 1; inline void ModelOptimizationPolicy_Graph::clear_level() { _impl_.level_ = 0; } inline int32_t ModelOptimizationPolicy_Graph::_internal_level() const { return _impl_.level_; } inline int32_t ModelOptimizationPolicy_Graph::level() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Graph.level) return _internal_level(); } inline void ModelOptimizationPolicy_Graph::_internal_set_level(int32_t value) { _impl_.level_ = value; } inline void ModelOptimizationPolicy_Graph::set_level(int32_t value) { _internal_set_level(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.Graph.level) } // ------------------------------------------------------------------- // ModelOptimizationPolicy_Cuda_GraphSpec_Shape // repeated int64 dim = 1; inline int ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_internal_dim_size() const { return _impl_.dim_.size(); } inline int ModelOptimizationPolicy_Cuda_GraphSpec_Shape::dim_size() const { return _internal_dim_size(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::clear_dim() { _impl_.dim_.Clear(); } inline int64_t ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_internal_dim(int index) const { return _impl_.dim_.Get(index); } inline int64_t ModelOptimizationPolicy_Cuda_GraphSpec_Shape::dim(int index) const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape.dim) return _internal_dim(index); } inline void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::set_dim(int index, int64_t value) { _impl_.dim_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape.dim) } inline void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_internal_add_dim(int64_t value) { _impl_.dim_.Add(value); } inline void ModelOptimizationPolicy_Cuda_GraphSpec_Shape::add_dim(int64_t value) { _internal_add_dim(value); // @@protoc_insertion_point(field_add:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape.dim) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_internal_dim() const { return _impl_.dim_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelOptimizationPolicy_Cuda_GraphSpec_Shape::dim() const { // @@protoc_insertion_point(field_list:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape.dim) return _internal_dim(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelOptimizationPolicy_Cuda_GraphSpec_Shape::_internal_mutable_dim() { return &_impl_.dim_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelOptimizationPolicy_Cuda_GraphSpec_Shape::mutable_dim() { // @@protoc_insertion_point(field_mutable_list:inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape.dim) return _internal_mutable_dim(); } // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound // int32 batch_size = 1; inline void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::clear_batch_size() { _impl_.batch_size_ = 0; } inline int32_t ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_internal_batch_size() const { return _impl_.batch_size_; } inline int32_t ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::batch_size() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound.batch_size) return _internal_batch_size(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_internal_set_batch_size(int32_t value) { _impl_.batch_size_ = value; } inline void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::set_batch_size(int32_t value) { _internal_set_batch_size(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound.batch_size) } // map input = 2; inline int ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_internal_input_size() const { return _impl_.input_.size(); } inline int ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::input_size() const { return _internal_input_size(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::clear_input() { _impl_.input_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >& ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_internal_input() const { return _impl_.input_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >& ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::input() const { // @@protoc_insertion_point(field_map:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound.input) return _internal_input(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >* ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::_internal_mutable_input() { return _impl_.input_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >* ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound::mutable_input() { // @@protoc_insertion_point(field_mutable_map:inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound.input) return _internal_mutable_input(); } // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ModelOptimizationPolicy_Cuda_GraphSpec // int32 batch_size = 1; inline void ModelOptimizationPolicy_Cuda_GraphSpec::clear_batch_size() { _impl_.batch_size_ = 0; } inline int32_t ModelOptimizationPolicy_Cuda_GraphSpec::_internal_batch_size() const { return _impl_.batch_size_; } inline int32_t ModelOptimizationPolicy_Cuda_GraphSpec::batch_size() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Cuda.GraphSpec.batch_size) return _internal_batch_size(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec::_internal_set_batch_size(int32_t value) { _impl_.batch_size_ = value; } inline void ModelOptimizationPolicy_Cuda_GraphSpec::set_batch_size(int32_t value) { _internal_set_batch_size(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.Cuda.GraphSpec.batch_size) } // map input = 2; inline int ModelOptimizationPolicy_Cuda_GraphSpec::_internal_input_size() const { return _impl_.input_.size(); } inline int ModelOptimizationPolicy_Cuda_GraphSpec::input_size() const { return _internal_input_size(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec::clear_input() { _impl_.input_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >& ModelOptimizationPolicy_Cuda_GraphSpec::_internal_input() const { return _impl_.input_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >& ModelOptimizationPolicy_Cuda_GraphSpec::input() const { // @@protoc_insertion_point(field_map:inference.ModelOptimizationPolicy.Cuda.GraphSpec.input) return _internal_input(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >* ModelOptimizationPolicy_Cuda_GraphSpec::_internal_mutable_input() { return _impl_.input_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_Shape >* ModelOptimizationPolicy_Cuda_GraphSpec::mutable_input() { // @@protoc_insertion_point(field_mutable_map:inference.ModelOptimizationPolicy.Cuda.GraphSpec.input) return _internal_mutable_input(); } // .inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound graph_lower_bound = 3; inline bool ModelOptimizationPolicy_Cuda_GraphSpec::_internal_has_graph_lower_bound() const { return this != internal_default_instance() && _impl_.graph_lower_bound_ != nullptr; } inline bool ModelOptimizationPolicy_Cuda_GraphSpec::has_graph_lower_bound() const { return _internal_has_graph_lower_bound(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec::clear_graph_lower_bound() { if (GetArenaForAllocation() == nullptr && _impl_.graph_lower_bound_ != nullptr) { delete _impl_.graph_lower_bound_; } _impl_.graph_lower_bound_ = nullptr; } inline const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& ModelOptimizationPolicy_Cuda_GraphSpec::_internal_graph_lower_bound() const { const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* p = _impl_.graph_lower_bound_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound_default_instance_); } inline const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound& ModelOptimizationPolicy_Cuda_GraphSpec::graph_lower_bound() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Cuda.GraphSpec.graph_lower_bound) return _internal_graph_lower_bound(); } inline void ModelOptimizationPolicy_Cuda_GraphSpec::unsafe_arena_set_allocated_graph_lower_bound( ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* graph_lower_bound) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.graph_lower_bound_); } _impl_.graph_lower_bound_ = graph_lower_bound; if (graph_lower_bound) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelOptimizationPolicy.Cuda.GraphSpec.graph_lower_bound) } inline ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* ModelOptimizationPolicy_Cuda_GraphSpec::release_graph_lower_bound() { ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* temp = _impl_.graph_lower_bound_; _impl_.graph_lower_bound_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* ModelOptimizationPolicy_Cuda_GraphSpec::unsafe_arena_release_graph_lower_bound() { // @@protoc_insertion_point(field_release:inference.ModelOptimizationPolicy.Cuda.GraphSpec.graph_lower_bound) ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* temp = _impl_.graph_lower_bound_; _impl_.graph_lower_bound_ = nullptr; return temp; } inline ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* ModelOptimizationPolicy_Cuda_GraphSpec::_internal_mutable_graph_lower_bound() { if (_impl_.graph_lower_bound_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound>(GetArenaForAllocation()); _impl_.graph_lower_bound_ = p; } return _impl_.graph_lower_bound_; } inline ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* ModelOptimizationPolicy_Cuda_GraphSpec::mutable_graph_lower_bound() { ::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* _msg = _internal_mutable_graph_lower_bound(); // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.Cuda.GraphSpec.graph_lower_bound) return _msg; } inline void ModelOptimizationPolicy_Cuda_GraphSpec::set_allocated_graph_lower_bound(::inference::ModelOptimizationPolicy_Cuda_GraphSpec_LowerBound* graph_lower_bound) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.graph_lower_bound_; } if (graph_lower_bound) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(graph_lower_bound); if (message_arena != submessage_arena) { graph_lower_bound = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, graph_lower_bound, submessage_arena); } } else { } _impl_.graph_lower_bound_ = graph_lower_bound; // @@protoc_insertion_point(field_set_allocated:inference.ModelOptimizationPolicy.Cuda.GraphSpec.graph_lower_bound) } // ------------------------------------------------------------------- // ModelOptimizationPolicy_Cuda // bool graphs = 1; inline void ModelOptimizationPolicy_Cuda::clear_graphs() { _impl_.graphs_ = false; } inline bool ModelOptimizationPolicy_Cuda::_internal_graphs() const { return _impl_.graphs_; } inline bool ModelOptimizationPolicy_Cuda::graphs() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Cuda.graphs) return _internal_graphs(); } inline void ModelOptimizationPolicy_Cuda::_internal_set_graphs(bool value) { _impl_.graphs_ = value; } inline void ModelOptimizationPolicy_Cuda::set_graphs(bool value) { _internal_set_graphs(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.Cuda.graphs) } // bool busy_wait_events = 2; inline void ModelOptimizationPolicy_Cuda::clear_busy_wait_events() { _impl_.busy_wait_events_ = false; } inline bool ModelOptimizationPolicy_Cuda::_internal_busy_wait_events() const { return _impl_.busy_wait_events_; } inline bool ModelOptimizationPolicy_Cuda::busy_wait_events() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Cuda.busy_wait_events) return _internal_busy_wait_events(); } inline void ModelOptimizationPolicy_Cuda::_internal_set_busy_wait_events(bool value) { _impl_.busy_wait_events_ = value; } inline void ModelOptimizationPolicy_Cuda::set_busy_wait_events(bool value) { _internal_set_busy_wait_events(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.Cuda.busy_wait_events) } // repeated .inference.ModelOptimizationPolicy.Cuda.GraphSpec graph_spec = 3; inline int ModelOptimizationPolicy_Cuda::_internal_graph_spec_size() const { return _impl_.graph_spec_.size(); } inline int ModelOptimizationPolicy_Cuda::graph_spec_size() const { return _internal_graph_spec_size(); } inline void ModelOptimizationPolicy_Cuda::clear_graph_spec() { _impl_.graph_spec_.Clear(); } inline ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* ModelOptimizationPolicy_Cuda::mutable_graph_spec(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.Cuda.graph_spec) return _impl_.graph_spec_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec >* ModelOptimizationPolicy_Cuda::mutable_graph_spec() { // @@protoc_insertion_point(field_mutable_list:inference.ModelOptimizationPolicy.Cuda.graph_spec) return &_impl_.graph_spec_; } inline const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec& ModelOptimizationPolicy_Cuda::_internal_graph_spec(int index) const { return _impl_.graph_spec_.Get(index); } inline const ::inference::ModelOptimizationPolicy_Cuda_GraphSpec& ModelOptimizationPolicy_Cuda::graph_spec(int index) const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Cuda.graph_spec) return _internal_graph_spec(index); } inline ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* ModelOptimizationPolicy_Cuda::_internal_add_graph_spec() { return _impl_.graph_spec_.Add(); } inline ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* ModelOptimizationPolicy_Cuda::add_graph_spec() { ::inference::ModelOptimizationPolicy_Cuda_GraphSpec* _add = _internal_add_graph_spec(); // @@protoc_insertion_point(field_add:inference.ModelOptimizationPolicy.Cuda.graph_spec) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_Cuda_GraphSpec >& ModelOptimizationPolicy_Cuda::graph_spec() const { // @@protoc_insertion_point(field_list:inference.ModelOptimizationPolicy.Cuda.graph_spec) return _impl_.graph_spec_; } // bool output_copy_stream = 4; inline void ModelOptimizationPolicy_Cuda::clear_output_copy_stream() { _impl_.output_copy_stream_ = false; } inline bool ModelOptimizationPolicy_Cuda::_internal_output_copy_stream() const { return _impl_.output_copy_stream_; } inline bool ModelOptimizationPolicy_Cuda::output_copy_stream() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.Cuda.output_copy_stream) return _internal_output_copy_stream(); } inline void ModelOptimizationPolicy_Cuda::_internal_set_output_copy_stream(bool value) { _impl_.output_copy_stream_ = value; } inline void ModelOptimizationPolicy_Cuda::set_output_copy_stream(bool value) { _internal_set_output_copy_stream(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.Cuda.output_copy_stream) } // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ModelOptimizationPolicy_ExecutionAccelerators_Accelerator // string name = 1; inline void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::name() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.name) } inline std::string* ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.name) return _s; } inline const std::string& ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_internal_name() const { return _impl_.name_.Get(); } inline void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::release_name() { // @@protoc_insertion_point(field_release:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.name) return _impl_.name_.Release(); } inline void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.name) } // map parameters = 2; inline int ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_internal_parameters_size() const { return _impl_.parameters_.size(); } inline int ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::parameters_size() const { return _internal_parameters_size(); } inline void ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::clear_parameters() { _impl_.parameters_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_internal_parameters() const { return _impl_.parameters_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::parameters() const { // @@protoc_insertion_point(field_map:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.parameters) return _internal_parameters(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::_internal_mutable_parameters() { return _impl_.parameters_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelOptimizationPolicy_ExecutionAccelerators_Accelerator::mutable_parameters() { // @@protoc_insertion_point(field_mutable_map:inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.parameters) return _internal_mutable_parameters(); } // ------------------------------------------------------------------- // ModelOptimizationPolicy_ExecutionAccelerators // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator gpu_execution_accelerator = 1; inline int ModelOptimizationPolicy_ExecutionAccelerators::_internal_gpu_execution_accelerator_size() const { return _impl_.gpu_execution_accelerator_.size(); } inline int ModelOptimizationPolicy_ExecutionAccelerators::gpu_execution_accelerator_size() const { return _internal_gpu_execution_accelerator_size(); } inline void ModelOptimizationPolicy_ExecutionAccelerators::clear_gpu_execution_accelerator() { _impl_.gpu_execution_accelerator_.Clear(); } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* ModelOptimizationPolicy_ExecutionAccelerators::mutable_gpu_execution_accelerator(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.ExecutionAccelerators.gpu_execution_accelerator) return _impl_.gpu_execution_accelerator_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >* ModelOptimizationPolicy_ExecutionAccelerators::mutable_gpu_execution_accelerator() { // @@protoc_insertion_point(field_mutable_list:inference.ModelOptimizationPolicy.ExecutionAccelerators.gpu_execution_accelerator) return &_impl_.gpu_execution_accelerator_; } inline const ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& ModelOptimizationPolicy_ExecutionAccelerators::_internal_gpu_execution_accelerator(int index) const { return _impl_.gpu_execution_accelerator_.Get(index); } inline const ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& ModelOptimizationPolicy_ExecutionAccelerators::gpu_execution_accelerator(int index) const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.ExecutionAccelerators.gpu_execution_accelerator) return _internal_gpu_execution_accelerator(index); } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* ModelOptimizationPolicy_ExecutionAccelerators::_internal_add_gpu_execution_accelerator() { return _impl_.gpu_execution_accelerator_.Add(); } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* ModelOptimizationPolicy_ExecutionAccelerators::add_gpu_execution_accelerator() { ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* _add = _internal_add_gpu_execution_accelerator(); // @@protoc_insertion_point(field_add:inference.ModelOptimizationPolicy.ExecutionAccelerators.gpu_execution_accelerator) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >& ModelOptimizationPolicy_ExecutionAccelerators::gpu_execution_accelerator() const { // @@protoc_insertion_point(field_list:inference.ModelOptimizationPolicy.ExecutionAccelerators.gpu_execution_accelerator) return _impl_.gpu_execution_accelerator_; } // repeated .inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator cpu_execution_accelerator = 2; inline int ModelOptimizationPolicy_ExecutionAccelerators::_internal_cpu_execution_accelerator_size() const { return _impl_.cpu_execution_accelerator_.size(); } inline int ModelOptimizationPolicy_ExecutionAccelerators::cpu_execution_accelerator_size() const { return _internal_cpu_execution_accelerator_size(); } inline void ModelOptimizationPolicy_ExecutionAccelerators::clear_cpu_execution_accelerator() { _impl_.cpu_execution_accelerator_.Clear(); } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* ModelOptimizationPolicy_ExecutionAccelerators::mutable_cpu_execution_accelerator(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.ExecutionAccelerators.cpu_execution_accelerator) return _impl_.cpu_execution_accelerator_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >* ModelOptimizationPolicy_ExecutionAccelerators::mutable_cpu_execution_accelerator() { // @@protoc_insertion_point(field_mutable_list:inference.ModelOptimizationPolicy.ExecutionAccelerators.cpu_execution_accelerator) return &_impl_.cpu_execution_accelerator_; } inline const ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& ModelOptimizationPolicy_ExecutionAccelerators::_internal_cpu_execution_accelerator(int index) const { return _impl_.cpu_execution_accelerator_.Get(index); } inline const ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator& ModelOptimizationPolicy_ExecutionAccelerators::cpu_execution_accelerator(int index) const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.ExecutionAccelerators.cpu_execution_accelerator) return _internal_cpu_execution_accelerator(index); } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* ModelOptimizationPolicy_ExecutionAccelerators::_internal_add_cpu_execution_accelerator() { return _impl_.cpu_execution_accelerator_.Add(); } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* ModelOptimizationPolicy_ExecutionAccelerators::add_cpu_execution_accelerator() { ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator* _add = _internal_add_cpu_execution_accelerator(); // @@protoc_insertion_point(field_add:inference.ModelOptimizationPolicy.ExecutionAccelerators.cpu_execution_accelerator) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOptimizationPolicy_ExecutionAccelerators_Accelerator >& ModelOptimizationPolicy_ExecutionAccelerators::cpu_execution_accelerator() const { // @@protoc_insertion_point(field_list:inference.ModelOptimizationPolicy.ExecutionAccelerators.cpu_execution_accelerator) return _impl_.cpu_execution_accelerator_; } // ------------------------------------------------------------------- // ModelOptimizationPolicy_PinnedMemoryBuffer // bool enable = 1; inline void ModelOptimizationPolicy_PinnedMemoryBuffer::clear_enable() { _impl_.enable_ = false; } inline bool ModelOptimizationPolicy_PinnedMemoryBuffer::_internal_enable() const { return _impl_.enable_; } inline bool ModelOptimizationPolicy_PinnedMemoryBuffer::enable() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.PinnedMemoryBuffer.enable) return _internal_enable(); } inline void ModelOptimizationPolicy_PinnedMemoryBuffer::_internal_set_enable(bool value) { _impl_.enable_ = value; } inline void ModelOptimizationPolicy_PinnedMemoryBuffer::set_enable(bool value) { _internal_set_enable(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.PinnedMemoryBuffer.enable) } // ------------------------------------------------------------------- // ModelOptimizationPolicy // .inference.ModelOptimizationPolicy.Graph graph = 1; inline bool ModelOptimizationPolicy::_internal_has_graph() const { return this != internal_default_instance() && _impl_.graph_ != nullptr; } inline bool ModelOptimizationPolicy::has_graph() const { return _internal_has_graph(); } inline void ModelOptimizationPolicy::clear_graph() { if (GetArenaForAllocation() == nullptr && _impl_.graph_ != nullptr) { delete _impl_.graph_; } _impl_.graph_ = nullptr; } inline const ::inference::ModelOptimizationPolicy_Graph& ModelOptimizationPolicy::_internal_graph() const { const ::inference::ModelOptimizationPolicy_Graph* p = _impl_.graph_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelOptimizationPolicy_Graph_default_instance_); } inline const ::inference::ModelOptimizationPolicy_Graph& ModelOptimizationPolicy::graph() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.graph) return _internal_graph(); } inline void ModelOptimizationPolicy::unsafe_arena_set_allocated_graph( ::inference::ModelOptimizationPolicy_Graph* graph) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.graph_); } _impl_.graph_ = graph; if (graph) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelOptimizationPolicy.graph) } inline ::inference::ModelOptimizationPolicy_Graph* ModelOptimizationPolicy::release_graph() { ::inference::ModelOptimizationPolicy_Graph* temp = _impl_.graph_; _impl_.graph_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelOptimizationPolicy_Graph* ModelOptimizationPolicy::unsafe_arena_release_graph() { // @@protoc_insertion_point(field_release:inference.ModelOptimizationPolicy.graph) ::inference::ModelOptimizationPolicy_Graph* temp = _impl_.graph_; _impl_.graph_ = nullptr; return temp; } inline ::inference::ModelOptimizationPolicy_Graph* ModelOptimizationPolicy::_internal_mutable_graph() { if (_impl_.graph_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelOptimizationPolicy_Graph>(GetArenaForAllocation()); _impl_.graph_ = p; } return _impl_.graph_; } inline ::inference::ModelOptimizationPolicy_Graph* ModelOptimizationPolicy::mutable_graph() { ::inference::ModelOptimizationPolicy_Graph* _msg = _internal_mutable_graph(); // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.graph) return _msg; } inline void ModelOptimizationPolicy::set_allocated_graph(::inference::ModelOptimizationPolicy_Graph* graph) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.graph_; } if (graph) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(graph); if (message_arena != submessage_arena) { graph = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, graph, submessage_arena); } } else { } _impl_.graph_ = graph; // @@protoc_insertion_point(field_set_allocated:inference.ModelOptimizationPolicy.graph) } // .inference.ModelOptimizationPolicy.ModelPriority priority = 2; inline void ModelOptimizationPolicy::clear_priority() { _impl_.priority_ = 0; } inline ::inference::ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy::_internal_priority() const { return static_cast< ::inference::ModelOptimizationPolicy_ModelPriority >(_impl_.priority_); } inline ::inference::ModelOptimizationPolicy_ModelPriority ModelOptimizationPolicy::priority() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.priority) return _internal_priority(); } inline void ModelOptimizationPolicy::_internal_set_priority(::inference::ModelOptimizationPolicy_ModelPriority value) { _impl_.priority_ = value; } inline void ModelOptimizationPolicy::set_priority(::inference::ModelOptimizationPolicy_ModelPriority value) { _internal_set_priority(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.priority) } // .inference.ModelOptimizationPolicy.Cuda cuda = 3; inline bool ModelOptimizationPolicy::_internal_has_cuda() const { return this != internal_default_instance() && _impl_.cuda_ != nullptr; } inline bool ModelOptimizationPolicy::has_cuda() const { return _internal_has_cuda(); } inline void ModelOptimizationPolicy::clear_cuda() { if (GetArenaForAllocation() == nullptr && _impl_.cuda_ != nullptr) { delete _impl_.cuda_; } _impl_.cuda_ = nullptr; } inline const ::inference::ModelOptimizationPolicy_Cuda& ModelOptimizationPolicy::_internal_cuda() const { const ::inference::ModelOptimizationPolicy_Cuda* p = _impl_.cuda_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelOptimizationPolicy_Cuda_default_instance_); } inline const ::inference::ModelOptimizationPolicy_Cuda& ModelOptimizationPolicy::cuda() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.cuda) return _internal_cuda(); } inline void ModelOptimizationPolicy::unsafe_arena_set_allocated_cuda( ::inference::ModelOptimizationPolicy_Cuda* cuda) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.cuda_); } _impl_.cuda_ = cuda; if (cuda) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelOptimizationPolicy.cuda) } inline ::inference::ModelOptimizationPolicy_Cuda* ModelOptimizationPolicy::release_cuda() { ::inference::ModelOptimizationPolicy_Cuda* temp = _impl_.cuda_; _impl_.cuda_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelOptimizationPolicy_Cuda* ModelOptimizationPolicy::unsafe_arena_release_cuda() { // @@protoc_insertion_point(field_release:inference.ModelOptimizationPolicy.cuda) ::inference::ModelOptimizationPolicy_Cuda* temp = _impl_.cuda_; _impl_.cuda_ = nullptr; return temp; } inline ::inference::ModelOptimizationPolicy_Cuda* ModelOptimizationPolicy::_internal_mutable_cuda() { if (_impl_.cuda_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelOptimizationPolicy_Cuda>(GetArenaForAllocation()); _impl_.cuda_ = p; } return _impl_.cuda_; } inline ::inference::ModelOptimizationPolicy_Cuda* ModelOptimizationPolicy::mutable_cuda() { ::inference::ModelOptimizationPolicy_Cuda* _msg = _internal_mutable_cuda(); // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.cuda) return _msg; } inline void ModelOptimizationPolicy::set_allocated_cuda(::inference::ModelOptimizationPolicy_Cuda* cuda) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.cuda_; } if (cuda) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(cuda); if (message_arena != submessage_arena) { cuda = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, cuda, submessage_arena); } } else { } _impl_.cuda_ = cuda; // @@protoc_insertion_point(field_set_allocated:inference.ModelOptimizationPolicy.cuda) } // .inference.ModelOptimizationPolicy.ExecutionAccelerators execution_accelerators = 4; inline bool ModelOptimizationPolicy::_internal_has_execution_accelerators() const { return this != internal_default_instance() && _impl_.execution_accelerators_ != nullptr; } inline bool ModelOptimizationPolicy::has_execution_accelerators() const { return _internal_has_execution_accelerators(); } inline void ModelOptimizationPolicy::clear_execution_accelerators() { if (GetArenaForAllocation() == nullptr && _impl_.execution_accelerators_ != nullptr) { delete _impl_.execution_accelerators_; } _impl_.execution_accelerators_ = nullptr; } inline const ::inference::ModelOptimizationPolicy_ExecutionAccelerators& ModelOptimizationPolicy::_internal_execution_accelerators() const { const ::inference::ModelOptimizationPolicy_ExecutionAccelerators* p = _impl_.execution_accelerators_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelOptimizationPolicy_ExecutionAccelerators_default_instance_); } inline const ::inference::ModelOptimizationPolicy_ExecutionAccelerators& ModelOptimizationPolicy::execution_accelerators() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.execution_accelerators) return _internal_execution_accelerators(); } inline void ModelOptimizationPolicy::unsafe_arena_set_allocated_execution_accelerators( ::inference::ModelOptimizationPolicy_ExecutionAccelerators* execution_accelerators) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.execution_accelerators_); } _impl_.execution_accelerators_ = execution_accelerators; if (execution_accelerators) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelOptimizationPolicy.execution_accelerators) } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators* ModelOptimizationPolicy::release_execution_accelerators() { ::inference::ModelOptimizationPolicy_ExecutionAccelerators* temp = _impl_.execution_accelerators_; _impl_.execution_accelerators_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators* ModelOptimizationPolicy::unsafe_arena_release_execution_accelerators() { // @@protoc_insertion_point(field_release:inference.ModelOptimizationPolicy.execution_accelerators) ::inference::ModelOptimizationPolicy_ExecutionAccelerators* temp = _impl_.execution_accelerators_; _impl_.execution_accelerators_ = nullptr; return temp; } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators* ModelOptimizationPolicy::_internal_mutable_execution_accelerators() { if (_impl_.execution_accelerators_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelOptimizationPolicy_ExecutionAccelerators>(GetArenaForAllocation()); _impl_.execution_accelerators_ = p; } return _impl_.execution_accelerators_; } inline ::inference::ModelOptimizationPolicy_ExecutionAccelerators* ModelOptimizationPolicy::mutable_execution_accelerators() { ::inference::ModelOptimizationPolicy_ExecutionAccelerators* _msg = _internal_mutable_execution_accelerators(); // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.execution_accelerators) return _msg; } inline void ModelOptimizationPolicy::set_allocated_execution_accelerators(::inference::ModelOptimizationPolicy_ExecutionAccelerators* execution_accelerators) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.execution_accelerators_; } if (execution_accelerators) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(execution_accelerators); if (message_arena != submessage_arena) { execution_accelerators = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, execution_accelerators, submessage_arena); } } else { } _impl_.execution_accelerators_ = execution_accelerators; // @@protoc_insertion_point(field_set_allocated:inference.ModelOptimizationPolicy.execution_accelerators) } // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer input_pinned_memory = 5; inline bool ModelOptimizationPolicy::_internal_has_input_pinned_memory() const { return this != internal_default_instance() && _impl_.input_pinned_memory_ != nullptr; } inline bool ModelOptimizationPolicy::has_input_pinned_memory() const { return _internal_has_input_pinned_memory(); } inline void ModelOptimizationPolicy::clear_input_pinned_memory() { if (GetArenaForAllocation() == nullptr && _impl_.input_pinned_memory_ != nullptr) { delete _impl_.input_pinned_memory_; } _impl_.input_pinned_memory_ = nullptr; } inline const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& ModelOptimizationPolicy::_internal_input_pinned_memory() const { const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* p = _impl_.input_pinned_memory_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelOptimizationPolicy_PinnedMemoryBuffer_default_instance_); } inline const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& ModelOptimizationPolicy::input_pinned_memory() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.input_pinned_memory) return _internal_input_pinned_memory(); } inline void ModelOptimizationPolicy::unsafe_arena_set_allocated_input_pinned_memory( ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* input_pinned_memory) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.input_pinned_memory_); } _impl_.input_pinned_memory_ = input_pinned_memory; if (input_pinned_memory) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelOptimizationPolicy.input_pinned_memory) } inline ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* ModelOptimizationPolicy::release_input_pinned_memory() { ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* temp = _impl_.input_pinned_memory_; _impl_.input_pinned_memory_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* ModelOptimizationPolicy::unsafe_arena_release_input_pinned_memory() { // @@protoc_insertion_point(field_release:inference.ModelOptimizationPolicy.input_pinned_memory) ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* temp = _impl_.input_pinned_memory_; _impl_.input_pinned_memory_ = nullptr; return temp; } inline ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* ModelOptimizationPolicy::_internal_mutable_input_pinned_memory() { if (_impl_.input_pinned_memory_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelOptimizationPolicy_PinnedMemoryBuffer>(GetArenaForAllocation()); _impl_.input_pinned_memory_ = p; } return _impl_.input_pinned_memory_; } inline ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* ModelOptimizationPolicy::mutable_input_pinned_memory() { ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* _msg = _internal_mutable_input_pinned_memory(); // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.input_pinned_memory) return _msg; } inline void ModelOptimizationPolicy::set_allocated_input_pinned_memory(::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* input_pinned_memory) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.input_pinned_memory_; } if (input_pinned_memory) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(input_pinned_memory); if (message_arena != submessage_arena) { input_pinned_memory = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, input_pinned_memory, submessage_arena); } } else { } _impl_.input_pinned_memory_ = input_pinned_memory; // @@protoc_insertion_point(field_set_allocated:inference.ModelOptimizationPolicy.input_pinned_memory) } // .inference.ModelOptimizationPolicy.PinnedMemoryBuffer output_pinned_memory = 6; inline bool ModelOptimizationPolicy::_internal_has_output_pinned_memory() const { return this != internal_default_instance() && _impl_.output_pinned_memory_ != nullptr; } inline bool ModelOptimizationPolicy::has_output_pinned_memory() const { return _internal_has_output_pinned_memory(); } inline void ModelOptimizationPolicy::clear_output_pinned_memory() { if (GetArenaForAllocation() == nullptr && _impl_.output_pinned_memory_ != nullptr) { delete _impl_.output_pinned_memory_; } _impl_.output_pinned_memory_ = nullptr; } inline const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& ModelOptimizationPolicy::_internal_output_pinned_memory() const { const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* p = _impl_.output_pinned_memory_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelOptimizationPolicy_PinnedMemoryBuffer_default_instance_); } inline const ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer& ModelOptimizationPolicy::output_pinned_memory() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.output_pinned_memory) return _internal_output_pinned_memory(); } inline void ModelOptimizationPolicy::unsafe_arena_set_allocated_output_pinned_memory( ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* output_pinned_memory) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.output_pinned_memory_); } _impl_.output_pinned_memory_ = output_pinned_memory; if (output_pinned_memory) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelOptimizationPolicy.output_pinned_memory) } inline ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* ModelOptimizationPolicy::release_output_pinned_memory() { ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* temp = _impl_.output_pinned_memory_; _impl_.output_pinned_memory_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* ModelOptimizationPolicy::unsafe_arena_release_output_pinned_memory() { // @@protoc_insertion_point(field_release:inference.ModelOptimizationPolicy.output_pinned_memory) ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* temp = _impl_.output_pinned_memory_; _impl_.output_pinned_memory_ = nullptr; return temp; } inline ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* ModelOptimizationPolicy::_internal_mutable_output_pinned_memory() { if (_impl_.output_pinned_memory_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelOptimizationPolicy_PinnedMemoryBuffer>(GetArenaForAllocation()); _impl_.output_pinned_memory_ = p; } return _impl_.output_pinned_memory_; } inline ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* ModelOptimizationPolicy::mutable_output_pinned_memory() { ::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* _msg = _internal_mutable_output_pinned_memory(); // @@protoc_insertion_point(field_mutable:inference.ModelOptimizationPolicy.output_pinned_memory) return _msg; } inline void ModelOptimizationPolicy::set_allocated_output_pinned_memory(::inference::ModelOptimizationPolicy_PinnedMemoryBuffer* output_pinned_memory) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.output_pinned_memory_; } if (output_pinned_memory) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(output_pinned_memory); if (message_arena != submessage_arena) { output_pinned_memory = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, output_pinned_memory, submessage_arena); } } else { } _impl_.output_pinned_memory_ = output_pinned_memory; // @@protoc_insertion_point(field_set_allocated:inference.ModelOptimizationPolicy.output_pinned_memory) } // uint32 gather_kernel_buffer_threshold = 7; inline void ModelOptimizationPolicy::clear_gather_kernel_buffer_threshold() { _impl_.gather_kernel_buffer_threshold_ = 0u; } inline uint32_t ModelOptimizationPolicy::_internal_gather_kernel_buffer_threshold() const { return _impl_.gather_kernel_buffer_threshold_; } inline uint32_t ModelOptimizationPolicy::gather_kernel_buffer_threshold() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.gather_kernel_buffer_threshold) return _internal_gather_kernel_buffer_threshold(); } inline void ModelOptimizationPolicy::_internal_set_gather_kernel_buffer_threshold(uint32_t value) { _impl_.gather_kernel_buffer_threshold_ = value; } inline void ModelOptimizationPolicy::set_gather_kernel_buffer_threshold(uint32_t value) { _internal_set_gather_kernel_buffer_threshold(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.gather_kernel_buffer_threshold) } // bool eager_batching = 8; inline void ModelOptimizationPolicy::clear_eager_batching() { _impl_.eager_batching_ = false; } inline bool ModelOptimizationPolicy::_internal_eager_batching() const { return _impl_.eager_batching_; } inline bool ModelOptimizationPolicy::eager_batching() const { // @@protoc_insertion_point(field_get:inference.ModelOptimizationPolicy.eager_batching) return _internal_eager_batching(); } inline void ModelOptimizationPolicy::_internal_set_eager_batching(bool value) { _impl_.eager_batching_ = value; } inline void ModelOptimizationPolicy::set_eager_batching(bool value) { _internal_set_eager_batching(value); // @@protoc_insertion_point(field_set:inference.ModelOptimizationPolicy.eager_batching) } // ------------------------------------------------------------------- // ModelQueuePolicy // .inference.ModelQueuePolicy.TimeoutAction timeout_action = 1; inline void ModelQueuePolicy::clear_timeout_action() { _impl_.timeout_action_ = 0; } inline ::inference::ModelQueuePolicy_TimeoutAction ModelQueuePolicy::_internal_timeout_action() const { return static_cast< ::inference::ModelQueuePolicy_TimeoutAction >(_impl_.timeout_action_); } inline ::inference::ModelQueuePolicy_TimeoutAction ModelQueuePolicy::timeout_action() const { // @@protoc_insertion_point(field_get:inference.ModelQueuePolicy.timeout_action) return _internal_timeout_action(); } inline void ModelQueuePolicy::_internal_set_timeout_action(::inference::ModelQueuePolicy_TimeoutAction value) { _impl_.timeout_action_ = value; } inline void ModelQueuePolicy::set_timeout_action(::inference::ModelQueuePolicy_TimeoutAction value) { _internal_set_timeout_action(value); // @@protoc_insertion_point(field_set:inference.ModelQueuePolicy.timeout_action) } // uint64 default_timeout_microseconds = 2; inline void ModelQueuePolicy::clear_default_timeout_microseconds() { _impl_.default_timeout_microseconds_ = uint64_t{0u}; } inline uint64_t ModelQueuePolicy::_internal_default_timeout_microseconds() const { return _impl_.default_timeout_microseconds_; } inline uint64_t ModelQueuePolicy::default_timeout_microseconds() const { // @@protoc_insertion_point(field_get:inference.ModelQueuePolicy.default_timeout_microseconds) return _internal_default_timeout_microseconds(); } inline void ModelQueuePolicy::_internal_set_default_timeout_microseconds(uint64_t value) { _impl_.default_timeout_microseconds_ = value; } inline void ModelQueuePolicy::set_default_timeout_microseconds(uint64_t value) { _internal_set_default_timeout_microseconds(value); // @@protoc_insertion_point(field_set:inference.ModelQueuePolicy.default_timeout_microseconds) } // bool allow_timeout_override = 3; inline void ModelQueuePolicy::clear_allow_timeout_override() { _impl_.allow_timeout_override_ = false; } inline bool ModelQueuePolicy::_internal_allow_timeout_override() const { return _impl_.allow_timeout_override_; } inline bool ModelQueuePolicy::allow_timeout_override() const { // @@protoc_insertion_point(field_get:inference.ModelQueuePolicy.allow_timeout_override) return _internal_allow_timeout_override(); } inline void ModelQueuePolicy::_internal_set_allow_timeout_override(bool value) { _impl_.allow_timeout_override_ = value; } inline void ModelQueuePolicy::set_allow_timeout_override(bool value) { _internal_set_allow_timeout_override(value); // @@protoc_insertion_point(field_set:inference.ModelQueuePolicy.allow_timeout_override) } // uint32 max_queue_size = 4; inline void ModelQueuePolicy::clear_max_queue_size() { _impl_.max_queue_size_ = 0u; } inline uint32_t ModelQueuePolicy::_internal_max_queue_size() const { return _impl_.max_queue_size_; } inline uint32_t ModelQueuePolicy::max_queue_size() const { // @@protoc_insertion_point(field_get:inference.ModelQueuePolicy.max_queue_size) return _internal_max_queue_size(); } inline void ModelQueuePolicy::_internal_set_max_queue_size(uint32_t value) { _impl_.max_queue_size_ = value; } inline void ModelQueuePolicy::set_max_queue_size(uint32_t value) { _internal_set_max_queue_size(value); // @@protoc_insertion_point(field_set:inference.ModelQueuePolicy.max_queue_size) } // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ModelDynamicBatching // repeated int32 preferred_batch_size = 1; inline int ModelDynamicBatching::_internal_preferred_batch_size_size() const { return _impl_.preferred_batch_size_.size(); } inline int ModelDynamicBatching::preferred_batch_size_size() const { return _internal_preferred_batch_size_size(); } inline void ModelDynamicBatching::clear_preferred_batch_size() { _impl_.preferred_batch_size_.Clear(); } inline int32_t ModelDynamicBatching::_internal_preferred_batch_size(int index) const { return _impl_.preferred_batch_size_.Get(index); } inline int32_t ModelDynamicBatching::preferred_batch_size(int index) const { // @@protoc_insertion_point(field_get:inference.ModelDynamicBatching.preferred_batch_size) return _internal_preferred_batch_size(index); } inline void ModelDynamicBatching::set_preferred_batch_size(int index, int32_t value) { _impl_.preferred_batch_size_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelDynamicBatching.preferred_batch_size) } inline void ModelDynamicBatching::_internal_add_preferred_batch_size(int32_t value) { _impl_.preferred_batch_size_.Add(value); } inline void ModelDynamicBatching::add_preferred_batch_size(int32_t value) { _internal_add_preferred_batch_size(value); // @@protoc_insertion_point(field_add:inference.ModelDynamicBatching.preferred_batch_size) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& ModelDynamicBatching::_internal_preferred_batch_size() const { return _impl_.preferred_batch_size_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& ModelDynamicBatching::preferred_batch_size() const { // @@protoc_insertion_point(field_list:inference.ModelDynamicBatching.preferred_batch_size) return _internal_preferred_batch_size(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* ModelDynamicBatching::_internal_mutable_preferred_batch_size() { return &_impl_.preferred_batch_size_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* ModelDynamicBatching::mutable_preferred_batch_size() { // @@protoc_insertion_point(field_mutable_list:inference.ModelDynamicBatching.preferred_batch_size) return _internal_mutable_preferred_batch_size(); } // uint64 max_queue_delay_microseconds = 2; inline void ModelDynamicBatching::clear_max_queue_delay_microseconds() { _impl_.max_queue_delay_microseconds_ = uint64_t{0u}; } inline uint64_t ModelDynamicBatching::_internal_max_queue_delay_microseconds() const { return _impl_.max_queue_delay_microseconds_; } inline uint64_t ModelDynamicBatching::max_queue_delay_microseconds() const { // @@protoc_insertion_point(field_get:inference.ModelDynamicBatching.max_queue_delay_microseconds) return _internal_max_queue_delay_microseconds(); } inline void ModelDynamicBatching::_internal_set_max_queue_delay_microseconds(uint64_t value) { _impl_.max_queue_delay_microseconds_ = value; } inline void ModelDynamicBatching::set_max_queue_delay_microseconds(uint64_t value) { _internal_set_max_queue_delay_microseconds(value); // @@protoc_insertion_point(field_set:inference.ModelDynamicBatching.max_queue_delay_microseconds) } // bool preserve_ordering = 3; inline void ModelDynamicBatching::clear_preserve_ordering() { _impl_.preserve_ordering_ = false; } inline bool ModelDynamicBatching::_internal_preserve_ordering() const { return _impl_.preserve_ordering_; } inline bool ModelDynamicBatching::preserve_ordering() const { // @@protoc_insertion_point(field_get:inference.ModelDynamicBatching.preserve_ordering) return _internal_preserve_ordering(); } inline void ModelDynamicBatching::_internal_set_preserve_ordering(bool value) { _impl_.preserve_ordering_ = value; } inline void ModelDynamicBatching::set_preserve_ordering(bool value) { _internal_set_preserve_ordering(value); // @@protoc_insertion_point(field_set:inference.ModelDynamicBatching.preserve_ordering) } // uint64 priority_levels = 4; inline void ModelDynamicBatching::clear_priority_levels() { _impl_.priority_levels_ = uint64_t{0u}; } inline uint64_t ModelDynamicBatching::_internal_priority_levels() const { return _impl_.priority_levels_; } inline uint64_t ModelDynamicBatching::priority_levels() const { // @@protoc_insertion_point(field_get:inference.ModelDynamicBatching.priority_levels) return _internal_priority_levels(); } inline void ModelDynamicBatching::_internal_set_priority_levels(uint64_t value) { _impl_.priority_levels_ = value; } inline void ModelDynamicBatching::set_priority_levels(uint64_t value) { _internal_set_priority_levels(value); // @@protoc_insertion_point(field_set:inference.ModelDynamicBatching.priority_levels) } // uint64 default_priority_level = 5; inline void ModelDynamicBatching::clear_default_priority_level() { _impl_.default_priority_level_ = uint64_t{0u}; } inline uint64_t ModelDynamicBatching::_internal_default_priority_level() const { return _impl_.default_priority_level_; } inline uint64_t ModelDynamicBatching::default_priority_level() const { // @@protoc_insertion_point(field_get:inference.ModelDynamicBatching.default_priority_level) return _internal_default_priority_level(); } inline void ModelDynamicBatching::_internal_set_default_priority_level(uint64_t value) { _impl_.default_priority_level_ = value; } inline void ModelDynamicBatching::set_default_priority_level(uint64_t value) { _internal_set_default_priority_level(value); // @@protoc_insertion_point(field_set:inference.ModelDynamicBatching.default_priority_level) } // .inference.ModelQueuePolicy default_queue_policy = 6; inline bool ModelDynamicBatching::_internal_has_default_queue_policy() const { return this != internal_default_instance() && _impl_.default_queue_policy_ != nullptr; } inline bool ModelDynamicBatching::has_default_queue_policy() const { return _internal_has_default_queue_policy(); } inline void ModelDynamicBatching::clear_default_queue_policy() { if (GetArenaForAllocation() == nullptr && _impl_.default_queue_policy_ != nullptr) { delete _impl_.default_queue_policy_; } _impl_.default_queue_policy_ = nullptr; } inline const ::inference::ModelQueuePolicy& ModelDynamicBatching::_internal_default_queue_policy() const { const ::inference::ModelQueuePolicy* p = _impl_.default_queue_policy_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelQueuePolicy_default_instance_); } inline const ::inference::ModelQueuePolicy& ModelDynamicBatching::default_queue_policy() const { // @@protoc_insertion_point(field_get:inference.ModelDynamicBatching.default_queue_policy) return _internal_default_queue_policy(); } inline void ModelDynamicBatching::unsafe_arena_set_allocated_default_queue_policy( ::inference::ModelQueuePolicy* default_queue_policy) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.default_queue_policy_); } _impl_.default_queue_policy_ = default_queue_policy; if (default_queue_policy) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelDynamicBatching.default_queue_policy) } inline ::inference::ModelQueuePolicy* ModelDynamicBatching::release_default_queue_policy() { ::inference::ModelQueuePolicy* temp = _impl_.default_queue_policy_; _impl_.default_queue_policy_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelQueuePolicy* ModelDynamicBatching::unsafe_arena_release_default_queue_policy() { // @@protoc_insertion_point(field_release:inference.ModelDynamicBatching.default_queue_policy) ::inference::ModelQueuePolicy* temp = _impl_.default_queue_policy_; _impl_.default_queue_policy_ = nullptr; return temp; } inline ::inference::ModelQueuePolicy* ModelDynamicBatching::_internal_mutable_default_queue_policy() { if (_impl_.default_queue_policy_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelQueuePolicy>(GetArenaForAllocation()); _impl_.default_queue_policy_ = p; } return _impl_.default_queue_policy_; } inline ::inference::ModelQueuePolicy* ModelDynamicBatching::mutable_default_queue_policy() { ::inference::ModelQueuePolicy* _msg = _internal_mutable_default_queue_policy(); // @@protoc_insertion_point(field_mutable:inference.ModelDynamicBatching.default_queue_policy) return _msg; } inline void ModelDynamicBatching::set_allocated_default_queue_policy(::inference::ModelQueuePolicy* default_queue_policy) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.default_queue_policy_; } if (default_queue_policy) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(default_queue_policy); if (message_arena != submessage_arena) { default_queue_policy = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, default_queue_policy, submessage_arena); } } else { } _impl_.default_queue_policy_ = default_queue_policy; // @@protoc_insertion_point(field_set_allocated:inference.ModelDynamicBatching.default_queue_policy) } // map priority_queue_policy = 7; inline int ModelDynamicBatching::_internal_priority_queue_policy_size() const { return _impl_.priority_queue_policy_.size(); } inline int ModelDynamicBatching::priority_queue_policy_size() const { return _internal_priority_queue_policy_size(); } inline void ModelDynamicBatching::clear_priority_queue_policy() { _impl_.priority_queue_policy_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >& ModelDynamicBatching::_internal_priority_queue_policy() const { return _impl_.priority_queue_policy_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >& ModelDynamicBatching::priority_queue_policy() const { // @@protoc_insertion_point(field_map:inference.ModelDynamicBatching.priority_queue_policy) return _internal_priority_queue_policy(); } inline ::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >* ModelDynamicBatching::_internal_mutable_priority_queue_policy() { return _impl_.priority_queue_policy_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< uint64_t, ::inference::ModelQueuePolicy >* ModelDynamicBatching::mutable_priority_queue_policy() { // @@protoc_insertion_point(field_mutable_map:inference.ModelDynamicBatching.priority_queue_policy) return _internal_mutable_priority_queue_policy(); } // ------------------------------------------------------------------- // ModelSequenceBatching_Control // .inference.ModelSequenceBatching.Control.Kind kind = 1; inline void ModelSequenceBatching_Control::clear_kind() { _impl_.kind_ = 0; } inline ::inference::ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control::_internal_kind() const { return static_cast< ::inference::ModelSequenceBatching_Control_Kind >(_impl_.kind_); } inline ::inference::ModelSequenceBatching_Control_Kind ModelSequenceBatching_Control::kind() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.Control.kind) return _internal_kind(); } inline void ModelSequenceBatching_Control::_internal_set_kind(::inference::ModelSequenceBatching_Control_Kind value) { _impl_.kind_ = value; } inline void ModelSequenceBatching_Control::set_kind(::inference::ModelSequenceBatching_Control_Kind value) { _internal_set_kind(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.Control.kind) } // repeated int32 int32_false_true = 2; inline int ModelSequenceBatching_Control::_internal_int32_false_true_size() const { return _impl_.int32_false_true_.size(); } inline int ModelSequenceBatching_Control::int32_false_true_size() const { return _internal_int32_false_true_size(); } inline void ModelSequenceBatching_Control::clear_int32_false_true() { _impl_.int32_false_true_.Clear(); } inline int32_t ModelSequenceBatching_Control::_internal_int32_false_true(int index) const { return _impl_.int32_false_true_.Get(index); } inline int32_t ModelSequenceBatching_Control::int32_false_true(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.Control.int32_false_true) return _internal_int32_false_true(index); } inline void ModelSequenceBatching_Control::set_int32_false_true(int index, int32_t value) { _impl_.int32_false_true_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.Control.int32_false_true) } inline void ModelSequenceBatching_Control::_internal_add_int32_false_true(int32_t value) { _impl_.int32_false_true_.Add(value); } inline void ModelSequenceBatching_Control::add_int32_false_true(int32_t value) { _internal_add_int32_false_true(value); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.Control.int32_false_true) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& ModelSequenceBatching_Control::_internal_int32_false_true() const { return _impl_.int32_false_true_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& ModelSequenceBatching_Control::int32_false_true() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.Control.int32_false_true) return _internal_int32_false_true(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* ModelSequenceBatching_Control::_internal_mutable_int32_false_true() { return &_impl_.int32_false_true_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* ModelSequenceBatching_Control::mutable_int32_false_true() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.Control.int32_false_true) return _internal_mutable_int32_false_true(); } // repeated float fp32_false_true = 3; inline int ModelSequenceBatching_Control::_internal_fp32_false_true_size() const { return _impl_.fp32_false_true_.size(); } inline int ModelSequenceBatching_Control::fp32_false_true_size() const { return _internal_fp32_false_true_size(); } inline void ModelSequenceBatching_Control::clear_fp32_false_true() { _impl_.fp32_false_true_.Clear(); } inline float ModelSequenceBatching_Control::_internal_fp32_false_true(int index) const { return _impl_.fp32_false_true_.Get(index); } inline float ModelSequenceBatching_Control::fp32_false_true(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.Control.fp32_false_true) return _internal_fp32_false_true(index); } inline void ModelSequenceBatching_Control::set_fp32_false_true(int index, float value) { _impl_.fp32_false_true_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.Control.fp32_false_true) } inline void ModelSequenceBatching_Control::_internal_add_fp32_false_true(float value) { _impl_.fp32_false_true_.Add(value); } inline void ModelSequenceBatching_Control::add_fp32_false_true(float value) { _internal_add_fp32_false_true(value); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.Control.fp32_false_true) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& ModelSequenceBatching_Control::_internal_fp32_false_true() const { return _impl_.fp32_false_true_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& ModelSequenceBatching_Control::fp32_false_true() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.Control.fp32_false_true) return _internal_fp32_false_true(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* ModelSequenceBatching_Control::_internal_mutable_fp32_false_true() { return &_impl_.fp32_false_true_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* ModelSequenceBatching_Control::mutable_fp32_false_true() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.Control.fp32_false_true) return _internal_mutable_fp32_false_true(); } // repeated bool bool_false_true = 5; inline int ModelSequenceBatching_Control::_internal_bool_false_true_size() const { return _impl_.bool_false_true_.size(); } inline int ModelSequenceBatching_Control::bool_false_true_size() const { return _internal_bool_false_true_size(); } inline void ModelSequenceBatching_Control::clear_bool_false_true() { _impl_.bool_false_true_.Clear(); } inline bool ModelSequenceBatching_Control::_internal_bool_false_true(int index) const { return _impl_.bool_false_true_.Get(index); } inline bool ModelSequenceBatching_Control::bool_false_true(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.Control.bool_false_true) return _internal_bool_false_true(index); } inline void ModelSequenceBatching_Control::set_bool_false_true(int index, bool value) { _impl_.bool_false_true_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.Control.bool_false_true) } inline void ModelSequenceBatching_Control::_internal_add_bool_false_true(bool value) { _impl_.bool_false_true_.Add(value); } inline void ModelSequenceBatching_Control::add_bool_false_true(bool value) { _internal_add_bool_false_true(value); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.Control.bool_false_true) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool >& ModelSequenceBatching_Control::_internal_bool_false_true() const { return _impl_.bool_false_true_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool >& ModelSequenceBatching_Control::bool_false_true() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.Control.bool_false_true) return _internal_bool_false_true(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool >* ModelSequenceBatching_Control::_internal_mutable_bool_false_true() { return &_impl_.bool_false_true_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< bool >* ModelSequenceBatching_Control::mutable_bool_false_true() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.Control.bool_false_true) return _internal_mutable_bool_false_true(); } // .inference.DataType data_type = 4; inline void ModelSequenceBatching_Control::clear_data_type() { _impl_.data_type_ = 0; } inline ::inference::DataType ModelSequenceBatching_Control::_internal_data_type() const { return static_cast< ::inference::DataType >(_impl_.data_type_); } inline ::inference::DataType ModelSequenceBatching_Control::data_type() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.Control.data_type) return _internal_data_type(); } inline void ModelSequenceBatching_Control::_internal_set_data_type(::inference::DataType value) { _impl_.data_type_ = value; } inline void ModelSequenceBatching_Control::set_data_type(::inference::DataType value) { _internal_set_data_type(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.Control.data_type) } // ------------------------------------------------------------------- // ModelSequenceBatching_ControlInput // string name = 1; inline void ModelSequenceBatching_ControlInput::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelSequenceBatching_ControlInput::name() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.ControlInput.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelSequenceBatching_ControlInput::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.ControlInput.name) } inline std::string* ModelSequenceBatching_ControlInput::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.ControlInput.name) return _s; } inline const std::string& ModelSequenceBatching_ControlInput::_internal_name() const { return _impl_.name_.Get(); } inline void ModelSequenceBatching_ControlInput::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_ControlInput::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_ControlInput::release_name() { // @@protoc_insertion_point(field_release:inference.ModelSequenceBatching.ControlInput.name) return _impl_.name_.Release(); } inline void ModelSequenceBatching_ControlInput::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelSequenceBatching.ControlInput.name) } // repeated .inference.ModelSequenceBatching.Control control = 2; inline int ModelSequenceBatching_ControlInput::_internal_control_size() const { return _impl_.control_.size(); } inline int ModelSequenceBatching_ControlInput::control_size() const { return _internal_control_size(); } inline void ModelSequenceBatching_ControlInput::clear_control() { _impl_.control_.Clear(); } inline ::inference::ModelSequenceBatching_Control* ModelSequenceBatching_ControlInput::mutable_control(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.ControlInput.control) return _impl_.control_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_Control >* ModelSequenceBatching_ControlInput::mutable_control() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.ControlInput.control) return &_impl_.control_; } inline const ::inference::ModelSequenceBatching_Control& ModelSequenceBatching_ControlInput::_internal_control(int index) const { return _impl_.control_.Get(index); } inline const ::inference::ModelSequenceBatching_Control& ModelSequenceBatching_ControlInput::control(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.ControlInput.control) return _internal_control(index); } inline ::inference::ModelSequenceBatching_Control* ModelSequenceBatching_ControlInput::_internal_add_control() { return _impl_.control_.Add(); } inline ::inference::ModelSequenceBatching_Control* ModelSequenceBatching_ControlInput::add_control() { ::inference::ModelSequenceBatching_Control* _add = _internal_add_control(); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.ControlInput.control) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_Control >& ModelSequenceBatching_ControlInput::control() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.ControlInput.control) return _impl_.control_; } // ------------------------------------------------------------------- // ModelSequenceBatching_InitialState // .inference.DataType data_type = 1; inline void ModelSequenceBatching_InitialState::clear_data_type() { _impl_.data_type_ = 0; } inline ::inference::DataType ModelSequenceBatching_InitialState::_internal_data_type() const { return static_cast< ::inference::DataType >(_impl_.data_type_); } inline ::inference::DataType ModelSequenceBatching_InitialState::data_type() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.InitialState.data_type) return _internal_data_type(); } inline void ModelSequenceBatching_InitialState::_internal_set_data_type(::inference::DataType value) { _impl_.data_type_ = value; } inline void ModelSequenceBatching_InitialState::set_data_type(::inference::DataType value) { _internal_set_data_type(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.InitialState.data_type) } // repeated int64 dims = 2; inline int ModelSequenceBatching_InitialState::_internal_dims_size() const { return _impl_.dims_.size(); } inline int ModelSequenceBatching_InitialState::dims_size() const { return _internal_dims_size(); } inline void ModelSequenceBatching_InitialState::clear_dims() { _impl_.dims_.Clear(); } inline int64_t ModelSequenceBatching_InitialState::_internal_dims(int index) const { return _impl_.dims_.Get(index); } inline int64_t ModelSequenceBatching_InitialState::dims(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.InitialState.dims) return _internal_dims(index); } inline void ModelSequenceBatching_InitialState::set_dims(int index, int64_t value) { _impl_.dims_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.InitialState.dims) } inline void ModelSequenceBatching_InitialState::_internal_add_dims(int64_t value) { _impl_.dims_.Add(value); } inline void ModelSequenceBatching_InitialState::add_dims(int64_t value) { _internal_add_dims(value); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.InitialState.dims) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelSequenceBatching_InitialState::_internal_dims() const { return _impl_.dims_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelSequenceBatching_InitialState::dims() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.InitialState.dims) return _internal_dims(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelSequenceBatching_InitialState::_internal_mutable_dims() { return &_impl_.dims_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelSequenceBatching_InitialState::mutable_dims() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.InitialState.dims) return _internal_mutable_dims(); } // bool zero_data = 3; inline bool ModelSequenceBatching_InitialState::_internal_has_zero_data() const { return state_data_case() == kZeroData; } inline bool ModelSequenceBatching_InitialState::has_zero_data() const { return _internal_has_zero_data(); } inline void ModelSequenceBatching_InitialState::set_has_zero_data() { _impl_._oneof_case_[0] = kZeroData; } inline void ModelSequenceBatching_InitialState::clear_zero_data() { if (_internal_has_zero_data()) { _impl_.state_data_.zero_data_ = false; clear_has_state_data(); } } inline bool ModelSequenceBatching_InitialState::_internal_zero_data() const { if (_internal_has_zero_data()) { return _impl_.state_data_.zero_data_; } return false; } inline void ModelSequenceBatching_InitialState::_internal_set_zero_data(bool value) { if (!_internal_has_zero_data()) { clear_state_data(); set_has_zero_data(); } _impl_.state_data_.zero_data_ = value; } inline bool ModelSequenceBatching_InitialState::zero_data() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.InitialState.zero_data) return _internal_zero_data(); } inline void ModelSequenceBatching_InitialState::set_zero_data(bool value) { _internal_set_zero_data(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.InitialState.zero_data) } // string data_file = 4; inline bool ModelSequenceBatching_InitialState::_internal_has_data_file() const { return state_data_case() == kDataFile; } inline bool ModelSequenceBatching_InitialState::has_data_file() const { return _internal_has_data_file(); } inline void ModelSequenceBatching_InitialState::set_has_data_file() { _impl_._oneof_case_[0] = kDataFile; } inline void ModelSequenceBatching_InitialState::clear_data_file() { if (_internal_has_data_file()) { _impl_.state_data_.data_file_.Destroy(); clear_has_state_data(); } } inline const std::string& ModelSequenceBatching_InitialState::data_file() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.InitialState.data_file) return _internal_data_file(); } template inline void ModelSequenceBatching_InitialState::set_data_file(ArgT0&& arg0, ArgT... args) { if (!_internal_has_data_file()) { clear_state_data(); set_has_data_file(); _impl_.state_data_.data_file_.InitDefault(); } _impl_.state_data_.data_file_.Set( static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.InitialState.data_file) } inline std::string* ModelSequenceBatching_InitialState::mutable_data_file() { std::string* _s = _internal_mutable_data_file(); // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.InitialState.data_file) return _s; } inline const std::string& ModelSequenceBatching_InitialState::_internal_data_file() const { if (_internal_has_data_file()) { return _impl_.state_data_.data_file_.Get(); } return ::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(); } inline void ModelSequenceBatching_InitialState::_internal_set_data_file(const std::string& value) { if (!_internal_has_data_file()) { clear_state_data(); set_has_data_file(); _impl_.state_data_.data_file_.InitDefault(); } _impl_.state_data_.data_file_.Set(value, GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_InitialState::_internal_mutable_data_file() { if (!_internal_has_data_file()) { clear_state_data(); set_has_data_file(); _impl_.state_data_.data_file_.InitDefault(); } return _impl_.state_data_.data_file_.Mutable( GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_InitialState::release_data_file() { // @@protoc_insertion_point(field_release:inference.ModelSequenceBatching.InitialState.data_file) if (_internal_has_data_file()) { clear_has_state_data(); return _impl_.state_data_.data_file_.Release(); } else { return nullptr; } } inline void ModelSequenceBatching_InitialState::set_allocated_data_file(std::string* data_file) { if (has_state_data()) { clear_state_data(); } if (data_file != nullptr) { set_has_data_file(); _impl_.state_data_.data_file_.InitAllocated(data_file, GetArenaForAllocation()); } // @@protoc_insertion_point(field_set_allocated:inference.ModelSequenceBatching.InitialState.data_file) } // string name = 5; inline void ModelSequenceBatching_InitialState::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelSequenceBatching_InitialState::name() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.InitialState.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelSequenceBatching_InitialState::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.InitialState.name) } inline std::string* ModelSequenceBatching_InitialState::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.InitialState.name) return _s; } inline const std::string& ModelSequenceBatching_InitialState::_internal_name() const { return _impl_.name_.Get(); } inline void ModelSequenceBatching_InitialState::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_InitialState::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_InitialState::release_name() { // @@protoc_insertion_point(field_release:inference.ModelSequenceBatching.InitialState.name) return _impl_.name_.Release(); } inline void ModelSequenceBatching_InitialState::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelSequenceBatching.InitialState.name) } inline bool ModelSequenceBatching_InitialState::has_state_data() const { return state_data_case() != STATE_DATA_NOT_SET; } inline void ModelSequenceBatching_InitialState::clear_has_state_data() { _impl_._oneof_case_[0] = STATE_DATA_NOT_SET; } inline ModelSequenceBatching_InitialState::StateDataCase ModelSequenceBatching_InitialState::state_data_case() const { return ModelSequenceBatching_InitialState::StateDataCase(_impl_._oneof_case_[0]); } // ------------------------------------------------------------------- // ModelSequenceBatching_State // string input_name = 1; inline void ModelSequenceBatching_State::clear_input_name() { _impl_.input_name_.ClearToEmpty(); } inline const std::string& ModelSequenceBatching_State::input_name() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.State.input_name) return _internal_input_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelSequenceBatching_State::set_input_name(ArgT0&& arg0, ArgT... args) { _impl_.input_name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.State.input_name) } inline std::string* ModelSequenceBatching_State::mutable_input_name() { std::string* _s = _internal_mutable_input_name(); // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.State.input_name) return _s; } inline const std::string& ModelSequenceBatching_State::_internal_input_name() const { return _impl_.input_name_.Get(); } inline void ModelSequenceBatching_State::_internal_set_input_name(const std::string& value) { _impl_.input_name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_State::_internal_mutable_input_name() { return _impl_.input_name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_State::release_input_name() { // @@protoc_insertion_point(field_release:inference.ModelSequenceBatching.State.input_name) return _impl_.input_name_.Release(); } inline void ModelSequenceBatching_State::set_allocated_input_name(std::string* input_name) { if (input_name != nullptr) { } else { } _impl_.input_name_.SetAllocated(input_name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.input_name_.IsDefault()) { _impl_.input_name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelSequenceBatching.State.input_name) } // string output_name = 2; inline void ModelSequenceBatching_State::clear_output_name() { _impl_.output_name_.ClearToEmpty(); } inline const std::string& ModelSequenceBatching_State::output_name() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.State.output_name) return _internal_output_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelSequenceBatching_State::set_output_name(ArgT0&& arg0, ArgT... args) { _impl_.output_name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.State.output_name) } inline std::string* ModelSequenceBatching_State::mutable_output_name() { std::string* _s = _internal_mutable_output_name(); // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.State.output_name) return _s; } inline const std::string& ModelSequenceBatching_State::_internal_output_name() const { return _impl_.output_name_.Get(); } inline void ModelSequenceBatching_State::_internal_set_output_name(const std::string& value) { _impl_.output_name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_State::_internal_mutable_output_name() { return _impl_.output_name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelSequenceBatching_State::release_output_name() { // @@protoc_insertion_point(field_release:inference.ModelSequenceBatching.State.output_name) return _impl_.output_name_.Release(); } inline void ModelSequenceBatching_State::set_allocated_output_name(std::string* output_name) { if (output_name != nullptr) { } else { } _impl_.output_name_.SetAllocated(output_name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.output_name_.IsDefault()) { _impl_.output_name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelSequenceBatching.State.output_name) } // .inference.DataType data_type = 3; inline void ModelSequenceBatching_State::clear_data_type() { _impl_.data_type_ = 0; } inline ::inference::DataType ModelSequenceBatching_State::_internal_data_type() const { return static_cast< ::inference::DataType >(_impl_.data_type_); } inline ::inference::DataType ModelSequenceBatching_State::data_type() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.State.data_type) return _internal_data_type(); } inline void ModelSequenceBatching_State::_internal_set_data_type(::inference::DataType value) { _impl_.data_type_ = value; } inline void ModelSequenceBatching_State::set_data_type(::inference::DataType value) { _internal_set_data_type(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.State.data_type) } // repeated int64 dims = 4; inline int ModelSequenceBatching_State::_internal_dims_size() const { return _impl_.dims_.size(); } inline int ModelSequenceBatching_State::dims_size() const { return _internal_dims_size(); } inline void ModelSequenceBatching_State::clear_dims() { _impl_.dims_.Clear(); } inline int64_t ModelSequenceBatching_State::_internal_dims(int index) const { return _impl_.dims_.Get(index); } inline int64_t ModelSequenceBatching_State::dims(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.State.dims) return _internal_dims(index); } inline void ModelSequenceBatching_State::set_dims(int index, int64_t value) { _impl_.dims_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.State.dims) } inline void ModelSequenceBatching_State::_internal_add_dims(int64_t value) { _impl_.dims_.Add(value); } inline void ModelSequenceBatching_State::add_dims(int64_t value) { _internal_add_dims(value); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.State.dims) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelSequenceBatching_State::_internal_dims() const { return _impl_.dims_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelSequenceBatching_State::dims() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.State.dims) return _internal_dims(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelSequenceBatching_State::_internal_mutable_dims() { return &_impl_.dims_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelSequenceBatching_State::mutable_dims() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.State.dims) return _internal_mutable_dims(); } // repeated .inference.ModelSequenceBatching.InitialState initial_state = 5; inline int ModelSequenceBatching_State::_internal_initial_state_size() const { return _impl_.initial_state_.size(); } inline int ModelSequenceBatching_State::initial_state_size() const { return _internal_initial_state_size(); } inline void ModelSequenceBatching_State::clear_initial_state() { _impl_.initial_state_.Clear(); } inline ::inference::ModelSequenceBatching_InitialState* ModelSequenceBatching_State::mutable_initial_state(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.State.initial_state) return _impl_.initial_state_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_InitialState >* ModelSequenceBatching_State::mutable_initial_state() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.State.initial_state) return &_impl_.initial_state_; } inline const ::inference::ModelSequenceBatching_InitialState& ModelSequenceBatching_State::_internal_initial_state(int index) const { return _impl_.initial_state_.Get(index); } inline const ::inference::ModelSequenceBatching_InitialState& ModelSequenceBatching_State::initial_state(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.State.initial_state) return _internal_initial_state(index); } inline ::inference::ModelSequenceBatching_InitialState* ModelSequenceBatching_State::_internal_add_initial_state() { return _impl_.initial_state_.Add(); } inline ::inference::ModelSequenceBatching_InitialState* ModelSequenceBatching_State::add_initial_state() { ::inference::ModelSequenceBatching_InitialState* _add = _internal_add_initial_state(); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.State.initial_state) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_InitialState >& ModelSequenceBatching_State::initial_state() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.State.initial_state) return _impl_.initial_state_; } // bool use_same_buffer_for_input_output = 6; inline void ModelSequenceBatching_State::clear_use_same_buffer_for_input_output() { _impl_.use_same_buffer_for_input_output_ = false; } inline bool ModelSequenceBatching_State::_internal_use_same_buffer_for_input_output() const { return _impl_.use_same_buffer_for_input_output_; } inline bool ModelSequenceBatching_State::use_same_buffer_for_input_output() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.State.use_same_buffer_for_input_output) return _internal_use_same_buffer_for_input_output(); } inline void ModelSequenceBatching_State::_internal_set_use_same_buffer_for_input_output(bool value) { _impl_.use_same_buffer_for_input_output_ = value; } inline void ModelSequenceBatching_State::set_use_same_buffer_for_input_output(bool value) { _internal_set_use_same_buffer_for_input_output(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.State.use_same_buffer_for_input_output) } // bool use_growable_memory = 7; inline void ModelSequenceBatching_State::clear_use_growable_memory() { _impl_.use_growable_memory_ = false; } inline bool ModelSequenceBatching_State::_internal_use_growable_memory() const { return _impl_.use_growable_memory_; } inline bool ModelSequenceBatching_State::use_growable_memory() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.State.use_growable_memory) return _internal_use_growable_memory(); } inline void ModelSequenceBatching_State::_internal_set_use_growable_memory(bool value) { _impl_.use_growable_memory_ = value; } inline void ModelSequenceBatching_State::set_use_growable_memory(bool value) { _internal_set_use_growable_memory(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.State.use_growable_memory) } // ------------------------------------------------------------------- // ModelSequenceBatching_StrategyDirect // uint64 max_queue_delay_microseconds = 1; inline void ModelSequenceBatching_StrategyDirect::clear_max_queue_delay_microseconds() { _impl_.max_queue_delay_microseconds_ = uint64_t{0u}; } inline uint64_t ModelSequenceBatching_StrategyDirect::_internal_max_queue_delay_microseconds() const { return _impl_.max_queue_delay_microseconds_; } inline uint64_t ModelSequenceBatching_StrategyDirect::max_queue_delay_microseconds() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.StrategyDirect.max_queue_delay_microseconds) return _internal_max_queue_delay_microseconds(); } inline void ModelSequenceBatching_StrategyDirect::_internal_set_max_queue_delay_microseconds(uint64_t value) { _impl_.max_queue_delay_microseconds_ = value; } inline void ModelSequenceBatching_StrategyDirect::set_max_queue_delay_microseconds(uint64_t value) { _internal_set_max_queue_delay_microseconds(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.StrategyDirect.max_queue_delay_microseconds) } // float minimum_slot_utilization = 2; inline void ModelSequenceBatching_StrategyDirect::clear_minimum_slot_utilization() { _impl_.minimum_slot_utilization_ = 0; } inline float ModelSequenceBatching_StrategyDirect::_internal_minimum_slot_utilization() const { return _impl_.minimum_slot_utilization_; } inline float ModelSequenceBatching_StrategyDirect::minimum_slot_utilization() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.StrategyDirect.minimum_slot_utilization) return _internal_minimum_slot_utilization(); } inline void ModelSequenceBatching_StrategyDirect::_internal_set_minimum_slot_utilization(float value) { _impl_.minimum_slot_utilization_ = value; } inline void ModelSequenceBatching_StrategyDirect::set_minimum_slot_utilization(float value) { _internal_set_minimum_slot_utilization(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.StrategyDirect.minimum_slot_utilization) } // ------------------------------------------------------------------- // ModelSequenceBatching_StrategyOldest // int32 max_candidate_sequences = 1; inline void ModelSequenceBatching_StrategyOldest::clear_max_candidate_sequences() { _impl_.max_candidate_sequences_ = 0; } inline int32_t ModelSequenceBatching_StrategyOldest::_internal_max_candidate_sequences() const { return _impl_.max_candidate_sequences_; } inline int32_t ModelSequenceBatching_StrategyOldest::max_candidate_sequences() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.StrategyOldest.max_candidate_sequences) return _internal_max_candidate_sequences(); } inline void ModelSequenceBatching_StrategyOldest::_internal_set_max_candidate_sequences(int32_t value) { _impl_.max_candidate_sequences_ = value; } inline void ModelSequenceBatching_StrategyOldest::set_max_candidate_sequences(int32_t value) { _internal_set_max_candidate_sequences(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.StrategyOldest.max_candidate_sequences) } // repeated int32 preferred_batch_size = 2; inline int ModelSequenceBatching_StrategyOldest::_internal_preferred_batch_size_size() const { return _impl_.preferred_batch_size_.size(); } inline int ModelSequenceBatching_StrategyOldest::preferred_batch_size_size() const { return _internal_preferred_batch_size_size(); } inline void ModelSequenceBatching_StrategyOldest::clear_preferred_batch_size() { _impl_.preferred_batch_size_.Clear(); } inline int32_t ModelSequenceBatching_StrategyOldest::_internal_preferred_batch_size(int index) const { return _impl_.preferred_batch_size_.Get(index); } inline int32_t ModelSequenceBatching_StrategyOldest::preferred_batch_size(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.StrategyOldest.preferred_batch_size) return _internal_preferred_batch_size(index); } inline void ModelSequenceBatching_StrategyOldest::set_preferred_batch_size(int index, int32_t value) { _impl_.preferred_batch_size_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.StrategyOldest.preferred_batch_size) } inline void ModelSequenceBatching_StrategyOldest::_internal_add_preferred_batch_size(int32_t value) { _impl_.preferred_batch_size_.Add(value); } inline void ModelSequenceBatching_StrategyOldest::add_preferred_batch_size(int32_t value) { _internal_add_preferred_batch_size(value); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.StrategyOldest.preferred_batch_size) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& ModelSequenceBatching_StrategyOldest::_internal_preferred_batch_size() const { return _impl_.preferred_batch_size_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >& ModelSequenceBatching_StrategyOldest::preferred_batch_size() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.StrategyOldest.preferred_batch_size) return _internal_preferred_batch_size(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* ModelSequenceBatching_StrategyOldest::_internal_mutable_preferred_batch_size() { return &_impl_.preferred_batch_size_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t >* ModelSequenceBatching_StrategyOldest::mutable_preferred_batch_size() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.StrategyOldest.preferred_batch_size) return _internal_mutable_preferred_batch_size(); } // uint64 max_queue_delay_microseconds = 3; inline void ModelSequenceBatching_StrategyOldest::clear_max_queue_delay_microseconds() { _impl_.max_queue_delay_microseconds_ = uint64_t{0u}; } inline uint64_t ModelSequenceBatching_StrategyOldest::_internal_max_queue_delay_microseconds() const { return _impl_.max_queue_delay_microseconds_; } inline uint64_t ModelSequenceBatching_StrategyOldest::max_queue_delay_microseconds() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.StrategyOldest.max_queue_delay_microseconds) return _internal_max_queue_delay_microseconds(); } inline void ModelSequenceBatching_StrategyOldest::_internal_set_max_queue_delay_microseconds(uint64_t value) { _impl_.max_queue_delay_microseconds_ = value; } inline void ModelSequenceBatching_StrategyOldest::set_max_queue_delay_microseconds(uint64_t value) { _internal_set_max_queue_delay_microseconds(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.StrategyOldest.max_queue_delay_microseconds) } // bool preserve_ordering = 4; inline void ModelSequenceBatching_StrategyOldest::clear_preserve_ordering() { _impl_.preserve_ordering_ = false; } inline bool ModelSequenceBatching_StrategyOldest::_internal_preserve_ordering() const { return _impl_.preserve_ordering_; } inline bool ModelSequenceBatching_StrategyOldest::preserve_ordering() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.StrategyOldest.preserve_ordering) return _internal_preserve_ordering(); } inline void ModelSequenceBatching_StrategyOldest::_internal_set_preserve_ordering(bool value) { _impl_.preserve_ordering_ = value; } inline void ModelSequenceBatching_StrategyOldest::set_preserve_ordering(bool value) { _internal_set_preserve_ordering(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.StrategyOldest.preserve_ordering) } // ------------------------------------------------------------------- // ModelSequenceBatching // .inference.ModelSequenceBatching.StrategyDirect direct = 3; inline bool ModelSequenceBatching::_internal_has_direct() const { return strategy_choice_case() == kDirect; } inline bool ModelSequenceBatching::has_direct() const { return _internal_has_direct(); } inline void ModelSequenceBatching::set_has_direct() { _impl_._oneof_case_[0] = kDirect; } inline void ModelSequenceBatching::clear_direct() { if (_internal_has_direct()) { if (GetArenaForAllocation() == nullptr) { delete _impl_.strategy_choice_.direct_; } clear_has_strategy_choice(); } } inline ::inference::ModelSequenceBatching_StrategyDirect* ModelSequenceBatching::release_direct() { // @@protoc_insertion_point(field_release:inference.ModelSequenceBatching.direct) if (_internal_has_direct()) { clear_has_strategy_choice(); ::inference::ModelSequenceBatching_StrategyDirect* temp = _impl_.strategy_choice_.direct_; if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } _impl_.strategy_choice_.direct_ = nullptr; return temp; } else { return nullptr; } } inline const ::inference::ModelSequenceBatching_StrategyDirect& ModelSequenceBatching::_internal_direct() const { return _internal_has_direct() ? *_impl_.strategy_choice_.direct_ : reinterpret_cast< ::inference::ModelSequenceBatching_StrategyDirect&>(::inference::_ModelSequenceBatching_StrategyDirect_default_instance_); } inline const ::inference::ModelSequenceBatching_StrategyDirect& ModelSequenceBatching::direct() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.direct) return _internal_direct(); } inline ::inference::ModelSequenceBatching_StrategyDirect* ModelSequenceBatching::unsafe_arena_release_direct() { // @@protoc_insertion_point(field_unsafe_arena_release:inference.ModelSequenceBatching.direct) if (_internal_has_direct()) { clear_has_strategy_choice(); ::inference::ModelSequenceBatching_StrategyDirect* temp = _impl_.strategy_choice_.direct_; _impl_.strategy_choice_.direct_ = nullptr; return temp; } else { return nullptr; } } inline void ModelSequenceBatching::unsafe_arena_set_allocated_direct(::inference::ModelSequenceBatching_StrategyDirect* direct) { clear_strategy_choice(); if (direct) { set_has_direct(); _impl_.strategy_choice_.direct_ = direct; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelSequenceBatching.direct) } inline ::inference::ModelSequenceBatching_StrategyDirect* ModelSequenceBatching::_internal_mutable_direct() { if (!_internal_has_direct()) { clear_strategy_choice(); set_has_direct(); _impl_.strategy_choice_.direct_ = CreateMaybeMessage< ::inference::ModelSequenceBatching_StrategyDirect >(GetArenaForAllocation()); } return _impl_.strategy_choice_.direct_; } inline ::inference::ModelSequenceBatching_StrategyDirect* ModelSequenceBatching::mutable_direct() { ::inference::ModelSequenceBatching_StrategyDirect* _msg = _internal_mutable_direct(); // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.direct) return _msg; } // .inference.ModelSequenceBatching.StrategyOldest oldest = 4; inline bool ModelSequenceBatching::_internal_has_oldest() const { return strategy_choice_case() == kOldest; } inline bool ModelSequenceBatching::has_oldest() const { return _internal_has_oldest(); } inline void ModelSequenceBatching::set_has_oldest() { _impl_._oneof_case_[0] = kOldest; } inline void ModelSequenceBatching::clear_oldest() { if (_internal_has_oldest()) { if (GetArenaForAllocation() == nullptr) { delete _impl_.strategy_choice_.oldest_; } clear_has_strategy_choice(); } } inline ::inference::ModelSequenceBatching_StrategyOldest* ModelSequenceBatching::release_oldest() { // @@protoc_insertion_point(field_release:inference.ModelSequenceBatching.oldest) if (_internal_has_oldest()) { clear_has_strategy_choice(); ::inference::ModelSequenceBatching_StrategyOldest* temp = _impl_.strategy_choice_.oldest_; if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } _impl_.strategy_choice_.oldest_ = nullptr; return temp; } else { return nullptr; } } inline const ::inference::ModelSequenceBatching_StrategyOldest& ModelSequenceBatching::_internal_oldest() const { return _internal_has_oldest() ? *_impl_.strategy_choice_.oldest_ : reinterpret_cast< ::inference::ModelSequenceBatching_StrategyOldest&>(::inference::_ModelSequenceBatching_StrategyOldest_default_instance_); } inline const ::inference::ModelSequenceBatching_StrategyOldest& ModelSequenceBatching::oldest() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.oldest) return _internal_oldest(); } inline ::inference::ModelSequenceBatching_StrategyOldest* ModelSequenceBatching::unsafe_arena_release_oldest() { // @@protoc_insertion_point(field_unsafe_arena_release:inference.ModelSequenceBatching.oldest) if (_internal_has_oldest()) { clear_has_strategy_choice(); ::inference::ModelSequenceBatching_StrategyOldest* temp = _impl_.strategy_choice_.oldest_; _impl_.strategy_choice_.oldest_ = nullptr; return temp; } else { return nullptr; } } inline void ModelSequenceBatching::unsafe_arena_set_allocated_oldest(::inference::ModelSequenceBatching_StrategyOldest* oldest) { clear_strategy_choice(); if (oldest) { set_has_oldest(); _impl_.strategy_choice_.oldest_ = oldest; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelSequenceBatching.oldest) } inline ::inference::ModelSequenceBatching_StrategyOldest* ModelSequenceBatching::_internal_mutable_oldest() { if (!_internal_has_oldest()) { clear_strategy_choice(); set_has_oldest(); _impl_.strategy_choice_.oldest_ = CreateMaybeMessage< ::inference::ModelSequenceBatching_StrategyOldest >(GetArenaForAllocation()); } return _impl_.strategy_choice_.oldest_; } inline ::inference::ModelSequenceBatching_StrategyOldest* ModelSequenceBatching::mutable_oldest() { ::inference::ModelSequenceBatching_StrategyOldest* _msg = _internal_mutable_oldest(); // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.oldest) return _msg; } // uint64 max_sequence_idle_microseconds = 1; inline void ModelSequenceBatching::clear_max_sequence_idle_microseconds() { _impl_.max_sequence_idle_microseconds_ = uint64_t{0u}; } inline uint64_t ModelSequenceBatching::_internal_max_sequence_idle_microseconds() const { return _impl_.max_sequence_idle_microseconds_; } inline uint64_t ModelSequenceBatching::max_sequence_idle_microseconds() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.max_sequence_idle_microseconds) return _internal_max_sequence_idle_microseconds(); } inline void ModelSequenceBatching::_internal_set_max_sequence_idle_microseconds(uint64_t value) { _impl_.max_sequence_idle_microseconds_ = value; } inline void ModelSequenceBatching::set_max_sequence_idle_microseconds(uint64_t value) { _internal_set_max_sequence_idle_microseconds(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.max_sequence_idle_microseconds) } // repeated .inference.ModelSequenceBatching.ControlInput control_input = 2; inline int ModelSequenceBatching::_internal_control_input_size() const { return _impl_.control_input_.size(); } inline int ModelSequenceBatching::control_input_size() const { return _internal_control_input_size(); } inline void ModelSequenceBatching::clear_control_input() { _impl_.control_input_.Clear(); } inline ::inference::ModelSequenceBatching_ControlInput* ModelSequenceBatching::mutable_control_input(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.control_input) return _impl_.control_input_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_ControlInput >* ModelSequenceBatching::mutable_control_input() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.control_input) return &_impl_.control_input_; } inline const ::inference::ModelSequenceBatching_ControlInput& ModelSequenceBatching::_internal_control_input(int index) const { return _impl_.control_input_.Get(index); } inline const ::inference::ModelSequenceBatching_ControlInput& ModelSequenceBatching::control_input(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.control_input) return _internal_control_input(index); } inline ::inference::ModelSequenceBatching_ControlInput* ModelSequenceBatching::_internal_add_control_input() { return _impl_.control_input_.Add(); } inline ::inference::ModelSequenceBatching_ControlInput* ModelSequenceBatching::add_control_input() { ::inference::ModelSequenceBatching_ControlInput* _add = _internal_add_control_input(); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.control_input) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_ControlInput >& ModelSequenceBatching::control_input() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.control_input) return _impl_.control_input_; } // repeated .inference.ModelSequenceBatching.State state = 5; inline int ModelSequenceBatching::_internal_state_size() const { return _impl_.state_.size(); } inline int ModelSequenceBatching::state_size() const { return _internal_state_size(); } inline void ModelSequenceBatching::clear_state() { _impl_.state_.Clear(); } inline ::inference::ModelSequenceBatching_State* ModelSequenceBatching::mutable_state(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelSequenceBatching.state) return _impl_.state_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_State >* ModelSequenceBatching::mutable_state() { // @@protoc_insertion_point(field_mutable_list:inference.ModelSequenceBatching.state) return &_impl_.state_; } inline const ::inference::ModelSequenceBatching_State& ModelSequenceBatching::_internal_state(int index) const { return _impl_.state_.Get(index); } inline const ::inference::ModelSequenceBatching_State& ModelSequenceBatching::state(int index) const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.state) return _internal_state(index); } inline ::inference::ModelSequenceBatching_State* ModelSequenceBatching::_internal_add_state() { return _impl_.state_.Add(); } inline ::inference::ModelSequenceBatching_State* ModelSequenceBatching::add_state() { ::inference::ModelSequenceBatching_State* _add = _internal_add_state(); // @@protoc_insertion_point(field_add:inference.ModelSequenceBatching.state) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelSequenceBatching_State >& ModelSequenceBatching::state() const { // @@protoc_insertion_point(field_list:inference.ModelSequenceBatching.state) return _impl_.state_; } // bool iterative_sequence = 6; inline void ModelSequenceBatching::clear_iterative_sequence() { _impl_.iterative_sequence_ = false; } inline bool ModelSequenceBatching::_internal_iterative_sequence() const { return _impl_.iterative_sequence_; } inline bool ModelSequenceBatching::iterative_sequence() const { // @@protoc_insertion_point(field_get:inference.ModelSequenceBatching.iterative_sequence) return _internal_iterative_sequence(); } inline void ModelSequenceBatching::_internal_set_iterative_sequence(bool value) { _impl_.iterative_sequence_ = value; } inline void ModelSequenceBatching::set_iterative_sequence(bool value) { _internal_set_iterative_sequence(value); // @@protoc_insertion_point(field_set:inference.ModelSequenceBatching.iterative_sequence) } inline bool ModelSequenceBatching::has_strategy_choice() const { return strategy_choice_case() != STRATEGY_CHOICE_NOT_SET; } inline void ModelSequenceBatching::clear_has_strategy_choice() { _impl_._oneof_case_[0] = STRATEGY_CHOICE_NOT_SET; } inline ModelSequenceBatching::StrategyChoiceCase ModelSequenceBatching::strategy_choice_case() const { return ModelSequenceBatching::StrategyChoiceCase(_impl_._oneof_case_[0]); } // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ModelEnsembling_Step // string model_name = 1; inline void ModelEnsembling_Step::clear_model_name() { _impl_.model_name_.ClearToEmpty(); } inline const std::string& ModelEnsembling_Step::model_name() const { // @@protoc_insertion_point(field_get:inference.ModelEnsembling.Step.model_name) return _internal_model_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelEnsembling_Step::set_model_name(ArgT0&& arg0, ArgT... args) { _impl_.model_name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelEnsembling.Step.model_name) } inline std::string* ModelEnsembling_Step::mutable_model_name() { std::string* _s = _internal_mutable_model_name(); // @@protoc_insertion_point(field_mutable:inference.ModelEnsembling.Step.model_name) return _s; } inline const std::string& ModelEnsembling_Step::_internal_model_name() const { return _impl_.model_name_.Get(); } inline void ModelEnsembling_Step::_internal_set_model_name(const std::string& value) { _impl_.model_name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelEnsembling_Step::_internal_mutable_model_name() { return _impl_.model_name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelEnsembling_Step::release_model_name() { // @@protoc_insertion_point(field_release:inference.ModelEnsembling.Step.model_name) return _impl_.model_name_.Release(); } inline void ModelEnsembling_Step::set_allocated_model_name(std::string* model_name) { if (model_name != nullptr) { } else { } _impl_.model_name_.SetAllocated(model_name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.model_name_.IsDefault()) { _impl_.model_name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelEnsembling.Step.model_name) } // int64 model_version = 2; inline void ModelEnsembling_Step::clear_model_version() { _impl_.model_version_ = int64_t{0}; } inline int64_t ModelEnsembling_Step::_internal_model_version() const { return _impl_.model_version_; } inline int64_t ModelEnsembling_Step::model_version() const { // @@protoc_insertion_point(field_get:inference.ModelEnsembling.Step.model_version) return _internal_model_version(); } inline void ModelEnsembling_Step::_internal_set_model_version(int64_t value) { _impl_.model_version_ = value; } inline void ModelEnsembling_Step::set_model_version(int64_t value) { _internal_set_model_version(value); // @@protoc_insertion_point(field_set:inference.ModelEnsembling.Step.model_version) } // map input_map = 3; inline int ModelEnsembling_Step::_internal_input_map_size() const { return _impl_.input_map_.size(); } inline int ModelEnsembling_Step::input_map_size() const { return _internal_input_map_size(); } inline void ModelEnsembling_Step::clear_input_map() { _impl_.input_map_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelEnsembling_Step::_internal_input_map() const { return _impl_.input_map_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelEnsembling_Step::input_map() const { // @@protoc_insertion_point(field_map:inference.ModelEnsembling.Step.input_map) return _internal_input_map(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelEnsembling_Step::_internal_mutable_input_map() { return _impl_.input_map_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelEnsembling_Step::mutable_input_map() { // @@protoc_insertion_point(field_mutable_map:inference.ModelEnsembling.Step.input_map) return _internal_mutable_input_map(); } // map output_map = 4; inline int ModelEnsembling_Step::_internal_output_map_size() const { return _impl_.output_map_.size(); } inline int ModelEnsembling_Step::output_map_size() const { return _internal_output_map_size(); } inline void ModelEnsembling_Step::clear_output_map() { _impl_.output_map_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelEnsembling_Step::_internal_output_map() const { return _impl_.output_map_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelEnsembling_Step::output_map() const { // @@protoc_insertion_point(field_map:inference.ModelEnsembling.Step.output_map) return _internal_output_map(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelEnsembling_Step::_internal_mutable_output_map() { return _impl_.output_map_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelEnsembling_Step::mutable_output_map() { // @@protoc_insertion_point(field_mutable_map:inference.ModelEnsembling.Step.output_map) return _internal_mutable_output_map(); } // string model_namespace = 5; inline void ModelEnsembling_Step::clear_model_namespace() { _impl_.model_namespace_.ClearToEmpty(); } inline const std::string& ModelEnsembling_Step::model_namespace() const { // @@protoc_insertion_point(field_get:inference.ModelEnsembling.Step.model_namespace) return _internal_model_namespace(); } template inline PROTOBUF_ALWAYS_INLINE void ModelEnsembling_Step::set_model_namespace(ArgT0&& arg0, ArgT... args) { _impl_.model_namespace_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelEnsembling.Step.model_namespace) } inline std::string* ModelEnsembling_Step::mutable_model_namespace() { std::string* _s = _internal_mutable_model_namespace(); // @@protoc_insertion_point(field_mutable:inference.ModelEnsembling.Step.model_namespace) return _s; } inline const std::string& ModelEnsembling_Step::_internal_model_namespace() const { return _impl_.model_namespace_.Get(); } inline void ModelEnsembling_Step::_internal_set_model_namespace(const std::string& value) { _impl_.model_namespace_.Set(value, GetArenaForAllocation()); } inline std::string* ModelEnsembling_Step::_internal_mutable_model_namespace() { return _impl_.model_namespace_.Mutable(GetArenaForAllocation()); } inline std::string* ModelEnsembling_Step::release_model_namespace() { // @@protoc_insertion_point(field_release:inference.ModelEnsembling.Step.model_namespace) return _impl_.model_namespace_.Release(); } inline void ModelEnsembling_Step::set_allocated_model_namespace(std::string* model_namespace) { if (model_namespace != nullptr) { } else { } _impl_.model_namespace_.SetAllocated(model_namespace, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.model_namespace_.IsDefault()) { _impl_.model_namespace_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelEnsembling.Step.model_namespace) } // ------------------------------------------------------------------- // ModelEnsembling // repeated .inference.ModelEnsembling.Step step = 1; inline int ModelEnsembling::_internal_step_size() const { return _impl_.step_.size(); } inline int ModelEnsembling::step_size() const { return _internal_step_size(); } inline void ModelEnsembling::clear_step() { _impl_.step_.Clear(); } inline ::inference::ModelEnsembling_Step* ModelEnsembling::mutable_step(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelEnsembling.step) return _impl_.step_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelEnsembling_Step >* ModelEnsembling::mutable_step() { // @@protoc_insertion_point(field_mutable_list:inference.ModelEnsembling.step) return &_impl_.step_; } inline const ::inference::ModelEnsembling_Step& ModelEnsembling::_internal_step(int index) const { return _impl_.step_.Get(index); } inline const ::inference::ModelEnsembling_Step& ModelEnsembling::step(int index) const { // @@protoc_insertion_point(field_get:inference.ModelEnsembling.step) return _internal_step(index); } inline ::inference::ModelEnsembling_Step* ModelEnsembling::_internal_add_step() { return _impl_.step_.Add(); } inline ::inference::ModelEnsembling_Step* ModelEnsembling::add_step() { ::inference::ModelEnsembling_Step* _add = _internal_add_step(); // @@protoc_insertion_point(field_add:inference.ModelEnsembling.step) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelEnsembling_Step >& ModelEnsembling::step() const { // @@protoc_insertion_point(field_list:inference.ModelEnsembling.step) return _impl_.step_; } // ------------------------------------------------------------------- // ModelParameter // string string_value = 1; inline void ModelParameter::clear_string_value() { _impl_.string_value_.ClearToEmpty(); } inline const std::string& ModelParameter::string_value() const { // @@protoc_insertion_point(field_get:inference.ModelParameter.string_value) return _internal_string_value(); } template inline PROTOBUF_ALWAYS_INLINE void ModelParameter::set_string_value(ArgT0&& arg0, ArgT... args) { _impl_.string_value_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelParameter.string_value) } inline std::string* ModelParameter::mutable_string_value() { std::string* _s = _internal_mutable_string_value(); // @@protoc_insertion_point(field_mutable:inference.ModelParameter.string_value) return _s; } inline const std::string& ModelParameter::_internal_string_value() const { return _impl_.string_value_.Get(); } inline void ModelParameter::_internal_set_string_value(const std::string& value) { _impl_.string_value_.Set(value, GetArenaForAllocation()); } inline std::string* ModelParameter::_internal_mutable_string_value() { return _impl_.string_value_.Mutable(GetArenaForAllocation()); } inline std::string* ModelParameter::release_string_value() { // @@protoc_insertion_point(field_release:inference.ModelParameter.string_value) return _impl_.string_value_.Release(); } inline void ModelParameter::set_allocated_string_value(std::string* string_value) { if (string_value != nullptr) { } else { } _impl_.string_value_.SetAllocated(string_value, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.string_value_.IsDefault()) { _impl_.string_value_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelParameter.string_value) } // ------------------------------------------------------------------- // ModelWarmup_Input // .inference.DataType data_type = 1; inline void ModelWarmup_Input::clear_data_type() { _impl_.data_type_ = 0; } inline ::inference::DataType ModelWarmup_Input::_internal_data_type() const { return static_cast< ::inference::DataType >(_impl_.data_type_); } inline ::inference::DataType ModelWarmup_Input::data_type() const { // @@protoc_insertion_point(field_get:inference.ModelWarmup.Input.data_type) return _internal_data_type(); } inline void ModelWarmup_Input::_internal_set_data_type(::inference::DataType value) { _impl_.data_type_ = value; } inline void ModelWarmup_Input::set_data_type(::inference::DataType value) { _internal_set_data_type(value); // @@protoc_insertion_point(field_set:inference.ModelWarmup.Input.data_type) } // repeated int64 dims = 2; inline int ModelWarmup_Input::_internal_dims_size() const { return _impl_.dims_.size(); } inline int ModelWarmup_Input::dims_size() const { return _internal_dims_size(); } inline void ModelWarmup_Input::clear_dims() { _impl_.dims_.Clear(); } inline int64_t ModelWarmup_Input::_internal_dims(int index) const { return _impl_.dims_.Get(index); } inline int64_t ModelWarmup_Input::dims(int index) const { // @@protoc_insertion_point(field_get:inference.ModelWarmup.Input.dims) return _internal_dims(index); } inline void ModelWarmup_Input::set_dims(int index, int64_t value) { _impl_.dims_.Set(index, value); // @@protoc_insertion_point(field_set:inference.ModelWarmup.Input.dims) } inline void ModelWarmup_Input::_internal_add_dims(int64_t value) { _impl_.dims_.Add(value); } inline void ModelWarmup_Input::add_dims(int64_t value) { _internal_add_dims(value); // @@protoc_insertion_point(field_add:inference.ModelWarmup.Input.dims) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelWarmup_Input::_internal_dims() const { return _impl_.dims_; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& ModelWarmup_Input::dims() const { // @@protoc_insertion_point(field_list:inference.ModelWarmup.Input.dims) return _internal_dims(); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelWarmup_Input::_internal_mutable_dims() { return &_impl_.dims_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* ModelWarmup_Input::mutable_dims() { // @@protoc_insertion_point(field_mutable_list:inference.ModelWarmup.Input.dims) return _internal_mutable_dims(); } // bool zero_data = 3; inline bool ModelWarmup_Input::_internal_has_zero_data() const { return input_data_type_case() == kZeroData; } inline bool ModelWarmup_Input::has_zero_data() const { return _internal_has_zero_data(); } inline void ModelWarmup_Input::set_has_zero_data() { _impl_._oneof_case_[0] = kZeroData; } inline void ModelWarmup_Input::clear_zero_data() { if (_internal_has_zero_data()) { _impl_.input_data_type_.zero_data_ = false; clear_has_input_data_type(); } } inline bool ModelWarmup_Input::_internal_zero_data() const { if (_internal_has_zero_data()) { return _impl_.input_data_type_.zero_data_; } return false; } inline void ModelWarmup_Input::_internal_set_zero_data(bool value) { if (!_internal_has_zero_data()) { clear_input_data_type(); set_has_zero_data(); } _impl_.input_data_type_.zero_data_ = value; } inline bool ModelWarmup_Input::zero_data() const { // @@protoc_insertion_point(field_get:inference.ModelWarmup.Input.zero_data) return _internal_zero_data(); } inline void ModelWarmup_Input::set_zero_data(bool value) { _internal_set_zero_data(value); // @@protoc_insertion_point(field_set:inference.ModelWarmup.Input.zero_data) } // bool random_data = 4; inline bool ModelWarmup_Input::_internal_has_random_data() const { return input_data_type_case() == kRandomData; } inline bool ModelWarmup_Input::has_random_data() const { return _internal_has_random_data(); } inline void ModelWarmup_Input::set_has_random_data() { _impl_._oneof_case_[0] = kRandomData; } inline void ModelWarmup_Input::clear_random_data() { if (_internal_has_random_data()) { _impl_.input_data_type_.random_data_ = false; clear_has_input_data_type(); } } inline bool ModelWarmup_Input::_internal_random_data() const { if (_internal_has_random_data()) { return _impl_.input_data_type_.random_data_; } return false; } inline void ModelWarmup_Input::_internal_set_random_data(bool value) { if (!_internal_has_random_data()) { clear_input_data_type(); set_has_random_data(); } _impl_.input_data_type_.random_data_ = value; } inline bool ModelWarmup_Input::random_data() const { // @@protoc_insertion_point(field_get:inference.ModelWarmup.Input.random_data) return _internal_random_data(); } inline void ModelWarmup_Input::set_random_data(bool value) { _internal_set_random_data(value); // @@protoc_insertion_point(field_set:inference.ModelWarmup.Input.random_data) } // string input_data_file = 5; inline bool ModelWarmup_Input::_internal_has_input_data_file() const { return input_data_type_case() == kInputDataFile; } inline bool ModelWarmup_Input::has_input_data_file() const { return _internal_has_input_data_file(); } inline void ModelWarmup_Input::set_has_input_data_file() { _impl_._oneof_case_[0] = kInputDataFile; } inline void ModelWarmup_Input::clear_input_data_file() { if (_internal_has_input_data_file()) { _impl_.input_data_type_.input_data_file_.Destroy(); clear_has_input_data_type(); } } inline const std::string& ModelWarmup_Input::input_data_file() const { // @@protoc_insertion_point(field_get:inference.ModelWarmup.Input.input_data_file) return _internal_input_data_file(); } template inline void ModelWarmup_Input::set_input_data_file(ArgT0&& arg0, ArgT... args) { if (!_internal_has_input_data_file()) { clear_input_data_type(); set_has_input_data_file(); _impl_.input_data_type_.input_data_file_.InitDefault(); } _impl_.input_data_type_.input_data_file_.Set( static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelWarmup.Input.input_data_file) } inline std::string* ModelWarmup_Input::mutable_input_data_file() { std::string* _s = _internal_mutable_input_data_file(); // @@protoc_insertion_point(field_mutable:inference.ModelWarmup.Input.input_data_file) return _s; } inline const std::string& ModelWarmup_Input::_internal_input_data_file() const { if (_internal_has_input_data_file()) { return _impl_.input_data_type_.input_data_file_.Get(); } return ::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(); } inline void ModelWarmup_Input::_internal_set_input_data_file(const std::string& value) { if (!_internal_has_input_data_file()) { clear_input_data_type(); set_has_input_data_file(); _impl_.input_data_type_.input_data_file_.InitDefault(); } _impl_.input_data_type_.input_data_file_.Set(value, GetArenaForAllocation()); } inline std::string* ModelWarmup_Input::_internal_mutable_input_data_file() { if (!_internal_has_input_data_file()) { clear_input_data_type(); set_has_input_data_file(); _impl_.input_data_type_.input_data_file_.InitDefault(); } return _impl_.input_data_type_.input_data_file_.Mutable( GetArenaForAllocation()); } inline std::string* ModelWarmup_Input::release_input_data_file() { // @@protoc_insertion_point(field_release:inference.ModelWarmup.Input.input_data_file) if (_internal_has_input_data_file()) { clear_has_input_data_type(); return _impl_.input_data_type_.input_data_file_.Release(); } else { return nullptr; } } inline void ModelWarmup_Input::set_allocated_input_data_file(std::string* input_data_file) { if (has_input_data_type()) { clear_input_data_type(); } if (input_data_file != nullptr) { set_has_input_data_file(); _impl_.input_data_type_.input_data_file_.InitAllocated(input_data_file, GetArenaForAllocation()); } // @@protoc_insertion_point(field_set_allocated:inference.ModelWarmup.Input.input_data_file) } inline bool ModelWarmup_Input::has_input_data_type() const { return input_data_type_case() != INPUT_DATA_TYPE_NOT_SET; } inline void ModelWarmup_Input::clear_has_input_data_type() { _impl_._oneof_case_[0] = INPUT_DATA_TYPE_NOT_SET; } inline ModelWarmup_Input::InputDataTypeCase ModelWarmup_Input::input_data_type_case() const { return ModelWarmup_Input::InputDataTypeCase(_impl_._oneof_case_[0]); } // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ModelWarmup // string name = 1; inline void ModelWarmup::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelWarmup::name() const { // @@protoc_insertion_point(field_get:inference.ModelWarmup.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelWarmup::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelWarmup.name) } inline std::string* ModelWarmup::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelWarmup.name) return _s; } inline const std::string& ModelWarmup::_internal_name() const { return _impl_.name_.Get(); } inline void ModelWarmup::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelWarmup::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelWarmup::release_name() { // @@protoc_insertion_point(field_release:inference.ModelWarmup.name) return _impl_.name_.Release(); } inline void ModelWarmup::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelWarmup.name) } // uint32 batch_size = 2; inline void ModelWarmup::clear_batch_size() { _impl_.batch_size_ = 0u; } inline uint32_t ModelWarmup::_internal_batch_size() const { return _impl_.batch_size_; } inline uint32_t ModelWarmup::batch_size() const { // @@protoc_insertion_point(field_get:inference.ModelWarmup.batch_size) return _internal_batch_size(); } inline void ModelWarmup::_internal_set_batch_size(uint32_t value) { _impl_.batch_size_ = value; } inline void ModelWarmup::set_batch_size(uint32_t value) { _internal_set_batch_size(value); // @@protoc_insertion_point(field_set:inference.ModelWarmup.batch_size) } // map inputs = 3; inline int ModelWarmup::_internal_inputs_size() const { return _impl_.inputs_.size(); } inline int ModelWarmup::inputs_size() const { return _internal_inputs_size(); } inline void ModelWarmup::clear_inputs() { _impl_.inputs_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >& ModelWarmup::_internal_inputs() const { return _impl_.inputs_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >& ModelWarmup::inputs() const { // @@protoc_insertion_point(field_map:inference.ModelWarmup.inputs) return _internal_inputs(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >* ModelWarmup::_internal_mutable_inputs() { return _impl_.inputs_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelWarmup_Input >* ModelWarmup::mutable_inputs() { // @@protoc_insertion_point(field_mutable_map:inference.ModelWarmup.inputs) return _internal_mutable_inputs(); } // uint32 count = 4; inline void ModelWarmup::clear_count() { _impl_.count_ = 0u; } inline uint32_t ModelWarmup::_internal_count() const { return _impl_.count_; } inline uint32_t ModelWarmup::count() const { // @@protoc_insertion_point(field_get:inference.ModelWarmup.count) return _internal_count(); } inline void ModelWarmup::_internal_set_count(uint32_t value) { _impl_.count_ = value; } inline void ModelWarmup::set_count(uint32_t value) { _internal_set_count(value); // @@protoc_insertion_point(field_set:inference.ModelWarmup.count) } // ------------------------------------------------------------------- // ModelOperations // repeated string op_library_filename = 1; inline int ModelOperations::_internal_op_library_filename_size() const { return _impl_.op_library_filename_.size(); } inline int ModelOperations::op_library_filename_size() const { return _internal_op_library_filename_size(); } inline void ModelOperations::clear_op_library_filename() { _impl_.op_library_filename_.Clear(); } inline std::string* ModelOperations::add_op_library_filename() { std::string* _s = _internal_add_op_library_filename(); // @@protoc_insertion_point(field_add_mutable:inference.ModelOperations.op_library_filename) return _s; } inline const std::string& ModelOperations::_internal_op_library_filename(int index) const { return _impl_.op_library_filename_.Get(index); } inline const std::string& ModelOperations::op_library_filename(int index) const { // @@protoc_insertion_point(field_get:inference.ModelOperations.op_library_filename) return _internal_op_library_filename(index); } inline std::string* ModelOperations::mutable_op_library_filename(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelOperations.op_library_filename) return _impl_.op_library_filename_.Mutable(index); } inline void ModelOperations::set_op_library_filename(int index, const std::string& value) { _impl_.op_library_filename_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set:inference.ModelOperations.op_library_filename) } inline void ModelOperations::set_op_library_filename(int index, std::string&& value) { _impl_.op_library_filename_.Mutable(index)->assign(std::move(value)); // @@protoc_insertion_point(field_set:inference.ModelOperations.op_library_filename) } inline void ModelOperations::set_op_library_filename(int index, const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.op_library_filename_.Mutable(index)->assign(value); // @@protoc_insertion_point(field_set_char:inference.ModelOperations.op_library_filename) } inline void ModelOperations::set_op_library_filename(int index, const char* value, size_t size) { _impl_.op_library_filename_.Mutable(index)->assign( reinterpret_cast(value), size); // @@protoc_insertion_point(field_set_pointer:inference.ModelOperations.op_library_filename) } inline std::string* ModelOperations::_internal_add_op_library_filename() { return _impl_.op_library_filename_.Add(); } inline void ModelOperations::add_op_library_filename(const std::string& value) { _impl_.op_library_filename_.Add()->assign(value); // @@protoc_insertion_point(field_add:inference.ModelOperations.op_library_filename) } inline void ModelOperations::add_op_library_filename(std::string&& value) { _impl_.op_library_filename_.Add(std::move(value)); // @@protoc_insertion_point(field_add:inference.ModelOperations.op_library_filename) } inline void ModelOperations::add_op_library_filename(const char* value) { GOOGLE_DCHECK(value != nullptr); _impl_.op_library_filename_.Add()->assign(value); // @@protoc_insertion_point(field_add_char:inference.ModelOperations.op_library_filename) } inline void ModelOperations::add_op_library_filename(const char* value, size_t size) { _impl_.op_library_filename_.Add()->assign(reinterpret_cast(value), size); // @@protoc_insertion_point(field_add_pointer:inference.ModelOperations.op_library_filename) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& ModelOperations::op_library_filename() const { // @@protoc_insertion_point(field_list:inference.ModelOperations.op_library_filename) return _impl_.op_library_filename_; } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* ModelOperations::mutable_op_library_filename() { // @@protoc_insertion_point(field_mutable_list:inference.ModelOperations.op_library_filename) return &_impl_.op_library_filename_; } // ------------------------------------------------------------------- // ModelTransactionPolicy // bool decoupled = 1; inline void ModelTransactionPolicy::clear_decoupled() { _impl_.decoupled_ = false; } inline bool ModelTransactionPolicy::_internal_decoupled() const { return _impl_.decoupled_; } inline bool ModelTransactionPolicy::decoupled() const { // @@protoc_insertion_point(field_get:inference.ModelTransactionPolicy.decoupled) return _internal_decoupled(); } inline void ModelTransactionPolicy::_internal_set_decoupled(bool value) { _impl_.decoupled_ = value; } inline void ModelTransactionPolicy::set_decoupled(bool value) { _internal_set_decoupled(value); // @@protoc_insertion_point(field_set:inference.ModelTransactionPolicy.decoupled) } // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ModelRepositoryAgents_Agent // string name = 1; inline void ModelRepositoryAgents_Agent::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelRepositoryAgents_Agent::name() const { // @@protoc_insertion_point(field_get:inference.ModelRepositoryAgents.Agent.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelRepositoryAgents_Agent::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelRepositoryAgents.Agent.name) } inline std::string* ModelRepositoryAgents_Agent::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelRepositoryAgents.Agent.name) return _s; } inline const std::string& ModelRepositoryAgents_Agent::_internal_name() const { return _impl_.name_.Get(); } inline void ModelRepositoryAgents_Agent::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelRepositoryAgents_Agent::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelRepositoryAgents_Agent::release_name() { // @@protoc_insertion_point(field_release:inference.ModelRepositoryAgents.Agent.name) return _impl_.name_.Release(); } inline void ModelRepositoryAgents_Agent::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelRepositoryAgents.Agent.name) } // map parameters = 2; inline int ModelRepositoryAgents_Agent::_internal_parameters_size() const { return _impl_.parameters_.size(); } inline int ModelRepositoryAgents_Agent::parameters_size() const { return _internal_parameters_size(); } inline void ModelRepositoryAgents_Agent::clear_parameters() { _impl_.parameters_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelRepositoryAgents_Agent::_internal_parameters() const { return _impl_.parameters_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelRepositoryAgents_Agent::parameters() const { // @@protoc_insertion_point(field_map:inference.ModelRepositoryAgents.Agent.parameters) return _internal_parameters(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelRepositoryAgents_Agent::_internal_mutable_parameters() { return _impl_.parameters_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelRepositoryAgents_Agent::mutable_parameters() { // @@protoc_insertion_point(field_mutable_map:inference.ModelRepositoryAgents.Agent.parameters) return _internal_mutable_parameters(); } // ------------------------------------------------------------------- // ModelRepositoryAgents // repeated .inference.ModelRepositoryAgents.Agent agents = 1; inline int ModelRepositoryAgents::_internal_agents_size() const { return _impl_.agents_.size(); } inline int ModelRepositoryAgents::agents_size() const { return _internal_agents_size(); } inline void ModelRepositoryAgents::clear_agents() { _impl_.agents_.Clear(); } inline ::inference::ModelRepositoryAgents_Agent* ModelRepositoryAgents::mutable_agents(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelRepositoryAgents.agents) return _impl_.agents_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRepositoryAgents_Agent >* ModelRepositoryAgents::mutable_agents() { // @@protoc_insertion_point(field_mutable_list:inference.ModelRepositoryAgents.agents) return &_impl_.agents_; } inline const ::inference::ModelRepositoryAgents_Agent& ModelRepositoryAgents::_internal_agents(int index) const { return _impl_.agents_.Get(index); } inline const ::inference::ModelRepositoryAgents_Agent& ModelRepositoryAgents::agents(int index) const { // @@protoc_insertion_point(field_get:inference.ModelRepositoryAgents.agents) return _internal_agents(index); } inline ::inference::ModelRepositoryAgents_Agent* ModelRepositoryAgents::_internal_add_agents() { return _impl_.agents_.Add(); } inline ::inference::ModelRepositoryAgents_Agent* ModelRepositoryAgents::add_agents() { ::inference::ModelRepositoryAgents_Agent* _add = _internal_add_agents(); // @@protoc_insertion_point(field_add:inference.ModelRepositoryAgents.agents) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelRepositoryAgents_Agent >& ModelRepositoryAgents::agents() const { // @@protoc_insertion_point(field_list:inference.ModelRepositoryAgents.agents) return _impl_.agents_; } // ------------------------------------------------------------------- // ModelResponseCache // bool enable = 1; inline void ModelResponseCache::clear_enable() { _impl_.enable_ = false; } inline bool ModelResponseCache::_internal_enable() const { return _impl_.enable_; } inline bool ModelResponseCache::enable() const { // @@protoc_insertion_point(field_get:inference.ModelResponseCache.enable) return _internal_enable(); } inline void ModelResponseCache::_internal_set_enable(bool value) { _impl_.enable_ = value; } inline void ModelResponseCache::set_enable(bool value) { _internal_set_enable(value); // @@protoc_insertion_point(field_set:inference.ModelResponseCache.enable) } // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ModelConfig // string name = 1; inline void ModelConfig::clear_name() { _impl_.name_.ClearToEmpty(); } inline const std::string& ModelConfig::name() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.name) return _internal_name(); } template inline PROTOBUF_ALWAYS_INLINE void ModelConfig::set_name(ArgT0&& arg0, ArgT... args) { _impl_.name_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelConfig.name) } inline std::string* ModelConfig::mutable_name() { std::string* _s = _internal_mutable_name(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.name) return _s; } inline const std::string& ModelConfig::_internal_name() const { return _impl_.name_.Get(); } inline void ModelConfig::_internal_set_name(const std::string& value) { _impl_.name_.Set(value, GetArenaForAllocation()); } inline std::string* ModelConfig::_internal_mutable_name() { return _impl_.name_.Mutable(GetArenaForAllocation()); } inline std::string* ModelConfig::release_name() { // @@protoc_insertion_point(field_release:inference.ModelConfig.name) return _impl_.name_.Release(); } inline void ModelConfig::set_allocated_name(std::string* name) { if (name != nullptr) { } else { } _impl_.name_.SetAllocated(name, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.name_.IsDefault()) { _impl_.name_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.name) } // string platform = 2; inline void ModelConfig::clear_platform() { _impl_.platform_.ClearToEmpty(); } inline const std::string& ModelConfig::platform() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.platform) return _internal_platform(); } template inline PROTOBUF_ALWAYS_INLINE void ModelConfig::set_platform(ArgT0&& arg0, ArgT... args) { _impl_.platform_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelConfig.platform) } inline std::string* ModelConfig::mutable_platform() { std::string* _s = _internal_mutable_platform(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.platform) return _s; } inline const std::string& ModelConfig::_internal_platform() const { return _impl_.platform_.Get(); } inline void ModelConfig::_internal_set_platform(const std::string& value) { _impl_.platform_.Set(value, GetArenaForAllocation()); } inline std::string* ModelConfig::_internal_mutable_platform() { return _impl_.platform_.Mutable(GetArenaForAllocation()); } inline std::string* ModelConfig::release_platform() { // @@protoc_insertion_point(field_release:inference.ModelConfig.platform) return _impl_.platform_.Release(); } inline void ModelConfig::set_allocated_platform(std::string* platform) { if (platform != nullptr) { } else { } _impl_.platform_.SetAllocated(platform, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.platform_.IsDefault()) { _impl_.platform_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.platform) } // string backend = 17; inline void ModelConfig::clear_backend() { _impl_.backend_.ClearToEmpty(); } inline const std::string& ModelConfig::backend() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.backend) return _internal_backend(); } template inline PROTOBUF_ALWAYS_INLINE void ModelConfig::set_backend(ArgT0&& arg0, ArgT... args) { _impl_.backend_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelConfig.backend) } inline std::string* ModelConfig::mutable_backend() { std::string* _s = _internal_mutable_backend(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.backend) return _s; } inline const std::string& ModelConfig::_internal_backend() const { return _impl_.backend_.Get(); } inline void ModelConfig::_internal_set_backend(const std::string& value) { _impl_.backend_.Set(value, GetArenaForAllocation()); } inline std::string* ModelConfig::_internal_mutable_backend() { return _impl_.backend_.Mutable(GetArenaForAllocation()); } inline std::string* ModelConfig::release_backend() { // @@protoc_insertion_point(field_release:inference.ModelConfig.backend) return _impl_.backend_.Release(); } inline void ModelConfig::set_allocated_backend(std::string* backend) { if (backend != nullptr) { } else { } _impl_.backend_.SetAllocated(backend, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.backend_.IsDefault()) { _impl_.backend_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.backend) } // string runtime = 25; inline void ModelConfig::clear_runtime() { _impl_.runtime_.ClearToEmpty(); } inline const std::string& ModelConfig::runtime() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.runtime) return _internal_runtime(); } template inline PROTOBUF_ALWAYS_INLINE void ModelConfig::set_runtime(ArgT0&& arg0, ArgT... args) { _impl_.runtime_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelConfig.runtime) } inline std::string* ModelConfig::mutable_runtime() { std::string* _s = _internal_mutable_runtime(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.runtime) return _s; } inline const std::string& ModelConfig::_internal_runtime() const { return _impl_.runtime_.Get(); } inline void ModelConfig::_internal_set_runtime(const std::string& value) { _impl_.runtime_.Set(value, GetArenaForAllocation()); } inline std::string* ModelConfig::_internal_mutable_runtime() { return _impl_.runtime_.Mutable(GetArenaForAllocation()); } inline std::string* ModelConfig::release_runtime() { // @@protoc_insertion_point(field_release:inference.ModelConfig.runtime) return _impl_.runtime_.Release(); } inline void ModelConfig::set_allocated_runtime(std::string* runtime) { if (runtime != nullptr) { } else { } _impl_.runtime_.SetAllocated(runtime, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.runtime_.IsDefault()) { _impl_.runtime_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.runtime) } // .inference.ModelVersionPolicy version_policy = 3; inline bool ModelConfig::_internal_has_version_policy() const { return this != internal_default_instance() && _impl_.version_policy_ != nullptr; } inline bool ModelConfig::has_version_policy() const { return _internal_has_version_policy(); } inline void ModelConfig::clear_version_policy() { if (GetArenaForAllocation() == nullptr && _impl_.version_policy_ != nullptr) { delete _impl_.version_policy_; } _impl_.version_policy_ = nullptr; } inline const ::inference::ModelVersionPolicy& ModelConfig::_internal_version_policy() const { const ::inference::ModelVersionPolicy* p = _impl_.version_policy_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelVersionPolicy_default_instance_); } inline const ::inference::ModelVersionPolicy& ModelConfig::version_policy() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.version_policy) return _internal_version_policy(); } inline void ModelConfig::unsafe_arena_set_allocated_version_policy( ::inference::ModelVersionPolicy* version_policy) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.version_policy_); } _impl_.version_policy_ = version_policy; if (version_policy) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.version_policy) } inline ::inference::ModelVersionPolicy* ModelConfig::release_version_policy() { ::inference::ModelVersionPolicy* temp = _impl_.version_policy_; _impl_.version_policy_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelVersionPolicy* ModelConfig::unsafe_arena_release_version_policy() { // @@protoc_insertion_point(field_release:inference.ModelConfig.version_policy) ::inference::ModelVersionPolicy* temp = _impl_.version_policy_; _impl_.version_policy_ = nullptr; return temp; } inline ::inference::ModelVersionPolicy* ModelConfig::_internal_mutable_version_policy() { if (_impl_.version_policy_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelVersionPolicy>(GetArenaForAllocation()); _impl_.version_policy_ = p; } return _impl_.version_policy_; } inline ::inference::ModelVersionPolicy* ModelConfig::mutable_version_policy() { ::inference::ModelVersionPolicy* _msg = _internal_mutable_version_policy(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.version_policy) return _msg; } inline void ModelConfig::set_allocated_version_policy(::inference::ModelVersionPolicy* version_policy) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.version_policy_; } if (version_policy) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(version_policy); if (message_arena != submessage_arena) { version_policy = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, version_policy, submessage_arena); } } else { } _impl_.version_policy_ = version_policy; // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.version_policy) } // int32 max_batch_size = 4; inline void ModelConfig::clear_max_batch_size() { _impl_.max_batch_size_ = 0; } inline int32_t ModelConfig::_internal_max_batch_size() const { return _impl_.max_batch_size_; } inline int32_t ModelConfig::max_batch_size() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.max_batch_size) return _internal_max_batch_size(); } inline void ModelConfig::_internal_set_max_batch_size(int32_t value) { _impl_.max_batch_size_ = value; } inline void ModelConfig::set_max_batch_size(int32_t value) { _internal_set_max_batch_size(value); // @@protoc_insertion_point(field_set:inference.ModelConfig.max_batch_size) } // repeated .inference.ModelInput input = 5; inline int ModelConfig::_internal_input_size() const { return _impl_.input_.size(); } inline int ModelConfig::input_size() const { return _internal_input_size(); } inline void ModelConfig::clear_input() { _impl_.input_.Clear(); } inline ::inference::ModelInput* ModelConfig::mutable_input(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelConfig.input) return _impl_.input_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInput >* ModelConfig::mutable_input() { // @@protoc_insertion_point(field_mutable_list:inference.ModelConfig.input) return &_impl_.input_; } inline const ::inference::ModelInput& ModelConfig::_internal_input(int index) const { return _impl_.input_.Get(index); } inline const ::inference::ModelInput& ModelConfig::input(int index) const { // @@protoc_insertion_point(field_get:inference.ModelConfig.input) return _internal_input(index); } inline ::inference::ModelInput* ModelConfig::_internal_add_input() { return _impl_.input_.Add(); } inline ::inference::ModelInput* ModelConfig::add_input() { ::inference::ModelInput* _add = _internal_add_input(); // @@protoc_insertion_point(field_add:inference.ModelConfig.input) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInput >& ModelConfig::input() const { // @@protoc_insertion_point(field_list:inference.ModelConfig.input) return _impl_.input_; } // repeated .inference.ModelOutput output = 6; inline int ModelConfig::_internal_output_size() const { return _impl_.output_.size(); } inline int ModelConfig::output_size() const { return _internal_output_size(); } inline void ModelConfig::clear_output() { _impl_.output_.Clear(); } inline ::inference::ModelOutput* ModelConfig::mutable_output(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelConfig.output) return _impl_.output_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOutput >* ModelConfig::mutable_output() { // @@protoc_insertion_point(field_mutable_list:inference.ModelConfig.output) return &_impl_.output_; } inline const ::inference::ModelOutput& ModelConfig::_internal_output(int index) const { return _impl_.output_.Get(index); } inline const ::inference::ModelOutput& ModelConfig::output(int index) const { // @@protoc_insertion_point(field_get:inference.ModelConfig.output) return _internal_output(index); } inline ::inference::ModelOutput* ModelConfig::_internal_add_output() { return _impl_.output_.Add(); } inline ::inference::ModelOutput* ModelConfig::add_output() { ::inference::ModelOutput* _add = _internal_add_output(); // @@protoc_insertion_point(field_add:inference.ModelConfig.output) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelOutput >& ModelConfig::output() const { // @@protoc_insertion_point(field_list:inference.ModelConfig.output) return _impl_.output_; } // repeated .inference.BatchInput batch_input = 20; inline int ModelConfig::_internal_batch_input_size() const { return _impl_.batch_input_.size(); } inline int ModelConfig::batch_input_size() const { return _internal_batch_input_size(); } inline void ModelConfig::clear_batch_input() { _impl_.batch_input_.Clear(); } inline ::inference::BatchInput* ModelConfig::mutable_batch_input(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelConfig.batch_input) return _impl_.batch_input_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchInput >* ModelConfig::mutable_batch_input() { // @@protoc_insertion_point(field_mutable_list:inference.ModelConfig.batch_input) return &_impl_.batch_input_; } inline const ::inference::BatchInput& ModelConfig::_internal_batch_input(int index) const { return _impl_.batch_input_.Get(index); } inline const ::inference::BatchInput& ModelConfig::batch_input(int index) const { // @@protoc_insertion_point(field_get:inference.ModelConfig.batch_input) return _internal_batch_input(index); } inline ::inference::BatchInput* ModelConfig::_internal_add_batch_input() { return _impl_.batch_input_.Add(); } inline ::inference::BatchInput* ModelConfig::add_batch_input() { ::inference::BatchInput* _add = _internal_add_batch_input(); // @@protoc_insertion_point(field_add:inference.ModelConfig.batch_input) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchInput >& ModelConfig::batch_input() const { // @@protoc_insertion_point(field_list:inference.ModelConfig.batch_input) return _impl_.batch_input_; } // repeated .inference.BatchOutput batch_output = 21; inline int ModelConfig::_internal_batch_output_size() const { return _impl_.batch_output_.size(); } inline int ModelConfig::batch_output_size() const { return _internal_batch_output_size(); } inline void ModelConfig::clear_batch_output() { _impl_.batch_output_.Clear(); } inline ::inference::BatchOutput* ModelConfig::mutable_batch_output(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelConfig.batch_output) return _impl_.batch_output_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchOutput >* ModelConfig::mutable_batch_output() { // @@protoc_insertion_point(field_mutable_list:inference.ModelConfig.batch_output) return &_impl_.batch_output_; } inline const ::inference::BatchOutput& ModelConfig::_internal_batch_output(int index) const { return _impl_.batch_output_.Get(index); } inline const ::inference::BatchOutput& ModelConfig::batch_output(int index) const { // @@protoc_insertion_point(field_get:inference.ModelConfig.batch_output) return _internal_batch_output(index); } inline ::inference::BatchOutput* ModelConfig::_internal_add_batch_output() { return _impl_.batch_output_.Add(); } inline ::inference::BatchOutput* ModelConfig::add_batch_output() { ::inference::BatchOutput* _add = _internal_add_batch_output(); // @@protoc_insertion_point(field_add:inference.ModelConfig.batch_output) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::BatchOutput >& ModelConfig::batch_output() const { // @@protoc_insertion_point(field_list:inference.ModelConfig.batch_output) return _impl_.batch_output_; } // .inference.ModelOptimizationPolicy optimization = 12; inline bool ModelConfig::_internal_has_optimization() const { return this != internal_default_instance() && _impl_.optimization_ != nullptr; } inline bool ModelConfig::has_optimization() const { return _internal_has_optimization(); } inline void ModelConfig::clear_optimization() { if (GetArenaForAllocation() == nullptr && _impl_.optimization_ != nullptr) { delete _impl_.optimization_; } _impl_.optimization_ = nullptr; } inline const ::inference::ModelOptimizationPolicy& ModelConfig::_internal_optimization() const { const ::inference::ModelOptimizationPolicy* p = _impl_.optimization_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelOptimizationPolicy_default_instance_); } inline const ::inference::ModelOptimizationPolicy& ModelConfig::optimization() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.optimization) return _internal_optimization(); } inline void ModelConfig::unsafe_arena_set_allocated_optimization( ::inference::ModelOptimizationPolicy* optimization) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.optimization_); } _impl_.optimization_ = optimization; if (optimization) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.optimization) } inline ::inference::ModelOptimizationPolicy* ModelConfig::release_optimization() { ::inference::ModelOptimizationPolicy* temp = _impl_.optimization_; _impl_.optimization_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelOptimizationPolicy* ModelConfig::unsafe_arena_release_optimization() { // @@protoc_insertion_point(field_release:inference.ModelConfig.optimization) ::inference::ModelOptimizationPolicy* temp = _impl_.optimization_; _impl_.optimization_ = nullptr; return temp; } inline ::inference::ModelOptimizationPolicy* ModelConfig::_internal_mutable_optimization() { if (_impl_.optimization_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelOptimizationPolicy>(GetArenaForAllocation()); _impl_.optimization_ = p; } return _impl_.optimization_; } inline ::inference::ModelOptimizationPolicy* ModelConfig::mutable_optimization() { ::inference::ModelOptimizationPolicy* _msg = _internal_mutable_optimization(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.optimization) return _msg; } inline void ModelConfig::set_allocated_optimization(::inference::ModelOptimizationPolicy* optimization) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.optimization_; } if (optimization) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(optimization); if (message_arena != submessage_arena) { optimization = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, optimization, submessage_arena); } } else { } _impl_.optimization_ = optimization; // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.optimization) } // .inference.ModelDynamicBatching dynamic_batching = 11; inline bool ModelConfig::_internal_has_dynamic_batching() const { return scheduling_choice_case() == kDynamicBatching; } inline bool ModelConfig::has_dynamic_batching() const { return _internal_has_dynamic_batching(); } inline void ModelConfig::set_has_dynamic_batching() { _impl_._oneof_case_[0] = kDynamicBatching; } inline void ModelConfig::clear_dynamic_batching() { if (_internal_has_dynamic_batching()) { if (GetArenaForAllocation() == nullptr) { delete _impl_.scheduling_choice_.dynamic_batching_; } clear_has_scheduling_choice(); } } inline ::inference::ModelDynamicBatching* ModelConfig::release_dynamic_batching() { // @@protoc_insertion_point(field_release:inference.ModelConfig.dynamic_batching) if (_internal_has_dynamic_batching()) { clear_has_scheduling_choice(); ::inference::ModelDynamicBatching* temp = _impl_.scheduling_choice_.dynamic_batching_; if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } _impl_.scheduling_choice_.dynamic_batching_ = nullptr; return temp; } else { return nullptr; } } inline const ::inference::ModelDynamicBatching& ModelConfig::_internal_dynamic_batching() const { return _internal_has_dynamic_batching() ? *_impl_.scheduling_choice_.dynamic_batching_ : reinterpret_cast< ::inference::ModelDynamicBatching&>(::inference::_ModelDynamicBatching_default_instance_); } inline const ::inference::ModelDynamicBatching& ModelConfig::dynamic_batching() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.dynamic_batching) return _internal_dynamic_batching(); } inline ::inference::ModelDynamicBatching* ModelConfig::unsafe_arena_release_dynamic_batching() { // @@protoc_insertion_point(field_unsafe_arena_release:inference.ModelConfig.dynamic_batching) if (_internal_has_dynamic_batching()) { clear_has_scheduling_choice(); ::inference::ModelDynamicBatching* temp = _impl_.scheduling_choice_.dynamic_batching_; _impl_.scheduling_choice_.dynamic_batching_ = nullptr; return temp; } else { return nullptr; } } inline void ModelConfig::unsafe_arena_set_allocated_dynamic_batching(::inference::ModelDynamicBatching* dynamic_batching) { clear_scheduling_choice(); if (dynamic_batching) { set_has_dynamic_batching(); _impl_.scheduling_choice_.dynamic_batching_ = dynamic_batching; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.dynamic_batching) } inline ::inference::ModelDynamicBatching* ModelConfig::_internal_mutable_dynamic_batching() { if (!_internal_has_dynamic_batching()) { clear_scheduling_choice(); set_has_dynamic_batching(); _impl_.scheduling_choice_.dynamic_batching_ = CreateMaybeMessage< ::inference::ModelDynamicBatching >(GetArenaForAllocation()); } return _impl_.scheduling_choice_.dynamic_batching_; } inline ::inference::ModelDynamicBatching* ModelConfig::mutable_dynamic_batching() { ::inference::ModelDynamicBatching* _msg = _internal_mutable_dynamic_batching(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.dynamic_batching) return _msg; } // .inference.ModelSequenceBatching sequence_batching = 13; inline bool ModelConfig::_internal_has_sequence_batching() const { return scheduling_choice_case() == kSequenceBatching; } inline bool ModelConfig::has_sequence_batching() const { return _internal_has_sequence_batching(); } inline void ModelConfig::set_has_sequence_batching() { _impl_._oneof_case_[0] = kSequenceBatching; } inline void ModelConfig::clear_sequence_batching() { if (_internal_has_sequence_batching()) { if (GetArenaForAllocation() == nullptr) { delete _impl_.scheduling_choice_.sequence_batching_; } clear_has_scheduling_choice(); } } inline ::inference::ModelSequenceBatching* ModelConfig::release_sequence_batching() { // @@protoc_insertion_point(field_release:inference.ModelConfig.sequence_batching) if (_internal_has_sequence_batching()) { clear_has_scheduling_choice(); ::inference::ModelSequenceBatching* temp = _impl_.scheduling_choice_.sequence_batching_; if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } _impl_.scheduling_choice_.sequence_batching_ = nullptr; return temp; } else { return nullptr; } } inline const ::inference::ModelSequenceBatching& ModelConfig::_internal_sequence_batching() const { return _internal_has_sequence_batching() ? *_impl_.scheduling_choice_.sequence_batching_ : reinterpret_cast< ::inference::ModelSequenceBatching&>(::inference::_ModelSequenceBatching_default_instance_); } inline const ::inference::ModelSequenceBatching& ModelConfig::sequence_batching() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.sequence_batching) return _internal_sequence_batching(); } inline ::inference::ModelSequenceBatching* ModelConfig::unsafe_arena_release_sequence_batching() { // @@protoc_insertion_point(field_unsafe_arena_release:inference.ModelConfig.sequence_batching) if (_internal_has_sequence_batching()) { clear_has_scheduling_choice(); ::inference::ModelSequenceBatching* temp = _impl_.scheduling_choice_.sequence_batching_; _impl_.scheduling_choice_.sequence_batching_ = nullptr; return temp; } else { return nullptr; } } inline void ModelConfig::unsafe_arena_set_allocated_sequence_batching(::inference::ModelSequenceBatching* sequence_batching) { clear_scheduling_choice(); if (sequence_batching) { set_has_sequence_batching(); _impl_.scheduling_choice_.sequence_batching_ = sequence_batching; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.sequence_batching) } inline ::inference::ModelSequenceBatching* ModelConfig::_internal_mutable_sequence_batching() { if (!_internal_has_sequence_batching()) { clear_scheduling_choice(); set_has_sequence_batching(); _impl_.scheduling_choice_.sequence_batching_ = CreateMaybeMessage< ::inference::ModelSequenceBatching >(GetArenaForAllocation()); } return _impl_.scheduling_choice_.sequence_batching_; } inline ::inference::ModelSequenceBatching* ModelConfig::mutable_sequence_batching() { ::inference::ModelSequenceBatching* _msg = _internal_mutable_sequence_batching(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.sequence_batching) return _msg; } // .inference.ModelEnsembling ensemble_scheduling = 15; inline bool ModelConfig::_internal_has_ensemble_scheduling() const { return scheduling_choice_case() == kEnsembleScheduling; } inline bool ModelConfig::has_ensemble_scheduling() const { return _internal_has_ensemble_scheduling(); } inline void ModelConfig::set_has_ensemble_scheduling() { _impl_._oneof_case_[0] = kEnsembleScheduling; } inline void ModelConfig::clear_ensemble_scheduling() { if (_internal_has_ensemble_scheduling()) { if (GetArenaForAllocation() == nullptr) { delete _impl_.scheduling_choice_.ensemble_scheduling_; } clear_has_scheduling_choice(); } } inline ::inference::ModelEnsembling* ModelConfig::release_ensemble_scheduling() { // @@protoc_insertion_point(field_release:inference.ModelConfig.ensemble_scheduling) if (_internal_has_ensemble_scheduling()) { clear_has_scheduling_choice(); ::inference::ModelEnsembling* temp = _impl_.scheduling_choice_.ensemble_scheduling_; if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } _impl_.scheduling_choice_.ensemble_scheduling_ = nullptr; return temp; } else { return nullptr; } } inline const ::inference::ModelEnsembling& ModelConfig::_internal_ensemble_scheduling() const { return _internal_has_ensemble_scheduling() ? *_impl_.scheduling_choice_.ensemble_scheduling_ : reinterpret_cast< ::inference::ModelEnsembling&>(::inference::_ModelEnsembling_default_instance_); } inline const ::inference::ModelEnsembling& ModelConfig::ensemble_scheduling() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.ensemble_scheduling) return _internal_ensemble_scheduling(); } inline ::inference::ModelEnsembling* ModelConfig::unsafe_arena_release_ensemble_scheduling() { // @@protoc_insertion_point(field_unsafe_arena_release:inference.ModelConfig.ensemble_scheduling) if (_internal_has_ensemble_scheduling()) { clear_has_scheduling_choice(); ::inference::ModelEnsembling* temp = _impl_.scheduling_choice_.ensemble_scheduling_; _impl_.scheduling_choice_.ensemble_scheduling_ = nullptr; return temp; } else { return nullptr; } } inline void ModelConfig::unsafe_arena_set_allocated_ensemble_scheduling(::inference::ModelEnsembling* ensemble_scheduling) { clear_scheduling_choice(); if (ensemble_scheduling) { set_has_ensemble_scheduling(); _impl_.scheduling_choice_.ensemble_scheduling_ = ensemble_scheduling; } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.ensemble_scheduling) } inline ::inference::ModelEnsembling* ModelConfig::_internal_mutable_ensemble_scheduling() { if (!_internal_has_ensemble_scheduling()) { clear_scheduling_choice(); set_has_ensemble_scheduling(); _impl_.scheduling_choice_.ensemble_scheduling_ = CreateMaybeMessage< ::inference::ModelEnsembling >(GetArenaForAllocation()); } return _impl_.scheduling_choice_.ensemble_scheduling_; } inline ::inference::ModelEnsembling* ModelConfig::mutable_ensemble_scheduling() { ::inference::ModelEnsembling* _msg = _internal_mutable_ensemble_scheduling(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.ensemble_scheduling) return _msg; } // repeated .inference.ModelInstanceGroup instance_group = 7; inline int ModelConfig::_internal_instance_group_size() const { return _impl_.instance_group_.size(); } inline int ModelConfig::instance_group_size() const { return _internal_instance_group_size(); } inline void ModelConfig::clear_instance_group() { _impl_.instance_group_.Clear(); } inline ::inference::ModelInstanceGroup* ModelConfig::mutable_instance_group(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelConfig.instance_group) return _impl_.instance_group_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup >* ModelConfig::mutable_instance_group() { // @@protoc_insertion_point(field_mutable_list:inference.ModelConfig.instance_group) return &_impl_.instance_group_; } inline const ::inference::ModelInstanceGroup& ModelConfig::_internal_instance_group(int index) const { return _impl_.instance_group_.Get(index); } inline const ::inference::ModelInstanceGroup& ModelConfig::instance_group(int index) const { // @@protoc_insertion_point(field_get:inference.ModelConfig.instance_group) return _internal_instance_group(index); } inline ::inference::ModelInstanceGroup* ModelConfig::_internal_add_instance_group() { return _impl_.instance_group_.Add(); } inline ::inference::ModelInstanceGroup* ModelConfig::add_instance_group() { ::inference::ModelInstanceGroup* _add = _internal_add_instance_group(); // @@protoc_insertion_point(field_add:inference.ModelConfig.instance_group) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelInstanceGroup >& ModelConfig::instance_group() const { // @@protoc_insertion_point(field_list:inference.ModelConfig.instance_group) return _impl_.instance_group_; } // string default_model_filename = 8; inline void ModelConfig::clear_default_model_filename() { _impl_.default_model_filename_.ClearToEmpty(); } inline const std::string& ModelConfig::default_model_filename() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.default_model_filename) return _internal_default_model_filename(); } template inline PROTOBUF_ALWAYS_INLINE void ModelConfig::set_default_model_filename(ArgT0&& arg0, ArgT... args) { _impl_.default_model_filename_.Set(static_cast(arg0), args..., GetArenaForAllocation()); // @@protoc_insertion_point(field_set:inference.ModelConfig.default_model_filename) } inline std::string* ModelConfig::mutable_default_model_filename() { std::string* _s = _internal_mutable_default_model_filename(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.default_model_filename) return _s; } inline const std::string& ModelConfig::_internal_default_model_filename() const { return _impl_.default_model_filename_.Get(); } inline void ModelConfig::_internal_set_default_model_filename(const std::string& value) { _impl_.default_model_filename_.Set(value, GetArenaForAllocation()); } inline std::string* ModelConfig::_internal_mutable_default_model_filename() { return _impl_.default_model_filename_.Mutable(GetArenaForAllocation()); } inline std::string* ModelConfig::release_default_model_filename() { // @@protoc_insertion_point(field_release:inference.ModelConfig.default_model_filename) return _impl_.default_model_filename_.Release(); } inline void ModelConfig::set_allocated_default_model_filename(std::string* default_model_filename) { if (default_model_filename != nullptr) { } else { } _impl_.default_model_filename_.SetAllocated(default_model_filename, GetArenaForAllocation()); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING if (_impl_.default_model_filename_.IsDefault()) { _impl_.default_model_filename_.Set("", GetArenaForAllocation()); } #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.default_model_filename) } // map cc_model_filenames = 9; inline int ModelConfig::_internal_cc_model_filenames_size() const { return _impl_.cc_model_filenames_.size(); } inline int ModelConfig::cc_model_filenames_size() const { return _internal_cc_model_filenames_size(); } inline void ModelConfig::clear_cc_model_filenames() { _impl_.cc_model_filenames_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelConfig::_internal_cc_model_filenames() const { return _impl_.cc_model_filenames_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelConfig::cc_model_filenames() const { // @@protoc_insertion_point(field_map:inference.ModelConfig.cc_model_filenames) return _internal_cc_model_filenames(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelConfig::_internal_mutable_cc_model_filenames() { return _impl_.cc_model_filenames_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelConfig::mutable_cc_model_filenames() { // @@protoc_insertion_point(field_mutable_map:inference.ModelConfig.cc_model_filenames) return _internal_mutable_cc_model_filenames(); } // map metric_tags = 10; inline int ModelConfig::_internal_metric_tags_size() const { return _impl_.metric_tags_.size(); } inline int ModelConfig::metric_tags_size() const { return _internal_metric_tags_size(); } inline void ModelConfig::clear_metric_tags() { _impl_.metric_tags_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelConfig::_internal_metric_tags() const { return _impl_.metric_tags_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& ModelConfig::metric_tags() const { // @@protoc_insertion_point(field_map:inference.ModelConfig.metric_tags) return _internal_metric_tags(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelConfig::_internal_mutable_metric_tags() { return _impl_.metric_tags_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* ModelConfig::mutable_metric_tags() { // @@protoc_insertion_point(field_mutable_map:inference.ModelConfig.metric_tags) return _internal_mutable_metric_tags(); } // map parameters = 14; inline int ModelConfig::_internal_parameters_size() const { return _impl_.parameters_.size(); } inline int ModelConfig::parameters_size() const { return _internal_parameters_size(); } inline void ModelConfig::clear_parameters() { _impl_.parameters_.Clear(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >& ModelConfig::_internal_parameters() const { return _impl_.parameters_.GetMap(); } inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >& ModelConfig::parameters() const { // @@protoc_insertion_point(field_map:inference.ModelConfig.parameters) return _internal_parameters(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >* ModelConfig::_internal_mutable_parameters() { return _impl_.parameters_.MutableMap(); } inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, ::inference::ModelParameter >* ModelConfig::mutable_parameters() { // @@protoc_insertion_point(field_mutable_map:inference.ModelConfig.parameters) return _internal_mutable_parameters(); } // repeated .inference.ModelWarmup model_warmup = 16; inline int ModelConfig::_internal_model_warmup_size() const { return _impl_.model_warmup_.size(); } inline int ModelConfig::model_warmup_size() const { return _internal_model_warmup_size(); } inline void ModelConfig::clear_model_warmup() { _impl_.model_warmup_.Clear(); } inline ::inference::ModelWarmup* ModelConfig::mutable_model_warmup(int index) { // @@protoc_insertion_point(field_mutable:inference.ModelConfig.model_warmup) return _impl_.model_warmup_.Mutable(index); } inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelWarmup >* ModelConfig::mutable_model_warmup() { // @@protoc_insertion_point(field_mutable_list:inference.ModelConfig.model_warmup) return &_impl_.model_warmup_; } inline const ::inference::ModelWarmup& ModelConfig::_internal_model_warmup(int index) const { return _impl_.model_warmup_.Get(index); } inline const ::inference::ModelWarmup& ModelConfig::model_warmup(int index) const { // @@protoc_insertion_point(field_get:inference.ModelConfig.model_warmup) return _internal_model_warmup(index); } inline ::inference::ModelWarmup* ModelConfig::_internal_add_model_warmup() { return _impl_.model_warmup_.Add(); } inline ::inference::ModelWarmup* ModelConfig::add_model_warmup() { ::inference::ModelWarmup* _add = _internal_add_model_warmup(); // @@protoc_insertion_point(field_add:inference.ModelConfig.model_warmup) return _add; } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::inference::ModelWarmup >& ModelConfig::model_warmup() const { // @@protoc_insertion_point(field_list:inference.ModelConfig.model_warmup) return _impl_.model_warmup_; } // .inference.ModelOperations model_operations = 18; inline bool ModelConfig::_internal_has_model_operations() const { return this != internal_default_instance() && _impl_.model_operations_ != nullptr; } inline bool ModelConfig::has_model_operations() const { return _internal_has_model_operations(); } inline void ModelConfig::clear_model_operations() { if (GetArenaForAllocation() == nullptr && _impl_.model_operations_ != nullptr) { delete _impl_.model_operations_; } _impl_.model_operations_ = nullptr; } inline const ::inference::ModelOperations& ModelConfig::_internal_model_operations() const { const ::inference::ModelOperations* p = _impl_.model_operations_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelOperations_default_instance_); } inline const ::inference::ModelOperations& ModelConfig::model_operations() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.model_operations) return _internal_model_operations(); } inline void ModelConfig::unsafe_arena_set_allocated_model_operations( ::inference::ModelOperations* model_operations) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.model_operations_); } _impl_.model_operations_ = model_operations; if (model_operations) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.model_operations) } inline ::inference::ModelOperations* ModelConfig::release_model_operations() { ::inference::ModelOperations* temp = _impl_.model_operations_; _impl_.model_operations_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelOperations* ModelConfig::unsafe_arena_release_model_operations() { // @@protoc_insertion_point(field_release:inference.ModelConfig.model_operations) ::inference::ModelOperations* temp = _impl_.model_operations_; _impl_.model_operations_ = nullptr; return temp; } inline ::inference::ModelOperations* ModelConfig::_internal_mutable_model_operations() { if (_impl_.model_operations_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelOperations>(GetArenaForAllocation()); _impl_.model_operations_ = p; } return _impl_.model_operations_; } inline ::inference::ModelOperations* ModelConfig::mutable_model_operations() { ::inference::ModelOperations* _msg = _internal_mutable_model_operations(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.model_operations) return _msg; } inline void ModelConfig::set_allocated_model_operations(::inference::ModelOperations* model_operations) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.model_operations_; } if (model_operations) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(model_operations); if (message_arena != submessage_arena) { model_operations = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, model_operations, submessage_arena); } } else { } _impl_.model_operations_ = model_operations; // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.model_operations) } // .inference.ModelTransactionPolicy model_transaction_policy = 19; inline bool ModelConfig::_internal_has_model_transaction_policy() const { return this != internal_default_instance() && _impl_.model_transaction_policy_ != nullptr; } inline bool ModelConfig::has_model_transaction_policy() const { return _internal_has_model_transaction_policy(); } inline void ModelConfig::clear_model_transaction_policy() { if (GetArenaForAllocation() == nullptr && _impl_.model_transaction_policy_ != nullptr) { delete _impl_.model_transaction_policy_; } _impl_.model_transaction_policy_ = nullptr; } inline const ::inference::ModelTransactionPolicy& ModelConfig::_internal_model_transaction_policy() const { const ::inference::ModelTransactionPolicy* p = _impl_.model_transaction_policy_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelTransactionPolicy_default_instance_); } inline const ::inference::ModelTransactionPolicy& ModelConfig::model_transaction_policy() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.model_transaction_policy) return _internal_model_transaction_policy(); } inline void ModelConfig::unsafe_arena_set_allocated_model_transaction_policy( ::inference::ModelTransactionPolicy* model_transaction_policy) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.model_transaction_policy_); } _impl_.model_transaction_policy_ = model_transaction_policy; if (model_transaction_policy) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.model_transaction_policy) } inline ::inference::ModelTransactionPolicy* ModelConfig::release_model_transaction_policy() { ::inference::ModelTransactionPolicy* temp = _impl_.model_transaction_policy_; _impl_.model_transaction_policy_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelTransactionPolicy* ModelConfig::unsafe_arena_release_model_transaction_policy() { // @@protoc_insertion_point(field_release:inference.ModelConfig.model_transaction_policy) ::inference::ModelTransactionPolicy* temp = _impl_.model_transaction_policy_; _impl_.model_transaction_policy_ = nullptr; return temp; } inline ::inference::ModelTransactionPolicy* ModelConfig::_internal_mutable_model_transaction_policy() { if (_impl_.model_transaction_policy_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelTransactionPolicy>(GetArenaForAllocation()); _impl_.model_transaction_policy_ = p; } return _impl_.model_transaction_policy_; } inline ::inference::ModelTransactionPolicy* ModelConfig::mutable_model_transaction_policy() { ::inference::ModelTransactionPolicy* _msg = _internal_mutable_model_transaction_policy(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.model_transaction_policy) return _msg; } inline void ModelConfig::set_allocated_model_transaction_policy(::inference::ModelTransactionPolicy* model_transaction_policy) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.model_transaction_policy_; } if (model_transaction_policy) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(model_transaction_policy); if (message_arena != submessage_arena) { model_transaction_policy = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, model_transaction_policy, submessage_arena); } } else { } _impl_.model_transaction_policy_ = model_transaction_policy; // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.model_transaction_policy) } // .inference.ModelRepositoryAgents model_repository_agents = 23; inline bool ModelConfig::_internal_has_model_repository_agents() const { return this != internal_default_instance() && _impl_.model_repository_agents_ != nullptr; } inline bool ModelConfig::has_model_repository_agents() const { return _internal_has_model_repository_agents(); } inline void ModelConfig::clear_model_repository_agents() { if (GetArenaForAllocation() == nullptr && _impl_.model_repository_agents_ != nullptr) { delete _impl_.model_repository_agents_; } _impl_.model_repository_agents_ = nullptr; } inline const ::inference::ModelRepositoryAgents& ModelConfig::_internal_model_repository_agents() const { const ::inference::ModelRepositoryAgents* p = _impl_.model_repository_agents_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelRepositoryAgents_default_instance_); } inline const ::inference::ModelRepositoryAgents& ModelConfig::model_repository_agents() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.model_repository_agents) return _internal_model_repository_agents(); } inline void ModelConfig::unsafe_arena_set_allocated_model_repository_agents( ::inference::ModelRepositoryAgents* model_repository_agents) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.model_repository_agents_); } _impl_.model_repository_agents_ = model_repository_agents; if (model_repository_agents) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.model_repository_agents) } inline ::inference::ModelRepositoryAgents* ModelConfig::release_model_repository_agents() { ::inference::ModelRepositoryAgents* temp = _impl_.model_repository_agents_; _impl_.model_repository_agents_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelRepositoryAgents* ModelConfig::unsafe_arena_release_model_repository_agents() { // @@protoc_insertion_point(field_release:inference.ModelConfig.model_repository_agents) ::inference::ModelRepositoryAgents* temp = _impl_.model_repository_agents_; _impl_.model_repository_agents_ = nullptr; return temp; } inline ::inference::ModelRepositoryAgents* ModelConfig::_internal_mutable_model_repository_agents() { if (_impl_.model_repository_agents_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelRepositoryAgents>(GetArenaForAllocation()); _impl_.model_repository_agents_ = p; } return _impl_.model_repository_agents_; } inline ::inference::ModelRepositoryAgents* ModelConfig::mutable_model_repository_agents() { ::inference::ModelRepositoryAgents* _msg = _internal_mutable_model_repository_agents(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.model_repository_agents) return _msg; } inline void ModelConfig::set_allocated_model_repository_agents(::inference::ModelRepositoryAgents* model_repository_agents) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.model_repository_agents_; } if (model_repository_agents) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(model_repository_agents); if (message_arena != submessage_arena) { model_repository_agents = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, model_repository_agents, submessage_arena); } } else { } _impl_.model_repository_agents_ = model_repository_agents; // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.model_repository_agents) } // .inference.ModelResponseCache response_cache = 24; inline bool ModelConfig::_internal_has_response_cache() const { return this != internal_default_instance() && _impl_.response_cache_ != nullptr; } inline bool ModelConfig::has_response_cache() const { return _internal_has_response_cache(); } inline void ModelConfig::clear_response_cache() { if (GetArenaForAllocation() == nullptr && _impl_.response_cache_ != nullptr) { delete _impl_.response_cache_; } _impl_.response_cache_ = nullptr; } inline const ::inference::ModelResponseCache& ModelConfig::_internal_response_cache() const { const ::inference::ModelResponseCache* p = _impl_.response_cache_; return p != nullptr ? *p : reinterpret_cast( ::inference::_ModelResponseCache_default_instance_); } inline const ::inference::ModelResponseCache& ModelConfig::response_cache() const { // @@protoc_insertion_point(field_get:inference.ModelConfig.response_cache) return _internal_response_cache(); } inline void ModelConfig::unsafe_arena_set_allocated_response_cache( ::inference::ModelResponseCache* response_cache) { if (GetArenaForAllocation() == nullptr) { delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.response_cache_); } _impl_.response_cache_ = response_cache; if (response_cache) { } else { } // @@protoc_insertion_point(field_unsafe_arena_set_allocated:inference.ModelConfig.response_cache) } inline ::inference::ModelResponseCache* ModelConfig::release_response_cache() { ::inference::ModelResponseCache* temp = _impl_.response_cache_; _impl_.response_cache_ = nullptr; #ifdef PROTOBUF_FORCE_COPY_IN_RELEASE auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); if (GetArenaForAllocation() == nullptr) { delete old; } #else // PROTOBUF_FORCE_COPY_IN_RELEASE if (GetArenaForAllocation() != nullptr) { temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); } #endif // !PROTOBUF_FORCE_COPY_IN_RELEASE return temp; } inline ::inference::ModelResponseCache* ModelConfig::unsafe_arena_release_response_cache() { // @@protoc_insertion_point(field_release:inference.ModelConfig.response_cache) ::inference::ModelResponseCache* temp = _impl_.response_cache_; _impl_.response_cache_ = nullptr; return temp; } inline ::inference::ModelResponseCache* ModelConfig::_internal_mutable_response_cache() { if (_impl_.response_cache_ == nullptr) { auto* p = CreateMaybeMessage<::inference::ModelResponseCache>(GetArenaForAllocation()); _impl_.response_cache_ = p; } return _impl_.response_cache_; } inline ::inference::ModelResponseCache* ModelConfig::mutable_response_cache() { ::inference::ModelResponseCache* _msg = _internal_mutable_response_cache(); // @@protoc_insertion_point(field_mutable:inference.ModelConfig.response_cache) return _msg; } inline void ModelConfig::set_allocated_response_cache(::inference::ModelResponseCache* response_cache) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); if (message_arena == nullptr) { delete _impl_.response_cache_; } if (response_cache) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = ::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(response_cache); if (message_arena != submessage_arena) { response_cache = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( message_arena, response_cache, submessage_arena); } } else { } _impl_.response_cache_ = response_cache; // @@protoc_insertion_point(field_set_allocated:inference.ModelConfig.response_cache) } inline bool ModelConfig::has_scheduling_choice() const { return scheduling_choice_case() != SCHEDULING_CHOICE_NOT_SET; } inline void ModelConfig::clear_has_scheduling_choice() { _impl_._oneof_case_[0] = SCHEDULING_CHOICE_NOT_SET; } inline ModelConfig::SchedulingChoiceCase ModelConfig::scheduling_choice_case() const { return ModelConfig::SchedulingChoiceCase(_impl_._oneof_case_[0]); } #ifdef __GNUC__ #pragma GCC diagnostic pop #endif // __GNUC__ // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // ------------------------------------------------------------------- // @@protoc_insertion_point(namespace_scope) } // namespace inference PROTOBUF_NAMESPACE_OPEN template <> struct is_proto_enum< ::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind>() { return ::inference::ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_descriptor(); } template <> struct is_proto_enum< ::inference::ModelInstanceGroup_Kind> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::ModelInstanceGroup_Kind>() { return ::inference::ModelInstanceGroup_Kind_descriptor(); } template <> struct is_proto_enum< ::inference::ModelInput_Format> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::ModelInput_Format>() { return ::inference::ModelInput_Format_descriptor(); } template <> struct is_proto_enum< ::inference::BatchInput_Kind> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::BatchInput_Kind>() { return ::inference::BatchInput_Kind_descriptor(); } template <> struct is_proto_enum< ::inference::BatchOutput_Kind> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::BatchOutput_Kind>() { return ::inference::BatchOutput_Kind_descriptor(); } template <> struct is_proto_enum< ::inference::ModelOptimizationPolicy_ModelPriority> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::ModelOptimizationPolicy_ModelPriority>() { return ::inference::ModelOptimizationPolicy_ModelPriority_descriptor(); } template <> struct is_proto_enum< ::inference::ModelQueuePolicy_TimeoutAction> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::ModelQueuePolicy_TimeoutAction>() { return ::inference::ModelQueuePolicy_TimeoutAction_descriptor(); } template <> struct is_proto_enum< ::inference::ModelSequenceBatching_Control_Kind> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::ModelSequenceBatching_Control_Kind>() { return ::inference::ModelSequenceBatching_Control_Kind_descriptor(); } template <> struct is_proto_enum< ::inference::DataType> : ::std::true_type {}; template <> inline const EnumDescriptor* GetEnumDescriptor< ::inference::DataType>() { return ::inference::DataType_descriptor(); } PROTOBUF_NAMESPACE_CLOSE // @@protoc_insertion_point(global_scope) #include #endif // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_model_5fconfig_2eproto kytensor-2.49.0.7/core/src/model_config_cuda.cc000066400000000000000000000044441527052371200213120ustar00rootroot00000000000000// Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "model_config_cuda.h" #include namespace triton { namespace core { int GetCudaStreamPriority( inference::ModelOptimizationPolicy::ModelPriority priority) { // Default priority is 0 int cuda_stream_priority = 0; int min, max; cudaError_t cuerr = cudaDeviceGetStreamPriorityRange(&min, &max); if ((cuerr != cudaErrorNoDevice) && (cuerr != cudaSuccess)) { return 0; } switch (priority) { case inference::ModelOptimizationPolicy::PRIORITY_MAX: cuda_stream_priority = max; break; case inference::ModelOptimizationPolicy::PRIORITY_MIN: cuda_stream_priority = min; break; default: cuda_stream_priority = 0; break; } return cuda_stream_priority; } }} // namespace triton::core kytensor-2.49.0.7/core/src/model_config_cuda.h000066400000000000000000000037461527052371200211600ustar00rootroot00000000000000// Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "model_config.pb.h" namespace triton { namespace core { /// Get the CUDA stream priority for a given ModelPriority /// \param priority The inference::ModelOptimizationPolicy::ModelPriority /// priority. \param cuda_stream_priority Returns the CUDA stream priority. /// \return The error status. int GetCudaStreamPriority( inference::ModelOptimizationPolicy::ModelPriority priority); }} // namespace triton::core kytensor-2.49.0.7/core/src/model_config_utils.cc000066400000000000000000002504151527052371200215370ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "model_config_utils.h" #include #include #include #include #include #include "constants.h" #include "cuda_utils.h" #include "filesystem/api.h" #include "triton/common/logging.h" #define TRITONJSON_STATUSTYPE triton::core::Status #define TRITONJSON_STATUSRETURN(M) \ return triton::core::Status(triton::core::Status::Code::INTERNAL, (M)) #define TRITONJSON_STATUSSUCCESS triton::core::Status::Success #include "triton/common/triton_json.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace core { namespace { #ifdef TRITON_ENABLE_ENSEMBLE struct EnsembleTensor { EnsembleTensor(const std::string& name, bool isOutput) : name{name}, isOutput(isOutput) { } const std::string name; bool isOutput{false}; bool ready{false}; std::vector prev_nodes; std::vector next_nodes; }; /// Build a graph that represents the data flow in the ensemble specified in /// given model config. the node (ensemble tensor) in the graph can be looked /// up using its name as key. /// \param ensemble_config The model configuration that specifies /// ensemble_scheduling field. /// \param keyed_ensemble_graph Returned the ensemble graph. /// \return The error status. A non-OK status indicates the build fails because /// the ensemble configuration is not valid. Status BuildEnsembleGraph( const inference::ModelConfig& config, std::unordered_map& keyed_ensemble_graph) { keyed_ensemble_graph.clear(); size_t step_idx = 0; for (const auto& element : config.ensemble_scheduling().step()) { if (element.model_name().empty()) { return Status( Status::Code::INVALID_ARG, "must specify 'model_name' in step " + std::to_string(step_idx) + " of ensemble '" + config.name() + "'"); } if (element.input_map().size() == 0) { return Status( Status::Code::INVALID_ARG, "must specify 'input_map' in step " + std::to_string(step_idx) + " of ensemble '" + config.name() + "'"); } if (element.output_map().size() == 0) { return Status( Status::Code::INVALID_ARG, "must specify 'output_map' in step " + std::to_string(step_idx) + " of ensemble '" + config.name() + "'"); } // Link ensemble tensors std::vector tensor_as_output; for (const auto& output_map : element.output_map()) { auto it = keyed_ensemble_graph.find(output_map.second); if (it != keyed_ensemble_graph.end()) { if (it->second.isOutput) { return Status( Status::Code::INVALID_ARG, "ensemble tensor '" + it->first + "' can appear in an output map only once for ensemble '" + config.name() + "' step " + std::to_string(step_idx)); } else { it->second.isOutput = true; } } else { it = keyed_ensemble_graph .emplace(std::make_pair( output_map.second, EnsembleTensor(output_map.second, true))) .first; } tensor_as_output.push_back(&(it->second)); } std::set model_inputs; for (const auto& input_map : element.input_map()) { if (model_inputs.find(input_map.first) != model_inputs.end()) { return Status( Status::Code::INVALID_ARG, "input '" + input_map.first + "' in model '" + element.model_name() + "' is mapped to multiple ensemble tensors for ensemble '" + config.name() + "' step " + std::to_string(step_idx)); } else { model_inputs.emplace(input_map.first); } auto it = keyed_ensemble_graph.find(input_map.second); if (it == keyed_ensemble_graph.end()) { it = keyed_ensemble_graph .emplace(std::make_pair( input_map.second, EnsembleTensor(input_map.second, false))) .first; } for (auto output : tensor_as_output) { output->prev_nodes.push_back(&(it->second)); it->second.next_nodes.push_back(output); } } step_idx++; } return Status::Success; } Status ValidateEnsembleSchedulingConfig(const inference::ModelConfig& config) { if (config.platform() != kEnsemblePlatform) { return Status( Status::Code::INVALID_ARG, "ensemble scheduling cannot be set for model '" + config.name() + "' whose platform is not " + kEnsemblePlatform); } if (config.instance_group().size() != 0) { return Status( Status::Code::INVALID_ARG, "instance group should not be specified for ensemble '" + config.name() + "'"); } if (config.has_optimization()) { return Status( Status::Code::INVALID_ARG, "optimization should not be specified for ensemble '" + config.name() + "'"); } if (config.model_warmup_size() != 0) { return Status( Status::Code::INVALID_ARG, "model_warmup can not be specified for ensemble '" + config.name() + "'"); } // Make sure step is not empty and all fields are set if (config.ensemble_scheduling().step_size() == 0) { return Status( Status::Code::INVALID_ARG, "must specify 'step' for ensemble '" + config.name() + "'"); } std::unordered_map tensors; RETURN_IF_ERROR(BuildEnsembleGraph(config, tensors)); // check data flow std::deque ready_queue; for (const auto& input : config.input()) { auto it = tensors.find(input.name()); if (it == tensors.end()) { return Status( Status::Code::INVALID_ARG, "ensemble input '" + input.name() + "' for ensemble " + config.name() + "' is not used"); } it->second.ready = true; ready_queue.push_back(&(it->second)); } while (!ready_queue.empty()) { auto& ready_node = ready_queue.front(); for (auto& next_node : ready_node->next_nodes) { if (next_node->ready) { continue; } bool next_node_ready = true; for (auto& prev_node : next_node->prev_nodes) { if (!prev_node->ready) { next_node_ready = false; break; } } next_node->ready = next_node_ready; if (next_node_ready) { ready_queue.push_back(next_node); } } ready_queue.pop_front(); } std::set outputs; for (const auto& output : config.output()) { auto it = tensors.find(output.name()); if (it == tensors.end()) { return Status( Status::Code::INVALID_ARG, "ensemble output '" + output.name() + "' for ensemble " + config.name() + "' is not used"); } if (!it->second.ready) { std::string error_message = "output '" + output.name() + "' for ensemble '" + config.name() + "' is not written"; // recurrsively check 'prev_nodes' for the source of not-ready state std::vector* prev_nodes = &it->second.prev_nodes; auto last_not_ready_node = &it->second; // there can be circular dependency so remember seen names to break it std::set seen_names; while ((prev_nodes != nullptr) && (!prev_nodes->empty())) { const auto& nodes = *prev_nodes; // make sure while loop will terminate if no not-ready source is seen prev_nodes = nullptr; for (const auto& node : nodes) { if ((!node->ready) && (seen_names.find(node->name) == seen_names.end())) { seen_names.emplace(node->name); last_not_ready_node = node; prev_nodes = &node->prev_nodes; break; } } } // there is not-ready source if (last_not_ready_node->name != it->second.name) { error_message += ": at least one of its depending tensors, '" + last_not_ready_node->name + "', is not connected"; } return Status(Status::Code::INVALID_ARG, error_message); } else { outputs.insert(it->first); } } // Check redundant ensemble tensors for (const auto& tensor : tensors) { // skip ensemble outputs as they have been checked and can have no // next nodes if (outputs.find(tensor.first) != outputs.end()) { continue; } if (!tensor.second.ready || (tensor.second.next_nodes.size() == 0)) { return Status( Status::Code::INVALID_ARG, "ensemble tensor '" + tensor.first + "' is unused in ensemble '" + config.name() + "'"); } } return Status::Success; } #endif // TRITON_ENABLE_ENSEMBLE template Status ValidateIOShape( const ModelIO& io, int32_t max_batch_size, const std::string& message_prefix = "") { if (io.name().empty()) { return Status( Status::Code::INVALID_ARG, message_prefix + "must specify 'name'"); } std::string message_prefix_with_name = message_prefix + std::string("'" + io.name() + "' "); if (io.data_type() == inference::DataType::TYPE_INVALID) { return Status( Status::Code::INVALID_ARG, message_prefix_with_name + "must specify 'data_type'"); } if (io.dims_size() == 0) { return Status( Status::Code::INVALID_ARG, message_prefix_with_name + "must specify 'dims'"); } // If the configuration is non-batching, then no input or output // reshape can be empty as that would mean that input or output was // always empty (no data). if (io.has_reshape() && (io.reshape().shape_size() == 0) && (max_batch_size == 0)) { return Status( Status::Code::INVALID_ARG, message_prefix_with_name + "cannot have empty reshape for non-batching model as scalar " "tensors are not supported"); } for (auto dim : io.dims()) { // Dimension cannot be 0. if ((dim < 1) && (dim != triton::common::WILDCARD_DIM)) { return Status( Status::Code::INVALID_ARG, message_prefix_with_name + "dimension must be integer >= 1, or " + std::to_string(triton::common::WILDCARD_DIM) + " to indicate a variable-size dimension"); } } if (io.has_reshape()) { // Zeros are not allowed in reshape. for (auto dim : io.reshape().shape()) { if ((dim < 1) && (dim != triton::common::WILDCARD_DIM)) { return Status( Status::Code::INVALID_ARG, message_prefix_with_name + "reshape dimensions must be integer >= 1, or " + std::to_string(triton::common::WILDCARD_DIM) + " to indicate a variable-size dimension"); } } const int64_t dims_size = triton::common::GetElementCount(io.dims()); const int64_t reshape_size = triton::common::GetElementCount(io.reshape().shape()); // dims and reshape must both have same element count // or both have variable-size dimension. // Special case for empty reshape... expect dims to have element // count of 1. if ((dims_size != reshape_size) && ((reshape_size != 0) || (dims_size != 1))) { return Status( Status::Code::INVALID_ARG, message_prefix_with_name + "has different size for dims and reshape"); } // shape contains variable-size dimension, in this case we compare if // each pair of the trunks separated by variable-size dimension has // the same element count. For instance, from [2, 4, -1, 6] to [8, -1, 1, 6] // is valid reshape as 2 * 4 = 8 and 6 = 1 * 6. if (dims_size == -1) { std::vector dim_element_cnts; std::vector reshape_element_cnts; int64_t current_cnt = 1; for (const auto& dim : io.dims()) { if (dim != -1) { current_cnt *= dim; } else { dim_element_cnts.push_back(current_cnt); current_cnt = 1; } } dim_element_cnts.push_back(current_cnt); current_cnt = 1; for (const auto& dim : io.reshape().shape()) { if (dim != -1) { current_cnt *= dim; } else { reshape_element_cnts.push_back(current_cnt); current_cnt = 1; } } reshape_element_cnts.push_back(current_cnt); if (dim_element_cnts.size() != reshape_element_cnts.size()) { return Status( Status::Code::INVALID_ARG, message_prefix_with_name + "has different number of variable-size dimensions for dims " "and reshape"); } for (size_t idx = 0; idx < dim_element_cnts.size(); idx++) { if (dim_element_cnts[idx] != reshape_element_cnts[idx]) { return Status( Status::Code::INVALID_ARG, message_prefix_with_name + "has different size for dims and reshape"); } } } } return Status::Success; } /// Validate that Non-linear format inputs or outputs are specified correctly /// in a model configuration. template Status ValidateNonLinearFormatIO( const ModelIO& io, const std::string& platform, bool is_input) { if (!io.is_non_linear_format_io()) { // Nothing to validate as the tensor is not non-linear format. return Status::Success; } if (platform != kTensorRTPlanPlatform) { return Status( Status::Code::INVALID_ARG, "Non-linear IO format is only supported for the TensorRT platform"); } if (io.dims_size() != 3) { std::string io_type = is_input ? "input" : "output"; return Status( Status::Code::INVALID_ARG, "Non-linear IO format " + io_type + " requires 3 dims"); } return Status::Success; } } // namespace Status GetModelVersionFromPath(const std::string& path, int64_t* version) { auto version_dir = BaseName(path); // Determine the version from the last segment of 'path' try { *version = std::atoll(version_dir.c_str()); } catch (...) { return Status( Status::Code::INTERNAL, "unable to determine model version from " + path); } return Status::Success; } Status GetBooleanSequenceControlProperties( const inference::ModelSequenceBatching& batcher, const std::string& model_name, const inference::ModelSequenceBatching::Control::Kind control_kind, const bool required, std::string* tensor_name, inference::DataType* tensor_datatype, float* fp32_false_value, float* fp32_true_value, int32_t* int32_false_value, int32_t* int32_true_value, bool* bool_false_value, bool* bool_true_value) { // Make sure same tensor is not configured for multiple controls std::set seen_tensors; // Make sure the control kind is not mentioned multiple times. bool seen_control = false; for (const auto& control_input : batcher.control_input()) { if (control_input.name().empty()) { return Status( Status::Code::INVALID_ARG, "sequence batching control tensor must have a name for " + model_name); } if (seen_tensors.find(control_input.name()) != seen_tensors.end()) { return Status( Status::Code::INVALID_ARG, "sequence batching control tensor '" + control_input.name() + "' is specified for multiple control kinds for " + model_name); } seen_tensors.insert(control_input.name()); for (const auto& c : control_input.control()) { if (c.kind() == control_kind) { if (seen_control) { return Status( Status::Code::INVALID_ARG, "sequence batching specifies multiple " + inference::ModelSequenceBatching_Control_Kind_Name( control_kind) + " tensors for " + model_name); } *tensor_name = control_input.name(); seen_control = true; // Make sure only one of int, float, or bool type is specified. if (!((c.int32_false_true_size() != 0) || (c.fp32_false_true_size() != 0) || (c.bool_false_true_size() != 0))) { return Status( Status::Code::INVALID_ARG, "sequence batching must specify either 'int32_false_true', " "'fp32_false_true' or 'bool_false_true' for " + inference::ModelSequenceBatching_Control_Kind_Name( control_kind) + " for " + model_name); } else if ( ((c.int32_false_true_size() != 0) && (c.fp32_false_true_size() != 0)) || ((c.int32_false_true_size() != 0) && (c.bool_false_true_size() != 0)) || ((c.fp32_false_true_size() != 0) && (c.bool_false_true_size() != 0))) { return Status( Status::Code::INVALID_ARG, "sequence batching specifies more than one from " "'int32_false_true', 'fp32_false_true' and 'bool_false_true' " "for " + inference::ModelSequenceBatching_Control_Kind_Name( control_kind) + " for " + model_name); } if (c.int32_false_true_size() > 0) { if (c.int32_false_true_size() != 2) { return Status( Status::Code::INVALID_ARG, "sequence batching control 'int32_false_true' must have " "exactly 2 entries for " + inference::ModelSequenceBatching_Control_Kind_Name( control_kind) + " for " + model_name); } if (tensor_datatype != nullptr) { *tensor_datatype = inference::DataType::TYPE_INT32; } if (int32_false_value != nullptr) { *int32_false_value = c.int32_false_true(0); } if (int32_true_value != nullptr) { *int32_true_value = c.int32_false_true(1); } } else if (c.fp32_false_true_size() > 0) { if (c.fp32_false_true_size() != 2) { return Status( Status::Code::INVALID_ARG, "sequence batching control 'fp32_false_true' must have exactly " "2 entries for " + inference::ModelSequenceBatching_Control_Kind_Name( control_kind) + " for " + model_name); } if (tensor_datatype != nullptr) { *tensor_datatype = inference::DataType::TYPE_FP32; } if (fp32_false_value != nullptr) { *fp32_false_value = c.fp32_false_true(0); } if (fp32_true_value != nullptr) { *fp32_true_value = c.fp32_false_true(1); } } else { if (c.bool_false_true_size() != 2) { return Status( Status::Code::INVALID_ARG, "sequence batching control 'bool_false_true' must have exactly " "2 entries for " + inference::ModelSequenceBatching_Control_Kind_Name( control_kind) + " for " + model_name); } if (tensor_datatype != nullptr) { *tensor_datatype = inference::DataType::TYPE_BOOL; } if (bool_false_value != nullptr) { *bool_false_value = c.bool_false_true(0); } if (bool_true_value != nullptr) { *bool_true_value = c.bool_false_true(1); } } } } } if (!seen_control) { if (required) { return Status( Status::Code::INVALID_ARG, "sequence batching control tensor must specify a " + inference::ModelSequenceBatching_Control_Kind_Name(control_kind) + " value for " + model_name); } tensor_name->clear(); } return Status::Success; } Status GetTypedSequenceControlProperties( const inference::ModelSequenceBatching& batcher, const std::string& model_name, const inference::ModelSequenceBatching::Control::Kind control_kind, const bool required, std::string* tensor_name, inference::DataType* tensor_datatype) { // Make sure same tensor is not configured for multiple controls std::set seen_tensors; // Make sure the control kind is not mentioned multiple times. bool seen_control = false; for (const auto& control_input : batcher.control_input()) { if (control_input.name().empty()) { return Status( Status::Code::INVALID_ARG, "sequence batching control tensor must have a name for " + model_name); } if (seen_tensors.find(control_input.name()) != seen_tensors.end()) { return Status( Status::Code::INVALID_ARG, "sequence batching control tensor '" + control_input.name() + "' is specified for multiple control kinds for " + model_name); } seen_tensors.insert(control_input.name()); for (const auto& c : control_input.control()) { if (c.kind() == control_kind) { if (seen_control) { return Status( Status::Code::INVALID_ARG, "sequence batching specifies multiple " + inference::ModelSequenceBatching_Control_Kind_Name( control_kind) + " tensors for " + model_name); } *tensor_name = control_input.name(); if (tensor_datatype != nullptr) { *tensor_datatype = c.data_type(); } seen_control = true; if ((c.int32_false_true_size() > 0) || (c.fp32_false_true_size() > 0) || (c.bool_false_true_size() > 0)) { return Status( Status::Code::INVALID_ARG, "sequence batching must not specify either 'int32_false_true', " "'fp32_false_true' or 'bool_false_true' for " + inference::ModelSequenceBatching_Control_Kind_Name( control_kind) + " for " + model_name); } } } } if (!seen_control) { if (required) { return Status( Status::Code::INVALID_ARG, "sequence batching control tensor must specify a " + inference::ModelSequenceBatching_Control_Kind_Name(control_kind) + " value for " + model_name); } tensor_name->clear(); } return Status::Success; } Status GetNormalizedModelConfig( const std::string& model_name, const std::string& path, const double min_compute_capability, inference::ModelConfig* config) { // Server-side autofill only sets certain backend fields for the models that // belong to limited backends for backwards-compatibility. See TensorRT // backend, ONNX Runtime backend, OpenVINO backend, TensorFLow backend, and // PyTorch backend. // Extracting detailed information is delegated to the backend implementation // to auto-complete. RETURN_IF_ERROR( AutoCompleteBackendFields(model_name, std::string(path), config)); LOG_PROTOBUF_VERBOSE(1, "Server side auto-completed config: ", (*config)); RETURN_IF_ERROR(NormalizeModelConfig(min_compute_capability, config)); return Status::Success; } Status NormalizeModelConfig( const double min_compute_capability, inference::ModelConfig* config) { // If version_policy is not specified, default to Latest 1 version. if (!config->has_version_policy()) { inference::ModelVersionPolicy::Latest latest; latest.set_num_versions(1); config->mutable_version_policy()->mutable_latest()->CopyFrom(latest); } // If dynamic batching is specified... if (config->has_dynamic_batching()) { // If preferred batch size is not specified set it to // max-batch-size. if (config->dynamic_batching().preferred_batch_size().size() == 0) { auto mutable_preferred_batch_size = config->mutable_dynamic_batching()->mutable_preferred_batch_size(); if (config->max_batch_size() > 0) { mutable_preferred_batch_size->Add(config->max_batch_size()); } } } // If sequence batching is specified... if (config->has_sequence_batching()) { // Set default idle is not specified. if (config->sequence_batching().max_sequence_idle_microseconds() == 0) { config->mutable_sequence_batching()->set_max_sequence_idle_microseconds( SEQUENCE_IDLE_DEFAULT_MICROSECONDS); } if (config->sequence_batching().has_oldest()) { // If preferred batch size is not specified set it to // max-batch-size. if (config->sequence_batching().oldest().preferred_batch_size().size() == 0) { auto mutable_preferred_batch_size = config->mutable_sequence_batching() ->mutable_oldest() ->mutable_preferred_batch_size(); if (config->max_batch_size() > 0) { mutable_preferred_batch_size->Add(config->max_batch_size()); } } } } // If model ensembling is specified, don't attempt to normalize instance_group // as it is not allowed in ensemble scheduling if (!config->has_ensemble_scheduling()) { auto optimization = config->mutable_optimization(); if (!optimization->has_input_pinned_memory()) { optimization->mutable_input_pinned_memory()->set_enable(true); } if (!optimization->has_output_pinned_memory()) { optimization->mutable_output_pinned_memory()->set_enable(true); } } return Status::Success; } Status NormalizeInstanceGroup( const double min_compute_capability, const std::vector& preferred_groups, inference::ModelConfig* config) { // Instance group setting doesn't apply to ensemble if (config->has_ensemble_scheduling()) { return Status::Success; } // Creates a set of supported GPU device ids std::set supported_gpus; #ifdef TRITON_ENABLE_GPU // Get the total number of GPUs from the runtime library. Status status = GetSupportedGPUs(&supported_gpus, min_compute_capability); if (!status.IsOk()) { return status; } #endif // TRITON_ENABLE_GPU // Make sure there is at least one instance_group. if (config->instance_group().empty()) { inference::ModelInstanceGroup* group = config->add_instance_group(); group->set_name(config->name()); for (const auto& pg : preferred_groups) { // handle preferred GPU setting differently based on kind if (pg.kind() == inference::ModelInstanceGroup::KIND_GPU) { // Don't use preferred group with KIND_GPU if there is no GPU. if (supported_gpus.empty()) { continue; } // If preferred group sets GPUs, limit deployment onto those that // are also listed in supported gpus if (!pg.gpus().empty()) { for (const int32_t gid : pg.gpus()) { if (supported_gpus.find(gid) != supported_gpus.end()) { group->add_gpus(gid); } } } } else if (pg.kind() == inference::ModelInstanceGroup::KIND_AUTO) { // if AUTO, then set preferred GPU as is, to align with KIND_AUTO // deduction specified below for (const int32_t gid : pg.gpus()) { group->add_gpus(gid); } } group->set_kind(pg.kind()); group->set_count(pg.count()); // Found a valid preferred group. break; } } // Assign default name, kind and count to each instance group that // doesn't give those values explicitly. For KIND_GPU, set GPUs to // all available if not specified explicitly. size_t cnt = 0; for (auto& group : *config->mutable_instance_group()) { // Name if (group.name().empty()) { group.set_name(config->name() + "_" + std::to_string(cnt)); } cnt++; // For KIND_AUTO... if there are no GPUs or if any of the listed // 'gpu's are not present, then use KIND_CPU. if (group.kind() == inference::ModelInstanceGroup::KIND_AUTO) { if (supported_gpus.empty()) { group.set_kind(inference::ModelInstanceGroup::KIND_CPU); } else { for (const int32_t gid : group.gpus()) { if (supported_gpus.find(gid) == supported_gpus.end()) { group.set_kind(inference::ModelInstanceGroup::KIND_CPU); break; } } } if (group.kind() == inference::ModelInstanceGroup::KIND_AUTO) { group.set_kind(inference::ModelInstanceGroup::KIND_GPU); } } // KIND is resolved at this point for (const auto& pg : preferred_groups) { if (group.kind() != pg.kind()) { continue; } // Limit the GPU setting within what is specified in the preferred group, // if no available GPU then skip to next preferred group if ((group.kind() == inference::ModelInstanceGroup::KIND_GPU) && group.gpus().empty() && !pg.gpus().empty()) { for (const int32_t gid : pg.gpus()) { if (supported_gpus.find(gid) != supported_gpus.end()) { group.add_gpus(gid); } } if (group.gpus().empty()) { continue; } } if ((group.count() < 1) && (pg.count() > 0)) { group.set_count(pg.count()); } } // Set Triton default if the fields are not set from preferred group // Count if (group.count() < 1) { RETURN_IF_ERROR(SetDefaultInstanceCount(&group, config->backend())); } // GPUs if ((group.kind() == inference::ModelInstanceGroup::KIND_GPU) && (group.gpus().size() == 0)) { for (auto d : supported_gpus) { group.add_gpus(d); } } } return Status::Success; } Status LocalizePythonBackendExecutionEnvironmentPath( const std::string& model_path, inference::ModelConfig* config, std::shared_ptr* localized_model_dir) { if (config->backend() == kPythonBackend) { if (config->parameters().contains("EXECUTION_ENV_PATH")) { // Read EXECUTION_ENV_PATH std::string exec_env_path = config->parameters().at("EXECUTION_ENV_PATH").string_value(); // Replace model directory variable with model_path std::string model_dir_var = "$$TRITON_MODEL_DIRECTORY"; if (exec_env_path.substr(0, model_dir_var.size()) == model_dir_var) { exec_env_path.replace(0, model_dir_var.size(), model_path); } // Collapse any .. in the path std::string abs_exec_env_path; std::size_t prev_pos = exec_env_path.size(); std::size_t pos = exec_env_path.find_last_of('/', prev_pos - 1); int skip = 0; while (pos != std::string::npos && prev_pos > 0) { if (!skip) { abs_exec_env_path = exec_env_path.substr(pos, prev_pos - pos) + abs_exec_env_path; } skip = skip > 0 ? skip - 1 : skip; if (pos >= 3 && exec_env_path.substr(pos - 3, 3) == "/..") { skip += 2; } prev_pos = pos; pos = exec_env_path.find_last_of('/', prev_pos - 1); } abs_exec_env_path = exec_env_path.substr(0, prev_pos) + abs_exec_env_path; // Localize iff abs_exec_env_path is outside the model directory std::string model_path_slash = model_path.back() == '/' ? model_path : model_path + "/"; if (abs_exec_env_path.substr(0, model_path_slash.size()) != model_path_slash) { // Localize the file std::shared_ptr localized_exec_env_path; RETURN_IF_ERROR( LocalizePath(abs_exec_env_path, &localized_exec_env_path)); // Persist the localized temporary path (*localized_model_dir) ->other_localized_path.push_back(localized_exec_env_path); // Rewrite EXECUTION_ENV_PATH config->mutable_parameters() ->at("EXECUTION_ENV_PATH") .set_string_value(localized_exec_env_path->Path()); } } } return Status::Success; } Status SetPythonBasedBackendExecutionEnvironment( const std::string& backend_libdir, inference::ModelConfig* model_config) { if (!model_config->parameters().contains("EXECUTION_ENV_PATH")) { std::string env_name = "pb_exec_env_" + model_config->runtime() + ".tar.gz"; std::string env_path = JoinPath({backend_libdir, std::move(env_name)}); bool env_path_exist; RETURN_IF_ERROR(FileExists(env_path, &env_path_exist)); if (env_path_exist) { inference::ModelParameter model_param; model_param.set_string_value(env_path); (*model_config->mutable_parameters())["EXECUTION_ENV_PATH"] = std::move(model_param); } } return Status::Success; } Status SetDefaultInstanceCount( inference::ModelInstanceGroup* group, const std::string& backend) { group->set_count(1); // Backends opt into the default_cpu_instance_count since // some backends (pytorch, OpenVINO) don't perform well/have high overhead // when using multiple instances. const int default_cpu_instance_count = 2; bool use_default_cpu_instance_count = (backend == kTensorFlowBackend) || (backend == kOnnxRuntimeBackend); if (group->kind() == inference::ModelInstanceGroup::KIND_CPU && use_default_cpu_instance_count) { group->set_count(default_cpu_instance_count); } return Status::Success; } Status AutoCompleteBackendFields( const std::string& model_name, const std::string& model_path, inference::ModelConfig* config) { std::set version_dirs; RETURN_IF_ERROR(GetDirectorySubdirs(model_path, &version_dirs)); // There must be at least one version directory that we can inspect to // attempt to determine the platform. If not, we skip autofill with file name. // For now we allow multiple versions and only inspect the first version // directory to ensure it is valid. We can add more aggressive checks later. const bool has_version = (version_dirs.size() != 0); const auto version_path = has_version ? JoinPath({model_path, *(version_dirs.begin())}) : ""; std::set version_dir_content; if (has_version) { RETURN_IF_ERROR(GetDirectoryContents(version_path, &version_dir_content)); } // If the model name is not given in the configuration, set if based // on the model path. if (config->name().empty()) { config->set_name(model_name); } // Trying to fill the 'backend', 'default_model_filename' field. // TensorFlow // For TF backend, the platform is required if (config->platform().empty()) { // Check 'backend', 'default_model_filename', and the actual directory // to determine the platform if (config->backend().empty() || (config->backend() == kTensorFlowBackend)) { if (config->default_model_filename() == kTensorFlowSavedModelFilename) { config->set_platform(kTensorFlowSavedModelPlatform); } else if ( config->default_model_filename() == kTensorFlowGraphDefFilename) { config->set_platform(kTensorFlowGraphDefPlatform); } else if (config->default_model_filename().empty() && has_version) { bool is_dir = false; if (version_dir_content.find(kTensorFlowSavedModelFilename) != version_dir_content.end()) { RETURN_IF_ERROR(IsDirectory( JoinPath({version_path, kTensorFlowSavedModelFilename}), &is_dir)); if (is_dir) { config->set_platform(kTensorFlowSavedModelPlatform); } } if (version_dir_content.find(kTensorFlowGraphDefFilename) != version_dir_content.end()) { RETURN_IF_ERROR(IsDirectory( JoinPath({version_path, kTensorFlowGraphDefFilename}), &is_dir)); if (!is_dir) { config->set_platform(kTensorFlowGraphDefPlatform); } } } } } // Fill 'backend' and 'default_model_filename' if missing if ((config->platform() == kTensorFlowSavedModelPlatform) || (config->platform() == kTensorFlowGraphDefPlatform)) { if (config->backend().empty()) { config->set_backend(kTensorFlowBackend); } if (config->default_model_filename().empty()) { if (config->platform() == kTensorFlowSavedModelPlatform) { config->set_default_model_filename(kTensorFlowSavedModelFilename); } else { config->set_default_model_filename(kTensorFlowGraphDefFilename); } } return Status::Success; } // TensorRT if (config->backend().empty()) { if ((config->platform() == kTensorRTPlanPlatform) || (config->default_model_filename() == kTensorRTPlanFilename)) { config->set_backend(kTensorRTBackend); } else if ( config->platform().empty() && config->default_model_filename().empty() && has_version) { bool is_dir = false; if (version_dir_content.find(kTensorRTPlanFilename) != version_dir_content.end()) { RETURN_IF_ERROR(IsDirectory( JoinPath({version_path, kTensorRTPlanFilename}), &is_dir)); if (!is_dir) { config->set_backend(kTensorRTBackend); } } } } if (config->backend() == kTensorRTBackend) { if (config->platform().empty()) { config->set_platform(kTensorRTPlanPlatform); } if (config->default_model_filename().empty()) { config->set_default_model_filename(kTensorRTPlanFilename); } return Status::Success; } // ONNXRuntime if (config->backend().empty()) { if ((config->platform() == kOnnxRuntimeOnnxPlatform) || (config->default_model_filename() == kOnnxRuntimeOnnxFilename)) { config->set_backend(kOnnxRuntimeBackend); } else if ( config->platform().empty() && config->default_model_filename().empty() && has_version) { if (version_dir_content.find(kOnnxRuntimeOnnxFilename) != version_dir_content.end()) { // ONNX model can be a file or a directory in the case of large model config->set_backend(kOnnxRuntimeBackend); } } } if (config->backend() == kOnnxRuntimeBackend) { if (config->platform().empty()) { config->set_platform(kOnnxRuntimeOnnxPlatform); } if (config->default_model_filename().empty()) { config->set_default_model_filename(kOnnxRuntimeOnnxFilename); } return Status::Success; } // OpenVINO if (config->backend().empty()) { if (config->default_model_filename() == kOpenVINORuntimeOpenVINOFilename) { config->set_backend(kOpenVINORuntimeBackend); } else if ( config->platform().empty() && config->default_model_filename().empty() && has_version) { if (version_dir_content.find(kOpenVINORuntimeOpenVINOFilename) != version_dir_content.end()) { config->set_backend(kOpenVINORuntimeBackend); } } } if (config->backend() == kOpenVINORuntimeBackend) { if (config->default_model_filename().empty()) { config->set_default_model_filename(kOpenVINORuntimeOpenVINOFilename); } return Status::Success; } // PyTorch if (config->backend().empty()) { if ((config->platform() == kPyTorchLibTorchPlatform) || (config->default_model_filename() == kPyTorchLibTorchFilename)) { config->set_backend(kPyTorchBackend); } else if ( config->platform().empty() && config->default_model_filename().empty() && has_version) { bool is_dir = false; if (version_dir_content.find(kPyTorchLibTorchFilename) != version_dir_content.end()) { RETURN_IF_ERROR(IsDirectory( JoinPath({version_path, kPyTorchLibTorchFilename}), &is_dir)); if (!is_dir) { config->set_backend(kPyTorchBackend); } } } } if (config->backend() == kPyTorchBackend) { if (config->platform().empty()) { // do not introduce new platforms, new runtimes may ignore this field. config->set_platform(kPyTorchLibTorchPlatform); } if (config->runtime() != kPythonFilename && config->default_model_filename().empty()) { config->set_default_model_filename(kPyTorchLibTorchFilename); } return Status::Success; } // Python if (config->backend().empty()) { if (config->default_model_filename() == kPythonFilename) { config->set_backend(kPythonBackend); } else if ( config->platform().empty() && config->default_model_filename().empty() && has_version) { if (version_dir_content.find(kPythonFilename) != version_dir_content.end()) { config->set_backend(kPythonBackend); } } } if (config->backend() == kPythonBackend) { if (config->default_model_filename().empty()) { config->set_default_model_filename(kPythonFilename); } return Status::Success; } // Custom Backend // For now, only do the narrowest case, where no info is given in the config. if (config->backend().empty() && config->platform().empty() && config->default_model_filename().empty()) { LOG_VERBOSE(1) << "Could not infer supported backend, so attempting " "autofill of custom backend."; // Since we lazily load the backends, we let the model tell us what backend // to load. We must assume that if the model name conforms to the required // shape, we parse the backend name out of the model file name. i.e. // model.identity will set the backend to "identity". const std::string delimiter = "."; size_t pos = model_name.find(delimiter, 0); if (pos == std::string::npos) { return Status( triton::common::Error::Code::INVALID_ARG, ("Invalid model name: Could not determine backend for model '" + model_name + "' with no backend in model configuration. Expected model name of " "the form 'model.'.")); } const std::string backend_name = model_name.substr(pos + 1, std::string::npos); config->set_backend(backend_name); config->set_default_model_filename( (std::string("model.") + backend_name).c_str()); return Status::Success; } return Status::Success; } Status ValidateModelIOConfig(const inference::ModelConfig& config) { Status status; for (const auto& io : config.input()) { status = ValidateModelInput(io, config.max_batch_size(), config.platform()); if (!status.IsOk()) { return Status( status.StatusCode(), status.Message() + " for " + config.name()); } } for (const auto& io : config.output()) { status = ValidateModelOutput(io, config.max_batch_size(), config.platform()); if (!status.IsOk()) { return Status( status.StatusCode(), status.Message() + " for " + config.name()); } } status = ValidateBatchIO(config); if (!status.IsOk()) { return Status( status.StatusCode(), status.Message() + " for " + config.name()); } return Status::Success; } Status ValidateBatchIO(const inference::ModelConfig& config) { std::set input_names; std::set output_names; for (const auto& io : config.input()) { input_names.emplace(io.name()); } for (const auto& io : config.output()) { output_names.emplace(io.name()); } for (const auto& batch_io : config.batch_input()) { switch (batch_io.kind()) { case inference::BatchInput::BATCH_ELEMENT_COUNT: case inference::BatchInput::BATCH_ACCUMULATED_ELEMENT_COUNT: case inference::BatchInput::BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO: case inference::BatchInput::BATCH_MAX_ELEMENT_COUNT_AS_SHAPE: case inference::BatchInput::BATCH_ITEM_SHAPE: case inference::BatchInput::BATCH_ITEM_SHAPE_FLATTEN: { if (batch_io.source_input_size() != 1) { return Status( Status::Code::INVALID_ARG, "batch input kind '" + inference::BatchInput::Kind_Name(batch_io.kind()) + "' expects 1 source input, got " + std::to_string(batch_io.source_input_size())); } break; } default: return Status( Status::Code::INVALID_ARG, "unknown batch input kind '" + inference::BatchInput::Kind_Name(batch_io.kind()) + "'"); } if ((batch_io.data_type() != inference::DataType::TYPE_INT32) && (batch_io.data_type() != inference::DataType::TYPE_FP32)) { return Status( Status::Code::INVALID_ARG, "batch input data type must be TYPE_INT32 or TYPE_FP32"); } for (const auto& source_name : batch_io.source_input()) { if (input_names.find(source_name) == input_names.end()) { return Status( Status::Code::INVALID_ARG, "unknown source input name '" + source_name + "'"); } } } for (const auto& batch_io : config.batch_output()) { switch (batch_io.kind()) { case inference::BatchOutput::BATCH_SCATTER_WITH_INPUT_SHAPE: { if (batch_io.source_input_size() != 1) { return Status( Status::Code::INVALID_ARG, "batch output kind '" + inference::BatchOutput::Kind_Name(batch_io.kind()) + "' expects 1 source input, got " + std::to_string(batch_io.source_input_size())); } break; } default: return Status( Status::Code::INVALID_ARG, "unknown batch output kind '" + inference::BatchOutput::Kind_Name(batch_io.kind()) + "'"); } for (const auto& source_name : batch_io.source_input()) { if (input_names.find(source_name) == input_names.end()) { return Status( Status::Code::INVALID_ARG, "unknown source input name '" + source_name + "'"); } } std::set target_names; for (const auto& target_name : batch_io.target_name()) { if (output_names.find(target_name) == output_names.end()) { return Status( Status::Code::INVALID_ARG, "unknown target output name '" + target_name + "'"); } if (target_names.emplace(target_name).second == false) { return Status( Status::Code::INVALID_ARG, "target output name '" + target_name + "' can only be specified once"); } } } return Status::Success; } Status ValidateModelConfig( const inference::ModelConfig& config, const double min_compute_capability) { if (config.name().empty()) { return Status( Status::Code::INVALID_ARG, "model configuration must specify 'name'"); } if (config.backend().empty()) { // Expect backend is not empty unless it is ensemble platform. #ifdef TRITON_ENABLE_ENSEMBLE if (config.platform() != kEnsemblePlatform) #endif // TRITON_ENABLE_ENSEMBLE return Status( Status::Code::INVALID_ARG, "unexpected platform type '" + config.platform() + "' for " + config.name()); } #ifdef TRITON_ENABLE_ENSEMBLE else if (config.platform() == kEnsemblePlatform) { return Status( Status::Code::INVALID_ARG, "Ensemble model '" + config.name() + "' must have platform type '" + config.platform() + "' and empty backend type"); } #endif // TRITON_ENABLE_ENSEMBLE if (config.platform().empty() && config.backend().empty()) { return Status( Status::Code::INVALID_ARG, "must specify 'platform' or 'backend' for '" + config.name() + "'"); } // Ensure both platform and backend are referring to known backend, // and allow all platforms for a user-provided unknown backend. auto backend_type = GetBackendType(config.backend()); if ((backend_type != BackendType::BACKEND_TYPE_UNKNOWN) && (backend_type != GetBackendTypeFromPlatform(config.platform()))) { return Status( Status::Code::INVALID_ARG, "unexpected 'platform' and 'backend' pair, got:" + config.platform() + ", " + config.backend()); } if (config.max_batch_size() < 0) { return Status( Status::Code::INVALID_ARG, "'max_batch_size' must be non-negative value for " + config.name()); } if (!config.has_version_policy()) { return Status( Status::Code::INVALID_ARG, "must specify 'version policy' for " + config.name()); } // If dynamic batching is specified make sure the preferred batch // sizes are positive and don't exceed maximum batch size. if (config.has_dynamic_batching()) { for (const auto size : config.dynamic_batching().preferred_batch_size()) { if (size <= 0) { return Status( Status::Code::INVALID_ARG, "dynamic batching preferred size must be positive for " + config.name()); } if (size > config.max_batch_size()) { return Status( Status::Code::INVALID_ARG, "dynamic batching preferred size must be <= max batch size for " + config.name()); } } // Priority queue is specified const auto priority_levels = config.dynamic_batching().priority_levels(); if (priority_levels != 0) { if ((config.dynamic_batching().default_priority_level() == 0) || (config.dynamic_batching().default_priority_level() > priority_levels)) { return Status( Status::Code::INVALID_ARG, "default priority level must be in range [1, " + std::to_string(priority_levels) + "] for " + config.name()); } for (const auto& queue_policy : config.dynamic_batching().priority_queue_policy()) { if ((queue_policy.first == 0) || (queue_policy.first > priority_levels)) { return Status( Status::Code::INVALID_ARG, "priority queue policy must have priority level in range [1, " + std::to_string(priority_levels) + "] for " + config.name()); } } } // preserve ordering option will conflict with priorities and delay policy if (config.dynamic_batching().preserve_ordering()) { if (priority_levels > 1) { return Status( Status::Code::INVALID_ARG, "Only one priority level is allowed when 'preserve_ordering' is " "true for " + config.name()); } const auto& default_policy = config.dynamic_batching().default_queue_policy(); if ((default_policy.default_timeout_microseconds() != 0) && (default_policy.timeout_action() == inference::ModelQueuePolicy::DELAY)) { return Status( Status::Code::INVALID_ARG, "Queue policy can not have DELAY as timeout action when " "'preserve_ordering' is true for " + config.name()); } // Also need to check policy in 'priority_queue_policy' // for single priority case for (const auto& policy : config.dynamic_batching().priority_queue_policy()) { if ((policy.second.default_timeout_microseconds() != 0) && (policy.second.timeout_action() == inference::ModelQueuePolicy::DELAY)) { return Status( Status::Code::INVALID_ARG, "Queue policy can not have DELAY as timeout action when " "'preserve_ordering' is true for " + config.name()); } } } } // If sequence batching is specified make sure the control is // specified correctly. if (config.has_sequence_batching()) { // FIXME: DLIS-4034 - Response Cache does not yet support sequence batcher. if (config.response_cache().enable()) { return Status( Status::Code::INVALID_ARG, "Response Cache does not currently support model " + config.name() + " with sequence batching scheduler. Please disable the response " "cache."); } const auto& batcher = config.sequence_batching(); // Check boolean controls... std::string tensor_name; RETURN_IF_ERROR(GetBooleanSequenceControlProperties( batcher, config.name(), inference::ModelSequenceBatching::Control::CONTROL_SEQUENCE_START, false /* required */, &tensor_name, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)); RETURN_IF_ERROR(GetBooleanSequenceControlProperties( batcher, config.name(), inference::ModelSequenceBatching::Control::CONTROL_SEQUENCE_END, false /* required */, &tensor_name, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)); RETURN_IF_ERROR(GetBooleanSequenceControlProperties( batcher, config.name(), inference::ModelSequenceBatching::Control::CONTROL_SEQUENCE_READY, false /* required */, &tensor_name, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)); // Check CORRID control and make sure it is one of the allowed types. inference::DataType tensor_datatype; RETURN_IF_ERROR(GetTypedSequenceControlProperties( batcher, config.name(), inference::ModelSequenceBatching::Control::CONTROL_SEQUENCE_CORRID, false /* required */, &tensor_name, &tensor_datatype)); if (!tensor_name.empty()) { if ((tensor_datatype != inference::DataType::TYPE_UINT64) && (tensor_datatype != inference::DataType::TYPE_INT64) && (tensor_datatype != inference::DataType::TYPE_UINT32) && (tensor_datatype != inference::DataType::TYPE_INT32) && (tensor_datatype != inference::DataType::TYPE_STRING)) { return Status( Status::Code::INVALID_ARG, "unexpected data type for control " + inference::ModelSequenceBatching_Control_Kind_Name( inference::ModelSequenceBatching::Control:: CONTROL_SEQUENCE_CORRID) + " for " + config.name() + ". Allowed data types are TYPE_UINT64, TYPE_INT64, " "TYPE_UINT32, " "TYPE_INT32 and TYPE_STRING"); } } // If oldest-first strategy is enabled make sure the preferred // batch sizes are positive and don't exceed maximum batch size. if (config.sequence_batching().has_oldest()) { for (const auto size : config.sequence_batching().oldest().preferred_batch_size()) { if (size <= 0) { return Status( Status::Code::INVALID_ARG, "sequence batching preferred batch size must be positive for " + config.name()); } if (size > config.max_batch_size()) { return Status( Status::Code::INVALID_ARG, "sequence batching preferred batch size must be <= max batch " "size for " + config.name()); } } } // If direct strategy is enabled make sure the minimum slot utilization is // in range (0.0, 1.0] if (config.sequence_batching().has_direct()) { if ((config.sequence_batching().direct().minimum_slot_utilization() < 0.0) || (config.sequence_batching().direct().minimum_slot_utilization() > 1.0)) { return Status( Status::Code::INVALID_ARG, "sequence batching minimum slot utilization must be in range " "(0.0, 1.0] for " + config.name()); } } } // If ensemble scheduling is specified, validate it. Otherwise, // must validate platform and instance_group if (config.has_ensemble_scheduling()) { #ifdef TRITON_ENABLE_ENSEMBLE RETURN_IF_ERROR(ValidateEnsembleSchedulingConfig(config)); #else return Status( Status::Code::INVALID_ARG, "ensemble scheduling not supported"); #endif // TRITON_ENABLE_ENSEMBLE } #ifdef TRITON_ENABLE_ENSEMBLE else if (config.platform() == kEnsemblePlatform) { return Status( Status::Code::INVALID_ARG, "ensemble scheduling must be set for ensemble " + config.name() + " whose platform is " + kEnsemblePlatform); } #endif // TRITON_ENABLE_ENSEMBLE // FIXME: DLIS-3916 - Response Cache does not yet support decoupled models if (config.model_transaction_policy().decoupled() && config.response_cache().enable()) { return Status( Status::Code::INVALID_ARG, "Response Cache does not currently support model " + config.name() + " with 'decoupled' transaction policy. Please disable the response" " cache."); } return Status::Success; } Status ValidateInstanceGroup( const inference::ModelConfig& config, const double min_compute_capability) { // Instance group setting doesn't apply to ensemble if (config.has_ensemble_scheduling()) { return Status::Success; } if (config.instance_group().size() == 0) { return Status( Status::Code::INVALID_ARG, "must specify one or more 'instance group's for " + config.name()); } // Make sure KIND_GPU instance group specifies at least one GPU and // doesn't specify a non-existent GPU. Make sure non-KIND_GPU does // not specify any GPUs. #ifdef TRITON_ENABLE_GPU std::set supported_gpus; Status status = GetSupportedGPUs(&supported_gpus, min_compute_capability); if (!status.IsOk()) { return status; } #endif // TRITON_ENABLE_GPU for (const auto& group : config.instance_group()) { if (group.kind() == inference::ModelInstanceGroup::KIND_MODEL) { if (group.gpus().size() > 0) { return Status( Status::Code::INVALID_ARG, "instance group " + group.name() + " of model " + config.name() + " has kind KIND_MODEL but specifies one or more GPUs"); } } else if (group.kind() == inference::ModelInstanceGroup::KIND_GPU) { #if !defined(TRITON_ENABLE_GPU) && !defined(TRITON_ENABLE_MALI_GPU) return Status( Status::Code::INVALID_ARG, "instance group " + group.name() + " of model " + config.name() + " has kind KIND_GPU but server does not support GPUs"); #elif defined(TRITON_ENABLE_GPU) if (group.gpus().size() == 0) { if (supported_gpus.size() == 0) { return Status( Status::Code::INVALID_ARG, "instance group " + group.name() + " of model " + config.name() + " has kind KIND_GPU but no GPUs are available"); } else { return Status( Status::Code::INVALID_ARG, "instance group " + group.name() + " of model " + config.name() + " has kind KIND_GPU but specifies no GPUs"); } } for (const int32_t gid : group.gpus()) { if (supported_gpus.find(gid) == supported_gpus.end()) { std::string supported_gpus_str; for (const auto& cc : supported_gpus) { if (!supported_gpus_str.empty()) { supported_gpus_str += ", "; } supported_gpus_str += std::to_string(cc); } return Status( Status::Code::INVALID_ARG, "instance group " + group.name() + " of model " + config.name() + " specifies invalid or unsupported gpu id " + std::to_string(gid) + ". GPUs with at least the minimum required CUDA compute " "compatibility of " + std::to_string(min_compute_capability) + " are: " + supported_gpus_str); } } #endif // ! TRITON_ENABLE_GPU && ! TRITON_ENABLE_MALI_GPU } else if (group.kind() == inference::ModelInstanceGroup::KIND_CPU) { if (group.gpus().size() > 0) { return Status( Status::Code::INVALID_ARG, "instance group " + group.name() + " of model " + config.name() + " has kind KIND_CPU but specifies one or more GPUs"); } } else { return Status( Status::Code::INTERNAL, "instance group " + group.name() + " of model " + config.name() + " has unexpected kind KIND_AUTO"); } if ((config.platform() != kTensorRTPlanPlatform) && !group.profile().empty()) { return Status( Status::Code::INVALID_ARG, "instance group " + group.name() + " of model " + config.name() + " and platform " + config.platform() + "specifies profile field which is only supported for " "TensorRT models"); } else if (!group.profile().empty()) { for (const auto& profile : group.profile()) { int profile_index; RETURN_IF_ERROR(GetProfileIndex(profile, &profile_index)); if (profile_index < 0) { return Status( Status::Code::INVALID_ARG, "instance group " + group.name() + " of model " + config.name() + " and platform " + config.platform() + " specifies invalid profile " + profile + ". The field should contain the string representation of a " "non-negative integer."); } } } } return Status::Success; } Status ValidateModelInput( const inference::ModelInput& io, int32_t max_batch_size, const std::string& platform) { RETURN_IF_ERROR(ValidateIOShape(io, max_batch_size, "model input ")); if (((io.format() == inference::ModelInput::FORMAT_NHWC) || (io.format() == inference::ModelInput::FORMAT_NCHW)) && (io.dims_size() != 3)) { return Status( Status::Code::INVALID_ARG, "model input NHWC/NCHW require 3 dims"); } if ((platform != kTensorRTPlanPlatform) && io.is_shape_tensor()) { return Status( Status::Code::INVALID_ARG, "shape tensors are only supported for TensorRT platform"); } RETURN_IF_ERROR(ValidateNonLinearFormatIO(io, platform, true /* is_input*/)); return Status::Success; } Status CheckAllowedModelInput( const inference::ModelInput& io, const std::set& allowed) { if (allowed.find(io.name()) == allowed.end()) { std::string astr; for (const auto& a : allowed) { if (!astr.empty()) { astr.append(", "); } astr.append(a); } return Status( Status::Code::INVALID_ARG, "unexpected inference input '" + io.name() + "', allowed inputs are: " + astr); } return Status::Success; } Status ValidateModelOutput( const inference::ModelOutput& io, int32_t max_batch_size, const std::string& platform) { RETURN_IF_ERROR(ValidateIOShape(io, max_batch_size, "model output ")); if ((platform != kTensorRTPlanPlatform) && io.is_shape_tensor()) { return Status( Status::Code::INVALID_ARG, "shape tensors are only supported for TensorRT platform"); } RETURN_IF_ERROR(ValidateNonLinearFormatIO(io, platform, false /* is_input*/)); return Status::Success; } Status CheckAllowedModelOutput( const inference::ModelOutput& io, const std::set& allowed) { if (allowed.find(io.name()) == allowed.end()) { std::string astr; for (const auto& a : allowed) { if (!astr.empty()) { astr.append(", "); } astr.append(a); } return Status( Status::Code::INVALID_ARG, "unexpected inference output '" + io.name() + "', allowed outputs are: " + astr); } return Status::Success; } Status ParseBoolParameter( const std::string& key, std::string value, bool* parsed_value) { std::transform( value.begin(), value.end(), value.begin(), [](unsigned char c) { return std::tolower(c); }); if ((value == "true") || (value == "1")) { *parsed_value = true; } else if ((value == "false") || (value == "0")) { *parsed_value = false; } else { return Status( Status::Code::INVALID_ARG, "failed to convert " + key + " '" + value + "' to boolean value"); } return Status::Success; } Status ParseLongLongParameter( const std::string& key, const std::string& value, int64_t* parsed_value) { try { *parsed_value = std::stoll(value); } catch (const std::invalid_argument& ia) { return Status( Status::Code::INVALID_ARG, "failed to convert " + key + " '" + value + "' to integral number"); } return Status::Success; } Status GetProfileIndex(const std::string& profile_name, int* profile_index) { if (profile_name.empty()) { return Status(Status::Code::INVALID_ARG, "profile name must not be empty"); } try { *profile_index = stoi(profile_name); } catch (const std::invalid_argument& ia) { return Status( Status::Code::INVALID_ARG, "unable to parse '" + profile_name + "': " + ia.what()); } return Status::Success; } namespace { Status CollectInt64Fields( google::protobuf::Message* message, const std::string& prefix, std::set* int64_fields) { const google::protobuf::Descriptor* desc = message->GetDescriptor(); const google::protobuf::Reflection* refl = message->GetReflection(); for (int i = 0; i < desc->field_count(); ++i) { const google::protobuf::FieldDescriptor* field = desc->field(i); const std::string fullname = prefix + "::" + field->name(); switch (field->type()) { case google::protobuf::FieldDescriptor::TYPE_MESSAGE: { if (field->is_repeated()) { int rsize = refl->FieldSize(*message, field); if (rsize == 0) { refl->AddMessage(message, field); } rsize = refl->FieldSize(*message, field); for (int r = 0; r < rsize; ++r) { RETURN_IF_ERROR(CollectInt64Fields( refl->MutableRepeatedMessage(message, field, r), fullname, int64_fields)); } } else { RETURN_IF_ERROR(CollectInt64Fields( refl->MutableMessage(message, field), fullname, int64_fields)); } } break; case google::protobuf::FieldDescriptor::TYPE_INT64: case google::protobuf::FieldDescriptor::TYPE_UINT64: case google::protobuf::FieldDescriptor::TYPE_SINT64: case google::protobuf::FieldDescriptor::TYPE_FIXED64: case google::protobuf::FieldDescriptor::TYPE_SFIXED64: int64_fields->insert(fullname); break; default: break; } } return Status::Success; } Status ValidateModelConfigInt64() { // Must initialize a dummy ModelConfig so that all fields are // visited. inference::ModelConfig config; std::set int64_fields; RETURN_IF_ERROR(CollectInt64Fields(&config, "ModelConfig", &int64_fields)); LOG_VERBOSE(1) << "ModelConfig 64-bit fields:"; for (const auto& f : int64_fields) { LOG_VERBOSE(1) << "\t" << f; } // We expect to find exactly the following fields. If we get an // error from this code ModelConfig has added or removed a 64-bit // field and we need to adjust here and in ModelConfigToJson below. std::set expected{ "ModelConfig::input::dims", "ModelConfig::input::reshape::shape", "ModelConfig::output::dims", "ModelConfig::output::reshape::shape", "ModelConfig::version_policy::specific::versions", "ModelConfig::dynamic_batching::max_queue_delay_microseconds", "ModelConfig::dynamic_batching::default_queue_policy::default_timeout_" "microseconds", "ModelConfig::dynamic_batching::priority_queue_policy::value::default_" "timeout_microseconds", "ModelConfig::dynamic_batching::priority_levels", "ModelConfig::dynamic_batching::priority_queue_policy::key", "ModelConfig::dynamic_batching::default_priority_level", "ModelConfig::sequence_batching::direct::max_queue_delay_microseconds", "ModelConfig::sequence_batching::state::dims", "ModelConfig::sequence_batching::state::initial_state::dims", "ModelConfig::sequence_batching::oldest::max_queue_delay_microseconds", "ModelConfig::sequence_batching::max_sequence_idle_microseconds", "ModelConfig::ensemble_scheduling::step::model_version", "ModelConfig::model_warmup::inputs::value::dims", "ModelConfig::optimization::cuda::graph_spec::input::value::dim", "ModelConfig::optimization::cuda::graph_spec::graph_lower_bound::input::" "value::dim", "ModelConfig::instance_group::secondary_devices::device_id"}; if (int64_fields != expected) { return Status( Status::Code::INTERNAL, "ModelConfig 64-bit field needs update"); } return Status::Success; } Status FixUInt( triton::common::TritonJson::Value& document, triton::common::TritonJson::Value& io, const std::string& name) { triton::common::TritonJson::Value str_value; if (!io.Find(name.c_str(), &str_value)) { return Status::Success; } std::string str; RETURN_IF_ERROR(str_value.AsString(&str)); uint64_t d; try { d = std::strtoull(str.c_str(), nullptr, 10); } catch (...) { return Status( Status::Code::INTERNAL, (std::string("unable to convert '") + str + "' to unsigned integer")); } str_value.SetUInt(d); return Status::Success; } Status FixInt( triton::common::TritonJson::Value& document, triton::common::TritonJson::Value& io, const std::string& name) { triton::common::TritonJson::Value str_value; if (!io.Find(name.c_str(), &str_value)) { return Status::Success; } std::string str; RETURN_IF_ERROR(str_value.AsString(&str)); int64_t d; try { d = std::atoll(str.c_str()); } catch (...) { return Status( Status::Code::INTERNAL, (std::string("unable to convert '") + str + "' to integer")); } str_value.SetInt(d); return Status::Success; } Status FixIntArray( triton::common::TritonJson::Value& document, triton::common::TritonJson::Value& io, const std::string& name) { triton::common::TritonJson::Value fixed_shape_array( document, triton::common::TritonJson::ValueType::ARRAY); if (!io.Find(name.c_str())) { return Status::Success; } triton::common::TritonJson::Value shape_array; RETURN_IF_ERROR(io.MemberAsArray(name.c_str(), &shape_array)); for (size_t i = 0; i < shape_array.ArraySize(); ++i) { std::string str; RETURN_IF_ERROR(shape_array.IndexAsString(i, &str)); int64_t d; try { d = std::atoll(str.c_str()); } catch (...) { return Status( Status::Code::INTERNAL, (std::string("unable to convert '") + str + "' to integer")); } RETURN_IF_ERROR(fixed_shape_array.AppendInt(d)); } shape_array.Swap(fixed_shape_array); fixed_shape_array.Release(); return Status::Success; } Status FixObjectArray( triton::common::TritonJson::Value& document, triton::common::TritonJson::Value& arr, const std::string& name) { for (size_t i = 0; i < arr.ArraySize(); ++i) { triton::common::TritonJson::Value obj; RETURN_IF_ERROR(arr.IndexAsObject(i, &obj)); RETURN_IF_ERROR(FixInt(document, obj, name)); } return Status::Success; } } // namespace Status ModelConfigToJson( const inference::ModelConfig& config, const uint32_t config_version, std::string* json_str) { // Currently only support 'config_version' 1, which is the json // representation of the ModelConfig protobuf with the int64 fields // fixes to be actual numbers instead of the string madness done by // protobuf. if (config_version != 1) { return Status( Status::Code::INVALID_ARG, std::string("model configuration version ") + std::to_string(config_version) + " not supported, supported versions are: 1"); } // Config will have 0 byte size if all fields are with default value, // in other word the config is empty. if (config.ByteSizeLong() == 0) { json_str->clear(); return Status::Success; } std::string config_json_str; ::google::protobuf::util::JsonPrintOptions options; options.preserve_proto_field_names = true; options.always_print_primitive_fields = true; ::google::protobuf::util::MessageToJsonString( config, &config_json_str, options); // We need to verify that every field 64-bit field in the // ModelConfig protobuf is being handled. We hardcode the known // fields and check just once to make sure everything has been // handled. We could have this check in a separately compiled CI // test but it is convenient to keep it here close to the code below // that actually fixes the 64-bit fields. { static std::once_flag fonce; Status status = Status::Success; std::call_once(fonce, [&status] { status = ValidateModelConfigInt64(); }); RETURN_IF_ERROR(status); } // In the json produced by protobuf, int64 and uint64 values are // represented as strings. Protobuf doesn't provide an option to // disable this (sigh) so we need to fix it up here as we want the // json representation of the config to be reasonable json... triton::common::TritonJson::Value config_json; config_json.Parse(config_json_str); // Fix input::dims, input::reshape::shape, output::dims, // output::reshape::shape for (std::string name : {"input", "output"}) { triton::common::TritonJson::Value ios; RETURN_IF_ERROR(config_json.MemberAsArray(name.c_str(), &ios)); for (size_t i = 0; i < ios.ArraySize(); ++i) { triton::common::TritonJson::Value io; RETURN_IF_ERROR(ios.IndexAsObject(i, &io)); RETURN_IF_ERROR(FixIntArray(config_json, io, "dims")); triton::common::TritonJson::Value reshape; if (io.Find("reshape", &reshape)) { RETURN_IF_ERROR(FixIntArray(config_json, reshape, "shape")); } } } // Fix version_policy::specific::versions { triton::common::TritonJson::Value vp; if (config_json.Find("version_policy", &vp)) { triton::common::TritonJson::Value specific; if (vp.Find("specific", &specific)) { RETURN_IF_ERROR(FixIntArray(config_json, specific, "versions")); } } } // Fix dynamic_batching::max_queue_delay_microseconds, // dynamic_batching::default_queue_policy::default_timeout_microseconds, // dynamic_batching::priority_queue_policy::value::default_timeout_microseconds // dynamic_batching::priority_levels // dynamic_batching::default_priority_level // dynamic_batching::priority_queue_policy::key is left as JSON only allows // strings for keys { triton::common::TritonJson::Value db; if (config_json.Find("dynamic_batching", &db)) { RETURN_IF_ERROR(FixUInt(config_json, db, "max_queue_delay_microseconds")); RETURN_IF_ERROR(FixUInt(config_json, db, "priority_levels")); RETURN_IF_ERROR(FixUInt(config_json, db, "default_priority_level")); triton::common::TritonJson::Value dqp; if (db.Find("default_queue_policy", &dqp)) { RETURN_IF_ERROR( FixUInt(config_json, dqp, "default_timeout_microseconds")); } triton::common::TritonJson::Value pqp; if (db.Find("priority_queue_policy", &pqp)) { // Iterate over each member in 'pqp' and fix... std::vector members; RETURN_IF_ERROR(pqp.Members(&members)); for (const auto& m : members) { triton::common::TritonJson::Value el; RETURN_IF_ERROR(pqp.MemberAsObject(m.c_str(), &el)); RETURN_IF_ERROR( FixUInt(config_json, el, "default_timeout_microseconds")); } } } } // Fix sequence_batching::oldest::max_queue_delay_microseconds, // sequence_batching::direct::max_queue_delay_microseconds, // sequence_batching::max_sequence_idle_microseconds { triton::common::TritonJson::Value sb; if (config_json.Find("sequence_batching", &sb)) { RETURN_IF_ERROR( FixUInt(config_json, sb, "max_sequence_idle_microseconds")); triton::common::TritonJson::Value oldest; if (sb.Find("oldest", &oldest)) { RETURN_IF_ERROR( FixUInt(config_json, oldest, "max_queue_delay_microseconds")); } triton::common::TritonJson::Value direct; if (sb.Find("direct", &direct)) { RETURN_IF_ERROR( FixUInt(config_json, direct, "max_queue_delay_microseconds")); } triton::common::TritonJson::Value states; if (sb.Find("state", &states)) { for (size_t i = 0; i < states.ArraySize(); ++i) { triton::common::TritonJson::Value state; RETURN_IF_ERROR(states.IndexAsObject(i, &state)); RETURN_IF_ERROR(FixIntArray(config_json, state, "dims")); triton::common::TritonJson::Value initial_state; if (sb.Find("initial_state", &initial_state)) { RETURN_IF_ERROR(FixIntArray(config_json, initial_state, "dims")); } } } } } // Fix ensemble_scheduling::step::model_version. { triton::common::TritonJson::Value ens; if (config_json.Find("ensemble_scheduling", &ens)) { triton::common::TritonJson::Value step; if (ens.Find("step", &step)) { RETURN_IF_ERROR(FixObjectArray(config_json, step, "model_version")); } } } // Fix model_warmup::inputs::value::dims. { triton::common::TritonJson::Value warmups; if (config_json.Find("model_warmup", &warmups)) { for (size_t i = 0; i < warmups.ArraySize(); ++i) { triton::common::TritonJson::Value warmup; RETURN_IF_ERROR(warmups.IndexAsObject(i, &warmup)); triton::common::TritonJson::Value inputs; if (warmup.Find("inputs", &inputs)) { std::vector members; RETURN_IF_ERROR(inputs.Members(&members)); for (const auto& m : members) { triton::common::TritonJson::Value input; RETURN_IF_ERROR(inputs.MemberAsObject(m.c_str(), &input)); RETURN_IF_ERROR(FixIntArray(config_json, input, "dims")); } } } } } // Convert fixed json back the string... triton::common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(config_json.Write(&buffer)); *json_str = std::move(buffer.MutableContents()); return Status::Success; } Status JsonToModelConfig( const std::string& json_config, const uint32_t config_version, inference::ModelConfig* protobuf_config) { // Currently only support 'config_version' 1, which is the json // representation of the ModelConfig protobuf matches the representation in // ModelConfigToJson(). if (config_version != 1) { return Status( Status::Code::INVALID_ARG, std::string("model configuration version ") + std::to_string(config_version) + " not supported, supported versions are: 1"); } ::google::protobuf::util::JsonParseOptions options; options.case_insensitive_enum_parsing = true; options.ignore_unknown_fields = false; auto err = ::google::protobuf::util::JsonStringToMessage( json_config, protobuf_config, options); if (!err.ok()) { return Status(Status::Code::INVALID_ARG, std::string(err.message())); } return Status::Success; } BackendType GetBackendTypeFromPlatform(const std::string& platform_name) { if ((platform_name == kTensorFlowGraphDefPlatform) || (platform_name == kTensorFlowSavedModelPlatform)) { return BackendType::BACKEND_TYPE_TENSORFLOW; } if (platform_name == kTensorRTPlanPlatform) { return BackendType::BACKEND_TYPE_TENSORRT; } if (platform_name == kOnnxRuntimeOnnxPlatform) { return BackendType::BACKEND_TYPE_ONNXRUNTIME; } if (platform_name == kPyTorchLibTorchPlatform) { return BackendType::BACKEND_TYPE_PYTORCH; } return BackendType::BACKEND_TYPE_UNKNOWN; } /// Get the BackendType value for a backend name. /// \param backend_name The backend name. /// \return The BackendType or BackendType::UNKNOWN if the platform string /// is not recognized. BackendType GetBackendType(const std::string& backend_name) { if (backend_name == kTensorFlowBackend) { return BackendType::BACKEND_TYPE_TENSORFLOW; } if (backend_name == kTensorRTBackend) { return BackendType::BACKEND_TYPE_TENSORRT; } if (backend_name == kOnnxRuntimeBackend) { return BackendType::BACKEND_TYPE_ONNXRUNTIME; } if (backend_name == kPyTorchBackend) { return BackendType::BACKEND_TYPE_PYTORCH; } return BackendType::BACKEND_TYPE_UNKNOWN; } TRITONSERVER_DataType DataTypeToTriton(const inference::DataType dtype) { switch (dtype) { case inference::DataType::TYPE_BOOL: return TRITONSERVER_TYPE_BOOL; case inference::DataType::TYPE_UINT8: return TRITONSERVER_TYPE_UINT8; case inference::DataType::TYPE_UINT16: return TRITONSERVER_TYPE_UINT16; case inference::DataType::TYPE_UINT32: return TRITONSERVER_TYPE_UINT32; case inference::DataType::TYPE_UINT64: return TRITONSERVER_TYPE_UINT64; case inference::DataType::TYPE_INT8: return TRITONSERVER_TYPE_INT8; case inference::DataType::TYPE_INT16: return TRITONSERVER_TYPE_INT16; case inference::DataType::TYPE_INT32: return TRITONSERVER_TYPE_INT32; case inference::DataType::TYPE_INT64: return TRITONSERVER_TYPE_INT64; case inference::DataType::TYPE_FP16: return TRITONSERVER_TYPE_FP16; case inference::DataType::TYPE_FP32: return TRITONSERVER_TYPE_FP32; case inference::DataType::TYPE_FP64: return TRITONSERVER_TYPE_FP64; case inference::DataType::TYPE_STRING: return TRITONSERVER_TYPE_BYTES; case inference::DataType::TYPE_BF16: return TRITONSERVER_TYPE_BF16; default: break; } return TRITONSERVER_TYPE_INVALID; } inference::DataType TritonToDataType(const TRITONSERVER_DataType dtype) { switch (dtype) { case TRITONSERVER_TYPE_BOOL: return inference::DataType::TYPE_BOOL; case TRITONSERVER_TYPE_UINT8: return inference::DataType::TYPE_UINT8; case TRITONSERVER_TYPE_UINT16: return inference::DataType::TYPE_UINT16; case TRITONSERVER_TYPE_UINT32: return inference::DataType::TYPE_UINT32; case TRITONSERVER_TYPE_UINT64: return inference::DataType::TYPE_UINT64; case TRITONSERVER_TYPE_INT8: return inference::DataType::TYPE_INT8; case TRITONSERVER_TYPE_INT16: return inference::DataType::TYPE_INT16; case TRITONSERVER_TYPE_INT32: return inference::DataType::TYPE_INT32; case TRITONSERVER_TYPE_INT64: return inference::DataType::TYPE_INT64; case TRITONSERVER_TYPE_FP16: return inference::DataType::TYPE_FP16; case TRITONSERVER_TYPE_FP32: return inference::DataType::TYPE_FP32; case TRITONSERVER_TYPE_FP64: return inference::DataType::TYPE_FP64; case TRITONSERVER_TYPE_BYTES: return inference::DataType::TYPE_STRING; case TRITONSERVER_TYPE_BF16: return inference::DataType::TYPE_BF16; default: break; } return inference::DataType::TYPE_INVALID; } bool ConfigChangeRequiresReload( const inference::ModelConfig& old_config, const inference::ModelConfig& new_config) { ::google::protobuf::util::MessageDifferencer pb_diff; pb_diff.IgnoreField( old_config.descriptor()->FindFieldByLowercaseName("instance_group")); pb_diff.IgnoreField( old_config.descriptor()->FindFieldByLowercaseName("version_policy")); return !pb_diff.Compare(old_config, new_config); } bool EquivalentInInstanceConfig( const inference::ModelInstanceGroup& instance_config_lhs, const inference::ModelInstanceGroup& instance_config_rhs) { ::google::protobuf::util::MessageDifferencer pb_diff; pb_diff.IgnoreField( instance_config_lhs.descriptor()->FindFieldByLowercaseName("name")); pb_diff.IgnoreField( instance_config_lhs.descriptor()->FindFieldByLowercaseName("count")); return pb_diff.Compare(instance_config_lhs, instance_config_rhs); } std::string InstanceConfigSignature(const inference::ModelInstanceGroup& instance_config) { inference::ModelInstanceGroup config = instance_config; *config.mutable_name() = "[Normalized]"; config.set_count(1); return config.SerializeAsString(); } }} // namespace triton::core kytensor-2.49.0.7/core/src/model_config_utils.h000066400000000000000000000346311527052371200214010ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "filesystem/api.h" #include "model_config.pb.h" #include "status.h" #include "triton/common/model_config.h" #include "tritonserver_apis.h" namespace triton { namespace core { /// Enumeration for the different backend types. enum BackendType { BACKEND_TYPE_UNKNOWN = 0, BACKEND_TYPE_TENSORRT = 1, BACKEND_TYPE_TENSORFLOW = 2, BACKEND_TYPE_ONNXRUNTIME = 3, BACKEND_TYPE_PYTORCH = 4 }; // Get version of a model from the path containing the model /// definition file. /// \param path The path to the model definition file. /// \param version Returns the version. /// \return The error status. Status GetModelVersionFromPath(const std::string& path, int64_t* version); /// Get the tensor name, false value, and true value for a boolean /// sequence batcher control kind. If 'required' is true then must /// find a tensor for the control. If 'required' is false, return /// 'tensor_name' as empty-string if the control is not mapped to any /// tensor. Status GetBooleanSequenceControlProperties( const inference::ModelSequenceBatching& batcher, const std::string& model_name, const inference::ModelSequenceBatching::Control::Kind control_kind, const bool required, std::string* tensor_name, inference::DataType* tensor_datatype, float* fp32_false_value, float* fp32_true_value, int32_t* int32_false_value, int32_t* int32_true_value, bool* bool_false_value, bool* bool_true_value); /// Get the tensor name and datatype for a non-boolean sequence /// batcher control kind. If 'required' is true then must find a /// tensor for the control. If 'required' is false, return /// 'tensor_name' as empty-string if the control is not mapped to any /// tensor. 'tensor_datatype' returns the required datatype for the /// control. Status GetTypedSequenceControlProperties( const inference::ModelSequenceBatching& batcher, const std::string& model_name, const inference::ModelSequenceBatching::Control::Kind control_kind, const bool required, std::string* tensor_name, inference::DataType* tensor_datatype); /// Read a ModelConfig and normalize it as expected by model backends. /// \param path The full-path to the directory containing the /// model configuration. /// \param min_compute_capability The minimum support CUDA compute /// capability. /// \param config Returns the normalized model configuration. /// \return The error status. Status GetNormalizedModelConfig( const std::string& model_name, const std::string& path, const double min_compute_capability, inference::ModelConfig* config); /// Auto-complete backend related fields (platform, backend and default model /// filename) if not set, note that only Triton recognized backends will be /// checked. /// \param model_name The name of the model. /// \param model_path The full-path to the directory containing the /// model configuration. /// \param config Returns the auto-completed model configuration. /// \return The error status. Status AutoCompleteBackendFields( const std::string& model_name, const std::string& model_path, inference::ModelConfig* config); /// Detects and adds missing fields in the model configuration. /// \param min_compute_capability The minimum supported CUDA compute /// capability. /// \param config The model configuration /// \return The error status Status NormalizeModelConfig( const double min_compute_capability, inference::ModelConfig* config); /// [FIXME] better formalize config normalization / validation /// Detects and adds missing fields in instance group setting. /// \param min_compute_capability The minimum supported CUDA compute /// capability. /// \param config The model configuration /// \return The error status Status NormalizeInstanceGroup( const double min_compute_capability, const std::vector& preferred_groups, inference::ModelConfig* config); /// [FIXME] Remove once a more permanent solution is implemented (DLIS-4211) /// Localize EXECUTION_ENV_PATH in python backend. /// \param model_path The full-path to the directory containing the model /// configuration, before localization. /// \param config The model configuration /// \param localized_model_dir The localized model directory /// \return The error status Status LocalizePythonBackendExecutionEnvironmentPath( const std::string& model_path, inference::ModelConfig* config, std::shared_ptr* localized_model_dir); /// Set execution environments for Python based backends, if the execution /// environments is not already specified and the execution environment file /// '/pb_exec_.tar.gz' exists. /// \param backend_libdir The backend runtime library directory path. /// \param model_config The model configuration. /// \return The error status. Status SetPythonBasedBackendExecutionEnvironment( const std::string& backend_libdir, inference::ModelConfig* model_config); /// Auto-complete the instance count based on instance kind and backend name. /// \param group The instance group to set the count for. /// \param backend The backend name to check against. /// \return The error status. Status SetDefaultInstanceCount( inference::ModelInstanceGroup* group, const std::string& backend); /// Validate that a model is specified correctly, except for model inputs /// and outputs. ValidateModelIOConfig() should be called to /// validate model inputs and outputs. /// \param config The model configuration to validate. /// \param min_compute_capability The minimum support CUDA compute /// capability. /// \return The error status. A non-OK status indicates the configuration /// is not valid. Status ValidateModelConfig( const inference::ModelConfig& config, const double min_compute_capability); /// [FIXME] better formalize config normalization / validation /// Validate instance group setting. /// \param config The model configuration to validate. /// \param min_compute_capability The minimum support CUDA compute /// capability. /// \return The error status. A non-OK status indicates the configuration /// is not valid. Status ValidateInstanceGroup( const inference::ModelConfig& config, const double min_compute_capability); /// Validate that a model inputs and outputs are specified correctly. /// \param config The model configuration to validate. /// \return The error status. A non-OK status indicates the configuration /// is not valid. Status ValidateModelIOConfig(const inference::ModelConfig& config); /// Validate that input is specified correctly in a model /// configuration. /// \param io The model input. /// \param max_batch_size The max batch size specified in model configuration. /// \param platform The platform name /// \return The error status. A non-OK status indicates the input /// is not valid. Status ValidateModelInput( const inference::ModelInput& io, int32_t max_batch_size, const std::string& platform); /// Validate that an input matches one of the allowed input names. /// \param io The model input. /// \param allowed The set of allowed input names. /// \return The error status. A non-OK status indicates the input /// is not valid. Status CheckAllowedModelInput( const inference::ModelInput& io, const std::set& allowed); /// Validate that an output is specified correctly in a model /// configuration. /// \param io The model output. /// \param max_batch_size The max batch size specified in model configuration. /// \param platform The platform name /// \return The error status. A non-OK status indicates the output /// is not valid. Status ValidateModelOutput( const inference::ModelOutput& io, int32_t max_batch_size, const std::string& platform); /// Validate that an output matches one of the allowed output names. /// \param io The model output. /// \param allowed The set of allowed output names. /// \return The error status. A non-OK status indicates the output /// is not valid. Status CheckAllowedModelOutput( const inference::ModelOutput& io, const std::set& allowed); /// Validate that a model batch inputs and batch outputs are specified /// correctly. /// \param config The model configuration to validate.. /// \return The error status. A non-OK status indicates the batch inputs or /// batch outputs are not valid. Status ValidateBatchIO(const inference::ModelConfig& config); /// Parse the 'value' of the parameter 'key' into a boolean value. /// \param key The name of the parameter. /// \param value The value of the parameter in string. /// \param parsed_value Return the boolean of the parameter. /// \return The error status. A non-OK status indicates failure on parsing the /// value. Status ParseBoolParameter( const std::string& key, std::string value, bool* parsed_value); /// Parse the 'value' of the parameter 'key' into a long long integer value. /// \param key The name of the parameter. /// \param value The value of the parameter in string. /// \param parsed_value Return the numerical value of the parameter. /// \return The error status. A non-OK status indicates failure on parsing the /// value. Status ParseLongLongParameter( const std::string& key, const std::string& value, int64_t* parsed_value); /// Obtain the 'profile_index' of the 'profile_name'. /// \param profile_name The name of the profile. /// \param profile_index Return the index of the profile. /// \return The error status. A non-OK status indicates failure on getting the /// value. Status GetProfileIndex(const std::string& profile_name, int* profile_index); /// Convert a model configuration protobuf to the equivalent json. /// \param config The protobuf model configuration. /// \param config_version The model configuration will be returned in /// a format matching this version. If the configuration cannot be /// represented in the requested version's format then an error will /// be returned. /// \param json Returns the equivalent JSON. /// \return The error status. Status ModelConfigToJson( const inference::ModelConfig& config, const uint32_t config_version, std::string* json_str); /// Convert a model configuration JSON to the equivalent protobuf. /// \param config The JSON model configuration. /// \param config_version The model configuration will be returned in /// a format matching this version. If the configuration cannot be /// represented in the requested version's format then an error will /// be returned. /// \param protobuf Returns the equivalent protobuf. /// \return The error status. Status JsonToModelConfig( const std::string& json_config, const uint32_t config_version, inference::ModelConfig* protobuf_config); /// Get the BackendType value for a platform name. /// \param platform_name The platform name. /// \return The BackendType or BackendType::UNKNOWN if the platform string /// is not recognized. BackendType GetBackendTypeFromPlatform(const std::string& platform_name); /// Get the BackendType value for a backend name. /// \param backend_name The backend name. /// \return The BackendType or BackendType::UNKNOWN if the platform string /// is not recognized. BackendType GetBackendType(const std::string& backend_name); /// Get the Triton server data type corresponding to a data type. /// \param dtype The data type. /// \return The Triton server data type. TRITONSERVER_DataType DataTypeToTriton(const inference::DataType dtype); /// Get the data type corresponding to a Triton server data type. /// \param dtype The Triton server data type. /// \return The data type. inference::DataType TritonToDataType(const TRITONSERVER_DataType dtype); /// Check if non instance group and non version policy settings on the model /// configs are equivalent. /// \param old_config The old model config. /// \param new_config The new model config. /// \return False if the model configs are equivalent in all non instance group /// and non version policy settings. True if they differ in non instance group /// and non version policy settings. bool ConfigChangeRequiresReload( const inference::ModelConfig& old_config, const inference::ModelConfig& new_config); /// Check if both model instance configs are equivalent. 'name' and 'count' /// fields do not alter the functionality of the instance and hence excluded /// from checking. /// \param instance_config_lhs The left hand side instance config. /// \param instance_config_rhs The right hand side instance config. /// \return True if instance configs are the same without checking 'name' and /// 'count' fields. False if they are different without checking the fields. bool EquivalentInInstanceConfig( const inference::ModelInstanceGroup& instance_config_lhs, const inference::ModelInstanceGroup& instance_config_rhs); /// Obtain a signature identifying the instance config. 'name' and 'count' /// fields do not alter the functionality of the instance and hence excluded /// from altering the signature. /// \param instance_config The instance config. /// \return Signature identifying the instance config. std::string InstanceConfigSignature( const inference::ModelInstanceGroup& instance_config); }} // namespace triton::core kytensor-2.49.0.7/core/src/model_repository_manager/000077500000000000000000000000001527052371200224455ustar00rootroot00000000000000kytensor-2.49.0.7/core/src/model_repository_manager/model_lifecycle.cc000066400000000000000000000737411527052371200261070ustar00rootroot00000000000000// Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #include "model_lifecycle.h" #include #include #include #include #include #include "backend_model.h" #include "constants.h" #include "filesystem/api.h" #include "model.h" #include "model_config_utils.h" #include "repo_agent.h" #include "triton/common/logging.h" #include "triton/common/thread_pool.h" #ifdef TRITON_ENABLE_ENSEMBLE #include "ensemble_scheduler/ensemble_model.h" #endif // TRITON_ENABLE_ENSEMBLE #include "server.h" namespace triton { namespace core { const std::string& ModelReadyStateString(ModelReadyState state) { switch (state) { case ModelReadyState::UNKNOWN: { static std::string m("UNKNOWN"); return m; } case ModelReadyState::READY: { static std::string m("READY"); return m; } case ModelReadyState::UNAVAILABLE: { static std::string m("UNAVAILABLE"); return m; } case ModelReadyState::LOADING: { static std::string m("LOADING"); return m; } case ModelReadyState::UNLOADING: { static std::string m("UNLOADING"); return m; } } static std::string m(""); return m; } namespace { Status VersionsToLoad( const std::string model_path, const ModelIdentifier& model_id, const inference::ModelConfig& model_config, std::set* versions) { versions->clear(); // Get integral number of the version directory std::set subdirs; RETURN_IF_ERROR(GetDirectorySubdirs(model_path, &subdirs)); std::set> existing_versions; for (const auto& subdir : subdirs) { static const std::vector skip_dirs{ kWarmupDataFolder, kInitialStateFolder, kModelConfigFolder}; if (std::find(skip_dirs.begin(), skip_dirs.end(), subdir) != skip_dirs.end()) { continue; } if ((subdir.length() > 1) && (subdir.front() == '0')) { LOG_WARNING << "ignore version directory '" << subdir << "' which contains leading zeros in its directory name"; continue; } try { int64_t version = std::stoll(subdir); existing_versions.insert(version); } catch (const std::invalid_argument& ia) { LOG_WARNING << "ignore version directory '" << subdir << "' which fails to convert to integral number"; } } if (model_config.version_policy().has_specific()) { for (const auto& v : model_config.version_policy().specific().versions()) { // Only load the specific versions that are presented in model directory bool version_not_exist = existing_versions.insert(v).second; if (!version_not_exist) { versions->emplace(v); } else { LOG_ERROR << "version " << v << " is specified for model '" << model_id << "', but the version directory is not present"; } } } else { if (model_config.version_policy().has_latest()) { // std::set is sorted with std::greater for (const auto& v : existing_versions) { if (versions->size() >= model_config.version_policy().latest().num_versions()) { break; } versions->emplace(v); } } else { // all versions->insert(existing_versions.begin(), existing_versions.end()); } } return Status::Success; } // Use smart pointer with custom deleter so that model state will be updated // to UNAVAILABLE if all smart pointer copies are out of scope struct ModelDeleter { ModelDeleter(std::function OnDestroyModel) : OnDestroyModel_(std::move(OnDestroyModel)) { } void operator()(Model* model) { // The actual model object must be destroyed in a different // thread. This thread could have a callstack that includes the // model itself because this deleter could be triggered by // a request release or response send in the model. Following // delete will lead to the model destructor which may wait on this // same thread... so deadlock if we don't use a different thread // here. std::function destroy_fn = OnDestroyModel_; std::thread dthd([model, destroy_fn]() { delete model; destroy_fn(); }); dthd.detach(); } // Use to inform the ModelLifeCycle that the model handle is destroyed std::function OnDestroyModel_; }; } // namespace Status ModelLifeCycle::Create( InferenceServer* server, const ModelLifeCycleOptions& options, std::unique_ptr* life_cycle) { std::unique_ptr local_life_cycle( new ModelLifeCycle(server, options)); *life_cycle = std::move(local_life_cycle); return Status::Success; } const ModelStateMap ModelLifeCycle::LiveModelStates(bool strict_readiness) { LOG_VERBOSE(2) << "LiveModelStates()"; std::lock_guard map_lock(map_mtx_); ModelStateMap live_model_states; for (auto& model_version : map_) { bool live = false; VersionStateMap version_map; for (auto& version_model : model_version.second) { std::lock_guard lock(version_model.second->mtx_); if (strict_readiness && version_model.second->state_ != ModelReadyState::READY) { continue; } // At least one version is live (ready / loading / unloading) if ((version_model.second->state_ != ModelReadyState::UNKNOWN) && (version_model.second->state_ != ModelReadyState::UNAVAILABLE)) { live = true; version_map[version_model.first] = std::make_pair( version_model.second->state_, version_model.second->state_reason_); } } if (live) { live_model_states[model_version.first] = std::move(version_map); } } return live_model_states; } Status ModelLifeCycle::StopAllModels() { LOG_VERBOSE(2) << "StopAllModels()"; std::lock_guard map_lock(map_mtx_); for (auto& model_version : map_) { for (auto& version_model : model_version.second) { if (version_model.second != nullptr) { std::lock_guard lock(version_model.second->mtx_); if (version_model.second->model_ != nullptr) { version_model.second->model_->Stop(); } } } } return Status::Success; } const std::set> ModelLifeCycle::InflightStatus() { LOG_VERBOSE(2) << "InflightStatus()"; std::lock_guard map_lock(map_mtx_); std::set> inflight_status; for (auto& model_version : map_) { for (auto& version_model : model_version.second) { if (version_model.second != nullptr) { std::lock_guard lock(version_model.second->mtx_); if (version_model.second->model_ != nullptr) { const auto cnt = version_model.second->model_->InflightInferenceCount(); if (cnt != 0) { inflight_status.emplace( model_version.first, version_model.first, cnt); } } } } } return inflight_status; } size_t ModelLifeCycle::BackgroundModelsSize() { LOG_VERBOSE(2) << "BackgroundModelsSize()"; std::lock_guard map_lock(map_mtx_); return background_models_.size(); } const ModelStateMap ModelLifeCycle::ModelStates() { LOG_VERBOSE(2) << "ModelStates()"; std::lock_guard map_lock(map_mtx_); ModelStateMap model_states; for (auto& model_version : map_) { VersionStateMap version_map; for (auto& version_model : model_version.second) { std::lock_guard lock(version_model.second->mtx_); version_map[version_model.first] = std::make_pair( version_model.second->state_, version_model.second->state_reason_); } model_states[model_version.first] = std::move(version_map); } return model_states; } const VersionStateMap ModelLifeCycle::VersionStates(const ModelIdentifier& model_id) { LOG_VERBOSE(2) << "VersionStates() '" << model_id << "'"; std::lock_guard map_lock(map_mtx_); VersionStateMap version_map; auto mit = map_.find(model_id); if (mit != map_.end()) { for (auto& version_model : mit->second) { std::lock_guard lock(version_model.second->mtx_); version_map[version_model.first] = std::make_pair( version_model.second->state_, version_model.second->state_reason_); } } return version_map; } Status ModelLifeCycle::ModelState( const ModelIdentifier& model_id, const int64_t model_version, ModelReadyState* state) { std::lock_guard map_lock(map_mtx_); auto mit = map_.find(model_id); if (mit != map_.end()) { auto vit = mit->second.find(model_version); if (vit != mit->second.end()) { std::lock_guard lock(vit->second->mtx_); *state = vit->second->state_; return Status::Success; } } return Status( Status::Code::NOT_FOUND, "model '" + model_id.str() + "', version " + std::to_string(model_version) + " is not found"); } Status ModelLifeCycle::GetModel( const ModelIdentifier& model_id, const int64_t version, std::shared_ptr* model) { LOG_VERBOSE(2) << "GetModel() '" << model_id << "' version " << version; std::lock_guard map_lock(map_mtx_); auto mit = map_.find(model_id); if (mit == map_.end()) { return Status( Status::Code::NOT_FOUND, "'" + model_id.str() + "' is not found"); } auto vit = mit->second.find(version); if (vit == mit->second.end()) { if (version != -1) { return Status( Status::Code::NOT_FOUND, "'" + model_id.str() + "' version " + std::to_string(version) + " is not found"); } // The case where the request is asking for latest version int64_t latest = -1; static_assert( std::is_same< decltype(mit->second)::key_compare, std::less>::value, "Below assume versions are sorted in specific order"); for (auto version_model = mit->second.rbegin(); version_model != mit->second.rend(); ++version_model) { std::lock_guard lock(version_model->second->mtx_); if (version_model->second->state_ == ModelReadyState::READY) { latest = version_model->first; *model = version_model->second->model_; break; } } if (latest == -1) { return Status( Status::Code::NOT_FOUND, "'" + model_id.str() + "' has no available versions"); } } else { std::lock_guard lock(vit->second->mtx_); if (vit->second->state_ == ModelReadyState::READY) { *model = vit->second->model_; } else { return Status( Status::Code::UNAVAILABLE, "'" + model_id.str() + "' version " + std::to_string(version) + " is not at ready state"); } } return Status::Success; } Status ModelLifeCycle::AsyncUnload(const ModelIdentifier& model_id) { LOG_VERBOSE(2) << "AsyncUnload() '" << model_id << "'"; std::lock_guard map_lock(map_mtx_); auto it = map_.find(model_id); if (it == map_.end()) { return Status( Status::Code::INVALID_ARG, "Model to be unloaded has not been served"); } // Get the existing agent models and notify the unload action const uint64_t now_ns = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); for (auto& version : it->second) { auto& model_info = version.second; std::lock_guard lock(model_info->mtx_); model_info->last_update_ns_ = now_ns; // Unload serving model, for model that is in LOADING state, // the updated timestamp will be recognized that there is newer update // on the model info and the load should be aborted if (model_info->state_ == ModelReadyState::READY) { if (model_info->agent_model_list_ != nullptr) { // Only log the error because the model should be unloaded regardless auto status = model_info->agent_model_list_->InvokeAgentModels( TRITONREPOAGENT_ACTION_UNLOAD); if (!status.IsOk()) { LOG_ERROR << "Agent model returns error on TRITONREPOAGENT_ACTION_UNLOAD: " << status.AsString(); } } // unload model_info->Release(); } } return Status::Success; } Status ModelLifeCycle::AsyncLoad( const ModelIdentifier& model_id, const std::string& model_path, const inference::ModelConfig& model_config, const bool is_config_provided, const ModelTimestamp& prev_timestamp, const ModelTimestamp& curr_timestamp, const std::shared_ptr& agent_model_list, std::function&& OnComplete) { LOG_VERBOSE(2) << "AsyncLoad() '" << model_id << "'"; std::lock_guard map_lock(map_mtx_); auto it = map_.find(model_id); if (it == map_.end()) { it = map_.emplace(std::make_pair(model_id, VersionMap())).first; } std::set versions; RETURN_IF_ERROR( VersionsToLoad(model_path, model_id, model_config, &versions)); if (versions.empty()) { return Status( Status::Code::INVALID_ARG, "at least one version must be available under the version policy of " "model '" + model_id.str() + "'"); } const uint64_t now_ns = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); std::shared_ptr load_tracker( new LoadTracker(versions.size(), now_ns)); for (const auto& version : versions) { std::unique_ptr linfo( new ModelInfo(model_path, model_config, now_ns)); ModelInfo* model_info = linfo.get(); LOG_INFO << "loading: " << model_id << ":" << version; model_info->state_ = ModelReadyState::LOADING; model_info->state_reason_.clear(); model_info->agent_model_list_ = agent_model_list; auto res = it->second.emplace( std::make_pair(version, std::unique_ptr())); if (res.second) { res.first->second = std::move(linfo); } else { // There is already a record of this model version. Check if the version // model is being served, if so, the re-load of the version // should be performed in background to avoid version downtime. // Otherwise, swap and monitor state for newly loading model. auto& serving_model = res.first->second; std::lock_guard lock(serving_model->mtx_); if (serving_model->state_ == ModelReadyState::READY) { // The model is currently being served. Check if the model load could // be completed with a simple config update. if (!serving_model->is_ensemble_ && !prev_timestamp.IsModelVersionModified(curr_timestamp, version) && !ConfigChangeRequiresReload( serving_model->model_config_, model_config)) { // Update the model model_info = serving_model.get(); model_info->last_update_ns_ = now_ns; load_pool_->Enqueue([this, model_id, version, model_info, model_config, OnComplete, load_tracker]() { UpdateModelConfig(model_id, version, model_info, model_config); OnLoadComplete( model_id, version, model_info, true /* is_update */, OnComplete, load_tracker); }); continue; // move to the next version } // A full model load is required. background_models_[(uintptr_t)model_info] = std::move(linfo); } else { // swap the monitoring model info serving_model.swap(linfo); // further check the state, put to 'background_models_' to keep // the object valid if the model is LOADING / UNLOADING, because // the model info will be accessed by a different thread once the // operation is completed if ((linfo->state_ == ModelReadyState::LOADING) || (linfo->state_ == ModelReadyState::UNLOADING)) { ModelInfo* key = linfo.get(); background_models_[(uintptr_t)key] = std::move(linfo); } } } // Load model asynchronously via thread pool load_pool_->Enqueue([this, model_id, version, model_info, OnComplete, load_tracker, is_config_provided]() { for (size_t retry = 0; retry <= options_.load_retry; ++retry) { model_info->state_ = ModelReadyState::LOADING; CreateModel(model_id, version, model_info, is_config_provided); // Model state will be changed to NOT loading if failed to load, // so the model is loaded if state is LOADING. if (model_info->state_ == ModelReadyState::LOADING) { break; } } OnLoadComplete( model_id, version, model_info, false /* is_update */, OnComplete, load_tracker); }); } return Status::Success; } void ModelLifeCycle::CreateModel( const ModelIdentifier& model_id, const int64_t version, ModelInfo* model_info, const bool is_config_provided) { LOG_VERBOSE(2) << "CreateModel() '" << model_id << "' version " << version; const auto& model_config = model_info->model_config_; // Create model Status status; std::unique_ptr is; // If 'backend' is specified in the config then use the new triton // backend. if (!model_config.backend().empty()) { std::unique_ptr model; status = TritonModel::Create( server_, model_info->model_path_, options_.backend_cmdline_config_map, options_.host_policy_map, model_id, version, model_config, is_config_provided, &model); is.reset(model.release()); } else { #ifdef TRITON_ENABLE_ENSEMBLE if (model_info->is_ensemble_) { status = EnsembleModel::Create( server_, model_info->model_path_, model_id, version, model_config, is_config_provided, options_.min_compute_capability, &is); // Complete label provider with label information from involved models // Must be done here because involved models may not be able to // obtained from server because this may happen during server // initialization. if (status.IsOk()) { std::set no_label_outputs; const auto& label_provider = is->GetLabelProvider(); for (const auto& output : model_config.output()) { if (label_provider->GetLabel(output.name(), 0).empty()) { no_label_outputs.emplace(output.name()); } } for (const auto& element : model_config.ensemble_scheduling().step()) { for (const auto& pair : element.output_map()) { // Found model that produce one of the missing output if (no_label_outputs.find(pair.second) != no_label_outputs.end()) { std::shared_ptr model; // Safe to obtain model because the ensemble can't be loaded // until the involved models are ready GetModel( {element.model_namespace(), element.model_name()}, element.model_version(), &model); label_provider->AddLabels( pair.second, model->GetLabelProvider()->GetLabels(pair.first)); } } } } } else #endif // TRITON_ENABLE_ENSEMBLE { status = Status( Status::Code::INVALID_ARG, "unknown platform '" + model_config.platform() + "'"); } } std::lock_guard lock(model_info->mtx_); if (status.IsOk()) { // [FIXME] better way to manage agent model lifecycle // Let the deleter also holds a shared pointer copy of agent model list, // because the reference in ModelInfo can be cleared before the Model object // is destroyed, and we want agent model to be valid for receiving // UNLOAD_COMPLETE signal (see ~TritonRepoAgentModelList for detail) auto agent_model_list = model_info->agent_model_list_; model_info->model_.reset( is.release(), ModelDeleter([this, model_id, version, model_info, agent_model_list]() mutable { LOG_VERBOSE(2) << "OnDestroy callback() '" << model_id << "' version " << version; LOG_INFO << "successfully unloaded '" << model_id << "' version " << version; // Update model state as it is fully unloaded { std::lock_guard lock(model_info->mtx_); model_info->state_ = ModelReadyState::UNAVAILABLE; model_info->state_reason_ = "unloaded"; } // Check if the model info is in background, if so, remove from the // map std::lock_guard lk(this->map_mtx_); auto it = this->background_models_.find((uintptr_t)model_info); if (it != this->background_models_.end()) { this->background_models_.erase(it); } })); } else { LOG_ERROR << "failed to load '" << model_id << "' version " << version << ": " << status.AsString(); model_info->state_ = ModelReadyState::UNAVAILABLE; model_info->state_reason_ = status.AsString(); } } void ModelLifeCycle::UpdateModelConfig( const ModelIdentifier& model_id, const int64_t version, ModelInfo* model_info, const inference::ModelConfig& new_model_config) { LOG_VERBOSE(2) << "UpdateModelConfig() '" << model_id << "' version " << version; std::unique_lock model_info_lock(model_info->mtx_); // Make sure the state reason is empty before attempting to update. model_info->state_reason_.clear(); // Downcast 'Model' to 'TritonModel'. TritonModel* model = dynamic_cast(model_info->model_.get()); if (model == nullptr) { model_info->state_reason_ = "Unable to downcast '" + model_id.str() + "' from 'Model' to 'TritonModel' during model update."; return; } // Update model instance group. The 'model_info->mtx_' may be released while // the model is updating its instance group, because no model info from this // model lifecycle object is being accessed by the model during the update, // and model repository manager will ensure no concurrent update on the same // model may take place. Therefore, releasing the lock allows the model to be // accessed which enables inference during update. model_info_lock.unlock(); Status status = model->UpdateInstanceGroup(new_model_config); model_info_lock.lock(); if (!status.IsOk()) { model_info->state_reason_ = status.AsString(); return; } // Write new config into 'model_info'. model_info->model_config_ = new_model_config; } void ModelLifeCycle::OnLoadComplete( const ModelIdentifier& model_id, const int64_t version, ModelInfo* model_info, const bool is_update, const std::function& OnComplete, std::shared_ptr load_tracker) { LOG_VERBOSE(2) << "OnLoadComplete() '" << model_id << "' version " << version; std::lock_guard tracker_lock(load_tracker->mtx_); // Track 'model_info' for newly created models in 'load_set_'. if (!is_update) { load_tracker->load_set_[version] = model_info; } { std::lock_guard model_info_lock(model_info->mtx_); // Write failed to load reason into tracker, if any. // A newly created model is at 'LOADING' state unless it failed to load. // An updated model has an empty state reason unless it failed to update. if ((!is_update && model_info->state_ != ModelReadyState::LOADING) || (is_update && !model_info->state_reason_.empty())) { load_tracker->load_failed_ = true; load_tracker->reason_ += ("version " + std::to_string(version) + " is at " + ModelReadyStateString(model_info->state_) + " state: " + model_info->state_reason_ + ";"); } } // Check if all versions are completed and finish the load. if (++load_tracker->completed_version_cnt_ == load_tracker->affected_version_cnt_) { OnLoadFinal(model_id, model_info, OnComplete, load_tracker); } } void ModelLifeCycle::OnLoadFinal( const ModelIdentifier& model_id, ModelInfo* model_info, const std::function& OnComplete, std::shared_ptr load_tracker) { LOG_VERBOSE(2) << "OnLoadFinal() '" << model_id << "' for all version(s)"; std::lock_guard model_lifecycle_map_lock(map_mtx_); auto it = map_.find(model_id); // Check if the load is the latest frontground action on the model for (const auto& version_info : it->second) { std::lock_guard lock(version_info.second->mtx_); if (version_info.second->last_update_ns_ > load_tracker->last_update_ns_) { load_tracker->load_failed_ = true; load_tracker->reason_ = "Newer operation has been applied to the model lifecycle, current " "load operation is out-dated."; break; } } if (load_tracker->load_failed_) { // Copy agent list out of `ModelInfo` as it needs to be invoked after all // 'ModelInfo' for all model versions are reset. std::shared_ptr lagent_list = model_info->agent_model_list_; // If any of the versions fails to load, abort the load and unload all newly // loaded versions for (auto& loaded : load_tracker->load_set_) { // Unload directly, the object is being managed either in frontground // or background std::lock_guard lock(loaded.second->mtx_); if (loaded.second->model_ != nullptr) { loaded.second->Release(); } } // Invoke agent list. if (lagent_list) { auto status = lagent_list->InvokeAgentModels(TRITONREPOAGENT_ACTION_LOAD_FAIL); if (!status.IsOk()) { LOG_ERROR << "Agent model returns error on " "TRITONREPOAGENT_ACTION_LOAD_FAIL: " << status.AsString(); } } LOG_INFO << "failed to load '" << model_id << "'"; } else { // Unload any previous loaded versions that are still available for (auto& version_info : it->second) { auto& mi = version_info.second; std::lock_guard info_lk(mi->mtx_); if (mi->state_ == ModelReadyState::READY && mi->last_update_ns_ < load_tracker->last_update_ns_) { if (mi->agent_model_list_ != nullptr) { auto status = mi->agent_model_list_->InvokeAgentModels( TRITONREPOAGENT_ACTION_UNLOAD); if (!status.IsOk()) { LOG_ERROR << "Agent model returns error on " "TRITONREPOAGENT_ACTION_UNLOAD: " << status.AsString(); } } mi->Release(); } } // Mark current versions ready and track info in foreground for (auto& loaded : load_tracker->load_set_) { std::lock_guard curr_info_lk(loaded.second->mtx_); loaded.second->state_ = ModelReadyState::READY; loaded.second->state_reason_.clear(); auto bit = background_models_.find((uintptr_t)loaded.second); // Check if the version model is loaded in background, if so, // replace and unload the current serving version if (bit != background_models_.end()) { auto vit = it->second.find(loaded.first); // Need to lock the previous model info for in case the model is // loading / unloading, this ensure the model state is consistent // even when the load / unload is completed. std::lock_guard prev_info_lk(vit->second->mtx_); // swap previous info into local unique pointer auto linfo = std::move(bit->second); vit->second.swap(linfo); background_models_.erase(bit); // if previous info is under change, put into 'background_models_' if ((linfo->state_ == ModelReadyState::LOADING) || (linfo->state_ == ModelReadyState::UNLOADING)) { ModelInfo* key = linfo.get(); background_models_[(uintptr_t)key] = std::move(linfo); } } } if (model_info->agent_model_list_) { auto status = model_info->agent_model_list_->InvokeAgentModels( TRITONREPOAGENT_ACTION_LOAD_COMPLETE); if (!status.IsOk()) { LOG_ERROR << "Agent model returns error on " "TRITONREPOAGENT_ACTION_LOAD_COMPLETE: " << status.AsString(); } } LOG_INFO << "successfully loaded '" << model_id << "'"; } if (OnComplete) { OnComplete( load_tracker->load_failed_ ? Status(Status::Code::INVALID_ARG, load_tracker->reason_) : Status::Success); } } }} // namespace triton::core kytensor-2.49.0.7/core/src/model_repository_manager/model_lifecycle.h000066400000000000000000000303621527052371200257410ustar00rootroot00000000000000// Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once #include #include #include #include #include "infer_parameter.h" #include "model.h" #include "model_config.pb.h" #include "repo_agent.h" #include "status.h" #include "triton/common/model_config.h" #include "triton/common/thread_pool.h" namespace triton { namespace core { struct ModelLifeCycleOptions { explicit ModelLifeCycleOptions( const double min_compute_capability, const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map, const triton::common::HostPolicyCmdlineConfigMap& host_policy_map, const unsigned int model_load_thread_count, const size_t load_retry) : min_compute_capability(min_compute_capability), backend_cmdline_config_map(backend_cmdline_config_map), host_policy_map(host_policy_map), model_load_thread_count(model_load_thread_count), load_retry(load_retry) { } // The minimum supported CUDA compute capability. const double min_compute_capability; // The backend configuration settings specified on the command-line const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map; // The host policy setting used when loading models. const triton::common::HostPolicyCmdlineConfigMap& host_policy_map; // Number of the threads to use for concurrently loading models const unsigned int model_load_thread_count; // Number of retry on model loading before considering the load has failed. const size_t load_retry{0}; }; /// Readiness status for models. enum class ModelReadyState { // The model is in an unknown state. The model is not available for // inferencing. UNKNOWN, // The model is ready and available for inferencing. READY, // The model is unavailable, indicating that the model failed to // load or has been implicitly or explicitly unloaded. The model is // not available for inferencing. UNAVAILABLE, // The model is being loaded by the inference server. The model is // not available for inferencing. LOADING, // The model is being unloaded by the inference server. The model is // not available for inferencing. UNLOADING }; /// Get the string representation for a ModelReadyState const std::string& ModelReadyStateString(ModelReadyState state); using VersionStateMap = std::map>; using ModelStateMap = std::map; // Helper class to manage the lifecycle of a list of associated agent models class TritonRepoAgentModelList { public: TritonRepoAgentModelList() : last_action_type_(TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE){}; ~TritonRepoAgentModelList() { // Using destructor to finish the unload lifecycle without // explicitly managing the last step in ModelLifecycle. if (last_action_type_ == TRITONREPOAGENT_ACTION_UNLOAD) { InvokeAgentModels(TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE); } } Status AddAgentModel(std::unique_ptr&& agent_model) { agent_models_.emplace_back(std::move(agent_model)); return Status::Success; } size_t Size() { return agent_models_.size(); } TritonRepoAgentModel* Back() { return agent_models_.back().get(); } Status InvokeAgentModels(const TRITONREPOAGENT_ActionType action_type) { // Special handling for the current model lifecycle implementation, // the repo agent may be asked to perform UNLOAD action multiple times, // and the requests after the first should be ignored. const bool duplicate_unload = (action_type == TRITONREPOAGENT_ACTION_UNLOAD) && (last_action_type_ == TRITONREPOAGENT_ACTION_UNLOAD); if (duplicate_unload) { return Status::Success; } last_action_type_ = action_type; switch (action_type) { case TRITONREPOAGENT_ACTION_LOAD: case TRITONREPOAGENT_ACTION_UNLOAD: { for (size_t idx = 0; idx < agent_models_.size(); ++idx) { RETURN_IF_ERROR(agent_models_[idx]->InvokeAgent(action_type)); } break; } case TRITONREPOAGENT_ACTION_LOAD_COMPLETE: case TRITONREPOAGENT_ACTION_LOAD_FAIL: case TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: { // reverse order for (size_t one_pass_idx = agent_models_.size(); one_pass_idx > 0; --one_pass_idx) { RETURN_IF_ERROR( agent_models_[one_pass_idx - 1]->InvokeAgent(action_type)); } break; } } return Status::Success; } private: DISALLOW_COPY_AND_ASSIGN(TritonRepoAgentModelList); std::vector> agent_models_; TRITONREPOAGENT_ActionType last_action_type_; }; class InferenceServer; class ModelTimestamp; class ModelLifeCycle { public: static Status Create( InferenceServer* server, const ModelLifeCycleOptions& options, std::unique_ptr* life_cycle); ~ModelLifeCycle() { // Explicitly clean up thread pool first to clean up any pending callbacks // that may modify model lifecycle members load_pool_.reset(); map_.clear(); } // Start loading model with specified versions asynchronously. // All versions that are being served will be unloaded only after // the load is finished successfully. Status AsyncLoad( const ModelIdentifier& model_id, const std::string& model_path, const inference::ModelConfig& model_config, const bool is_config_provided, const ModelTimestamp& prev_timestamp, const ModelTimestamp& curr_timestamp, const std::shared_ptr& agent_model_list, std::function&& OnComplete); // Unload model asynchronously. Status AsyncUnload(const ModelIdentifier& model_id); // Get specified version of the model. Latest ready version will // be retrieved if 'version' is -1. Return error if the version specified is // not found or it is not ready. Status GetModel( const ModelIdentifier& model_id, const int64_t version, std::shared_ptr* model); // Get the ModelStateMap representation of the live models. A model is // live if at least one of the versions is not unknown nor unavailable. // If 'strict_readiness' is true, a model is only live if // at least one of the versions is ready. const ModelStateMap LiveModelStates(bool strict_readiness = false); // Get the ModelStateMap representation of the models. const ModelStateMap ModelStates(); // Get the VersionStateMap representation of the specified model. const VersionStateMap VersionStates(const ModelIdentifier& model_id); // Get the state of a specific model version. Status ModelState( const ModelIdentifier& model_id, const int64_t model_version, ModelReadyState* state); // Instruct the model to stop accepting new inference requests. Status StopAllModels(); // Return the number of in-flight inference if any, model versions // that don't have in-flight inferences will not be included. const std::set> InflightStatus(); // Return the number of model(s) in the background. size_t BackgroundModelsSize(); private: struct ModelInfo { ModelInfo( const std::string& model_path, const inference::ModelConfig& model_config, const uint64_t last_update_ns) : model_config_(model_config), model_path_(model_path), #ifdef TRITON_ENABLE_ENSEMBLE is_ensemble_(model_config.platform() == kEnsemblePlatform), #else is_ensemble_(false), #endif // TRITON_ENABLE_ENSEMBLE last_update_ns_(last_update_ns), state_(ModelReadyState::UNKNOWN) { } // Release the flyweight in ModelInfo object, reflect as 'UNLOADING' in // model state. Note that 'mtx_' should be acquired before invoking this // function to prevent possible data race. void Release() { state_ = ModelReadyState::UNLOADING; state_reason_.clear(); agent_model_list_.reset(); model_.reset(); } inference::ModelConfig model_config_; const std::string model_path_; const bool is_ensemble_; std::mutex mtx_; uint64_t last_update_ns_; ModelReadyState state_; std::string state_reason_; // flyweight std::shared_ptr agent_model_list_; std::shared_ptr model_; }; struct LoadTracker { LoadTracker( const size_t affected_version_cnt, const uint64_t last_update_ns) : last_update_ns_(last_update_ns), affected_version_cnt_(affected_version_cnt), load_failed_(false), completed_version_cnt_(0) { } const uint64_t last_update_ns_; const size_t affected_version_cnt_; std::mutex mtx_; bool load_failed_; std::string reason_; size_t completed_version_cnt_; std::map load_set_; }; ModelLifeCycle(InferenceServer* server, const ModelLifeCycleOptions& options) : server_(server), options_(options) { load_pool_.reset(new triton::common::ThreadPool( std::max(1u, options_.model_load_thread_count))); } // Create a new model, the 'model_id' can either be a new or existing model. void CreateModel( const ModelIdentifier& model_id, const int64_t version, ModelInfo* model_info, const bool is_config_provided); // Update model to the new config. It is the responsibility of the caller to // ensure the model can be updated in-place without a complete reload. // Currently, only model instances can be updated. void UpdateModelConfig( const ModelIdentifier& model_id, const int64_t version, ModelInfo* model_info, const inference::ModelConfig& new_model_config); // Update 'load_tracker' to the latest info in 'model_info' after loading // **each** model version. void OnLoadComplete( const ModelIdentifier& model_id, const int64_t version, ModelInfo* model_info, const bool is_update, const std::function& OnComplete, std::shared_ptr load_tracker); // Helper function for 'OnLoadComplete()' to finish final operations after // loading **all** model versions. void OnLoadFinal( const ModelIdentifier& model_id, ModelInfo* model_info, const std::function& OnComplete, std::shared_ptr load_tracker); // Mutex for 'map_' and 'background_models_' std::mutex map_mtx_; using VersionMap = std::map>; using ModelMap = std::map; ModelMap map_; // Models that are being loaded / unloaded in background std::map> background_models_; InferenceServer* server_; const ModelLifeCycleOptions options_; // Fixed-size thread pool to load models at specified concurrency std::unique_ptr load_pool_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/model_repository_manager/model_repository_manager.cc000066400000000000000000002536411527052371200300600ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #include "model_repository_manager.h" #include #include #include #include #include #include #include "backend_model.h" #include "constants.h" #include "ensemble_scheduler/ensemble_utils.h" #include "filesystem/api.h" #include "model.h" #include "model_config_utils.h" #include "triton/common/logging.h" #ifdef TRITON_ENABLE_ENSEMBLE #include "ensemble_scheduler/ensemble_model.h" #endif // TRITON_ENABLE_ENSEMBLE namespace triton { namespace core { namespace { template void AddToSet(const std::set& src, std::set* dest) { // std::set::merge() can be used if move to >= C++17, // note this is different from merge: copy item instead of move for (const auto& item : src) { dest->emplace(item); } } static std::string file_prefix = "file:"; // Internal repo agent used for model file override class LocalizeRepoAgent : public TritonRepoAgent { public: LocalizeRepoAgent() : TritonRepoAgent("ModelRepositoryManager::LocalizeRepoAgent") { // Callbacks below interact with TritonRepoAgentModel directly knowing that // it is the internal implementation of TRITONREPOAGENT_AgentModel model_action_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { auto agent_model = reinterpret_cast(model); switch (action_type) { case TRITONREPOAGENT_ACTION_LOAD: { // localize the override files for model loading, // as currently the model is expected to load from local directory const char* temp_dir_cstr = nullptr; RETURN_TRITONSERVER_ERROR_IF_ERROR( agent_model->AcquireMutableLocation( TRITONREPOAGENT_ARTIFACT_FILESYSTEM, &temp_dir_cstr)); const std::string temp_dir = std::filesystem::canonical(temp_dir_cstr).string(); const auto& files = *reinterpret_cast*>( agent_model->State()); bool found_config = false; for (const auto& file : files) { if (file->Name() == "config") { if (file->Type() != TRITONSERVER_PARAMETER_STRING) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Config parameter 'config' must have string type for its " "value"); } inference::ModelConfig config; RETURN_TRITONSERVER_ERROR_IF_ERROR(JsonToModelConfig( file->ValueString(), 1 /* config_version */, &config)); RETURN_TRITONSERVER_ERROR_IF_ERROR(WriteTextProto( JoinPath({temp_dir, kModelConfigPbTxt}), config)); found_config = true; } else if (file->Name().rfind(file_prefix, 0) == 0) { if (file->Type() != TRITONSERVER_PARAMETER_BYTES) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("File parameter '") + file->Name() + "' must have bytes type for its value") .c_str()); } // Resolve any relative paths or symlinks, and enforce that target // directory stays within model directory for security. const std::string file_path = std::filesystem::weakly_canonical( JoinPath( {temp_dir, file->Name().substr(file_prefix.size())})) .string(); if (file_path.rfind(temp_dir, 0) != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Invalid file parameter '") + file->Name() + "' with normalized path '" + file_path + "' must stay within model directory.") .c_str()); } // Save model override file to the instructed directory using the // temporary model directory as the basepath. const std::string dir = DirName(file_path); bool dir_exist = false; RETURN_TRITONSERVER_ERROR_IF_ERROR(FileExists(dir, &dir_exist)); if (dir_exist) { bool is_dir = false; RETURN_TRITONSERVER_ERROR_IF_ERROR(IsDirectory(dir, &is_dir)); if (!is_dir) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Invalid file parameter '") + file->Name() + "', directory has been created as a file") .c_str()); } } else { RETURN_TRITONSERVER_ERROR_IF_ERROR( MakeDirectory(dir, true /* recursive */)); } // Write file contents at specified path RETURN_TRITONSERVER_ERROR_IF_ERROR(WriteBinaryFile( file_path, reinterpret_cast(file->ValuePointer()), file->ValueByteSize())); } } if (!found_config) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Load parameter 'config' must be specified for model file " "override"); } // Commit the temporary directory RETURN_TRITONSERVER_ERROR_IF_ERROR(agent_model->SetLocation( TRITONREPOAGENT_ARTIFACT_FILESYSTEM, temp_dir_cstr)); break; } default: break; } return nullptr; // success }; model_fini_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) -> TRITONSERVER_Error* { auto agent_model = reinterpret_cast(model); RETURN_TRITONSERVER_ERROR_IF_ERROR(agent_model->DeleteMutableLocation()); return nullptr; // success }; } }; /// Get the model config path to load for the model. const std::string GetModelConfigFullPath( const std::string& model_dir_path, const std::string& custom_config_name) { // "--model-config-name" is set. Select custom config from // "/configs" folder if config file exists. if (!custom_config_name.empty()) { bool custom_config_exists = false; const std::string custom_config_path = JoinPath( {model_dir_path, kModelConfigFolder, custom_config_name + kPbTxtExtension}); Status status = FileExists(custom_config_path, &custom_config_exists); if (!status.IsOk()) { LOG_ERROR << "Failed to get model configuration full path for '" << model_dir_path << "': " << status.AsString(); return ""; } if (custom_config_exists) { return custom_config_path; } } // "--model-config-name" is not set or custom config file does not exist. return JoinPath({model_dir_path, kModelConfigPbTxt}); } Status CreateAgentModelListWithLoadAction( const inference::ModelConfig& original_model_config, const std::string& original_model_path, const std::string& model_config_name, std::shared_ptr* agent_model_list) { if (original_model_config.has_model_repository_agents()) { // Trick to append user specified repo agent on top of internal ones std::shared_ptr lagent_model_list; if (*agent_model_list != nullptr) { lagent_model_list = std::move(*agent_model_list); } else { lagent_model_list.reset(new TritonRepoAgentModelList()); } FileSystemType filesystem_type; RETURN_IF_ERROR(GetFileSystemType(original_model_path, &filesystem_type)); TRITONREPOAGENT_ArtifactType artifact_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; if (filesystem_type != FileSystemType::LOCAL) { artifact_type = TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM; } const char* location = original_model_path.c_str(); inference::ModelConfig model_config = original_model_config; for (const auto& agent_config : original_model_config.model_repository_agents().agents()) { std::shared_ptr agent; RETURN_IF_ERROR( TritonRepoAgentManager::CreateAgent(agent_config.name(), &agent)); TritonRepoAgent::Parameters agent_params; for (const auto& parameter : agent_config.parameters()) { agent_params.emplace_back(parameter.first, parameter.second); } std::unique_ptr agent_model; if (lagent_model_list->Size() != 0) { lagent_model_list->Back()->Location(&artifact_type, &location); const auto config_path = GetModelConfigFullPath(location, model_config_name); if (!ReadTextProto(config_path, &model_config).IsOk()) { model_config.Clear(); } } RETURN_IF_ERROR(TritonRepoAgentModel::Create( artifact_type, location, model_config, agent, agent_params, &agent_model)); RETURN_IF_ERROR(agent_model->InvokeAgent(TRITONREPOAGENT_ACTION_LOAD)); lagent_model_list->AddAgentModel(std::move(agent_model)); } *agent_model_list = std::move(lagent_model_list); } return Status::Success; } // Return the latest modification time in ns for all files/folders starting // from the path. The modification time will be 0 if there is an error. int64_t GetPathModifiedTime(const std::string& path) { // If there is an error in any step the fall-back default // modification time is 0. This means that in error cases 'path' // will show as not modified. This is the safe fall-back to avoid // assuming a model is constantly being modified. bool path_is_dir; Status status = IsDirectory(path, &path_is_dir); if (!status.IsOk()) { LOG_ERROR << "Failed to determine modification time for '" << path << "': " << status.AsString(); return 0; } // If 'path' is a file return its mtime. Otherwise, using the modification // time of the directory as baseline in case of file deletion int64_t mtime = 0; status = FileModificationTime(path, &mtime); if (!status.IsOk()) { LOG_ERROR << "Failed to determine modification time for '" << path << "': " << status.AsString(); return 0; } if (!path_is_dir) { return mtime; } // 'path' is a directory. Return the most recent mtime of the // contents of the directory. std::set contents; status = GetDirectoryContents(path, &contents); if (!status.IsOk()) { LOG_ERROR << "Failed to determine modification time for '" << path << "': " << status.AsString(); return 0; } for (const auto& child : contents) { const auto full_path = JoinPath({path, child}); mtime = std::max(mtime, GetPathModifiedTime(full_path)); } return mtime; } } // namespace ModelTimestamp::ModelTimestamp( const std::string& model_dir_path, const std::string& model_config_path) { // DLIS-7221: Raise an exception when failed to create timestamp bool init_success = ModelDirectoryPathIsValid(model_dir_path) && ReadModelDirectoryTimestamp(model_dir_path, model_config_path) && ReadModelDirectoryContentTimestamps(model_dir_path, model_config_path); if (!init_success) { // Same as calling the default constructor. All timestamps are presumed to // be 0 when comparing. model_timestamps_.clear(); model_config_content_name_.clear(); } } bool ModelTimestamp::ModelDirectoryPathIsValid(const std::string& path) const { bool is_dir; Status status = IsDirectory(path, &is_dir); if (!status.IsOk()) { LOG_ERROR << "Failed to determine modification time for '" << path << "': " << status.AsString(); return false; } if (!is_dir) { LOG_ERROR << "Failed to determine modification time for '" << path << "': Model directory path is not a directory"; return false; } return true; } bool ModelTimestamp::ReadModelDirectoryTimestamp( const std::string& model_dir_path, const std::string& model_config_path) { int64_t model_dir_time = 0; Status status = FileModificationTime(model_dir_path, &model_dir_time); if (!status.IsOk()) { LOG_ERROR << "Failed to determine modification time for '" << model_dir_path << "': " << status.AsString(); return false; } model_timestamps_.emplace("", model_dir_time); return true; } bool ModelTimestamp::ReadModelDirectoryContentTimestamps( const std::string& model_dir_path, const std::string& model_config_path) { std::set dir_contents; Status status = GetDirectoryContents(model_dir_path, &dir_contents); if (!status.IsOk()) { LOG_ERROR << "Failed to determine modification time for '" << model_dir_path << "': " << status.AsString(); return false; } for (const auto& content_name : dir_contents) { const auto content_path = JoinPath({model_dir_path, content_name}); bool is_model_config = model_config_path.rfind(content_path, 0) == 0; if (is_model_config) { if (!model_config_content_name_.empty()) { LOG_ERROR << "Failed to determine modification time for '" << model_dir_path << "': Duplicate model config is detected"; return false; } model_config_content_name_ = content_name; } model_timestamps_.emplace(content_name, GetPathModifiedTime(content_path)); } return true; } bool ModelTimestamp::IsModified(const ModelTimestamp& new_timestamp) const { int64_t old_modified_time = GetModifiedTime(); int64_t new_modified_time = new_timestamp.GetModifiedTime(); return new_modified_time > old_modified_time; } bool ModelTimestamp::IsModelVersionModified( const ModelTimestamp& new_timestamp, const int64_t version) const { int64_t old_modified_time = std::max( GetModelVersionModifiedTime(version), GetNonModelConfigNorVersionNorDirModifiedTime()); int64_t new_modified_time = std::max( new_timestamp.GetModelVersionModifiedTime(version), new_timestamp.GetNonModelConfigNorVersionNorDirModifiedTime()); return new_modified_time > old_modified_time; } int64_t ModelTimestamp::GetModifiedTime() const { int64_t modified_time = 0; for (const auto& pair : model_timestamps_) { const int64_t time = pair.second; modified_time = std::max(modified_time, time); } return modified_time; } int64_t ModelTimestamp::GetModelVersionModifiedTime(const int64_t version) const { int64_t modified_time = 0; auto itr = model_timestamps_.find(std::to_string(version)); if (itr != model_timestamps_.end()) { modified_time = itr->second; } return modified_time; } int64_t ModelTimestamp::GetNonModelConfigNorVersionNorDirModifiedTime() const { // Get modified time excluding time from model config, model version // directory(s) and model directory. int64_t modified_time = 0; for (const auto& pair : model_timestamps_) { const std::string& content_name = pair.first; bool found_non_digit_in_content_name = false; for (const char& c : content_name) { if (std::isdigit(c) == 0) { found_non_digit_in_content_name = true; break; } } // All model version directory(s) will not 'found_non_digit_in_content_name' // as they are all digit(s). // Model directory name will not 'found_non_digit_in_content_name' as it is // empty. if (found_non_digit_in_content_name && content_name != model_config_content_name_) { const int64_t time = pair.second; modified_time = std::max(modified_time, time); } } return modified_time; } void ModelTimestamp::SetModelConfigModifiedTime(const int64_t time_ns) { if (model_config_content_name_.empty()) { LOG_ERROR << "Failed to set config modification time: " "model_config_content_name_ is empty"; return; } model_timestamps_[model_config_content_name_] = time_ns; } ModelRepositoryManager::ModelRepositoryManager( const std::set& repository_paths, const bool autofill, const std::string& model_config_name, const bool polling_enabled, const bool model_control_enabled, const double min_compute_capability, const bool enable_model_namespacing, std::unique_ptr life_cycle) : autofill_(autofill), model_config_name_(model_config_name), polling_enabled_(polling_enabled), model_control_enabled_(model_control_enabled), min_compute_capability_(min_compute_capability), dependency_graph_(&global_map_), enable_model_namespacing_(enable_model_namespacing), repository_paths_(repository_paths), model_life_cycle_(std::move(life_cycle)) { if (enable_model_namespacing_) { find_identifier_fn_ = [this](const std::string& n, ModelIdentifier* i) { return FindModelIdentifier(n, i); }; } else { find_identifier_fn_ = [this](const std::string& n, ModelIdentifier* i) { return Status::Success; }; } } ModelRepositoryManager::~ModelRepositoryManager() {} Status ModelRepositoryManager::Create( InferenceServer* server, const std::string& server_version, const std::set& repository_paths, const std::set& startup_models, const bool strict_model_config, const std::string& model_config_name, const bool polling_enabled, const bool model_control_enabled, const ModelLifeCycleOptions& life_cycle_options, const bool enable_model_namespacing, std::unique_ptr* model_repository_manager) { // The rest only matters if repository path is valid directory for (const auto& path : repository_paths) { bool path_is_dir; RETURN_IF_ERROR(IsDirectory(path, &path_is_dir)); if (!path_is_dir) { return Status( Status::Code::INVALID_ARG, "repository path is not a valid directory"); } } if (polling_enabled && model_control_enabled) { return Status( Status::Code::INVALID_ARG, "cannot enable both polling and explicit model control"); } std::unique_ptr life_cycle; RETURN_IF_ERROR( ModelLifeCycle::Create(server, life_cycle_options, &life_cycle)); // Not setting the smart pointer directly to simplify clean up std::unique_ptr local_manager( new ModelRepositoryManager( repository_paths, !strict_model_config, model_config_name, polling_enabled, model_control_enabled, life_cycle_options.min_compute_capability, enable_model_namespacing, std::move(life_cycle))); *model_repository_manager = std::move(local_manager); // Support loading all models on startup in explicit model control mode with // special startup_model name "*". This does not imply support for pattern // matching in model names. bool load_all_models_on_startup = false; if ((startup_models.find("*") != startup_models.end()) && model_control_enabled) { if (startup_models.size() > 1) { return Status( Status::Code::INVALID_ARG, "Wildcard model name '*' must be the ONLY startup model " "if specified at all."); } load_all_models_on_startup = true; } bool all_models_polled = true; if (!model_control_enabled || load_all_models_on_startup) { // only error happens before model load / unload will be return // model loading / unloading error will be printed but ignored RETURN_IF_ERROR( (*model_repository_manager)->PollAndUpdateInternal(&all_models_polled)); } else { // Load each specified startup_model std::unordered_map> models; for (const auto& model_name : startup_models) { models[model_name]; } RETURN_IF_ERROR( (*model_repository_manager) ->LoadUnloadModels( models, ActionType::LOAD, false, &all_models_polled)); } if (!all_models_polled) { return Status(Status::Code::INTERNAL, "failed to load all models"); } // Some models may failed to be loaded after model manager is created, // return proper error and let function caller decide whether to proceed. for (const auto& model : (*model_repository_manager)->infos_) { const auto version_states = (*model_repository_manager) ->model_life_cycle_->VersionStates(model.first); // Return general error message, detail of each model's loading state // is logged separately. if (version_states.empty()) { return Status(Status::Code::INTERNAL, "failed to load all models"); } for (const auto& state : version_states) { if (state.second.first != ModelReadyState::READY) { return Status(Status::Code::INTERNAL, "failed to load all models"); } } } return Status::Success; } Status ModelRepositoryManager::PollAndUpdate() { if (!polling_enabled_) { return Status(Status::Code::UNAVAILABLE, "polling is disabled"); } bool all_models_polled; return PollAndUpdateInternal(&all_models_polled); } Status ModelRepositoryManager::PollAndUpdateInternal(bool* all_models_polled) { // Serialize all operations that change model state std::lock_guard lock(mu_); std::set added, deleted, modified, unmodified; // We don't modify 'infos_' in place to minimize how long we need to // hold the lock and also prevent any partial changes to do an error // during processing. ModelInfoMap new_infos; // Each subdirectory of repository path is a model directory from // which we read the model configuration. std::unordered_map> model_names; RETURN_IF_ERROR(Poll( model_names, &added, &deleted, &modified, &unmodified, &new_infos, all_models_polled)); // Anything in 'infos_' that is not in "added", "modified", or // "unmodified" is deleted. for (const auto& pr : infos_) { if ((added.find(pr.first) == added.end()) && (modified.find(pr.first) == modified.end()) && (unmodified.find(pr.first) == unmodified.end())) { deleted.insert(pr.first); } } // Nothing to do if no model adds, deletes or modifies. if (added.empty() && deleted.empty() && modified.empty()) { return Status::Success; } infos_.Swap(new_infos); dependency_graph_.UpdateGraph(infos_, added, deleted, modified); for (const auto& name : deleted) { model_life_cycle_->AsyncUnload(name); } // model loading / unloading error will be printed but ignored LoadModelByDependency(&dependency_graph_, &infos_); return Status::Success; } std::map ModelRepositoryManager::LoadModelByDependency( DependencyGraph* dependency_graph, ModelInfoMap* infos) { // [FIXME] This function involves iterative interaction between // ModelLifeCycle object and DependencyGraph object, should better // encapsulate the interaction: // Each iteration: // - Check dependency graph for nodes that are ready for lifecycle changes: // - load if all dependencies are satisfied and the node is 'healthy' // - unload otherwise (should revisit this, logically will only happen in // ensemble, the ensemble is requested to be re-loaded, at this point // it is too late to revert model changes so the ensemble will not be // available afterwards. In other words, ensemble is not covered in // the model availability claim: if re-load results in model not // available, the model state will be reverted) // - Apply changes to lifecycle // - Reflect the lifecycle change result to the dependency graph so that the // downstream nodes may be ready for lifecycle changes. // Repeat until no more changes can be made. std::map res; struct ModelState { ModelState(DependencyNode* node) : node_(node), status_(Status::Success) {} DependencyNode* node_; Status status_; std::promise ready_; }; NodeSet loaded_models; auto set_pair = ModelsToLoadUnload(loaded_models, res, dependency_graph); // Loop until all model are loaded / unloaded while ((!set_pair.first.empty()) || (!set_pair.second.empty())) { loaded_models.clear(); // Unload invalid models first for (auto& invalid_model : set_pair.second) { model_life_cycle_->AsyncUnload(invalid_model->model_id_); res[invalid_model->model_id_] = invalid_model->status_; LOG_ERROR << invalid_model->status_.AsString(); invalid_model->loaded_versions_ = std::set(); loaded_models.emplace(invalid_model); } // load valid models and wait for load results std::vector> model_states; for (auto& valid_model : set_pair.first) { model_states.emplace_back(new ModelState(valid_model)); auto model_state = model_states.back().get(); const auto itr = infos->Find(valid_model->model_id_); auto status = model_life_cycle_->AsyncLoad( valid_model->model_id_, itr->second->model_path_, valid_model->model_config_, itr->second->is_config_provided_, itr->second->prev_mtime_ns_, itr->second->mtime_nsec_, itr->second->agent_model_list_, [model_state](Status load_status) { model_state->status_ = load_status; model_state->ready_.set_value(); }); if (!status.IsOk()) { model_state->status_ = status; model_state->ready_.set_value(); LOG_ERROR << "failed to load model '" << valid_model->model_id_.str() << "': " << status.Message(); } loaded_models.emplace(valid_model); } for (auto& model_state : model_states) { model_state->ready_.get_future().wait(); res[model_state->node_->model_id_] = model_state->status_; const auto version_state = model_life_cycle_->VersionStates(model_state->node_->model_id_); model_state->node_->loaded_versions_.clear(); for (const auto& vs : version_state) { if (vs.second.first == ModelReadyState::READY) { model_state->node_->loaded_versions_.emplace(vs.first); } } // If the model failed to load, should revert the timestamp to // ensure the next load request will attempt to load the model again // for operation idempotence. See comment on 'infos_' if (!model_state->status_.IsOk()) { auto& model_info = infos->Find(model_state->node_->model_id_)->second; model_info->mtime_nsec_ = model_info->prev_mtime_ns_; } } set_pair = ModelsToLoadUnload(loaded_models, res, dependency_graph); } // Clear temporary stored agent model list after all loads are triggered for (auto& info : *infos) { info.second->agent_model_list_.reset(); } return res; } Status ModelRepositoryManager::LoadUnloadModel( const std::unordered_map< std::string, std::vector>& models, const ActionType type, const bool unload_dependents) { if (!model_control_enabled_) { return Status( Status::Code::UNAVAILABLE, "explicit model load / unload is not allowed if polling is enabled"); } if (models.size() > 1) { return Status( Status::Code::UNSUPPORTED, "explicit load / unload multiple models is not currently supported"); } const auto& model_name = models.begin()->first; // Need ModelIdentifier to retrieve model state in lifecycle object, // which will not be available after graph update. So make a copy first // to check if unload works as intended. std::set to_be_unloaded_ids; if (type == ActionType::UNLOAD) { std::lock_guard lock(mu_); // guard global_map_ const auto& git = global_map_.find(model_name); if (git != global_map_.end()) { to_be_unloaded_ids = git->second; } } bool polled = true; bool no_parallel_conflict = true; do { // Function will block during a conflict, so safe to retry immediately. RETURN_IF_ERROR(LoadUnloadModels( models, type, unload_dependents, &polled, &no_parallel_conflict)); } while (!no_parallel_conflict); // Check if model is loaded / unloaded properly if (!polled) { return Status( Status::Code::INTERNAL, "failed to load '" + model_name + "', failed to poll from model repository"); } // Use global map to retrieve the set of models being updated with the given // name if (type == ActionType::LOAD) { std::lock_guard lock(mu_); // guard global_map_ and infos_ const auto& git = global_map_.find(model_name); if (git == global_map_.end()) { return Status( Status::Code::INTERNAL, "failed to load '" + model_name + "', unexpected miss in global map"); } for (const auto& model_id : git->second) { const auto version_states = model_life_cycle_->VersionStates(model_id); if (version_states.empty()) { return Status( Status::Code::INTERNAL, "failed to load '" + model_name + "', no version is available"); } auto it = infos_.Find(model_id); if (it == infos_.end()) { return Status( Status::Code::INTERNAL, "failed to load '" + model_name + "', failed to poll from model repository"); } } } else { for (const auto& model_id : to_be_unloaded_ids) { std::string ready_version_str; const auto version_states = model_life_cycle_->VersionStates(model_id); for (const auto& version_state : version_states) { if (version_state.second.first == ModelReadyState::READY) { ready_version_str += std::to_string(version_state.first); ready_version_str += ","; } } if (!ready_version_str.empty()) { ready_version_str.pop_back(); return Status( Status::Code::INTERNAL, "failed to unload '" + model_name + "', versions that are still available: " + ready_version_str); } } } return Status::Success; } Status ModelRepositoryManager::LoadUnloadModels( const std::unordered_map< std::string, std::vector>& models, const ActionType type, const bool unload_dependents, bool* all_models_polled, bool* no_parallel_conflict) { *all_models_polled = true; if (no_parallel_conflict != nullptr) { *no_parallel_conflict = true; } std::unique_lock lock(mu_); // Prepare new model infos related to file system accordingly, do not write // yet until committed to load/unload the models. // Also poll the added, deleted, modified, unmodified models. ModelInfoMap new_infos(infos_); std::set added, deleted, modified, unmodified; switch (type) { case ActionType::LOAD: RETURN_IF_ERROR(PollModels( models, &added, &deleted, &modified, &unmodified, &new_infos, all_models_polled)); break; case ActionType::UNLOAD: for (const auto& model : models) { // Within the class, model is referred by ModelIdentifier, so it needs // to check global map to find identifier associated with the name. auto git = global_map_.find(model.first); if (git != global_map_.end()) { for (const auto& mid : git->second) { deleted.insert(mid); } } } break; default: return Status(Status::Code::INTERNAL, "Invalid action type"); } std::set deleted_dependents; // Make a copy of the dependency graph for rollback if there is a collision // between the models to be loaded/unloaded by this thread with another // thread actively loading/unloading models. The global map referenced by the // dependency graph is managed by this object, which is the responsibility of // this object to make a copy. std::unordered_map> new_global_map( global_map_); DependencyGraph new_dependency_graph(dependency_graph_, &new_global_map); // Update the new dependency graph, and get models affected by the update. std::set affected_models = new_dependency_graph.UpdateGraph( new_infos, added, deleted, modified, unload_dependents ? &deleted_dependents : nullptr); affected_models.insert(unmodified.begin(), unmodified.end()); // Check for collision in affected models and try to lock those models. std::shared_ptr retry_notify_cv; auto conflict_model = new_dependency_graph.LockNodes(affected_models, &retry_notify_cv); if (conflict_model) { // A collision is found. Since info is only written to local copy, so it is // safe to rollback by simply returning the conflict information. LOG_VERBOSE(2) << "Load/Unload conflict '" << conflict_model->str() << "'"; if (no_parallel_conflict != nullptr) { *no_parallel_conflict = false; retry_notify_cv->wait(lock); return Status::Success; } return Status( Status::Code::INTERNAL, "a related model '" + conflict_model->str() + "' to a load/unload request is currently loading or unloading"); } // The models are in 'deleted' either when they are asked to be unloaded or // they are not found / are duplicated across all model repositories. // Explicitly remove them from new infos. for (const auto& name : (unload_dependents ? deleted_dependents : deleted)) { new_infos.Erase(name); } // At this point, it is committed to load/unload the models, so write the // local copy into this object. The state of the models cannot be modified by // another thread while they are locked. Thus, it is safe to release the // mutex while actively loading/unloading models, but the mutex must be // re-held when reading/updating any model info to avoid any corruption. infos_ = new_infos; global_map_ = new_global_map; dependency_graph_.Assign(new_dependency_graph, &global_map_); // Unlock the affected models. // This serve 2 purposes: // 1. The unlocked state will be writeback to the dependency graph owned by // this object, after the load/unload is completed. // 2. It doubles as an indicator that all the locked models will be handled // by another thread, and this thread must not attempt to load/unload // those locked models. new_dependency_graph.UnlockNodes(affected_models); // Release the mutex for parallel loading/unloading lock.unlock(); // Unload 'deleted' models. for (const auto& name : (unload_dependents ? deleted_dependents : deleted)) { model_life_cycle_->AsyncUnload(name); } // Load/Unload the models affected const auto& load_status = LoadModelByDependency(&new_dependency_graph, &new_infos); // Re-held the mutex after loading/unloading lock.lock(); // Write updated info back to this object after model load/unload. infos_.Writeback(new_infos, affected_models); dependency_graph_.Writeback(new_dependency_graph, affected_models); // Check the load status of the requested models. if (type == ActionType::LOAD) { std::string load_error_message = ""; for (const auto& model : models) { auto git = global_map_.find(model.first); if (git == global_map_.end()) { // skip the model name that is not found, requested name didn't pass // polling continue; } for (const auto& model_id : git->second) { auto it = load_status.find(model_id); // If 'model.first' not in load status, it means the (re-)load is not // necessary because there is no change in the model's directory if ((it != load_status.end()) && !it->second.IsOk()) { load_error_message += ("load failed for model '" + model_id.str() + "': " + it->second.Message() + "\n"); } } } if (!load_error_message.empty()) { return Status(Status::Code::INVALID_ARG, load_error_message); } } return Status::Success; } Status ModelRepositoryManager::PollModels( const std::unordered_map< std::string, std::vector>& models, std::set* added, std::set* deleted, std::set* modified, std::set* unmodified, ModelInfoMap* infos, bool* all_models_polled) { *all_models_polled = true; std::set checked_models; auto current_models = models; ModelInfoMap new_infos; while (!current_models.empty()) { bool polled = true; RETURN_IF_ERROR(Poll( current_models, added, deleted, modified, unmodified, &new_infos, &polled)); *all_models_polled &= polled; // A polling chain is needed if the polled model is ensemble, so that all // (up-to-date) composing models are exposed to Triton. std::set next_models; // The intention here is to iteratively expand the collection of models // to be polled for this load operation, if ensemble is involved, until // the collection is self-contained. // [FIXME] Every Poll() above add new entries in existing 'new_infos' // (and 'added' etc.), so iterating on the whole structure means // re-examination on certain entries. std::set temp_checked; for (auto& info : new_infos) { // if the model is not "checked", it is model polled at current // iteration, check config for more models to be polled const bool checked = (checked_models.find(info.first.name_) != checked_models.end()); if (!checked) { // Don't add the model name to 'checked_models' as there may be same // name model polled but with different namespace temp_checked.emplace(info.first.name_); const auto& config = info.second->model_config_; if (config.has_ensemble_scheduling()) { for (const auto& step : config.ensemble_scheduling().step()) { next_models.emplace(step.model_name()); } } } } AddToSet(temp_checked, &checked_models); // trim the model if it has been checked current_models.clear(); for (const auto& name : next_models) { const bool checked = (checked_models.find(name) != checked_models.end()); if (!checked) { current_models[name]; } } } // After all polls, only models in the initial set ('models') are // explicitly loaded. for (auto& info : new_infos) { info.second->explicitly_load_ = (models.find(info.first.name_) != models.end()); } // Only update the infos when all validation is completed for (const auto& model_name : *added) { auto nitr = new_infos.Find(model_name); infos->Emplace(model_name, std::move(nitr->second)); } for (const auto& model_name : *modified) { auto nitr = new_infos.Find(model_name); auto itr = infos->Find(model_name); // Also check the 'explicitly_load_' in previous snapshot, if the model // has been explicitly loaded, we shouldn't change its state as it may be // polled as composing model and flag is set to false. nitr->second->explicitly_load_ |= itr->second->explicitly_load_; itr->second = std::move(nitr->second); } return Status::Success; } Status ModelRepositoryManager::UnloadAllModels() { // Find all the models to be unloaded. std::unordered_map> models; { std::lock_guard lock(mu_); for (const auto& pair : infos_) { // Only the model name is needed. models[pair.first.name_]; } } // Unload all models found. bool polled; return LoadUnloadModels( models, ActionType::UNLOAD, true /* unload_dependents */, &polled, nullptr /* no_parallel_conflict */); } Status ModelRepositoryManager::StopAllModels() { return model_life_cycle_->StopAllModels(); } const std::set> ModelRepositoryManager::InflightStatus() { return model_life_cycle_->InflightStatus(); } size_t ModelRepositoryManager::BackgroundModelsSize() { return model_life_cycle_->BackgroundModelsSize(); } const ModelStateMap ModelRepositoryManager::LiveModelStates(bool strict_readiness) { return model_life_cycle_->LiveModelStates(strict_readiness); } const ModelStateMap ModelRepositoryManager::ModelStates() { return model_life_cycle_->ModelStates(); } const VersionStateMap ModelRepositoryManager::VersionStates(const std::string& model_name) { ModelIdentifier model_id("", model_name); if (find_identifier_fn_(model_name, &model_id).IsOk()) { return model_life_cycle_->VersionStates(model_id); } return {}; } Status ModelRepositoryManager::ModelState( const std::string& model_name, const int64_t model_version, ModelReadyState* state) { ModelIdentifier model_id("", model_name); RETURN_IF_ERROR(find_identifier_fn_(model_name, &model_id)); return model_life_cycle_->ModelState(model_id, model_version, state); } Status ModelRepositoryManager::RepositoryIndex( const bool ready_only, std::vector* index) { std::set seen_models; std::set duplicate_models; for (const auto& repository_path : repository_paths_) { const std::string model_namespace = (enable_model_namespacing_ ? repository_path : ""); // For any mapped models in this repository, save the mapping // from their subdirectory name to model name. std::map models_in_repo; for (const auto& mapping_it : model_mappings_) { if (mapping_it.second.first == repository_path) { models_in_repo.emplace( BaseName(mapping_it.second.second), mapping_it.first); } } std::set subdirs; RETURN_IF_ERROR(GetDirectorySubdirs(repository_path, &subdirs)); for (const auto& subdir : subdirs) { auto model_id = ModelIdentifier(model_namespace, subdir); auto model_it = models_in_repo.find(subdir); if (model_it != models_in_repo.end()) { model_id.name_ = model_it->second; } if (seen_models.find(model_id) != seen_models.end()) { duplicate_models.insert(model_id); } seen_models.insert(model_id); } } ModelStateMap states = ModelStates(); for (const auto& model_id : seen_models) { // If the same model appears in multiple repositories then show it // as unavailable since duplicate models are not allowed to load. if (duplicate_models.find(model_id) != duplicate_models.end()) { index->emplace_back( model_id, -1 /* version */, ModelReadyState::UNAVAILABLE, MODEL_READY_REASON_DUPLICATE); continue; } // If there is any version/state/reason associated with the model // then include that in the index. auto sitr = states.find(model_id); if (sitr == states.end()) { if (!ready_only) { index->emplace_back(model_id); } } else { for (const auto& pr : sitr->second) { if (!ready_only || (pr.second.first == ModelReadyState::READY)) { index->emplace_back( model_id, pr.first, pr.second.first, pr.second.second); } } } } return Status::Success; } Status ModelRepositoryManager::GetModel( const std::string& model_name, const int64_t model_version, std::shared_ptr* model) { ModelIdentifier model_id("", model_name); RETURN_IF_ERROR(find_identifier_fn_(model_name, &model_id)); return GetModel(model_id, model_version, model); } Status ModelRepositoryManager::GetModel( const ModelIdentifier& model_id, const int64_t model_version, std::shared_ptr* model) { Status status = model_life_cycle_->GetModel(model_id, model_version, model); if (!status.IsOk()) { model->reset(); status = Status( status.ErrorCode(), "Request for unknown model: " + status.Message()); } return status; } Status ModelRepositoryManager::FindModelIdentifier( const std::string& model_name, ModelIdentifier* model_id) { auto git = global_map_.find(model_name); if (git == global_map_.end()) { return Status( Status::Code::INVALID_ARG, "Request for unknown model: '" + model_name + "' is not found"); } switch (git->second.size()) { case 0: return Status( Status::Code::NOT_FOUND, "Identifier of model '" + model_name + "' is not found in global map"); break; case 1: { *model_id = *git->second.begin(); break; } default: return Status( Status::Code::INVALID_ARG, "There are " + std::to_string(git->second.size()) + " identifiers of model '" + model_name + "' in global map, model namespace must be provided to resolve " "ambiguity."); break; } return Status::Success; } Status ModelRepositoryManager::Poll( const std::unordered_map< std::string, std::vector>& models, std::set* added, std::set* deleted, std::set* modified, std::set* unmodified, ModelInfoMap* updated_infos, bool* all_models_polled) { *all_models_polled = true; // empty path is the special case to indicate the model should be loaded // from override file content in 'models'. std::map model_to_path; // If no model is specified, poll all models in all model repositories. // Otherwise, only poll the specified models if (models.empty()) { std::set duplicated_models; for (const auto& repository_path : repository_paths_) { // collapse namespace based on whether it is enabled const std::string model_namespace = (enable_model_namespacing_ ? repository_path : ""); // Model name mapping is not checked in direct polling, because // it is set when registering additional model repository which is limited // to explicit mode, so there is no intersection of the code paths. // [TODO] revisit the above statement. std::set subdirs; Status status = GetDirectorySubdirs(repository_path, &subdirs); if (!status.IsOk()) { LOG_ERROR << "failed to poll model repository '" << repository_path << "': " << status.Message(); *all_models_polled = false; } else { for (const auto& subdir : subdirs) { const ModelIdentifier model_identifier(model_namespace, subdir); if (!model_to_path .emplace( model_identifier, JoinPath({repository_path, subdir})) .second) { duplicated_models.insert(model_identifier); *all_models_polled = false; } } } } // If the model is not unique, mark as deleted to unload it for (const auto& model : duplicated_models) { model_to_path.erase(model); deleted->insert(model); LOG_ERROR << "failed to poll model '" << model << "': not unique across all model repositories"; } } // If models are specified, this is explicit model control mode. else { for (const auto& model : models) { // Skip repository polling if override model files if (ModelDirectoryOverride(model.second)) { // [TODO] once namespace is exposed to external user, should allow and // check if the namespace is provided as well,. model_to_path.emplace(ModelIdentifier("", model.first), ""); continue; } // Check model mapping first to see if matching model to load. bool exists = false; auto model_it = model_mappings_.find(model.first); if (model_it != model_mappings_.end()) { bool exists_in_this_repo = false; auto full_path = model_it->second.second; Status status = FileExists(full_path, &exists_in_this_repo); if (!status.IsOk()) { LOG_ERROR << "failed to poll mapped path '" << full_path << "' for model '" << model.first << "': " << status.Message(); *all_models_polled = false; } if (exists_in_this_repo) { // collapse namespace based on whether it is enabled const std::string model_namespace = (enable_model_namespacing_ ? model_it->second.first : ""); model_to_path.emplace( ModelIdentifier(model_namespace, model.first), model_it->second.second); exists = true; } else { LOG_ERROR << "mapped path '" << full_path << "' does not exist for model '" << model.first << "'"; exists = false; } } else { for (const auto& repository_path : repository_paths_) { bool exists_in_this_repo = false; const auto full_path = JoinPath({repository_path, model.first}); Status status = FileExists(full_path, &exists_in_this_repo); if (!status.IsOk()) { LOG_ERROR << "failed to poll model repository '" << repository_path << "' for model '" << model.first << "': " << status.Message(); *all_models_polled = false; } else if (exists_in_this_repo) { // [FIXME] Revisit model_mapping: The intention is to check if // the model name is mapped to something other than the folder name. // If so, we shouldn't consider the model is found in this repo. // If the intention is correct, the current check is not accurate, // it skips the repo as long as at least one model within is mapped. // // Check to make sure this directory is not mapped. // If mapped, continue to next repository path. bool mapped = false; for (auto const& mapping : model_mappings_) { if (mapping.second.second == full_path) { mapped = true; break; } } if (mapped) { continue; } // collapse namespace based on whether it is enabled const std::string model_namespace = (enable_model_namespacing_ ? repository_path : ""); auto res = model_to_path.emplace( ModelIdentifier(model_namespace, model.first), JoinPath({repository_path, model.first})); if (res.second) { exists = true; } else { exists = false; model_to_path.erase(res.first); LOG_ERROR << "failed to poll model '" << model.first << "': not unique across all model repositories"; break; } } // else: model doesn't exist in this repository_path } } // For an explicitly specified model that does not exist, we do not mark // it as deleted, we simply mark that we could not poll all models. if (!exists) { *all_models_polled = false; LOG_ERROR << "failed to poll model '" << model.first << "': model not found in any model repository."; } } } // Poll each of the models. If error happens during polling the model, // its state will fallback to the state before the polling. for (const auto& pair : model_to_path) { std::unique_ptr model_info; const auto& model_name = pair.first.name_; // Load with parameters will be appiled to all models with the same // name (namespace can be different), unless namespace is specified // in the future. const auto& mit = models.find(model_name); static std::vector empty_params; auto status = InitializeModelInfo( pair.first, pair.second, ((mit == models.end()) ? empty_params : mit->second), &model_info); const auto& iitr = infos_.Find(pair.first); const bool invalid_add = (!status.IsOk()) && (iitr == infos_.end()); if (!invalid_add) { const auto& ret = updated_infos->Emplace(pair.first, nullptr); if (!ret.second) { return Status( Status::Code::ALREADY_EXISTS, "unexpected model info for model '" + pair.first.str() + "'"); } // Classify load state and set updated info if (model_info == nullptr) { ret.first->second.reset(new ModelInfo(*iitr->second)); unmodified->insert(pair.first); } else { ret.first->second = std::move(model_info); if (iitr != infos_.end()) { modified->insert(pair.first); } else { added->insert(pair.first); } } } if (!status.IsOk()) { LOG_ERROR << "Poll failed for model directory '" << pair.first << "': " << status.Message(); *all_models_polled = false; } } return Status::Success; } bool ModelRepositoryManager::ModelDirectoryOverride( const std::vector& model_params) { for (const auto& param : model_params) { if (param->Name().rfind(file_prefix, 0) == 0) { // param name starts with prefix if user provides override file return true; } } return false; } Status ModelRepositoryManager::InitializeModelInfo( const ModelIdentifier& model_id, const std::string& path, const std::vector& params, std::unique_ptr* info) { std::unique_ptr linfo(new ModelInfo()); linfo->model_path_ = path; bool unmodified = false; const auto iitr = infos_.Find(model_id); // Set 'prev_mtime_ns_' if there is existing ModelInfo if (iitr != infos_.end()) { linfo->prev_mtime_ns_ = iitr->second->mtime_nsec_; } else { linfo->prev_mtime_ns_ = ModelTimestamp(); } // Set 'mtime_nsec_' and override 'model_path_' if current path is empty // (file override is specified) if (linfo->model_path_.empty()) { // Need to localize the override files, use repo agent to manage // the lifecycle of the localized files std::shared_ptr localize_agent(new LocalizeRepoAgent()); std::unique_ptr localize_agent_model; RETURN_IF_ERROR(TritonRepoAgentModel::Create( TRITONREPOAGENT_ARTIFACT_FILESYSTEM, "", inference::ModelConfig(), localize_agent, {}, &localize_agent_model)); // Set agent model state so the repo agent can access the encoded files // Using const_cast here but we are safe as the RepoAgent will not // modify the state localize_agent_model->SetState( const_cast(reinterpret_cast(¶ms))); RETURN_IF_ERROR( localize_agent_model->InvokeAgent(TRITONREPOAGENT_ACTION_LOAD)); const char* location; TRITONREPOAGENT_ArtifactType type; RETURN_IF_ERROR(localize_agent_model->Location(&type, &location)); // For file override, set 'mtime_nsec_' to minimum value so that // the next load without override will trigger re-load to undo // the override while the local files may still be unchanged. linfo->mtime_nsec_ = ModelTimestamp(); linfo->model_path_ = location; linfo->model_config_path_ = JoinPath({location, kModelConfigPbTxt}); linfo->agent_model_list_.reset(new TritonRepoAgentModelList()); linfo->agent_model_list_->AddAgentModel(std::move(localize_agent_model)); } else { linfo->model_config_path_ = GetModelConfigFullPath(linfo->model_path_, model_config_name_); // Model is not loaded. if (iitr == infos_.end()) { linfo->mtime_nsec_ = ModelTimestamp(linfo->model_path_, linfo->model_config_path_); } else { // Check the current timestamps to determine if model actually has been // modified linfo->mtime_nsec_ = linfo->prev_mtime_ns_; ModelTimestamp new_mtime_ns = ModelTimestamp(linfo->model_path_, linfo->model_config_path_); unmodified = !linfo->mtime_nsec_.IsModified(new_mtime_ns); linfo->mtime_nsec_ = new_mtime_ns; } } // Set 'model_config_' bool parsed_config = false; // Check if there is config override for (const auto& override_parameter : params) { if ((override_parameter->Name() == "config") && (override_parameter->Type() == TRITONSERVER_PARAMETER_STRING)) { // When override happens, set 'mtime_nsec_' to minimum value so that // the next load without override will trigger re-load to undo // the override while the local files may still be unchanged. auto time_since_epoch = std::chrono::system_clock::now().time_since_epoch(); linfo->mtime_nsec_.SetModelConfigModifiedTime( std::chrono::duration_cast(time_since_epoch) .count()); unmodified = false; const std::string& override_config = override_parameter->ValueString(); auto err = JsonToModelConfig( override_config, 1 /* config_version */, &linfo->model_config_); if (!err.IsOk()) { return Status( Status::Code::INVALID_ARG, "Invalid config override: " + std::string(err.Message())); } parsed_config = true; break; } else if (override_parameter->Name().rfind(file_prefix, 0) != 0) { return Status( Status::Code::INVALID_ARG, "Unrecognized load parameter '" + override_parameter->Name() + "' with type '" + TRITONSERVER_ParameterTypeString(override_parameter->Type()) + "'"); } } // Polling model is considered unmodified by this point and can be returned // with info == nullptr if (unmodified) { return Status::Success; } // Create the associated repo agent models when a model is to be loaded, // this must be done before normalizing model config as agents might // redirect to use the model config at a different location if (!parsed_config) { const auto config_path = linfo->model_config_path_; bool model_config_exists = false; RETURN_IF_ERROR(FileExists(config_path, &model_config_exists)); // model config can be missing if auto fill is set if (autofill_ && !model_config_exists) { linfo->model_config_.Clear(); } else { RETURN_IF_ERROR(ReadTextProto(config_path, &linfo->model_config_)); parsed_config = true; } } if (parsed_config) { RETURN_IF_ERROR(CreateAgentModelListWithLoadAction( linfo->model_config_, linfo->model_path_, model_config_name_, &linfo->agent_model_list_)); if (linfo->agent_model_list_ != nullptr) { // Get the latest repository path const char* location; TRITONREPOAGENT_ArtifactType artifact_type; RETURN_IF_ERROR(linfo->agent_model_list_->Back()->Location( &artifact_type, &location)); auto latest_path = std::string(location); linfo->model_path_ = latest_path; // [TODO] should try to read the config again at the latest location? } } linfo->is_config_provided_ = parsed_config; // [FIXME] better document / interact with config, the actual config to be // associated with the model is not finalized and valid for read until now. // Repo agent / config overwrite may provide different configs than // the one in storage. // Try to automatically generate missing parts of the model // configuration (autofill) that don't require model detail RETURN_IF_ERROR(GetNormalizedModelConfig( model_id.name_, linfo->model_path_, min_compute_capability_, &linfo->model_config_)); // Note that the model inputs and outputs are not validated until // the model model is initialized as they may not be auto-completed // until model is initialized. RETURN_IF_ERROR( ValidateModelConfig(linfo->model_config_, min_compute_capability_)); if (!autofill_) { RETURN_IF_ERROR(ValidateModelIOConfig(linfo->model_config_)); } // If the model is mapped, update its config name based on the // mapping. if (model_mappings_.find(model_id.name_) != model_mappings_.end()) { linfo->model_config_.set_name(model_id.name_); } else { // If there is no model mapping, make sure the name of the model // matches the name of the directory. This is a somewhat arbitrary // requirement but seems like good practice to require it of the user. // It also acts as a check to make sure we don't have two different // models with the same name. if (linfo->model_config_.name() != model_id.name_) { return Status( Status::Code::INVALID_ARG, "unexpected directory name '" + model_id.name_ + "' for model '" + linfo->model_config_.name() + "', directory name must equal model name"); } } *info = std::move(linfo); return Status::Success; } Status ModelRepositoryManager::RegisterModelRepository( const std::string& repository, const std::unordered_map& model_mapping) { if (!model_control_enabled_) { return Status( Status::Code::UNSUPPORTED, "repository registration is not allowed if model control mode is not " "EXPLICIT"); } bool is_directory = false; auto status = IsDirectory(repository, &is_directory); if (!status.IsOk() || !is_directory) { return Status( Status::Code::INVALID_ARG, (std::string("failed to register '") + repository + "', repository not found") .c_str()); } { // Serialize all operations that change model state std::lock_guard lock(mu_); // Check repository and mapped models do not yet exist. if (repository_paths_.find(repository) != repository_paths_.end()) { return Status( Status::Code::ALREADY_EXISTS, "model repository '" + repository + "' has already been registered"); } for (const auto& mapping : model_mapping) { if (model_mappings_.find(mapping.first) != model_mappings_.end()) { return Status( Status::Code::ALREADY_EXISTS, (std::string("failed to register '") + mapping.first + "', there is a conflicting mapping for '" + std::string(mapping.first) + "'") .c_str()); } } repository_paths_.emplace(repository); for (const auto& mapping : model_mapping) { model_mappings_.emplace( mapping.first, std::make_pair(repository, JoinPath({repository, mapping.second}))); } } LOG_INFO << "Model repository registered: " << repository; return Status::Success; } Status ModelRepositoryManager::UnregisterModelRepository(const std::string& repository) { if (!model_control_enabled_) { return Status( Status::Code::UNSUPPORTED, "repository unregistration is not allowed if model control mode is not " "EXPLICIT"); } { std::lock_guard lock(mu_); if (repository_paths_.erase(repository) != 1) { return Status( Status::Code::INVALID_ARG, "failed to unregister '" + repository + "', repository not found"); } std::set models_to_delete; for (auto const& mapping : model_mappings_) { if (mapping.second.first == repository) { models_to_delete.insert(mapping.first); } } for (auto const& model : models_to_delete) { model_mappings_.erase(model); } } LOG_INFO << "Model repository unregistered: " << repository; return Status::Success; } std::pair ModelRepositoryManager::ModelsToLoadUnload( const NodeSet& loaded_models, const std::map& model_load_status, DependencyGraph* dependency_graph) { // std::pair res; // first call to this function if (loaded_models.empty()) { for (auto& pair : *(dependency_graph->MutableNodes())) { auto node = pair.second.get(); // only care about nodes that are affected by the update // nodes is locked if and only if it is being updated by another thread if (!node->checked_ && !node->is_locked_) { if (CheckNode(node, model_load_status)) { if (node->status_.IsOk()) { res.first.emplace(node); } else { res.second.emplace(node); } } } } } else { for (const auto& model : loaded_models) { for (auto node : model->downstreams_) { // only care about nodes that are affected by the update // nodes is locked if and only if it is being updated by another thread if (!node->checked_ && !node->is_locked_) { if (CheckNode(node, model_load_status)) { if (node->status_.IsOk()) { res.first.emplace(node); } else { res.second.emplace(node); } } } } } } for (auto& node : res.first) { node->checked_ = true; } for (auto& node : res.second) { node->checked_ = true; } return res; } bool ModelRepositoryManager::CheckNode( DependencyNode* node, const std::map& model_load_status) { bool node_ready = true; // if the node is in invalid status, mark as ready as we know // it should not be loaded if (node->status_.IsOk()) { for (auto& upstream : node->upstreams_) { if (!upstream.first->checked_) { node_ready = false; break; } if (!upstream.first->status_.IsOk()) { node->status_ = Status( Status::Code::INVALID_ARG, "ensemble '" + node->model_id_.str() + "' depends on '" + upstream.first->model_id_.str() + "' which is not valid. Model '" + upstream.first->model_id_.str() + "' failed with error: " + upstream.first->status_.Message()); } else if (upstream.first->loaded_versions_.empty()) { auto model_load_stat = model_load_status.find(upstream.first->model_id_); if ((model_load_stat != model_load_status.end()) && model_load_stat->second.IsOk()) { node->status_ = Status( Status::Code::INVALID_ARG, "ensemble '" + node->model_id_.str() + "' depends on '" + upstream.first->model_id_.str() + "' which has no loaded version."); } else { // [FIXME] See https://jirasw.nvidia.com/browse/DLIS-4459 node->status_ = Status( Status::Code::INVALID_ARG, "ensemble '" + node->model_id_.str() + "' depends on '" + upstream.first->model_id_.str() + "' which has no loaded version. Model '" + upstream.first->model_id_.str() + "' loading failed with error: " + model_load_stat->second.Message()); } } else { for (const auto& required_version : upstream.second) { if (required_version == -1) { continue; } auto it = upstream.first->loaded_versions_.find(required_version); if (it == upstream.first->loaded_versions_.end()) { node->status_ = Status( Status::Code::INVALID_ARG, "ensemble '" + node->model_id_.str() + "' depends on '" + upstream.first->model_id_.str() + "' whose required version " + std::to_string(required_version) + " is not loaded."); } } } if (!node->status_.IsOk()) { break; } } #ifdef TRITON_ENABLE_ENSEMBLE // Validate ensemble config if the node is ready. By this point, the // depending models are loaded and their configs are completed if (node_ready && node->status_.IsOk()) { node->status_ = ValidateEnsembleConfig(this, node); } #endif // TRITON_ENABLE_ENSEMBLE } return node_ready; } std::set ModelRepositoryManager::DependencyGraph::UpdateGraph( const ModelInfoMap& model_infos, const std::set& added, const std::set& deleted, const std::set& modified, std::set* deleted_dependents) { // Update dependency graph in following steps: // 1. Apply all node changes. // 2. Connect the edges of all affected nodes. // 3. Connectivity check. // If the state of a node is changed, all its downstreams will be affected. // deleted, drop from dependency_graph, add to missing_nodes if downstreams is // not empty affected_nodes are all ensembles as only ensembles are depending // on other models std::set affected_nodes, deleted_nodes; const bool cascading_removal = (deleted_dependents != nullptr); std::tie(affected_nodes, deleted_nodes) = RemoveNodes(deleted, cascading_removal); if (cascading_removal) { deleted_dependents->swap(deleted_nodes); } AddToSet(UpdateNodes(modified, model_infos), &affected_nodes); AddToSet(AddNodes(added, model_infos), &affected_nodes); for (auto& node_id : affected_nodes) { ConnectDependencyGraph(node_id); } for (auto& node_id : affected_nodes) { CircularDependencyCheck(node_id); } // Add deleted nodes to affected nodes const auto& deleted_ref = cascading_removal ? *deleted_dependents : deleted_nodes; affected_nodes.insert(deleted_ref.begin(), deleted_ref.end()); return affected_nodes; } std::pair, std::set> ModelRepositoryManager::DependencyGraph::RemoveNodes( const std::set& nodes, const bool cascading_removal) { std::set all_affected_nodes, all_removed_nodes; std::set curr_removal = nodes; while (!curr_removal.empty()) { std::set next_removal; for (const auto& model_id : curr_removal) { const auto affected_nodes = RemoveNode(model_id); // Check if the upstream should be removed as well, // a node should be removed if cascading removal is requested, // was not explicitly loaded and now doesn't have any downstreams. // For the concern that the node may then have downstreams from // 'added' / 'modified' nodes, it is not possible based on the situations // where dependency graph will be updated: // - POLL/NONE : There can be additions and deletions within single // operation, but all nodes are marked explicitly loaded. // - EXPLICIT : Each operation can either be "load" or "unload", so there // will not be bi-directional changes regarding downstreams. if (cascading_removal) { const auto& upstreams = affected_nodes.first; for (const auto& upstream : upstreams) { auto unode = FindNode(upstream, false /* allow_fuzzy_matching */); if (unode && (unode->downstreams_.empty()) && (!unode->explicitly_load_)) { next_removal.emplace(upstream); } } } // The downstreams will need to be re-evaluated once the node // changes are in place const auto& downstreams = affected_nodes.second; // std::set::merge() can be used if move to >= C++17 for (const auto& id : downstreams) { all_affected_nodes.emplace(id); } all_removed_nodes.emplace(model_id); } curr_removal.swap(next_removal); } // Exclude removed nodes from affected nodes. for (const auto& removed_node : all_removed_nodes) { all_affected_nodes.erase(removed_node); } return {std::move(all_affected_nodes), std::move(all_removed_nodes)}; } std::set ModelRepositoryManager::DependencyGraph::UpdateNodes( const std::set& nodes, const ModelInfoMap& model_infos) { std::set updated_nodes; // modified, invalidate (uncheck) all downstreams for (const auto& model_id : nodes) { auto it = nodes_.find(model_id); if (it != nodes_.end()) { auto& node = it->second; UncheckDownstream(node->downstreams_); // remove all upstream reference, because the // config may be changed and should rebuild dependency for (auto& upstream : node->upstreams_) { upstream.first->DisconnectDownstream(node.get()); } for (const auto& model_name : node->missing_upstreams_) { auto mit = missing_nodes_.find(model_name); mit->second.erase(it->first); } // Update model info stored in the node ModelInfo* info = model_infos.At(model_id).get(); node->model_config_ = info->model_config_; node->explicitly_load_ = info->explicitly_load_; node->upstreams_.clear(); node->checked_ = false; node->status_ = Status::Success; updated_nodes.emplace(model_id); } } return updated_nodes; } std::set ModelRepositoryManager::DependencyGraph::AddNodes( const std::set& nodes, const ModelInfoMap& model_infos) { std::set affected_nodes; // added, add to dependency_graph, if in missing_node, invalidate (uncheck) // and associate all downstreams, remove from missing_node for (const auto& model_id : nodes) { std::unique_ptr added_node(new DependencyNode(model_id)); ModelInfo* info = model_infos.At(model_id).get(); added_node->model_config_ = info->model_config_; added_node->explicitly_load_ = info->explicitly_load_; // Check if this model name is needed for some nodes, simply mark those // nodes affected to re-evaluate them later auto it = missing_nodes_.find(model_id.name_); if (it != missing_nodes_.end()) { for (const auto& dependent_node_id : it->second) { auto dependent_node = FindNode(dependent_node_id, false /* allow_fuzzy_matching */); if (dependent_node) { UncheckDownstream({dependent_node}); affected_nodes.emplace(dependent_node_id); } } } // Add nodes to all reference affected_nodes.emplace(model_id); (*global_map_ptr_)[model_id.name_].emplace(model_id); nodes_.emplace(std::make_pair(model_id, std::move(added_node))); } return affected_nodes; } void ModelRepositoryManager::DependencyGraph::ConnectDependencyGraph( const ModelIdentifier& node_id) { // Check the node's model config to determine if it depends on other models // and if those models are present auto updated_node = FindNode(node_id, false /* allow_fuzzy_matching */); updated_node->upstreams_.clear(); updated_node->missing_upstreams_.clear(); updated_node->connected_ = true; if (updated_node->model_config_.has_ensemble_scheduling()) { auto mutable_steps = updated_node->model_config_.mutable_ensemble_scheduling() ->mutable_step(); for (auto& step : *mutable_steps) { // Build model identifier to look for the composing model, // currently using the same namespace as the 'updated_node' for // preferred identifier. // [enhancement] May allow user-specified namespace once allowed in // ensemble config. const ModelIdentifier preferred_id( updated_node->model_id_.namespace_, step.model_name()); auto node = FindNode(preferred_id, true /* allow_fuzzy_matching */); if (node) { // Set 'model_namespace' field in model config to allow direct model // lookup at later stage. step.set_model_namespace(node->model_id_.namespace_); // Add the current node to its upstream node->downstreams_.emplace(updated_node); // Add the upstream to the current node auto res = updated_node->upstreams_.emplace( node, std::set({step.model_version()})); // If map insertion doesn't happen, the same model is required in // different step, insert the version to existing required version set. if (!res.second) { res.first->second.insert(step.model_version()); } } else { updated_node->connected_ = false; updated_node->missing_upstreams_.emplace(step.model_name()); } // If no node is found or the found node is not exact matched, // just record that in 'missing_nodes_' so the current node will // be rechecked whenever a new candidate is added. if (!node || (node->model_id_ != preferred_id)) { missing_nodes_[preferred_id.name_].emplace(updated_node->model_id_); } // Only update status if there is no failure from earlier stages if (updated_node->status_.IsOk() && !updated_node->connected_) { std::string name_list; for (const auto& missing_name : updated_node->missing_upstreams_) { if (!name_list.empty()) { name_list += ", "; } name_list += missing_name; } updated_node->status_ = Status( Status::Code::INVALID_ARG, "ensemble " + updated_node->model_id_.str() + " contains models that are not available or ambiguous: " + name_list); } } } } std::pair, std::set> ModelRepositoryManager::DependencyGraph::RemoveNode( const ModelIdentifier& model_id) { auto it = nodes_.find(model_id); // no-op if not found: "node" has been removed if (it == nodes_.end()) { return {{}, {}}; } std::set upstreams, downstreams; auto& node = it->second; // remove this node from its upstreams for (auto& upstream : node->upstreams_) { auto& upstream_node = upstream.first; upstream_node->DisconnectDownstream(node.get()); upstreams.emplace(upstream_node->model_id_); } // remove this node from its downstreams UncheckDownstream(node->downstreams_); for (auto& downstream : node->downstreams_) { downstream->DisconnectUpstream(node.get()); downstreams.emplace(downstream->model_id_); } // Drop the node from all references and only track it on removed nodes (*global_map_ptr_)[model_id.name_].erase(model_id); for (const auto& model_name : node->missing_upstreams_) { auto mit = missing_nodes_.find(model_name); mit->second.erase(model_id); } removed_nodes_.emplace(std::make_pair(model_id, std::move(node))); nodes_.erase(it); return {std::move(upstreams), std::move(downstreams)}; } ModelRepositoryManager::DependencyNode* ModelRepositoryManager::DependencyGraph::FindNode( const ModelIdentifier& model_id, const bool allow_fuzzy_matching) const { const auto git = nodes_.find(model_id); if (git != nodes_.end()) { return git->second.get(); } else if (allow_fuzzy_matching) { const auto gmit = global_map_ptr_->find(model_id.name_); if ((gmit != global_map_ptr_->end()) && (gmit->second.size() == 1)) { const auto git = nodes_.find(*gmit->second.begin()); if (git != nodes_.end()) { return git->second.get(); } } } return nullptr; } void ModelRepositoryManager::DependencyGraph::UncheckDownstream( const std::set& downstreams) { for (auto& node : downstreams) { if (node->checked_) { node->checked_ = false; node->status_ = Status::Success; UncheckDownstream(node->downstreams_); } } } void ModelRepositoryManager::DependencyGraph::CircularDependencyCheck( const ModelIdentifier& node_id) { auto node = FindNode(node_id, false /* allow_fuzzy_matching */); if (node->status_.IsOk()) { node->status_ = CircularDependencyCheck(node, node); } } Status ModelRepositoryManager::DependencyGraph::CircularDependencyCheck( DependencyNode* current_node, const DependencyNode* start_node) { // Note that traverse order is towards upstream for the correct dependency // ordering. for (auto& upstream : current_node->upstreams_) { auto& upstream_node = upstream.first; if (upstream_node == start_node) { return Status( Status::Code::INVALID_ARG, "circular dependency between ensembles: " + start_node->model_id_.str() + " -> ... -> " + current_node->model_id_.str() + " -> " + start_node->model_id_.str()); } else { // Only return error when detect circular dependency, propagate the error // so all node in the chain will be set. const auto status = CircularDependencyCheck(upstream_node, start_node); if (!status.IsOk()) { current_node->status_ = status; return status; } } } return Status::Success; } ModelRepositoryManager::DependencyGraph::DependencyGraph( const DependencyGraph& rhs, std::unordered_map>* global_map) : global_map_ptr_(global_map), missing_nodes_(rhs.missing_nodes_) { // Copy nodes_ for (auto& pair : rhs.nodes_) { nodes_.emplace(pair.first, std::make_unique(*pair.second)); } // Re-map pointers from rhs to this for (auto& pair : nodes_) { // upstreams_ std::unordered_map> new_upstreams; for (auto& p : pair.second->upstreams_) { new_upstreams.emplace(nodes_.at(p.first->model_id_).get(), p.second); } pair.second->upstreams_.swap(new_upstreams); // downstreams_ std::set new_downstreams; for (auto* node : pair.second->downstreams_) { new_downstreams.emplace(nodes_.at(node->model_id_).get()); } pair.second->downstreams_.swap(new_downstreams); } // Copy removed_nodes_ for (auto& pair : rhs.removed_nodes_) { removed_nodes_.emplace( pair.first, std::make_unique(*pair.second)); } } void ModelRepositoryManager::DependencyGraph::Assign( const DependencyGraph& rhs, std::unordered_map>* global_map) { DependencyGraph tmp(rhs, global_map); Swap(tmp); } void ModelRepositoryManager::DependencyGraph::Swap(DependencyGraph& rhs) { std::swap(global_map_ptr_, rhs.global_map_ptr_); nodes_.swap(rhs.nodes_); missing_nodes_.swap(rhs.missing_nodes_); removed_nodes_.swap(rhs.removed_nodes_); } ModelRepositoryManager::DependencyNode* ModelRepositoryManager::DependencyGraph::GetNode( const ModelIdentifier& model_id) const { const auto it = removed_nodes_.find(model_id); if (it != removed_nodes_.end()) { return it->second.get(); } auto* node = FindNode(model_id, false /* allow_fuzzy_matching */); if (node == nullptr) { throw std::invalid_argument("model_id '" + model_id.str() + "' not found"); } return node; } std::unique_ptr ModelRepositoryManager::DependencyGraph::LockNodes( const std::set& nodes, std::shared_ptr* retry_notify_cv) { for (const auto& model_id : nodes) { auto* node = GetNode(model_id); if (node->is_locked_) { if (retry_notify_cv != nullptr) { *retry_notify_cv = node->retry_notify_cv_; } return std::make_unique(model_id); } node->is_locked_ = true; } return std::unique_ptr(nullptr); } std::unique_ptr ModelRepositoryManager::DependencyGraph::UnlockNodes( const std::set& nodes) { for (const auto& model_id : nodes) { auto* node = GetNode(model_id); if (!node->is_locked_) { return std::make_unique(model_id); } node->is_locked_ = false; } return std::unique_ptr(nullptr); } void ModelRepositoryManager::DependencyGraph::Writeback( const DependencyGraph& updated_dependency_graph, const std::set& affected_models) { for (const auto& model_id : affected_models) { auto* node = GetNode(model_id); auto* updated_node = updated_dependency_graph.GetNode(model_id); // Writeback node->Writeback(*updated_node); // Erase removed nodes at last to complete its lifecycle removed_nodes_.erase(model_id); } } void ModelRepositoryManager::DependencyNode::Writeback( const DependencyNode& updated_dependency_node) { // Writeback status_ = updated_dependency_node.status_; checked_ = updated_dependency_node.checked_; loaded_versions_ = updated_dependency_node.loaded_versions_; is_locked_ = updated_dependency_node.is_locked_; // Notify retry(s) retry_notify_cv_->notify_all(); } ModelRepositoryManager::ModelInfoMap::ModelInfoMap(const ModelInfoMap& rhs) { for (auto& pair : rhs.map_) { map_.emplace(pair.first, std::make_unique(*pair.second)); } } ModelRepositoryManager::ModelInfoMap& ModelRepositoryManager::ModelInfoMap::operator=(const ModelInfoMap& rhs) { ModelInfoMap tmp(rhs); Swap(tmp); return *this; } void ModelRepositoryManager::ModelInfoMap::Writeback( const ModelInfoMap& updated_model_info, const std::set& affected_models) { for (auto& model_id : affected_models) { // An affected model can be deleted, so the presence of the model must be // checked. auto itr = updated_model_info.map_.find(model_id); if (itr != updated_model_info.map_.end()) { auto* updated_info = itr->second.get(); auto* info = map_.at(model_id).get(); // Writeback info->mtime_nsec_ = updated_info->mtime_nsec_; } } } }} // namespace triton::core kytensor-2.49.0.7/core/src/model_repository_manager/model_repository_manager.h000066400000000000000000000757421527052371200277260ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once #include #include #include #include #include #include "infer_parameter.h" #include "model_config.pb.h" #include "model_lifecycle.h" #include "status.h" #include "triton/common/model_config.h" namespace triton { namespace core { class InferenceServer; class Model; // [FIXME] should have separated load / unload functions for clarity enum ActionType { NO_ACTION, LOAD, UNLOAD }; /// Predefined reason strings #define MODEL_READY_REASON_DUPLICATE "model appears in two or more repositories" // Information about timestamps in model directory. class ModelTimestamp { public: ModelTimestamp() {} ModelTimestamp( const std::string& model_dir_path, const std::string& model_config_path); bool IsModified(const ModelTimestamp& new_timestamp) const; bool IsModelVersionModified( const ModelTimestamp& new_timestamp, const int64_t version) const; void SetModelConfigModifiedTime(const int64_t time_ns); private: bool ModelDirectoryPathIsValid(const std::string& path) const; bool ReadModelDirectoryTimestamp( const std::string& model_dir_path, const std::string& model_config_path); bool ReadModelDirectoryContentTimestamps( const std::string& model_dir_path, const std::string& model_config_path); int64_t GetModifiedTime() const; int64_t GetModelVersionModifiedTime(const int64_t version) const; int64_t GetNonModelConfigNorVersionNorDirModifiedTime() const; // Timestamps of all contents in the model directory, i.e. // - "": ns timestamp of model config (directory) // - "1": ns timestamp of model version 1 // - "": ns timestamp of the model directory std::unordered_map model_timestamps_; // If empty, model config is not detected on the model directory. std::string model_config_content_name_; }; /// An object to manage the model repository active in the server. class ModelRepositoryManager { public: // Index information for a model. struct ModelIndex { ModelIndex(const ModelIdentifier& n) : name_only_(true), namespace_(n.namespace_), name_(n.name_), version_(-1), state_(ModelReadyState::UNKNOWN) { } ModelIndex( const ModelIdentifier& n, const int64_t v, const ModelReadyState s, const std::string& r) : name_only_(false), namespace_(n.namespace_), name_(n.name_), version_(v), state_(s), reason_(r) { } const bool name_only_; const std::string namespace_; const std::string name_; const int64_t version_; const ModelReadyState state_; const std::string reason_; }; // Information about the model. struct ModelInfo { ModelInfo( const ModelTimestamp& mtime_nsec, const ModelTimestamp& prev_mtime_ns, const std::string& model_path, const std::string& model_config_path) : mtime_nsec_(mtime_nsec), prev_mtime_ns_(prev_mtime_ns), explicitly_load_(true), model_path_(model_path), model_config_path_(model_config_path), is_config_provided_(false) { } ModelInfo() : explicitly_load_(true), is_config_provided_(false) {} ModelTimestamp mtime_nsec_; ModelTimestamp prev_mtime_ns_; bool explicitly_load_; inference::ModelConfig model_config_; std::string model_path_; std::string model_config_path_; // Temporary location to hold agent model list before creating the model // the ownership must transfer to ModelLifeCycle to ensure // the agent model life cycle is handled properly. std::shared_ptr agent_model_list_; bool is_config_provided_; }; // Map from model name to information about the model. class ModelInfoMap { public: using MapType = std::unordered_map>; using Iterator = MapType::iterator; using ConstIterator = MapType::const_iterator; ModelInfoMap() = default; ModelInfoMap(const ModelInfoMap& rhs); ModelInfoMap& operator=(const ModelInfoMap& rhs); Iterator begin() noexcept { return map_.begin(); } ConstIterator begin() const noexcept { return map_.begin(); } ConstIterator cbegin() const noexcept { return map_.cbegin(); } Iterator end() noexcept { return map_.end(); } ConstIterator end() const noexcept { return map_.end(); } ConstIterator cend() const noexcept { return map_.cend(); } std::pair Emplace( const ModelIdentifier& model_id, std::unique_ptr&& model_info) { return map_.emplace(model_id, std::move(model_info)); } size_t Erase(const ModelIdentifier& key) { return map_.erase(key); } void Swap(ModelInfoMap& rhs) { map_.swap(rhs.map_); } std::unique_ptr& At(const ModelIdentifier& key) { return map_.at(key); } const std::unique_ptr& At(const ModelIdentifier& key) const { return map_.at(key); } Iterator Find(const ModelIdentifier& key) { return map_.find(key); } ConstIterator Find(const ModelIdentifier& key) const { return map_.find(key); } // Write updated model info back to this object after model load/unload. void Writeback( const ModelInfoMap& updated_model_info, const std::set& affected_models); private: MapType map_; }; /// A basic unit in dependency graph that records the models seen by the model /// repository manager. struct DependencyNode { DependencyNode(const ModelIdentifier& model_id) : status_(Status::Success), model_id_(model_id), checked_(false), connected_(false), is_locked_(false), retry_notify_cv_(new std::condition_variable()) { } void DisconnectUpstream(DependencyNode* upstream) { upstreams_.erase(upstream); } void DisconnectDownstream(DependencyNode* downstream) { downstreams_.erase(downstream); } void Writeback(const DependencyNode& updated_dependency_node); // Overall status Status status_; // Poll info ModelIdentifier model_id_; bool explicitly_load_; inference::ModelConfig model_config_; // Graph info // Whether the node has been checked for performing lifecycle change // (load / unload) bool checked_; // Whether the node has connected: all required upstreams are found bool connected_; // store only the model names for missing upstreams, and remove from it // only if the upstream is exactly matched. This variable works with // 'missing_nodes_' in DependencyGraph to provide bi-directional lookup for // dependency resolution (exact / fuzzy match). // i.e. the node will look for upstream node that has matching identifier, // but upstream node with different namespace can still be used if not // found. std::set missing_upstreams_; std::unordered_map> upstreams_; std::set downstreams_; // Lifecycle info std::set loaded_versions_; // Loading/Unloading info // when locked, there is another thread loading/unloading model(s) that // depend on the current state of this model. // i.e. this model is being unloaded, or this is an ensemble dependency. bool is_locked_; // when there is a load/unload conflict, related to the model represented // by this node, the calling thread should wait on this condition variable // until notified to retry. // this is an optional measure to reduce the number of retry before the // conflict is resolved, which is not to be relied upon to determine if the // conflict has been resolved and will remain resolved during the retry. // i.e. it is safe to move forward without waiting until notified. std::shared_ptr retry_notify_cv_; }; // Interface for dependency graph operations class DependencyGraph { public: // Create a placeholder object to be assigned later. Do not use the // placeholder object before it is assigned. DependencyGraph() : global_map_ptr_(nullptr) {} // Passing pointer of 'global_map'. // There is coupling between the dependency graph and global map: global map // will be used to resolve dependency (node connectivity), and global map // will be updated to reflect node changes. DependencyGraph( std::unordered_map>* global_map) : global_map_ptr_(global_map) { } DependencyGraph(const DependencyGraph&) = delete; // Copy from rhs, but set global_map_ptr_ to the provided global_map. DependencyGraph( const DependencyGraph& rhs, std::unordered_map>* global_map); DependencyGraph& operator=(const DependencyGraph&) = delete; // Discard all state of this object, and copy state from rhs, but set // global_map_ptr_ to the provided global_map. void Assign( const DependencyGraph& rhs, std::unordered_map>* global_map); void Swap(DependencyGraph& rhs); std::unordered_map>* MutableNodes() { return &nodes_; } // Look up node in dependency graph with matching model identifier. If not // found and fuzzy match is allowed, a node in different namespace will be // returned if it is the only node with the same name DependencyNode* FindNode( const ModelIdentifier& model_id, const bool allow_fuzzy_matching) const; // Set the nodes to lock state. If a node is already locked, then the // identifier of the node is returned. Otherwise, nullptr is returned. // The retry notifier can be provided optionally. If a node is already // locked, it will be set to a notifier that the calling thread can wait on // until notified to retry, to reduce the number of retries. std::unique_ptr LockNodes( const std::set& nodes, std::shared_ptr* retry_notify_cv = nullptr); // Set the nodes to unlock state. If a node is already unlocked, then the // identifier of the node is returned. Otherwise, nullptr is returned. std::unique_ptr UnlockNodes( const std::set& nodes); // Write updated graph back to this object after model load/unload. Only // models specified in affected models will be updated, and the update is // limited to a few node variables that could be changed after load/unload. // This will also notify load/unload conflict to retry for affected models. void Writeback( const DependencyGraph& updated_dependency_graph, const std::set& affected_models); /// Update the dependency graph based on the poll result /// \param model_infos The latest infos for updating the dependency graph. /// \param added The names of the models added to the repository. /// \param deleted The names of the models removed from the repository. /// \param modified The names of the models remaining in the repository /// that have been changed. /// \param deleted_dependents The names of dependent models to be removed /// from the repository. /// \return Set of model ids that are affected by this update. std::set UpdateGraph( const ModelInfoMap& model_infos, const std::set& added, const std::set& deleted, const std::set& modified, std::set* deleted_dependents = nullptr); private: // Find a node in the dependency graph with the exact matching model // identifier. The disconnected portion of the graph is looked up first, see // 'removed_nodes_'. If not found, then the connected portion of the graph // is looked up, see 'nodes_'. If not found, an exception is thrown. DependencyNode* GetNode(const ModelIdentifier& model_id) const; // Remove the given set of nodes, return two sets of nodes: The first set // contains existing nodes to be re-evaluated, because they depend on // the nodes removed; the second set contains all the nodes removed in this // operation. // 'cascading_removal' control whether other nodes will be removed. If true, // it will also remove the nodes that were added to dependency graph only // because they were needed by the node in 'nodes'. In other words, they // will no longer be needed once the 'nodes' are removed and can be removed // from the dependency graph as well. Such an operation will be recursively // applied and thus called "cascading". std::pair, std::set> RemoveNodes( const std::set& nodes, const bool cascading_removal); // Remove the node of the given identifier from dependency graph, // and its reference in other nodes. // Returns two sets of identifiers of the existing nodes that were linked to // the removed node. The first set of the returned identifier is the // "upstreams" of the node (i.e. composing models of the ensemble), the // second set is the "downstreams" of the node (i.e. the model is required // by other ensembles) std::pair, std::set> RemoveNode( const ModelIdentifier& model_id); // Update the given set of nodes to reflect the latest model information // polled, returns existing nodes to be re-evaluated, including the modified // node. std::set UpdateNodes( const std::set& nodes, const ModelInfoMap& model_infos); // Add the given set of nodes to the dependency graph, // returns existing nodes to be re-evaluated, including the added node. std::set AddNodes( const std::set& nodes, const ModelInfoMap& model_infos); // Helper function on the 'node_id' to construct the edges between nodes // in the dependency graph. The node status will be updated if the node // has missing edges. This function should be called after all node changes // are made to the dependency graph. void ConnectDependencyGraph(const ModelIdentifier& node_id); // Check if there is circular dependency on the given node, the node // status will be updated if the node has circular dependency. void CircularDependencyCheck(const ModelIdentifier& node_id); // Recursive version for internal use. Status CircularDependencyCheck( DependencyNode* current_node, const DependencyNode* start_node); // Recursively uncheck the downstream, so the downstreams will need to be // re-checked at later stage to propagate the impact of upstream changes. void UncheckDownstream(const std::set& downstreams); std::unordered_map>* global_map_ptr_; std::unordered_map> nodes_; // A list of model names that there are nodes depending on but not present. // Note that the key is not ModelIdentifier to allow more flexible matching. std::unordered_map> missing_nodes_; // A set of nodes that are disconnected from the graph but not yet // forgotten. Since the nodes are disconnected, their upstream and // downstream pointers are always invalid and must not be used. std::unordered_map> removed_nodes_; }; ~ModelRepositoryManager(); /// Create a manager for a repository. /// \param server The pointer to the inference server. /// \param server_version The version of the inference server. /// \param repository_paths A set of file-system paths of the repositories. /// \param startup_models A set of models to be loaded at startup /// if model control is enabled. /// \param strict_model_config If false attempt to autofill missing required /// information in each model configuration. /// \param polling_enabled If true, then PollAndUpdate() is allowed. /// Otherwise, it is not allowed. /// \param model_control_enabled If true, then LoadUnloadModel() is allowed /// and the models in the model repository will not be loaded at startup. /// Otherwise, LoadUnloadModel() is not allowed and the models will be loaded. /// Cannot be set to true if polling_enabled is true. /// \param model_config_name Custom model config name to load for all models. /// Fall back to default config file if empty. /// \param life_cycle_options The options to configure ModelLifeCycle. /// \param model_repository_manager Return the model repository manager. /// \return The error status. static Status Create( InferenceServer* server, const std::string& server_version, const std::set& repository_paths, const std::set& startup_models, const bool strict_model_config, const std::string& model_config_name, const bool polling_enabled, const bool model_control_enabled, const ModelLifeCycleOptions& life_cycle_options, const bool enable_model_namespacing, std::unique_ptr* model_repository_manager); /// Poll the model repository to determine the new set of models and /// compare with the current set. And serve the new set of models based /// on their version policy. Status PollAndUpdate(); /// Load or unload a specified model. /// \param models The models and the parameters to be loaded or unloaded. /// Expect the number of models to be exactly one. /// \param type The type action to be performed. If the action is LOAD and /// the model has been loaded, the model will be re-loaded. /// \return error status. Return "NOT_FOUND" if it tries to load /// a non-existing model or if it tries to unload a model that hasn't been /// loaded. Status LoadUnloadModel( const std::unordered_map< std::string, std::vector>& models, const ActionType type, const bool unload_dependents); /// Unload all models that are tracked by the model repository manager. If a /// model is loading or unloading when this function is called, or a model /// failed to unload, an error is returned. New models may be loaded while /// this function is unloading models. This function should be called before /// shutting down the model repository manager. /// \return error status. Status UnloadAllModels(); /// Instruct all models to stop accepting new inference requests. However, /// the models are still capable of processing inference requests /// if the model considers them as part of the in-flight inference. /// \return error status. Status StopAllModels(); /// \return the number of in-flight inferences for the all versions of all /// models. The set element will be a tuple of . Note that a model version will not be included /// if it doesn't have in-flight inferences. const std::set> InflightStatus(); /// \return the number of model(s) in the background. size_t BackgroundModelsSize(); /// \param strict_readiness If true, only models that have at least one /// ready version will be considered as live. Otherwise, the models that /// have loading / unloading versions will also be live. /// \return the state of all versions of all live models. const ModelStateMap LiveModelStates(bool strict_readiness = false); /// \return the state of all versions of all models that have every /// been (attempted) loaded over the lifetime of the server. const ModelStateMap ModelStates(); /// \return the states of all versions of a specific model. const VersionStateMap VersionStates(const std::string& model_name); /// \return the ready-state of a specific model version. Status ModelState( const std::string& model_name, const int64_t model_version, ModelReadyState* state); /// Obtain the specified model. /// \param model_name The name of the model. /// \param model_version The version of the model. /// \param model Return the model object. /// \return error status. Status GetModel( const std::string& model_name, const int64_t model_version, std::shared_ptr* model); /// Obtain the specified model. /// \param model_id The identifier of the model. /// \param model_version The version of the model. /// \param model Return the model object. /// \return error status. Status GetModel( const ModelIdentifier& model_id, const int64_t model_version, std::shared_ptr* model); Status FindModelIdentifier( const std::string& model_name, ModelIdentifier* model_id); /// Get the index of all models in all repositories. /// \param ready_only If true return only index of models that are ready. /// \param index Returns the index. /// \return error status. Status RepositoryIndex(const bool ready_only, std::vector* index); // Register model repository path. /// \param repository Path to model repository. /// \param model_mapping Mapping with (overridden) model name as key, subdir /// name as value. /// \return error status Status RegisterModelRepository( const std::string& repository, const std::unordered_map& model_mapping); // Unregister model repository path. /// \param repository Path to model repository. /// \return error status Status UnregisterModelRepository(const std::string& repository); private: // Set of DependencyNode using NodeSet = std::set; ModelRepositoryManager( const std::set& repository_paths, const bool autofill, const std::string& model_config_name, const bool polling_enabled, const bool model_control_enabled, const double min_compute_capability, const bool enable_model_namespacing, std::unique_ptr life_cycle); /// The internal function that are called in Create() and PollAndUpdate(). Status PollAndUpdateInternal(bool* all_models_polled); /// The internal function that load or unload a set of models. /// If 'no_parallel_conflict' is provided and there is a conflict, then the /// function will block until the conflict is resolved and set the correct /// value into the variable, and return with success status for retrying. /// If 'no_parallel_conflict' is not provided and there is a conflict, then /// the function will return immediately with an error status. Status LoadUnloadModels( const std::unordered_map< std::string, std::vector>& models, const ActionType type, const bool unload_dependents, bool* all_models_polled, bool* no_parallel_conflict = nullptr); /// Helper function for LoadUnloadModels() to find the set of added, deleted, /// modified and unmodified models. Also update the provided model infos. /// This function will not update the model info held by this object, it is /// the responsibility of LoadUnloadModels() to do so. Status PollModels( const std::unordered_map< std::string, std::vector>& models, std::set* added, std::set* deleted, std::set* modified, std::set* unmodified, ModelInfoMap* infos, bool* all_models_polled); /// Poll the requested models in the model repository and /// compare with the current set. Return the additions, deletions, /// and modifications that have occurred. This function will not updated /// the current model info, it is caller's responsibility to do so. /// \param models The map from models to be polled to their associated /// parameters. /// \param added The names of the models added to the repository. /// \param deleted The names of the models removed from the repository. /// \param modified The names of the models remaining in the /// repository that have been changed. /// \param unmodified The names of the models remaining in the /// repository that have not changed. /// \param updated_infos The model infos retrieved from the poll. /// \param all_models_polled Return true if all models are polled and /// their model configuration are validated successfully. Instead of aborting /// the polling, the models that fail will be ignored and their model infos /// will stay in the previous state. /// \return The error status. Status Poll( const std::unordered_map< std::string, std::vector>& models, std::set* added, std::set* deleted, std::set* modified, std::set* unmodified, ModelInfoMap* updated_infos, bool* all_models_polled); /// Helper function for Poll() to initialize ModelInfo for the model. /// \param model_id The identifier of the model. /// \param path The model path. Empty path means the model is provided via /// 'params' /// \param params The model parameters provided for polling model. /// \param info Return the updated ModelInfo. 'nullptr' will be returned if /// existing ModelInfo for the model should be reused. /// \return The error status. Status InitializeModelInfo( const ModelIdentifier& model_id, const std::string& path, const std::vector& params, std::unique_ptr* info); /// Load models based on the dependency graph. The function will iteratively /// load models that all the models they depend on has been loaded, and unload /// models if their dependencies are no longer satisfied. /// \param dependency_graph The dependency graph. /// \param infos Model infos to be updated along the load. /// \return The status of the model loads. std::map LoadModelByDependency( DependencyGraph* dependency_graph, ModelInfoMap* infos); /// Get the models to be loaded / unloaded based on the model loaded in /// previous iteration. /// \param loaded_models The models loaded / unloaded in previous iteration. /// Unloaded models will be represented as models with no loaded versions. /// \param model_load_status For checking ensemble dependency(s) status. /// \param dependency_graph The dependency graph corresponds to this load. /// \return A pair of node set containing models to be loaded and models to be /// unloaded for the next iteration. std::pair ModelsToLoadUnload( const NodeSet& loaded_models, const std::map& model_load_status, DependencyGraph* dependency_graph); /// Check if the node is ready for the next iteration. A node is ready if the /// node is invalid (containing invalid model config or its dependencies /// failed to load) or all of its dependencies are satisfied. \param node The /// node to be checked. \param model_load_status For checking ensemble /// dependency(s) status. \return True if the node is ready. False otherwise. bool CheckNode( DependencyNode* node, const std::map& model_load_status); bool ModelDirectoryOverride( const std::vector& model_params); const bool autofill_; const std::string model_config_name_; const bool polling_enabled_; const bool model_control_enabled_; const double min_compute_capability_; std::mutex mu_; // Dependency graph // WAR to avoid change in behavior (mainly error reporting) if model namespace // is not enabled. std::function find_identifier_fn_; // A map from model name to model identifiers that share the same model name std::unordered_map> global_map_; DependencyGraph dependency_graph_; // Repository specific.. const bool enable_model_namespacing_; // [FIXME] Better document below: 'infos_' is the in memory representation of // repo polling results. It is a intermediate layer between the set of models // in storage and the models being served, and it doesn't directly reflect the // state of the two above. // // There are three stages in terms of changing model: // - poll model // - resolve dependency // - load model // // 'infos_' is a long living object across model changes to serve as // differentiator between polls (i.e. is the model newly added? modified?), // so it is a subset of storage models at the time of polling. Note that // it can be marked "dirty" to handle "fallback / revert" in model lifecycle: // if the model is currently being served and re-load action is required, the // lifecycle will keep the "older" model if the newer storage model is // malformed => re-load failure. Marking info "dirty" in this case to ensure // the model to be polled again even though the storage model hasn't been // changed, for operation idempotence. // (https://github.com/triton-inference-server/server/issues/3802) ModelInfoMap infos_; std::set repository_paths_; // Mappings from (overridden) model names to a pair of their repository and // absolute path // [DLIS-4596] key should be updated to contain namespace to work with enabled // namespace. Also need to revisit with repository side of things. std::unordered_map> model_mappings_; // Model lifecycle std::unique_ptr model_life_cycle_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/numa_utils.cc000066400000000000000000000162101527052371200200430ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "numa_utils.h" #ifndef _WIN32 #include #include #endif #include "triton/common/logging.h" namespace triton { namespace core { namespace { std::string VectorToString(const std::vector& vec) { std::string str("["); for (const auto& element : vec) { str += std::to_string(element); str += ","; } str += "]"; return str; } Status ParseIntOption(const std::string& msg, const std::string& arg, int* value) { try { *value = std::stoi(arg); } catch (const std::invalid_argument& ia) { return Status( Status::Code::INVALID_ARG, msg + ": Can't parse '" + arg + "' to integer"); } return Status::Success; } } // namespace // NUMA setting will be ignored on Windows platform #ifdef _WIN32 Status SetNumaConfigOnThread( const triton::common::HostPolicyCmdlineConfig& host_policy) { return Status::Success; } Status SetNumaMemoryPolicy(const triton::common::HostPolicyCmdlineConfig& host_policy) { return Status::Success; } Status GetNumaMemoryPolicyNodeMask(unsigned long* node_mask) { *node_mask = 0; return Status::Success; } Status ResetNumaMemoryPolicy() { return Status::Success; } Status SetNumaThreadAffinity( std::thread::native_handle_type thread, const triton::common::HostPolicyCmdlineConfig& host_policy) { return Status::Success; } #else // Use variable to make sure no NUMA related function is actually called // if Triton is not running with NUMA awareness. i.e. Extra docker permission // is needed to call the NUMA functions and this ensures backward compatibility. thread_local bool numa_set = false; Status SetNumaConfigOnThread( const triton::common::HostPolicyCmdlineConfig& host_policy) { // Set thread affinity RETURN_IF_ERROR(SetNumaThreadAffinity(pthread_self(), host_policy)); // Set memory policy RETURN_IF_ERROR(SetNumaMemoryPolicy(host_policy)); return Status::Success; } Status SetNumaMemoryPolicy(const triton::common::HostPolicyCmdlineConfig& host_policy) { const auto it = host_policy.find("numa-node"); if (it != host_policy.end()) { int node_id; RETURN_IF_ERROR( ParseIntOption("Parsing 'numa-node' value", it->second, &node_id)); LOG_VERBOSE(1) << "Thread is binding to NUMA node " << it->second << ". Max NUMA node count: " << (numa_max_node() + 1); numa_set = true; unsigned long node_mask = 1UL << node_id; if (set_mempolicy(MPOL_BIND, &node_mask, (numa_max_node() + 1) + 1) != 0) { return Status( Status::Code::INTERNAL, std::string("Unable to set NUMA memory policy: ") + strerror(errno)); } } return Status::Success; } Status GetNumaMemoryPolicyNodeMask(unsigned long* node_mask) { *node_mask = 0; int mode; if (numa_set && get_mempolicy(&mode, node_mask, numa_max_node() + 1, NULL, 0) != 0) { return Status( Status::Code::INTERNAL, std::string("Unable to get NUMA node for current thread: ") + strerror(errno)); } return Status::Success; } Status ResetNumaMemoryPolicy() { if (numa_set && (set_mempolicy(MPOL_DEFAULT, nullptr, 0) != 0)) { return Status( Status::Code::INTERNAL, std::string("Unable to reset NUMA memory policy: ") + strerror(errno)); } numa_set = false; return Status::Success; } Status SetNumaThreadAffinity( std::thread::native_handle_type thread, const triton::common::HostPolicyCmdlineConfig& host_policy) { const auto it = host_policy.find("cpu-cores"); if (it != host_policy.end()) { // Parse CPUs std::vector cpus; { const auto& cpu_str = it->second; auto delim_cpus = cpu_str.find(","); int current_pos = 0; while (true) { auto delim_range = cpu_str.find("-", current_pos); if (delim_range == std::string::npos) { return Status( Status::Code::INVALID_ARG, std::string("host policy setting 'cpu-cores' format is " "'-'. Got ") + cpu_str.substr( current_pos, ((delim_cpus == std::string::npos) ? (cpu_str.length() + 1) : delim_cpus) - current_pos)); } int lower, upper; RETURN_IF_ERROR(ParseIntOption( "Parsing 'cpu-cores' value", cpu_str.substr(current_pos, delim_range - current_pos), &lower)); RETURN_IF_ERROR(ParseIntOption( "Parsing 'cpu-cores' value", (delim_cpus == std::string::npos) ? cpu_str.substr(delim_range + 1) : cpu_str.substr( delim_range + 1, delim_cpus - (delim_range + 1)), &upper)); for (; lower <= upper; ++lower) { cpus.push_back(lower); } // break if the processed range is the last specified range if (delim_cpus != std::string::npos) { current_pos = delim_cpus + 1; delim_cpus = cpu_str.find(",", current_pos); } else { break; } } } LOG_VERBOSE(1) << "Thread is binding to one of the CPUs: " << VectorToString(cpus); numa_set = true; cpu_set_t cpuset; CPU_ZERO(&cpuset); for (int cpu : cpus) { CPU_SET(cpu, &cpuset); } if (pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset) != 0) { return Status( Status::Code::INTERNAL, std::string("Unable to set NUMA thread affinity: ") + strerror(errno)); } } return Status::Success; } #endif }} // namespace triton::core kytensor-2.49.0.7/core/src/numa_utils.h000066400000000000000000000047331527052371200177140ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include "status.h" #include "triton/common/model_config.h" #include "tritonserver_apis.h" namespace triton { namespace core { // Helper function to set memory policy and thread affinity on current thread Status SetNumaConfigOnThread( const triton::common::HostPolicyCmdlineConfig& host_policy); // Restrict the memory allocation to specific NUMA node. Status SetNumaMemoryPolicy( const triton::common::HostPolicyCmdlineConfig& host_policy); // Retrieve the node mask used to set memory policy for the current thread Status GetNumaMemoryPolicyNodeMask(unsigned long* node_mask); // Reset the memory allocation setting. Status ResetNumaMemoryPolicy(); // Set a thread affinity to be on specific cpus. Status SetNumaThreadAffinity( std::thread::native_handle_type thread, const triton::common::HostPolicyCmdlineConfig& host_policy); }} // namespace triton::core kytensor-2.49.0.7/core/src/payload.cc000066400000000000000000000135741527052371200173260ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "payload.h" namespace triton { namespace core { Payload::Payload() : op_type_(Operation::INFER_RUN), requests_(std::vector>()), OnCallback_([]() {}), instance_(nullptr), state_(State::UNINITIALIZED), batcher_start_ns_(0), saturated_(false), user_pointer_(nullptr) { exec_mu_.reset(new std::mutex()); } const Status& Payload::MergePayload(std::shared_ptr& payload) { if ((payload->GetOpType() != Operation::INFER_RUN) || (op_type_ != Operation::INFER_RUN)) { static Status op_type_error( Status::Code::INTERNAL, "Attempted to merge payloads of type that are not INFER_RUN"); return op_type_error; } if (payload->GetInstance() != instance_) { static Status instance_error( Status::Code::INTERNAL, "Attempted to merge payloads of mismatching instance"); return instance_error; } if ((payload->GetState() != State::EXECUTING) || (state_ != State::EXECUTING)) { static Status state_error( Status::Code::INTERNAL, "Attempted to merge payloads that are not in executing state"); return state_error; } // Skip comparison if not initialized (required), here assume either all // payloads are initialized or otherwise. if (required_equal_inputs_.Initialized() && !required_equal_inputs_.HasEqualInputs(*payload->Requests().begin())) { static Status shape_error( Status::Code::INVALID_ARG, "Attempted to merge payloads that has non-equal inputs"); return shape_error; } requests_.insert( requests_.end(), std::make_move_iterator(payload->Requests().begin()), std::make_move_iterator(payload->Requests().end())); payload->Callback(); return Status::Success; } void Payload::Reset(const Operation op_type, TritonModelInstance* instance) { op_type_ = op_type; requests_.clear(); OnCallback_ = []() {}; release_callbacks_.clear(); instance_ = instance; state_ = State::UNINITIALIZED; status_.reset(new std::promise()); required_equal_inputs_ = RequiredEqualInputs(); batcher_start_ns_ = 0; saturated_ = false; user_pointer_ = nullptr; } void Payload::Release() { op_type_ = Operation::INFER_RUN; requests_.clear(); OnCallback_ = []() {}; release_callbacks_.clear(); instance_ = nullptr; state_ = State::RELEASED; required_equal_inputs_ = RequiredEqualInputs(); batcher_start_ns_ = 0; saturated_ = false; user_pointer_ = nullptr; } size_t Payload::BatchSize() { size_t batch_size = 0; for (const auto& request : requests_) { batch_size += std::max(1U, request->BatchSize()); } return batch_size; } void Payload::ReserveRequests(size_t size) { requests_.reserve(size); } void Payload::AddRequest(std::unique_ptr request) { if ((batcher_start_ns_ == 0) || (batcher_start_ns_ > request->BatcherStartNs())) { batcher_start_ns_ = request->BatcherStartNs(); } requests_.push_back(std::move(request)); } void Payload::SetCallback(std::function OnCallback) { OnCallback_ = OnCallback; } void Payload::SetInstance(TritonModelInstance* model_instance) { instance_ = model_instance; } void Payload::AddInternalReleaseCallback(std::function&& callback) { release_callbacks_.emplace_back(std::move(callback)); } void Payload::MarkSaturated() { saturated_ = true; } void Payload::SetState(Payload::State state) { state_ = state; } Status Payload::Wait() { return status_->get_future().get(); } void Payload::Callback() { OnCallback_(); } void Payload::OnRelease() { // Invoke the release callbacks added internally before releasing the // request to user provided callback. for (auto it = release_callbacks_.rbegin(); it != release_callbacks_.rend(); it++) { (*it)(); } release_callbacks_.clear(); } void Payload::Execute(bool* should_exit) { *should_exit = false; Status status; switch (op_type_) { case Operation::INFER_RUN: status = instance_->Schedule(std::move(requests_)); break; case Operation::INIT: status = instance_->Initialize(); break; case Operation::WARM_UP: status = instance_->WarmUp(); break; case Operation::EXIT: *should_exit = true; } status_->set_value(status); // Call specified callback, notifying that execution has completed. Callback(); } }} // namespace triton::core kytensor-2.49.0.7/core/src/payload.h000066400000000000000000000104051527052371200171560ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include "backend_model_instance.h" #include "infer_request.h" #include "scheduler_utils.h" #include "status.h" namespace triton { namespace core { class Payload { public: enum Operation { INFER_RUN = 0, INIT = 1, WARM_UP = 2, EXIT = 3 }; enum State { UNINITIALIZED = 0, READY = 1, REQUESTED = 2, SCHEDULED = 3, EXECUTING = 4, RELEASED = 5 }; Payload(); void Reset(const Operation op_type, TritonModelInstance* instance = nullptr); const Status& MergePayload(std::shared_ptr& payload); Operation GetOpType() { return op_type_; } std::mutex* GetExecMutex() { return exec_mu_.get(); } size_t RequestCount() { return requests_.size(); } size_t BatchSize(); void ReserveRequests(size_t size); void AddRequest(std::unique_ptr request); std::vector>& Requests() { return requests_; } uint64_t BatcherStartNs() { return batcher_start_ns_; } // Callback used for internal optimizations around payload dequeueing and // execution, such as informing schedulers that payload slot(s) are available. // Only a single callback of this form is used. For resource cleanup, see // the OnRelease callbacks. void Callback(); void SetCallback(std::function OnCallback); // Callbacks used for any resource cleanup when a payload is about to be // released. Some payloads may be released early before execution, such as // paylods can be merged together for efficiency. Multiple release callbacks // may be specified. void OnRelease(); void AddInternalReleaseCallback(std::function&& callback); void SetInstance(TritonModelInstance* model_instance); TritonModelInstance* GetInstance() { return instance_; } void MarkSaturated(); bool IsSaturated() { return saturated_; } RequiredEqualInputs* MutableRequiredEqualInputs() { return &required_equal_inputs_; } void** UserPointerAddr() { return &user_pointer_; } State GetState() { return state_; } void SetState(State state); void Execute(bool* should_exit); Status Wait(); void Release(); private: Operation op_type_; std::vector> requests_; std::function OnCallback_; std::vector> release_callbacks_; TritonModelInstance* instance_; State state_; std::unique_ptr> status_; std::unique_ptr exec_mu_; uint64_t batcher_start_ns_; RequiredEqualInputs required_equal_inputs_; bool saturated_; // Pointer for use with user-supplied batching strategy. void* user_pointer_ = nullptr; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/pinned_memory_manager.cc000066400000000000000000000334571527052371200222360ustar00rootroot00000000000000// Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #include "pinned_memory_manager.h" #include #include "numa_utils.h" #include "triton/common/logging.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace core { namespace { std::string PointerToString(void* ptr) { std::stringstream ss; ss << ptr; return ss.str(); } Status ParseIntOption(const std::string& msg, const std::string& arg, int* value) { try { *value = std::stoi(arg); } catch (const std::invalid_argument& ia) { return Status( Status::Code::INVALID_ARG, msg + ": Can't parse '" + arg + "' to integer"); } return Status::Success; } } // namespace std::unique_ptr PinnedMemoryManager::instance_; uint64_t PinnedMemoryManager::pinned_memory_byte_size_ = 0; std::mutex PinnedMemoryManager::allocated_buffer_mtx_; std::vector> PinnedMemoryManager::allocated_pinned_memory_buffers_; PinnedMemoryManager::PinnedMemory::PinnedMemory( void* pinned_memory_buffer, uint64_t size) : pinned_memory_buffer_(pinned_memory_buffer) { used_pinned_memory_byte_size_ = 0; if (pinned_memory_buffer_ != nullptr) { managed_pinned_memory_ = boost::interprocess::managed_external_buffer( boost::interprocess::create_only_t{}, pinned_memory_buffer_, size); } } PinnedMemoryManager::PinnedMemory::~PinnedMemory() { #ifdef TRITON_ENABLE_GPU if (pinned_memory_buffer_ != nullptr) { cudaFreeHost(pinned_memory_buffer_); } #endif // TRITON_ENABLE_GPU } void* PinnedMemoryManager::PinnedMemory::Allocate(uint64_t size) { std::lock_guard lk(buffer_mtx_); void* ptr = managed_pinned_memory_.allocate(size, std::nothrow_t{}); used_pinned_memory_byte_size_ += size; allocated_memory_info_.emplace(ptr, size); return ptr; } void PinnedMemoryManager::PinnedMemory::Deallocate(void* ptr) { std::lock_guard lk(buffer_mtx_); managed_pinned_memory_.deallocate(ptr); auto it = allocated_memory_info_.find(ptr); if (it != allocated_memory_info_.end()) { used_pinned_memory_byte_size_ -= it->second; allocated_memory_info_.erase(it); } } uint64_t PinnedMemoryManager::PinnedMemory::GetUsedPinnedMemorySizeInternal() { std::lock_guard lk(buffer_mtx_); return used_pinned_memory_byte_size_; } PinnedMemoryManager::~PinnedMemoryManager() { // Clean up allocated_pinned_memory_buffers_.clear(); for (const auto& memory_info : memory_info_) { const auto& is_pinned = memory_info.second.first; if (!is_pinned) { free(memory_info.first); } } } void PinnedMemoryManager::AddPinnedMemoryBuffer( const std::shared_ptr& pinned_memory_buffer, unsigned long node_mask) { pinned_memory_buffers_[node_mask] = pinned_memory_buffer; { std::lock_guard lk(allocated_buffer_mtx_); allocated_pinned_memory_buffers_.push_back(pinned_memory_buffer); } } Status PinnedMemoryManager::AllocInternal( void** ptr, uint64_t size, TRITONSERVER_MemoryType* allocated_type, bool allow_nonpinned_fallback, PinnedMemory* pinned_memory_buffer) { auto status = Status::Success; if (pinned_memory_buffer->pinned_memory_buffer_ != nullptr) { *ptr = pinned_memory_buffer->Allocate(size); *allocated_type = TRITONSERVER_MEMORY_CPU_PINNED; if (*ptr == nullptr) { status = Status( Status::Code::INTERNAL, "failed to allocate pinned system memory"); } } else { status = Status( Status::Code::INTERNAL, "failed to allocate pinned system memory: no pinned memory pool"); } bool is_pinned = true; if ((!status.IsOk()) && allow_nonpinned_fallback) { static bool warning_logged = false; if (!warning_logged) { LOG_WARNING << status.Message() << ", falling back to non-pinned system memory"; warning_logged = true; } *ptr = malloc(size); *allocated_type = TRITONSERVER_MEMORY_CPU; is_pinned = false; if (*ptr == nullptr) { status = Status( Status::Code::INTERNAL, "failed to allocate non-pinned system memory"); } else { status = Status::Success; } } // keep track of allocated buffer or clean up { std::lock_guard lk(info_mtx_); if (status.IsOk()) { auto res = memory_info_.emplace( *ptr, std::make_pair(is_pinned, pinned_memory_buffer)); if (!res.second) { status = Status( Status::Code::INTERNAL, "unexpected memory address collision, '" + PointerToString(*ptr) + "' has been managed"); } LOG_VERBOSE(1) << (is_pinned ? "" : "non-") << "pinned memory allocation: " << "size " << size << ", addr " << *ptr; } } if ((!status.IsOk()) && (*ptr != nullptr)) { if (is_pinned) { pinned_memory_buffer->Deallocate(*ptr); } else { free(*ptr); } } return status; } Status PinnedMemoryManager::FreeInternal(void* ptr) { bool is_pinned = true; PinnedMemory* pinned_memory_buffer = nullptr; { std::lock_guard lk(info_mtx_); auto it = memory_info_.find(ptr); if (it != memory_info_.end()) { is_pinned = it->second.first; pinned_memory_buffer = it->second.second; LOG_VERBOSE(1) << (is_pinned ? "" : "non-") << "pinned memory deallocation: " << "addr " << ptr; memory_info_.erase(it); } else { return Status( Status::Code::INTERNAL, "unexpected memory address '" + PointerToString(ptr) + "' is not being managed"); } } if (is_pinned) { pinned_memory_buffer->Deallocate(ptr); } else { free(ptr); } return Status::Success; } void PinnedMemoryManager::Reset() { instance_.reset(); } Status PinnedMemoryManager::Create(const Options& options) { if (instance_ != nullptr) { LOG_WARNING << "New pinned memory pool of size " << options.pinned_memory_pool_byte_size_ << " could not be created since one already exists" << " of size " << pinned_memory_byte_size_; return Status::Success; } instance_.reset(new PinnedMemoryManager()); if (options.host_policy_map_.empty()) { void* buffer = nullptr; if (options.pinned_memory_pool_byte_size_ > 0) { #ifdef TRITON_ENABLE_GPU auto err = cudaHostAlloc( &buffer, options.pinned_memory_pool_byte_size_, cudaHostAllocPortable); if (err != cudaSuccess) { buffer = nullptr; LOG_WARNING << "Unable to allocate pinned system memory, pinned memory " "pool will not be available: " << std::string(cudaGetErrorString(err)); } else if (options.pinned_memory_pool_byte_size_ != 0) { LOG_INFO << "Pinned memory pool is created at '" << PointerToString(buffer) << "' with size " << options.pinned_memory_pool_byte_size_; } #endif // TRITON_ENABLE_GPU } else { LOG_INFO << "Pinned memory pool disabled"; } try { instance_->AddPinnedMemoryBuffer( std::shared_ptr( new PinnedMemory(buffer, options.pinned_memory_pool_byte_size_)), 0); } catch (const std::exception& ex) { return Status( Status::Code::INTERNAL, "Failed to add Pinned Memory buffer: " + std::string(ex.what())); } } else { // Create only one buffer / manager should be created for one node, // and all associated devices should request memory from the shared manager std::map numa_map; for (const auto& host_policy : options.host_policy_map_) { const auto numa_it = host_policy.second.find("numa-node"); if (numa_it != host_policy.second.end()) { int32_t numa_id; if (ParseIntOption("Parsing NUMA node", numa_it->second, &numa_id) .IsOk()) { numa_map.emplace(numa_id, host_policy.first); } } } for (const auto& node_policy : numa_map) { auto status = SetNumaMemoryPolicy(options.host_policy_map_.at(node_policy.second)); if (!status.IsOk()) { LOG_WARNING << "Unable to allocate pinned system memory for NUMA node " << node_policy.first << ": " << status.AsString(); continue; } unsigned long node_mask; status = GetNumaMemoryPolicyNodeMask(&node_mask); if (!status.IsOk()) { LOG_WARNING << "Unable to get NUMA node set for current thread: " << status.AsString(); continue; } void* buffer = nullptr; if (options.pinned_memory_pool_byte_size_ > 0) { #ifdef TRITON_ENABLE_GPU auto err = cudaHostAlloc( &buffer, options.pinned_memory_pool_byte_size_, cudaHostAllocPortable); if (err != cudaSuccess) { buffer = nullptr; LOG_WARNING << "Unable to allocate pinned system memory, pinned memory " "pool will not be available: " << std::string(cudaGetErrorString(err)); } else if (options.pinned_memory_pool_byte_size_ != 0) { LOG_INFO << "Pinned memory pool is created at '" << PointerToString(buffer) << "' with size " << options.pinned_memory_pool_byte_size_; } else { LOG_INFO << "Pinned memory pool disabled"; } #endif // TRITON_ENABLE_GPU } else { LOG_INFO << "Pinned memory pool disabled"; } ResetNumaMemoryPolicy(); try { instance_->AddPinnedMemoryBuffer( std::shared_ptr(new PinnedMemory( buffer, options.pinned_memory_pool_byte_size_)), node_mask); } catch (const std::exception& ex) { return Status( Status::Code::INTERNAL, "Failed to add Pinned Memory buffer with host policy: " + std::string(ex.what())); } // If no pinned memory is allocated, add an empty entry where all // allocation will be on normal system memory if (instance_->pinned_memory_buffers_.empty()) { try { instance_->AddPinnedMemoryBuffer( std::shared_ptr(new PinnedMemory( nullptr, options.pinned_memory_pool_byte_size_)), 0); } catch (const std::exception& ex) { return Status( Status::Code::INTERNAL, "Failed to add empty Pinned Memory entry: " + std::string(ex.what())); } } } } pinned_memory_byte_size_ = options.pinned_memory_pool_byte_size_; return Status::Success; } Status PinnedMemoryManager::Alloc( void** ptr, uint64_t size, TRITONSERVER_MemoryType* allocated_type, bool allow_nonpinned_fallback) { if (instance_ == nullptr) { return Status( Status::Code::UNAVAILABLE, "PinnedMemoryManager has not been created"); } auto pinned_memory_buffer = instance_->pinned_memory_buffers_.begin()->second.get(); if (instance_->pinned_memory_buffers_.size() > 1) { unsigned long node_mask; if (GetNumaMemoryPolicyNodeMask(&node_mask).IsOk()) { auto it = instance_->pinned_memory_buffers_.find(node_mask); if (it != instance_->pinned_memory_buffers_.end()) { pinned_memory_buffer = it->second.get(); } } } return instance_->AllocInternal( ptr, size, allocated_type, allow_nonpinned_fallback, pinned_memory_buffer); } Status PinnedMemoryManager::Free(void* ptr) { if (instance_ == nullptr) { return Status( Status::Code::UNAVAILABLE, "PinnedMemoryManager has not been created"); } return instance_->FreeInternal(ptr); } uint64_t PinnedMemoryManager::GetTotalPinnedMemoryByteSize() { return pinned_memory_byte_size_; } uint64_t PinnedMemoryManager::GetUsedPinnedMemoryByteSize() { std::lock_guard lk(allocated_buffer_mtx_); uint64_t used_pinned_memory_size = 0; if (!allocated_pinned_memory_buffers_.empty()) { for (const auto& it : allocated_pinned_memory_buffers_) { used_pinned_memory_size += it->GetUsedPinnedMemorySizeInternal(); } } return used_pinned_memory_size; } }} // namespace triton::core kytensor-2.49.0.7/core/src/pinned_memory_manager.h000066400000000000000000000112621527052371200220660ustar00rootroot00000000000000// Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once #include #include #include #include #include "status.h" #include "triton/common/model_config.h" namespace triton { namespace core { // This is a singleton class responsible for maintaining pinned memory pool // used by the inference server. Pinned memory allocations and deallocations // must be requested via functions provided by this class. class PinnedMemoryManager { public: // Options to configure pinned memory manager. struct Options { Options( uint64_t b = 0, const triton::common::HostPolicyCmdlineConfigMap& host_policy_map = {}) : pinned_memory_pool_byte_size_(b), host_policy_map_(host_policy_map) { } uint64_t pinned_memory_pool_byte_size_; triton::common::HostPolicyCmdlineConfigMap host_policy_map_; }; ~PinnedMemoryManager(); // Create the pinned memory manager based on 'options' specified. // Return Status object indicating success or failure. static Status Create(const Options& options); // Provide explicit control on ending the memory manager lifecycle, // CUDA resource must be cleaned up before CUDA context is destroyed. static void Reset(); // Allocate pinned memory with the requested 'size' and return the pointer // in 'ptr'. If 'allow_nonpinned_fallback' is true, regular system memory // will be allocated as fallback in the case where pinned memory fails to // be allocated. // Return Status object indicating success or failure. static Status Alloc( void** ptr, uint64_t size, TRITONSERVER_MemoryType* allocated_type, bool allow_nonpinned_fallback); // Free the memory allocated by the pinned memory manager. // Return Status object indicating success or failure. static Status Free(void* ptr); static uint64_t GetTotalPinnedMemoryByteSize(); static uint64_t GetUsedPinnedMemoryByteSize(); private: class PinnedMemory { public: PinnedMemory(void* pinned_memory_buffer, uint64_t size); ~PinnedMemory(); void* Allocate(uint64_t size); void Deallocate(void* ptr); uint64_t GetUsedPinnedMemorySizeInternal(); void* pinned_memory_buffer_; std::mutex buffer_mtx_; uint64_t used_pinned_memory_byte_size_; std::map allocated_memory_info_; boost::interprocess::managed_external_buffer managed_pinned_memory_; }; PinnedMemoryManager() = default; Status AllocInternal( void** ptr, uint64_t size, TRITONSERVER_MemoryType* allocated_type, bool allow_nonpinned_fallback, PinnedMemory* pinned_memory_buffer); Status FreeInternal(void* ptr); void AddPinnedMemoryBuffer( const std::shared_ptr& pinned_memory_buffer, unsigned long node_mask); static std::unique_ptr instance_; static uint64_t pinned_memory_byte_size_; static std::mutex allocated_buffer_mtx_; static std::vector> allocated_pinned_memory_buffers_; std::mutex info_mtx_; std::map> memory_info_; std::map> pinned_memory_buffers_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/rate_limiter.cc000066400000000000000000001115131527052371200203450ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "rate_limiter.h" #include #include "triton/common/logging.h" namespace triton { namespace core { constexpr size_t MAX_PAYLOAD_BUCKET_COUNT = 1000; //========================================================================= // Core Implementation //========================================================================= Status RateLimiter::Create( const bool ignore_resources_and_priority, const RateLimiter::ResourceMap& resource_map, std::unique_ptr* rate_limiter) { std::unique_ptr local_rate_limiter( new RateLimiter(ignore_resources_and_priority, resource_map)); *rate_limiter = std::move(local_rate_limiter); return Status::Success; } Status RateLimiter::RegisterModelInstance( TritonModelInstance* triton_model_instance, const RateLimiterConfig& rate_limiter_config) { { std::lock_guard lk1(model_ctx_mtx_); std::lock_guard lk2(model_instance_ctx_mtx_); auto& model_context = model_contexts_[triton_model_instance->Model()]; auto& model_instances = model_instance_ctxs_[triton_model_instance->Model()]; auto pair_it = model_instances.emplace( triton_model_instance, std::unique_ptr(new ModelInstanceContext( triton_model_instance, &model_context, rate_limiter_config, [this](ModelInstanceContext* instance) { OnStage(instance); }, [this](ModelInstanceContext* instance) { OnRelease(instance); }))); model_context.AddAvailableInstance(pair_it.first->second.get()); model_context.AddSpecificRequestQueue(pair_it.first->second.get()); if (!ignore_resources_and_priority_) { // As there can be multiple models being loaded concurrently, need // to hold a lock to protect the resource counts. // Without this serialization instances of other models might fail // to load because of the resource constraints in this instance. std::lock_guard lk(resource_manager_mtx_); const auto& status = resource_manager_->AddModelInstance(pair_it.first->second.get()); if (!status.IsOk()) { LOG_STATUS_ERROR( resource_manager_->RemoveModelInstance(pair_it.first->second.get()), "Cannot remove instance from resource manager"); return status; } } } InitializePayloadQueues(triton_model_instance); return Status::Success; } void RateLimiter::UnregisterModelInstance(TritonModelInstance* triton_model_instance) { std::lock_guard lk1(model_ctx_mtx_); std::lock_guard lk2(model_instance_ctx_mtx_); const TritonModel* model = triton_model_instance->Model(); auto& model_context = model_contexts_[model]; auto& model_instances = model_instance_ctxs_[model]; auto i_it = model_instances.find(triton_model_instance); if (i_it != model_instances.end()) { if (!ignore_resources_and_priority_) { LOG_STATUS_ERROR( resource_manager_->RemoveModelInstance(i_it->second.get()), "Cannot remove instance from resource manager"); } model_context.RemoveInstance(i_it->second.get()); model_instances.erase(i_it); } { std::lock_guard lk(payload_queues_mu_); auto p_it = payload_queues_.find(model); if (p_it != payload_queues_.end()) { auto s_it = p_it->second->specific_queues_.find(triton_model_instance); if (s_it != p_it->second->specific_queues_.end()) { p_it->second->specific_queues_.erase(s_it); } } } } void RateLimiter::UnregisterModel(const TritonModel* model) { { std::lock_guard lk1(model_ctx_mtx_); std::lock_guard lk2(model_instance_ctx_mtx_); auto& model_context = model_contexts_[model]; model_context.RequestRemoval(); for (const auto& instance : model_instance_ctxs_[model]) { if (!ignore_resources_and_priority_) { LOG_STATUS_ERROR( resource_manager_->RemoveModelInstance(instance.second.get()), "Cannot remove instance from resource manager"); } } model_instance_ctxs_.erase(model); model_contexts_.erase(model); } { std::lock_guard lk(payload_queues_mu_); if (payload_queues_.find(model) != payload_queues_.end()) { payload_queues_.erase(model); } } } void RateLimiter::WaitForConsumer( const TritonModel* model, const TritonModelInstance* model_instance) { PayloadQueue* payload_queue = nullptr; { std::lock_guard lk(payload_queues_mu_); if (payload_queues_.find(model) == payload_queues_.end()) { LOG_ERROR << "Unable to find the payload queue for the model " << model->Name(); return; } payload_queue = payload_queues_[model].get(); } if (model_instance == nullptr) { payload_queue->queue_->WaitForConsumer(); } else { payload_queue->specific_queues_[model_instance]->WaitForConsumer(); } } int RateLimiter::WaitingConsumerCount( const TritonModel* model, const TritonModelInstance* model_instance) { PayloadQueue* payload_queue = nullptr; { std::lock_guard lk(payload_queues_mu_); if (payload_queues_.find(model) == payload_queues_.end()) { LOG_ERROR << "Unable to find the payload queue for the model " << model->Name(); return 0; } payload_queue = payload_queues_[model].get(); } if (model_instance == nullptr) { return payload_queue->queue_->WaitingConsumerCount(); } else { return payload_queue->specific_queues_[model_instance] ->WaitingConsumerCount(); } } bool RateLimiter::PayloadSlotAvailable( const TritonModel* model, const TritonModelInstance* model_instance, const bool support_prefetching, const bool force_non_blocking) { bool result; if (support_prefetching) { PayloadQueue* payload_queue = nullptr; { std::lock_guard lk(payload_queues_mu_); payload_queue = payload_queues_[model].get(); } { std::lock_guard lk(payload_queue->mu_); // The logic below sets cap on the number of payloads that // can be pre-fetched. For per-model batcher the cap is // twice the number of model instances. For per-instance // batcher the cap is 2. size_t multiplier = (model_instance == nullptr) ? payload_queue->specific_queues_.size() : 1; result = payload_queue->queue_->Size() < (2 * multiplier); } } else { result = true; if (force_non_blocking) { result = (WaitingConsumerCount(model, model_instance) > 0); } else { WaitForConsumer(model, model_instance); } } return result; } Status RateLimiter::EnqueuePayload( const TritonModel* model, std::shared_ptr payload) { auto pinstance = payload->GetInstance(); PayloadQueue* payload_queue = nullptr; { std::lock_guard lk(payload_queues_mu_); if (payload_queues_.find(model) == payload_queues_.end()) { return Status( Status::Code::INTERNAL, "Unable to find the payload queue for the model " + model->Name()); } payload_queue = payload_queues_[model].get(); } // Update the pending consumer counts to prevent additional // requests from getting enqueued. if (pinstance != nullptr) { payload_queue->specific_queues_[pinstance]->DecrementConsumerCount(); } payload_queue->queue_->DecrementConsumerCount(); { std::lock_guard lk(payload_queue->mu_); payload->SetState(Payload::State::REQUESTED); if (ignore_resources_and_priority_) { SchedulePayload(pinstance, payload_queue, payload); } } if (ignore_resources_and_priority_) { if (pinstance == nullptr) { payload_queue->cv_.notify_one(); } else { payload_queue->cv_.notify_all(); } } else { StandardScheduleFunc sched_func = [this, payload_queue, payload](ModelInstanceContext* mi) { { std::lock_guard lk(payload_queue->mu_); auto cb = [mi]() { mi->Release(); }; payload->AddInternalReleaseCallback(cb); this->SchedulePayload(mi->RawInstance(), payload_queue, payload); } if (mi->RawInstance() == nullptr) { payload_queue->cv_.notify_one(); } else { payload_queue->cv_.notify_all(); } }; DeferPayloadSchedule(sched_func, model, payload->GetInstance()); } return Status::Success; } void RateLimiter::DequeuePayload( std::deque& instances, std::shared_ptr* payload) { payload->reset(); PayloadQueue* payload_queue = nullptr; auto model = instances[0]->Model(); { std::lock_guard lk(payload_queues_mu_); if (payload_queues_.find(model) == payload_queues_.end()) { LOG_ERROR << "Unable to find the payload queue for the model " << model->Name(); return; } payload_queue = payload_queues_[model].get(); } // Update the queue to reflect availability of a waiting // consumer. payload_queue->queue_->IncrementConsumerCount(); for (const auto instance : instances) { payload_queue->specific_queues_[instance]->IncrementConsumerCount(); } std::vector> merged_payloads; size_t instance_index = std::numeric_limits::max(); { std::unique_lock lk(payload_queue->mu_); payload_queue->cv_.wait(lk, [&instances, &instance_index, payload_queue]() { bool empty = payload_queue->queue_->Empty(); if (empty) { instance_index = 0; for (const auto instance : instances) { empty = payload_queue->specific_queues_[instance]->Empty(); if (empty) { instance_index++; } else { break; } } } return !empty; }); if (instance_index < instances.size()) { TritonModelInstance* instance = instances[instance_index]; if (!payload_queue->specific_queues_[instance]->Empty()) { payload_queue->specific_queues_[instance]->Dequeue( payload, &merged_payloads); } } else { payload_queue->queue_->Dequeue(payload, &merged_payloads); } } for (auto& merge_payload : merged_payloads) { PayloadRelease(merge_payload); } // Call specified callback, notifying that payloads have been dequeued/merged. (*payload)->Callback(); if ((*payload)->GetInstance() == nullptr) { (*payload)->SetInstance(instances.front()); // Enqueue did not specify the specific instance to // run with the payload. Hence, need to explicitly // decrement the consumer count for the instance // which got allocated. payload_queue->specific_queues_[instances.front()] ->DecrementConsumerCount(); instances.pop_front(); } else { instances.erase(instances.begin() + instance_index); } // Decrement the counts from the remaining specific // instance handling as there will be no consumer for // these queues. // FIXME: DLIS-5238 For more accurate handling, the // consumer count for the instances that were not // requested should be decremented upon the // EnqueuePayload too. This will need instance // association to be derived via instances fed into // DequeuePayload call. // However, as multiple instances are provided to // DequeuePayload call only when using device-blocking // and a single consumer thread, we are decrementing the // specific instance consumer count as an approximation. for (const auto instance : instances) { payload_queue->specific_queues_[instance]->DecrementConsumerCount(); } } std::shared_ptr RateLimiter::GetPayload( const Payload::Operation op_type, TritonModelInstance* instance) { std::shared_ptr payload; if (max_payload_bucket_count_ > 0) { std::lock_guard lock(payload_mu_); if (!payload_bucket_.empty()) { payload = payload_bucket_.back(); payload_bucket_.pop_back(); } if (payload.get() == nullptr && (!payloads_in_use_.empty())) { // Just checking the front of the queue instead the entire queue for // an available payload to save time. if (payloads_in_use_.front().use_count() == 1) { payload = payloads_in_use_.front(); payloads_in_use_.pop_front(); } } } if (payload.get() == nullptr) { payload.reset(new Payload()); } payload->Reset(op_type, instance); return payload; } void RateLimiter::PayloadRelease(std::shared_ptr& payload) { // If this is an exit payload, the instance must not be staged once marked as // available, so mark the instance as removing. if (payload->GetOpType() == Payload::Operation::EXIT) { std::lock_guard lk(model_instance_ctx_mtx_); auto it_model = model_instance_ctxs_.find(payload->GetInstance()->Model()); if (it_model == model_instance_ctxs_.end()) { LOG_ERROR << "Should not print this! Releasing payload containing an " "instance of an unknown model."; return; } auto it_instance = it_model->second.find(payload->GetInstance()); if (it_instance == it_model->second.end()) { LOG_ERROR << "Should not print this! Releasing payload containing an " "unknown instance."; return; } it_instance->second->RequestRemoval(); // mark the instance as removing } payload->OnRelease(); if (max_payload_bucket_count_ > 0) { std::lock_guard lock(payload_mu_); if (payloads_in_use_.size() + payload_bucket_.size() < max_payload_bucket_count_) { // Release iff the payload shared_ptr is uniquely held. if (payload.use_count() == 1) { payload->Release(); payload_bucket_.push_back(std::move(payload)); return; } else { payloads_in_use_.push_back(std::move(payload)); } } } } RateLimiter::RateLimiter( const bool ignore_resources_and_priority, const ResourceMap& resource_map) : ignore_resources_and_priority_(ignore_resources_and_priority), max_payload_bucket_count_(MAX_PAYLOAD_BUCKET_COUNT) { ResourceManager::Create(resource_map, &resource_manager_); } void RateLimiter::InitializePayloadQueues(const TritonModelInstance* instance) { auto& config = instance->Model()->Config(); uint64_t max_queue_delay_microseconds; if (config.has_sequence_batching()) { const auto& batcher_config = config.sequence_batching(); if (batcher_config.has_oldest()) { max_queue_delay_microseconds = batcher_config.oldest().max_queue_delay_microseconds(); } else { max_queue_delay_microseconds = 0; } } else if (config.has_dynamic_batching()) { max_queue_delay_microseconds = config.dynamic_batching().max_queue_delay_microseconds(); } else { max_queue_delay_microseconds = 0; } PayloadQueue* payload_queue = nullptr; { std::lock_guard lk(payload_queues_mu_); if (payload_queues_.find(instance->Model()) == payload_queues_.end()) { payload_queues_.emplace( instance->Model(), new PayloadQueue( config.max_batch_size(), max_queue_delay_microseconds * 1000)); } payload_queue = payload_queues_[instance->Model()].get(); } { // NOTE: payload_queue can have a data race because instance->Model() // is the same for multiple instances of same model, so protect it when // creating model instances in parallel. std::lock_guard lk(payload_queue->mu_); if (payload_queue->specific_queues_.find(instance) == payload_queue->specific_queues_.end()) { payload_queue->specific_queues_.emplace( instance, new InstanceQueue( config.max_batch_size(), max_queue_delay_microseconds * 1000)); } } } Status RateLimiter::DeferPayloadSchedule( const StandardScheduleFunc& OnSchedule, const TritonModel* model, TritonModelInstance* triton_model_instance) { std::lock_guard lk(model_ctx_mtx_); auto itr = model_contexts_.find(model); if (itr == model_contexts_.end()) { return Status( Status::Code::INTERNAL, "Requested model is not yet registered with rate limiter"); } if (itr->second.IsRemovalInProgress()) { return Status( Status::Code::INTERNAL, "New model requests can not be made to a model that is being " "removed"); } itr->second.EnqueueModelInstanceRequest(OnSchedule, triton_model_instance); itr->second.StageInstanceIfAvailable(triton_model_instance); return Status::Success; } void RateLimiter::SchedulePayload( TritonModelInstance* tmi, PayloadQueue* payload_queue, const std::shared_ptr& payload) { if (tmi == nullptr) { payload_queue->queue_->Enqueue(payload); } else { payload_queue->specific_queues_[tmi]->Enqueue(payload); } payload->SetState(Payload::State::SCHEDULED); } void RateLimiter::OnStage(ModelInstanceContext* instance) { { std::lock_guard lk(staged_instances_mtx_); staged_instances_.push(instance); } AttemptAllocation(); } void RateLimiter::OnRelease(ModelInstanceContext* instance) { { std::lock_guard lk(model_ctx_mtx_); auto& model_context = model_contexts_[instance->RawInstance()->Model()]; model_context.AddAvailableInstance(instance); resource_manager_->ReleaseResources(instance); if (model_context.ContainsPendingRequests(instance)) { model_context.StageInstanceIfAvailable(instance->RawInstance()); } } AttemptAllocation(); } void RateLimiter::AttemptAllocation() { std::lock_guard lk(staged_instances_mtx_); if (!staged_instances_.empty()) { ModelInstanceContext* instance = staged_instances_.top(); if (resource_manager_->AllocateResources(instance)) { staged_instances_.pop(); instance->Allocate(); } } } //========================================================================= // ModelContext Implementation //========================================================================= Status RateLimiter::ModelContext::EnqueueModelInstanceRequest( const StandardScheduleFunc& OnSchedule, TritonModelInstance* triton_model_instance) { std::lock_guard lk(sched_request_queue_mtx_); if (triton_model_instance == nullptr) { generic_sched_request_queue_.push(OnSchedule); } else { auto it = specific_sched_request_queues_.find(triton_model_instance); if (it != specific_sched_request_queues_.end()) { it->second.push(OnSchedule); } else { return Status( Status::Code::INTERNAL, "instance not added to specific request queue"); } } return Status::Success; } void RateLimiter::ModelContext::AddAvailableInstance(ModelInstanceContext* instance) { std::lock_guard lk(avbl_instances_mtx_); avbl_instances_.push(instance); instance->MarkAvailable(); } void RateLimiter::ModelContext::StageInstanceIfAvailable( TritonModelInstance* req_instance) { std::lock_guard lk1(sched_request_queue_mtx_); std::lock_guard lk2(avbl_instances_mtx_); PriorityQueue backup_queue; while (!avbl_instances_.empty()) { ModelInstanceContext* instance = avbl_instances_.top(); if (instance->IsRemovalInProgress() || (req_instance != nullptr && instance->RawInstance() != req_instance)) { // Skip staging the available instance if either it is being removed or it // is not the requested instance (if specified). backup_queue.push(instance); avbl_instances_.pop(); continue; } if (!specific_sched_request_queues_[instance->RawInstance()].empty()) { // Prioritize the specific requests for the available model // instance highest priority. const StandardScheduleFunc func = specific_sched_request_queues_[instance->RawInstance()].front(); specific_sched_request_queues_[instance->RawInstance()].pop(); instance->Stage(func); } else if (!generic_sched_request_queue_.empty()) { // If request is for generic model instance then use the // instance with the highest priority. const StandardScheduleFunc func = generic_sched_request_queue_.front(); generic_sched_request_queue_.pop(); instance->Stage(func); } else { // If there are requests for a specific model instance then backup // the model instance and keep searching through the available // model instances. The prioritization will be taken care of in the // staging priority queue. backup_queue.push(instance); } avbl_instances_.pop(); } // Restore the backup queue if (!backup_queue.empty()) { avbl_instances_.swap(backup_queue); } } void RateLimiter::ModelContext::AllocateInstanceIfAvailable() { std::lock_guard lk1(sched_request_queue_mtx_); std::lock_guard lk2(avbl_instances_mtx_); PriorityQueue backup_queue; while (!avbl_instances_.empty()) { ModelInstanceContext* instance = avbl_instances_.top(); if (!specific_sched_request_queues_[instance->RawInstance()].empty()) { // Prioritize the specific requests for the available model // instance highest priority. const StandardScheduleFunc func = specific_sched_request_queues_[instance->RawInstance()].front(); specific_sched_request_queues_[instance->RawInstance()].pop(); instance->DirectAllocate(func); } else if (!generic_sched_request_queue_.empty()) { // If request is for generic model instance then use the // instance with the highest priority. const StandardScheduleFunc func = generic_sched_request_queue_.front(); generic_sched_request_queue_.pop(); instance->DirectAllocate(func); } else { // If there are requests for a specific model instance then backup // the model instance and keep searching through the available // model instances. The prioritization will be taken care of in the // staging priority queue. backup_queue.push(instance); } avbl_instances_.pop(); } // Restore the backup queue if (!backup_queue.empty()) { avbl_instances_.swap(backup_queue); } } void RateLimiter::ModelContext::AddSpecificRequestQueue( ModelInstanceContext* instance) { std::lock_guard lk(sched_request_queue_mtx_); specific_sched_request_queues_[instance->RawInstance()]; } bool RateLimiter::ModelContext::ContainsPendingRequests( ModelInstanceContext* instance) { std::lock_guard lk(sched_request_queue_mtx_); return (generic_sched_request_queue_.size() != 0) || (specific_sched_request_queues_[instance->RawInstance()].size() != 0); } void RateLimiter::ModelContext::RemoveInstance(ModelInstanceContext* instance) { std::lock_guard lk1(sched_request_queue_mtx_); std::lock_guard lk2(avbl_instances_mtx_); PriorityQueue new_avbl_instances; while (!avbl_instances_.empty()) { ModelInstanceContext* curr_instance = avbl_instances_.top(); if (curr_instance != instance) { new_avbl_instances.push(curr_instance); } avbl_instances_.pop(); } avbl_instances_.swap(new_avbl_instances); specific_sched_request_queues_.erase(instance->RawInstance()); } //========================================================================= // ModelInstanceContext Implementation //========================================================================= RateLimiter::ModelInstanceContext::ModelInstanceContext( TritonModelInstance* triton_model_instance, RateLimiter::ModelContext* model_context, const RateLimiter::RateLimiterConfig& rate_limiter_config, RateLimiter::StandardStageFunc OnStage, RateLimiter::StandardReleaseFunc OnRelease) : triton_model_instance_(triton_model_instance), model_context_(model_context), rate_limiter_config_(rate_limiter_config), OnStage_(OnStage), OnRelease_(OnRelease), exec_count_(0), state_(AVAILABLE), removal_in_progress_(false) { } void RateLimiter::ModelInstanceContext::MarkAvailable() { std::lock_guard lk(state_mtx_); state_ = AVAILABLE; } Status RateLimiter::ModelInstanceContext::Stage(StandardScheduleFunc OnSchedule) { { std::lock_guard lk(state_mtx_); if (state_ != AVAILABLE) { return Status( Status::Code::INTERNAL, "Can not stage a model instance that is not yet available"); } state_ = STAGED; OnSchedule_ = OnSchedule; } OnStage_(this); return Status::Success; } Status RateLimiter::ModelInstanceContext::Allocate() { { std::lock_guard lk(state_mtx_); if (state_ != STAGED) { return Status( Status::Code::INTERNAL, "Can not allocate a model instance that is not yet staged"); } state_ = ALLOCATED; } OnSchedule_(this); return Status::Success; } Status RateLimiter::ModelInstanceContext::DirectAllocate( StandardScheduleFunc OnSchedule) { { std::lock_guard lk(state_mtx_); if (state_ != AVAILABLE) { return Status( Status::Code::INTERNAL, "Can not allocate a model instance that is not yet available"); } state_ = ALLOCATED; } OnSchedule(this); return Status::Success; } void RateLimiter::ModelInstanceContext::Release() { exec_count_++; OnRelease_(this); } double RateLimiter::ModelInstanceContext::ScaledPriority() { // TODO: Different schemes for the prioritization of // model instance can be added here. // The priority of instance is 1 by default. If specified // as 0, the priority is still treated as 1. auto priority = std::max(rate_limiter_config_.priority(), 1u); return (exec_count_ * priority); } void RateLimiter::ModelInstanceContext::RequestRemoval() { std::unique_lock lk(state_mtx_); removal_in_progress_ = true; } bool RateLimiter::ModelInstanceContext::IsRemovalInProgress() { std::unique_lock lk(state_mtx_); return removal_in_progress_; } //========================================================================= // ResourceManager Implementation //========================================================================= Status RateLimiter::ResourceManager::Create( const ResourceMap& resource_map, std::unique_ptr* resource_manager) { std::unique_ptr local_resource_manager( new ResourceManager(resource_map)); *resource_manager = std::move(local_resource_manager); return Status::Success; } Status RateLimiter::ResourceManager::AddModelInstance( const ModelInstanceContext* instance) { // Add instance into model resources. std::lock_guard lk1(model_resources_mtx_); auto pr = model_resources_.emplace(std::make_pair(instance, ResourceMap())); for (const auto& resource : instance->GetRateLimiterConfig()->resources()) { if (resource.global()) { (pr.first->second[GLOBAL_RESOURCE_KEY])[resource.name()] = resource.count(); } else { (pr.first->second[instance->RawInstance()->DeviceId()])[resource.name()] = resource.count(); } } // Increase max resource if needed. std::lock_guard lk2(max_resources_mtx_); UpdateMaxResource(pr.first->second); return ParseAndValidateResources(); } Status RateLimiter::ResourceManager::RemoveModelInstance( const ModelInstanceContext* instance) { // Find instance from model resources. std::lock_guard lk1(model_resources_mtx_); const auto& itr = model_resources_.find(instance); if (itr == model_resources_.end()) { return Status(Status::Code::INTERNAL, "Cannot find the instance to remove"); } // Check if max resources need to be updated. bool update_needed = false; std::lock_guard lk2(max_resources_mtx_); for (const auto& resource_device_map : itr->second) { auto ditr = max_resources_.find(resource_device_map.first); if (ditr != max_resources_.end()) { for (const auto& resource : resource_device_map.second) { auto ritr = ditr->second.find(resource.first); if (ritr != ditr->second.end() && ritr->second <= resource.second) { update_needed = true; if (ritr->second < resource.second) { LOG_ERROR << "Should not print this! Removing an instance with " "resource above max resource."; } break; } } } if (update_needed) { break; } } // Remove instance from model resources. model_resources_.erase(instance); // Re-compute max resource if needed. if (update_needed) { ComputeResourceLimits(); } return ParseAndValidateResources(); } void RateLimiter::ResourceManager::ComputeResourceLimits() { // Obtain the maximum resource across all the instances and use it as the // default available. max_resources_.clear(); for (const auto& instance_resources : model_resources_) { UpdateMaxResource(instance_resources.second); } } void RateLimiter::ResourceManager::UpdateMaxResource( const ResourceMap& instance_resource_map) { for (const auto& resource_device_map : instance_resource_map) { auto ditr = max_resources_.find(resource_device_map.first); if (ditr == max_resources_.end()) { ditr = max_resources_ .emplace(resource_device_map.first, resource_device_map.second) .first; } else { for (const auto& resource : resource_device_map.second) { auto ritr = ditr->second.find(resource.first); if (ritr == ditr->second.end()) { ritr = ditr->second.emplace(resource.first, resource.second).first; } else { if (ritr->second < resource.second) { ritr->second = resource.second; } } } } } } Status RateLimiter::ResourceManager::ParseAndValidateResources() { if (!explicit_max_resources_.empty()) { RETURN_IF_ERROR(ParseAndValidateExplicitResources()); } RETURN_IF_ERROR(ValidateMaxResources()); if (LOG_VERBOSE_IS_ON(1)) { std::string resource_map_str{"\nMax Resource Map===>\n"}; for (const auto& ditr : max_resources_) { if (!ditr.second.empty()) { std::string device_str{ (ditr.first == GLOBAL_RESOURCE_KEY) ? "GLOBAL" : std::to_string(ditr.first)}; resource_map_str += "\tDevice: " + device_str + "\n"; for (const auto& ritr : ditr.second) { resource_map_str += "\t\tResource: " + ritr.first + "\t Count: " + std::to_string(ritr.second) + "\n"; } } } LOG_VERBOSE(1) << resource_map_str; } return Status::Success; } Status RateLimiter::ResourceManager::ValidateMaxResources() { for (const auto& global_resource : max_resources_[GLOBAL_RESOURCE_KEY]) { for (const auto& ditr : max_resources_) { if (ditr.first != GLOBAL_RESOURCE_KEY) { for (const auto& ritr : ditr.second) { if (global_resource.first.compare(ritr.first) == 0) { return Status( Status::Code::INVALID_ARG, (std::string("Resource \"") + ritr.first + "\" is present as both global and device-specific resource in " "the model configuration.") .c_str()); } } } } } return Status::Success; } Status RateLimiter::ResourceManager::ParseAndValidateExplicitResources() { for (auto& ditr : max_resources_) { for (auto& ritr : ditr.second) { // If not specified explicitly, consider the resource to be unavailable. size_t resource_count = 0; if (ditr.first == GLOBAL_RESOURCE_KEY) { // Ignore the device specification... will search for all resources in // the map... for (const auto& exp_ditr : explicit_max_resources_) { for (const auto& exp_ritr : exp_ditr.second) { if (ritr.first.compare(exp_ritr.first) == 0) { if (resource_count < exp_ritr.second) { resource_count = exp_ritr.second; } } } } } else { // Search only for the device specific or per-device resources... // device-specific for (const auto& exp_ritr : explicit_max_resources_[ditr.first]) { if (ritr.first.compare(exp_ritr.first) == 0) { if (resource_count < exp_ritr.second) { resource_count = exp_ritr.second; } } } // per-device for (const auto& exp_ritr : explicit_max_resources_[PER_DEVICE_RESOURCE_KEY]) { if (ritr.first.compare(exp_ritr.first) == 0) { if (resource_count < exp_ritr.second) { resource_count = exp_ritr.second; } } } } if (resource_count < ritr.second) { return Status( Status::Code::INVALID_ARG, (std::string("Resource count for \"") + ritr.first + "\" is limited to " + std::to_string(resource_count) + " which will prevent scheduling of one or more model " "instances, the minimum required count is " + std::to_string(ritr.second)) .c_str()); } else { ritr.second = resource_count; } } } return Status::Success; } bool RateLimiter::ResourceManager::AllocateResources( const ModelInstanceContext* instance) { std::lock_guard lk1(model_resources_mtx_); std::lock_guard lk2(allocated_resources_mtx_); const auto& itr = model_resources_.find(instance); if (itr == model_resources_.end()) { return false; } else { // First pass to verify if resources are available { std::lock_guard lk3(max_resources_mtx_); for (const auto& ditr : itr->second) { auto allocated_ditr = allocated_resources_.find(ditr.first); if (allocated_ditr == allocated_resources_.end()) { allocated_ditr = allocated_resources_ .emplace(ditr.first, std::map()) .first; } for (const auto& ritr : ditr.second) { auto allocated_ritr = allocated_ditr->second.find(ritr.first); if (allocated_ritr == allocated_ditr->second.end()) { allocated_ritr = allocated_ditr->second.emplace(ritr.first, 0).first; } if ((allocated_ritr->second + ritr.second) > (max_resources_[ditr.first])[ritr.first]) { return false; } } } } // Second pass to actually allocate the resources for (const auto& ditr : itr->second) { for (const auto& ritr : ditr.second) { (allocated_resources_[ditr.first])[ritr.first] += ritr.second; } } } return true; } Status RateLimiter::ResourceManager::ReleaseResources( const ModelInstanceContext* instance) { std::lock_guard lk1(model_resources_mtx_); std::lock_guard lk2(allocated_resources_mtx_); const auto& itr = model_resources_.find(instance); if (itr == model_resources_.end()) { return Status( Status::Code::INTERNAL, "Unable find the instance resources to release"); } else { for (const auto& ditr : itr->second) { for (const auto& ritr : ditr.second) { (allocated_resources_[ditr.first])[ritr.first] -= ritr.second; } } } return Status::Success; } RateLimiter::ResourceManager::ResourceManager(const ResourceMap& resource_map) : explicit_max_resources_(resource_map) { } }} // namespace triton::core kytensor-2.49.0.7/core/src/rate_limiter.h000066400000000000000000000365661527052371200202250ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include "backend_model.h" #include "backend_model_instance.h" #include "instance_queue.h" #include "model_config.pb.h" #include "payload.h" #include "status.h" namespace triton { namespace core { // Limits the rate at which requests are dispatched to the model instances class RateLimiter { public: using RateLimiterConfig = inference::ModelRateLimiter; using ResourceMap = std::map>; enum RESOURCE_KIND_KEY { // Key for holding global resources GLOBAL_RESOURCE_KEY = -2, // Key for holding resources per each device PER_DEVICE_RESOURCE_KEY = -1 }; /// Creates a rate limiter object which will funnel the requests to /// the model instances. A typical lifetime of the model instance within /// RateLimiter transition from available -> staged -> allocated -> available. /// The transition from available to staged occurs when a request is /// registered for the model. Depending upon the resource availability and /// priority, the RateLimiter will transition an instance to allocated state /// at some point in the future. The staged state is skipped when /// configured to ignore the resource constraints. The cycle in this case /// will be available -> allocated -> available. /// \param ignore_resources_and_priority Whether or not to ignore resource /// constraints and cross-model priority. An available instance is directly /// allocated when true. /// \param resource_map The map to the available resource count provided /// explicitly. /// \return Status object indicating success or failure. static Status Create( const bool ignore_resources_and_priority, const ResourceMap& resource_map, std::unique_ptr* rate_limiter); /// Registers the model instance with the rate limiter. /// \param instance The pointer to the TritonModelInstance object to register /// with the rate limiter. /// \param rate_limiter_config The rate limiter configuration associated with /// the model instance. /// \return Status object indicating success or failure. Status RegisterModelInstance( TritonModelInstance* instance, const RateLimiterConfig& rate_limiter_config); /// Unregisters the model instance with the rate limiter. /// \param instance The pointer to the TritonModelInstance object to /// unregister with the rate limiter. void UnregisterModelInstance(TritonModelInstance* instance); /// Remove model from the set of models being managed by the rate limiter. /// \param model The pointer to TritonModel object to be removed. void UnregisterModel(const TritonModel* model); /// Returns true if there is a payload slot available for the given model. /// Note the function can be a blocking call when support_prefetching is /// false. In this case, the function will block until a slot is available to /// start building the payload. force_non_blocking option can be set to True /// to allow function to return back with availability. /// \param model The pointer to TritonModel object to query for. /// \param model_instance The pointer to TritonMode /// \param support_prefetching Whether or not pre-fetching of payloads is /// enabled. /// \param force_non_blocking When set true, function will not block for /// the availability of the slot. /// \return slot availability in boolean. bool PayloadSlotAvailable( const TritonModel* model, const TritonModelInstance* model_instance, const bool support_prefetching, const bool force_non_blocking = false); /// Enqueues the payload to rate limiter for scheduling on the given model. /// \param model The pointer to TritonModel object to be removed. /// \param payload The shared pointer to the payload object. /// \return Status object indicating success or failure. Status EnqueuePayload( const TritonModel* model, std::shared_ptr payload); /// Returns the payload that has been scheduled for the given set of model /// instances. Note that this call is blocking and depends upon the /// availability of payloads in the rate limiter for the triton model /// instance. /// \param instance The pointers to TritonModelInstance objects whose /// payload is being requested. /// \param payload The shared pointer to the payload object. void DequeuePayload( std::deque& instance, std::shared_ptr* payload); /// Returns a new payload object. /// \param op_type The operation type for the payload. /// \param instance Optional field that providess the model instance that must /// be used for the execution of the payload. Default is nullptr which allows /// any model instance to execute the payload. /// \return The shared pointer to a new payload object. std::shared_ptr GetPayload( const Payload::Operation op_type, TritonModelInstance* instance = nullptr); /// Releases the given payload object back to the rate limiter. /// \param payload The payload to release. void PayloadRelease(std::shared_ptr& payload); private: class ModelInstanceContext; class ModelContext; struct PayloadQueue; using StandardReleaseFunc = std::function; using StandardScheduleFunc = std::function; using StandardStageFunc = std::function; // Holds the state of the model instance. class ModelInstanceContext { public: friend class RateLimiter; friend class ResourceManager; enum State { AVAILABLE, STAGED, ALLOCATED, REMOVED }; void Release(); TritonModelInstance* RawInstance() const { return triton_model_instance_; } private: ModelInstanceContext( TritonModelInstance* triton_model_instance, ModelContext* model_context, const RateLimiterConfig& rate_limiter_config, StandardStageFunc OnStage, StandardReleaseFunc OnRelease); const RateLimiterConfig* GetRateLimiterConfig() const { return &rate_limiter_config_; } void MarkAvailable(); double ScaledPriority(); Status Stage(StandardScheduleFunc OnSchedule); Status Allocate(); Status DirectAllocate(StandardScheduleFunc OnSchedule); void RequestRemoval(); bool IsRemovalInProgress(); TritonModelInstance* triton_model_instance_; ModelContext* model_context_; RateLimiterConfig rate_limiter_config_; StandardStageFunc OnStage_; StandardReleaseFunc OnRelease_; std::atomic exec_count_; State state_; bool removal_in_progress_; std::mutex state_mtx_; StandardScheduleFunc OnSchedule_; }; class ScaledPriorityComparator { public: bool operator()(ModelInstanceContext* a, ModelInstanceContext* b) { return a->ScaledPriority() > b->ScaledPriority(); } }; using PriorityQueue = std::priority_queue< ModelInstanceContext*, std::vector, ScaledPriorityComparator>; // Holds the active context to a model class ModelContext { public: ModelContext() : removal_in_progress_(false) {} // Enqueue request for obtaining a model instance for scheduling // a inference payload execution. Status EnqueueModelInstanceRequest( const StandardScheduleFunc& OnSchedule, TritonModelInstance* triton_model_instance); // Marks the given instance of the model as available and ready // to be staged. void AddAvailableInstance(ModelInstanceContext* instance); // Attempts to stage the instance given upon its availability. void StageInstanceIfAvailable(TritonModelInstance* triton_model_instance); // Allocates one of the staged model instance for execution if resources // are available in the system. void AllocateInstanceIfAvailable(); // Adds a queue in the model context for holding requests meant for // running on the given instance. void AddSpecificRequestQueue(ModelInstanceContext* instance); // Whether or not there are any requests waiting for execution on the given // model instance. bool ContainsPendingRequests(ModelInstanceContext* instance); // Remove the given instance of the model. `WaitForRemoval()` on the // instance should have been called and returned. void RemoveInstance(ModelInstanceContext* instance); // Starts the removal of the model context from scheduling purposes. // Will wait for all enqueued model instance requests to complete. void RequestRemoval() { removal_in_progress_ = true; } // Whether or not model context is decommissioned bool IsRemovalInProgress() { return removal_in_progress_; } private: bool removal_in_progress_; // Queue holding pending scheduling request std::queue generic_sched_request_queue_; std::map> specific_sched_request_queues_; std::recursive_mutex sched_request_queue_mtx_; // The set of instances that are available at the moment PriorityQueue avbl_instances_; std::recursive_mutex avbl_instances_mtx_; }; // Manages and keep track of resource allocation to the model instances. class ResourceManager { public: static Status Create( const ResourceMap& resource_map, std::unique_ptr* resource_manager); // Adds the model instance to the resource manager Status AddModelInstance(const ModelInstanceContext* instance); // Removes the model instance from the resource manager Status RemoveModelInstance(const ModelInstanceContext* instance); // Allocate resources for the given model instance. Returns // false if resources are not available at this time. bool AllocateResources(const ModelInstanceContext* instance); // Releases resources held by the given model instance back to // the available pool. Status ReleaseResources(const ModelInstanceContext* instance); private: ResourceManager(const ResourceMap& resource_map); void ComputeResourceLimits(); void UpdateMaxResource(const ResourceMap& instance_resource_map); Status ParseAndValidateResources(); Status ValidateMaxResources(); Status ParseAndValidateExplicitResources(); ResourceMap explicit_max_resources_; std::map model_resources_; std::mutex model_resources_mtx_; ResourceMap max_resources_; std::mutex max_resources_mtx_; ResourceMap allocated_resources_; std::mutex allocated_resources_mtx_; }; RateLimiter( const bool ignore_resources_and_priority, const ResourceMap& resource_map); // Initializes payload queues for the given model instance. The queue // holds payloads that get scheduled by rate limiter. void InitializePayloadQueues(const TritonModelInstance* instance); // Should wait till a consumer registers a pending dequeue request // for the given instance(s) of the model. This implies that the // call will wait for an idle runner. void WaitForConsumer( const TritonModel* model, const TritonModelInstance* model_instance); // Returns the number of consumers who have a pending dequeue request for // the given instance(s) of the model. int WaitingConsumerCount( const TritonModel* model, const TritonModelInstance* model_instance); // Defers scheduling of the payload to the future. Rate Limiter will // schedule the payload execution based upon the resource availability/ // Note that OnSchedule function should only schedule(enqueued in payload // queue) the payload and not execute it. Status DeferPayloadSchedule( const StandardScheduleFunc& OnSchedule, const TritonModel* model, TritonModelInstance* instance = nullptr); // Callback function to stage the instance. void OnStage(ModelInstanceContext* instance_ptr); // Callback function to release resources allocated to the instance // and attempt allocating available resources to next staged instance. void OnRelease(ModelInstanceContext* instance_ptr); // Attempt allocating the resources for the staged instance with // highest priority. void AttemptAllocation(); // Schedules the payload for execution on model instance. void SchedulePayload( TritonModelInstance* tmi, PayloadQueue* payload_queue, const std::shared_ptr& payload); bool ignore_resources_and_priority_; // Instance context for the models std::map< const TritonModel*, std::map< const TritonModelInstance*, std::unique_ptr>> model_instance_ctxs_; std::mutex model_instance_ctx_mtx_; // Running context of the models std::map model_contexts_; std::mutex model_ctx_mtx_; // Holds the model instances that have been staged PriorityQueue staged_instances_; std::recursive_mutex staged_instances_mtx_; // Manager to keep track of the resource allocations std::unique_ptr resource_manager_; std::mutex resource_manager_mtx_; // Mutex to serialize Payload [de]allocation std::mutex payload_mu_; // Mutex to serialize Payload Queues deallocation std::mutex payload_queues_mu_; // Keep some number of Payload objects for reuse to avoid the overhead // of creating a Payload for every new request. const size_t max_payload_bucket_count_; std::vector> payload_bucket_; std::deque> payloads_in_use_; struct PayloadQueue { explicit PayloadQueue(size_t max_batch_size, uint64_t max_queue_delay_ns) { queue_.reset(new InstanceQueue(max_batch_size, max_queue_delay_ns)); } std::unique_ptr queue_; std::map> specific_queues_; std::mutex mu_; std::condition_variable cv_; }; std::map> payload_queues_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/repo_agent.cc000066400000000000000000000474111527052371200200150ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "repo_agent.h" #include #include "filesystem/api.h" #include "shared_library.h" #include "triton/common/logging.h" #include "tritonserver_apis.h" // For unknown reason, windows will not export the TRITONREPOAGENT_* // functions declared with dllexport in tritonrepoagent.h. To get // those functions exported it is (also?) necessary to mark the // definitions in this file with dllexport as well. #if defined(_MSC_VER) #define TRITONAPI_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONAPI_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONAPI_DECLSPEC #endif namespace triton { namespace core { std::string TritonRepoAgentLibraryName(const std::string& agent_name) { #ifdef _WIN32 return std::string("tritonrepoagent_") + agent_name + ".dll"; #else return std::string("libtritonrepoagent_") + agent_name + ".so"; #endif } std::string TRITONREPOAGENT_ActionTypeString(const TRITONREPOAGENT_ActionType type) { switch (type) { case TRITONREPOAGENT_ACTION_LOAD: return "TRITONREPOAGENT_ACTION_LOAD"; case TRITONREPOAGENT_ACTION_LOAD_COMPLETE: return "TRITONREPOAGENT_ACTION_LOAD_COMPLETE"; case TRITONREPOAGENT_ACTION_LOAD_FAIL: return "TRITONREPOAGENT_ACTION_LOAD_FAIL"; case TRITONREPOAGENT_ACTION_UNLOAD: return "TRITONREPOAGENT_ACTION_UNLOAD"; case TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: return "TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE"; } return "Unknown TRITONREPOAGENT_ActionType"; } std::string TRITONREPOAGENT_ArtifactTypeString(const TRITONREPOAGENT_ArtifactType type) { switch (type) { case TRITONREPOAGENT_ARTIFACT_FILESYSTEM: return "TRITONREPOAGENT_ARTIFACT_FILESYSTEM"; case TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: return "TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM"; } return "Unknown TRITONREPOAGENT_ArtifactType"; } // // TritonRepoAgent // Status TritonRepoAgent::Create( const std::string& name, const std::string& libpath, std::shared_ptr* agent) { std::shared_ptr lagent(new TritonRepoAgent(name)); { std::unique_ptr slib; RETURN_IF_ERROR(SharedLibrary::Acquire(&slib)); RETURN_IF_ERROR(slib->OpenLibraryHandle(libpath, &lagent->dlhandle_)); RETURN_IF_ERROR(slib->GetEntrypoint( lagent->dlhandle_, "TRITONREPOAGENT_Initialize", true /* optional */, reinterpret_cast(&lagent->init_fn_))); RETURN_IF_ERROR(slib->GetEntrypoint( lagent->dlhandle_, "TRITONREPOAGENT_Finalize", true /* optional */, reinterpret_cast(&lagent->fini_fn_))); RETURN_IF_ERROR(slib->GetEntrypoint( lagent->dlhandle_, "TRITONREPOAGENT_ModelInitialize", true /* optional */, reinterpret_cast(&lagent->model_init_fn_))); RETURN_IF_ERROR(slib->GetEntrypoint( lagent->dlhandle_, "TRITONREPOAGENT_ModelFinalize", true /* optional */, reinterpret_cast(&lagent->model_fini_fn_))); RETURN_IF_ERROR(slib->GetEntrypoint( lagent->dlhandle_, "TRITONREPOAGENT_ModelAction", false /* optional */, reinterpret_cast(&lagent->model_action_fn_))); } // Initialize if needed if (lagent->init_fn_ != nullptr) { RETURN_IF_TRITONSERVER_ERROR(lagent->init_fn_( reinterpret_cast(lagent.get()))); } *agent = std::move(lagent); return Status::Success; } TritonRepoAgent::~TritonRepoAgent() { // Finalize if needed if (fini_fn_ != nullptr) { auto err = fini_fn_(reinterpret_cast(this)); if (err != nullptr) { LOG_ERROR << "~TritonRepoAgent: " << Status( TritonCodeToStatusCode(TRITONSERVER_ErrorCode(err)), TRITONSERVER_ErrorMessage(err)) .AsString(); TRITONSERVER_ErrorDelete(err); }; } { std::unique_ptr slib; LOG_STATUS_ERROR(SharedLibrary::Acquire(&slib), "~TritonRepoAgent"); LOG_STATUS_ERROR(slib->CloseLibraryHandle(dlhandle_), "~TritonRepoAgent"); } } // // TritonRepoAgentModel // Status TritonRepoAgentModel::Create( const TRITONREPOAGENT_ArtifactType type, const std::string& location, const inference::ModelConfig& config, const std::shared_ptr& agent, const TritonRepoAgent::Parameters& agent_parameters, std::unique_ptr* agent_model) { std::unique_ptr lagent_model(new TritonRepoAgentModel( type, location, config, agent, agent_parameters)); if (agent->AgentModelInitFn() != nullptr) { RETURN_IF_TRITONSERVER_ERROR(agent->AgentModelInitFn()( reinterpret_cast(agent.get()), reinterpret_cast(lagent_model.get()))); } *agent_model = std::move(lagent_model); return Status::Success; } TritonRepoAgentModel::~TritonRepoAgentModel() { // Need to ensure the proper lifecycle is informed if (action_type_set_) { switch (current_action_type_) { case TRITONREPOAGENT_ACTION_LOAD: LOG_TRITONSERVER_ERROR( agent_->AgentModelActionFn()( reinterpret_cast(agent_.get()), reinterpret_cast(this), TRITONREPOAGENT_ACTION_LOAD_FAIL), "Inform TRITONREPOAGENT_ACTION_LOAD_FAIL"); break; case TRITONREPOAGENT_ACTION_LOAD_COMPLETE: LOG_TRITONSERVER_ERROR( agent_->AgentModelActionFn()( reinterpret_cast(agent_.get()), reinterpret_cast(this), TRITONREPOAGENT_ACTION_UNLOAD), "Inform TRITONREPOAGENT_ACTION_UNLOAD"); // Fallthough is not yet an language feature until C++17 LOG_TRITONSERVER_ERROR( agent_->AgentModelActionFn()( reinterpret_cast(agent_.get()), reinterpret_cast(this), TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE), "Inform TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE"); break; case TRITONREPOAGENT_ACTION_UNLOAD: LOG_TRITONSERVER_ERROR( agent_->AgentModelActionFn()( reinterpret_cast(agent_.get()), reinterpret_cast(this), TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE), "Inform TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE"); break; case TRITONREPOAGENT_ACTION_LOAD_FAIL: case TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: break; } } if (agent_->AgentModelFiniFn() != nullptr) { LOG_TRITONSERVER_ERROR( agent_->AgentModelFiniFn()( reinterpret_cast(agent_.get()), reinterpret_cast(this)), "~TritonRepoAgentModel"); } if (!acquired_location_.empty()) { DeleteMutableLocation(); } } Status TritonRepoAgentModel::InvokeAgent(const TRITONREPOAGENT_ActionType action_type) { if ((!action_type_set_) && (action_type != TRITONREPOAGENT_ACTION_LOAD)) { return Status( Status::Code::INTERNAL, "Unexpected lifecycle start state " + TRITONREPOAGENT_ActionTypeString(action_type)); } switch (action_type) { case TRITONREPOAGENT_ACTION_LOAD: if (action_type_set_) { return Status( Status::Code::INTERNAL, "Unexpected lifecycle state transition from " + TRITONREPOAGENT_ActionTypeString(current_action_type_) + " to " + TRITONREPOAGENT_ActionTypeString(action_type)); } break; case TRITONREPOAGENT_ACTION_LOAD_COMPLETE: case TRITONREPOAGENT_ACTION_LOAD_FAIL: if (current_action_type_ != TRITONREPOAGENT_ACTION_LOAD) { return Status( Status::Code::INTERNAL, "Unexpected lifecycle state transition from " + TRITONREPOAGENT_ActionTypeString(current_action_type_) + " to " + TRITONREPOAGENT_ActionTypeString(action_type)); } break; case TRITONREPOAGENT_ACTION_UNLOAD: if (current_action_type_ != TRITONREPOAGENT_ACTION_LOAD_COMPLETE) { return Status( Status::Code::INTERNAL, "Unexpected lifecycle state transition from " + TRITONREPOAGENT_ActionTypeString(current_action_type_) + " to " + TRITONREPOAGENT_ActionTypeString(action_type)); } break; case TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: if (current_action_type_ != TRITONREPOAGENT_ACTION_UNLOAD) { return Status( Status::Code::INTERNAL, "Unexpected lifecycle state transition from " + TRITONREPOAGENT_ActionTypeString(current_action_type_) + " to " + TRITONREPOAGENT_ActionTypeString(action_type)); } break; } current_action_type_ = action_type; action_type_set_ = true; RETURN_IF_TRITONSERVER_ERROR(agent_->AgentModelActionFn()( reinterpret_cast(agent_.get()), reinterpret_cast(this), action_type)); return Status::Success; } Status TritonRepoAgentModel::SetLocation( const TRITONREPOAGENT_ArtifactType type, const std::string& location) { if (current_action_type_ != TRITONREPOAGENT_ACTION_LOAD) { return Status( Status::Code::INVALID_ARG, "location can only be updated during TRITONREPOAGENT_ACTION_LOAD, " "current action type is " + (action_type_set_ ? TRITONREPOAGENT_ActionTypeString(current_action_type_) : "not set")); } type_ = type; location_ = location; return Status::Success; } Status TritonRepoAgentModel::Location( TRITONREPOAGENT_ArtifactType* type, const char** location) { if (location_.empty()) { return Status( Status::Code::INTERNAL, "Model repository location is not set"); } *type = type_; *location = location_.c_str(); return Status::Success; } Status TritonRepoAgentModel::AcquireMutableLocation( const TRITONREPOAGENT_ArtifactType type, const char** location) { if (type != TRITONREPOAGENT_ARTIFACT_FILESYSTEM) { return Status( Status::Code::INVALID_ARG, "Unexpected artifact type, expects " "'TRITONREPOAGENT_ARTIFACT_FILESYSTEM'"); } if (acquired_location_.empty()) { std::string lacquired_location; RETURN_IF_ERROR( MakeTemporaryDirectory(FileSystemType::LOCAL, &lacquired_location)); acquired_location_.swap(lacquired_location); acquired_type_ = type; } *location = acquired_location_.c_str(); return Status::Success; } Status TritonRepoAgentModel::DeleteMutableLocation() { if (acquired_location_.empty()) { return Status( Status::Code::UNAVAILABLE, "No mutable location to be deleted"); } auto status = DeletePath(acquired_location_); if (!status.IsOk()) { LOG_ERROR << "Failed to delete previously acquired location '" << acquired_location_ << "': " << status.AsString(); } acquired_location_.clear(); return Status::Success; } // // TritonRepoAgentManager // TritonRepoAgentManager& TritonRepoAgentManager::Singleton() { static TritonRepoAgentManager triton_repo_agent_manager; return triton_repo_agent_manager; } Status TritonRepoAgentManager::SetGlobalSearchPath(const std::string& path) { auto& singleton_manager = Singleton(); std::lock_guard lock(singleton_manager.mu_); singleton_manager.global_search_path_ = path; return Status::Success; } Status TritonRepoAgentManager::CreateAgent( const std::string& agent_name, std::shared_ptr* agent) { auto& singleton_manager = Singleton(); std::lock_guard lock(singleton_manager.mu_); // Get the path to the agent shared library. Search path is global // agent directory. FIXME expose global path as Triton option const std::vector search_paths = { JoinPath({singleton_manager.global_search_path_, agent_name})}; std::string agent_libname = TritonRepoAgentLibraryName(agent_name); std::string libpath; for (const auto& path : search_paths) { const auto full_path = JoinPath({path, agent_libname}); bool exists = false; RETURN_IF_ERROR(FileExists(full_path, &exists)); if (exists) { libpath = full_path; break; } } if (libpath.empty()) { return Status( Status::Code::INVALID_ARG, "unable to find '" + agent_libname + "' for repo agent '" + agent_name + "', searched: " + singleton_manager.global_search_path_); } const auto& itr = singleton_manager.agent_map_.find(libpath); if (itr != singleton_manager.agent_map_.end()) { // Found in map. If the weak_ptr is still valid that means that // there are other models using the agent and we just reuse that // same agent. If the weak_ptr is not valid then agent has been // unloaded so we need to remove the weak_ptr from the map and // create the agent again. *agent = itr->second.lock(); if (*agent != nullptr) { return Status::Success; } singleton_manager.agent_map_.erase(itr); } RETURN_IF_ERROR(TritonRepoAgent::Create(agent_name, libpath, agent)); singleton_manager.agent_map_.insert({libpath, *agent}); return Status::Success; } Status TritonRepoAgentManager::AgentState( std::unique_ptr>* agent_state) { auto& singleton_manager = Singleton(); std::lock_guard lock(singleton_manager.mu_); std::unique_ptr> agent_state_map( new std::unordered_map); for (const auto& agent_pair : singleton_manager.agent_map_) { auto& libpath = agent_pair.first; auto agent = agent_pair.second.lock(); if (agent != nullptr) { agent_state_map->insert({agent->Name(), libpath}); } } *agent_state = std::move(agent_state_map); return Status::Success; } extern "C" { TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ApiVersion(uint32_t* major, uint32_t* minor) { *major = TRITONREPOAGENT_API_VERSION_MAJOR; *minor = TRITONREPOAGENT_API_VERSION_MINOR; return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelRepositoryLocation( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, TRITONREPOAGENT_ArtifactType* artifact_type, const char** location) { TritonRepoAgentModel* tam = reinterpret_cast(model); RETURN_TRITONSERVER_ERROR_IF_ERROR(tam->Location(artifact_type, location)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelRepositoryLocationAcquire( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ArtifactType artifact_type, const char** location) { TritonRepoAgentModel* tam = reinterpret_cast(model); RETURN_TRITONSERVER_ERROR_IF_ERROR( tam->AcquireMutableLocation(artifact_type, location)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelRepositoryLocationRelease( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const char* location) { TritonRepoAgentModel* tam = reinterpret_cast(model); RETURN_TRITONSERVER_ERROR_IF_ERROR(tam->DeleteMutableLocation()); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelRepositoryUpdate( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ArtifactType artifact_type, const char* location) { TritonRepoAgentModel* tam = reinterpret_cast(model); RETURN_TRITONSERVER_ERROR_IF_ERROR(tam->SetLocation(artifact_type, location)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelParameterCount( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, uint32_t* count) { TritonRepoAgentModel* tam = reinterpret_cast(model); *count = tam->AgentParameters().size(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelParameter( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const uint32_t index, const char** parameter_name, const char** parameter_value) { TritonRepoAgentModel* tam = reinterpret_cast(model); const auto& params = tam->AgentParameters(); if (index >= params.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "index out of range for model parameters"); } *parameter_name = params[index].first.c_str(); *parameter_value = params[index].second.c_str(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelConfig( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const uint32_t config_version, TRITONSERVER_Message** model_config) { TritonRepoAgentModel* tam = reinterpret_cast(model); std::string model_config_json; RETURN_TRITONSERVER_ERROR_IF_ERROR( ModelConfigToJson(tam->Config(), config_version, &model_config_json)); return TRITONSERVER_MessageNewFromSerializedJson( model_config, model_config_json.c_str(), model_config_json.length()); } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelState(TRITONREPOAGENT_AgentModel* model, void** state) { TritonRepoAgentModel* tam = reinterpret_cast(model); *state = tam->State(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelSetState(TRITONREPOAGENT_AgentModel* model, void* state) { TritonRepoAgentModel* tam = reinterpret_cast(model); tam->SetState(state); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_State(TRITONREPOAGENT_Agent* agent, void** state) { TritonRepoAgent* ta = reinterpret_cast(agent); *state = ta->State(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_SetState(TRITONREPOAGENT_Agent* agent, void* state) { TritonRepoAgent* ta = reinterpret_cast(agent); ta->SetState(state); return nullptr; // success } } // extern C }} // namespace triton::core kytensor-2.49.0.7/core/src/repo_agent.h000066400000000000000000000147241527052371200176600ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "constants.h" #include "model_config_utils.h" #include "tritonserver_apis.h" namespace triton { namespace core { std::string TritonRepoAgentLibraryName(const std::string& agent_name); std::string TRITONREPOAGENT_ActionTypeString( const TRITONREPOAGENT_ActionType type); std::string TRITONREPOAGENT_ArtifactTypeString( const TRITONREPOAGENT_ArtifactType type); class TritonRepoAgent { public: using Parameters = std::vector>; typedef TRITONSERVER_Error* (*TritonRepoAgentInitFn_t)( TRITONREPOAGENT_Agent* agent); typedef TRITONSERVER_Error* (*TritonRepoAgentFiniFn_t)( TRITONREPOAGENT_Agent* agent); typedef TRITONSERVER_Error* (*TritonRepoAgentModelInitFn_t)( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model); typedef TRITONSERVER_Error* (*TritonRepoAgentModelFiniFn_t)( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model); typedef TRITONSERVER_Error* (*TritonRepoAgentModelActionFn_t)( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type); static Status Create( const std::string& name, const std::string& libpath, std::shared_ptr* agent); ~TritonRepoAgent(); const std::string& Name() { return name_; } void* State() { return state_; } void SetState(void* state) { state_ = state; } TritonRepoAgentModelActionFn_t AgentModelActionFn() const { return model_action_fn_; } TritonRepoAgentModelInitFn_t AgentModelInitFn() const { return model_init_fn_; } TritonRepoAgentModelFiniFn_t AgentModelFiniFn() const { return model_fini_fn_; } protected: DISALLOW_COPY_AND_ASSIGN(TritonRepoAgent); TritonRepoAgent(const std::string& name) : name_(name), state_(nullptr), dlhandle_(nullptr), init_fn_(nullptr), fini_fn_(nullptr), model_init_fn_(nullptr), model_fini_fn_(nullptr), model_action_fn_(nullptr) { } const std::string name_; void* state_; // dlopen / dlsym handles void* dlhandle_; TritonRepoAgentInitFn_t init_fn_; TritonRepoAgentFiniFn_t fini_fn_; TritonRepoAgentModelInitFn_t model_init_fn_; TritonRepoAgentModelFiniFn_t model_fini_fn_; TritonRepoAgentModelActionFn_t model_action_fn_; }; class TritonRepoAgentModel { public: static Status Create( const TRITONREPOAGENT_ArtifactType type, const std::string& location, const inference::ModelConfig& config, const std::shared_ptr& agent, const TritonRepoAgent::Parameters& agent_parameters, std::unique_ptr* agent_model); ~TritonRepoAgentModel(); void* State() { return state_; } void SetState(void* state) { state_ = state; } Status InvokeAgent(const TRITONREPOAGENT_ActionType action_type); const TritonRepoAgent::Parameters& AgentParameters() { return agent_parameters_; } Status SetLocation( const TRITONREPOAGENT_ArtifactType type, const std::string& location); Status Location(TRITONREPOAGENT_ArtifactType* type, const char** location); Status AcquireMutableLocation( const TRITONREPOAGENT_ArtifactType type, const char** location); Status DeleteMutableLocation(); const inference::ModelConfig Config() { return config_; } private: DISALLOW_COPY_AND_ASSIGN(TritonRepoAgentModel); TritonRepoAgentModel( const TRITONREPOAGENT_ArtifactType type, const std::string& location, const inference::ModelConfig& config, const std::shared_ptr& agent, const TritonRepoAgent::Parameters& agent_parameters) : state_(nullptr), config_(config), agent_(agent), agent_parameters_(agent_parameters), type_(type), location_(location), action_type_set_(false), current_action_type_(TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE) { } void* state_; const inference::ModelConfig config_; const std::shared_ptr agent_; const TritonRepoAgent::Parameters agent_parameters_; TRITONREPOAGENT_ArtifactType type_; std::string location_; TRITONREPOAGENT_ArtifactType acquired_type_; std::string acquired_location_; bool action_type_set_; TRITONREPOAGENT_ActionType current_action_type_; }; class TritonRepoAgentManager { public: static Status SetGlobalSearchPath(const std::string& path); static Status CreateAgent( const std::string& agent_name, std::shared_ptr* agent); static Status AgentState( std::unique_ptr>* agent_state); private: DISALLOW_COPY_AND_ASSIGN(TritonRepoAgentManager); TritonRepoAgentManager() : global_search_path_("/opt/tritonserver/repoagents"){}; static TritonRepoAgentManager& Singleton(); std::mutex mu_; std::string global_search_path_; std::unordered_map> agent_map_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/response_allocator.h000066400000000000000000000061251527052371200214270ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "tritonserver_apis.h" namespace triton { namespace core { // // Implementation for TRITONSERVER_ResponseAllocator. // class ResponseAllocator { public: explicit ResponseAllocator( TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, TRITONSERVER_ResponseAllocatorStartFn_t start_fn) : alloc_fn_(alloc_fn), buffer_attributes_fn_(nullptr), query_fn_(nullptr), release_fn_(release_fn), start_fn_(start_fn) { } void SetQueryFunction(TRITONSERVER_ResponseAllocatorQueryFn_t query_fn) { query_fn_ = query_fn; } void SetBufferAttributesFunction( TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn) { buffer_attributes_fn_ = buffer_attributes_fn; } TRITONSERVER_ResponseAllocatorAllocFn_t AllocFn() const { return alloc_fn_; } TRITONSERVER_ResponseAllocatorBufferAttributesFn_t BufferAttributesFn() const { return buffer_attributes_fn_; } TRITONSERVER_ResponseAllocatorQueryFn_t QueryFn() const { return query_fn_; } TRITONSERVER_ResponseAllocatorReleaseFn_t ReleaseFn() const { return release_fn_; } TRITONSERVER_ResponseAllocatorStartFn_t StartFn() const { return start_fn_; } private: TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn_; TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn_; TRITONSERVER_ResponseAllocatorQueryFn_t query_fn_; TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn_; TRITONSERVER_ResponseAllocatorStartFn_t start_fn_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/scheduler.h000066400000000000000000000074051527052371200175110ustar00rootroot00000000000000// Copyright (c) 2018-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "infer_request.h" #include "status.h" namespace triton { namespace core { // Scheduler interface. class Scheduler { public: virtual ~Scheduler() {} // The prototype for the initialization function that will be called // by the "standard" schedulers created based on a model's // scheduling_choice settings. The init function is called once by // the runner that will later execute requests for 'runner_idx'. A // non-OK error status indicates an initialization error that // prevents scheduler from using the runner. using StandardInitFunc = std::function; // The prototype for the warmup function that will be called by the // "standard" schedulers created based on a model's // scheduling_choice settings. The warmup function is called once by // the runner that will later execute requests for 'runner_idx'. A // non-OK error status indicates an error that prevents scheduler // from sending warmup requests to the runner. using StandardWarmupFunc = std::function; // The prototype for the run function that will be called by the // "standard" schedulers created based on a model's // scheduling_choice settings. The run function must accept a // 'runner_idx' indicating which runner should execute the // 'requests'. Ownership of the 'requests' is transferred to the // runner which is responsible for generating responses and // releasing the requests. using StandardRunFunc = std::function>&& requests)>; // Enqueue a request with the scheduler. If Status::Success is returned // then the backend has taken ownership of the request object and so // 'request' will be nullptr. If non-success is returned then the // caller still retains ownership of 'request'. virtual Status Enqueue(std::unique_ptr& request) = 0; // Return the number of in-flight inferences tracked by the scheduler. virtual size_t InflightInferenceCount() = 0; // Instruct the scheduler to stop processing future requests unless they are // considered as in-flight. virtual void Stop() = 0; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/scheduler_utils.cc000066400000000000000000000445511527052371200210720ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "scheduler_utils.h" #include #include "constants.h" #include "triton/common/logging.h" namespace triton { namespace core { uint64_t CaptureTimeNs() { return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); } bool CacheLookUpUtil( std::unique_ptr& request, std::unique_ptr& cached_response, std::shared_ptr cache) { Status status; std::unique_ptr local_response; request->ResponseFactory()->CreateResponse(&local_response); std::string key = ""; if (!request->CacheKeyIsSet()) { status = cache->Hash(*request, &key); if (!status.IsOk()) { LOG_ERROR << "Failed to hash request: " << status.Message(); return false; } request->SetCacheKey(key); } else { key = request->CacheKey(); } request->CaptureCacheLookupStartNs(); status = cache->Lookup(local_response.get(), key); request->CaptureCacheLookupEndNs(); if (status.IsOk() && (local_response != nullptr)) { cached_response = std::move(local_response); return true; } return false; } Status RequiredEqualInputs::Initialize( const std::unique_ptr& request, const std::unordered_map& enforce_equal_shape_tensors, const bool has_optional_input) { has_optional_input_ = has_optional_input; required_inputs_.clear(); for (const auto& pr : request->ImmutableInputs()) { const InferenceRequest::Input* input = pr.second; const auto itr = enforce_equal_shape_tensors.find(input->Name()); if (itr != enforce_equal_shape_tensors.end()) { required_inputs_.emplace( std::piecewise_construct, std::forward_as_tuple(input->Name()), std::forward_as_tuple(input, itr->second)); } // When the model has optional inputs, overload 'required_inputs_' // to track the inputs involved in the batch else if (has_optional_input) { required_inputs_.emplace( std::piecewise_construct, std::forward_as_tuple(input->Name()), std::forward_as_tuple(nullptr, false)); } } init_ = true; return Status::Success; } bool RequiredEqualInputs::HasEqualInputs( const std::unique_ptr& request) { // If current request has different number of inputs, then dynamic batching // shouldn't be applied. if (has_optional_input_ && (request->ImmutableInputs().size() != required_inputs_.size())) { return false; } for (const auto& pr : request->ImmutableInputs()) { const InferenceRequest::Input* input = pr.second; const auto itr = required_inputs_.find(input->Name()); if (itr != required_inputs_.end()) { if (itr->second.first != nullptr) { // Make sure shape of input tensors is equal. if (!triton::common::CompareDims( itr->second.first->Shape(), input->Shape())) { return false; } // If necessary compare the contents as well... if (itr->second.second) { const auto& d1 = itr->second.first->Data(); const auto& d2 = input->Data(); // For now being conservative and assuming that content // comparison is for shape tensors which are likely to always // be in a single buffer. if ((d1->BufferCount() != 1) || (d2->BufferCount() != 1)) { return false; } size_t d1_byte_size, d2_byte_size; TRITONSERVER_MemoryType d1_memory_type, d2_memory_type; int64_t d1_memory_id, d2_memory_id; const char* d1_buffer = d1->BufferAt( 0 /* idx */, &d1_byte_size, &d1_memory_type, &d1_memory_id); const char* d2_buffer = d2->BufferAt( 0 /* idx */, &d2_byte_size, &d2_memory_type, &d2_memory_id); // Tensor must be same size and in in CPU memory so that it // can be easily compared. If not return false conservatively. if ((d1_byte_size != d2_byte_size) || (d1_buffer == nullptr) || (d2_buffer == nullptr) || (d1_memory_type == TRITONSERVER_MEMORY_GPU) || (d2_memory_type == TRITONSERVER_MEMORY_GPU)) { return false; } if (strncmp(d1_buffer, d2_buffer, d1_byte_size) != 0) { return false; } } } } else if (has_optional_input_) { // If the model has optional inputs, the current request must contains all // inputs that in the first request (tracked in 'required_inputs_'). return false; } } return true; } Status PriorityQueue::PolicyQueue::Enqueue(std::unique_ptr& request) { if ((max_queue_size_ != 0) && (Size() >= max_queue_size_)) { return Status( Status::Code::UNAVAILABLE, request->LogRequest() + "Exceeds maximum queue size"); } queue_.emplace_back(std::move(request)); auto timeout_us = default_timeout_us_; if (allow_timeout_override_) { auto override_timeout_us = queue_.back()->TimeoutMicroseconds(); if (override_timeout_us != 0 && override_timeout_us < timeout_us) { timeout_us = override_timeout_us; } } if (timeout_us != 0) { timeout_timestamp_ns_.emplace_back( std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count() + timeout_us * 1000); } else { timeout_timestamp_ns_.emplace_back(0); } return Status::Success; } Status PriorityQueue::PolicyQueue::Dequeue(std::unique_ptr* request) { if (!queue_.empty()) { *request = std::move(queue_.front()); queue_.pop_front(); timeout_timestamp_ns_.pop_front(); } else { *request = std::move(delayed_queue_.front()); delayed_queue_.pop_front(); } return Status::Success; } bool PriorityQueue::PolicyQueue::ApplyPolicy( size_t idx, size_t* rejected_count, size_t* rejected_batch_size, size_t* cancelled_count, size_t* cancelled_batch_size) { uint64_t now_nanoseconds = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); if (idx < queue_.size()) { size_t curr_idx = idx; // Advance curr_idx until a request that goes into a batch, if not already. while (curr_idx < queue_.size()) { // Cancel request at curr_idx if it is marked as cancelled. if (queue_[curr_idx]->IsCancelled()) { cancelled_queue_.emplace_back(std::move(queue_[curr_idx])); *cancelled_count += 1; *cancelled_batch_size += std::max(1U, cancelled_queue_.back()->BatchSize()); curr_idx++; } // Delay or reject request at curr_idx if it is expired. else if ( (timeout_timestamp_ns_[curr_idx] != 0) && (now_nanoseconds > timeout_timestamp_ns_[curr_idx])) { if (timeout_action_ == inference::ModelQueuePolicy::DELAY) { delayed_queue_.emplace_back(std::move(queue_[curr_idx])); } else { rejected_queue_.emplace_back(std::move(queue_[curr_idx])); *rejected_count += 1; *rejected_batch_size += std::max(1U, rejected_queue_.back()->BatchSize()); } curr_idx++; } // Request at curr_idx is unexpired and non-cancelled. else { break; } } // Erase requests that are not going into a batch from queue_. // Use range erasure on deque as all erasure functions are linear, // this implies in the edge case where this function is always called on // 'bad' index can be O(n^2). However, for data structures that are O(1) // erasure, the traversal may not be as efficient due to cache miss // (elements not stored contiguously). queue_.erase(queue_.begin() + idx, queue_.begin() + curr_idx); timeout_timestamp_ns_.erase( timeout_timestamp_ns_.begin() + idx, timeout_timestamp_ns_.begin() + curr_idx); // Check if idx is still in range after erasing requests. if (idx < queue_.size()) { return true; } } // At this point, idx >= queue_.size(). // If the item is in delayed queue, then return true. Otherwise, false // meaning the queue has no item with this 'idx'. return ((idx - queue_.size()) < delayed_queue_.size()); } size_t PriorityQueue::PolicyQueue::RejectTimeoutRequests() { if (timeout_action_ != inference::ModelQueuePolicy::REJECT) { return 0; } size_t rejected_count = 0; uint64_t now_nanoseconds = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); size_t idx = 0; while (idx < queue_.size()) { if (timeout_timestamp_ns_[idx] != 0 && now_nanoseconds > timeout_timestamp_ns_[idx]) { rejected_count++; rejected_queue_.emplace_back(std::move(queue_[idx])); queue_.erase(queue_.begin() + idx); timeout_timestamp_ns_.erase(timeout_timestamp_ns_.begin() + idx); } else { idx++; } } return rejected_count; } void PriorityQueue::PolicyQueue::ReleaseRejectedQueue( std::deque>* requests) { rejected_queue_.swap(*requests); } void PriorityQueue::PolicyQueue::ReleaseCancelledQueue( std::deque>* requests) { cancelled_queue_.swap(*requests); } const std::unique_ptr& PriorityQueue::PolicyQueue::At(size_t idx) const { if (idx < queue_.size()) { return queue_[idx]; } else { return delayed_queue_[idx - queue_.size()]; } } uint64_t PriorityQueue::PolicyQueue::TimeoutAt(size_t idx) { if (idx < queue_.size()) { return timeout_timestamp_ns_[idx]; } else { return 0; } } bool PriorityQueue::PolicyQueue::ReadyForErasure() { size_t total_size = Size() + rejected_queue_.size() + cancelled_queue_.size(); return !keep_instantiated_ && total_size == 0; } PriorityQueue::PriorityQueue() : size_(0), front_priority_level_(0), default_policy_() { queues_.emplace(0, PolicyQueue(default_policy_, true)); front_priority_level_ = queues_.begin()->first; ResetCursor(); } PriorityQueue::PriorityQueue( const inference::ModelQueuePolicy& default_queue_policy, uint64_t priority_levels, const ModelQueuePolicyMap queue_policy_map) : size_(0), default_policy_(default_queue_policy) { // Permanently instantiate PolicyQueue with keep_instantiate=true // to prevent them from being erased & created during scheduling if (priority_levels == 0) { // Only default policy is instantiated queues_.emplace(0, PolicyQueue(default_policy_, true)); support_prefetching_ = (default_policy_.default_timeout_microseconds() == 0) && (!default_policy_.allow_timeout_override()) && (default_policy_.max_queue_size() == 0); } else { // All priorities with user-given policy are instantiated. We do not // permanently add default PolicyQueue because those will be dynamically // created and erased to keep memory footprint low for (const auto& qp : queue_policy_map) { queues_.emplace(qp.first, PolicyQueue(qp.second, true)); } support_prefetching_ = false; } front_priority_level_ = queues_.empty() ? 0 : queues_.begin()->first; ResetCursor(); } Status PriorityQueue::Enqueue( uint64_t priority_level, std::unique_ptr& request) { // Get corresponding PolicyQueue if it exists, otherwise insert it // via emplace with the default policy auto it = queues_.insert(std::make_pair(priority_level, default_policy_)); auto status = it.first->second.Enqueue(request); if (status.IsOk()) { size_++; front_priority_level_ = std::min(front_priority_level_, priority_level); // Invalidate the pending batch cursor if the enqueued item is placed // within the pending batch. At the same priority level the request is // guaranteed to be after pending batch if the batch hasn't reached // delayed queue. if (pending_cursor_.valid_ && ((priority_level < pending_cursor_.curr_it_->first) || ((priority_level == pending_cursor_.curr_it_->first) && (pending_cursor_.at_delayed_queue_)))) { pending_cursor_.valid_ = false; } } return status; } Status PriorityQueue::Dequeue(std::unique_ptr* request) { pending_cursor_.valid_ = false; auto it_start = queues_.lower_bound(front_priority_level_); for (auto it = it_start; it != queues_.end(); ++it) { if (!it->second.Empty()) { front_priority_level_ = it->first; RETURN_IF_ERROR(it->second.Dequeue(request)); size_--; if (it->second.ReadyForErasure()) { queues_.erase(it); } return Status::Success; } } return Status(Status::Code::UNAVAILABLE, "dequeue on empty queue"); } void PriorityQueue::RejectTimeoutRequests() { for (auto it = queues_.begin(); it != queues_.end(); it++) { size_t rejected_count = it->second.RejectTimeoutRequests(); size_ -= rejected_count; if (rejected_count > 0 && it->first == pending_cursor_.curr_it_->first) { pending_cursor_.valid_ = false; } } } void PriorityQueue::ReleaseSkippedRequests( std::vector>>* rejected_requests, std::vector>>* cancelled_requests) { std::vector>> reject_req( queues_.size()); std::vector>> cancel_req( queues_.size()); size_t idx = 0; for (auto it = queues_.begin(); it != queues_.end();) { it->second.ReleaseRejectedQueue(&reject_req[idx]); it->second.ReleaseCancelledQueue(&cancel_req[idx]); idx++; if (it->second.ReadyForErasure()) { // Invalidate the pending batch cursor if it points to // the item to be erased if (pending_cursor_.valid_ && it->first == pending_cursor_.curr_it_->first) { pending_cursor_.valid_ = false; } it = queues_.erase(it); // returns iterator following removed element } else { ++it; } } rejected_requests->swap(reject_req); cancelled_requests->swap(cancel_req); } bool PriorityQueue::IsCursorValid() { if (pending_cursor_.valid_) { return (uint64_t)std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count() < pending_cursor_.pending_batch_closest_timeout_ns_; } return false; } PriorityQueue::Cursor::Cursor(PriorityQueues::iterator start_it) : curr_it_(start_it), queue_idx_(0), at_delayed_queue_(false), pending_batch_closest_timeout_ns_(0), pending_batch_oldest_enqueue_time_ns_(0), pending_batch_count_(0), valid_(true) { } size_t PriorityQueue::ApplyPolicyAtCursor() { size_t rejected_batch_size = 0; size_t rejected_count = 0; size_t cancelled_batch_size = 0; size_t cancelled_count = 0; while (pending_cursor_.curr_it_ != queues_.end()) { if (!(pending_cursor_.curr_it_->second.ApplyPolicy( pending_cursor_.queue_idx_, &rejected_count, &rejected_batch_size, &cancelled_count, &cancelled_batch_size))) { if (size_ > pending_cursor_.pending_batch_count_ + rejected_count + cancelled_count) { pending_cursor_.curr_it_++; pending_cursor_.queue_idx_ = 0; continue; } } // Control reach here if the cursor points to a request that is candidate // for pending batch, or if all requests are in pending batch. break; } size_ -= rejected_count + cancelled_count; return rejected_batch_size + cancelled_batch_size; } void PriorityQueue::AdvanceCursor() { if (pending_cursor_.pending_batch_count_ >= size_) { return; } const auto& timeout_ns = pending_cursor_.curr_it_->second.TimeoutAt(pending_cursor_.queue_idx_); if (timeout_ns != 0) { if (pending_cursor_.pending_batch_closest_timeout_ns_ != 0) { pending_cursor_.pending_batch_closest_timeout_ns_ = std::min( pending_cursor_.pending_batch_closest_timeout_ns_, timeout_ns); } else { pending_cursor_.pending_batch_closest_timeout_ns_ = timeout_ns; } } uint64_t curr_enqueue_time_ns = pending_cursor_.curr_it_->second.At(pending_cursor_.queue_idx_) ->BatcherStartNs(); if (pending_cursor_.pending_batch_oldest_enqueue_time_ns_ != 0) { pending_cursor_.pending_batch_oldest_enqueue_time_ns_ = std::min( pending_cursor_.pending_batch_oldest_enqueue_time_ns_, curr_enqueue_time_ns); } else { pending_cursor_.pending_batch_oldest_enqueue_time_ns_ = curr_enqueue_time_ns; } ++pending_cursor_.queue_idx_; ++pending_cursor_.pending_batch_count_; // pending batch includes delayed request if (queue_idx_ - 1) points to // delayed queue. pending_cursor_.at_delayed_queue_ = (pending_cursor_.queue_idx_ > pending_cursor_.curr_it_->second.UnexpiredSize()); } }} // namespace triton::core kytensor-2.49.0.7/core/src/scheduler_utils.h000066400000000000000000000274541527052371200207370ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "cache_manager.h" #include "scheduler.h" namespace triton { namespace core { uint64_t CaptureTimeNs(); // Utility function called by the scheduler to lookup if the request is in the // cache and get the response. bool CacheLookUpUtil( std::unique_ptr& request, std::unique_ptr& cached_response, std::shared_ptr cache); struct RequiredEqualInputs { public: RequiredEqualInputs() : init_(false), has_optional_input_(false) {} Status Initialize( const std::unique_ptr& request, const std::unordered_map& enforce_equal_shape_tensors, const bool has_optional_input); bool HasEqualInputs(const std::unique_ptr& request); bool Initialized() { return init_; }; private: bool init_; bool has_optional_input_; // A collection of inputs in the request, an nullptr for // InferenceRequest::Input indicates that the inputs doesn't require // equality check std::unordered_map< std::string, std::pair> required_inputs_; }; // // PriorityQueue // using ModelQueuePolicyMap = ::google::protobuf::Map< ::google::protobuf::uint64, inference::ModelQueuePolicy>; class PriorityQueue { public: // Construct a queue with no priority level with default queue policy, // which will behave the same as regular queue. PriorityQueue(); // Construct a queue with 'priority_levels', the priority starts from 1. // Different priority level may follow different queue policies given by // 'queue_policy_map', otherwise, the 'default_queue_policy' will be used. PriorityQueue( const inference::ModelQueuePolicy& default_queue_policy, uint64_t priority_levels, const ModelQueuePolicyMap queue_policy_map); // Enqueue a request with priority set to 'priority_level'. If // Status::Success is returned then the queue has taken ownership of // the request object and so 'request' will be nullptr. If // non-success is returned then the caller still retains ownership // of 'request'. Status Enqueue( uint64_t priority_level, std::unique_ptr& request); // Dequeue the request at the front of the queue. Status Dequeue(std::unique_ptr* request); // Reject timed-out requests from 'queues_', if queue policy set to reject. // The cursor will be marked as invalid, if a request from the queue pointed // to by the cursor is rejected. void RejectTimeoutRequests(); // Retrieve the requests that are either rejected or cancelled. void ReleaseSkippedRequests( std::vector>>* rejected_requests, std::vector>>* cancelled_requests); // Return the number of requests in the queue, rejected requests are // not included. size_t Size() { return size_; } // Is the queue is empty? Rejected requests are not included. bool Empty() { return Size() == 0; } // Reset the cursor such that it is representing an empty pending batch. void ResetCursor() { pending_cursor_ = Cursor(queues_.begin()); } // Record the current cursor. The cursor can be restored to recorded state // by invoking SetCursorToMark(). Note that Enqueue(), Dequeue(), and // ResetCursor() will invalidate the marker, it is the function caller's // responsibility to ensure the marker is valid before calling // SetCursorToMark(). void MarkCursor() { current_mark_ = pending_cursor_; } // Apply the queue policy and alter the underlying queue accordingly. After // the function returns, the cursor may be at its end to indicate that // there no request after the pending batch. // Returns the total batch size of the newly rejected and cancelled requests. size_t ApplyPolicyAtCursor(); // Return the request at the cursor. const std::unique_ptr& RequestAtCursor() { return pending_cursor_.curr_it_->second.At(pending_cursor_.queue_idx_); } // Advance the cursor for pending batch. This function will not trigger the // queue policy. No effect if the cursor already reach the end of the queue. void AdvanceCursor(); // Whether the cursor reaches its end, bool CursorEnd() { return pending_cursor_.pending_batch_count_ == size_; } // Restore the cursor state to the marker. void SetCursorToMark() { pending_cursor_ = current_mark_; } // Whether the cursor is still valid. The cursor is valid only if the pending // batch is unchanged. bool IsCursorValid(); // Return the oldest queued time of requests in pending batch. uint64_t OldestEnqueueTime() { return pending_cursor_.pending_batch_oldest_enqueue_time_ns_; } // Return the closest timeout of requests in pending batch. uint64_t ClosestTimeout() { return pending_cursor_.pending_batch_closest_timeout_ns_; } // Return the number of requests in pending batch. size_t PendingBatchCount() { return pending_cursor_.pending_batch_count_; } // Whether the queue supports pre-fetching of the requests. bool SupportPrefetching() { return support_prefetching_; } private: class PolicyQueue { public: // Construct a policy queue with default policy, which will behave the same // as regular queue. PolicyQueue() : timeout_action_(inference::ModelQueuePolicy::REJECT), default_timeout_us_(0), allow_timeout_override_(false), max_queue_size_(0), keep_instantiated_(false) { } // Construct a policy queue with given 'policy'. PolicyQueue( const inference::ModelQueuePolicy& policy, bool keep_instantiated = false) : timeout_action_(policy.timeout_action()), default_timeout_us_(policy.default_timeout_microseconds()), allow_timeout_override_(policy.allow_timeout_override()), max_queue_size_(policy.max_queue_size()), keep_instantiated_(keep_instantiated) { } // Enqueue a request and set up its timeout accordingly. If // Status::Success is returned then the queue has taken ownership // of the request object and so 'request' will be nullptr. If // non-success is returned then the caller still retains ownership // of 'request'. Status Enqueue(std::unique_ptr& request); // Dequeue the request at the front of the queue. Status Dequeue(std::unique_ptr* request); // Apply the queue policy to the request at 'idx'. A request is rejected if // it is expired and not delayable. A request is cancelled if it is marked // as cancelled. // 'rejected_count' will be incremented by the number of the newly rejected // requests after applying the policy. // 'rejected_batch_size' will be incremented by the total batch size of the // newly rejected requests after applying the policy. // 'cancelled_count' will be incremented by the number of newly cancelled // requests after applying the policy. // 'cancelled_batch_size' will be incremented by the total batch size of the // newly cancelled requests after applying the policy. // Return true if the 'idx' still points to a request after applying the // policy, false otherwise. bool ApplyPolicy( size_t idx, size_t* rejected_count, size_t* rejected_batch_size, size_t* cancelled_count, size_t* cancelled_batch_size); // Move timed-out requests from 'queue_' to 'rejected_queue_', if // 'timeout_action_' is to reject. Return the number of requests rejected. size_t RejectTimeoutRequests(); // Return the rejected requests held by the queue. void ReleaseRejectedQueue( std::deque>* requests); // Return the cancelled requests held by the queue. void ReleaseCancelledQueue( std::deque>* requests); // Return the request at 'idx'. const std::unique_ptr& At(size_t idx) const; // Return the timeout timestamp of the request at 'idx', in ns. A value of 0 // indicates that the request doesn't specify a timeout. uint64_t TimeoutAt(size_t idx); // Return whether the queue is empty, rejected requests are not included. bool Empty() { return Size() == 0; } // Return the number of requests in the queue, rejected requests are not // included. size_t Size() { return queue_.size() + delayed_queue_.size(); } // Return the number of unexpired requests in the queue size_t UnexpiredSize() { return queue_.size(); } // Return whether this PolicyQueue can be erased, i.e. when all queues // are empty and should not be kept instantiated bool ReadyForErasure(); private: // Variables that define the policy for the queue const inference::ModelQueuePolicy::TimeoutAction timeout_action_; const uint64_t default_timeout_us_; const bool allow_timeout_override_; const uint32_t max_queue_size_; const bool keep_instantiated_; std::deque timeout_timestamp_ns_; std::deque> queue_; std::deque> delayed_queue_; std::deque> rejected_queue_; std::deque> cancelled_queue_; }; using PriorityQueues = std::map; // Cursor for tracking pending batch, the cursor points to the item after // the pending batch. struct Cursor { Cursor() = default; Cursor(PriorityQueues::iterator start_it); Cursor(const Cursor& rhs) = default; Cursor& operator=(const Cursor& rhs) = default; PriorityQueues::iterator curr_it_; size_t queue_idx_; bool at_delayed_queue_; uint64_t pending_batch_closest_timeout_ns_; uint64_t pending_batch_oldest_enqueue_time_ns_; size_t pending_batch_count_; bool valid_; }; PriorityQueues queues_; size_t size_; // Keep track of the priority level that the first request in the queue // is at to avoid traversing 'queues_' uint64_t front_priority_level_; inference::ModelQueuePolicy default_policy_; Cursor pending_cursor_; Cursor current_mark_; // Whether requests can be pre-fetched from the queue. bool support_prefetching_{true}; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/sequence_batch_scheduler/000077500000000000000000000000001527052371200223635ustar00rootroot00000000000000kytensor-2.49.0.7/core/src/sequence_batch_scheduler/sequence_batch_scheduler.cc000066400000000000000000002363461527052371200277170ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "sequence_batch_scheduler.h" #ifndef _WIN32 #include #include #include #endif #include #include "constants.h" #include "dynamic_batch_scheduler.h" #include "model_config_utils.h" #include "server.h" #include "triton/common/logging.h" namespace triton { namespace core { namespace { template inline uint64_t Now() { return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); } void SetThreadPriority(const int nice, const char* thread_name) { #ifndef _WIN32 if (setpriority(PRIO_PROCESS, syscall(SYS_gettid), nice) == 0) { LOG_VERBOSE(1) << "Starting " << thread_name << " thread at nice " << nice << "..."; } else { LOG_VERBOSE(1) << "Starting " << thread_name << " thread at default nice (requested nice " << nice << " failed)..."; } #else LOG_VERBOSE(1) << "Starting " << thread_name << " thread at default nice..."; #endif } bool IsAnyRequestCancelled(std::deque>& requests) { for (auto& req : requests) { if (req->IsCancelled()) { return true; } } return false; } void CancelRequests(std::vector>&& requests) { const static Status cancelled_status = Status(Status::Code::CANCELLED); for (auto& req : requests) { // Mark the request as cancelled before responding. Status status = req->Cancel(); if (!status.IsOk()) { LOG_ERROR << status.Message(); } // Respond the request as cancelled. InferenceRequest::RespondIfError( req, cancelled_status, true /* release_requests */, FailureReason::CANCELED); } } } // namespace Status SequenceBatchScheduler::Create( TritonModel* model, const std::vector>& new_instances, const std::unordered_map& enforce_equal_shape_tensors, std::unique_ptr* scheduler) { std::unique_ptr sched( new SequenceBatchScheduler(model, enforce_equal_shape_tensors)); // For debugging and testing, const char* dstr = getenv("TRITONSERVER_BACKLOG_DELAY_SCHEDULER"); sched->backlog_delay_cnt_ = 0; if (dstr != nullptr) { sched->backlog_delay_cnt_ = atoi(dstr); LOG_INFO << "Delaying scheduler until " << sched->backlog_delay_cnt_ << " backlog queued requests..."; } auto& config = model->Config(); // Sequencer if (config.sequence_batching().iterative_sequence()) { sched->sequencer_.reset(new IterativeSequencer(sched.get())); } else { sched->sequencer_.reset(new Sequencer()); } // Max sequence idle... sched->max_sequence_idle_microseconds_ = config.sequence_batching().max_sequence_idle_microseconds(); sched->max_batch_size_ = config.max_batch_size(); // Implicit States auto& states = config.sequence_batching().state(); for (const inference::ModelSequenceBatching_State& state : states) { sched->state_output_config_map_.insert({state.output_name(), state}); if (state.initial_state_size() > 1) { return Status( Status::Code::INVALID_ARG, std::string( std::string("initial_state field for state input '") + state.input_name() + "' must contain exactly one or zero element. Found '" + std::to_string(state.initial_state_size()) + "' elements.")); } // If the model configuration has initial_state field. if (state.initial_state_size() == 1) { auto& initial_state = state.initial_state(0); RETURN_IF_ERROR( sched->GenerateInitialStateData(initial_state, state, model)); } } // Get the number of candidate sequence slots to allow for each // runner. This is at least 1 even if the model doesn't support // batching. const size_t model_batch_size = std::max(1, config.max_batch_size()); sched->seq_slot_cnt_ = model_batch_size; if (config.sequence_batching().has_oldest()) { const auto& max_candidate_seqs = config.sequence_batching().oldest().max_candidate_sequences(); if (max_candidate_seqs > 0) { sched->seq_slot_cnt_ = max_candidate_seqs; } } // Create a batcher for each instance. RETURN_IF_ERROR(sched->CreateBatchers(new_instances)); // Create background threads that watch for different sequence states. sched->StartBackgroundThreads(); scheduler->reset(sched.release()); return Status::Success; } Status SequenceBatchScheduler::Update( const std::vector>& added_instances, const std::vector>& removed_instances) { std::lock_guard lk(mu_); // Add the new batchers. RETURN_IF_ERROR(CreateBatchers(added_instances)); // All sequence slots of the removed instances are pending to be removed. // Track them in 'pending_removal_seq_slots_' and they will be erased once // they become ready. for (auto& instance : removed_instances) { pending_removal_seq_slots_.emplace( instance.get(), std::make_pair(batchers_[instance.get()]->SeqSlotCnt(), instance)); } // Erase pending removal sequence slots that are already at ready. std::priority_queue< BatcherSequenceSlot, std::vector, BatcherSequenceSlotCompare> new_ready_batcher_seq_slots; while (!ready_batcher_seq_slots_.empty()) { auto& ready_seq_slot = ready_batcher_seq_slots_.top(); if (pending_removal_seq_slots_.find(ready_seq_slot.model_instance_) == pending_removal_seq_slots_.end()) { // The ready slot is not being removed. new_ready_batcher_seq_slots.push(ready_seq_slot); } else { // The ready slot is being removed, erase the slot. EraseBatcherSequenceSlot(ready_seq_slot); } ready_batcher_seq_slots_.pop(); } ready_batcher_seq_slots_.swap(new_ready_batcher_seq_slots); return Status::Success; } bool SequenceBatchScheduler::EraseBatcherSequenceSlot( const BatcherSequenceSlot& seq_slot) { // Find the sequence slot from pending removal sequence slots. auto it = pending_removal_seq_slots_.find(seq_slot.model_instance_); if (it == pending_removal_seq_slots_.end()) { // The sequence slot is not pending removal. return false; } LOG_VERBOSE(1) << "Removing slot for batcher " << seq_slot.model_instance_->Name() << ", slot " << seq_slot.seq_slot_; // Subtract the number of slots, and erase the batcher if no more slots left. size_t& remaining_slots = it->second.first; if (--remaining_slots == 0) { LOG_VERBOSE(1) << "Removing batcher " << seq_slot.model_instance_->Name(); // Erase batcher. auto batcher_it = batchers_.find(seq_slot.model_instance_); auto& pending_removal_batcher = batcher_it->second; removed_batchers_.emplace_back(std::move(pending_removal_batcher)); batchers_.erase(batcher_it); // Stop tracking the removed instance. auto& pending_removal_instance = it->second.second; removed_instances_.emplace_back(std::move(pending_removal_instance)); // Erase from debugging/testing info. queue_request_cnts_.erase(seq_slot.model_instance_); // Erase sequence slot from pending. pending_removal_seq_slots_.erase(it); // Notify the clean-up thread. clean_up_cv_.notify_one(); } // The sequence slot was pending removal. return true; } Status SequenceBatchScheduler::CreateBatchers( const std::vector>& instances) { auto& config = model_->Config(); // Based on the model configuration create input tensors for control // signals indicating sequence start, sequence continue, and // sequence not ready. std::shared_ptr start; std::shared_ptr end; std::shared_ptr startend; std::shared_ptr cont; std::shared_ptr notready; RETURN_IF_ERROR(CreateBooleanControlTensors( config, &start, &end, &startend, &cont, ¬ready)); bool has_optional_input = false; for (const auto& input : config.input()) { if (input.optional()) { has_optional_input = true; break; } } // Create one SequenceBatch object for each requested runner. The // SequenceBatch object has a thread that manages the batch of // requests. for (const auto& instance : instances) { bool init_state; std::unique_ptr sb; // Create the SequenceBatch derivative that handles the requested // scheduling strategy. if (config.sequence_batching().has_oldest()) { sb.reset(new OldestSequenceBatch( this, instance.get(), seq_slot_cnt_, enforce_equal_shape_tensors_, has_optional_input, start, end, startend, cont, notready, &init_state)); } else { sb.reset(new DirectSequenceBatch( this, instance.get(), seq_slot_cnt_, enforce_equal_shape_tensors_, has_optional_input, start, end, startend, cont, notready, &init_state)); } if (init_state) { batchers_.emplace(instance.get(), std::move(sb)); // All sequence slots in the batcher are initially ready for a // new sequence. for (size_t b = 0; b < seq_slot_cnt_; ++b) { ready_batcher_seq_slots_.push( SequenceBatchScheduler::BatcherSequenceSlot(instance.get(), b)); } } } if (batchers_.empty()) { return Status( Status::Code::INTERNAL, "Initialization failed for all sequence-batch scheduler threads"); } return Status::Success; } Status SequenceBatchScheduler::GenerateInitialStateData( const inference::ModelSequenceBatching_InitialState& initial_state, const inference::ModelSequenceBatching_State& state, TritonModel* model) { if (initial_state.data_type() != state.data_type()) { return Status( Status::Code::INVALID_ARG, std::string("The data type used for 'initial_state' field of state '") + state.input_name() + "' does not match the state data type."); } if (initial_state.name().size() == 0) { return Status( Status::Code::INVALID_ARG, std::string("Field 'name' must be set when using initial_state for " "state input '") + state.input_name() + "'."); } auto initial_state_itr = initial_state_.find(state.input_name()); if (initial_state_itr != initial_state_.end()) { return Status( Status::Code::INVALID_ARG, std::string("State input name '") + state.input_name() + "' specified more than once."); } if (initial_state.dims().size() != state.dims().size()) { return Status( Status::Code::INVALID_ARG, std::string( "Number of dimensions in 'initial_state' doesn't match the size of" " 'state' dimensions for state input '") + state.input_name() + "'. " + std::to_string(initial_state.dims().size()) + " != " + std::to_string(state.dims().size())); } const auto& initial_state_pair = initial_state_.emplace( std::piecewise_construct, std::forward_as_tuple(state.input_name()), std::forward_as_tuple(initial_state.name())); auto& initial_state_data = initial_state_pair.first->second; if (max_batch_size_ > 0) { initial_state_data.shape_.emplace_back(1); } // Check the dimensions to make sure it doesn't have variable-sized dims and // matches the state description. auto initial_state_dim = initial_state.dims().begin(); auto state_dim = state.dims().begin(); for (; initial_state_dim != initial_state.dims().end(); initial_state_dim++, state_dim++) { if (*initial_state_dim == -1) { return Status( Status::Code::INVALID_ARG, std::string("'initial_state' field for state input name '") + state.input_name() + "' contains variable dimensions."); } else { if (*state_dim != -1 && *initial_state_dim != *state_dim) { return Status( Status::Code::INVALID_ARG, std::string("'initial_state' dim for input name '") + state.input_name() + "' doesn't match 'state' dim description. " + std::to_string(*initial_state_dim) + " != " + std::to_string(*state_dim)); } } initial_state_data.shape_.emplace_back(*initial_state_dim); } // Calculate total memory byte size auto element_count = triton::common::GetElementCount(initial_state.dims()); size_t dtype_byte_size = triton::common::GetDataTypeByteSize(initial_state.data_type()); size_t total_byte_size = element_count * dtype_byte_size; // Custom handling for TYPE_BYTES if (dtype_byte_size == 0) { total_byte_size = sizeof(int32_t) * element_count; } switch (initial_state.state_data_case()) { case inference::ModelSequenceBatching_InitialState::StateDataCase:: kZeroData: { initial_state_data.data_ = std::make_shared( total_byte_size, TRITONSERVER_MEMORY_CPU /* memory_type */, 0 /* memory_type_id */); TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; char* data_ptr = initial_state_data.data_->MutableBuffer( &memory_type, &memory_type_id); memset(data_ptr, 0, total_byte_size); break; } case inference::ModelSequenceBatching_InitialState::StateDataCase:: kDataFile: { std::string file_input; RETURN_IF_ERROR(ReadTextFile( JoinPath( {model->LocalizedModelPath(), kInitialStateFolder, (initial_state.data_file())}), &file_input)); if (initial_state.data_type() == inference::DataType::TYPE_STRING) { total_byte_size = file_input.size(); } else if (total_byte_size > file_input.size()) { return Status( Status::Code::INVALID_ARG, "initial_state setting expects " + std::to_string(total_byte_size) + " bytes, but the data " "provided from " + initial_state.data_file() + "only has " + std::to_string(file_input.size()) + " bytes."); } TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; initial_state_data.data_ = std::make_shared( total_byte_size, TRITONSERVER_MEMORY_CPU /* memory_type */, 0 /* memory_type_id */); char* data_ptr = initial_state_data.data_->MutableBuffer( &memory_type, &memory_type_id); memcpy(data_ptr, file_input.data(), total_byte_size); break; } default: return Status( Status::Code::INVALID_ARG, std::string("initial_state setting expects state'") + state.input_name() + "' to have state_data set"); } return Status::Success; } SequenceBatchScheduler::~SequenceBatchScheduler() { StopBackgroundThreads(); // Release 'batchers_' before other member variables because 'batchers_' // can access 'this' and we need to make sure the member variables live // longer than 'batchers_' batchers_.clear(); removed_batchers_.clear(); } namespace { Status GetBooleanOverrideInputs( const std::string& tensor_name, const bool support_batching, const inference::DataType tensor_datatype, const float fp32_false_value, const float fp32_true_value, const int32_t int32_false_value, const int32_t int32_true_value, const bool bool_false_value, const bool bool_true_value, std::shared_ptr* true_override, std::shared_ptr* false_override) { TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; const std::vector tensor_shape{1}; std::vector tensor_shape_with_batch_dim{1}; if (support_batching) { tensor_shape_with_batch_dim.push_back(1); } const size_t size_p = triton::common::GetDataTypeByteSize(tensor_datatype); auto true_p = std::make_shared(size_p, TRITONSERVER_MEMORY_CPU, 0); char* true_p_ptr = true_p->MutableBuffer(&memory_type, &memory_type_id); if ((true_p_ptr == nullptr) || ((memory_type != TRITONSERVER_MEMORY_CPU) && (memory_type != TRITONSERVER_MEMORY_CPU_PINNED)) || (memory_type_id != 0)) { return Status( Status::Code::INTERNAL, "failed to allocate sequence control signal in CPU memory"); } auto false_p = std::make_shared(size_p, TRITONSERVER_MEMORY_CPU, 0); char* false_p_ptr = false_p->MutableBuffer(&memory_type, &memory_type_id); if ((false_p_ptr == nullptr) || ((memory_type != TRITONSERVER_MEMORY_CPU) && (memory_type != TRITONSERVER_MEMORY_CPU_PINNED)) || (memory_type_id != 0)) { return Status( Status::Code::INTERNAL, "failed to allocate sequence control signal in CPU memory"); } if (tensor_datatype == inference::DataType::TYPE_INT32) { *(reinterpret_cast(true_p_ptr)) = int32_true_value; *(reinterpret_cast(false_p_ptr)) = int32_false_value; } else if (tensor_datatype == inference::DataType::TYPE_FP32) { *(reinterpret_cast(true_p_ptr)) = fp32_true_value; *(reinterpret_cast(false_p_ptr)) = fp32_false_value; } else { *(reinterpret_cast(true_p_ptr)) = bool_true_value; *(reinterpret_cast(false_p_ptr)) = bool_false_value; } auto ltrue_override = std::make_shared( tensor_name, tensor_datatype, tensor_shape); *ltrue_override->MutableShape() = ltrue_override->OriginalShape(); *ltrue_override->MutableShapeWithBatchDim() = tensor_shape_with_batch_dim; RETURN_IF_ERROR(ltrue_override->SetData(true_p)); auto lfalse_override = std::make_shared( tensor_name, tensor_datatype, tensor_shape); *lfalse_override->MutableShape() = lfalse_override->OriginalShape(); *lfalse_override->MutableShapeWithBatchDim() = tensor_shape_with_batch_dim; RETURN_IF_ERROR(lfalse_override->SetData(false_p)); *true_override = std::move(ltrue_override); *false_override = std::move(lfalse_override); return Status::Success; } } // namespace Status SequenceBatchScheduler::CreateBooleanControlTensors( const inference::ModelConfig& config, std::shared_ptr* start_input_overrides, std::shared_ptr* end_input_overrides, std::shared_ptr* startend_input_overrides, std::shared_ptr* continue_input_overrides, std::shared_ptr* notready_input_overrides) { // Currently only batch-size 1 requests are supported so only need // to provide control vectors of that size. *start_input_overrides = std::make_shared(); *end_input_overrides = std::make_shared(); *startend_input_overrides = std::make_shared(); *continue_input_overrides = std::make_shared(); *notready_input_overrides = std::make_shared(); std::string tensor_name; inference::DataType tensor_datatype; int32_t int32_false_value, int32_true_value; float fp32_false_value, fp32_true_value; bool bool_false_value, bool_true_value; // START, optional { RETURN_IF_ERROR(GetBooleanSequenceControlProperties( config.sequence_batching(), config.name(), inference::ModelSequenceBatching::Control::CONTROL_SEQUENCE_START, false /* required */, &tensor_name, &tensor_datatype, &fp32_false_value, &fp32_true_value, &int32_false_value, &int32_true_value, &bool_false_value, &bool_true_value)); if (!tensor_name.empty()) { std::shared_ptr true_override; std::shared_ptr false_override; RETURN_IF_ERROR(GetBooleanOverrideInputs( tensor_name, config.max_batch_size() != 0, tensor_datatype, fp32_false_value, fp32_true_value, int32_false_value, int32_true_value, bool_false_value, bool_true_value, &true_override, &false_override)); (*start_input_overrides)->emplace_back(true_override); (*end_input_overrides)->emplace_back(false_override); (*startend_input_overrides)->emplace_back(true_override); (*continue_input_overrides)->emplace_back(false_override); (*notready_input_overrides)->emplace_back(false_override); } } // END, optional { RETURN_IF_ERROR(GetBooleanSequenceControlProperties( config.sequence_batching(), config.name(), inference::ModelSequenceBatching::Control::CONTROL_SEQUENCE_END, false /* required */, &tensor_name, &tensor_datatype, &fp32_false_value, &fp32_true_value, &int32_false_value, &int32_true_value, &bool_false_value, &bool_true_value)); if (!tensor_name.empty()) { std::shared_ptr true_override; std::shared_ptr false_override; RETURN_IF_ERROR(GetBooleanOverrideInputs( tensor_name, config.max_batch_size() != 0, tensor_datatype, fp32_false_value, fp32_true_value, int32_false_value, int32_true_value, bool_false_value, bool_true_value, &true_override, &false_override)); (*start_input_overrides)->emplace_back(false_override); (*end_input_overrides)->emplace_back(true_override); (*startend_input_overrides)->emplace_back(true_override); (*continue_input_overrides)->emplace_back(false_override); (*notready_input_overrides)->emplace_back(false_override); } } // READY, optional { RETURN_IF_ERROR(GetBooleanSequenceControlProperties( config.sequence_batching(), config.name(), inference::ModelSequenceBatching::Control::CONTROL_SEQUENCE_READY, false /* required */, &tensor_name, &tensor_datatype, &fp32_false_value, &fp32_true_value, &int32_false_value, &int32_true_value, &bool_false_value, &bool_true_value)); if (!tensor_name.empty()) { std::shared_ptr true_override; std::shared_ptr false_override; RETURN_IF_ERROR(GetBooleanOverrideInputs( tensor_name, config.max_batch_size() != 0, tensor_datatype, fp32_false_value, fp32_true_value, int32_false_value, int32_true_value, bool_false_value, bool_true_value, &true_override, &false_override)); (*start_input_overrides)->emplace_back(true_override); (*end_input_overrides)->emplace_back(true_override); (*startend_input_overrides)->emplace_back(true_override); (*continue_input_overrides)->emplace_back(true_override); (*notready_input_overrides)->emplace_back(false_override); } } return Status::Success; } Status SequenceBatchScheduler::Enqueue(std::unique_ptr& irequest) { // Queue timer starts at the beginning of the queueing and // scheduling process irequest->CaptureQueueStartNs(); INFER_TRACE_ACTIVITY( irequest->TraceProxy(), TRITONSERVER_TRACE_QUEUE_START, irequest->QueueStartNs()); // Record time at the beginning of the batcher queueing irequest->CaptureBatcherStartNs(); // For now the request must have batch-size 1 since the sequence // batcher does not yet support requests that are statically // batched. if (irequest->BatchSize() > 1) { return Status( Status::Code::INVALID_ARG, "inference request to model '" + irequest->ModelName() + "' must specify batch-size 1 due to requirements of sequence " "batcher"); } RETURN_IF_ERROR(sequencer_->SetupSequenceRequest(irequest)); const auto& correlation_id = irequest->CorrelationId(); BatcherSequenceSlot* target = nullptr; const bool seq_start = ((irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_START) != 0); const bool seq_end = ((irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_END) != 0); bool wake_reaper_thread = false; std::unique_lock lock(mu_); // Check if the request is one of the in-flight sequence (not starting new // sequence), we consider sequences in backlog as also in-flight. if (stop_ && seq_start) { return Status( Status::Code::UNAVAILABLE, "Server is stopping, scheduler for model has stopped accepting new " "inference requests"); } auto sb_itr = sequence_to_batcherseqslot_map_.find(correlation_id); auto bl_itr = sequence_to_backlog_map_.find(correlation_id); sequencer_->AddReleaseCallback( irequest, [this](std::unique_ptr& request, const uint32_t flags) -> Status { return sequencer_->RescheduleRequest(request, flags); }); // If this request is not starting a new sequence its correlation ID // should already be known with a target in either a sequence slot // or in the backlog. If it doesn't then the sequence wasn't started // correctly or there has been a correlation ID conflict. In either // case fail this request. if (!seq_start && (sb_itr == sequence_to_batcherseqslot_map_.end()) && (bl_itr == sequence_to_backlog_map_.end())) { std::string correlation_id_str{""}; if (correlation_id.Type() == InferenceRequest::SequenceId::DataType::STRING) { correlation_id_str = correlation_id.StringValue(); } else if ( correlation_id.Type() == InferenceRequest::SequenceId::DataType::UINT64) { correlation_id_str = std::to_string(correlation_id.UnsignedIntValue()); } return Status( Status::Code::INVALID_ARG, "inference request for sequence " + correlation_id_str + " to model '" + irequest->ModelName() + "' must specify the START flag on the first request of the " "sequence"); } // Record the timestamp of this request for the correlation ID. The // reaper thread will check to make sure that // max_sequence_idle_microseconds value is not exceed for any // sequence, and if it is it will release the sequence slot (if any) // allocated to that sequence. uint64_t now_us = Now(); correlation_id_timestamps_[correlation_id] = now_us; // If this request starts a new sequence but the correlation ID // already has an in-progress sequence then that previous sequence // did not end correctly, or there is a correlation ID conflict. In // this case we continue the new sequence (in either backlog or // sequence slot). It is ok for a backlog/slot to have multiple // starts... as long as it has a single end. The previous sequence // that was not correctly ended will have its existing requests // handled and then the new sequence will start. if (seq_start && ((sb_itr != sequence_to_batcherseqslot_map_.end()) || (bl_itr != sequence_to_backlog_map_.end()))) { LOG_WARNING << "sequence " << correlation_id << " for model '" << irequest->ModelName() << "' has a conflict. The previous sequence did not end before this " "sequence start. Previous sequence will be terminated early."; } // This request already has an assigned slot... if (sb_itr != sequence_to_batcherseqslot_map_.end()) { target = &sb_itr->second; } // This request already has a queue in the backlog... else if (bl_itr != sequence_to_backlog_map_.end()) { LOG_VERBOSE(1) << "Enqueuing CORRID " << correlation_id << " into existing backlog: " << irequest->ModelName(); auto& backlog = bl_itr->second; if (irequest->TimeoutMicroseconds() != 0) { backlog->expiration_timestamp_ = std::min( backlog->expiration_timestamp_, now_us + irequest->TimeoutMicroseconds()); if (backlog->expiration_timestamp_ < timeout_timestamp_) { timeout_timestamp_ = backlog->expiration_timestamp_; wake_reaper_thread = true; } } backlog->queue_->emplace_back(std::move(irequest)); // If the sequence is ending then forget correlation ID // connection to this backlog queue. If another sequence starts // with the same correlation ID it will be collected in another // backlog queue. if (seq_end) { sequence_to_backlog_map_.erase(bl_itr); } // Waking up reaper so it received latest timeout to be waited for, // shouldn't incur actual reaper work. if (wake_reaper_thread) { reaper_cv_.notify_all(); } return Status::Success; } // This request does not have an assigned backlog or sequence // slot. By the above checks it must be starting. If there is a free // sequence slot available then assign this sequence to that slot... else if (!ready_batcher_seq_slots_.empty()) { target = &sequence_to_batcherseqslot_map_[correlation_id]; *target = ready_batcher_seq_slots_.top(); ready_batcher_seq_slots_.pop(); } // Last option is to assign this request to the backlog... else { LOG_VERBOSE(1) << "Enqueuing CORRID " << correlation_id << " into new backlog: " << irequest->ModelName(); auto backlog = std::make_shared(); if (irequest->TimeoutMicroseconds() != 0) { backlog->expiration_timestamp_ = now_us + irequest->TimeoutMicroseconds(); if (backlog->expiration_timestamp_ < timeout_timestamp_) { timeout_timestamp_ = backlog->expiration_timestamp_; wake_reaper_thread = true; } } backlog_queues_.push_back(backlog); backlog->queue_->emplace_back(std::move(irequest)); if (!seq_end) { sequence_to_backlog_map_[correlation_id] = std::move(backlog); } // Waking up reaper so it received latest timeout to be waited for, // shouldn't incur actual reaper work. if (wake_reaper_thread) { reaper_cv_.notify_all(); } return Status::Success; } // Need to grab the target contents before the erase below since // that can free it. const TritonModelInstance* model_instance = target->model_instance_; const uint32_t seq_slot = target->seq_slot_; // At this point the request has been assigned to a sequence // slot. If the sequence is ending then stop tracking the // correlation. if (seq_end) { sequence_to_batcherseqslot_map_.erase(correlation_id); } // Enqueue request into batcher and sequence slot. Don't hold the // lock while enqueuing in a specific batcher. lock.unlock(); LOG_VERBOSE(1) << "Enqueuing CORRID " << correlation_id << " into batcher " << model_instance->Name() << ", sequence slot " << seq_slot << ": " << irequest->ModelName(); batchers_[model_instance]->Enqueue(seq_slot, correlation_id, irequest); return Status::Success; } void SequenceBatchScheduler::MarkRequestsCancelled( std::deque>* requests) { bool notify_clean_up = !requests->empty(); while (!requests->empty()) { auto& cancelled_request = requests->front(); if (cancelled_request) { cancelled_requests_.emplace_back(std::move(cancelled_request)); } requests->pop_front(); } if (notify_clean_up) { clean_up_cv_.notify_one(); } } InferenceRequest::SequenceId SequenceBatchScheduler::ReleaseSequenceSlot( const BatcherSequenceSlot& batcher_seq_slot, std::deque>* requests) { std::unique_lock lock(mu_); // If we are releasing the slot for a cancelled sequence, // we have to clean up the sequence // otherwise the reaper will try to clean it up again. if (!requests->empty() && requests->front()) { const InferenceRequest::SequenceId& corr_id = requests->front()->CorrelationId(); LOG_VERBOSE(1) << "Releasing canceled sequence CORRID " << corr_id; // Clean up the correlation id to sequence slot mapping, to avoid the reaper // from trying to release the same slot again on this instance of the // correlation id. sequence_to_batcherseqslot_map_.erase(corr_id); // Clean up the correlation id to sequence timeout timestamp mapping, to // avoid removal of a newer sequence from the backlog upon previous timeout // if the same id is re-used by the newer sequence. correlation_id_timestamps_.erase(corr_id); } // If there are any remaining requests on the releasing sequence slot, those // requests will be cancelled. MarkRequestsCancelled(requests); // If the instance behind the slot is pending to be removed, do not add the // slot back to ready and erase it instead. if (EraseBatcherSequenceSlot(batcher_seq_slot)) { // The slot is erased. return InferenceRequest::SequenceId(); } // If there is a backlogged sequence and it is requested, return it so that it // can use the newly available sequence slot. while (!backlog_queues_.empty()) { auto backlog = backlog_queues_.front()->queue_; backlog_queues_.pop_front(); if (backlog->empty()) { LOG_ERROR << "Should not print this! Unexpected empty backlog."; continue; } const auto& irequest = backlog->back(); const InferenceRequest::SequenceId& correlation_id = irequest->CorrelationId(); const bool seq_cancelled = IsAnyRequestCancelled(*backlog); // If the last queue entry is not an END request then the entire sequence is // not contained in the backlog. In that case must update backlog and // batcherseqslot maps so that future requests get directed to the batcher // sequence-slot instead of the backlog. const bool seq_end = ((irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_END) != 0); if (!seq_end) { // Since the correlation ID is being actively collected in the backlog, // there should not be any in-flight sequences with that same correlation // ID that have an assigned slot. if (sequence_to_batcherseqslot_map_.find(correlation_id) != sequence_to_batcherseqslot_map_.end()) { LOG_ERROR << irequest->LogRequest() << "internal: backlog sequence " << correlation_id << " conflicts with in-flight sequence for model '" << irequest->ModelName() << "'"; } sequence_to_backlog_map_.erase(correlation_id); if (!seq_cancelled) { sequence_to_batcherseqslot_map_[correlation_id] = batcher_seq_slot; } } if (seq_cancelled) { LOG_VERBOSE(1) << irequest->LogRequest() << "CORRID " << correlation_id << " sequence cancelled: " << irequest->ModelName(); MarkRequestsCancelled(backlog.get()); continue; } *requests = std::move(*backlog); LOG_VERBOSE(1) << irequest->LogRequest() << "CORRID " << correlation_id << " reusing batcher " << batcher_seq_slot.model_instance_->Name() << ", slot " << batcher_seq_slot.seq_slot_ << ": " << irequest->ModelName(); return correlation_id; } // There is no backlogged sequence so just release the batch slot LOG_VERBOSE(1) << "Freeing slot in batcher " << batcher_seq_slot.model_instance_->Name() << ", slot " << batcher_seq_slot.seq_slot_; ready_batcher_seq_slots_.push(batcher_seq_slot); return InferenceRequest::SequenceId(); } bool SequenceBatchScheduler::DelayScheduler( const TritonModelInstance* model_instance, const size_t cnt, const size_t total) { std::unique_lock lock(mu_); queue_request_cnts_[model_instance] = cnt; size_t seen = 0; for (auto c : queue_request_cnts_) { seen += c.second; } if (seen < total) { return true; } if (backlog_delay_cnt_ > 0) { size_t backlog_seen = 0; for (const auto& q : backlog_queues_) { backlog_seen += q->queue_->size(); } if (backlog_seen < backlog_delay_cnt_) { return true; } } return false; } void SequenceBatchScheduler::StartBackgroundThreads() { // Create a reaper thread that watches for idle sequences. reaper_thread_exit_ = false; reaper_thread_.reset( new std::thread([this]() { ReaperThread(10 /* nice */); })); // Create a clean-up thread that asynchronously erase removed resources. clean_up_thread_exit_ = false; clean_up_thread_.reset( new std::thread([this]() { CleanUpThread(20 /* nice */); })); } void SequenceBatchScheduler::StopBackgroundThreads() { // Exit the clean-up thread. clean_up_thread_exit_ = true; clean_up_cv_.notify_one(); if (clean_up_thread_ && clean_up_thread_->joinable()) { clean_up_thread_->join(); } // Exit the reaper thread. reaper_thread_exit_ = true; reaper_cv_.notify_one(); if (reaper_thread_ && reaper_thread_->joinable()) { reaper_thread_->join(); } } void SequenceBatchScheduler::ReaperThread(const int nice) { SetThreadPriority(nice, "sequence-batch reaper" /* thread_name */); const uint64_t backlog_idle_wait_microseconds = 50 * 1000; uint64_t idle_timestamp = Now() + max_sequence_idle_microseconds_; timeout_timestamp_ = std::numeric_limits::max(); while (!reaper_thread_exit_) { uint64_t now_us = Now(); // Reap idle assigned sequence if (now_us >= idle_timestamp) { uint64_t wait_microseconds = max_sequence_idle_microseconds_; BatcherSequenceSlotMap force_end_sequences; { std::unique_lock lock(mu_); for (auto cid_itr = correlation_id_timestamps_.cbegin(); cid_itr != correlation_id_timestamps_.cend();) { int64_t remaining_microseconds = (int64_t)max_sequence_idle_microseconds_ - (now_us - cid_itr->second); if (remaining_microseconds > 0) { wait_microseconds = std::min( wait_microseconds, (uint64_t)remaining_microseconds + 1); ++cid_itr; continue; } const InferenceRequest::SequenceId& idle_correlation_id = cid_itr->first; LOG_VERBOSE(1) << "Reaper: CORRID " << idle_correlation_id << ": max sequence idle exceeded"; auto idle_sb_itr = sequence_to_batcherseqslot_map_.find(idle_correlation_id); // If the idle correlation ID has an assigned sequence slot, // then release that assignment so it becomes available for // another sequence. Release is done by enqueuing and must be // done outside the lock, so just collect needed info here. if (idle_sb_itr != sequence_to_batcherseqslot_map_.end()) { force_end_sequences[idle_correlation_id] = idle_sb_itr->second; sequence_to_batcherseqslot_map_.erase(idle_correlation_id); cid_itr = correlation_id_timestamps_.erase(cid_itr); } else { // If the idle correlation ID is in the backlog, then just // need to increase the timeout so that we revisit it again in // the future to check if it is assigned to a sequence slot. auto idle_bl_itr = sequence_to_backlog_map_.find(idle_correlation_id); if (idle_bl_itr != sequence_to_backlog_map_.end()) { LOG_VERBOSE(1) << "Reaper: found idle CORRID " << idle_correlation_id; wait_microseconds = std::min(wait_microseconds, backlog_idle_wait_microseconds); ++cid_itr; } else { LOG_VERBOSE(1) << "Reaper: ignoring stale idle CORRID " << idle_correlation_id; cid_itr = correlation_id_timestamps_.erase(cid_itr); } } } } // Enqueue force-ends outside of the lock. for (const auto& pr : force_end_sequences) { const InferenceRequest::SequenceId& idle_correlation_id = pr.first; const TritonModelInstance* model_instance = pr.second.model_instance_; const uint32_t seq_slot = pr.second.seq_slot_; LOG_VERBOSE(1) << "Reaper: force-ending CORRID " << idle_correlation_id << " in batcher " << model_instance->Name() << ", slot " << seq_slot; // A slot assignment is released by enqueuing a request with a // null request. The scheduler thread will interpret the null // request as meaning it should release the sequence slot but // otherwise do nothing with the request. std::unique_ptr null_request; batchers_[model_instance]->Enqueue( seq_slot, idle_correlation_id, null_request); } // Update timestamp for next idle check idle_timestamp = now_us + wait_microseconds; } // Reap timed out backlog sequence if (now_us >= timeout_timestamp_) { timeout_timestamp_ = std::numeric_limits::max(); std::deque> expired_backlogs; { std::unique_lock lock(mu_); // Remove expired backlog from 'backlog_queues_' auto it = backlog_queues_.begin(); while (it != backlog_queues_.end()) { const auto queue_timestamp = (*it)->expiration_timestamp_; if (queue_timestamp > now_us) { timeout_timestamp_ = std::min(timeout_timestamp_, queue_timestamp); ++it; } else { // The queue expired, clear the records and reject the request // outside lock const auto& correlation_id = (*it)->queue_->front()->CorrelationId(); expired_backlogs.emplace_back(std::move(*it)); // Need to double check on 'sequence_to_backlog_map_', it may // be tracking a new sequence with the same ID which may not be // timing out. const auto& mit = sequence_to_backlog_map_.find(correlation_id); if ((mit != sequence_to_backlog_map_.end()) && (mit->second->expiration_timestamp_ <= now_us)) { sequence_to_backlog_map_.erase(mit); } it = backlog_queues_.erase(it); } } } // Reject timeout requests const static Status rejected_status = Status( Status::Code::UNAVAILABLE, "timeout of the corresponding sequence has been expired"); for (auto& backlog : expired_backlogs) { for (auto& req : *backlog->queue_) { InferenceRequest::RespondIfError( req, rejected_status, true /* release_requests */, FailureReason::REJECTED); } } } const auto wait_microseconds = std::min(idle_timestamp, timeout_timestamp_) - now_us; // Wait until the next timeout needs to be checked if (wait_microseconds > 0) { std::unique_lock lock(mu_); LOG_VERBOSE(2) << "Reaper: sleeping for " << wait_microseconds << "us..."; std::chrono::microseconds wait_timeout(wait_microseconds); reaper_cv_.wait_for(lock, wait_timeout); } } LOG_VERBOSE(1) << "Stopping sequence-batch reaper thread..."; } void SequenceBatchScheduler::CleanUpThread(const int nice) { SetThreadPriority(nice, "sequence-batch clean-up" /* thread_name */); while (!clean_up_thread_exit_) { // Removed resources should be destructed outside the lock. std::vector> removed_instances; std::vector> removed_batchers; std::vector> cancelled_requests; { std::unique_lock lk(mu_); clean_up_cv_.wait(lk, [this] { return clean_up_thread_exit_ || !removed_instances_.empty() || !removed_batchers_.empty() || !cancelled_requests_.empty(); }); removed_instances_.swap(removed_instances); removed_batchers_.swap(removed_batchers); cancelled_requests_.swap(cancelled_requests); } LOG_VERBOSE(2) << "Cleaning-up resources on sequence-batch clean-up thread..."; CancelRequests(std::move(cancelled_requests)); } LOG_VERBOSE(1) << "Stopping sequence-batch clean-up thread..."; } SequenceBatch::SequenceBatch( SequenceBatchScheduler* base, TritonModelInstance* model_instance, const size_t seq_slot_cnt, const std::unordered_map& enforce_equal_shape_tensors, const bool has_optional_input, const std::shared_ptr& start_input_overrides, const std::shared_ptr& end_input_overrides, const std::shared_ptr& startend_input_overrides, const std::shared_ptr& continue_input_overrides, const std::shared_ptr& notready_input_overrides) : base_(base), model_instance_(model_instance), seq_slot_cnt_(seq_slot_cnt), enforce_equal_shape_tensors_(enforce_equal_shape_tensors), has_optional_input_(has_optional_input), start_input_overrides_(start_input_overrides), end_input_overrides_(end_input_overrides), startend_input_overrides_(startend_input_overrides), continue_input_overrides_(continue_input_overrides), notready_input_overrides_(notready_input_overrides), sequence_states_(seq_slot_cnt) { } bool SequenceBatch::CreateCorrelationIDControl(const inference::ModelConfig& config) { // If model wants CORRID control then get the name of the input // tensor and initialize the override structure for each sequence // slot that is used to communicate the correlation ID. std::string correlation_id_tensor_name; inference::DataType correlation_id_datatype; Status corrid_status = GetTypedSequenceControlProperties( config.sequence_batching(), config.name(), inference::ModelSequenceBatching::Control::CONTROL_SEQUENCE_CORRID, false /* required */, &correlation_id_tensor_name, &correlation_id_datatype); if (!corrid_status.IsOk()) { LOG_ERROR << "failed validating CORRID control for sequence-batch " "scheduler thread " << model_instance_->Name() << ": " << corrid_status.Message(); return false; } if (!correlation_id_tensor_name.empty()) { if ((correlation_id_datatype != inference::DataType::TYPE_UINT64) && (correlation_id_datatype != inference::DataType::TYPE_INT64) && (correlation_id_datatype != inference::DataType::TYPE_UINT32) && (correlation_id_datatype != inference::DataType::TYPE_INT32) && (correlation_id_datatype != inference::DataType::TYPE_STRING)) { LOG_ERROR << "unexpected control data type, expected TYPE_UINT64, " "TYPE_INT64, TYPE_UINT32, TYPE_INT32, or TYPE_STRING for " << inference::ModelSequenceBatching_Control_Kind_Name( inference::ModelSequenceBatching::Control:: CONTROL_SEQUENCE_CORRID) << " for " << config.name(); return false; } const std::vector tensor_shape{1}; std::vector tensor_shape_with_batch_dim{1}; if (config.max_batch_size() != 0) { tensor_shape_with_batch_dim.push_back(1); } auto override = std::make_shared( correlation_id_tensor_name, correlation_id_datatype, tensor_shape); *override->MutableShape() = override->OriginalShape(); *override->MutableShapeWithBatchDim() = tensor_shape_with_batch_dim; seq_slot_corrid_override_ = std::move(override); } return true; } void SequenceBatch::SetControlTensors( std::unique_ptr& irequest, const int32_t seq_slot, const InferenceRequest::SequenceId& corrid, const bool not_ready) { const SequenceBatchScheduler::ControlInputs* controls; // Set the start, end, and ready control tensors appropriately... if (not_ready) { controls = notready_input_overrides_.get(); } else if ( (irequest->Flags() & (TRITONSERVER_REQUEST_FLAG_SEQUENCE_START | TRITONSERVER_REQUEST_FLAG_SEQUENCE_END)) == (TRITONSERVER_REQUEST_FLAG_SEQUENCE_START | TRITONSERVER_REQUEST_FLAG_SEQUENCE_END)) { controls = startend_input_overrides_.get(); } else if ( (irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_START) != 0) { controls = start_input_overrides_.get(); } else if ( (irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_END) != 0) { controls = end_input_overrides_.get(); } else { controls = continue_input_overrides_.get(); } for (const auto& control : *controls) { irequest->AddOverrideInput(control); } // Set correlation ID control tensor if requested by the model. if (seq_slot_corrid_override_ != nullptr) { auto& seq_corr_id = seq_slot_corrid_override_; size_t size_p = triton::common::GetDataTypeByteSize(seq_corr_id->DType()); if (seq_corr_id->DType() == inference::DataType::TYPE_STRING) { // 4 bytes for length of string plus pre-defined max string correlation id // length in bytes size_p = 4 + triton::core::STRING_CORRELATION_ID_MAX_LENGTH_BYTES; } TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; auto corrid_p = std::make_shared(size_p, TRITONSERVER_MEMORY_CPU, 0); char* corrid_p_ptr = corrid_p->MutableBuffer(&memory_type, &memory_type_id); if ((corrid_p_ptr == nullptr) || ((memory_type != TRITONSERVER_MEMORY_CPU) && (memory_type != TRITONSERVER_MEMORY_CPU_PINNED)) || (memory_type_id != 0)) { LOG_ERROR << "failed to allocate sequence CORRID control signal in CPU " "memory"; return; } auto override = std::make_shared( seq_corr_id->Name(), seq_corr_id->DType(), seq_corr_id->Shape()); *override->MutableShape() = override->OriginalShape(); *override->MutableShapeWithBatchDim() = seq_corr_id->ShapeWithBatchDim(); Status corrid_status = override->SetData(corrid_p); if (!corrid_status.IsOk()) { LOG_ERROR << "failed creating CORRID control for sequence-batch " "scheduler thread " << model_instance_->Name() << " for " << seq_corr_id->Name(); return; } if (corrid.Type() == InferenceRequest::SequenceId::DataType::STRING) { std::string correlation_id = corrid.StringValue(); uint32_t correlation_id_length = correlation_id.length(); memcpy(corrid_p_ptr, &correlation_id_length, sizeof(uint32_t)); memcpy( corrid_p_ptr + sizeof(uint32_t), correlation_id.c_str(), correlation_id_length); } else if ( corrid.Type() == InferenceRequest::SequenceId::DataType::UINT64) { uint64_t correlation_id = corrid.UnsignedIntValue(); const char* corrid_ptr = reinterpret_cast(&correlation_id); memcpy(corrid_p_ptr, corrid_ptr, size_p); } irequest->AddOverrideInput(override); } } void SequenceBatch::UpdateImplicitState( std::unique_ptr& irequest, const int32_t seq_slot) { // This should be executed only if the model has a states section. if (!base_->StateOutputConfigMap().empty()) { auto& sequence_states = sequence_states_[seq_slot]; // Initialize the input state if the sequence is starting. if ((irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_START) != 0) { sequence_states = nullptr; } // Create the state for the first request in the sequence. if (sequence_states == nullptr) { sequence_states.reset(new SequenceStates); Status status = sequence_states->Initialize( base_->StateOutputConfigMap(), base_->MaxBatchSize(), base_->InitialState(), model_instance_->Kind(), model_instance_->DeviceId(), model_instance_->Model()->Server()->CudaVirtualAddressSpaceSize()); if (!status.IsOk()) { LOG_ERROR << "Failed to initialize sequence state: " << status.Message(); } } irequest->SetSequenceStates(sequence_states); } } DirectSequenceBatch::DirectSequenceBatch( SequenceBatchScheduler* base, TritonModelInstance* model_instance, const size_t seq_slot_cnt, const std::unordered_map& enforce_equal_shape_tensors, const bool has_optional_input, const std::shared_ptr& start_input_overrides, const std::shared_ptr& end_input_overrides, const std::shared_ptr& startend_input_overrides, const std::shared_ptr& continue_input_overrides, const std::shared_ptr& notready_input_overrides, bool* is_initialized) : SequenceBatch( base, model_instance, seq_slot_cnt, enforce_equal_shape_tensors, has_optional_input, start_input_overrides, end_input_overrides, startend_input_overrides, continue_input_overrides, notready_input_overrides), scheduler_thread_exit_(false), scheduler_idle_(false), queues_(seq_slot_cnt), seq_slot_correlation_ids_(seq_slot_cnt, 0), max_active_seq_slot_(-1) { // Initialize to handle CORRID control. If error just exit // now... that means the corresponding model instance will not have // any runner and so will not get used for execution. const auto& config = model_instance_->Model()->Config(); if (!CreateCorrelationIDControl(config)) { *is_initialized = false; return; } max_batch_size_ = ((size_t)std::max(1, config.max_batch_size())); minimum_slot_utilization_ = config.sequence_batching().direct().minimum_slot_utilization(); pending_batch_delay_ns_ = config.sequence_batching().direct().max_queue_delay_microseconds() * 1000; // Create a scheduler thread associated with 'batcher_idx' that // executes the queued requests. const int nice = 0; NewPayload(); scheduler_thread_.reset( new std::thread([this, nice]() { BatcherThread(nice); })); *is_initialized = true; } DirectSequenceBatch::~DirectSequenceBatch() { // Wait until all queued requests begin execution. { std::unique_lock lk(mu_); queues_cv_.wait(lk, [this] { for (uint32_t seq_slot = 0; seq_slot < queues_.size(); seq_slot++) { if (!queues_[seq_slot].empty()) { LOG_VERBOSE(1) << "Waiting for slot " << seq_slot << " to begin execution before exiting"; return false; } } return true; }); } // Wait until the last enqueued payload completes execution. { std::unique_lock lk(payload_mu_); payload_cv_.wait(lk, [this] { if (!exec_complete_ || curr_payload_->RequestCount() > 0) { LOG_VERBOSE(1) << "Waiting for current payload to complete execution " "before exiting"; return false; } return true; }); } // Signal the scheduler thread to exit. scheduler_thread_exit_ = true; cv_.notify_one(); // It is possible for the scheduler thread to be the last holder of // a model object, and when that scheduler thread releases the // object the scheduler thread itself will destroy this // SequenceBatch object. So we need to check to make sure the // scheduler thread does not join it against itself and instead // detach it so there is not a problem when its thread object is // destroyed. if (scheduler_thread_->joinable()) { scheduler_thread_->join(); } } void DirectSequenceBatch::Enqueue( const uint32_t seq_slot, const InferenceRequest::SequenceId& correlation_id, std::unique_ptr& request) { bool wake_runner = false; { std::lock_guard lock(mu_); queues_[seq_slot].emplace_back(std::move(request)); seq_slot_correlation_ids_[seq_slot] = correlation_id; max_active_seq_slot_ = std::max(max_active_seq_slot_, static_cast(seq_slot)); // If runner is idle then wake it to service this request. We do // the actual wake outside of the lock to avoid having the woken // thread immediately block on the lock wake_runner = scheduler_idle_; } if (wake_runner) { cv_.notify_one(); } } void DirectSequenceBatch::NewPayload() { curr_payload_ = model_instance_->Model()->Server()->GetRateLimiter()->GetPayload( Payload::Operation::INFER_RUN, model_instance_); } void DirectSequenceBatch::BatcherThread(const int nice) { SetThreadPriority(nice, "Direct sequence-batch scheduler" /* thread_name */); // For debugging and testing, delay start of thread until queues // contain the specified number of entries (across all // SequenceBatchs in the scheduler). const char* dstr = getenv("TRITONSERVER_DELAY_SCHEDULER"); size_t delay_cnt = 0; if (dstr != nullptr) { delay_cnt = atoi(dstr); LOG_VERBOSE(1) << "Delaying scheduler thread " << model_instance_->Name() << " until " << delay_cnt << " queued requests..."; } const uint64_t default_wait_microseconds = 500 * 1000; exec_complete_ = true; // When there is optional input or input shape must be enforced, // the inputs in the requests must be examined for forming a batch const bool check_input = !enforce_equal_shape_tensors_.empty() || has_optional_input_; while (!scheduler_thread_exit_) { uint64_t wait_microseconds = default_wait_microseconds; // Wait till execution of the last enqueued payload is // complete. { std::unique_lock lk(payload_mu_); payload_cv_.wait(lk, [this] { return exec_complete_; }); } // Hold the lock for as short a time as possible. { std::unique_lock lock(mu_); if (delay_cnt > 0) { wait_microseconds = 10 * 1000; // Debugging/testing... wait until queues together contain at // least 'delay_cnt' items... size_t total_size = 0; for (const auto& q : queues_) { total_size += q.size(); } if (!base_->DelayScheduler(model_instance_, total_size, delay_cnt)) { delay_cnt = 0; } LOG_VERBOSE(1) << "Delaying scheduler thread " << model_instance_->Name() << " until " << delay_cnt << " queued requests, current total = " << total_size; } else { RequiredEqualInputs required_equal_inputs; InferenceRequest* null_irequest = nullptr; // Make one pass through the active slots to: // // 1) release any slots that have cancelled or forcibly ended // sequences // // 2) find a representative request that will provide: // // a) the shape, type, etc. information for null requests // // b) the required tensor shapes for the batch for the // case where ragged batching is not allowed // // 3) Determine the earliest enqueue time and number of ready // sequences if queue delay is enabled // int32_t max_seq_slot = -1; uint64_t earliest_enqueue_time_ns = UINT64_MAX; size_t ready_cnt = 0; for (int32_t seq_slot = 0; seq_slot <= max_active_seq_slot_; ++seq_slot) { std::deque>& queue = queues_[seq_slot]; if (!queue.empty()) { bool release_seq_slot = false; // If the request is nullptr then the sequence in the slot // has timed-out so release the slot for another sequence // from the backlog. if (queue.front() == nullptr) { queue.pop_front(); release_seq_slot = true; } // If the request is cancelled, then the sequence in the slot is // cancelled, so release the slot and cancel all queued requests of // the sequence. else if (queue.front()->IsCancelled()) { release_seq_slot = true; } if (release_seq_slot) { SequenceBatchScheduler::BatcherSequenceSlot batcher_seq_slot( model_instance_, seq_slot); seq_slot_correlation_ids_[seq_slot] = base_->ReleaseSequenceSlot(batcher_seq_slot, &queue); } } // Need to check queue again for contents since if released // above it may now be empty... if (!queue.empty()) { // For NULL requests need an InferenceRequest that can be // batched but has controls set to "not ready". Any // request can serve this purpose so grab a copy of the // first one. This first request is also used to // initialize 'required_equal_inputs' so we are sure that // this null request will have the correct shape for any // created batch. if (null_irequest == nullptr) { null_irequest = queue.front().get(); UpdateImplicitState(queue.front(), seq_slot); } // If this is the first non-null request capture the shape // of the tensors that don't support ragged so we can // compare them to later requests. if (!required_equal_inputs.Initialized() && check_input) { Status status = required_equal_inputs.Initialize( queue.front(), enforce_equal_shape_tensors_, has_optional_input_); if (!status.IsOk()) { LOG_ERROR << "internal: unexpecting failure initializing shape: " << status.Message(); } } earliest_enqueue_time_ns = std::min( earliest_enqueue_time_ns, queue.front()->BatcherStartNs()); ready_cnt++; max_seq_slot = seq_slot; } } if (max_seq_slot != -1) { if ((pending_batch_delay_ns_ == 0) || (minimum_slot_utilization_ == 0.0)) { wait_microseconds = 0; } else { // Compare the age of the oldest pending request to the maximum // batch queuing delay, and the size of the ready requests in the // batch, execute now if queuing delay is exceeded or the batch // size is large enough. Otherwise create a timer to wakeup a // thread to check again at the maximum allowed delay. uint64_t now_ns = Now(); uint64_t current_batch_delay_ns = (now_ns - earliest_enqueue_time_ns); if ((current_batch_delay_ns > pending_batch_delay_ns_) || (((float)ready_cnt) / max_batch_size_ >= minimum_slot_utilization_)) { wait_microseconds = 0; LOG_VERBOSE(1) << "start sequence batch execution. " << "current batch delay: " << current_batch_delay_ns << "; maximum delay allowed: " << pending_batch_delay_ns_ << "slot utilization: " << ready_cnt << "/" << max_batch_size_ << "; utilization threshold: " << minimum_slot_utilization_; } else { wait_microseconds = (pending_batch_delay_ns_ - current_batch_delay_ns) / 1000; // reset 'max_seq_slot' so that not request is pulled from the // queues max_seq_slot = -1; LOG_VERBOSE(1) << "defer sequence batch execution. " << "current batch delay: " << current_batch_delay_ns << "; maximum delay allowed: " << pending_batch_delay_ns_ << "slot utilization: " << ready_cnt << "/" << max_batch_size_ << "; utilization threshold: " << minimum_slot_utilization_; } } } // Collect requests from slot 0 to max_seq_slot. for (int32_t seq_slot = 0; seq_slot <= max_seq_slot; ++seq_slot) { bool end_of_sequence = false; bool use_null_request = false; std::deque>& queue = queues_[seq_slot]; // If 'seq_slot' doesn't have any requests then change the // request to send dummy/null input tensors for this // slot. We need this so that other requests stay in the // correct slot. if (queue.empty()) { use_null_request = true; } // If there are one or more tensors that don't support // ragged batch, then don't allow a request into an existing // batch if shape differs. else if (required_equal_inputs.Initialized() && check_input) { if (!required_equal_inputs.HasEqualInputs(queue.front())) { use_null_request = true; } } // Use null-request if necessary otherwise use the next // request in the queue... if (use_null_request) { std::unique_ptr ni( InferenceRequest::CopyAsNull(*null_irequest)); // Note that when the not-ready control input of the // request is "true" the model can't assume that any // other inputs are meaningful, including CORRID. So we // just use zero for that. SetControlTensors( ni, seq_slot, 0 /* corrid */, true /* not_ready */); // This should be executed only if the model has a states section. if (!base_->StateOutputConfigMap().empty()) { // For NULL requests we will be using a dummy state instead of the // real state stored in Triton. When the model is using variable // dimensions and batching, the null request's input state shapes // may be different from the actual shapes of the state for that // sequence. We create a dummy state in order to avoid corrupting // the actual state of the sequence. std::shared_ptr sequence_states( new SequenceStates); sequence_states->SetNullSequenceStates( null_irequest->GetSequenceStates()); ni->SetSequenceStates(sequence_states); } curr_payload_->AddRequest(std::move(ni)); } else { std::unique_ptr& irequest = queue.front(); // Set the control tensor values in the request. SetControlTensors(irequest, seq_slot, irequest->CorrelationId()); // Update the implicit state and set the input state tensors. UpdateImplicitState(irequest, seq_slot); if ((irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_END) != 0) { end_of_sequence = true; } curr_payload_->AddRequest(std::move(irequest)); queue.pop_front(); } if (curr_payload_->GetState() == Payload::State::UNINITIALIZED) { curr_payload_->SetState(Payload::State::READY); } // If the sequence has ended then attempt to refill the // sequence slot with a sequence from the backlog. If // there is no backlog show that the slot is no longer // active. if (end_of_sequence) { LOG_VERBOSE(1) << "End sequence CORRID " << seq_slot_correlation_ids_[seq_slot] << " in batcher " << model_instance_->Name() << ", slot " << seq_slot; // Should never be anything in a queue after the END // marker. If it happens that means we will clobber // that request if/when we swap in a backlog sequence // in ReleaseSequenceSlot below. if (!queue.empty()) { LOG_ERROR << "internal: unexpected requests after sequence " "end in slot " << seq_slot; } SequenceBatchScheduler::BatcherSequenceSlot batcher_seq_slot( model_instance_, seq_slot); seq_slot_correlation_ids_[seq_slot] = base_->ReleaseSequenceSlot(batcher_seq_slot, &queue); } } } // One or more sequences may have ended... find the new // 'max_active_seq_slot_'. while ((max_active_seq_slot_ >= 0) && (!seq_slot_correlation_ids_[max_active_seq_slot_].InSequence())) { max_active_seq_slot_--; } // If no requests are to be handled, wait for notification or // for the specified timeout before checking the queues again. if (wait_microseconds > 0) { scheduler_idle_ = true; std::chrono::microseconds wait_timeout(wait_microseconds); cv_.wait_for(lock, wait_timeout); scheduler_idle_ = false; } } // Requests could be removed from the queue, so notify a thread waiting for // requests removal from the queue. queues_cv_.notify_one(); if (curr_payload_->GetState() == Payload::State::READY) { // Add callback to signal the execution completion exec_complete_ = false; auto callback = [this]() { { std::unique_lock lk(payload_mu_); exec_complete_ = true; } payload_cv_.notify_all(); }; curr_payload_->AddInternalReleaseCallback(callback); curr_payload_->MarkSaturated(); // Enqueue the payload to RateLimiter model_instance_->Model()->Server()->GetRateLimiter()->EnqueuePayload( model_instance_->Model(), curr_payload_); NewPayload(); } } // end runner loop LOG_VERBOSE(1) << "Stopping Direct sequence-batch scheduler thread " << model_instance_->Name() << "..."; } OldestSequenceBatch::OldestSequenceBatch( SequenceBatchScheduler* base, TritonModelInstance* model_instance, const size_t seq_slot_cnt, const std::unordered_map& enforce_equal_shape_tensors, const bool has_optional_input, const std::shared_ptr& start_input_overrides, const std::shared_ptr& end_input_overrides, const std::shared_ptr& startend_input_overrides, const std::shared_ptr& continue_input_overrides, const std::shared_ptr& notready_input_overrides, bool* is_initialized) : SequenceBatch( base, model_instance, seq_slot_cnt, enforce_equal_shape_tensors, has_optional_input, start_input_overrides, end_input_overrides, startend_input_overrides, continue_input_overrides, notready_input_overrides), in_flight_(seq_slot_cnt, false), queues_(seq_slot_cnt) { // Initialize to handle CORRID control. If error just exit // now... that means the corresponding model instance will not have // any runner and so will not get used for execution. const auto& config = model_instance->Model()->Config(); if (!CreateCorrelationIDControl(config)) { *is_initialized = false; return; } // Create a dynamic batcher use to batch together sequences for // inference. std::set preferred_batch_sizes; for (const auto size : config.sequence_batching().oldest().preferred_batch_size()) { preferred_batch_sizes.insert(size); } // TODO: Provide appropriate request_cache_enable flag when caching // is enabled for sequence models. Status status = DynamicBatchScheduler::Create( model_instance->Model(), model_instance, triton::common::GetCpuNiceLevel(config), true /* dynamic_batching_enabled */, config.max_batch_size(), enforce_equal_shape_tensors_, config.sequence_batching().oldest().preserve_ordering(), preferred_batch_sizes, config.sequence_batching().oldest().max_queue_delay_microseconds(), &dynamic_batcher_); if (!status.IsOk()) { LOG_ERROR << "failed creating dynamic sequence batcher for OldestFirst " << model_instance->Name() << ": " << status.Message(); *is_initialized = false; return; } *is_initialized = true; } OldestSequenceBatch::~OldestSequenceBatch() { std::unique_lock lock(mu_); // Wait until all pending requests are completed. for (uint32_t seq_slot = 0; seq_slot < queues_.size(); seq_slot++) { while (in_flight_[seq_slot] || !queues_[seq_slot].empty()) { LOG_VERBOSE(1) << "Waiting for slot " << seq_slot << " with " << (in_flight_[seq_slot] ? "an" : "no") << " in-flight request and " << queues_[seq_slot].size() << " pending requests before exiting"; cv_.wait(lock); } } } void OldestSequenceBatch::CompleteAndNext(const uint32_t seq_slot) { { std::lock_guard lock(mu_); // We may enqueue 1 or more pending inferences triggered by the // completion. If the sequence has a pending inference then it needs // to be send to dynamic batcher since the "previous" inference just // completed. If this next inference ends up being the end of the // sequence (either from the END flag or because the sequence is // being force-ended) then we try to fill the now-free sequence slot // from the backlog and then send the first inference from that // sequence to the dynamic batcher... std::deque>& queue = queues_[seq_slot]; bool retry = true; while (retry) { retry = false; bool release_seq_slot = false; in_flight_[seq_slot] = false; // If the next sequence inference is ready in the queue then enqueue // it in the dynamic batcher now. if (!queue.empty()) { auto& irequest = queue.front(); bool retain_queue_front = false; // If the request is null then this inference request is from // the reaper thread indicating a timed-out sequence. Mark that // the sequence slot should be released but otherwise do // nothing. if (irequest == nullptr) { LOG_VERBOSE(1) << "force-end timed-out sequence in batcher " << model_instance_->Name() << ", slot " << seq_slot; release_seq_slot = true; } else if (irequest->IsCancelled()) { const InferenceRequest::SequenceId& correlation_id = irequest->CorrelationId(); LOG_VERBOSE(1) << irequest->LogRequest() << "force-end cancelled sequence CORRID " << correlation_id << " in batcher " << model_instance_->Name() << ", slot " << seq_slot; release_seq_slot = true; retain_queue_front = true; } else { const InferenceRequest::SequenceId& correlation_id = irequest->CorrelationId(); // After handling the last inference in a sequence we must // release the sequence slot to make it available to another // sequence. if ((irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_END) != 0) { LOG_VERBOSE(1) << irequest->LogRequest() << "end sequence CORRID " << correlation_id << " in batcher " << model_instance_->Name() << ", slot " << seq_slot; release_seq_slot = true; } // Add the appropriate control tensor values to the request. SetControlTensors(irequest, seq_slot, correlation_id); // Update the implicit state and set the input state tensors. UpdateImplicitState(irequest, seq_slot); LOG_VERBOSE(1) << irequest->LogRequest() << "issue to dynamic batcher CORRID " << correlation_id << " in batcher " << model_instance_->Name() << ", slot " << seq_slot; in_flight_[seq_slot] = true; base_->SequencerPtr()->AddReleaseCallback( irequest, [this, seq_slot]( std::unique_ptr& request, const uint32_t flags) -> Status { CompleteAndNext(seq_slot); return Status::Success; }); dynamic_batcher_->Enqueue(irequest); } if (!retain_queue_front) { queue.pop_front(); } } // If releasing the sequence slot then the sequence queue should be // empty and we can now assign a new sequence to the queue (from the // backlog). if (release_seq_slot) { // Unless the sequence is cancelled, there should never be anything in a // queue after the END marker. Any requests remaining in the queue will // be cancelled. if (!queue.empty()) { LOG_VERBOSE(2) << "requests remaining when releasing sequence slot " << seq_slot; } SequenceBatchScheduler::BatcherSequenceSlot batcher_seq_slot( model_instance_, seq_slot); const InferenceRequest::SequenceId& released_cid = base_->ReleaseSequenceSlot(batcher_seq_slot, &queue); if (released_cid.InSequence()) { LOG_VERBOSE(1) << "Enqueued new sequence containing " << queue.size() << " requests into OldestFirst batcher " << model_instance_->Name() << ", slot " << seq_slot; // If an inference is already in-flight in the dynamic batcher // in this sequence slot then can't process the new queue // inferences right now, because the in-flight request is // using slot resources like the CORRID override map. if (!in_flight_[seq_slot]) { retry = true; } } } } } // Opportunity for checking queueing and in-flight status. cv_.notify_all(); } void OldestSequenceBatch::Enqueue( const uint32_t seq_slot, const InferenceRequest::SequenceId& correlation_id, std::unique_ptr& request) { // Queue the new request... if there isn't already a request in // flight for this sequence then send one to the dynamic batcher // immediately. bool in_flight; { std::lock_guard lock(mu_); std::deque>& queue = queues_[seq_slot]; queue.emplace_back(std::move(request)); in_flight = in_flight_[seq_slot]; } if (!in_flight) { CompleteAndNext(seq_slot); } } }} // namespace triton::core kytensor-2.49.0.7/core/src/sequence_batch_scheduler/sequence_batch_scheduler.h000066400000000000000000000461321527052371200275510ustar00rootroot00000000000000// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include #include "backend_model.h" #include "backend_model_instance.h" #include "model_config.pb.h" #include "rate_limiter.h" #include "scheduler.h" #include "scheduler_utils.h" #include "sequence_state.h" #include "sequence_utils.h" #include "status.h" #include "triton/common/model_config.h" namespace triton { namespace core { class SequenceBatch; // Scheduler that implements batching across sequences of correlated // inferences. class SequenceBatchScheduler : public Scheduler { public: using ControlInputs = std::vector>; ~SequenceBatchScheduler(); // Create a scheduler to support a given number of runners and a run // function to call when a request is scheduled. static Status Create( TritonModel* model, const std::vector>& new_instances, const std::unordered_map& enforce_equal_shape_tensors, std::unique_ptr* scheduler); // \see Scheduler::Enqueue() Status Enqueue(std::unique_ptr& request) override; // \see Scheduler::InflightInferenceCount() size_t InflightInferenceCount() override { std::unique_lock lock(mu_); return sequence_to_batcherseqslot_map_.size(); } // \see Scheduler::Stop() void Stop() override { stop_ = true; } // Update the scheduler to the new set of model instances. This function // cannot be called concurrently. Status Update( const std::vector>& added_instances, const std::vector>& removed_instances); // A batcher-sequence_slot combination. The batcher is represented // by the index into 'batchers_'. struct BatcherSequenceSlot { BatcherSequenceSlot() = default; BatcherSequenceSlot(const BatcherSequenceSlot&) = default; BatcherSequenceSlot(TritonModelInstance* i, uint32_t s) : model_instance_(i), seq_slot_(s) { } TritonModelInstance* model_instance_; uint32_t seq_slot_; }; // Fill a sequence slot with a sequence from the backlog or show // that the sequence slot is no longer being used. InferenceRequest::SequenceId ReleaseSequenceSlot( const BatcherSequenceSlot& seq_slot, std::deque>* requests); // For debugging/testing, batcher reports how many waiting requests // and returns true if the batcher should continue waiting. bool DelayScheduler( const TritonModelInstance* model_instance, const size_t cnt, const size_t total); const std::unordered_map< std::string, const inference::ModelSequenceBatching_State&>& StateOutputConfigMap() { return state_output_config_map_; } size_t MaxBatchSize() { return max_batch_size_; } const std::unordered_map& InitialState() { return initial_state_; } Sequencer* SequencerPtr() { return sequencer_.get(); } private: SequenceBatchScheduler( TritonModel* model, const std::unordered_map& enforce_equal_shape_tensors) : model_(model), enforce_equal_shape_tensors_(enforce_equal_shape_tensors), stop_(false) { } void StartBackgroundThreads(); void StopBackgroundThreads(); Status CreateBooleanControlTensors( const inference::ModelConfig& config, std::shared_ptr* start_input_overrides, std::shared_ptr* end_input_overrides, std::shared_ptr* startend_input_overrides, std::shared_ptr* continue_input_overrides, std::shared_ptr* notready_input_overrides); Status GenerateInitialStateData( const inference::ModelSequenceBatching_InitialState& initial_state, const inference::ModelSequenceBatching_State& state, TritonModel* model); struct BatcherSequenceSlotCompare { bool operator()( const BatcherSequenceSlot& a, const BatcherSequenceSlot& b) const { return a.seq_slot_ > b.seq_slot_; } }; // Create a batcher for each of the provided instances. Status CreateBatchers( const std::vector>& instances); // Move requests so they will be cancelled by the CleanUpThread. void MarkRequestsCancelled( std::deque>* requests); // Erase the sequence slot from 'pending_removal_seq_slots_'. The batcher // behind the sequence slot will be removed when all sequence slots of the // batcher are removed. Return true if the sequence slot is pending removal. // Otherwise, false is returned. bool EraseBatcherSequenceSlot(const BatcherSequenceSlot& seq_slot); // A thread that monitors idle sequences. This thread is time sensitive that // all operations should be completed as quickly as possible to avoid blocking // the thread from starting its next iteration. void ReaperThread(const int nice); // A thread that asynchronously erase removed resources. This thread is // intended to destruct resources that might take some time to complete, // without preventing the scheduler from scheduling requests. void CleanUpThread(const int nice); // The 'TritonModel' and 'enforce_equal_shape_tensors' when this scheduler is // created. TritonModel* model_; std::unordered_map enforce_equal_shape_tensors_; // The number of candidate sequence slots. size_t seq_slot_cnt_; // The max_sequence_idle_microseconds value for this scheduler. uint64_t max_sequence_idle_microseconds_; // Whether this scheduler has stopped accepting new inference requests. bool stop_; // Mutex std::mutex mu_; // The reaper thread std::unique_ptr reaper_thread_; std::condition_variable reaper_cv_; std::atomic reaper_thread_exit_; // Need to share between enqueue thread and reaper thread because // the timeout may be shorten by new request uint64_t timeout_timestamp_; // The clean-up thread std::unique_ptr clean_up_thread_; std::condition_variable clean_up_cv_; std::atomic clean_up_thread_exit_; // Removed objects to be cleaned up std::vector> removed_instances_; std::vector> removed_batchers_; std::vector> cancelled_requests_; // Map from a model instance pointer that is pending to be removed from this // scheduler to a pair ["the number of sequence slots remaining for the // instance batcher", "the shared_ptr of the instance"]. The shared_ptr is // kept to ensure the instance is available until its sequence slots and // batcher are removed from this scheduler. std::unordered_map< const TritonModelInstance*, std::pair>> pending_removal_seq_slots_; // The SequenceBatchs being managed by this scheduler. std::unordered_map> batchers_; // Map from a request's correlation ID to the BatcherSequenceSlot // assigned to that correlation ID. using BatcherSequenceSlotMap = std::unordered_map; BatcherSequenceSlotMap sequence_to_batcherseqslot_map_; // The ordered backlog of sequences waiting for a free sequence slot. // The backlog queue keep track of the closest expiration timestamp among // the request of the corresponding sequence. Reaper thread will // sweep the queues on wake up and clear all timed out sequence. // See ReaperThread() for detail implementation. struct BacklogQueue { // Default to max value so it is not possible to time out unless specified. uint64_t expiration_timestamp_{std::numeric_limits::max()}; std::shared_ptr>> queue_{ std::make_shared>>()}; }; std::deque> backlog_queues_; // Map from a request's correlation ID to the backlog queue // collecting requests for that correlation ID. using BacklogMap = std::unordered_map< InferenceRequest::SequenceId, std::shared_ptr>; BacklogMap sequence_to_backlog_map_; // The batcher/sequence-slot locations ready to accept a new // sequence. Ordered from lowest sequence-slot-number to highest so // that all batchers grow at the same rate and attempt to remain as // small as possible. std::priority_queue< BatcherSequenceSlot, std::vector, BatcherSequenceSlotCompare> ready_batcher_seq_slots_; // For each correlation ID the most recently seen timestamp, in // microseconds, for a request using that correlation ID. std::unordered_map correlation_id_timestamps_; // Used for debugging/testing. size_t backlog_delay_cnt_; std::unordered_map queue_request_cnts_; // IO mapping between the output state name and the state configuration. std::unordered_map state_output_config_map_; size_t max_batch_size_; // Initial state used for implicit state. std::unordered_map initial_state_; std::unique_ptr sequencer_; }; // Base class for a scheduler that implements a particular scheduling // strategy for a model instance. class SequenceBatch { public: SequenceBatch( SequenceBatchScheduler* base, TritonModelInstance* model_instance, const size_t seq_slot_cnt, const std::unordered_map& enforce_equal_shape_tensors, const bool has_optional_input, const std::shared_ptr& start_input_overrides, const std::shared_ptr& end_input_overrides, const std::shared_ptr& startend_input_overrides, const std::shared_ptr& continue_input_overrides, const std::shared_ptr& notready_input_overrides); virtual ~SequenceBatch() = default; // Enqueue a request into the appropriate queue for the requested // sequence slot. This function takes ownership of 'request' so on // request 'request' will be nullptr. virtual void Enqueue( const uint32_t seq_slot, const InferenceRequest::SequenceId& correlation_id, std::unique_ptr& request) = 0; size_t SeqSlotCnt() { return seq_slot_cnt_; } protected: bool CreateCorrelationIDControl(const inference::ModelConfig& config); void SetControlTensors( std::unique_ptr& irequest, const int32_t seq_slot, const InferenceRequest::SequenceId& corr_id, const bool not_ready = false); // Update the implicit state and set the required input states. void UpdateImplicitState( std::unique_ptr& irequest, const int32_t seq_slot); // The controlling scheduler. SequenceBatchScheduler* const base_; // The identifier of this batcher within the controlling scheduler. TritonModelInstance* const model_instance_; // The number of candidate sequence slots. const size_t seq_slot_cnt_; // The input tensors that require shape checking before being // allowed in a batch. As a map from the tensor name to a bool. If // tensor is in map then its shape must match shape of same tensor // in requests already in the batch. If value is "true" then // additional tensor is treated as a shape tensor and the values // contained in the shape tensor must match same tensor already in // the batch. const std::unordered_map enforce_equal_shape_tensors_; // Store information on whether the model contains optional inputs. bool has_optional_input_; // The control values, delivered as input tensors, that should be // used when starting a sequence, continuing a sequence, ending a // sequence, and showing that a sequence has not input available. std::shared_ptr start_input_overrides_; std::shared_ptr end_input_overrides_; std::shared_ptr startend_input_overrides_; std::shared_ptr continue_input_overrides_; std::shared_ptr notready_input_overrides_; // The correlation ID override. Empty if model does not specify the // CONTROL_SEQUENCE_CORRID control. std::shared_ptr seq_slot_corrid_override_; // For each sequence slot store the optional state i/o tensors. std::vector> sequence_states_; }; // Scheduler that implements the Direct sequence scheduling strategy // for a model instance. class DirectSequenceBatch : public SequenceBatch { public: DirectSequenceBatch( SequenceBatchScheduler* base, TritonModelInstance* model_instance, const size_t seq_slot_cnt, const std::unordered_map& enforce_equal_shape_tensors, const bool has_optional_input, const std::shared_ptr& start_input_overrides, const std::shared_ptr& end_input_overrides, const std::shared_ptr& startend_input_overrides, const std::shared_ptr& continue_input_overrides, const std::shared_ptr& notready_input_overrides, bool* is_initialized); ~DirectSequenceBatch(); void Enqueue( const uint32_t seq_slot, const InferenceRequest::SequenceId& correlation_id, std::unique_ptr& request) override; private: void BatcherThread(const int nice); void NewPayload(); std::shared_ptr curr_payload_; // The thread scheduling requests that are queued in this batch. std::unique_ptr scheduler_thread_; std::atomic scheduler_thread_exit_; bool scheduler_idle_; // Mutex protecting correlation queues, etc. std::mutex mu_; std::condition_variable cv_; // Execution state of the last enqueued payload bool exec_complete_; // Mutex protecting execution state of payload std::mutex payload_mu_; std::condition_variable payload_cv_; // Queues holding inference requests. There are 'seq_slot_cnt' // queues, one for each sequence slot where requests assigned to // that slot are enqueued to wait for inferencing. std::vector>> queues_; // Notify when requests are removed from the queue. std::condition_variable queues_cv_; // Is each sequence slot active or not? A zero or empty value indicates // inactive, a non-zero/non-empty value indicates active and is the // correlation ID of the sequence active in the slot. An empty // queue for a sequence slot does not mean it's inactive... it // could just not have any requests pending at the moment. std::vector seq_slot_correlation_ids_; // The maximum active sequence slot. A value of -1 indicates that // no slots are active in the model. int32_t max_active_seq_slot_; size_t max_batch_size_; float minimum_slot_utilization_; uint64_t pending_batch_delay_ns_; }; // Scheduler that implements the oldest-first sequence scheduling // strategy for a model instance. class OldestSequenceBatch : public SequenceBatch { public: OldestSequenceBatch( SequenceBatchScheduler* base, TritonModelInstance* model_instance, const size_t seq_slot_cnt, const std::unordered_map& enforce_equal_shape_tensors, const bool has_optional_input, const std::shared_ptr& start_input_overrides, const std::shared_ptr& end_input_overrides, const std::shared_ptr& startend_input_overrides, const std::shared_ptr& continue_input_overrides, const std::shared_ptr& notready_input_overrides, bool* is_initialized); ~OldestSequenceBatch(); void Enqueue( const uint32_t seq_slot, const InferenceRequest::SequenceId& correlation_id, std::unique_ptr& request) override; private: void CompleteAndNext(const uint32_t seq_slot); // The dynamic batcher for this scheduler std::unique_ptr dynamic_batcher_; // Mutex protecting queues, etc. std::mutex mu_; // Condition variable notifying request queueing and/or in-flight completion. std::condition_variable cv_; // For each sequence slot, true if there is a request for that // sequence in-flight in the dynamic batcher. Used to ensure that at // most one request from each sequence can be scheduled at a time. std::vector in_flight_; // Queues holding inference requests. There are 'seq_slot_cnt' // queues, one for each sequence slot where requests assigned to // that slot are enqueued to wait for inferencing. std::vector>> queues_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/sequence_batch_scheduler/sequence_utils.cc000066400000000000000000000053421527052371200257260ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "sequence_utils.h" #include "sequence_batch_scheduler.h" namespace triton { namespace core { Status IterativeSequencer::RescheduleRequest( std::unique_ptr& request, const uint32_t flags) { if (flags & TRITONSERVER_REQUEST_RELEASE_RESCHEDULE) { // set flags to be not START and not END request->SetFlags(0); return base_->Enqueue(request); } // If not reschedule, use request cancellation to release sequence // resources. Check IsCancelled() to break the recursive calls if // the callback is called by cancellation. else if (!request->IsCancelled()) { // Use a null request to trigger sequence batcher cancellation so // additional request manipulation won't affect the actual request. std::unique_ptr ni( InferenceRequest::CopyAsNull(*request)); ni->SetCorrelationId(request->CorrelationId()); ni->SetFlags(TRITONSERVER_REQUEST_FLAG_SEQUENCE_END); ni->Cancel(); // result of enqueuing null request is not related to the actual request auto status = base_->Enqueue(ni); if (!status.IsOk()) { LOG_ERROR << status.AsString(); } } return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/sequence_batch_scheduler/sequence_utils.h000066400000000000000000000102341527052371200255640ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "infer_request.h" #include "status.h" namespace triton { namespace core { class SequenceBatchScheduler; class Sequencer { public: // Regular sequencer expects the request is well-prepared by the user when // sending inference request. virtual Status SetupSequenceRequest( std::unique_ptr& irequest) { // A request must have a correlation ID to be processed correctly by // this scheduler. A value of 0 (zero) or "" (empty) indicates that the // request doesn't have a correlation ID. const auto& correlation_id = irequest->CorrelationId(); if (!correlation_id.InSequence()) { return Status( Status::Code::INVALID_ARG, "inference request to model '" + irequest->ModelName() + "' must specify a non-zero or non-empty correlation ID"); } return Status::Success; } virtual void AddReleaseCallback( std::unique_ptr& irequest, InferenceRequest::InternalReleaseFn&& callback) { irequest->AddInternalReleaseCallback(std::move(callback)); } virtual Status RescheduleRequest( std::unique_ptr& request, const uint32_t flags) { // Sequencer will not reschedule requests return Status::Success; } }; class IterativeSequencer : public Sequencer { public: IterativeSequencer(SequenceBatchScheduler* base) : base_(base) {} // Iterative sequencer will prepare the request for sequence batcher if it is // not associated with an sequence Status SetupSequenceRequest( std::unique_ptr& irequest) override { // A request must have a correlation ID to be processed correctly by // this scheduler. A value of 0 (zero) or "" (empty) indicates that the // request doesn't have a correlation ID. const auto& correlation_id = irequest->CorrelationId(); if (!correlation_id.InSequence()) { irequest->SetCorrelationId(InferenceRequest::SequenceId(sequence_id_++)); irequest->SetFlags(TRITONSERVER_REQUEST_FLAG_SEQUENCE_START); } return Status::Success; } void AddReleaseCallback( std::unique_ptr& irequest, InferenceRequest::InternalReleaseFn&& callback) override { if (irequest->Flags() & TRITONSERVER_REQUEST_FLAG_SEQUENCE_START) { irequest->AddInternalReleaseCallback(std::move(callback)); } } Status RescheduleRequest( std::unique_ptr& request, const uint32_t flags) override; private: std::atomic sequence_id_{1}; SequenceBatchScheduler* const base_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/sequence_state.cc000066400000000000000000000400371527052371200206770ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "sequence_state.h" #include "cuda_utils.h" #include "memory.h" #include "triton/common/logging.h" namespace triton { namespace core { SequenceState::SequenceState() : data_(new MemoryReference) {} SequenceState::SequenceState( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count, bool use_single_buffer, bool use_growable_memory) : name_(name), datatype_(datatype), shape_(shape, shape + dim_count), data_(new MemoryReference), use_single_buffer_(use_single_buffer), use_growable_memory_(use_growable_memory) { } SequenceState::SequenceState( const std::string& name, const inference::DataType datatype, const std::vector& shape, bool use_single_buffer, bool use_growable_memory) : name_(name), datatype_(datatype), shape_(shape), data_(new MemoryReference), use_single_buffer_(use_single_buffer), use_growable_memory_(use_growable_memory) { } Status SequenceState::SetData(const std::shared_ptr& data) { if (data_->TotalByteSize() != 0) { return Status( Status::Code::INVALID_ARG, "state '" + name_ + "' already has data, can't overwrite"); } data_ = data; return Status::Success; } Status SequenceState::RemoveAllData() { data_ = std::make_shared(); return Status::Success; } Status SequenceState::SetStringDataToZero() { if (Data()->TotalByteSize() % 4 != 0) { return Status( Status::Code::INVALID_ARG, "The total byte size must be a multiple of 4 when setting the " "sequence state to zero."); } const std::shared_ptr& memory = reinterpret_cast&>(Data()); RETURN_IF_ERROR(memory->SetMemory(0 /* value */)); return Status::Success; } Status SequenceState::ResizeOrReallocate( void** buffer, const uint64_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { if (UseGrowableMemory()) { GrowableMemory* growable_memory = reinterpret_cast(Data().get()); RETURN_IF_ERROR(growable_memory->Resize(buffer_byte_size)); *buffer = growable_memory->MutableBuffer(memory_type, memory_type_id); } else { std::shared_ptr memory = std::make_shared( buffer_byte_size, *memory_type, *memory_type_id); *buffer = memory->MutableBuffer(memory_type, memory_type_id); RETURN_IF_ERROR(RemoveAllData()); RETURN_IF_ERROR(SetData(memory)); if (UseSingleBuffer()) { RETURN_IF_ERROR(OtherState()->RemoveAllData()); RETURN_IF_ERROR(OtherState()->SetData(memory)); } } return Status::Success; } Status SequenceStates::Initialize( const std::unordered_map< std::string, const inference::ModelSequenceBatching_State&>& state_output_config_map, const size_t max_batch_size, const std::unordered_map& initial_state, TRITONSERVER_InstanceGroupKind kind, int32_t device_id, const std::map& cuda_virtual_address_size) { input_states_.clear(); output_states_.clear(); for (auto& state : state_output_config_map) { auto& state_config = state.second; bool use_single_buffer = state_config.use_same_buffer_for_input_output(); bool use_growable_memory = state_config.use_growable_memory(); if (use_growable_memory) { if (!use_single_buffer) { return Status( Status::Code::INVALID_ARG, std::string("In order to 'use_growable_memory', you must also " "enable 'use_single_buffer' for state input name '") + state_config.input_name() + ("'.")); } if (kind != TRITONSERVER_INSTANCEGROUPKIND_GPU) { return Status( Status::Code::INVALID_ARG, std::string("'use_growable_memory' can only work with KIND_GPU " "model instances right now. Please either change the " "instance type to KIND_GPU or set " "'use_growable_memory' to false for state input '") + state_config.input_name() + ("'.")); } } std::vector dims; if (max_batch_size != 0) { dims.push_back(1); } // Convert the variable dimensions to 1 for the first request. for (auto& dim : state_config.dims()) { if (dim == -1) { dims.push_back(1); } else { dims.push_back(dim); } } std::shared_ptr data; auto initial_state_it = initial_state.find(state_config.input_name()); if (initial_state_it != initial_state.end()) { if (use_growable_memory) { std::unique_ptr growable_memory; RETURN_IF_ERROR(GrowableMemory::Create( growable_memory, initial_state_it->second.data_->TotalByteSize(), TRITONSERVER_MEMORY_GPU, device_id, cuda_virtual_address_size.at(device_id))); data = std::move(growable_memory); #ifdef TRITON_ENABLE_GPU TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; char* dst_buffer = data->MutableBuffer(&memory_type, &memory_type_id); char* initial_state_buffer = initial_state_it->second.data_->MutableBuffer( &memory_type, &memory_type_id); RETURN_IF_CUDA_ERR( cudaMemcpy( dst_buffer, initial_state_buffer, initial_state_it->second.data_->TotalByteSize(), cudaMemcpyHostToDevice), std::string("failed to copy initial state to GPU:")); #endif } else { data = std::make_shared( initial_state_it->second.data_->TotalByteSize(), TRITONSERVER_MEMORY_CPU, 0); TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; char* dst_buffer = data->MutableBuffer(&memory_type, &memory_type_id); char* initial_state_buffer = initial_state_it->second.data_->MutableBuffer( &memory_type, &memory_type_id); memcpy( dst_buffer, initial_state_buffer, initial_state_it->second.data_->TotalByteSize()); } } else { size_t state_size; if (state.second.data_type() == inference::DataType::TYPE_STRING) { auto element_count = triton::common::GetElementCount(dims); // Total number of bytes required is equal to the element count // multiplied by 4. state_size = 4 * element_count; } else { state_size = triton::common::GetByteSize(state.second.data_type(), dims); } if (use_growable_memory) { std::unique_ptr growable_memory; RETURN_IF_ERROR(GrowableMemory::Create( growable_memory, state_size, TRITONSERVER_MEMORY_GPU, device_id, cuda_virtual_address_size.at(device_id))); data = std::move(growable_memory); } else { data = std::make_shared( state_size, TRITONSERVER_MEMORY_CPU, 0); } } const auto& input_pair = input_states_.emplace( std::piecewise_construct, std::forward_as_tuple(state_config.input_name()), std::forward_as_tuple(new SequenceState( state_config.input_name(), state.second.data_type(), dims, use_single_buffer, use_growable_memory))); if (!input_pair.second) { LOG_WARNING << "Detected duplicate 'input_name' in the state configuration: '" << state_config.input_name() << ".' This state configuration will be ignored."; continue; } auto& input_tensor = input_pair.first->second; RETURN_IF_ERROR(input_tensor->SetData(data)); if (input_tensor->DType() == inference::DataType::TYPE_STRING) { RETURN_IF_ERROR(input_tensor->SetStringDataToZero()); } const auto& output_pair = output_states_.emplace( std::piecewise_construct, std::forward_as_tuple(state_config.output_name()), std::forward_as_tuple(new SequenceState( state_config.output_name(), state.second.data_type(), dims, use_single_buffer, use_growable_memory))); if (!output_pair.second) { // Remove the corresponding state from the input_states_map input_states_.erase(state_config.input_name()); LOG_WARNING << "Detected duplicate 'output_name' in the state " "configuration: '" << state_config.output_name() << "'. This state configuration will be ignored."; continue; } auto& output_tensor = output_pair.first->second; if (use_single_buffer) { output_tensor->SetData(data); } output_tensor->SetOtherState(input_tensor.get()); input_tensor->SetOtherState(output_tensor.get()); } return Status::Success; } Status SequenceStates::OutputState( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count, SequenceState** output_state) { const auto& output_state_itr = output_states_.find(name); // If the state name is not valid return an error. if (output_state_itr == output_states_.end()) { return Status( Status::Code::INVALID_ARG, "state '" + name + "' is not a valid state name."); } std::vector new_shape(shape, shape + dim_count); *output_states_[name]->MutableDType() = datatype; *output_states_[name]->MutableShape() = new_shape; auto& output_state_r = output_states_[name]; size_t iter_advance = std::distance(output_states_.begin(), output_states_.find(name)); // Find the input state corresponding to this output state. auto input_states_itr = input_states_.begin(); std::advance(input_states_itr, iter_advance); auto& input_state_r = input_states_[input_states_itr->first]; bool use_single_buffer = output_states_[name]->UseSingleBuffer(); if (output_state != nullptr) { *output_state = output_states_[name].get(); } output_state_r->SetStateUpdateCallback( [&output_state_r, &input_state_r, use_single_buffer]() { // Swap the internal memory if the size of the input and output state is // equal if (!use_single_buffer) { if (output_state_r->Data()->TotalByteSize() == input_state_r->Data()->TotalByteSize()) { std::shared_ptr temp_memory = input_state_r->Data(); RETURN_IF_ERROR(input_state_r->RemoveAllData()); RETURN_IF_ERROR(input_state_r->SetData(output_state_r->Data())); RETURN_IF_ERROR(output_state_r->RemoveAllData()); RETURN_IF_ERROR(output_state_r->SetData(temp_memory)); } else { // If the size of output state is different from the input state, // allocate a new memory for the input state with the same size as // output state. TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; const std::shared_ptr& input_memory = reinterpret_cast&>( input_state_r->Data()); input_memory->MutableBuffer(&memory_type, &memory_type_id); std::shared_ptr memory = std::make_shared( output_state_r->Data()->TotalByteSize(), memory_type, memory_type_id); RETURN_IF_ERROR(input_state_r->RemoveAllData()); RETURN_IF_ERROR(input_state_r->SetData(output_state_r->Data())); RETURN_IF_ERROR(output_state_r->RemoveAllData()); RETURN_IF_ERROR(output_state_r->SetData(memory)); } // Update the shape and data type of the output state if it doesn't // match the input state. if (input_state_r->Shape() != output_state_r->Shape()) { *input_state_r->MutableShape() = output_state_r->Shape(); } if (input_state_r->DType() != output_state_r->DType()) { *input_state_r->MutableDType() = output_state_r->DType(); } } return Status::Success; }); return Status::Success; } Status SequenceStates::OutputState( const std::string& name, const inference::DataType datatype, const std::vector& shape, SequenceState** output_state) { return OutputState(name, datatype, shape.data(), shape.size(), output_state); } std::shared_ptr SequenceStates::CopyAsNull(const std::shared_ptr& from) { std::shared_ptr lsequence_states; if (from != nullptr) { lsequence_states.reset(new SequenceStates); for (auto& from_input_state : from->InputStates()) { auto& from_input_state_tensor = from_input_state.second; const auto& input_pair = lsequence_states->input_states_.emplace( std::piecewise_construct, std::forward_as_tuple(from_input_state_tensor->Name()), std::forward_as_tuple(new SequenceState( from_input_state_tensor->Name(), from_input_state_tensor->DType(), from_input_state_tensor->Shape(), false /* use_single_buffer */, false /* use_growable_memory */))); auto& input_tensor = input_pair.first->second; std::shared_ptr data; if (from_input_state_tensor->DType() == inference::DataType::TYPE_STRING) { // Use all-zero input states for null requests. auto element_count = triton::common::GetElementCount(from_input_state_tensor->Shape()); auto state_size = 4 * element_count; data = std::make_shared( state_size, TRITONSERVER_MEMORY_CPU, 0); } else { data = std::make_shared( from_input_state_tensor->Data()->TotalByteSize(), TRITONSERVER_MEMORY_CPU, 0); } input_tensor->SetData(data); if (input_tensor->DType() == inference::DataType::TYPE_STRING) { input_tensor->SetStringDataToZero(); } } for (auto& from_output_state : from->OutputStates()) { auto& from_output_state_tensor = from_output_state.second; lsequence_states->output_states_.emplace( std::piecewise_construct, std::forward_as_tuple(from_output_state.first), std::forward_as_tuple(new SequenceState( from_output_state_tensor->Name(), from_output_state_tensor->DType(), from_output_state_tensor->Shape(), false /* reused_buffer */, false /* use_growable_memory */))); } } return lsequence_states; } }} // namespace triton::core kytensor-2.49.0.7/core/src/sequence_state.h000066400000000000000000000155461527052371200205500ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "memory.h" #include "status.h" #include "triton/common/model_config.h" #pragma once namespace triton { namespace core { // // Sequence state tensors. // class SequenceState { public: SequenceState(); SequenceState( const std::string& name, const inference::DataType datatype, const std::vector& shape, bool use_single_buffer, bool use_growable_memory); SequenceState( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count, bool use_single_buffer, bool use_growable_memory); // The name of the state tensor. const std::string& Name() const { return name_; } // Data type of the state tensor. inference::DataType DType() const { return datatype_; } // Mutable data type of the state tensor. inference::DataType* MutableDType() { return &datatype_; } // The shape of the state tensor after normalization. const std::vector& Shape() const { return shape_; } std::vector* MutableShape() { return &shape_; } // The data for this shape. std::shared_ptr& Data() { return data_; } // A boolean indicating whether a single buffer is used for both input or // output state or not. bool UseSingleBuffer() { return use_single_buffer_; } // Use growable memory or not bool UseGrowableMemory() { return use_growable_memory_; } // Set pointer to the other sequence state void SetOtherState(SequenceState* other) { other_state_ = other; } // Resize the sequence state buffer Status ResizeOrReallocate( void** buffer, const uint64_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); // For input state this would point to the output state. For output state this // would point to the input state. SequenceState* OtherState() { return other_state_; } // Set the data for this shape. Error if state already has some // data. Status SetData(const std::shared_ptr& data); // Sets state tensors that have type string to zero Status SetStringDataToZero(); // Remove all existing data for the state. Status RemoveAllData(); // Set the state update callback. void SetStateUpdateCallback(std::function&& state_update_cb) { state_update_cb_ = std::move(state_update_cb); } // Call the state update callback. This function will be called when // TRITONBACKEND_StateUpdate is called. Status Update() { return state_update_cb_(); } private: DISALLOW_COPY_AND_ASSIGN(SequenceState); std::string name_; inference::DataType datatype_; std::vector shape_; std::vector batch_dim_; std::shared_ptr data_; SequenceState* other_state_; bool use_single_buffer_; bool use_growable_memory_; std::function state_update_cb_ = []() { // By default calling the TRITONBACKEND_StateUpdate will return an error. return Status( Status::Code::INVALID_ARG, "TRITONBACKEND_StateUpdate called when sequence batching is disabled " "or the 'states' section of the model configuration is empty."); }; }; class SequenceStates { public: struct InitialStateData { InitialStateData(const std::string& state_init_name) : state_init_name_(state_init_name) { } std::string state_init_name_; std::vector shape_{}; std::shared_ptr data_; }; // Initialize the state tensors according to the state model configuration. // Will use a default value of 1 for the variable dimensions in the state // tensor configuration. Status Initialize( const std::unordered_map< std::string, const inference::ModelSequenceBatching_State&>& state_output_config_map, const size_t max_batch_size, const std::unordered_map& initial_state, TRITONSERVER_InstanceGroupKind kind, int32_t device_id, const std::map& cuda_virtual_address_size); // Get a buffer holding the output state. Status OutputState( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count, SequenceState** output_state); Status OutputState( const std::string& name, const inference::DataType datatype, const std::vector& shape, SequenceState** output_state); // Create a copy of the 'from' sequence states for NULL requests. static std::shared_ptr CopyAsNull( const std::shared_ptr& from); const std::map>& InputStates() { return input_states_; } std::map>& OutputStates() { return output_states_; } void SetNullSequenceStates(std::shared_ptr sequence_states) { null_sequence_states_ = sequence_states; is_null_request_ = true; } const std::shared_ptr& NullSequenceStates() { return null_sequence_states_; } bool IsNullRequest() { return is_null_request_; } private: std::map> input_states_; std::map> output_states_; std::shared_ptr null_sequence_states_; bool is_null_request_ = false; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/server.cc000066400000000000000000000535471527052371200172070ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "server.h" #include #include #include #include #include #include #include #include #include "backend_manager.h" #include "constants.h" #include "cuda_utils.h" #include "model.h" #include "model_config.pb.h" #include "model_config_utils.h" #include "pinned_memory_manager.h" #include "repo_agent.h" #include "triton/common/async_work_queue.h" #include "triton/common/logging.h" #include "triton/common/model_config.h" #include "triton/common/table_printer.h" #ifdef TRITON_ENABLE_GPU #include "cuda_block_manager.h" #include "cuda_memory_manager.h" #endif // TRITON_ENABLE_GPU namespace triton { namespace core { namespace { // Scoped increment / decrement of atomic class ScopedAtomicIncrement { public: explicit ScopedAtomicIncrement(std::atomic& counter) : counter_(counter) { counter_++; } ~ScopedAtomicIncrement() { counter_--; } private: std::atomic& counter_; }; } // namespace // // InferenceServer // InferenceServer::InferenceServer() : version_(TRITON_VERSION), ready_state_(ServerReadyState::SERVER_INVALID) { id_ = "triton"; extensions_.push_back("classification"); extensions_.push_back("sequence"); extensions_.push_back("model_repository"); extensions_.push_back("model_repository(unload_dependents)"); extensions_.push_back("schedule_policy"); extensions_.push_back("model_configuration"); extensions_.push_back("system_shared_memory"); extensions_.push_back("cuda_shared_memory"); extensions_.push_back("binary_tensor_data"); extensions_.push_back("parameters"); #ifdef TRITON_ENABLE_STATS extensions_.push_back("statistics"); #endif // TRITON_ENABLE_STATS #ifdef TRITON_ENABLE_TRACING extensions_.push_back("trace"); #endif // TRITON_ENABLE_TRACING #ifdef TRITON_ENABLE_LOGGING extensions_.push_back("logging"); #endif // TRITON_ENABLE_LOGGING strict_model_config_ = true; strict_readiness_ = true; exit_timeout_secs_ = 30; pinned_memory_pool_size_ = 1 << 28; buffer_manager_thread_count_ = 0; model_load_thread_count_ = 4; model_load_retry_count_ = 0; enable_model_namespacing_ = false; #ifdef TRITON_ENABLE_GPU min_supported_compute_capability_ = TRITON_MIN_COMPUTE_CAPABILITY; #else min_supported_compute_capability_ = 0.0; #endif // TRITON_ENABLE_GPU inflight_request_counter_ = 0; } Status InferenceServer::Init() { Status status; ready_state_ = ServerReadyState::SERVER_INITIALIZING; if (model_repository_paths_.empty()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return Status( Status::Code::INVALID_ARG, "--model-repository must be specified"); } // RepoAgentManager if (repoagent_dir_.empty()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return Status( Status::Code::INVALID_ARG, "--repoagent-directory can not be empty"); } status = TritonRepoAgentManager::SetGlobalSearchPath(repoagent_dir_); if (!status.IsOk()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return status; } // BackendManager status = TritonBackendManager::Create(&backend_manager_); if (!status.IsOk()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return status; } // CacheManager status = TritonCacheManager::Create(&cache_manager_, cache_dir_); if (!status.IsOk()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return status; } // Only a single global cache is supported at this time. if (cache_config_map_.size() > 1) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return Status( Status::Code::INVALID_ARG, "found multiple cache configurations, but only a single cache is " "currently supported"); } // Initialize each cache with its respective config for (const auto& iter : cache_config_map_) { const auto& name = iter.first; const auto& config = iter.second; std::shared_ptr cache; status = cache_manager_->CreateCache(name, config, &cache); if (!status.IsOk()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return status; } } if (buffer_manager_thread_count_ > 0) { status = CommonErrorToStatus(triton::common::AsyncWorkQueue::Initialize( buffer_manager_thread_count_)); if (!status.IsOk()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return status; } } std::unique_ptr local_rate_limiter; bool ignore_resources_and_priority = (rate_limit_mode_ == RateLimitMode::RL_OFF); status = RateLimiter::Create( ignore_resources_and_priority, rate_limit_resource_map_, &local_rate_limiter); rate_limiter_ = std::move(local_rate_limiter); if (!status.IsOk()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return status; } PinnedMemoryManager::Options options(pinned_memory_pool_size_); status = PinnedMemoryManager::Create(options); if (!status.IsOk()) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; return status; } #ifdef TRITON_ENABLE_GPU // Set the default CUDA memory pool size for GPUs where it is not // set explicitly. std::set supported_gpus; if (GetSupportedGPUs(&supported_gpus, min_supported_compute_capability_) .IsOk()) { for (const auto gpu : supported_gpus) { if (cuda_memory_pool_size_.find(gpu) == cuda_memory_pool_size_.end()) { cuda_memory_pool_size_[gpu] = 1 << 26; } if (cuda_virtual_address_space_size_.find(gpu) == cuda_virtual_address_space_size_.end()) { cuda_virtual_address_space_size_[gpu] = 1 << 30; } } } CudaMemoryManager::Options cuda_options( min_supported_compute_capability_, cuda_memory_pool_size_); status = CudaMemoryManager::Create(cuda_options); // If CUDA memory manager can't be created, just log error as the // server can still function properly if (!status.IsOk()) { LOG_ERROR << status.Message(); } status = CudaBlockManager::Create(min_supported_compute_capability_); // If CUDA memory manager can't be created, just log error as the // server can still function properly if (!status.IsOk()) { LOG_ERROR << status.Message(); } #endif // TRITON_ENABLE_GPU if (enable_peer_access_) { status = EnablePeerAccess(min_supported_compute_capability_); if (!status.IsOk()) { // failed to enable peer access is not critical, just inefficient. LOG_WARNING << status.Message(); } } // Create the model manager for the repository. Unless model control // is disabled, all models are eagerly loaded when the manager is created. bool polling_enabled = (model_control_mode_ == ModelControlMode::MODE_POLL); bool model_control_enabled = (model_control_mode_ == ModelControlMode::MODE_EXPLICIT); const ModelLifeCycleOptions life_cycle_options( min_supported_compute_capability_, backend_cmdline_config_map_, host_policy_map_, model_load_thread_count_, model_load_retry_count_); status = ModelRepositoryManager::Create( this, version_, model_repository_paths_, startup_models_, strict_model_config_, model_config_name_, polling_enabled, model_control_enabled, life_cycle_options, enable_model_namespacing_, &model_repository_manager_); if (!status.IsOk()) { if (model_repository_manager_ == nullptr) { ready_state_ = ServerReadyState::SERVER_FAILED_TO_INITIALIZE; } else { // If error is returned while the manager is set, we assume the // failure is due to a model not loading correctly so we just // continue if not exiting on error. ready_state_ = ServerReadyState::SERVER_READY; PrintBackendAndModelSummary(); } } else { ready_state_ = ServerReadyState::SERVER_READY; PrintBackendAndModelSummary(); } return status; } InferenceServer::~InferenceServer() { PinnedMemoryManager::Reset(); #ifdef TRITON_ENABLE_GPU CudaMemoryManager::Reset(); CudaBlockManager::Reset(); #endif // TRITON_ENABLE_GPU } Status InferenceServer::Stop(const bool force) { if (!force && (ready_state_ != ServerReadyState::SERVER_READY)) { return Status::Success; } ready_state_ = ServerReadyState::SERVER_EXITING; if (model_repository_manager_ == nullptr) { LOG_INFO << "No server context available. Exiting immediately."; return Status::Success; } else { LOG_INFO << "Waiting for in-flight requests to complete."; } Status status = model_repository_manager_->StopAllModels(); if (!status.IsOk()) { LOG_ERROR << status.Message(); } // Wait for all in-flight non-inference requests to complete and all // loaded models to unload, or for the exit timeout to expire. uint32_t exit_timeout_iters = exit_timeout_secs_; bool unloading_model = false; while (true) { if (!unloading_model) { // Check if all in-flight inference requests / sequences are completed const auto& inflight_status = model_repository_manager_->InflightStatus(); LOG_INFO << "Timeout " << exit_timeout_iters << ": Found " << inflight_status.size() << " model versions that have in-flight inferences"; for (const auto& inflight : inflight_status) { LOG_INFO << "Model '" << std::get<0>(inflight) << "' " << "(version " << std::get<1>(inflight) << ") has " << std::get<2>(inflight) << " in-flight inferences"; } if (inflight_status.size() == 0) { status = model_repository_manager_->UnloadAllModels(); if (!status.IsOk()) { LOG_WARNING << status.Message(); } else { unloading_model = true; LOG_INFO << "All models are stopped, unloading models"; continue; } } } else { const auto& live_models = model_repository_manager_->LiveModelStates(); size_t bg_models_size = model_repository_manager_->BackgroundModelsSize(); size_t num_models = live_models.size() + bg_models_size; LOG_INFO << "Timeout " << exit_timeout_iters << ": Found " << num_models << " live models and " << inflight_request_counter_ << " in-flight non-inference requests"; if (LOG_VERBOSE_IS_ON(1)) { for (const auto& m : live_models) { for (const auto& v : m.second) { LOG_VERBOSE(1) << m.first << " v" << v.first << ": " << ModelReadyStateString(v.second.first); } } } if ((num_models == 0) && (inflight_request_counter_ == 0)) { return Status::Success; } } if (exit_timeout_iters <= 0) { break; } exit_timeout_iters--; std::this_thread::sleep_for(std::chrono::seconds(1)); } return Status( Status::Code::INTERNAL, "Exit timeout expired. Exiting immediately."); } Status InferenceServer::PollModelRepository() { LOG_VERBOSE(1) << "Polling model repository"; // Look for changes and update the loaded model configurations // appropriately. if (ready_state_ == ServerReadyState::SERVER_READY) { ScopedAtomicIncrement inflight(inflight_request_counter_); RETURN_IF_ERROR(model_repository_manager_->PollAndUpdate()); } return Status::Success; } Status InferenceServer::IsLive(bool* live) { *live = false; if (ready_state_ == ServerReadyState::SERVER_EXITING) { return Status(Status::Code::UNAVAILABLE, "Server exiting"); } ScopedAtomicIncrement inflight(inflight_request_counter_); // Server is considered live if it can respond to this health // request and it was able to initialize. *live = ((ready_state_ != ServerReadyState::SERVER_INVALID) && (ready_state_ != ServerReadyState::SERVER_INITIALIZING) && (ready_state_ != ServerReadyState::SERVER_FAILED_TO_INITIALIZE)); return Status::Success; } Status InferenceServer::IsReady(bool* ready) { *ready = false; if (ready_state_ == ServerReadyState::SERVER_EXITING) { return Status(Status::Code::UNAVAILABLE, "Server exiting"); } ScopedAtomicIncrement inflight(inflight_request_counter_); // Server is considered ready if it is in the ready state. // Additionally can report ready only when all models are ready. *ready = (ready_state_ == ServerReadyState::SERVER_READY); if (*ready && strict_readiness_) { // Strict readiness... get the model status and make sure all // models are ready. const auto model_versions = model_repository_manager_->ModelStates(); for (const auto& mv : model_versions) { // If a model status is present but no version status, // the model is not ready as there is no proper version to be served if (mv.second.size() == 0) { *ready = false; goto strict_done; } for (const auto& vs : mv.second) { // Okay if model is not ready due to unload if ((vs.second.first != ModelReadyState::READY) && (vs.second.second != "unloaded")) { *ready = false; goto strict_done; } } } strict_done:; } return Status::Success; } Status InferenceServer::ModelIsReady( const std::string& model_name, const int64_t model_version, bool* ready) { *ready = false; if (ready_state_ != ServerReadyState::SERVER_READY) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } ScopedAtomicIncrement inflight(inflight_request_counter_); std::shared_ptr model; if (GetModel(model_name, model_version, &model).IsOk()) { ModelReadyState state; if (model_repository_manager_ ->ModelState(model_name, model->Version(), &state) .IsOk()) { *ready = (state == ModelReadyState::READY); } } return Status::Success; } Status InferenceServer::ModelReadyVersions( const std::string& model_name, std::vector* versions) { if (ready_state_ != ServerReadyState::SERVER_READY) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } ScopedAtomicIncrement inflight(inflight_request_counter_); const auto version_states = model_repository_manager_->VersionStates(model_name); for (const auto& pr : version_states) { if (pr.second.first == ModelReadyState::READY) { versions->push_back(pr.first); } } return Status::Success; } Status InferenceServer::ModelReadyVersions( std::map>* ready_model_versions) { if (ready_state_ != ServerReadyState::SERVER_READY) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } ScopedAtomicIncrement inflight(inflight_request_counter_); const auto model_versions = model_repository_manager_->LiveModelStates(true /* strict_readiness */); ready_model_versions->clear(); std::vector versions; for (const auto& mv_pair : model_versions) { for (const auto& vs_pair : mv_pair.second) { versions.emplace_back(vs_pair.first); } ready_model_versions->emplace(mv_pair.first.str(), std::move(versions)); } return Status::Success; } Status InferenceServer::RepositoryIndex( const bool ready_only, std::vector* index) { if (ready_state_ != ServerReadyState::SERVER_READY) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } ScopedAtomicIncrement inflight(inflight_request_counter_); return model_repository_manager_->RepositoryIndex(ready_only, index); } Status InferenceServer::InferAsync(std::unique_ptr& request) { // Allow inference request while server exiting to provide graceful // completion of inference sequence that spans multiple requests. if ((ready_state_ != ServerReadyState::SERVER_READY) && (ready_state_ != ServerReadyState::SERVER_EXITING)) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } #ifdef TRITON_ENABLE_STATS request->CaptureRequestStartNs(); INFER_TRACE_ACTIVITY( request->TraceProxy(), TRITONSERVER_TRACE_REQUEST_START, request->RequestStartNs()); #endif // TRITON_ENABLE_STATS return InferenceRequest::Run(request); } Status InferenceServer::LoadModel( const std::unordered_map< std::string, std::vector>& models) { if (ready_state_ != ServerReadyState::SERVER_READY) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } ScopedAtomicIncrement inflight(inflight_request_counter_); auto action_type = ActionType::LOAD; return model_repository_manager_->LoadUnloadModel( models, action_type, false /* unload_dependents */); } Status InferenceServer::UnloadModel( const std::string& model_name, const bool unload_dependents) { if (ready_state_ != ServerReadyState::SERVER_READY) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } ScopedAtomicIncrement inflight(inflight_request_counter_); auto action_type = ActionType::UNLOAD; return model_repository_manager_->LoadUnloadModel( {{model_name, {}}}, action_type, unload_dependents); } Status InferenceServer::PrintBackendAndModelSummary() { // Repository Agents Summary std::vector repoagent_headers; repoagent_headers.emplace_back("Repository Agent"); repoagent_headers.emplace_back("Path"); triton::common::TablePrinter repoagents_table(repoagent_headers); std::unique_ptr> repoagent_state; RETURN_IF_ERROR(TritonRepoAgentManager::AgentState(&repoagent_state)); for (const auto& repoagent_pair : *repoagent_state) { std::vector repoagent_record; repoagent_record.emplace_back(repoagent_pair.first); repoagent_record.emplace_back(repoagent_pair.second); repoagents_table.InsertRow(repoagent_record); } LOG_TABLE_INFO(repoagents_table); // Backends Summary std::vector backend_headers; backend_headers.emplace_back("Backend"); backend_headers.emplace_back("Path"); backend_headers.emplace_back("Config"); triton::common::TablePrinter backends_table(backend_headers); std::unique_ptr>> backend_state; RETURN_IF_ERROR(backend_manager_->BackendState(&backend_state)); for (const auto& backend_pair : *backend_state) { std::vector backend_record; // Backend Name backend_record.emplace_back(backend_pair.first); // Backend config and lib path for (const auto& backend_field : backend_pair.second) { backend_record.emplace_back(backend_field); } backends_table.InsertRow(backend_record); } LOG_TABLE_INFO(backends_table); // Models Summary auto model_states = model_repository_manager_->ModelStates(); std::vector model_headers; model_headers.emplace_back("Model"); model_headers.emplace_back("Version"); model_headers.emplace_back("Status"); triton::common::TablePrinter models_table(model_headers); for (const auto& model_state : model_states) { auto model_version_map = model_state.second; ModelIdentifier model_id = model_state.first; // If model_version_map size is zero, no version is found for this model if (model_version_map.size() == 0) { std::vector model_record; model_record.emplace_back(model_id.str()); model_record.emplace_back("-"); model_record.emplace_back("Not loaded: No model version was found"); models_table.InsertRow(model_record); } else { for (const auto& model_map : model_version_map) { std::vector model_record; std::string model_version = std::to_string(model_map.first); auto model_status_pair = model_map.second; std::string model_status = ModelReadyStateString(model_status_pair.first); if (model_status_pair.second != "") { model_status += ": " + model_status_pair.second; } model_record.emplace_back(model_id.str()); model_record.emplace_back(model_version); model_record.emplace_back(model_status); models_table.InsertRow(model_record); } } } LOG_TABLE_INFO(models_table); return Status::Success; } Status InferenceServer::RegisterModelRepository( const std::string& repository, const std::unordered_map& model_mapping) { return model_repository_manager_->RegisterModelRepository( repository, model_mapping); } Status InferenceServer::UnregisterModelRepository(const std::string& repository) { return model_repository_manager_->UnregisterModelRepository(repository); } }} // namespace triton::core kytensor-2.49.0.7/core/src/server.h000066400000000000000000000314611527052371200170400ustar00rootroot00000000000000// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include "backend_manager.h" #include "cache_manager.h" #include "infer_parameter.h" #include "model_config.pb.h" #include "model_repository_manager/model_repository_manager.h" #include "rate_limiter.h" #include "status.h" #include "triton/common/model_config.h" namespace triton { namespace core { // Maps cache name -> json config string using CacheConfigMap = std::unordered_map; class Model; class InferenceRequest; enum class ModelControlMode { MODE_NONE, MODE_POLL, MODE_EXPLICIT }; enum class RateLimitMode { RL_EXEC_COUNT, RL_OFF }; // Readiness status for the inference server. enum class ServerReadyState { // The server is in an invalid state and will likely not response // correctly to any requests. SERVER_INVALID, // The server is initializing. SERVER_INITIALIZING, // The server is ready and accepting requests. SERVER_READY, // The server is exiting and will not respond to requests. SERVER_EXITING, // The server did not initialize correctly. SERVER_FAILED_TO_INITIALIZE }; // Inference server information. class InferenceServer { public: // Construct an inference server. InferenceServer(); ~InferenceServer(); // Initialize the server. Return true on success, false otherwise. Status Init(); // Stop the server. Return true if all models are unloaded, false // if exit timeout occurs. If 'force' is true attempt to stop the // server even if it is not in a ready state. Status Stop(const bool force = false); // Check the model repository for changes and update server state // based on those changes. Status PollModelRepository(); // Server health Status IsLive(bool* live); Status IsReady(bool* ready); // Model health Status ModelIsReady( const std::string& model_name, const int64_t model_version, bool* ready); // Return the ready versions of specific model Status ModelReadyVersions( const std::string& model_name, std::vector* versions); // Return the ready versions of all models Status ModelReadyVersions( std::map>* model_versions); /// Get the index of all models in all repositories. /// \param ready_only If true return only index of models that are ready. /// \param index Returns the index. /// \return error status. Status RepositoryIndex( const bool ready_only, std::vector* index); // Inference. If Status::Success is returned then this function has // taken ownership of the request object and so 'request' will be // nullptr. If non-success is returned then the caller still retains // ownership of 'request'. Status InferAsync(std::unique_ptr& request); // Load the corresponding model. Reload the model if it has been loaded. Status LoadModel( const std::unordered_map< std::string, std::vector>& models); // Unload the corresponding model. Status UnloadModel( const std::string& model_name, const bool unload_dependents); // Print backends and models summary Status PrintBackendAndModelSummary(); // Register model repository path and associated mappings Status RegisterModelRepository( const std::string& repository, const std::unordered_map& model_mapping); // Unregister model repository path. Status UnregisterModelRepository(const std::string& repository); // Return the server version. const std::string& Version() const { return version_; } // Return the server extensions. const std::vector& Extensions() const { return extensions_; } // Get / set the ID of the server. const std::string& Id() const { return id_; } void SetId(const std::string& id) { id_ = id; } // Get / set the model repository path const std::set& ModelRepositoryPaths() const { return model_repository_paths_; } void SetModelRepositoryPaths(const std::set& p) { model_repository_paths_ = p; } // Get / set model control mode. ModelControlMode GetModelControlMode() const { return model_control_mode_; } void SetModelControlMode(ModelControlMode m) { model_control_mode_ = m; } // Get / set the startup models const std::set& StartupModels() const { return startup_models_; } void SetStartupModels(const std::set& m) { startup_models_ = m; } // Get / set strict model configuration enable. bool StrictModelConfigEnabled() const { return strict_model_config_; } void SetStrictModelConfigEnabled(bool e) { strict_model_config_ = e; } // Get / set custom model configuration file name. std::string ModelConfigName() const { return model_config_name_; } void SetModelConfigName(const std::string& name) { model_config_name_ = name; } // Get / set rate limiter mode. RateLimitMode RateLimiterMode() const { return rate_limit_mode_; } void SetRateLimiterMode(RateLimitMode m) { rate_limit_mode_ = m; } // Get / set rate limit resource counts const RateLimiter::ResourceMap& RateLimiterResources() const { return rate_limit_resource_map_; } void SetRateLimiterResources(const RateLimiter::ResourceMap& rm) { rate_limit_resource_map_ = rm; } // Get / set the pinned memory pool byte size. int64_t PinnedMemoryPoolByteSize() const { return pinned_memory_pool_size_; } void SetPinnedMemoryPoolByteSize(int64_t s) { pinned_memory_pool_size_ = std::max((int64_t)0, s); } // Get / set whether response cache will be enabled server-wide. // NOTE: Models still need caching enabled in individual model configs. bool ResponseCacheEnabled() { // Only return true if cache was enabled, and has been initialized return response_cache_enabled_ && CacheManager() && CacheManager()->Cache(); } void SetResponseCacheEnabled(bool e) { response_cache_enabled_ = e; } void SetCacheConfig(CacheConfigMap cfg) { cache_config_map_ = cfg; } std::string CacheDir() const { return cache_dir_; } void SetCacheDir(std::string dir) { cache_dir_ = dir; } // Get / set CUDA memory pool size const std::map& CudaMemoryPoolByteSize() const { return cuda_memory_pool_size_; } void SetCudaMemoryPoolByteSize(const std::map& s) { cuda_memory_pool_size_ = s; } // Get / set CUDA virtual address space size const std::map& CudaVirtualAddressSpaceSize() const { return cuda_virtual_address_space_size_; } void SetCudaVirtualAddressSpaceSize(const std::map& s) { cuda_virtual_address_space_size_ = s; } // Get / set the minimum support CUDA compute capability. double MinSupportedComputeCapability() const { return min_supported_compute_capability_; } void SetMinSupportedComputeCapability(double c) { min_supported_compute_capability_ = c; } // Get / set strict readiness enable. bool StrictReadinessEnabled() const { return strict_readiness_; } void SetStrictReadinessEnabled(bool e) { strict_readiness_ = e; } // Get / set the server exit timeout, in seconds. int32_t ExitTimeoutSeconds() const { return exit_timeout_secs_; } void SetExitTimeoutSeconds(int32_t s) { exit_timeout_secs_ = std::max(0, s); } void SetBufferManagerThreadCount(unsigned int c) { buffer_manager_thread_count_ = c; } void SetModelLoadThreadCount(unsigned int c) { model_load_thread_count_ = c; } void SetModelLoadRetryCount(unsigned int c) { model_load_retry_count_ = c; } void SetModelNamespacingEnabled(const bool e) { enable_model_namespacing_ = e; } void SetEnablePeerAccess(const bool e) { enable_peer_access_ = e; } // Set a backend command-line configuration void SetBackendCmdlineConfig( const triton::common::BackendCmdlineConfigMap& bc) { backend_cmdline_config_map_ = bc; } void SetHostPolicyCmdlineConfig( const triton::common::HostPolicyCmdlineConfigMap& hp) { host_policy_map_ = hp; } void SetRepoAgentDir(const std::string& d) { repoagent_dir_ = d; } // Return the requested model object. Status GetModel( const std::string& model_name, const int64_t model_version, std::shared_ptr* model) { // Allow model retrieval while server exiting to provide graceful // completion of inference sequence that spans multiple requests. if ((ready_state_ != ServerReadyState::SERVER_READY) && (ready_state_ != ServerReadyState::SERVER_EXITING)) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } return model_repository_manager_->GetModel( model_name, model_version, model); } // Return the requested model object. Status GetModel( const ModelIdentifier& model_id, const int64_t model_version, std::shared_ptr* model) { // Allow model retrieval while server exiting to provide graceful // completion of inference sequence that spans multiple requests. if ((ready_state_ != ServerReadyState::SERVER_READY) && (ready_state_ != ServerReadyState::SERVER_EXITING)) { return Status(Status::Code::UNAVAILABLE, "Server not ready"); } return model_repository_manager_->GetModel(model_id, model_version, model); } // Get the Backend Manager const std::shared_ptr& BackendManager() { return backend_manager_; } // Return the pointer to RateLimiter object. std::shared_ptr GetRateLimiter() { return rate_limiter_; } // Get the Cache Manager const std::shared_ptr& CacheManager() { return cache_manager_; } private: const std::string version_; std::string id_; std::vector extensions_; std::set model_repository_paths_; std::set startup_models_; ModelControlMode model_control_mode_; bool strict_model_config_; bool strict_readiness_; std::string model_config_name_; uint32_t exit_timeout_secs_; uint32_t buffer_manager_thread_count_; uint32_t model_load_thread_count_; uint32_t model_load_retry_count_; bool enable_model_namespacing_; bool enable_peer_access_; uint64_t pinned_memory_pool_size_; bool response_cache_enabled_; CacheConfigMap cache_config_map_; std::string cache_dir_; std::map cuda_memory_pool_size_; std::map cuda_virtual_address_space_size_; double min_supported_compute_capability_; triton::common::BackendCmdlineConfigMap backend_cmdline_config_map_; triton::common::HostPolicyCmdlineConfigMap host_policy_map_; std::string repoagent_dir_; RateLimitMode rate_limit_mode_; RateLimiter::ResourceMap rate_limit_resource_map_; // Current state of the inference server. ServerReadyState ready_state_; // Number of in-flight, non-inference requests. During shutdown we // attempt to wait for all in-flight non-inference requests to // complete before exiting (also wait for in-flight inference // requests but that is determined by model shared_ptr). std::atomic inflight_request_counter_; std::shared_ptr rate_limiter_; std::unique_ptr model_repository_manager_; std::shared_ptr backend_manager_; std::shared_ptr cache_manager_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/server_message.h000066400000000000000000000060001527052371200205330ustar00rootroot00000000000000// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "status.h" #define TRITONJSON_STATUSTYPE triton::core::Status #define TRITONJSON_STATUSRETURN(M) \ return triton::core::Status(triton::core::Status::Code::INTERNAL, (M)) #define TRITONJSON_STATUSSUCCESS triton::core::Status::Success #include "triton/common/triton_json.h" namespace triton { namespace core { // // Implementation for TRITONSERVER_Message. // class TritonServerMessage { public: TritonServerMessage(const triton::common::TritonJson::Value& msg) { json_buffer_.Clear(); msg.Write(&json_buffer_); base_ = json_buffer_.Base(); byte_size_ = json_buffer_.Size(); from_json_ = true; } TritonServerMessage(std::string&& msg) { str_buffer_ = std::move(msg); base_ = str_buffer_.data(); byte_size_ = str_buffer_.size(); from_json_ = false; } TritonServerMessage(const TritonServerMessage& rhs) { from_json_ = rhs.from_json_; if (from_json_) { json_buffer_ = rhs.json_buffer_; base_ = json_buffer_.Base(); byte_size_ = json_buffer_.Size(); } else { str_buffer_ = rhs.str_buffer_; base_ = str_buffer_.data(); byte_size_ = str_buffer_.size(); } } void Serialize(const char** base, size_t* byte_size) const { *base = base_; *byte_size = byte_size_; } private: bool from_json_; triton::common::TritonJson::WriteBuffer json_buffer_; std::string str_buffer_; const char* base_; size_t byte_size_; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/shared_library.cc000066400000000000000000000170031527052371200206560ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "shared_library.h" #include "filesystem/api.h" #include "mutex" #include "triton/common/logging.h" #ifdef TRITON_ENABLE_GPU #include #endif #ifdef _WIN32 // suppress the min and max definitions in Windef.h. #define NOMINMAX #include #else #include #endif namespace triton { namespace core { static std::mutex mu_; Status SharedLibrary::Acquire(std::unique_ptr* slib) { mu_.lock(); slib->reset(new SharedLibrary()); return Status::Success; } SharedLibrary::~SharedLibrary() { mu_.unlock(); } Status SharedLibrary::SetLibraryDirectory(const std::string& path) { #ifdef _WIN32 LOG_VERBOSE(1) << "SetLibraryDirectory: path = " << path; if (!SetDllDirectory(path.c_str())) { LPSTR err_buffer = nullptr; size_t size = FormatMessageA( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&err_buffer, 0, NULL); std::string errstr(err_buffer, size); LocalFree(err_buffer); return Status( Status::Code::NOT_FOUND, "unable to set dll path " + path + ": " + errstr); } #endif return Status::Success; } Status SharedLibrary::ResetLibraryDirectory() { #ifdef _WIN32 LOG_VERBOSE(1) << "ResetLibraryDirectory"; if (!SetDllDirectory(NULL)) { LPSTR err_buffer = nullptr; size_t size = FormatMessageA( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&err_buffer, 0, NULL); std::string errstr(err_buffer, size); LocalFree(err_buffer); return Status( Status::Code::NOT_FOUND, "unable to reset dll path: " + errstr); } #endif return Status::Success; } Status SharedLibrary::OpenLibraryHandle(const std::string& path, void** handle) { LOG_VERBOSE(1) << "OpenLibraryHandle: " << path; #ifdef TRITON_ENABLE_GPU // This call is to prevent a deadlock issue with dlopening backend shared // libraries and calling CUDA APIs in other threads. Since CUDA API also // dlopens some libraries and dlopen has an internal lock, it can create a // deadlock. Intentionally ignore the CUDA_ERROR (if any) for containers // running on a CPU. int device_count; cudaGetDeviceCount(&device_count); #endif #ifdef _WIN32 // Need to put shared library directory on the DLL path so that any // dependencies of the shared library are found const std::string library_dir = DirName(path); RETURN_IF_ERROR(SetLibraryDirectory(library_dir)); // HMODULE is typedef of void* // https://docs.microsoft.com/en-us/windows/win32/winprog/windows-data-types LOG_VERBOSE(1) << "OpenLibraryHandle: path = " << path; *handle = LoadLibrary(path.c_str()); // Remove the dll path added above... do this unconditionally before // check for failure in dll load. RETURN_IF_ERROR(ResetLibraryDirectory()); if (*handle == nullptr) { LPSTR err_buffer = nullptr; size_t size = FormatMessageA( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&err_buffer, 0, NULL); std::string errstr(err_buffer, size); LocalFree(err_buffer); return Status( Status::Code::NOT_FOUND, "unable to load shared library: " + errstr); } #else *handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); if (*handle == nullptr) { return Status( Status::Code::NOT_FOUND, "unable to load shared library: " + std::string(dlerror())); } #endif return Status::Success; } Status SharedLibrary::CloseLibraryHandle(void* handle) { if (handle != nullptr) { #ifdef _WIN32 if (FreeLibrary((HMODULE)handle) == 0) { LPSTR err_buffer = nullptr; size_t size = FormatMessageA( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&err_buffer, 0, NULL); std::string errstr(err_buffer, size); LocalFree(err_buffer); return Status( Status::Code::INTERNAL, "unable to unload shared library: " + errstr); } #else if (dlclose(handle) != 0) { return Status( Status::Code::INTERNAL, "unable to unload shared library: " + std::string(dlerror())); } #endif } return Status::Success; } Status SharedLibrary::GetEntrypoint( void* handle, const std::string& name, const bool optional, void** befn) { *befn = nullptr; #ifdef _WIN32 void* fn = GetProcAddress((HMODULE)handle, name.c_str()); if ((fn == nullptr) && !optional) { LPSTR err_buffer = nullptr; size_t size = FormatMessageA( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&err_buffer, 0, NULL); std::string errstr(err_buffer, size); LocalFree(err_buffer); return Status( Status::Code::NOT_FOUND, "unable to find '" + name + "' entrypoint in custom library: " + errstr); } #else dlerror(); void* fn = dlsym(handle, name.c_str()); const char* dlsym_error = dlerror(); if (dlsym_error != nullptr) { if (optional) { return Status::Success; } std::string errstr(dlsym_error); // need copy as dlclose overwrites return Status( Status::Code::NOT_FOUND, "unable to find required entrypoint '" + name + "' in shared library: " + errstr); } if (fn == nullptr) { if (optional) { return Status::Success; } return Status( Status::Code::NOT_FOUND, "unable to find required entrypoint '" + name + "' in shared library"); } #endif *befn = fn; return Status::Success; } }} // namespace triton::core kytensor-2.49.0.7/core/src/shared_library.h000066400000000000000000000056331527052371200205260ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "constants.h" #include "status.h" namespace triton { namespace core { // SharedLibrary // // Utility functions for shared libraries. Because some operations // require serialization, this object cannot be directly constructed // and must instead be accessed using Acquire(). class SharedLibrary { public: // Acquire a SharedLibrary object exclusively. Any other attempts to // concurrently acquire a SharedLibrary object will block. // object. Ownership is released by destroying the SharedLibrary // object. static Status Acquire(std::unique_ptr* slib); ~SharedLibrary(); // Configuration so that dependent libraries will be searched for in // 'path' during OpenLibraryHandle. Status SetLibraryDirectory(const std::string& path); // Reset any configuration done by SetLibraryDirectory. Status ResetLibraryDirectory(); // Open shared library and return generic handle. Status OpenLibraryHandle(const std::string& path, void** handle); // Close shared library. Status CloseLibraryHandle(void* handle); // Get a generic pointer for an entrypoint into a shared library. Status GetEntrypoint( void* handle, const std::string& name, const bool optional, void** befn); private: DISALLOW_COPY_AND_ASSIGN(SharedLibrary); explicit SharedLibrary() = default; }; }} // namespace triton::core kytensor-2.49.0.7/core/src/status.cc000066400000000000000000000064711527052371200172160ustar00rootroot00000000000000// Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "status.h" namespace triton { namespace core { const Status Status::Success(Status::Code::SUCCESS); Status::Code TritonCodeToStatusCode(TRITONSERVER_Error_Code code) { switch (code) { case TRITONSERVER_ERROR_UNKNOWN: return Status::Code::UNKNOWN; case TRITONSERVER_ERROR_INTERNAL: return Status::Code::INTERNAL; case TRITONSERVER_ERROR_NOT_FOUND: return Status::Code::NOT_FOUND; case TRITONSERVER_ERROR_INVALID_ARG: return Status::Code::INVALID_ARG; case TRITONSERVER_ERROR_UNAVAILABLE: return Status::Code::UNAVAILABLE; case TRITONSERVER_ERROR_UNSUPPORTED: return Status::Code::UNSUPPORTED; case TRITONSERVER_ERROR_ALREADY_EXISTS: return Status::Code::ALREADY_EXISTS; case TRITONSERVER_ERROR_CANCELLED: return Status::Code::CANCELLED; default: break; } return Status::Code::UNKNOWN; } TRITONSERVER_Error_Code StatusCodeToTritonCode(Status::Code status_code) { switch (status_code) { case Status::Code::UNKNOWN: return TRITONSERVER_ERROR_UNKNOWN; case Status::Code::INTERNAL: return TRITONSERVER_ERROR_INTERNAL; case Status::Code::NOT_FOUND: return TRITONSERVER_ERROR_NOT_FOUND; case Status::Code::INVALID_ARG: return TRITONSERVER_ERROR_INVALID_ARG; case Status::Code::UNAVAILABLE: return TRITONSERVER_ERROR_UNAVAILABLE; case Status::Code::UNSUPPORTED: return TRITONSERVER_ERROR_UNSUPPORTED; case Status::Code::ALREADY_EXISTS: return TRITONSERVER_ERROR_ALREADY_EXISTS; case Status::Code::CANCELLED: return TRITONSERVER_ERROR_CANCELLED; default: break; } return TRITONSERVER_ERROR_UNKNOWN; } Status CommonErrorToStatus(const triton::common::Error& error) { return Status(error); } }} // namespace triton::core kytensor-2.49.0.7/core/src/status.h000066400000000000000000000103611527052371200170510ustar00rootroot00000000000000// Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "triton/common/error.h" #include "tritonserver_apis.h" namespace triton { namespace core { class Status : public triton::common::Error { public: // Construct a status from a code with no message. explicit Status(Code code = Code::SUCCESS) : Error(code) {} // Construct a status from a code and message. explicit Status(Code code, const std::string& msg) : Error(code, msg) {} // Construct a status from a code and message. explicit Status(const Error& error) : Error(error) {} // Convenience "success" value. Can be used as Error::Success to // indicate no error. static const Status Success; // Return the code for this status. Code StatusCode() const { return code_; } }; // Return the Status::Code corresponding to a // TRITONSERVER_Error_Code. Status::Code TritonCodeToStatusCode(TRITONSERVER_Error_Code code); // Return the TRITONSERVER_Error_Code corresponding to a // Status::Code. TRITONSERVER_Error_Code StatusCodeToTritonCode(Status::Code status_code); // Converts the common Error to Status object Status CommonErrorToStatus(const triton::common::Error& error); // If status is non-OK, return the Status. #define RETURN_IF_ERROR(S) \ do { \ const Status& status__ = (S); \ if (!status__.IsOk()) { \ return status__; \ } \ } while (false) // If TRITONSERVER error is non-OK, return the corresponding status. #define RETURN_IF_TRITONSERVER_ERROR(E) \ do { \ TRITONSERVER_Error* err__ = (E); \ if (err__ != nullptr) { \ Status status__ = Status( \ TritonCodeToStatusCode(TRITONSERVER_ErrorCode(err__)), \ TRITONSERVER_ErrorMessage(err__)); \ TRITONSERVER_ErrorDelete(err__); \ return status__; \ } \ } while (false) // If status is non-OK, return the corresponding TRITONSERVER_Error. #define RETURN_TRITONSERVER_ERROR_IF_ERROR(S) \ do { \ const Status& status__ = (S); \ if (!status__.IsOk()) { \ return TRITONSERVER_ErrorNew( \ StatusCodeToTritonCode(status__.StatusCode()), \ status__.Message().c_str()); \ } \ } while (false) }} // namespace triton::core kytensor-2.49.0.7/core/src/test/000077500000000000000000000000001527052371200163335ustar00rootroot00000000000000kytensor-2.49.0.7/core/src/test/CMakeLists.txt000066400000000000000000000357541527052371200211110ustar00rootroot00000000000000# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) # # Unit tests # find_package(GTest REQUIRED) # # CudaMemoryManger # set( CUDA_MEMORY_MANAGER_SRCS ../cuda_block_manager.cc ../cuda_memory_manager.cc ../cuda_utils.cc ../status.cc ) set( CUDA_MEMORY_MANAGER_HDRS ../cuda_block_manager.h ../cuda_memory_manager.h ../cuda_utils.h ../status.h ) # # PinnedMemoryManger # set( PINNED_MEMORY_MANAGER_SRCS ../cuda_utils.cc ../numa_utils.cc ../pinned_memory_manager.cc ../status.cc ) set( PINNED_MEMORY_MANAGER_HDRS ../cuda_utils.h ../numa_utils.h ../pinned_memory_manager.h ../status.h ) # # Memory # set( MEMORY_SRCS ../buffer_attributes.cc ../memory.cc ) set( MEMORY_HDRS ../buffer_attributes.h ../memory.h ) # # Unit test for TritonCache # if(${TRITON_ENABLE_GPU}) add_executable( response_cache_test response_cache_test.cc ../cache_manager.cc ../cache_manager.h ../cache_entry.cc ../cache_entry.h ../filesystem/api.cc ../filesystem/api.h ../shared_library.cc ../shared_library.h ../status.cc ../status.h ../constants.h ${MEMORY_SRCS} ${CUDA_MEMORY_MANAGER_SRCS} ${PINNED_MEMORY_MANAGER_SRCS} ${MEMORY_HDRS} ${CUDA_MEMORY_MANAGER_HDRS} ${PINNED_MEMORY_MANAGER_HDRS} ) set_target_properties( response_cache_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( response_cache_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ${CNMEM_PATH}/include ${Boost_INCLUDE_DIRS} ) target_compile_definitions( response_cache_test PRIVATE TRITON_ENABLE_LOGGING=1 TRITON_ENABLE_GPU=1 TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) find_library(CNMEM_LIBRARY NAMES cnmem PATHS ${CNMEM_PATH}/lib) target_link_libraries( response_cache_test PRIVATE triton-common-error # from repo-common triton-common-logging # from repo-common triton-common-model-config # from repo-common proto-library # from repo-common triton-core GTest::gtest GTest::gtest_main protobuf::libprotobuf ${CNMEM_LIBRARY} CUDA::cudart ) if (NOT WIN32) target_link_libraries( response_cache_test PRIVATE dl numa ) endif() install( TARGETS response_cache_test RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_GPU # # Unit test for Query # add_executable( query_test query_test.cc ) set_target_properties( query_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( query_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ) target_link_libraries( query_test PRIVATE triton-common-error # from repo-common triton-core GTest::gtest GTest::gtest_main GTest::gmock ) install( TARGETS query_test RUNTIME DESTINATION bin ) # # Unit test for Memory # if(${TRITON_ENABLE_GPU}) add_executable( memory_test memory_test.cc ../constants.h ${MEMORY_SRCS} ${CUDA_MEMORY_MANAGER_SRCS} ${PINNED_MEMORY_MANAGER_SRCS} ${MEMORY_HDRS} ${CUDA_MEMORY_MANAGER_HDRS} ${PINNED_MEMORY_MANAGER_HDRS} ) set_target_properties( memory_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( memory_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ${CNMEM_PATH}/include ${Boost_INCLUDE_DIRS} ) target_compile_definitions( memory_test PRIVATE TRITON_ENABLE_LOGGING=1 TRITON_ENABLE_GPU=1 TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) find_library(CNMEM_LIBRARY NAMES cnmem PATHS ${CNMEM_PATH}/lib) target_link_libraries( memory_test PRIVATE triton-common-error # from repo-common triton-common-logging # from repo-common proto-library # from repo-common GTest::gtest GTest::gtest_main protobuf::libprotobuf ${CNMEM_LIBRARY} CUDA::cudart ) if (NOT WIN32) target_link_libraries( memory_test PRIVATE dl numa ) endif() install( TARGETS memory_test RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_GPU # # Unit test for PinnedMemoryManager # if(${TRITON_ENABLE_GPU}) add_executable( pinned_memory_manager_test pinned_memory_manager_test.cc ${PINNED_MEMORY_MANAGER_SRCS} ${PINNED_MEMORY_MANAGER_HDRS} ) set_target_properties( pinned_memory_manager_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( pinned_memory_manager_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ${CNMEM_PATH}/include ${Boost_INCLUDE_DIRS} ) target_compile_definitions( pinned_memory_manager_test PRIVATE TRITON_ENABLE_LOGGING=1 TRITON_ENABLE_GPU=1 TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) find_library(CNMEM_LIBRARY NAMES cnmem PATHS ${CNMEM_PATH}/lib) target_link_libraries( pinned_memory_manager_test PRIVATE triton-common-error # from repo-common triton-common-logging # from repo-common proto-library # from repo-common GTest::gtest GTest::gtest_main protobuf::libprotobuf ${CNMEM_LIBRARY} CUDA::cudart ) if (NOT WIN32) target_link_libraries( pinned_memory_manager_test PRIVATE dl numa ) endif() install( TARGETS pinned_memory_manager_test RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_GPU # # Unit test for AsycWorkQueue # add_executable( async_work_queue_test async_work_queue_test.cc ) set_target_properties( async_work_queue_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( async_work_queue_test PRIVATE ${GTEST_INCLUDE_DIRS} ) target_link_libraries( async_work_queue_test PRIVATE triton-common-thread-pool # from repo-common triton-common-async-work-queue # from repo-common triton-common-error # from repo-common GTest::gtest GTest::gtest_main ) install( TARGETS async_work_queue_test RUNTIME DESTINATION bin ) # # Unit test for TritonRepoAgent ... (TODO specify the other classes) # add_executable( repo_agent_test repo_agent_test.cc ../repo_agent.cc ../status.cc ../filesystem/api.cc ../model_config_utils.cc ../repo_agent.h ../shared_library.h ../status.h ../filesystem/api.h ../model_config_utils.h ) set_target_properties( repo_agent_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( repo_agent_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ) target_link_libraries( repo_agent_test PRIVATE triton-common-error # from repo-common triton-common-model-config # from repo-common triton-common-json # from repo-common triton-common-logging # from repo-common proto-library # from repo-common GTest::gtest GTest::gtest_main protobuf::libprotobuf ) install( TARGETS repo_agent_test RUNTIME DESTINATION bin ) if(${TRITON_ENABLE_METRICS}) # # Unit test for Generic Metrics API # add_executable( metrics_api_test metrics_api_test.cc ${PINNED_MEMORY_MANAGER_SRCS} ${PINNED_MEMORY_MANAGER_HDRS} ../metric_family.cc ../metric_family.h ../metrics.cc ../metrics.h ../infer_parameter.cc ../infer_parameter.h ) set_target_properties( metrics_api_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( metrics_api_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ) target_compile_definitions( metrics_api_test PRIVATE TRITON_ENABLE_LOGGING=1 TRITON_ENABLE_METRICS=1 ) target_link_libraries( metrics_api_test PRIVATE triton-common-error # from repo-common triton-common-logging # from repo-common proto-library # from repo-common triton-core GTest::gtest GTest::gtest_main GTest::gmock prometheus-cpp::core protobuf::libprotobuf ) if (TRITON_ENABLE_GPU) target_link_libraries( metrics_api_test PRIVATE ${CNMEM_LIBRARY} CUDA::cudart ) endif() if (NOT WIN32) target_link_libraries( metrics_api_test PRIVATE dl numa ) endif() install( TARGETS metrics_api_test RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_METRICS # # Unit test for Model Repository Register API # add_executable( register_api_test register_api_test.cc ) set_target_properties( register_api_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( register_api_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ) target_link_libraries( register_api_test PRIVATE triton-common-error # from repo-common triton-common-logging # from repo-common triton-core GTest::gtest GTest::gtest_main ) install( TARGETS register_api_test RUNTIME DESTINATION bin ) # # Backend Output Detail Unittest # add_executable( backend_output_detail_test backend_output_detail_test.cc ) target_include_directories( backend_output_detail_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ) set_target_properties( backend_output_detail_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_link_libraries( backend_output_detail_test PRIVATE triton-common-error # from repo-common triton-common-logging # from repo-common triton-core GTest::gtest GTest::gtest_main ) install( TARGETS backend_output_detail_test RUNTIME DESTINATION bin ) # # Request Cancellation Unittest # add_executable( request_cancellation_test request_cancellation_test.cc ) set_target_properties( request_cancellation_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( request_cancellation_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ) target_link_libraries( request_cancellation_test PRIVATE triton-common-error # from repo-common triton-common-logging # from repo-common triton-core GTest::gtest GTest::gtest_main ) install( TARGETS request_cancellation_test RUNTIME DESTINATION bin ) # # Unit test for Input Byte Size Validation # if(${TRITON_ENABLE_GPU}) add_executable( input_byte_size_test input_byte_size_test.cc ../constants.h ${CUDA_MEMORY_MANAGER_SRCS} ${CUDA_MEMORY_MANAGER_HDRS} ) target_include_directories( input_byte_size_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ${CNMEM_PATH}/include ) target_compile_definitions( input_byte_size_test PRIVATE TRITON_ENABLE_LOGGING=1 TRITON_ENABLE_GPU=1 TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) find_library(CNMEM_LIBRARY NAMES cnmem PATHS ${CNMEM_PATH}/lib) target_link_libraries( input_byte_size_test PRIVATE dl triton-common-error # from repo-common triton-core triton-common-logging # from repo-common proto-library # from repo-common GTest::gtest GTest::gtest_main GTest::gmock protobuf::libprotobuf CUDA::cudart ${CNMEM_LIBRARY} ) else() add_executable( input_byte_size_test input_byte_size_test.cc ) target_include_directories( input_byte_size_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ) target_link_libraries( input_byte_size_test PRIVATE triton-common-error # from repo-common triton-core GTest::gtest GTest::gtest_main GTest::gmock ) endif() # TRITON_ENABLE_GPU set_target_properties( input_byte_size_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) install( TARGETS input_byte_size_test RUNTIME DESTINATION bin ) kytensor-2.49.0.7/core/src/test/async_work_queue_test.cc000066400000000000000000000204421527052371200232660ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton/common/async_work_queue.h" #include #include #include #include #include #include #include #include #include "gtest/gtest.h" namespace tc = triton::common; namespace { // Wrapper of AsyncWorkQueue class to expose Reset() for unit testing class TestingAsyncWorkQueue : public tc::AsyncWorkQueue { public: static void Reset() { AsyncWorkQueue::Reset(); } }; class AsyncWorkQueueTest : public ::testing::Test { protected: void TearDown() override { TestingAsyncWorkQueue::Reset(); } }; TEST_F(AsyncWorkQueueTest, InitZeroWorker) { auto error = tc::AsyncWorkQueue::Initialize(0); EXPECT_FALSE(error.IsOk()) << "Expect error when initialized with 0 worker"; } TEST_F(AsyncWorkQueueTest, InitOneWorker) { auto error = tc::AsyncWorkQueue::Initialize(1); EXPECT_TRUE(error.IsOk()) << error.Message(); } TEST_F(AsyncWorkQueueTest, InitFourWorker) { auto error = tc::AsyncWorkQueue::Initialize(4); EXPECT_TRUE(error.IsOk()) << error.Message(); } TEST_F(AsyncWorkQueueTest, InitTwice) { auto error = tc::AsyncWorkQueue::Initialize(4); EXPECT_TRUE(error.IsOk()) << error.Message(); error = tc::AsyncWorkQueue::Initialize(2); EXPECT_FALSE(error.IsOk()) << "Expect error from initializing twice"; } TEST_F(AsyncWorkQueueTest, WorkerCountUninitialized) { EXPECT_EQ(tc::AsyncWorkQueue::WorkerCount(), (size_t)0) << "Expect 0 worker count for uninitialized queue"; } TEST_F(AsyncWorkQueueTest, WorkerCountInitialized) { auto error = tc::AsyncWorkQueue::Initialize(4); EXPECT_TRUE(error.IsOk()) << error.Message(); EXPECT_EQ(tc::AsyncWorkQueue::WorkerCount(), (size_t)4) << "Expect 4 worker count for initialized queue"; } TEST_F(AsyncWorkQueueTest, RunTasksInParallel) { auto AddTwoFn = [](const std::vector& lhs, const std::vector& rhs, std::promise>* res) { std::vector lres; lres.reserve(lhs.size()); for (size_t idx = 0; idx < lhs.size(); idx++) { lres.push_back(lhs[idx] + rhs[idx]); } res->set_value(lres); }; size_t task_count = 8; std::vector> operands; std::vector> expected_results; { // Use large element count to reduce the async work queue overhead size_t element_count = 1 << 24; auto RandHalfIntFn = std::bind( std::uniform_int_distribution<>{ std::numeric_limits::min() / 2, std::numeric_limits::max() / 2}, std::default_random_engine{}); for (size_t tc = 0; tc < task_count + 1; tc++) { expected_results.push_back(std::vector()); operands.push_back(std::vector()); operands.back().reserve(element_count); for (size_t ec = 0; ec < element_count; ec++) { operands.back().push_back(RandHalfIntFn()); } } } // Get serialized time as baseline and store expected results uint64_t serialized_duration = 0; { std::vector>> res(task_count); auto start_ts = std::chrono::duration_cast( std::chrono::high_resolution_clock::now().time_since_epoch()) .count(); for (size_t count = 0; count < task_count; count++) { AddTwoFn(operands[count], operands[count + 1], &res[count]); } auto end_ts = std::chrono::duration_cast( std::chrono::high_resolution_clock::now().time_since_epoch()) .count(); for (size_t count = 0; count < task_count; count++) { expected_results[count] = std::move(res[count].get_future().get()); } serialized_duration = end_ts - start_ts; } auto error = tc::AsyncWorkQueue::Initialize(4); ASSERT_TRUE(error.IsOk()) << error.Message(); uint64_t parallelized_duration = 0; { std::vector>> ps(task_count); std::vector>> fs; for (auto& p : ps) { fs.emplace_back(std::move(p.get_future())); } auto start_ts = std::chrono::duration_cast( std::chrono::high_resolution_clock::now().time_since_epoch()) .count(); for (size_t count = 0; count < task_count; count++) { tc::AsyncWorkQueue::AddTask([&AddTwoFn, &operands, &ps, count]() mutable { AddTwoFn(operands[count], operands[count + 1], &ps[count]); }); } for (size_t count = 0; count < task_count; count++) { fs[count].wait(); } auto end_ts = std::chrono::duration_cast( std::chrono::high_resolution_clock::now().time_since_epoch()) .count(); parallelized_duration = end_ts - start_ts; // FIXME manual testing shows parallelized time is between 30% to 33.3% // for 128 M total elements, but is flaky in CI so relax the comparison // for parallel to simply be faster than serial by any amount. EXPECT_LT(parallelized_duration, serialized_duration) << "Expected parallelized work was completed faster than serialized " "time"; for (size_t count = 0; count < task_count; count++) { auto res = std::move(fs[count].get()); EXPECT_EQ(res, expected_results[count]) << "Mismatched parallelized result"; } } } TEST_F(AsyncWorkQueueTest, RunTasksFIFO) { auto CaptureTimestampFn = [](std::promise* res) { res->set_value( std::chrono::duration_cast( std::chrono::high_resolution_clock::now().time_since_epoch()) .count()); std::this_thread::sleep_for(std::chrono::milliseconds(100)); }; size_t task_count = 8; std::vector> ps(task_count); auto error = tc::AsyncWorkQueue::Initialize(2); ASSERT_TRUE(error.IsOk()) << error.Message(); std::vector> barrier(2); tc::AsyncWorkQueue::AddTask([&barrier]() mutable { barrier[0].get_future().get(); std::this_thread::sleep_for(std::chrono::milliseconds(100)); }); tc::AsyncWorkQueue::AddTask([&barrier]() mutable { barrier[1].get_future().get(); std::this_thread::sleep_for(std::chrono::milliseconds(200)); }); for (size_t count = 0; count < task_count; count++) { tc::AsyncWorkQueue::AddTask([count, &CaptureTimestampFn, &ps]() mutable { CaptureTimestampFn(&ps[count]); }); } // Signal to start the work barrier[0].set_value(); barrier[1].set_value(); uint64_t prev_ts = 0; for (size_t count = 0; count < task_count; count++) { uint64_t curr_ts = ps[count].get_future().get(); EXPECT_LT(prev_ts, curr_ts) << "Expected async work is processed in FIFO order"; } } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/backend_output_detail_test.cc000066400000000000000000000253621527052371200242420ustar00rootroot00000000000000// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include #include #include "gtest/gtest.h" #include "triton/core/tritonbackend.h" #include "triton/core/tritonserver.h" namespace { #define FAIL_TEST_IF_ERR(X, MSG) \ do { \ std::shared_ptr err__((X), TRITONSERVER_ErrorDelete); \ ASSERT_TRUE((err__ == nullptr)) \ << "error: " << (MSG) << ": " \ << TRITONSERVER_ErrorCodeString(err__.get()) << " - " \ << TRITONSERVER_ErrorMessage(err__.get()); \ } while (false) TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { *actual_memory_type = TRITONSERVER_MEMORY_CPU; *actual_memory_type_id = preferred_memory_type_id; if (byte_size == 0) { *buffer = nullptr; *buffer_userp = nullptr; } else { void* allocated_ptr = nullptr; allocated_ptr = malloc(byte_size); if (allocated_ptr != nullptr) { *buffer = allocated_ptr; *buffer_userp = new std::string(tensor_name); } } return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { return nullptr; // Success } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { TRITONSERVER_InferenceRequestDelete(request); } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { if (response != nullptr) { // Notify that the completion. std::promise* p = reinterpret_cast*>(userp); p->set_value(response); delete p; } } class BackendOutputDetailTest : public ::testing::Test { protected: static void SetUpTestSuite() { // Create the server... TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, "./models"), "setting model repository path"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetBackendDirectory( server_options, "/opt/tritonserver/backends"), "setting backend directory"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetRepoAgentDirectory( server_options, "/opt/tritonserver/repoagents"), "setting repository agent directory"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), "setting strict model configuration"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerNew(&server_, server_options), "creating server"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); } static void TearDownTestSuite() { FAIL_TEST_IF_ERR(TRITONSERVER_ServerDelete(server_), "deleting server"); } void SetUp() override { ASSERT_TRUE(server_ != nullptr) << "Server has not created"; // Wait until the server is both live and ready. size_t health_iters = 0; while (true) { bool live, ready; FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsLive(server_, &live), "unable to get server liveness"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsReady(server_, &ready), "unable to get server readiness"); if (live && ready) { break; } if (++health_iters >= 10) { FAIL() << "failed to find healthy inference server"; } std::this_thread::sleep_for(std::chrono::milliseconds(500)); } // Create allocator with common callback FAIL_TEST_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator_, ResponseAlloc, ResponseRelease, nullptr /* start_fn */), "creating response allocator"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "add_sub", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); std::vector input0_shape({16}); std::vector input1_shape({16}); const auto input0_byte_size = sizeof(input0_data_[0]) * input0_data_.size(); const auto input1_byte_size = sizeof(input1_data_[0]) * input1_data_.size(); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_FP32, input0_shape.data(), input0_shape.size()), "setting input0 for the request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest_, "INPUT0", input0_data_.data(), input0_byte_size, TRITONSERVER_MEMORY_CPU, 0), "assigning INPUT data"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT1", TRITONSERVER_TYPE_FP32, input1_shape.data(), input1_shape.size()), "setting input1 for the request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest_, "INPUT1", input1_data_.data(), input1_byte_size, TRITONSERVER_MEMORY_CPU, 0), "assigning INPUT1 data"); } void TearDown() override { unsetenv("TEST_ANONYMOUS"); unsetenv("TEST_BYTE_SIZE"); FAIL_TEST_IF_ERR( TRITONSERVER_ResponseAllocatorDelete(allocator_), "deleting response allocator"); } static TRITONSERVER_Server* server_; TRITONSERVER_ResponseAllocator* allocator_ = nullptr; static std::vector input0_data_; static std::vector input1_data_; TRITONSERVER_InferenceRequest* irequest_ = nullptr; }; TRITONSERVER_Server* BackendOutputDetailTest::server_ = nullptr; std::vector BackendOutputDetailTest::input0_data_(16, 1); std::vector BackendOutputDetailTest::input1_data_(16, 1); TEST_F(BackendOutputDetailTest, DefaultInference) { auto p = new std::promise(); std::future future = p->get_future(); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(p)), "setting response callback"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "running inference"); uint32_t output_count; const char* output_name; TRITONSERVER_DataType output_datatype; const int64_t* output_shape; uint64_t dims_count; std::vector names = {"OUTPUT0", "OUTPUT1"}; TRITONSERVER_InferenceResponse* response = future.get(); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceResponseError(response), "error with inference response"); ASSERT_TRUE(response != nullptr) << "Expect successful inference"; FAIL_TEST_IF_ERR( TRITONSERVER_InferenceResponseOutputCount(response, &output_count), "getting output count"); ASSERT_EQ(output_count, size_t(2)); for (size_t idx = 0; idx < output_count; idx++) { // Cast the response from a TRITONSERVER_Response to a // TRITONBACKEND_Response. This is not recommended and not allowed for // backend developers as this cast is unsupported. However, for the purposes // of our own internal testing we do so here in order to validate the // functionality of our backend APIs. TRITONBACKEND_Response* backend_response = reinterpret_cast(response); FAIL_TEST_IF_ERR( TRITONBACKEND_InferenceResponseOutput( backend_response, idx, &output_name, &output_datatype, &output_shape, &dims_count), "getting output details by index"); EXPECT_EQ(*output_name, *names[idx]); EXPECT_EQ(output_datatype, TRITONSERVER_TYPE_FP32); EXPECT_EQ(*output_shape, int64_t(16)); EXPECT_EQ(dims_count, int64_t(1)); FAIL_TEST_IF_ERR( TRITONBACKEND_InferenceResponseOutputByName( backend_response, names[idx], &output_datatype, &output_shape, &dims_count), "getting output details by name"); EXPECT_EQ(output_datatype, TRITONSERVER_TYPE_FP32); EXPECT_EQ(*output_shape, int64_t(16)); EXPECT_EQ(dims_count, int64_t(1)); } TRITONSERVER_InferenceResponseDelete(response); } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/input_byte_size_test.cc000066400000000000000000000625411527052371200231250ustar00rootroot00000000000000// Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include #include #include #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "triton/core/tritonserver.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace { #define FAIL_TEST_IF_ERR(X, MSG) \ do { \ std::shared_ptr err__((X), TRITONSERVER_ErrorDelete); \ ASSERT_TRUE((err__ == nullptr)) \ << "error: " << (MSG) << ": " \ << TRITONSERVER_ErrorCodeString(err__.get()) << " - " \ << TRITONSERVER_ErrorMessage(err__.get()); \ } while (false) #define FAIL_TEST_IF_SUCCESS(X, MSG, ERR_MSG) \ do { \ std::shared_ptr err__((X), TRITONSERVER_ErrorDelete); \ ASSERT_FALSE((err__ == nullptr)) << "error: " << (MSG) << ": "; \ ASSERT_THAT( \ TRITONSERVER_ErrorMessage(err__.get()), ::testing::HasSubstr(ERR_MSG)) \ << "error: " \ << "Unexpected error message: " \ << TRITONSERVER_ErrorCodeString(err__.get()) << " - " \ << TRITONSERVER_ErrorMessage(err__.get()); \ } while (false) #ifdef TRITON_ENABLE_GPU #define FAIL_IF_CUDA_ERR(X, MSG) \ do { \ cudaError_t err__ = (X); \ if (err__ != cudaSuccess) { \ std::cerr << "error: " << (MSG) << ": " << cudaGetErrorString(err__) \ << std::endl; \ exit(1); \ } \ } while (false) #endif // TRITON_ENABLE_GPU TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { *actual_memory_type = TRITONSERVER_MEMORY_CPU; *actual_memory_type_id = preferred_memory_type_id; if (byte_size == 0) { *buffer = nullptr; *buffer_userp = nullptr; } else { void* allocated_ptr = nullptr; allocated_ptr = malloc(byte_size); if (allocated_ptr != nullptr) { *buffer = allocated_ptr; *buffer_userp = new std::string(tensor_name); } } return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { free(buffer); return nullptr; // Success } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestDelete(request), "deleting inference request"); } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { if (response != nullptr) { // Notify that the completion. auto p = reinterpret_cast*>(userp); p->set_value(response); } } void SplitBytesInput( const char input_data[], const size_t input_data_size, size_t first_portion_size, TRITONSERVER_MemoryType memory_type, TRITONSERVER_InferenceRequest* irequest) { // Append first buffer FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, "INPUT0", input_data, first_portion_size, memory_type, 0), "assigning INPUT data"); // Append second buffer const size_t second_portion_size = input_data_size - first_portion_size; FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, "INPUT0", input_data + first_portion_size, second_portion_size, memory_type, 0), "assigning INPUT data"); } class InputByteSizeTest : public ::testing::Test { protected: static void SetUpTestSuite() { // Prepare input array. Concatenate first 4 bytes representing the size // of input with the input string bytes. const size_t input_element_size = std::strlen(input_element_); std::memcpy( input_data_string_, reinterpret_cast(&input_element_size), kElementSizeIndicator_); std::memcpy( input_data_string_ + kElementSizeIndicator_, input_element_, std::strlen(input_element_)); // Create the server... TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, "./models"), "setting model repository path"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetBackendDirectory( server_options, "/opt/tritonserver/backends"), "setting backend directory"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetRepoAgentDirectory( server_options, "/opt/tritonserver/repoagents"), "setting repository agent directory"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), "setting strict model configuration"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerNew(&server_, server_options), "creating server"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); } static void TearDownTestSuite() { FAIL_TEST_IF_ERR(TRITONSERVER_ServerDelete(server_), "deleting server"); } void SetUp() override { ASSERT_TRUE(server_ != nullptr) << "Server has not created"; // Wait until the server is both live and ready. size_t health_iters = 0; while (true) { bool live, ready; FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsLive(server_, &live), "unable to get server liveness"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsReady(server_, &ready), "unable to get server readiness"); if (live && ready) { break; } if (++health_iters >= 10) { FAIL() << "failed to find healthy inference server"; } std::this_thread::sleep_for(std::chrono::milliseconds(500)); } // Create allocator with common callback FAIL_TEST_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator_, ResponseAlloc, ResponseRelease, nullptr /* start_fn */), "creating response allocator"); } void TearDown() override { FAIL_TEST_IF_ERR( TRITONSERVER_ResponseAllocatorDelete(allocator_), "deleting response allocator"); } static TRITONSERVER_Server* server_; TRITONSERVER_ResponseAllocator* allocator_ = nullptr; static constexpr char input_element_[] = "Example input string"; static constexpr size_t kElementSizeIndicator_ = sizeof(uint32_t); static char input_data_string_[]; TRITONSERVER_InferenceRequest* irequest_ = nullptr; TRITONSERVER_InferenceResponse* response_ = nullptr; std::promise completed_; }; TRITONSERVER_Server* InputByteSizeTest::server_ = nullptr; char InputByteSizeTest::input_data_string_ [kElementSizeIndicator_ + std::strlen(input_element_)] = {}; TEST_F(InputByteSizeTest, ValidInputByteSize) { // Create an inference request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "pt_identity", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Define input shape and data std::vector shape{1, 8}; std::vector input_data(8, 1); const auto input0_byte_size = sizeof(input_data[0]) * input_data.size(); // Set input for the request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_FP32, shape.data(), shape.size()), "setting input for the request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest_, "INPUT0", input_data.data(), input0_byte_size, TRITONSERVER_MEMORY_CPU, 0), "assigning INPUT data"); std::promise p; std::future future = p.get_future(); // Set response callback FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(&p)), "setting response callback"); // Run inference FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "running inference"); // Get the inference response response_ = future.get(); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceResponseError(response_), "response status"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceResponseDelete(response_), "deleting inference response"); ASSERT_TRUE(response_ != nullptr) << "Expect successful inference"; } TEST_F(InputByteSizeTest, InputByteSizeMismatch) { // Create an inference request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "pt_identity", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Define input shape and data std::vector shape{1, 8}; std::vector input_data(10, 1); const auto input0_byte_size = sizeof(input_data[0]) * input_data.size(); // Set input for the request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_FP32, shape.data(), shape.size()), "setting input for the request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest_, "INPUT0", input_data.data(), input0_byte_size, TRITONSERVER_MEMORY_CPU, 0), "assigning INPUT data"); std::promise p; std::future future = p.get_future(); // Set response callback FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(&p)), "setting response callback"); // Run inference FAIL_TEST_IF_SUCCESS( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "expect error with inference request", "input byte size mismatch for input 'INPUT0' for model 'pt_identity'. " "Expected 32, got 40"); // Need to manually delete request, otherwise server will not shut down. FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestDelete(irequest_), "deleting inference request"); } TEST_F(InputByteSizeTest, ValidStringInputByteSize) { // Create an inference request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "simple_identity", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Define input shape std::vector shape({1, 3}); const size_t input_data_size = sizeof(input_data_string_); const size_t input_element_size = std::strlen(input_element_); // Set input for the request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_BYTES, shape.data(), shape.size()), "setting input for the request"); SplitBytesInput( input_data_string_, input_data_size, kElementSizeIndicator_, TRITONSERVER_MEMORY_CPU, irequest_); SplitBytesInput( input_data_string_, input_data_size, kElementSizeIndicator_ + input_element_size, TRITONSERVER_MEMORY_CPU, irequest_); SplitBytesInput( input_data_string_, input_data_size, kElementSizeIndicator_ + input_element_size / 2, TRITONSERVER_MEMORY_CPU, irequest_); std::promise p; std::future future = p.get_future(); // Set response callback FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(&p)), "setting response callback"); // Run inference FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "running inference"); // Get the inference response response_ = future.get(); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceResponseError(response_), "response status"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceResponseDelete(response_), "deleting inference response"); ASSERT_TRUE(response_ != nullptr) << "Expect successful inference"; } TEST_F(InputByteSizeTest, StringCountMismatch) { // Create an inference request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "simple_identity", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Shape is larger than actual string count std::vector shape({1, 3}); const size_t input_data_size = sizeof(input_data_string_); const size_t input_element_size = std::strlen(input_element_); // Set input for the request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_BYTES, shape.data(), shape.size()), "setting input for the request"); SplitBytesInput( input_data_string_, input_data_size, kElementSizeIndicator_, TRITONSERVER_MEMORY_CPU, irequest_); SplitBytesInput( input_data_string_, input_data_size, kElementSizeIndicator_ + input_element_size, TRITONSERVER_MEMORY_CPU, irequest_); // Run inference FAIL_TEST_IF_SUCCESS( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "expect error with inference request", "expected 3 string elements for inference input 'INPUT0', got 2"); // Need to manually delete request, otherwise server will not shut down. FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestDelete(irequest_), "deleting inference request"); // Create an inference request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "simple_identity", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Shape is smaller than actual string count shape = {1, 1}; // Set input for the request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_BYTES, shape.data(), shape.size()), "setting input for the request"); SplitBytesInput( input_data_string_, input_data_size, kElementSizeIndicator_, TRITONSERVER_MEMORY_CPU, irequest_); SplitBytesInput( input_data_string_, input_data_size, kElementSizeIndicator_ + input_element_size, TRITONSERVER_MEMORY_CPU, irequest_); // Run inference FAIL_TEST_IF_SUCCESS( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "expect error with inference request", "expected 1 string elements for inference input 'INPUT0', got 2"); // Need to manually delete request, otherwise server will not shut down. FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestDelete(irequest_), "deleting inference request"); } TEST_F(InputByteSizeTest, StringSizeMisalign) { // Create an inference request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "simple_identity", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Define input shape std::vector shape({1, 1}); const size_t input_data_size = sizeof(input_data_string_); // Set input for the request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_BYTES, shape.data(), shape.size()), "setting input for the request"); // Split element size indicator into two buffers SplitBytesInput( input_data_string_, input_data_size, 2, TRITONSERVER_MEMORY_CPU, irequest_); std::promise p; std::future future = p.get_future(); // Set response callback FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(&p)), "setting response callback"); // Run inference FAIL_TEST_IF_SUCCESS( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "expect error with inference request", "element byte size indicator exceeds the end of the buffer"); // Need to manually delete request, otherwise server will not shut down. FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestDelete(irequest_), "deleting inference request"); } #ifdef TRITON_ENABLE_GPU TEST_F(InputByteSizeTest, StringCountMismatchGPU) { const size_t input_data_size = sizeof(input_data_string_); const size_t input_element_size = std::strlen(input_element_); // Allocate CUDA shared memory char* d_input_data; FAIL_IF_CUDA_ERR(cudaSetDevice(0), "setting CUDA device to device 0"); FAIL_IF_CUDA_ERR( cudaMalloc((void**)&d_input_data, input_data_size), "allocating GPU memory for INPUT0 data"); FAIL_IF_CUDA_ERR( cudaMemcpy( (void*)d_input_data, input_data_string_, input_data_size, cudaMemcpyHostToDevice), "setting INPUT0 data in GPU memory"); // Create an inference request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "simple_identity", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Shape is larger than actual string count std::vector shape({1, 3}); // Set input for the request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_BYTES, shape.data(), shape.size()), "setting input for the request"); // Assign input data using CUDA shared memory SplitBytesInput( d_input_data, input_data_size, kElementSizeIndicator_, TRITONSERVER_MEMORY_GPU, irequest_); SplitBytesInput( d_input_data, input_data_size, kElementSizeIndicator_ + input_element_size, TRITONSERVER_MEMORY_GPU, irequest_); std::promise p1; std::future future1 = p1.get_future(); // Set response callback FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(&p1)), "setting response callback"); // Run inference FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "running inference"); // Currently byte_size check for string input is skipped at core level thus // should not receive the above error message. See details in // InferenceRequest::ValidateBytesInputs in infer_request.cc. response_ = future1.get(); FAIL_TEST_IF_SUCCESS( TRITONSERVER_InferenceResponseError(response_), "response status", "expected 3 strings for inference input 'INPUT0', got 2"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceResponseDelete(response_), "deleting inference response"); ASSERT_TRUE(response_ != nullptr) << "Expect successful inference"; // Create an inference request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "simple_identity", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Shape is smaller than actual string count shape = {1, 1}; // Set input for the request FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_BYTES, shape.data(), shape.size()), "setting input for the request"); // Assign input data using CUDA shared memory SplitBytesInput( d_input_data, input_data_size, kElementSizeIndicator_, TRITONSERVER_MEMORY_GPU, irequest_); SplitBytesInput( d_input_data, input_data_size, kElementSizeIndicator_ + input_element_size, TRITONSERVER_MEMORY_GPU, irequest_); std::promise p2; std::future future2 = p2.get_future(); // Set response callback FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(&p2)), "setting response callback"); // Run inference FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "running inference"); // Currently byte_size check for string input is skipped at core level thus // should not receive the above error message. See details in // InferenceRequest::ValidateBytesInputs in infer_request.cc. response_ = future2.get(); FAIL_TEST_IF_SUCCESS( TRITONSERVER_InferenceResponseError(response_), "response status", "unexpected number of string elements 2 for inference input 'INPUT0', " "expecting 1"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceResponseDelete(response_), "deleting inference response"); ASSERT_TRUE(response_ != nullptr) << "Expect successful inference"; // Clean up CUDA resources FAIL_IF_CUDA_ERR(cudaFree(d_input_data), "releasing GPU memory"); } #endif // TRITON_ENABLE_GPU } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/memory_test.cc000066400000000000000000000444061527052371200212210ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "memory.h" #include #include "cuda_block_manager.h" #include "cuda_memory_manager.h" #include "cuda_utils.h" #include "gtest/gtest.h" #include "pinned_memory_manager.h" #include "triton/core/tritonserver.h" namespace tc = triton::core; namespace { #define CHECK_POINTER_ATTRIBUTES(ptr__, type__, device__) \ do { \ cudaPointerAttributes attr; \ auto cuerr = cudaPointerGetAttributes(&attr, ptr__); \ ASSERT_TRUE(cuerr == cudaSuccess) \ << "Failed to get CUDA pointer attributes: " \ << cudaGetErrorString(cuerr); \ EXPECT_TRUE(attr.type == type__) \ << "Expect pointer with type " << type__ << ", got: " << attr.type; \ if (attr.type == cudaMemoryTypeDevice) { \ EXPECT_TRUE(attr.device == device__) \ << "Expect allocation on CUDA device " << device__ \ << ", got: " << attr.device; \ } \ } while (false) // Wrapper of CudaMemoryManager class to expose Reset() for unit testing class TestingCudaMemoryManager : public tc::CudaMemoryManager { public: static void Reset() { CudaMemoryManager::Reset(); } }; class CudaMemoryManagerTest : public ::testing::Test { protected: void SetUp() override { // Default memory manager options options_.min_supported_compute_capability_ = 6.0; options_.memory_pool_byte_size_ = {{0, 1 << 10}}; } void TearDown() override { TestingCudaMemoryManager::Reset(); } tc::CudaMemoryManager::Options options_; }; TEST_F(CudaMemoryManagerTest, InitOOM) { // Set to reserve too much memory double cc = 6.0; std::map s{{0, uint64_t(1) << 40 /* 1024 GB */}}; const tc::CudaMemoryManager::Options options{cc, s}; auto status = tc::CudaMemoryManager::Create(options); EXPECT_FALSE(status.IsOk()) << "Expect creation error"; } TEST_F(CudaMemoryManagerTest, InitSuccess) { double cc = 6.0; std::map s{{0, 1 << 10 /* 1024 bytes */}}; const tc::CudaMemoryManager::Options options{cc, s}; auto status = tc::CudaMemoryManager::Create(options); EXPECT_TRUE(status.IsOk()) << status.Message(); } TEST_F(CudaMemoryManagerTest, InitNoDeviceConfig) { double cc = 6.0; std::map s; const tc::CudaMemoryManager::Options options{cc, s}; auto status = tc::CudaMemoryManager::Create(options); EXPECT_TRUE(status.IsOk()) << status.Message(); void* ptr = nullptr; status = tc::CudaMemoryManager::Alloc(&ptr, 1, 0); ASSERT_FALSE(status.IsOk()) << "Unexpected successful allocation"; } TEST_F(CudaMemoryManagerTest, InitZeroByte) { double cc = 6.0; std::map s{{0, 0}}; const tc::CudaMemoryManager::Options options{cc, s}; auto status = tc::CudaMemoryManager::Create(options); EXPECT_TRUE(status.IsOk()) << status.Message(); void* ptr = nullptr; status = tc::CudaMemoryManager::Alloc(&ptr, 1, 0); ASSERT_FALSE(status.IsOk()) << "Unexpected successful allocation"; } TEST_F(CudaMemoryManagerTest, AllocSuccess) { auto status = tc::CudaMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); void* ptr = nullptr; status = tc::CudaMemoryManager::Alloc(&ptr, 1024, 0); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(ptr) << "Expect pointer to allocated buffer"; // check if returned pointer is CUDA pointer CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeDevice, 0); } TEST_F(CudaMemoryManagerTest, AllocFail) { auto status = tc::CudaMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); void* ptr = nullptr; status = tc::CudaMemoryManager::Alloc(&ptr, 2048, 0); ASSERT_FALSE(status.IsOk()) << "Unexpected successful allocation"; } TEST_F(CudaMemoryManagerTest, MultipleAlloc) { auto status = tc::CudaMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); void* first_ptr = nullptr; status = tc::CudaMemoryManager::Alloc(&first_ptr, 600, 0); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(first_ptr) << "Expect pointer to allocated buffer"; // check if returned pointer is CUDA pointer CHECK_POINTER_ATTRIBUTES(first_ptr, cudaMemoryTypeDevice, 0); // 512 + 600 > 1024 void* second_ptr = nullptr; status = tc::CudaMemoryManager::Alloc(&second_ptr, 512, 0); ASSERT_FALSE(status.IsOk()) << "Unexpected successful allocation"; // Free the first pointer and retry the second one status = tc::CudaMemoryManager::Free(first_ptr, 0); EXPECT_TRUE(status.IsOk()) << status.Message(); status = tc::CudaMemoryManager::Alloc(&second_ptr, 512, 0); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(second_ptr) << "Expect pointer to allocated buffer"; // check if returned pointer is CUDA pointer CHECK_POINTER_ATTRIBUTES(second_ptr, cudaMemoryTypeDevice, 0); } TEST_F(CudaMemoryManagerTest, MultipleDevice) { std::set supported_gpus; auto status = tc::GetSupportedGPUs( &supported_gpus, options_.min_supported_compute_capability_); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_GE(supported_gpus.size(), size_t(2)) << "Test requires at least two supported CUDA devices"; { double cc = 6.0; std::map s; // Only enough memory is only reserved in one of the devices s[*supported_gpus.begin()] = 32; s[*(++supported_gpus.begin())] = 1024; const tc::CudaMemoryManager::Options options{cc, s}; status = tc::CudaMemoryManager::Create(options); ASSERT_TRUE(status.IsOk()) << status.Message(); } void* ptr = nullptr; // Allocation on small device int small_device = *supported_gpus.begin(); status = tc::CudaMemoryManager::Alloc(&ptr, 1024, small_device); ASSERT_FALSE(status.IsOk()) << "Unexpected successful allocation"; // Allocation on large device int large_device = *(++supported_gpus.begin()); status = tc::CudaMemoryManager::Alloc(&ptr, 1024, large_device); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(ptr) << "Expect pointer to allocated buffer"; // check if returned pointer is CUDA pointer CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeDevice, large_device); // Free allocation ... status = tc::CudaMemoryManager::Free(ptr, small_device); EXPECT_FALSE(status.IsOk()) << "Unexpected deallocation on wrong device"; status = tc::CudaMemoryManager::Free(ptr, large_device); EXPECT_TRUE(status.IsOk()) << status.Message(); } class AllocatedMemoryTest : public ::testing::Test { protected: // Per-test-suite set-up. static void SetUpTestSuite() { // Pinned memory manager { tc::PinnedMemoryManager::Options options{1024}; auto status = tc::PinnedMemoryManager::Create(options); ASSERT_TRUE(status.IsOk()) << status.Message(); } } // Set up CUDA memory manager per test for special fallback case void SetUp() override { tc::CudaMemoryManager::Options options{6.0, {{0, 1 << 10}}}; auto status = tc::CudaMemoryManager::Create(options); ASSERT_TRUE(status.IsOk()) << status.Message(); } void TearDown() override { TestingCudaMemoryManager::Reset(); } }; TEST_F(AllocatedMemoryTest, AllocGPU) { size_t expect_size = 512, actual_size; TRITONSERVER_MemoryType expect_type = TRITONSERVER_MEMORY_GPU, actual_type; int64_t expect_id = 0, actual_id; tc::AllocatedMemory memory(expect_size, expect_type, expect_id); auto ptr = memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(expect_type, actual_type) << "Expect type: " << expect_type << ", got: " << actual_type; EXPECT_EQ(expect_id, actual_id) << "Expect id: " << expect_id << ", got: " << actual_id; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeDevice, expect_id); } TEST_F(AllocatedMemoryTest, AllocPinned) { size_t expect_size = 512, actual_size; TRITONSERVER_MemoryType expect_type = TRITONSERVER_MEMORY_CPU_PINNED, actual_type; int64_t expect_id = 0, actual_id; tc::AllocatedMemory memory(expect_size, expect_type, expect_id); auto ptr = memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(expect_type, actual_type) << "Expect type: " << expect_type << ", got: " << actual_type; EXPECT_EQ(expect_id, actual_id) << "Expect id: " << expect_id << ", got: " << actual_id; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeHost, expect_id); } TEST_F(AllocatedMemoryTest, AllocFallback) { // Each allocation uses half of the target reserved memory size_t expect_size = 600, actual_size; TRITONSERVER_MemoryType expect_type = TRITONSERVER_MEMORY_GPU, actual_type; int64_t expect_id = 0, actual_id; // First allocation tc::AllocatedMemory cuda_memory(expect_size, expect_type, expect_id); auto ptr = cuda_memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(expect_type, actual_type) << "Expect type: " << expect_type << ", got: " << actual_type; EXPECT_EQ(expect_id, actual_id) << "Expect id: " << expect_id << ", got: " << actual_id; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeDevice, expect_id); // Second allocation, should trigger fallback from CUDA -> pinned memory tc::AllocatedMemory pinned_memory(expect_size, expect_type, expect_id); ptr = pinned_memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(TRITONSERVER_MEMORY_CPU_PINNED, actual_type) << "Expect type: " << TRITONSERVER_MEMORY_CPU_PINNED << ", got: " << actual_type; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeHost, expect_id); // Third allocation, CUDA -> pinned -> non-pinned tc::AllocatedMemory system_memory(expect_size, expect_type, expect_id); ptr = system_memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(TRITONSERVER_MEMORY_CPU, actual_type) << "Expect type: " << TRITONSERVER_MEMORY_CPU_PINNED << ", got: " << actual_type; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeUnregistered, expect_id); } TEST_F(AllocatedMemoryTest, AllocFallbackNoCuda) { // Test fallback in the case where CUDA memory manager is not properly created TestingCudaMemoryManager::Reset(); size_t expect_size = 600, actual_size; TRITONSERVER_MemoryType expect_type = TRITONSERVER_MEMORY_GPU, actual_type; int64_t expect_id = 0, actual_id; // CUDA memory allocation should trigger fallback to allocate pinned memory tc::AllocatedMemory pinned_memory(expect_size, expect_type, expect_id); auto ptr = pinned_memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(TRITONSERVER_MEMORY_CPU_PINNED, actual_type) << "Expect type: " << TRITONSERVER_MEMORY_CPU_PINNED << ", got: " << actual_type; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeHost, expect_id); } TEST_F(AllocatedMemoryTest, Release) { // Similar to above, but verify that the memory will be released once // out of scope // Each allocation uses half of the target reserved memory size_t expect_size = 600, actual_size; TRITONSERVER_MemoryType expect_type = TRITONSERVER_MEMORY_GPU, actual_type; int64_t expect_id = 0, actual_id; { // First allocation tc::AllocatedMemory cuda_memory(expect_size, expect_type, expect_id); auto ptr = cuda_memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(expect_type, actual_type) << "Expect type: " << expect_type << ", got: " << actual_type; EXPECT_EQ(expect_id, actual_id) << "Expect id: " << expect_id << ", got: " << actual_id; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeDevice, expect_id); // Second allocation, should trigger fallback from CUDA -> pinned memory tc::AllocatedMemory pinned_memory(expect_size, expect_type, expect_id); ptr = pinned_memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(TRITONSERVER_MEMORY_CPU_PINNED, actual_type) << "Expect type: " << TRITONSERVER_MEMORY_CPU_PINNED << ", got: " << actual_type; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeHost, expect_id); } // Third allocation, should not trigger fallback tc::AllocatedMemory memory(expect_size, expect_type, expect_id); auto ptr = memory.BufferAt(0, &actual_size, &actual_type, &actual_id); EXPECT_EQ(expect_size, actual_size) << "Expect size: " << expect_size << ", got: " << actual_size; EXPECT_EQ(expect_type, actual_type) << "Expect type: " << expect_type << ", got: " << actual_type; // Sanity check on the pointer property CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeDevice, expect_id); } class GrowableMemoryTest : public ::testing::Test { protected: // Per-test-suite set-up. static void SetUpTestSuite() { auto status = tc::CudaBlockManager::Create(6.0); EXPECT_TRUE(status.IsOk()) << status.Message(); } void TearDown() override { tc::CudaBlockManager::Reset(); } }; TEST_F(GrowableMemoryTest, AllocGPU) { std::unique_ptr growable_memory; size_t block_size = tc::CudaBlockManager::BlockSize(); auto status = tc::GrowableMemory::Create( growable_memory, block_size, TRITONSERVER_MEMORY_GPU, 0, block_size * 20); EXPECT_TRUE(status.IsOk()) << status.Message(); auto blocks = growable_memory->GetAllocation()->Blocks(); TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; char* buffer = growable_memory->MutableBuffer(&memory_type, &memory_type_id); EXPECT_EQ(memory_type, TRITONSERVER_MEMORY_GPU); EXPECT_EQ(memory_type_id, 0); CHECK_POINTER_ATTRIBUTES(buffer, cudaMemoryTypeDevice, 0); // Make sure the memory is destructed without issues. growable_memory.reset(nullptr); // Allocate new memory with the same size. status = tc::GrowableMemory::Create( growable_memory, block_size, TRITONSERVER_MEMORY_GPU, 0, block_size * 20); buffer = growable_memory->MutableBuffer(&memory_type, &memory_type_id); EXPECT_EQ(memory_type, TRITONSERVER_MEMORY_GPU); EXPECT_EQ(memory_type_id, 0); CHECK_POINTER_ATTRIBUTES(buffer, cudaMemoryTypeDevice, 0); EXPECT_TRUE(status.IsOk()) << status.Message(); auto new_blocks = growable_memory->GetAllocation()->Blocks(); ASSERT_EQ(blocks[0], new_blocks[0]) << "Expected the same blocks found different blocks"; // Grow the memory status = growable_memory->Resize(block_size * 3); buffer = growable_memory->MutableBuffer(&memory_type, &memory_type_id); EXPECT_EQ(memory_type, TRITONSERVER_MEMORY_GPU); EXPECT_EQ(memory_type_id, 0); CHECK_POINTER_ATTRIBUTES(buffer, cudaMemoryTypeDevice, 0); EXPECT_TRUE(status.IsOk()) << status.Message(); ASSERT_EQ(growable_memory->GetAllocation()->Blocks().size(), 3) << "expected 3 blocks."; // Try to grow the memory beyond the virtual address size status = growable_memory->Resize(block_size * 21); EXPECT_FALSE(status.IsOk()) << status.Message(); // Try to allocate memory larger than the block size status = tc::GrowableMemory::Create( growable_memory, block_size * 21, TRITONSERVER_MEMORY_GPU, 0, block_size * 20); EXPECT_FALSE(status.IsOk()) << status.Message(); // Request a CPU memory and make sure you get GPU memory back status = tc::GrowableMemory::Create( growable_memory, block_size, TRITONSERVER_MEMORY_CPU, 0, block_size * 20); buffer = growable_memory->MutableBuffer(&memory_type, &memory_type_id); EXPECT_EQ(memory_type, TRITONSERVER_MEMORY_GPU); EXPECT_EQ(memory_type_id, 0); } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/metrics_api_test.cc000066400000000000000000001005101527052371200221750ustar00rootroot00000000000000// Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights // reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef TRITON_ENABLE_METRICS #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "metric_family.h" #include "triton/common/logging.h" #include "triton/core/tritonserver.h" namespace tc = triton::core; namespace { using ::testing::HasSubstr; #define FAIL_TEST_IF_ERR(X, MSG) \ do { \ std::shared_ptr err__((X), TRITONSERVER_ErrorDelete); \ ASSERT_TRUE((err__ == nullptr)) \ << "error: " << (MSG) << ": " \ << TRITONSERVER_ErrorCodeString(err__.get()) << " - " \ << TRITONSERVER_ErrorMessage(err__.get()); \ } while (false) /* Helpers */ // Get serialized metrics string from C API void GetMetrics(TRITONSERVER_Server* server, std::string* metrics_str) { // Check metrics via C API ASSERT_NE(server, nullptr); TRITONSERVER_Metrics* metrics = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_ServerMetrics(server, &metrics), "fetch metrics"); const char* base; size_t byte_size; FAIL_TEST_IF_ERR( TRITONSERVER_MetricsFormatted( metrics, TRITONSERVER_METRIC_PROMETHEUS, &base, &byte_size), "format metrics string"); *metrics_str = std::string(base, byte_size); TRITONSERVER_MetricsDelete(metrics); } // Count number of times substr appears in s int CountMatches(const std::string s, const std::string substr) { int num_matches = 0; std::string::size_type pos = 0; while ((pos = s.find(substr, pos)) != std::string::npos) { num_matches++; pos += substr.length(); } return num_matches; } int NumMetricMatches(TRITONSERVER_Server* server, const std::string substr) { std::string metrics_str; GetMetrics(server, &metrics_str); const int num_matches = CountMatches(metrics_str, substr); return num_matches; } // Add two metrics with the same labels from the same metric family // and verify they refer to the same metric/value void DupeMetricHelper( TRITONSERVER_Server* server, std::vector labels) { // Create metric family TRITONSERVER_MetricFamily* family = nullptr; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_COUNTER; const char* name = "dupe_metric_test"; const char* description = "dupe metric description"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family1"); // Create metric TRITONSERVER_Metric* metric1 = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric1, family, labels.data(), labels.size()), "Creating new metric"); // Create duplicate metric TRITONSERVER_Metric* metric2 = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric2, family, labels.data(), labels.size()), "Creating new metric"); // Verify dupe metrics reference same underlying metric double value1 = -1; double value2 = -1; double inc = 7.5; // Verify initial values of zero FAIL_TEST_IF_ERR( TRITONSERVER_MetricValue(metric1, &value1), "query metric value after increment"); ASSERT_EQ(value1, 0); FAIL_TEST_IF_ERR( TRITONSERVER_MetricValue(metric2, &value2), "query metric value after increment"); ASSERT_EQ(value2, 0); // Increment metric 1, check metric 2 == metric 1 FAIL_TEST_IF_ERR( TRITONSERVER_MetricIncrement(metric1, inc), "increase metric value"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricValue(metric1, &value1), "query metric value after increment"); ASSERT_EQ(value1, inc); FAIL_TEST_IF_ERR( TRITONSERVER_MetricValue(metric2, &value2), "query metric value after increment"); ASSERT_EQ(value1, value2); std::cout << "metric1 value: " << value1 << " == metric2 value: " << value2 << std::endl; // Assert custom metric/family remains when there's still a reference to it FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric1), "delete metric1"); ASSERT_EQ(NumMetricMatches(server, description), 1); // Assert custom metric/family not displayed after all metrics are deleted FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric2), "delete metric2"); ASSERT_EQ(NumMetricMatches(server, description), 0); FAIL_TEST_IF_ERR(TRITONSERVER_MetricFamilyDelete(family), "delete family"); } void MetricAPIHelper(TRITONSERVER_Metric* metric, TRITONSERVER_MetricKind kind) { double value = -1; double prev_value = -1; FAIL_TEST_IF_ERR( TRITONSERVER_MetricValue(metric, &value), "query metric initial value"); // Value should be zero initially ASSERT_EQ(value, 0.0); // Increment positively double increment = 1729.0; prev_value = value; FAIL_TEST_IF_ERR( TRITONSERVER_MetricIncrement(metric, increment), "increase metric value"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricValue(metric, &value), "query metric value after positive increment"); ASSERT_EQ(value, prev_value + increment); // Increment negatively double decrement = -3.14; prev_value = value; auto err = TRITONSERVER_MetricIncrement(metric, decrement); switch (kind) { case TRITONSERVER_METRIC_KIND_COUNTER: { ASSERT_NE(err, nullptr); break; } case TRITONSERVER_METRIC_KIND_GAUGE: { ASSERT_EQ(err, nullptr); FAIL_TEST_IF_ERR( TRITONSERVER_MetricValue(metric, &value), "query metric value after negative increment"); ASSERT_EQ(value, prev_value + decrement); break; } default: ASSERT_TRUE(false); break; } // Set double set_value = 42.0; err = TRITONSERVER_MetricSet(metric, set_value); switch (kind) { case TRITONSERVER_METRIC_KIND_COUNTER: { ASSERT_NE(err, nullptr); break; } case TRITONSERVER_METRIC_KIND_GAUGE: { ASSERT_EQ(err, nullptr); FAIL_TEST_IF_ERR( TRITONSERVER_MetricValue(metric, &value), "query metric value after set"); ASSERT_EQ(value, set_value); break; } default: ASSERT_TRUE(false); break; } // MetricKind TRITONSERVER_MetricKind kind_tmp; FAIL_TEST_IF_ERR( TRITONSERVER_GetMetricKind(metric, &kind_tmp), "query metric kind"); ASSERT_EQ(kind_tmp, kind); TRITONSERVER_ErrorDelete(err); } // Calculate the cumulative_counts vector based on data and buckets. std::vector GetCumulativeCounts( const std::vector& data, const std::vector& buckets) { std::vector cumulative_counts(buckets.size() + 1, 0); for (double datum : data) { int i = 0; for (; i < buckets.size(); ++i) { if (datum <= buckets[i]) { cumulative_counts[i]++; break; } } if (i == buckets.size()) { cumulative_counts[i]++; } } for (int i = 1; i < cumulative_counts.size(); ++i) { cumulative_counts[i] += cumulative_counts[i - 1]; } return cumulative_counts; } void HistogramAPIHelper( TRITONSERVER_Server* server, TRITONSERVER_Metric* metric, std::vector buckets, std::string metric_name, std::string labels_str) { // Observe data std::vector data{0.05, 1.5, 6.0}; double sum = 0.0; for (auto datum : data) { FAIL_TEST_IF_ERR( TRITONSERVER_MetricObserve(metric, datum), "observe metric value"); sum += datum; } std::vector cumulative_counts = GetCumulativeCounts(data, buckets); // Collect formatted output std::string metrics_str; GetMetrics(server, &metrics_str); // All strings in this function are generated from ostringstream to make sure // there is no trailing zeros. For example, std::to_string(7.55) is "7.550000" // which is inconsistent with prometheus text_serializer output "7.55". // custom_histogram_example_count{example1="histogram_label1",example2="histogram_label2"} // 3 std::ostringstream count_ss; count_ss << metric_name << "_count{" << labels_str << "} " << data.size(); ASSERT_EQ(NumMetricMatches(server, count_ss.str()), 1); // custom_histogram_example_sum{example1="histogram_label1",example2="histogram_label2"} // 7.55 std::ostringstream sum_ss; sum_ss << metric_name << "_sum{" << labels_str << "} " << sum; ASSERT_EQ(NumMetricMatches(server, sum_ss.str()), 1); // custom_histogram_example_bucket{example1="histogram_label1",example2="histogram_label2" std::ostringstream bucket_partial_ss; bucket_partial_ss << metric_name << "_bucket{" << labels_str; ASSERT_EQ( NumMetricMatches(server, bucket_partial_ss.str()), cumulative_counts.size()); for (uint64_t i = 0; i < cumulative_counts.size(); ++i) { // custom_histogram_example_bucket{example1="histogram_label1",example2="histogram_label2",le="0.1"} // 1 std::ostringstream le_ss; if (i < buckets.size()) { le_ss << "\"" << buckets[i] << "\""; } else { le_ss << "\"+Inf\""; } std::ostringstream bucket_ss; bucket_ss << metric_name << "_bucket{" << labels_str << ",le=" << le_ss.str() << "} " << cumulative_counts[i]; ASSERT_EQ(NumMetricMatches(server, bucket_ss.str()), 1); } } // Test Fixture class MetricsApiTest : public ::testing::Test { protected: // Run only once before entire set of tests static void SetUpTestSuite() {} // Run only once after entire set of tests static void TearDownTestSuite() {} // Run before each test void SetUp() override { // Create server object to pass when retrieving metrics. // NOTE: It is currently not required to pass a valid server object to // TRITONSERVER_ServerMetrics, but is more future-proof to include. TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); // Mute info output for the sake of this test, less output FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetLogInfo(server_options, false), "disabling log INFO for brevity"); // This test doesn't require the use of any models, so we use "." as repo // and set ModelControlMode to EXPLICIT to avoid attempting to load models FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath(server_options, "."), "setting model repository path"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelControlMode( server_options, TRITONSERVER_MODEL_CONTROL_EXPLICIT), "setting model control mode"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerNew(&server_, server_options), "creating server"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); } // Run after each test void TearDown() override { FAIL_TEST_IF_ERR(TRITONSERVER_ServerDelete(server_), "deleting server"); } static TRITONSERVER_Server* server_; }; TRITONSERVER_Server* MetricsApiTest::server_ = nullptr; // Test end-to-end flow of Generic Metrics API for Counter metric TEST_F(MetricsApiTest, TestCounterEndToEnd) { // Create metric family TRITONSERVER_MetricFamily* family; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_COUNTER; const char* name = "custom_counter_example"; const char* description = "this is an example counter metric added via API."; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // Create metric TRITONSERVER_Metric* metric; std::vector labels; labels.emplace_back(TRITONSERVER_ParameterNew( "example1", TRITONSERVER_PARAMETER_STRING, "counter_label1")); labels.emplace_back(TRITONSERVER_ParameterNew( "example2", TRITONSERVER_PARAMETER_STRING, "counter_label2")); FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric, family, labels.data(), labels.size()), "Creating new metric"); for (const auto label : labels) { TRITONSERVER_ParameterDelete(const_cast(label)); } // Run through metric APIs and assert correctness MetricAPIHelper(metric, kind); // Assert custom metric is reported and found in output ASSERT_EQ(NumMetricMatches(server_, description), 1); // Cleanup FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric), "delete metric"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyDelete(family), "delete metric family"); // Assert custom metric/family is unregistered and no longer in output ASSERT_EQ(NumMetricMatches(server_, description), 0); } // Test end-to-end flow of Generic Metrics API for Gauge metric TEST_F(MetricsApiTest, TestGaugeEndToEnd) { // Create metric family TRITONSERVER_MetricFamily* family; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_GAUGE; const char* name = "custom_gauge_example"; const char* description = "this is an example gauge metric added via API."; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // Create metric TRITONSERVER_Metric* metric; std::vector labels; labels.emplace_back(TRITONSERVER_ParameterNew( "example1", TRITONSERVER_PARAMETER_STRING, "gauge_label1")); labels.emplace_back(TRITONSERVER_ParameterNew( "example2", TRITONSERVER_PARAMETER_STRING, "gauge_label2")); FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric, family, labels.data(), labels.size()), "Creating new metric"); for (const auto label : labels) { TRITONSERVER_ParameterDelete(const_cast(label)); } // Run through metric APIs and assert correctness MetricAPIHelper(metric, kind); // Assert custom metric is reported and found in output ASSERT_EQ(NumMetricMatches(server_, description), 1); // Cleanup FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric), "delete metric"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyDelete(family), "delete metric family"); // Assert custom metric/family is unregistered and no longer in output ASSERT_EQ(NumMetricMatches(server_, description), 0); } // Test end-to-end flow of Generic Metrics API for Histogram metric TEST_F(MetricsApiTest, TestHistogramEndToEnd) { // Create metric family TRITONSERVER_MetricFamily* family; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_HISTOGRAM; const char* name = "custom_histogram_example"; const char* description = "this is an example histogram metric added via API."; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // Create metric TRITONSERVER_Metric* metric; // Create labels std::vector labels; labels.emplace_back(TRITONSERVER_ParameterNew( "example1", TRITONSERVER_PARAMETER_STRING, "histogram_label1")); labels.emplace_back(TRITONSERVER_ParameterNew( "example2", TRITONSERVER_PARAMETER_STRING, "histogram_label2")); // Create metric args std::vector buckets = {0.1, 1.0, 2.5, 5.0, 10.0}; TRITONSERVER_MetricArgs* args; FAIL_TEST_IF_ERR( TRITONSERVER_MetricArgsNew(&args), "Creating new metric args"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricArgsSetHistogram(args, buckets.data(), buckets.size()), "setting histogram metric args"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricNewWithArgs( &metric, family, labels.data(), labels.size(), args), "Creating new metric"); for (const auto label : labels) { TRITONSERVER_ParameterDelete(const_cast(label)); } FAIL_TEST_IF_ERR(TRITONSERVER_MetricArgsDelete(args), "delete metric args"); // Run through metric APIs and assert correctness std::string labels_str = "example1=\"histogram_label1\",example2=\"histogram_label2\""; HistogramAPIHelper(server_, metric, buckets, name, labels_str); // Assert custom metric is reported and found in output ASSERT_EQ(NumMetricMatches(server_, description), 1); // Cleanup FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric), "delete metric"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyDelete(family), "delete metric family"); // Assert custom metric/family is unregistered and no longer in output ASSERT_EQ(NumMetricMatches(server_, description), 0); } // Test create a histogram metric without creating metric arguments TEST_F(MetricsApiTest, TestHistogramNoMetricArgs) { // Create metric family TRITONSERVER_MetricFamily* family; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_HISTOGRAM; const char* name = "no_metric_args"; const char* description = "no metric args description"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // MetricArgs not created TRITONSERVER_MetricArgs* args = nullptr; // Create metric std::vector labels; TRITONSERVER_Metric* metric = nullptr; auto err = TRITONSERVER_MetricNewWithArgs( &metric, family, labels.data(), labels.size(), args); EXPECT_THAT( TRITONSERVER_ErrorMessage(err), HasSubstr("Bucket boundaries not found in Metric args")); // Cleanup FAIL_TEST_IF_ERR(TRITONSERVER_MetricArgsDelete(args), "delete metric args"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyDelete(family), "delete metric family"); } // Test create a histogram metric without setting metric arguments TEST_F(MetricsApiTest, TestHistogramMetricArgsNotset) { // Create metric family TRITONSERVER_MetricFamily* family; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_HISTOGRAM; const char* name = "metric_args_not_set"; const char* description = "metric args not set description"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // Create metric args object without setting it TRITONSERVER_MetricArgs* args; FAIL_TEST_IF_ERR( TRITONSERVER_MetricArgsNew(&args), "Creating new metric args"); // Create metric std::vector labels; TRITONSERVER_Metric* metric = nullptr; auto err = TRITONSERVER_MetricNewWithArgs( &metric, family, labels.data(), labels.size(), args); EXPECT_THAT( TRITONSERVER_ErrorMessage(err), HasSubstr("Metric args not set to histogram kind")); // Cleanup FAIL_TEST_IF_ERR(TRITONSERVER_MetricArgsDelete(args), "delete metric args"); FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyDelete(family), "delete metric family"); } // Test that a duplicate metric family can't be added // with a conflicting type/kind TEST_F(MetricsApiTest, TestDupeMetricFamilyDiffKind) { // Create metric family TRITONSERVER_MetricFamily* family1 = nullptr; TRITONSERVER_MetricKind kind1 = TRITONSERVER_METRIC_KIND_COUNTER; const char* name = "diff_kind_test"; const char* description = "diff kind description"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family1, kind1, name, description), "Creating new metric family1"); // Create duplicate metric family with different kind TRITONSERVER_MetricFamily* family2 = nullptr; TRITONSERVER_MetricKind kind2 = TRITONSERVER_METRIC_KIND_GAUGE; // Expect this to fail, can't have duplicate name of different kind auto err = TRITONSERVER_MetricFamilyNew(&family2, kind2, name, description); ASSERT_NE(err, nullptr); ASSERT_EQ(family2, nullptr); TRITONSERVER_ErrorDelete(err); } // Test that a duplicate metric family name will still // return the original metric family even if the description // is changed TEST_F(MetricsApiTest, TestDupeMetricFamilyDiffDescription) { // Create metric family TRITONSERVER_MetricFamily* family1 = nullptr; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_COUNTER; const char* name = "diff_description_test"; const char* description1 = "first description"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family1, kind, name, description1), "Creating new metric family1"); // Create duplicate metric family TRITONSERVER_MetricFamily* family2 = nullptr; const char* description2 = "second description"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family2, kind, name, description2), "Creating new metric family2"); // Assert MetricFamily is not reported until metrics are added to them ASSERT_EQ(NumMetricMatches(server_, description1), 0); ASSERT_EQ(NumMetricMatches(server_, description2), 0); // Add metric to family2 only, this will be shared by family1 as well // since both families refer to the same underlying prometheus family std::vector labels; TRITONSERVER_Metric* metric2 = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric2, family2, labels.data(), labels.size()), "Creating new metric2"); // Assert MetricFamily is reported exactly once // This confirms attempting to add a duplicate returns the existing family ASSERT_EQ(NumMetricMatches(server_, description1), 1); // The first description will be taken/kept if adding a duplicate /// metric family name, even with a different description ASSERT_EQ(NumMetricMatches(server_, description2), 0); // Delete one of the metric family references // Specificailly, family1, because family2 is bound to metric2 FAIL_TEST_IF_ERR(TRITONSERVER_MetricFamilyDelete(family1), "delete family1"); // Assert custom metric/family remains when family2 still references it ASSERT_EQ(NumMetricMatches(server_, description1), 1); // Assert custom metric/family unregistered after last reference deleted FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric2), "delete metric2"); FAIL_TEST_IF_ERR(TRITONSERVER_MetricFamilyDelete(family2), "delete family2"); ASSERT_EQ(NumMetricMatches(server_, description1), 0); ASSERT_EQ(NumMetricMatches(server_, description2), 0); } // Test that adding a duplicate metric family will reuse the original // and not add another entry to registry TEST_F(MetricsApiTest, TestDupeMetricFamily) { // Create metric family TRITONSERVER_MetricFamily* family1 = nullptr; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_COUNTER; const char* name = "dupe_metric_family_test"; const char* description = "dupe metric family description"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family1, kind, name, description), "Creating new metric family1"); // Create duplicate metric family TRITONSERVER_MetricFamily* family2 = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family2, kind, name, description), "Creating new metric family2"); // Assert MetricFamily is not reported until metrics are added to them ASSERT_EQ(NumMetricMatches(server_, description), 0); // Create unique metrics for each family object. Both family objects // will refer to the same prometheus family in the registry, so both // metrics should be displayed under the family. const char* metric_key = "custom_metric_key"; std::vector labels1; labels1.emplace_back(TRITONSERVER_ParameterNew( metric_key, TRITONSERVER_PARAMETER_STRING, "label1")); TRITONSERVER_Metric* metric1 = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric1, family1, labels1.data(), labels1.size()), "Creating new metric1"); for (const auto label : labels1) { TRITONSERVER_ParameterDelete(const_cast(label)); } std::vector labels2; labels2.emplace_back(TRITONSERVER_ParameterNew( metric_key, TRITONSERVER_PARAMETER_STRING, "label2")); TRITONSERVER_Metric* metric2 = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric2, family2, labels2.data(), labels2.size()), "Creating new metric2"); for (const auto label : labels2) { TRITONSERVER_ParameterDelete(const_cast(label)); } // Assert MetricFamily is reported exactly once // This confirms attempting to add a duplicate returns the existing family ASSERT_EQ(NumMetricMatches(server_, description), 1); // Assert we have two unique metrics ASSERT_EQ(NumMetricMatches(server_, metric_key), 2); // Delete one of the metric family references FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric1), "delete metric1"); FAIL_TEST_IF_ERR(TRITONSERVER_MetricFamilyDelete(family1), "delete family1"); // Assert custom family remains when there's still a reference to it ASSERT_EQ(NumMetricMatches(server_, description), 1); // Assert only one remaining metric after deleting one ASSERT_EQ(NumMetricMatches(server_, metric_key), 1); // Assert custom metric/family unregistered after last reference deleted FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric2), "delete metric2"); FAIL_TEST_IF_ERR(TRITONSERVER_MetricFamilyDelete(family2), "delete family2"); ASSERT_EQ(NumMetricMatches(server_, description), 0); // Assert no remaining metrics after deleting both ASSERT_EQ(NumMetricMatches(server_, metric_key), 0); } // Test that adding a duplicate metric will refer to the same // underlying metric, and all instances will be updated TEST_F(MetricsApiTest, TestDupeMetricLabels) { std::vector labels; labels.emplace_back(TRITONSERVER_ParameterNew( "example1", TRITONSERVER_PARAMETER_STRING, "label1")); labels.emplace_back(TRITONSERVER_ParameterNew( "example2", TRITONSERVER_PARAMETER_STRING, "label2")); DupeMetricHelper(server_, labels); for (const auto label : labels) { TRITONSERVER_ParameterDelete(const_cast(label)); } } // Test that adding a duplicate metric will refer to the same // underlying metric, and all instances will be updated TEST_F(MetricsApiTest, TestDupeMetricEmptyLabels) { std::vector labels; DupeMetricHelper(server_, labels); } TEST_F(MetricsApiTest, TestOutOfOrderDelete) { // Create metric family TRITONSERVER_MetricFamily* family = nullptr; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_COUNTER; const char* name = "out_of_order_delete"; const char* description = "out of order delete test"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // Add metric to family std::vector labels; TRITONSERVER_Metric* metric = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric, family, labels.data(), labels.size()), "Creating new metric"); // Check that deleting metric family BEFORE metric fails auto err = TRITONSERVER_MetricFamilyDelete(family); EXPECT_THAT( TRITONSERVER_ErrorMessage(err), HasSubstr("Must call MetricDelete")); // Check that deleting in correct order still works after above failure FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric), "deleting metric"); FAIL_TEST_IF_ERR(TRITONSERVER_MetricFamilyDelete(family), "deleting family"); TRITONSERVER_ErrorDelete(err); } TEST_F(MetricsApiTest, TestMetricAfterFamilyDelete) { // Create metric family TRITONSERVER_MetricFamily* family = nullptr; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_GAUGE; const char* name = "use_metric_after_family_delete"; const char* description = "test using a metric after its family is deleted"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // Add metric to family std::vector labels; TRITONSERVER_Metric* metric = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric, family, labels.data(), labels.size()), "Creating new metric"); // Check that deleting metric family BEFORE metric fails auto err = TRITONSERVER_MetricFamilyDelete(family); EXPECT_THAT( TRITONSERVER_ErrorMessage(err), HasSubstr("Must call MetricDelete")); // Use internal implementation to force deletion since C API checks first // NOTE: This is for internal testing and should NOT be done by users. delete reinterpret_cast(family); // Expected API calls to fail since metric has been invalidated by // calling MetricFamilyDelete before MetricDelete double value = -1; err = TRITONSERVER_MetricValue(metric, &value); EXPECT_THAT(TRITONSERVER_ErrorMessage(err), HasSubstr("invalidated")); err = TRITONSERVER_MetricIncrement(metric, 1.0); EXPECT_THAT(TRITONSERVER_ErrorMessage(err), HasSubstr("invalidated")); err = TRITONSERVER_MetricSet(metric, 1.0); EXPECT_THAT(TRITONSERVER_ErrorMessage(err), HasSubstr("invalidated")); TRITONSERVER_ErrorDelete(err); } // This test serves as a reminder to consider the ability to access // internal core metrics via current metrics API and its implications. TEST_F(MetricsApiTest, TestCoreMetricAccess) { // Test accessing a metric family created in Triton Core // through prometheus directly. Technically this metric can be // updated manually by a user in addition to how the core manages // the metric, but this should generally not be done. TRITONSERVER_MetricFamily* family = nullptr; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_GAUGE; // Pick existing core metric name here. const char* name = "nv_gpu_power_limit"; const char* description = ""; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // DLIS-4072: If registry->Remove() is implemented in MetricFamily we will // we will probably want to make sure core metrics can not be deleted early. FAIL_TEST_IF_ERR(TRITONSERVER_MetricFamilyDelete(family), "delete family"); } TEST_F(MetricsApiTest, TestChildMetricTracking) { // Create metric family TRITONSERVER_MetricFamily* family = nullptr; TRITONSERVER_MetricKind kind = TRITONSERVER_METRIC_KIND_GAUGE; const char* name = "test_ref_counting"; const char* description = "test using metric ref counting"; FAIL_TEST_IF_ERR( TRITONSERVER_MetricFamilyNew(&family, kind, name, description), "Creating new metric family"); // Use internal implementation to verify correctness auto tc_family = reinterpret_cast(family); // Create metric TRITONSERVER_Metric* metric1 = nullptr; std::vector labels; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric1, family, labels.data(), labels.size()), "Creating new metric1"); ASSERT_EQ(tc_family->NumMetrics(), 1); // Create duplicate metric TRITONSERVER_Metric* metric2 = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_MetricNew(&metric2, family, labels.data(), labels.size()), "Creating new metric2"); ASSERT_EQ(tc_family->NumMetrics(), 2); FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric1), "delete metric1"); ASSERT_EQ(tc_family->NumMetrics(), 1); FAIL_TEST_IF_ERR(TRITONSERVER_MetricDelete(metric2), "delete metric2"); ASSERT_EQ(tc_family->NumMetrics(), 0); FAIL_TEST_IF_ERR(TRITONSERVER_MetricFamilyDelete(family), "delete family"); } } // namespace int main(int argc, char** argv) { #ifdef TRITON_ENABLE_LOGGING LOG_SET_VERBOSE(1); #endif // TRITON_ENABLE_LOGGING ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } #endif // TRITON_ENABLE_METRICS kytensor-2.49.0.7/core/src/test/pinned_memory_manager_test.cc000066400000000000000000000311001527052371200242330ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pinned_memory_manager.h" #include #include #include #include #include #include "gtest/gtest.h" #include "tritonserver_apis.h" namespace tc = triton::core; namespace { #define CHECK_POINTER_ATTRIBUTES(ptr__, type__, device__) \ do { \ cudaPointerAttributes attr; \ auto cuerr = cudaPointerGetAttributes(&attr, ptr__); \ ASSERT_TRUE(cuerr == cudaSuccess) \ << "Failed to get CUDA pointer attributes: " \ << cudaGetErrorString(cuerr); \ EXPECT_TRUE(attr.type == type__) \ << "Expect pointer with type " << type__ << ", got: " << attr.type; \ if (attr.type == cudaMemoryTypeDevice) { \ EXPECT_TRUE(attr.device == device__) \ << "Expect allocation on CUDA device " << device__ \ << ", got: " << attr.device; \ } \ } while (false) #define STORE_RESULT_AND_RETURN_IF_ERROR(metadata__, idx__, status__) \ do { \ if (!status__.IsOk()) { \ std::lock_guard lk(metadata__->mtx_); \ metadata__->results_[idx__] = status__.AsString(); \ return; \ } \ } while (false) struct MemoryWorkMetadata { MemoryWorkMetadata(size_t thread_count) : thread_count_(thread_count), ready_count_(0), results_(thread_count, "") { } size_t thread_count_; size_t ready_count_; std::vector results_; std::mutex mtx_; std::condition_variable cv_; }; void RunMemoryWork( size_t idx, size_t alloc_size, bool allow_nonpinned_fallback, MemoryWorkMetadata* metadata) { // Prepare variable to hold input / output std::unique_ptr input(new char[alloc_size]); std::unique_ptr output(new char[alloc_size]); // Wait until all threads are issued { std::unique_lock lk(metadata->mtx_); metadata->ready_count_++; if (metadata->ready_count_ != metadata->thread_count_) { while (metadata->ready_count_ != metadata->thread_count_) { metadata->cv_.wait(lk); } } metadata->cv_.notify_one(); } // Simulate receive input data -> alloc and write to input buffer // -> alloc and write to output buffer -> return output data TRITONSERVER_MemoryType allocated_type = TRITONSERVER_MEMORY_GPU; void* input_buffer = nullptr; STORE_RESULT_AND_RETURN_IF_ERROR( metadata, idx, tc::PinnedMemoryManager::Alloc( &input_buffer, alloc_size, &allocated_type, allow_nonpinned_fallback)); if ((!allow_nonpinned_fallback) && (allocated_type != TRITONSERVER_MEMORY_CPU_PINNED)) { tc::Status status( tc::Status::Code::INVALID_ARG, "returned memory buffer is not pinned"); STORE_RESULT_AND_RETURN_IF_ERROR(metadata, idx, status); } memcpy(input_buffer, input.get(), alloc_size); void* output_buffer = nullptr; STORE_RESULT_AND_RETURN_IF_ERROR( metadata, idx, tc::PinnedMemoryManager::Alloc( &output_buffer, alloc_size, &allocated_type, allow_nonpinned_fallback)); if ((!allow_nonpinned_fallback) && (allocated_type != TRITONSERVER_MEMORY_CPU_PINNED)) { tc::Status status( tc::Status::Code::INVALID_ARG, "returned memory buffer is not pinned"); STORE_RESULT_AND_RETURN_IF_ERROR(metadata, idx, status); } memcpy(output_buffer, input_buffer, alloc_size); memcpy(output.get(), output_buffer, alloc_size); for (size_t offset = 0; offset < alloc_size; offset++) { if (input.get()[offset] != output.get()[offset]) { std::lock_guard lk(metadata->mtx_); metadata->results_[idx] = std::string("mismatch between input and output for work idx ") + std::to_string(idx); return; } } } // Wrapper of PinnedMemoryManager class to expose Reset() for unit testing class TestingPinnedMemoryManager : public tc::PinnedMemoryManager { public: static void Reset() { PinnedMemoryManager::Reset(); } }; class PinnedMemoryManagerTest : public ::testing::Test { protected: void SetUp() override { // Default memory manager options options_.pinned_memory_pool_byte_size_ = 1 << 10; } void TearDown() override { TestingPinnedMemoryManager::Reset(); } tc::PinnedMemoryManager::Options options_; }; TEST_F(PinnedMemoryManagerTest, InitOOM) { // Set to reserve too much memory options_.pinned_memory_pool_byte_size_ = uint64_t(1) << 40 /* 1024 GB */; auto status = tc::PinnedMemoryManager::Create(options_); // For pinned memory manager, it will still be created for "CPU fallback" // allocation even if it fails to create pinned memory pool EXPECT_TRUE(status.IsOk()) << status.Message(); } TEST_F(PinnedMemoryManagerTest, InitSuccess) { auto status = tc::PinnedMemoryManager::Create(options_); EXPECT_TRUE(status.IsOk()) << status.Message(); } TEST_F(PinnedMemoryManagerTest, InitZeroByte) { options_.pinned_memory_pool_byte_size_ = 0; auto status = tc::PinnedMemoryManager::Create(options_); EXPECT_TRUE(status.IsOk()) << status.Message(); void* ptr = nullptr; TRITONSERVER_MemoryType allocated_type = TRITONSERVER_MEMORY_GPU; status = tc::PinnedMemoryManager::Alloc( &ptr, 1, &allocated_type, false /* allow_nonpinned_fallback */); ASSERT_FALSE(status.IsOk()) << "Unexpected successful allocation"; } TEST_F(PinnedMemoryManagerTest, AllocSuccess) { auto status = tc::PinnedMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); void* ptr = nullptr; TRITONSERVER_MemoryType allocated_type = TRITONSERVER_MEMORY_GPU; status = tc::PinnedMemoryManager::Alloc( &ptr, 512, &allocated_type, false /* allow_nonpinned_fallback */); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(ptr) << "Expect pointer to allocated buffer"; ASSERT_TRUE(allocated_type == TRITONSERVER_MEMORY_CPU_PINNED) << "Expect pointer to pinned memory"; // check if returned pointer is pinned memory pointer CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeHost, 0); } TEST_F(PinnedMemoryManagerTest, AllocFallbackSuccess) { auto status = tc::PinnedMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); void* ptr = nullptr; TRITONSERVER_MemoryType allocated_type = TRITONSERVER_MEMORY_GPU; status = tc::PinnedMemoryManager::Alloc( &ptr, 2048, &allocated_type, true /* allow_nonpinned_fallback */); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(ptr) << "Expect pointer to allocated buffer"; ASSERT_TRUE(allocated_type == TRITONSERVER_MEMORY_CPU) << "Expect pointer to non-pinned memory"; // check if returned pointer is non-pinned memory pointer CHECK_POINTER_ATTRIBUTES(ptr, cudaMemoryTypeUnregistered, 0); } TEST_F(PinnedMemoryManagerTest, AllocFail) { auto status = tc::PinnedMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); void* ptr = nullptr; TRITONSERVER_MemoryType allocated_type = TRITONSERVER_MEMORY_GPU; status = tc::PinnedMemoryManager::Alloc( &ptr, 2048, &allocated_type, false /* allow_nonpinned_fallback */); ASSERT_FALSE(status.IsOk()) << "Unexpected successful allocation"; } TEST_F(PinnedMemoryManagerTest, MultipleAlloc) { auto status = tc::PinnedMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); void* first_ptr = nullptr; TRITONSERVER_MemoryType allocated_type = TRITONSERVER_MEMORY_GPU; status = tc::PinnedMemoryManager::Alloc( &first_ptr, 600, &allocated_type, false /* allow_nonpinned_fallback */); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(first_ptr) << "Expect pointer to allocated buffer"; ASSERT_TRUE(allocated_type == TRITONSERVER_MEMORY_CPU_PINNED) << "Expect pointer to pinned memory"; // check if returned pointer is pinned memory pointer CHECK_POINTER_ATTRIBUTES(first_ptr, cudaMemoryTypeHost, 0); // 512 + 600 > 1024 void* second_ptr = nullptr; status = tc::PinnedMemoryManager::Alloc( &second_ptr, 512, &allocated_type, false /* allow_nonpinned_fallback */); ASSERT_FALSE(status.IsOk()) << "Unexpected successful allocation"; // Free the first pointer and retry the second one status = tc::PinnedMemoryManager::Free(first_ptr); EXPECT_TRUE(status.IsOk()) << status.Message(); status = tc::PinnedMemoryManager::Alloc( &second_ptr, 512, &allocated_type, false /* allow_nonpinned_fallback */); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(second_ptr) << "Expect pointer to allocated buffer"; ASSERT_TRUE(allocated_type == TRITONSERVER_MEMORY_CPU_PINNED) << "Expect pointer to pinned memory"; // check if returned pointer is pinned memory pointer CHECK_POINTER_ATTRIBUTES(second_ptr, cudaMemoryTypeHost, 0); } TEST_F(PinnedMemoryManagerTest, ParallelAlloc) { options_.pinned_memory_pool_byte_size_ = uint64_t(1) << 28 /* 256 MB */; auto status = tc::PinnedMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); // Create threads to perform operations on allocated memory in parallel // Seems like for 1 MB alloc size (2 MB for both input and output), // 100 threads is a good amount for pool manager not to use CPU fallback. size_t thread_count = 100; size_t allocated_size = 1 << 20 /* 1 MB */; MemoryWorkMetadata metadata(thread_count); std::vector threads; for (size_t idx = 0; idx < thread_count; idx++) { threads.emplace_back( std::thread(RunMemoryWork, idx, allocated_size, false, &metadata)); } for (size_t idx = 0; idx < thread_count; idx++) { threads[idx].join(); EXPECT_TRUE(metadata.results_[idx].empty()) << metadata.results_[idx]; } } TEST_F(PinnedMemoryManagerTest, ParallelAllocFallback) { options_.pinned_memory_pool_byte_size_ = uint64_t(1) << 28 /* 256 MB */; auto status = tc::PinnedMemoryManager::Create(options_); ASSERT_TRUE(status.IsOk()) << status.Message(); // Create threads to perform operations on allocated memory in parallel size_t thread_count = 128; size_t allocated_size = 1 << 24 /* 4 MB */; MemoryWorkMetadata metadata(thread_count); std::vector threads; for (size_t idx = 0; idx < thread_count; idx++) { threads.emplace_back( std::thread(RunMemoryWork, idx, allocated_size, true, &metadata)); } for (size_t idx = 0; idx < thread_count; idx++) { threads[idx].join(); EXPECT_TRUE(metadata.results_[idx].empty()) << metadata.results_[idx]; } } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/query_test.cc000066400000000000000000000316741527052371200210610ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include #include #include #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "triton/core/tritonserver.h" namespace { using ::testing::HasSubstr; #define FAIL_TEST_IF_ERR(X, MSG) \ do { \ std::shared_ptr err__((X), TRITONSERVER_ErrorDelete); \ ASSERT_TRUE((err__ == nullptr)) \ << "error: " << (MSG) << ": " \ << TRITONSERVER_ErrorCodeString(err__.get()) << " - " \ << TRITONSERVER_ErrorMessage(err__.get()); \ } while (false) using NameMap = std::map>; struct QueryTracker { QueryTracker( const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) : has_name_(tensor_name != nullptr), has_byte_size_(byte_size != nullptr), caller_preferred_type_(memory_type), caller_preferred_id_(memory_type_id) { if (has_name_) { name_ = tensor_name; } if (has_byte_size_) { byte_size_ = *byte_size; } } bool has_name_; bool has_byte_size_; std::string name_; size_t byte_size_; TRITONSERVER_MemoryType caller_preferred_type_; int64_t caller_preferred_id_; }; TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { auto& output_tracker = (reinterpret_cast, NameMap>*>(userp) ->second); output_tracker.emplace( tensor_name, std::make_tuple( preferred_memory_type, preferred_memory_type_id, byte_size)); return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { return nullptr; // Success } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { TRITONSERVER_InferenceRequestDelete(request); } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { if (response != nullptr) { // Notify that the completion. std::promise* p = reinterpret_cast*>(userp); p->set_value(TRITONSERVER_InferenceResponseError(response)); } TRITONSERVER_InferenceResponseDelete(response); } class QueryTest : public ::testing::Test { protected: static void SetUpTestSuite() { // Create the server... TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, "./models"), "setting model repository path"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetBackendDirectory( server_options, "/opt/tritonserver/backends"), "setting backend directory"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetRepoAgentDirectory( server_options, "/opt/tritonserver/repoagents"), "setting repository agent directory"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), "setting strict model configuration"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerNew(&server_, server_options), "creating server"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); } static void TearDownTestSuite() { FAIL_TEST_IF_ERR(TRITONSERVER_ServerDelete(server_), "deleting server"); } void SetUp() override { ASSERT_TRUE(server_ != nullptr) << "Server has not created"; // Wait until the server is both live and ready. size_t health_iters = 0; while (true) { bool live, ready; FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsLive(server_, &live), "unable to get server liveness"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsReady(server_, &ready), "unable to get server readiness"); if (live && ready) { break; } if (++health_iters >= 10) { FAIL() << "failed to find healthy inference server"; } std::this_thread::sleep_for(std::chrono::milliseconds(500)); } // Create allocator with common callback FAIL_TEST_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator_, ResponseAlloc, ResponseRelease, nullptr /* start_fn */), "creating response allocator"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest_, server_, "query", -1 /* model_version */), "creating inference request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); std::vector shape{1}; FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT", TRITONSERVER_TYPE_UINT8, shape.data(), shape.size()), "setting input for the request"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest_, "INPUT", input_data_.data(), input_data_.size(), TRITONSERVER_MEMORY_CPU, 0), "assigning INPUT data"); FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, reinterpret_cast(&output_info_), InferResponseComplete, reinterpret_cast(&completed_)), "setting response callback"); } void TearDown() override { unsetenv("TEST_ANONYMOUS"); unsetenv("TEST_BYTE_SIZE"); FAIL_TEST_IF_ERR( TRITONSERVER_ResponseAllocatorDelete(allocator_), "deleting response allocator"); } static TRITONSERVER_Server* server_; TRITONSERVER_ResponseAllocator* allocator_ = nullptr; static std::vector input_data_; TRITONSERVER_InferenceRequest* irequest_ = nullptr; std::promise completed_; std::pair, NameMap> output_info_; }; TRITONSERVER_Server* QueryTest::server_ = nullptr; std::vector QueryTest::input_data_{1}; TEST_F(QueryTest, DefaultQuery) { TRITONSERVER_ResponseAllocatorQueryFn_t query_fn = [](TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) -> TRITONSERVER_Error* { auto& query_tracker = (reinterpret_cast, NameMap>*>(userp) ->first); query_tracker.emplace_back( tensor_name, byte_size, *memory_type, *memory_type_id); *memory_type = TRITONSERVER_MEMORY_CPU; *memory_type_id = 0; return nullptr; }; FAIL_TEST_IF_ERR( TRITONSERVER_ResponseAllocatorSetQueryFunction(allocator_, query_fn), "setting response callback"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "running inference"); auto err = completed_.get_future().get(); ASSERT_TRUE(err == nullptr) << "Expect successful inference"; // Check query tracker to see if the query function is connected properly ASSERT_EQ(output_info_.first.size(), size_t(2)); for (size_t i = 0; i < output_info_.first.size(); ++i) { const auto& query_info = output_info_.first[i]; EXPECT_EQ(query_info.has_name_, true); EXPECT_EQ(query_info.name_, (std::string("OUTPUT") + std::to_string(i))); EXPECT_EQ(query_info.has_byte_size_, false); EXPECT_EQ( query_info.caller_preferred_type_, TRITONSERVER_MEMORY_CPU_PINNED); EXPECT_EQ(query_info.caller_preferred_id_, 1); } const auto& output_0 = output_info_.second["OUTPUT0"]; EXPECT_EQ(std::get<0>(output_0), TRITONSERVER_MEMORY_CPU); EXPECT_EQ(std::get<1>(output_0), int64_t(0)); EXPECT_EQ(std::get<2>(output_0), size_t(2)); const auto& output_1 = output_info_.second["OUTPUT1"]; EXPECT_EQ(std::get<0>(output_1), TRITONSERVER_MEMORY_CPU); EXPECT_EQ(std::get<1>(output_1), int64_t(0)); EXPECT_EQ(std::get<2>(output_1), size_t(2)); } TEST_F(QueryTest, NoQueryFn) { FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "running inference"); auto err = completed_.get_future().get(); ASSERT_TRUE(err != nullptr) << "Expect error"; EXPECT_EQ(TRITONSERVER_ErrorCode(err), TRITONSERVER_ERROR_UNAVAILABLE); EXPECT_THAT( TRITONSERVER_ErrorMessage(err), HasSubstr("Output properties are not available")); } TEST_F(QueryTest, UnnamedQuery) { setenv("TEST_ANONYMOUS", "", 1); setenv("TEST_BYTE_SIZE", "32", 1); TRITONSERVER_ResponseAllocatorQueryFn_t query_fn = [](TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) -> TRITONSERVER_Error* { auto& query_tracker = (reinterpret_cast, NameMap>*>(userp) ->first); query_tracker.emplace_back( tensor_name, byte_size, *memory_type, *memory_type_id); // Slightly different setting *memory_type = TRITONSERVER_MEMORY_GPU; *memory_type_id = 2; return nullptr; }; FAIL_TEST_IF_ERR( TRITONSERVER_ResponseAllocatorSetQueryFunction(allocator_, query_fn), "setting response callback"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */), "running inference"); auto err = completed_.get_future().get(); ASSERT_TRUE(err == nullptr) << "Expect successful inference"; // Check query tracker to see if the query function is connected properly ASSERT_EQ(output_info_.first.size(), size_t(1)); for (size_t i = 0; i < output_info_.first.size(); ++i) { const auto& query_info = output_info_.first[i]; EXPECT_EQ(query_info.has_name_, false); EXPECT_EQ(query_info.has_byte_size_, true); EXPECT_EQ(query_info.byte_size_, size_t(32)); EXPECT_EQ( query_info.caller_preferred_type_, TRITONSERVER_MEMORY_CPU_PINNED); EXPECT_EQ(query_info.caller_preferred_id_, 1); } const auto& output_0 = output_info_.second["OUTPUT0"]; EXPECT_EQ(std::get<0>(output_0), TRITONSERVER_MEMORY_GPU); EXPECT_EQ(std::get<1>(output_0), int64_t(2)); EXPECT_EQ(std::get<2>(output_0), size_t(16)); const auto& output_1 = output_info_.second["OUTPUT1"]; EXPECT_EQ(std::get<0>(output_1), TRITONSERVER_MEMORY_GPU); EXPECT_EQ(std::get<1>(output_1), int64_t(2)); EXPECT_EQ(std::get<2>(output_1), size_t(16)); } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/register_api_test.cc000066400000000000000000001060501527052371200223600ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "gtest/gtest.h" #include "triton/core/tritonserver.h" namespace { #define FAIL_TEST_IF_ERR(X, MSG) \ do { \ std::shared_ptr err__((X), TRITONSERVER_ErrorDelete); \ ASSERT_TRUE((err__ == nullptr)) \ << "error: " << (MSG) << ": " \ << TRITONSERVER_ErrorCodeString(err__.get()) << " - " \ << TRITONSERVER_ErrorMessage(err__.get()); \ } while (false) #define FAIL_TEST_IF_NOT_ERR(X, CODE, ERR_MSG, MSG) \ do { \ std::shared_ptr err__((X), TRITONSERVER_ErrorDelete); \ ASSERT_TRUE((err__ != nullptr)) << "expected error on: " << (MSG); \ if (err__ != nullptr) { \ EXPECT_EQ(TRITONSERVER_ErrorCode(err__.get()), (CODE)) << (MSG); \ EXPECT_STREQ(TRITONSERVER_ErrorMessage(err__.get()), (ERR_MSG)) \ << (MSG); \ } \ } while (false) // Test Fixture, this test suit expects the current directory to // have the following file structure: // - empty_models (empty directory) // - models_0 (contain model directory "model_0") // - models_1 (contain model directories "model_0", "model_1") // - models_2 (contain model directories "model_0" with config name // "mapped_name") class RegisterApiTest : public ::testing::Test { protected: void SetUp() override { // Create running server object. TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); // Triton expects at least one model repository is set at start, set to // an empty repository set ModelControlMode to EXPLICIT to avoid attempting // to load models. FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, "empty_models"), "setting model repository path"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelControlMode( server_options, TRITONSERVER_MODEL_CONTROL_EXPLICIT), "setting model control mode"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerNew(&server_, server_options), "creating server"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); ASSERT_TRUE(server_ != nullptr) << "server not created"; bool live = false; for (int i = 10; ((i > 0) && !live); --i) { FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsLive(server_, &live), "Is server live"); } ASSERT_TRUE(live) << "server not live"; } void TearDown() override { FAIL_TEST_IF_ERR(TRITONSERVER_ServerDelete(server_), "deleting server"); } TRITONSERVER_Server* server_ = nullptr; }; TEST_F(RegisterApiTest, Register) { // Request to load "model_0" which should fail FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), TRITONSERVER_ERROR_INTERNAL, "failed to load 'model_0', failed to poll from model repository", "loading model 'model_0'"); // Registering a repository "models_0" where contains "model_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", nullptr, 0), "registering model repository 'models_0'"); // Request to load "model_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), "loading model 'model_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_0", 1, &ready), "Is 'model_0' v1 ready"); ASSERT_TRUE(ready) << "Expect 'model_0' v1 to be ready, model directory is " "'models_0/model_0'"; } TEST_F(RegisterApiTest, RegisterWithMap) { // Registering a repository "models_0" where contains "model_0", but with // different name mapping const char* override_name = "name_0"; std::shared_ptr managed_param( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, override_name), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_param != nullptr) << "failed to create name mapping pair"; std::vector name_map{managed_param.get()}; FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", name_map.data(), name_map.size()), "registering model repository 'models_0'"); // Request to load "model_0" which should fail FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), TRITONSERVER_ERROR_INTERNAL, "failed to load 'model_0', failed to poll from model repository", "loading model 'model_0'"); // Request to load "name_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "name_0"), "loading model 'name_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "name_0", 1, &ready), "Is 'name_0' v1 ready"); ASSERT_TRUE(ready) << "Expect 'name_0' v1 to be ready, model directory is " "'models_0/model_0'"; } TEST_F(RegisterApiTest, RegisterTwice) { // Registering a startup repository FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "empty_models", nullptr, 0), TRITONSERVER_ERROR_ALREADY_EXISTS, "model repository 'empty_models' has already been registered", "registering model repository 'empty_models'"); } TEST_F(RegisterApiTest, RegisterTwice2) { // Registering the same repository twice FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", nullptr, 0), "registering model repository 'models_0'"); FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", nullptr, 0), TRITONSERVER_ERROR_ALREADY_EXISTS, "model repository 'models_0' has already been registered", "registering model repository 'models_0'"); } TEST_F(RegisterApiTest, RegisterWithMultiMap) { // Registering a repository "models_0" where contains "model_0", // and "model_0" is mapped to two different names std::vector override_names{"name_0", "name_1"}; std::vector> managed_params; std::vector name_map; for (const auto& name : override_names) { managed_params.emplace_back( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, name.c_str()), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_params.back() != nullptr) << "failed to create name mapping pair"; name_map.emplace_back(managed_params.back().get()); } // Such mapping should be allow as it is mapping to unique names FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", name_map.data(), name_map.size()), "registering model repository 'models_0'"); // Request to load "name_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "name_0"), "loading model 'name_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "name_0", 1, &ready), "Is 'name_0' v1 ready"); ASSERT_TRUE(ready) << "Expect 'name_0' v1 to be ready, model directory is " "'models_0/model_0'"; // Request to load "name_1" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "name_1"), "loading model 'name_1'"); ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "name_1", 1, &ready), "Is 'name_1' v1 ready"); ASSERT_TRUE(ready) << "Expect 'name_1' v1 to be ready, model directory is " "'models_0/model_0'"; } TEST_F(RegisterApiTest, RegisterWithRepeatedMap) { // Registering a repository "models_1" where contains "model_0" and "model_1", // map "model_0" to "model_1" which creates confliction, however, // in EXPLICIT mode, mapping lookup will have higher priority than // repository polling so the confliction will be resolved by always loading // the model from mapped directory. std::vector override_names{"model_1"}; std::vector> managed_params; std::vector name_map; managed_params.emplace_back( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, override_names[0].c_str()), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_params.back() != nullptr) << "failed to create name mapping pair"; name_map.emplace_back(managed_params.back().get()); // Such mapping should be allow as it is mapping to unique names FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_1", name_map.data(), name_map.size()), "registering model repository 'models_1'"); // Request to load "model_1" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_1"), "loading model 'model_1'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_1", 2, &ready), "Is 'model_1' ready"); ASSERT_TRUE(ready) << "Expect 'model_1' v2 to be ready, model directory is " "'models_1/model_0'"; } TEST_F(RegisterApiTest, RegisterWithRepeatedMap2) { // Registering a repository "models_1" where contains "model_0" and "model_1", // map both directories to the same name which creates confliction. Different // from 'RegisterWithRepeatedMap', the confliction within the mapping can't be // resolved and error should be returned std::vector dir_names{"model_0", "model_1"}; std::vector> managed_params; std::vector name_map; for (const auto& name : dir_names) { managed_params.emplace_back( TRITONSERVER_ParameterNew( name.c_str(), TRITONSERVER_PARAMETER_STRING, "name_0"), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_params.back() != nullptr) << "failed to create name mapping pair"; name_map.emplace_back(managed_params.back().get()); } // Register should fail FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_1", name_map.data(), name_map.size()), TRITONSERVER_ERROR_INVALID_ARG, "failed to register 'models_1', there is a conflicting mapping for " "'name_0'", "registering model repository 'models_1'"); } TEST_F(RegisterApiTest, RegisterMulti) { // Registering repository "models_0" and "model_1" without mappings, // there are duplicate models but it won't be checked until load std::vector name_map; FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", name_map.data(), name_map.size()), "registering model repository 'models_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_1", name_map.data(), name_map.size()), "registering model repository 'models_1'"); // Request to load "model_0" which should fail FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), TRITONSERVER_ERROR_INTERNAL, "failed to load 'model_0', failed to poll from model repository", "loading model 'model_0'"); // Request to load "model_1" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_1"), "loading model 'model_1'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_1", 3, &ready), "Is 'model_1' ready"); ASSERT_TRUE(ready) << "Expect 'model_1' v3 to be ready, model directory is " "'models_1/model_1'"; } TEST_F(RegisterApiTest, RegisterMultiWithMap) { // Registering repository "models_0" and "models_1" without mappings, // there are duplicate models but we provides a "override" map for "models_0", // from "model_0" to "model_0" which sets priority to resolve the conflict. std::vector override_names{"model_0"}; std::vector> managed_params; std::vector name_map; managed_params.emplace_back( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, override_names[0].c_str()), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_params.back() != nullptr) << "failed to create name mapping pair"; name_map.emplace_back(managed_params.back().get()); FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", name_map.data(), name_map.size()), "registering model repository 'models_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_1", nullptr, 0), "registering model repository 'models_1'"); // Request to load "model_0", "model_1" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), "loading model 'model_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_0", 1, &ready), "Is 'model_0' ready"); ASSERT_TRUE(ready) << "Expect 'model_0' v1 to be ready, model directory is " "'models_0/model_0'"; FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_1"), "loading model 'model_1'"); ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_1", 3, &ready), "Is 'model_1' ready"); ASSERT_TRUE(ready) << "Expect 'model_1' v3 to be ready, model directory is " "'models_1/model_1'"; } TEST_F(RegisterApiTest, RegisterMultiWithMap2) { // Registering repository "models_0" and "model_1s", // there are duplicate models but we provides a map for "models_1" // so they all have different name. std::vector override_names{"model_2"}; std::vector> managed_params; std::vector name_map; managed_params.emplace_back( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, override_names[0].c_str()), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_params.back() != nullptr) << "failed to create name mapping pair"; name_map.emplace_back(managed_params.back().get()); FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", nullptr, 0), "registering model repository 'models_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_1", name_map.data(), name_map.size()), "registering model repository 'models_1'"); // Request to load "model_0", "model_1", "model_2" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), "loading model 'model_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_0", 1, &ready), "Is 'model_0' ready"); ASSERT_TRUE(ready) << "Expect 'model_0' v1 to be ready, model directory is " "'models_0/model_0'"; FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_1"), "loading model 'model_1'"); ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_1", 3, &ready), "Is 'model_1' ready"); ASSERT_TRUE(ready) << "Expect 'model_1' v3 to be ready, model directory is " "'models_1/model_1'"; FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_2"), "loading model 'model_2'"); ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_2", 2, &ready), "Is 'model_2' ready"); ASSERT_TRUE(ready) << "Expect 'model_2' v2 to be ready, model directory is " "'models_1/model_0'"; } TEST_F(RegisterApiTest, RegisterMultiWithMap3) { // Registering repository "models_0" and "model_1s", // there are duplicate models but we provides a map for both // "models_0" and "models_1" so they all have different name. std::vector override_names{"name_0", "name_1"}; std::vector> managed_params; for (const auto& name : override_names) { managed_params.emplace_back( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, name.c_str()), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_params.back() != nullptr) << "failed to create name mapping pair"; } std::vector models_0_map{ managed_params[0].get()}; std::vector models_1_map{ managed_params[1].get()}; FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", models_0_map.data(), models_0_map.size()), "registering model repository 'models_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_1", models_1_map.data(), models_1_map.size()), "registering model repository 'models_1'"); // Request to load "model_0", "model_1", "model_2" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "name_0"), "loading model 'name_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "name_0", 1, &ready), "Is 'name_0' ready"); ASSERT_TRUE(ready) << "Expect 'name_0' v1 to be ready, model directory is " "'models_0/model_0'"; FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "name_1"), "loading model 'name_1'"); ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "name_1", 2, &ready), "Is 'name_1' ready"); ASSERT_TRUE(ready) << "Expect 'name_1' v2 to be ready, model directory is " "'models_1/model_0'"; FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_1"), "loading model 'model_1'"); ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_1", 3, &ready), "Is 'model_1' ready"); ASSERT_TRUE(ready) << "Expect 'model_1' v3 to be ready, model directory is " "'models_1/model_1'"; } TEST_F(RegisterApiTest, RegisterNonExistingRepo) { // Register should fail FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "unknown_repo", nullptr, 0), TRITONSERVER_ERROR_INVALID_ARG, "failed to register 'unknown_repo', repository not found", "registering model repository 'unknown_repo'"); } TEST_F(RegisterApiTest, UnregisterInvalidRepo) { FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "unknown_repo"), TRITONSERVER_ERROR_INVALID_ARG, "failed to unregister 'unknown_repo', repository not found", "unregistering model repository 'unknown_repo'"); } TEST_F(RegisterApiTest, Unregister) { FAIL_TEST_IF_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "empty_models"), "unregistering model repository 'empty_models'"); } TEST_F(RegisterApiTest, UnregisterTwice) { FAIL_TEST_IF_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "empty_models"), "unregistering model repository 'empty_models'"); FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "empty_models"), TRITONSERVER_ERROR_INVALID_ARG, "failed to unregister 'empty_models', repository not found", "unregistering model repository 'empty_models'"); } TEST_F(RegisterApiTest, UnregisterWithLoadedModel) { // Registering a repository "models_0" where contains "model_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", nullptr, 0), "registering model repository 'models_0'"); // Request to load "model_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), "loading model 'model_0'"); // Unregister and the model should still be loaded FAIL_TEST_IF_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "models_0"), "unregistering model repository 'models_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_0", 1, &ready), "Is 'model_0' ready"); ASSERT_TRUE(ready) << "Expect 'model_0' v1 to be ready, model directory is " "'models_0/model_0'"; // Request to load "model_0" which should fail FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), TRITONSERVER_ERROR_INTERNAL, "failed to load 'model_0', failed to poll from model repository", "loading model 'model_0'"); } TEST_F(RegisterApiTest, MultiRegister) { // Register / unregister a repository "models_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", nullptr, 0), "registering model repository 'models_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "models_0"), "unregistering model repository 'models_0'"); // Register / unregister "models_0" again FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", nullptr, 0), "registering model repository 'models_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "models_0"), "unregistering model repository 'models_0'"); } TEST_F(RegisterApiTest, RegisterMulti2) { // Registering repository "models_0" and "model_1" without mappings, // there are duplicate models but it won't be checked until load std::vector name_map; FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", name_map.data(), name_map.size()), "registering model repository 'models_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_1", name_map.data(), name_map.size()), "registering model repository 'models_1'"); // Request to load "model_0" which should fail FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), TRITONSERVER_ERROR_INTERNAL, "failed to load 'model_0', failed to poll from model repository", "loading model 'model_0'"); // Request to load "model_1" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_1"), "loading model 'model_1'"); // Unregister one of the repos and 'model_0' can be loaded as there is no // confliction FAIL_TEST_IF_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "models_1"), "unregistering model repository 'models_1'"); // Request to load "model_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), "loading model 'model_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_0", 1, &ready), "Is 'model_0' ready"); ASSERT_TRUE(ready) << "Expect 'model_0' v1 to be ready, model directory is " "'models_0/model_0'"; ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_1", 3, &ready), "Is 'model_1' ready"); ASSERT_TRUE(ready) << "Expect 'model_1' v3 to be ready, model directory is " "'models_1/model_1'"; } TEST_F(RegisterApiTest, DifferentMapping) { // With register and unregister, user can update a mapping for specific repo. std::vector override_names{"name_0"}; std::vector> managed_params; std::vector name_map; managed_params.emplace_back( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, override_names[0].c_str()), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_params.back() != nullptr) << "failed to create name mapping pair"; name_map.emplace_back(managed_params.back().get()); // First register without mapping FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", nullptr, 0), "registering model repository 'models_0'"); // Request to load "model_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), "loading model 'model_0'"); // Re-register with mapping FAIL_TEST_IF_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "models_0"), "unregistering model repository 'models_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", name_map.data(), name_map.size()), "registering model repository 'models_0'"); // Request to load "model_0" will fail, but load "name_0" is okay FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), TRITONSERVER_ERROR_INTERNAL, "failed to load 'model_0', failed to poll from model repository", "loading model 'model_0'"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "name_0"), "loading model 'name_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "name_0", 1, &ready), "Is 'name_0' ready"); ASSERT_TRUE(ready) << "Expect 'name_0' v1 to be ready, model directory is " "'models_0/model_0'"; // Verify that model_0 still exists in-memory ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "model_0", 1, &ready), "Is 'model_0' ready"); ASSERT_TRUE(ready) << "Expect 'model_0' v1 to be ready, model directory is " "'models_0/model_0'"; } TEST_F(RegisterApiTest, CorrectIndex) { // Registering a repository "models_0" where contains "model_0", but with // different name mapping const char* override_name = "name_0"; std::shared_ptr managed_param( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, override_name), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_param != nullptr) << "failed to create name mapping pair"; std::vector name_map{managed_param.get()}; FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", name_map.data(), name_map.size()), "registering model repository 'models_0'"); // Request to load "model_0" which should fail FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerLoadModel(server_, "model_0"), TRITONSERVER_ERROR_INTERNAL, "failed to load 'model_0', failed to poll from model repository", "loading model 'model_0'"); // Request to load "name_0" FAIL_TEST_IF_ERR( TRITONSERVER_ServerLoadModel(server_, "name_0"), "loading model 'name_0'"); bool ready = false; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIsReady(server_, "name_0", 1, &ready), "Is 'name_0' v1 ready"); ASSERT_TRUE(ready) << "Expect 'name_0' v1 to be ready, model directory is " "'models_0/model_0'"; TRITONSERVER_Message* repository_index; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIndex(server_, 1, &repository_index), "checking model indexes"); const char* base = nullptr; size_t byte_size = 0; FAIL_TEST_IF_ERR( TRITONSERVER_MessageSerializeToJson(repository_index, &base, &byte_size), "serializing index to Json"); const std::string search_msg = "[{\"name\":\"name_0\",\"version\":\"1\",\"state\":\"READY\"}]"; const std::string serialized_index(base, byte_size); EXPECT_EQ(serialized_index, search_msg) << "Returned index does not equal expected index"; } TEST_F(RegisterApiTest, CorrectIndexNotLoaded) { // Registering a repository "models_0" where contains "model_0", but with // different name mapping const char* override_name = "name_0"; std::shared_ptr managed_param( TRITONSERVER_ParameterNew( "model_0", TRITONSERVER_PARAMETER_STRING, override_name), TRITONSERVER_ParameterDelete); ASSERT_TRUE(managed_param != nullptr) << "failed to create name mapping pair"; std::vector name_map{managed_param.get()}; FAIL_TEST_IF_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "models_0", name_map.data(), name_map.size()), "registering model repository 'models_0'"); TRITONSERVER_Message* repository_index; FAIL_TEST_IF_ERR( TRITONSERVER_ServerModelIndex(server_, 0, &repository_index), "checking model indexes"); const char* base = nullptr; size_t byte_size = 0; FAIL_TEST_IF_ERR( TRITONSERVER_MessageSerializeToJson(repository_index, &base, &byte_size), "serializing index to Json"); const std::string search_msg = "[{\"name\":\"name_0\"}]"; const std::string serialized_index(base, byte_size); EXPECT_EQ(serialized_index, search_msg) << "Returned index does not equal expected index"; } // // Test Fixture that runs server with POLLING mode class PollingRegisterApiTest : public ::testing::Test { protected: void SetUp() override { // Create running server object. TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); // Triton expects at least one model repository is set at start, set to // an empty repository set ModelControlMode to EXPLICIT to avoid attempting // to load models. FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, "empty_models"), "setting model repository path"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelControlMode( server_options, TRITONSERVER_MODEL_CONTROL_POLL), "setting model control mode"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerNew(&server_, server_options), "creating server"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); ASSERT_TRUE(server_ != nullptr) << "server not created"; bool live = false; for (int i = 10; ((i > 0) && !live); --i) { FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsLive(server_, &live), "Is server live"); } ASSERT_TRUE(live) << "server not live"; } void TearDown() override { FAIL_TEST_IF_ERR(TRITONSERVER_ServerDelete(server_), "deleting server"); } TRITONSERVER_Server* server_ = nullptr; }; TEST_F(PollingRegisterApiTest, unsupported) { FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "empty_models", nullptr, 0), TRITONSERVER_ERROR_UNSUPPORTED, "repository registration is not allowed if model control mode is not " "EXPLICIT", "registering model repository 'empty_models'"); FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "empty_models"), TRITONSERVER_ERROR_UNSUPPORTED, "repository unregistration is not allowed if model control mode is not " "EXPLICIT", "unregistering model repository 'empty_models'"); } // Test Fixture that runs server with NONE mode class NoneRegisterApiTest : public ::testing::Test { protected: void SetUp() override { // Create running server object. TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); // Triton expects at least one model repository is set at start, set to // an empty repository set ModelControlMode to EXPLICIT to avoid attempting // to load models. FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, "empty_models"), "setting model repository path"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetModelControlMode( server_options, TRITONSERVER_MODEL_CONTROL_NONE), "setting model control mode"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerNew(&server_, server_options), "creating server"); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); ASSERT_TRUE(server_ != nullptr) << "server not created"; bool live = false; for (int i = 10; ((i > 0) && !live); --i) { FAIL_TEST_IF_ERR( TRITONSERVER_ServerIsLive(server_, &live), "Is server live"); } ASSERT_TRUE(live) << "server not live"; } void TearDown() override { FAIL_TEST_IF_ERR(TRITONSERVER_ServerDelete(server_), "deleting server"); } TRITONSERVER_Server* server_ = nullptr; }; TEST_F(NoneRegisterApiTest, unsupported) { FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerRegisterModelRepository( server_, "empty_models", nullptr, 0), TRITONSERVER_ERROR_UNSUPPORTED, "repository registration is not allowed if model control mode is not " "EXPLICIT", "registering model repository 'empty_models'"); FAIL_TEST_IF_NOT_ERR( TRITONSERVER_ServerUnregisterModelRepository(server_, "empty_models"), TRITONSERVER_ERROR_UNSUPPORTED, "repository unregistration is not allowed if model control mode is not " "EXPLICIT", "unregistering model repository 'empty_models'"); } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/repo_agent_test.cc000066400000000000000000002720501527052371200220320ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "repo_agent.h" #include #include #include #include #include #include #include #include "filesystem/api.h" #include "gtest/gtest.h" #include "server_message.h" #include "shared_library.h" namespace tc = triton::core; namespace { // // Duplication of TRITONSERVER_Error implementation // class TritonServerError { public: static TRITONSERVER_Error* Create( TRITONSERVER_Error_Code code, const char* msg); static TRITONSERVER_Error* Create(const tc::Status& status); TRITONSERVER_Error_Code Code() const { return code_; } const std::string& Message() const { return msg_; } private: TritonServerError(TRITONSERVER_Error_Code code, const std::string& msg) : code_(code), msg_(msg) { } TritonServerError(TRITONSERVER_Error_Code code, const char* msg) : code_(code), msg_(msg) { } TRITONSERVER_Error_Code code_; const std::string msg_; }; TRITONSERVER_Error* TritonServerError::Create(TRITONSERVER_Error_Code code, const char* msg) { return reinterpret_cast( new TritonServerError(code, msg)); } TRITONSERVER_Error* TritonServerError::Create(const tc::Status& status) { // If 'status' is success then return nullptr as that indicates // success if (status.IsOk()) { return nullptr; } return Create( tc::StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str()); } class MockSharedLibraryHandle { public: bool AddEntryPoint(const std::string& name, void* fn) { auto it = entry_points_.find(name); if (it == entry_points_.end()) { entry_points_.emplace(name, fn).second; return true; } else { it->second = fn; return false; } } bool GetEntryPoint(const std::string& name, void** fn) { auto it = entry_points_.find(name); if (it != entry_points_.end()) { *fn = it->second; return true; } return false; } private: std::map entry_points_; }; static std::map global_mock_agents; } // namespace #ifdef __cplusplus extern "C" { #endif TRITONSERVER_Error* TRITONSERVER_ErrorNew(TRITONSERVER_Error_Code code, const char* msg) { return reinterpret_cast( TritonServerError::Create(code, msg)); } void TRITONSERVER_ErrorDelete(TRITONSERVER_Error* error) { TritonServerError* lerror = reinterpret_cast(error); delete lerror; } TRITONSERVER_Error_Code TRITONSERVER_ErrorCode(TRITONSERVER_Error* error) { TritonServerError* lerror = reinterpret_cast(error); return lerror->Code(); } const char* TRITONSERVER_ErrorCodeString(TRITONSERVER_Error* error) { TritonServerError* lerror = reinterpret_cast(error); return tc::Status::CodeString(tc::TritonCodeToStatusCode(lerror->Code())); } const char* TRITONSERVER_ErrorMessage(TRITONSERVER_Error* error) { TritonServerError* lerror = reinterpret_cast(error); return lerror->Message().c_str(); } // // TRITONSERVER_Message // TRITONSERVER_Error* TRITONSERVER_MessageNewFromSerializedJson( TRITONSERVER_Message** message, const char* base, size_t byte_size) { *message = reinterpret_cast( new tc::TritonServerMessage({base, byte_size})); return nullptr; } TRITONSERVER_Error* TRITONSERVER_MessageSerializeToJson( TRITONSERVER_Message* message, const char** base, size_t* byte_size) { tc::TritonServerMessage* lmessage = reinterpret_cast(message); lmessage->Serialize(base, byte_size); return nullptr; // Success } #ifdef __cplusplus } #endif namespace triton { namespace core { Status SharedLibrary::Acquire(std::unique_ptr* slib) { slib->reset(new SharedLibrary()); return Status::Success; } SharedLibrary::~SharedLibrary() {} Status SharedLibrary::SetLibraryDirectory(const std::string& path) { return Status::Success; } Status SharedLibrary::ResetLibraryDirectory() { return Status::Success; } Status SharedLibrary::OpenLibraryHandle(const std::string& path, void** handle) { auto it = global_mock_agents.find(path); if (it != global_mock_agents.end()) { *handle = reinterpret_cast(&it->second); return Status::Success; } return Status( Status::Code::NOT_FOUND, "unable to load shared library: mock shared library is not set for " "path " + path); } Status SharedLibrary::CloseLibraryHandle(void* handle) { for (auto& global_mock_agent : global_mock_agents) { if (reinterpret_cast(&global_mock_agent.second) == handle) { return Status::Success; } } return Status( Status::Code::NOT_FOUND, "unable to unload shared library: handle does not matach any mock shared " "library"); } Status SharedLibrary::GetEntrypoint( void* handle, const std::string& name, const bool optional, void** fn) { auto mock_agent = reinterpret_cast(handle); bool found = mock_agent->GetEntryPoint(name, fn); if (!optional && !found) { return Status( Status::Code::NOT_FOUND, "unable to find required entrypoint '" + name + "' in shared library"); } return Status::Success; } }} // namespace triton::core namespace { class TritonRepoAgentTest : public ::testing::Test { protected: void TearDown() override { global_mock_agents.clear(); } }; TEST_F(TritonRepoAgentTest, Create) { // Set up agent with only action function defined, check agent properties tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t CheckNameModelActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { auto lagent = reinterpret_cast(agent); EXPECT_EQ(lagent->Name(), "minimal_agent") << "Expect action function is called with minimal agent"; return nullptr; }; auto agent_handle = MockSharedLibraryHandle(); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(CheckNameModelActionFn)); global_mock_agents.emplace("minimal_agent_path", agent_handle); std::shared_ptr minimal_agent; auto status = tc::TritonRepoAgent::Create( "minimal_agent", "minimal_agent_path", &minimal_agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); ASSERT_TRUE(minimal_agent->AgentModelActionFn() != nullptr) << "Expect action function is provided"; EXPECT_TRUE(minimal_agent->AgentModelInitFn() == nullptr) << "Unexpected model init function is provided"; EXPECT_TRUE(minimal_agent->AgentModelFiniFn() == nullptr) << "Unexpected model fini function is provided"; auto err = minimal_agent->AgentModelActionFn()( reinterpret_cast(minimal_agent.get()), nullptr, TRITONREPOAGENT_ACTION_LOAD); EXPECT_TRUE(err == nullptr) << "Expect successful action function invocation"; } TEST_F(TritonRepoAgentTest, CreateFailInvalidSharedLibrary) { // Passing a agent path that is not in global_mock_agents to // simulate failure on opening shared library handle std::shared_ptr invalid_agent; auto status = tc::TritonRepoAgent::Create( "invalid_agent", "invalid_agent_path", &invalid_agent); ASSERT_FALSE(status.IsOk()) << "Unexpected successful agent creation"; EXPECT_NE( status.Message().find("unable to load shared library"), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect 'unable to load shared library...'"; } TEST_F(TritonRepoAgentTest, CreateFailMissingEndpoint) { // Set up agent with nothing defined auto agent_handle = MockSharedLibraryHandle(); global_mock_agents.emplace("invalid_agent_path", agent_handle); std::shared_ptr invalid_agent; auto status = tc::TritonRepoAgent::Create( "invalid_agent", "invalid_agent_path", &invalid_agent); ASSERT_FALSE(status.IsOk()) << "Unexpected successful agent creation"; EXPECT_NE( status.Message().find("unable to find required entrypoint"), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect 'unable to find required entrypoint...'"; } TEST_F(TritonRepoAgentTest, Lifecycle) { // Set up agent with init / fini function defined tc::TritonRepoAgent::TritonRepoAgentInitFn_t InitFn = [](TRITONREPOAGENT_Agent* agent) -> TRITONSERVER_Error* { auto lagent = reinterpret_cast(agent); EXPECT_TRUE(lagent->State() == nullptr) << "Expect agent state is not set before initialization"; bool* state = new bool(false); lagent->SetState(reinterpret_cast(state)); return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentFiniFn_t FiniFn = [](TRITONREPOAGENT_Agent* agent) -> TRITONSERVER_Error* { auto lagent = reinterpret_cast(agent); bool* state = reinterpret_cast(lagent->State()); EXPECT_TRUE(state != nullptr) << "Expect agent state is set"; EXPECT_TRUE(*state) << "Expect state is set to true"; delete state; return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { auto lagent = reinterpret_cast(agent); bool* state = reinterpret_cast(lagent->State()); EXPECT_TRUE(state != nullptr) << "Expect agent state is set"; EXPECT_FALSE(*state) << "Expect state is set to false"; *state = true; return nullptr; }; auto agent_handle = MockSharedLibraryHandle(); agent_handle.AddEntryPoint( "TRITONREPOAGENT_Initialize", reinterpret_cast(InitFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_Finalize", reinterpret_cast(FiniFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ActionFn)); global_mock_agents.emplace("agent_path", agent_handle); std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); ASSERT_TRUE(agent->AgentModelActionFn() != nullptr) << "Expect action function is provided"; EXPECT_TRUE(agent->AgentModelInitFn() == nullptr) << "Unexpected model init function is provided"; EXPECT_TRUE(agent->AgentModelFiniFn() == nullptr) << "Unexpected model fini function is provided"; auto err = agent->AgentModelActionFn()( reinterpret_cast(agent.get()), nullptr, TRITONREPOAGENT_ACTION_LOAD); EXPECT_TRUE(err == nullptr) << "Expect successful action function invocation"; // Cause destructor to be called agent.reset(); } TEST_F(TritonRepoAgentTest, ModelLifecycle) { // Set up agent with model init / fini function defined tc::TritonRepoAgent::TritonRepoAgentModelInitFn_t InitFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) -> TRITONSERVER_Error* { auto lmodel_state = reinterpret_cast*, std::future*>*>( model); lmodel_state->first->set_value(); return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelFiniFn_t FiniFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) -> TRITONSERVER_Error* { auto lmodel_state = reinterpret_cast*, std::future*>*>( model); lmodel_state->second->get(); return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { auto lmodel_state = reinterpret_cast*, std::future*>*>( model); EXPECT_TRUE(lmodel_state->second->valid()) << "Expect promise value is set"; return nullptr; }; auto agent_handle = MockSharedLibraryHandle(); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelInitialize", reinterpret_cast(InitFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelFinalize", reinterpret_cast(FiniFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ActionFn)); global_mock_agents.emplace("agent_path", agent_handle); std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); ASSERT_TRUE(agent->AgentModelActionFn() != nullptr) << "Expect action function is provided"; ASSERT_TRUE(agent->AgentModelInitFn() != nullptr) << "Expect model init function is provided"; ASSERT_TRUE(agent->AgentModelFiniFn() != nullptr) << "Expect model fini function is provided"; std::promise p; auto f = p.get_future(); auto model_state = std::make_pair(&p, &f); // Simulate the model lifecycle auto err = agent->AgentModelInitFn()( reinterpret_cast(agent.get()), reinterpret_cast(&model_state)); EXPECT_TRUE(err == nullptr) << "Expect successful model init function invocation"; err = agent->AgentModelActionFn()( reinterpret_cast(agent.get()), reinterpret_cast(&model_state), TRITONREPOAGENT_ACTION_LOAD); EXPECT_TRUE(err == nullptr) << "Expect successful action function invocation"; err = agent->AgentModelFiniFn()( reinterpret_cast(agent.get()), reinterpret_cast(&model_state)); EXPECT_TRUE(err == nullptr) << "Expect successful model fini function invocation"; EXPECT_FALSE(f.valid()) << "Expect future value is retrieved"; } class TritonRepoAgentManagerTest : public ::testing::Test { public: static size_t agent_init_counter_; static size_t agent_fini_counter_; protected: void SetUp() override { // Set up agent with init / fini function defined tc::TritonRepoAgent::TritonRepoAgentInitFn_t InitFn = [](TRITONREPOAGENT_Agent* agent) -> TRITONSERVER_Error* { agent_init_counter_++; return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentFiniFn_t FiniFn = [](TRITONREPOAGENT_Agent* agent) -> TRITONSERVER_Error* { agent_fini_counter_++; return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { return nullptr; }; auto agent_handle = MockSharedLibraryHandle(); agent_handle.AddEntryPoint( "TRITONREPOAGENT_Initialize", reinterpret_cast(InitFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_Finalize", reinterpret_cast(FiniFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ActionFn)); // Reserve valid shared library paths because manager searches the libraries // via the FileSystem API const tc::FileSystemType type = tc::FileSystemType::LOCAL; auto status = tc::MakeTemporaryDirectory(type, &root_agent_path_); ASSERT_TRUE(status.IsOk()) << "TritonRepoAgentManagerTest set up failed: " "create temporary directory: " << status.AsString(); // FIXME make the following platform independent global_agent_path_ = tc::JoinPath({root_agent_path_, "global"}); int err = mkdir( global_agent_path_.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); ASSERT_EQ(err, 0) << "TritonRepoAgentManagerTest set up failed: create " "global agent directory: " << err; const std::set agent_names{"global_agent"}; for (const auto& agent_name : agent_names) { auto global_path_to_agent = tc::JoinPath({global_agent_path_, agent_name}); auto global_agent = tc::JoinPath( {global_path_to_agent, tc::TritonRepoAgentLibraryName(agent_name)}); err = mkdir( global_path_to_agent.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); ASSERT_EQ(err, 0) << "TritonRepoAgentManagerTest set up failed: create " "global agent directory: " << err; std::ofstream global_agent_file(global_agent); global_mock_agents.emplace(global_agent, agent_handle); } status = tc::TritonRepoAgentManager::SetGlobalSearchPath(global_agent_path_); ASSERT_TRUE(status.IsOk()) << "TritonRepoAgentManagerTest set up failed: " "create temporary directory: " << status.AsString(); } void TearDown() override { agent_init_counter_ = 0; agent_fini_counter_ = 0; if (!root_agent_path_.empty()) { // tc::DeleteDirectory(root_agent_path_); } global_mock_agents.clear(); } std::string root_agent_path_; std::string global_agent_path_; std::string local_agent_path_; }; size_t TritonRepoAgentManagerTest::agent_init_counter_ = 0; size_t TritonRepoAgentManagerTest::agent_fini_counter_ = 0; TEST_F(TritonRepoAgentManagerTest, CreateFailureFileNotExist) { // Passing a agent path that is not in global_mock_agents to // simulate failure on opening shared library handle std::shared_ptr invalid_agent; auto status = tc::TritonRepoAgentManager::CreateAgent( "invalid_agent_name", &invalid_agent); ASSERT_FALSE(status.IsOk()) << "Unexpected successful agent creation"; EXPECT_NE(status.Message().find("unable to find"), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect 'unable to find...'"; } TEST_F(TritonRepoAgentManagerTest, CreateGlobalAgent) { std::shared_ptr agent; auto status = tc::TritonRepoAgentManager::CreateAgent("global_agent", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation" << status.AsString(); agent.reset(); EXPECT_EQ(agent_init_counter_, (size_t)1) << "Expect 1 agent initialization"; EXPECT_EQ(agent_fini_counter_, (size_t)1) << "Expect 1 agent finalization"; } TEST_F(TritonRepoAgentManagerTest, AgentPersistence) { std::shared_ptr agent1; std::shared_ptr agent2; auto status = tc::TritonRepoAgentManager::CreateAgent("global_agent", &agent1); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation" << status.AsString(); EXPECT_EQ(agent_init_counter_, (size_t)1) << "Expect 1 agent initialization"; EXPECT_EQ(agent_fini_counter_, (size_t)0) << "Expect 0 agent finalization"; status = tc::TritonRepoAgentManager::CreateAgent("global_agent", &agent2); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation" << status.AsString(); EXPECT_EQ(agent_init_counter_, (size_t)1) << "Expect 1 agent initialization"; EXPECT_EQ(agent_fini_counter_, (size_t)0) << "Expect 0 agent finalization"; agent1.reset(); EXPECT_EQ(agent_init_counter_, (size_t)1) << "Expect 1 agent initialization"; EXPECT_EQ(agent_fini_counter_, (size_t)0) << "Expect 0 agent finalization"; agent2.reset(); EXPECT_EQ(agent_init_counter_, (size_t)1) << "Expect 1 agent initialization"; EXPECT_EQ(agent_fini_counter_, (size_t)1) << "Expect 1 agent finalization"; // Create again after all previous agents are reset status = tc::TritonRepoAgentManager::CreateAgent("global_agent", &agent1); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation" << status.AsString(); EXPECT_EQ(agent_init_counter_, (size_t)2) << "Expect 2 agent initialization"; EXPECT_EQ(agent_fini_counter_, (size_t)1) << "Expect 1 agent finalization"; agent1.reset(); EXPECT_EQ(agent_init_counter_, (size_t)2) << "Expect 2 agent initialization"; EXPECT_EQ(agent_fini_counter_, (size_t)2) << "Expect 2 agent finalization"; } class TritonRepoAgentModelTest : public ::testing::Test { protected: void SetUp() override { simple_config_.set_name("simple_config"); // Add a simple agent handle for convenience tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { return nullptr; }; auto agent_handle = MockSharedLibraryHandle(); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ActionFn)); global_mock_agents.emplace("simple_agent_path", agent_handle); // Add a agent handle for logging actions of the model tc::TritonRepoAgent::TritonRepoAgentModelInitFn_t LogInitFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) -> TRITONSERVER_Error* { auto lagent = reinterpret_cast(agent); auto state = reinterpret_cast*>(lagent->State()); if (state == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Agent state is not set"); } state->emplace_back("Model Initialized"); return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelFiniFn_t LogFiniFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) -> TRITONSERVER_Error* { auto lagent = reinterpret_cast(agent); auto state = reinterpret_cast*>(lagent->State()); if (state == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Agent state is not set"); } state->emplace_back("Model Finalized"); return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t LogActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { auto lagent = reinterpret_cast(agent); auto state = reinterpret_cast*>(lagent->State()); if (state == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Agent state is not set"); } state->emplace_back(tc::TRITONREPOAGENT_ActionTypeString(action_type)); return nullptr; }; auto log_agent_handle = MockSharedLibraryHandle(); log_agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelInitialize", reinterpret_cast(LogInitFn)); log_agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelFinalize", reinterpret_cast(LogFiniFn)); log_agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(LogActionFn)); global_mock_agents.emplace("log_agent_path", log_agent_handle); } void TearDown() override { global_mock_agents.clear(); } TRITONREPOAGENT_ArtifactType original_type_ = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; const std::string original_location_ = "/original"; inference::ModelConfig simple_config_; }; TEST_F(TritonRepoAgentModelTest, Create) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); EXPECT_EQ(model->Config().name(), simple_config_.name()) << "Expect the model contains the same config as simple config"; } TEST_F(TritonRepoAgentModelTest, CreateFailure) { // Create agent to be associated with the model, whose model init function // always returns error tc::TritonRepoAgent::TritonRepoAgentModelInitFn_t InitFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) -> TRITONSERVER_Error* { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Model initialization error"); }; tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { return nullptr; }; auto agent_handle = MockSharedLibraryHandle(); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelInitialize", reinterpret_cast(InitFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ActionFn)); global_mock_agents.emplace("agent_path", agent_handle); std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_FALSE(status.IsOk()) << "Unexpected successful model creation"; EXPECT_NE( status.Message().find("Model initialization error"), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect 'Model initialization error...'"; } TEST_F(TritonRepoAgentModelTest, Location) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); TRITONREPOAGENT_ArtifactType type; const char* location; status = model->Location(&type, &location); ASSERT_TRUE(status.IsOk()) << "Expect location is returned from Location()"; EXPECT_EQ(type, original_type_) << "Expect returned original filesystem type"; EXPECT_EQ(std::string(location), original_location_) << "Expect returned original location"; } TEST_F(TritonRepoAgentModelTest, SetLocationFailure) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); TRITONREPOAGENT_ArtifactType type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; const char* location = "/tmp"; status = model->SetLocation(type, location); ASSERT_FALSE(status.IsOk()) << "Expect error returned from SetLocation()"; EXPECT_NE( status.Message().find( "location can only be updated during TRITONREPOAGENT_ACTION_LOAD, " "current action type is not set"), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect 'location can only be updated during " "TRITONREPOAGENT_ACTION_LOAD, current action type is not set'"; } TEST_F(TritonRepoAgentModelTest, SetLocation) { static const TRITONREPOAGENT_ArtifactType new_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; static const std::string new_location = "/new_location"; // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); // Advance the model lifecycle to be able to set location status = model->InvokeAgent(TRITONREPOAGENT_ACTION_LOAD); EXPECT_TRUE(status.IsOk()) << "Expect successful agent invocation with TRITONREPOAGENT_ACTION_LOAD"; status = model->SetLocation(new_type, new_location); ASSERT_TRUE(status.IsOk()) << "Expect successful SetLocation() after invoking agent with " "TRITONREPOAGENT_ACTION_LOAD"; TRITONREPOAGENT_ArtifactType type = original_type_; const char* location = original_location_.c_str(); status = model->Location(&type, &location); ASSERT_TRUE(status.IsOk()) << "Expect location is returned from Location()"; EXPECT_EQ(type, new_type) << "Expect returned filesystem type is " << tc::TRITONREPOAGENT_ArtifactTypeString(new_type); EXPECT_EQ(std::string(location), new_location) << "Expect returned location is " << new_location; } TEST_F(TritonRepoAgentModelTest, SetLocationWrongActionFailure) { static const TRITONREPOAGENT_ArtifactType new_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; static const std::string new_location = "/new_location"; // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); // Advance the model lifecycle to be able to set location status = model->InvokeAgent(TRITONREPOAGENT_ACTION_LOAD); EXPECT_TRUE(status.IsOk()) << "Expect successful agent invocation with TRITONREPOAGENT_ACTION_LOAD"; status = model->InvokeAgent(TRITONREPOAGENT_ACTION_LOAD_COMPLETE); EXPECT_TRUE(status.IsOk()) << "Expect successful agent invocation with " "TRITONREPOAGENT_ACTION_LOAD_COMPLETE"; status = model->SetLocation(new_type, new_location); ASSERT_FALSE(status.IsOk()) << "Expect error returned from SetLocation()"; EXPECT_NE( status.Message().find( "location can only be updated during TRITONREPOAGENT_ACTION_LOAD, " "current action type is TRITONREPOAGENT_ACTION_LOAD_COMPLETE"), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect 'location can only be updated during " "TRITONREPOAGENT_ACTION_LOAD, current action type is " "TRITONREPOAGENT_ACTION_LOAD_COMPLETE'"; } TEST_F(TritonRepoAgentModelTest, SetLocationViaAgent) { static const TRITONREPOAGENT_ArtifactType new_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; static const std::string new_location = "/new_location"; // Create agent to be associated with the model tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { auto lmodel = reinterpret_cast(model); auto status = lmodel->SetLocation(new_type, new_location); return reinterpret_cast( TritonServerError::Create(status)); }; auto agent_handle = MockSharedLibraryHandle(); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ActionFn)); global_mock_agents.emplace("set_location_agent_path", agent_handle); std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "set_location_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); // Advance the model lifecycle to be able to set location status = model->InvokeAgent(TRITONREPOAGENT_ACTION_LOAD); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with TRITONREPOAGENT_ACTION_LOAD"; TRITONREPOAGENT_ArtifactType type = original_type_; const char* location = original_location_.c_str(); status = model->Location(&type, &location); ASSERT_TRUE(status.IsOk()) << "Expect location is returned from Location()"; EXPECT_EQ(type, new_type) << "Expect returned filesystem type is " << tc::TRITONREPOAGENT_ArtifactTypeString(new_type); EXPECT_EQ(std::string(location), new_location) << "Expect returned location is " << new_location; } TEST_F(TritonRepoAgentModelTest, DeleteLocationBeforeAcquire) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); status = model->DeleteMutableLocation(); ASSERT_FALSE(status.IsOk()) << "Expect error returned from DeleteMutableLocation()"; EXPECT_NE( status.Message().find("No mutable location to be deleted"), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect 'No mutable location to be deleted'"; } TEST_F(TritonRepoAgentModelTest, AcquireLocalLocationAndDelete) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); const char* acquired_location; status = model->AcquireMutableLocation( TRITONREPOAGENT_ARTIFACT_FILESYSTEM, &acquired_location); ASSERT_TRUE(status.IsOk()) << "Expect successful location acquisition: " << status.AsString(); // Check directory bool is_dir = false; status = tc::IsDirectory(acquired_location, &is_dir); ASSERT_TRUE(status.IsOk()) << "Expect location property can be checked: " << status.AsString(); EXPECT_TRUE(is_dir) << "Expect a directory is returned as mutable location"; tc::FileSystemType type = tc::FileSystemType::LOCAL; status = tc::GetFileSystemType(acquired_location, &type); ASSERT_TRUE(status.IsOk()) << "Expect location filesystem type can be checked: " << status.AsString(); EXPECT_EQ(type, tc::FileSystemType::LOCAL) << "Expect a local mutable location is acquired"; status = model->DeleteMutableLocation(); ASSERT_TRUE(status.IsOk()) << "Expect successful location deletion: " << status.AsString(); // Check directory bool exists = true; status = tc::FileExists(acquired_location, &exists); ASSERT_TRUE(status.IsOk()) << "Expect location property can be checked: " << status.AsString(); EXPECT_FALSE(exists) << "Expect the mutable location no longer exists"; } TEST_F(TritonRepoAgentModelTest, AcquireLocalLocationTwice) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); const char* acquired_location; status = model->AcquireMutableLocation( TRITONREPOAGENT_ARTIFACT_FILESYSTEM, &acquired_location); ASSERT_TRUE(status.IsOk()) << "Expect successful location acquisition: " << status.AsString(); // Acquire the same type again const char* second_acquired_location; status = model->AcquireMutableLocation( TRITONREPOAGENT_ARTIFACT_FILESYSTEM, &second_acquired_location); ASSERT_TRUE(status.IsOk()) << "Expect successful location acquisition: " << status.AsString(); EXPECT_EQ( std::string(acquired_location), std::string(second_acquired_location)) << "Expect the same location is returned"; } TEST_F(TritonRepoAgentModelTest, DeleteTwiceAfterAcquire) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); const char* acquired_location; status = model->AcquireMutableLocation( TRITONREPOAGENT_ARTIFACT_FILESYSTEM, &acquired_location); ASSERT_TRUE(status.IsOk()) << "Expect successful location acquisition: " << status.AsString(); status = model->DeleteMutableLocation(); ASSERT_TRUE(status.IsOk()) << "Expect successful location deletion: " << status.AsString(); status = model->DeleteMutableLocation(); ASSERT_FALSE(status.IsOk()) << "Expect error returned from DeleteMutableLocation()"; EXPECT_NE( status.Message().find("No mutable location to be deleted"), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect 'No mutable location to be deleted'"; } TEST_F(TritonRepoAgentModelTest, AcquireRemoteLocation) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); const char* acquired_location; status = model->AcquireMutableLocation( TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM, &acquired_location); ASSERT_FALSE(status.IsOk()) << "Expect error returned from AcquireMutableLocation()"; const std::string search_msg = "Unexpected artifact type, expects 'TRITONREPOAGENT_ARTIFACT_FILESYSTEM'"; EXPECT_NE(status.Message().find(search_msg), std::string::npos) << "Unexpected error message: '" << status.Message() << "', expect '" << search_msg << "'"; } TEST_F(TritonRepoAgentModelTest, AgentParameters) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model tc::TritonRepoAgent::Parameters expected_params{ {"key_a", "value_b"}, {"key_b", "value_b"}}; std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, expected_params, &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); auto agent_params = model->AgentParameters(); ASSERT_EQ(agent_params.size(), expected_params.size()); for (size_t idx = 0; idx < agent_params.size(); ++idx) { EXPECT_EQ(agent_params[idx].first, expected_params[idx].first); EXPECT_EQ(agent_params[idx].second, expected_params[idx].second); } } TEST_F(TritonRepoAgentModelTest, State) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "simple_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); // Create model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); auto state = model->State(); ASSERT_TRUE(state == nullptr) << "Expect state is not set"; bool state_value = true; model->SetState(reinterpret_cast(&state_value)); state = model->State(); ASSERT_TRUE(state != nullptr) << "Expect state is set"; EXPECT_EQ(*reinterpret_cast(state), state_value) << "Expect state value is true"; } TEST_F(TritonRepoAgentModelTest, EmptyLifeCycle) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "log_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::vector log; agent->SetState(reinterpret_cast(&log)); // Create and destroy model std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); model.reset(); // Check log ASSERT_EQ(log.size(), (size_t)2) << "Expect 2 state of model lifecycle is logged, got " << log.size(); EXPECT_EQ(log[0], "Model Initialized"); EXPECT_EQ(log[1], "Model Finalized"); } TEST_F(TritonRepoAgentModelTest, HalfLifeCycle) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "log_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::vector log; agent->SetState(reinterpret_cast(&log)); std::unique_ptr model; // Create and destroy model in situations that a full lifecycle should run std::vector> situations{ {TRITONREPOAGENT_ACTION_LOAD}, {TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_FAIL}}; std::vector expected_log{ "Model Initialized", "TRITONREPOAGENT_ACTION_LOAD", "TRITONREPOAGENT_ACTION_LOAD_FAIL", "Model Finalized"}; for (const auto& situation : situations) { log.clear(); status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : situation) { status = model->InvokeAgent(action); EXPECT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } model.reset(); // Check log ASSERT_EQ(log.size(), expected_log.size()) << "Expect " << expected_log.size() << " state of model lifecycle is logged, got " << log.size(); for (size_t i = 0; i < log.size(); ++i) { EXPECT_EQ(log[i], expected_log[i]); } } } TEST_F(TritonRepoAgentModelTest, FullLifeCycle) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "log_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::vector log; agent->SetState(reinterpret_cast(&log)); std::unique_ptr model; // Create and destroy model in situations that a full lifecycle should run std::vector> situations{ {TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_COMPLETE}, {TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_COMPLETE, TRITONREPOAGENT_ACTION_UNLOAD}, {TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_COMPLETE, TRITONREPOAGENT_ACTION_UNLOAD, TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE}}; std::vector expected_log{ "Model Initialized", "TRITONREPOAGENT_ACTION_LOAD", "TRITONREPOAGENT_ACTION_LOAD_COMPLETE", "TRITONREPOAGENT_ACTION_UNLOAD", "TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE", "Model Finalized"}; for (const auto& situation : situations) { log.clear(); status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : situation) { status = model->InvokeAgent(action); EXPECT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } model.reset(); // Check log ASSERT_EQ(log.size(), expected_log.size()) << "Expect " << expected_log.size() << " state of model lifecycle is logged, got " << log.size(); for (size_t i = 0; i < log.size(); ++i) { EXPECT_EQ(log[i], expected_log[i]); } } } TEST_F(TritonRepoAgentModelTest, WrongLifeCycle) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "log_agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::vector log; agent->SetState(reinterpret_cast(&log)); // Create model and run all action combinations std::vector> valid_lifecycles{ {TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_FAIL}, {TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_COMPLETE, TRITONREPOAGENT_ACTION_UNLOAD, TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE}}; std::vector available_actions{ TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_FAIL, TRITONREPOAGENT_ACTION_LOAD_COMPLETE, TRITONREPOAGENT_ACTION_UNLOAD, TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE}; std::map> valid_actions{ {TRITONREPOAGENT_ACTION_LOAD, {TRITONREPOAGENT_ACTION_LOAD_FAIL, TRITONREPOAGENT_ACTION_LOAD_COMPLETE}}, {TRITONREPOAGENT_ACTION_LOAD_FAIL, {}}, {TRITONREPOAGENT_ACTION_LOAD_COMPLETE, {TRITONREPOAGENT_ACTION_UNLOAD}}, {TRITONREPOAGENT_ACTION_UNLOAD, {TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE}}, {TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE, {}}}; for (const auto& valid_lifecycle : valid_lifecycles) { log.clear(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (size_t idx = 0; idx < valid_lifecycle.size(); ++idx) { const auto next_lifecycle_action = valid_lifecycle[idx]; // Handle the first action specially if (idx == 0) { for (const auto action : available_actions) { if (action == valid_lifecycle[0]) { continue; } status = model->InvokeAgent(action); if (status.IsOk()) { for (const auto& state_log : log) { EXPECT_TRUE(false) << state_log; } } ASSERT_FALSE(status.IsOk()) << "Unexpected successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action); } status = model->InvokeAgent(valid_lifecycle[0]); if (!status.IsOk()) { for (const auto& state_log : log) { EXPECT_TRUE(false) << state_log; } } ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(next_lifecycle_action) << ": " << status.AsString(); continue; } const auto& current_valid_actions = valid_actions[valid_lifecycle[idx - 1]]; for (const auto action : available_actions) { if (current_valid_actions.find(action) != current_valid_actions.end()) { continue; } status = model->InvokeAgent(action); if (status.IsOk()) { for (const auto& state_log : log) { EXPECT_TRUE(false) << state_log; } } ASSERT_FALSE(status.IsOk()) << "Unexpected successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action); } status = model->InvokeAgent(next_lifecycle_action); if (!status.IsOk()) { for (const auto& state_log : log) { EXPECT_TRUE(false) << state_log; } } ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(next_lifecycle_action) << ": " << status.AsString(); } } } class TritonRepoAgentAPITest : public ::testing::Test { public: static std::function agent_init_fn_; static std::function agent_fini_fn_; static std::function model_init_fn_; static std::function model_action_fn_; static std::function model_fini_fn_; protected: void SetUp() override { simple_config_.set_name("simple_config"); // Add a agent handle for flexible testing tc::TritonRepoAgent::TritonRepoAgentInitFn_t AgentInitFn = [](TRITONREPOAGENT_Agent* agent) -> TRITONSERVER_Error* { if (agent_init_fn_ != nullptr) { agent_init_fn_(agent); } return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentFiniFn_t AgentFiniFn = [](TRITONREPOAGENT_Agent* agent) -> TRITONSERVER_Error* { if (agent_fini_fn_ != nullptr) { agent_fini_fn_(agent); } return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelInitFn_t ModelInitFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) -> TRITONSERVER_Error* { if (model_init_fn_ != nullptr) { model_init_fn_(agent, model); } return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ModelActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { if (model_action_fn_ != nullptr) { model_action_fn_(agent, model); } return nullptr; }; tc::TritonRepoAgent::TritonRepoAgentModelFiniFn_t ModelFiniFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) -> TRITONSERVER_Error* { if (model_fini_fn_ != nullptr) { model_fini_fn_(agent, model); } return nullptr; }; auto agent_handle = MockSharedLibraryHandle(); agent_handle.AddEntryPoint( "TRITONREPOAGENT_Initialize", reinterpret_cast(AgentInitFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_Finalize", reinterpret_cast(AgentFiniFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelInitialize", reinterpret_cast(ModelInitFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ModelActionFn)); agent_handle.AddEntryPoint( "TRITONREPOAGENT_ModelFinalize", reinterpret_cast(ModelFiniFn)); global_mock_agents.emplace("agent_path", agent_handle); } void TearDown() override { global_mock_agents.clear(); agent_init_fn_ = nullptr; agent_fini_fn_ = nullptr; model_init_fn_ = nullptr; model_action_fn_ = nullptr; model_fini_fn_ = nullptr; } TRITONREPOAGENT_ArtifactType original_type_ = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; const std::string original_location_ = "/original"; inference::ModelConfig simple_config_; std::vector> valid_lifecycles_{ {TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_FAIL}, {TRITONREPOAGENT_ACTION_LOAD, TRITONREPOAGENT_ACTION_LOAD_COMPLETE, TRITONREPOAGENT_ACTION_UNLOAD, TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE}}; }; std::function TritonRepoAgentAPITest::agent_init_fn_ = nullptr; std::function TritonRepoAgentAPITest::agent_fini_fn_ = nullptr; std::function TritonRepoAgentAPITest::model_init_fn_ = nullptr; std::function TritonRepoAgentAPITest::model_action_fn_ = nullptr; std::function TritonRepoAgentAPITest::model_fini_fn_ = nullptr; TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_ApiVersion) { agent_init_fn_ = [](TRITONREPOAGENT_Agent* agent) { uint32_t major = 0; uint32_t minor = 0; auto err = TRITONREPOAGENT_ApiVersion(&major, &minor); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ApiVersion() invocation: " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_EQ(major, (uint32_t)TRITONREPOAGENT_API_VERSION_MAJOR) << "Unexpected major version"; EXPECT_EQ(minor, (uint32_t)TRITONREPOAGENT_API_VERSION_MINOR) << "Unexpected major version"; } }; agent_fini_fn_ = agent_init_fn_; model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { uint32_t major = 0; uint32_t minor = 0; auto err = TRITONREPOAGENT_ApiVersion(&major, &minor); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ApiVersion() invocation: " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_EQ(major, (uint32_t)TRITONREPOAGENT_API_VERSION_MAJOR) << "Unexpected major version"; EXPECT_EQ(minor, (uint32_t)TRITONREPOAGENT_API_VERSION_MINOR) << "Unexpected major version"; } }; model_action_fn_ = model_init_fn_; model_fini_fn_ = model_init_fn_; const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_ModelRepositoryLocation) { model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { TRITONREPOAGENT_ArtifactType artifact_type = TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM; const char* location = nullptr; auto err = TRITONREPOAGENT_ModelRepositoryLocation( agent, model, &artifact_type, &location); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelRepositoryLocation(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_EQ(artifact_type, TRITONREPOAGENT_ARTIFACT_FILESYSTEM) << "Unexpected artifact type"; EXPECT_EQ(std::string(location), "/original") << "Unexpected location"; } }; model_action_fn_ = model_init_fn_; model_fini_fn_ = model_init_fn_; const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F( TritonRepoAgentAPITest, TRITONREPOAGENT_ModelRepositoryLocationAcquireRemote) { model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { TRITONREPOAGENT_ArtifactType artifact_type = TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM; const char* location = nullptr; auto err = TRITONREPOAGENT_ModelRepositoryLocationAcquire( agent, model, artifact_type, &location); if (err != nullptr) { const std::string err_msg = TRITONSERVER_ErrorMessage(err); const std::string search_msg = "Unexpected artifact type, expects " "'TRITONREPOAGENT_ARTIFACT_FILESYSTEM'"; EXPECT_NE(err_msg.find(search_msg), std::string::npos) << "Unexpected error message: '" << err_msg << "', expect '" << search_msg << "'"; TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(false) << "Expect error returned from " "TRITONREPOAGENT_ModelRepositoryLocationAcquire()"; } }; model_action_fn_ = model_init_fn_; model_fini_fn_ = model_init_fn_; const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_ModelRepositoryLocationAcquire) { model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { // Acquire, acquire (same), release TRITONREPOAGENT_ArtifactType artifact_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; const char* location = nullptr; auto err = TRITONREPOAGENT_ModelRepositoryLocationAcquire( agent, model, artifact_type, &location); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful " "TRITONREPOAGENT_ModelRepositoryLocationAcquire(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } std::string acquired_location = location; err = TRITONREPOAGENT_ModelRepositoryLocationAcquire( agent, model, artifact_type, &location); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful " "TRITONREPOAGENT_ModelRepositoryLocationAcquire(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_EQ(acquired_location, std::string(location)) << "Expect the same location is acquired"; } }; model_action_fn_ = model_init_fn_; model_fini_fn_ = model_init_fn_; const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_ModelRepositoryLocationRelease) { model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { // release (fail), acquire, release const char* location = "nonexisting_location"; auto err = TRITONREPOAGENT_ModelRepositoryLocationRelease( agent, model, location); if (err != nullptr) { const std::string search_msg = "No mutable location to be deleted"; const std::string err_msg = TRITONSERVER_ErrorMessage(err); EXPECT_NE(err_msg.find(search_msg), std::string::npos) << "Unexpected error message: '" << err_msg << "', expect '" << search_msg << "'"; TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(false) << "Expect error returned from " "TRITONREPOAGENT_ModelRepositoryLocationRelease()"; } TRITONREPOAGENT_ArtifactType artifact_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; err = TRITONREPOAGENT_ModelRepositoryLocationAcquire( agent, model, artifact_type, &location); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelRepositoryLocation(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } err = TRITONREPOAGENT_ModelRepositoryLocationRelease( agent, model, location); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelRepositoryLocation(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } }; model_action_fn_ = model_init_fn_; model_fini_fn_ = model_init_fn_; const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_ModelRepositoryUpdate) { static std::string current_location = original_location_; static TRITONREPOAGENT_ArtifactType current_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { std::string new_location = current_location + "_new"; TRITONREPOAGENT_ArtifactType artifact_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; const char* location = new_location.c_str(); auto err = TRITONREPOAGENT_ModelRepositoryUpdate( agent, model, artifact_type, location); if (err != nullptr) { const std::string search_msg = "location can only be updated during TRITONREPOAGENT_ACTION_LOAD"; const std::string err_msg = TRITONSERVER_ErrorMessage(err); EXPECT_NE(err_msg.find(search_msg), std::string::npos) << "Unexpected error message: '" << err_msg << "', expect '" << search_msg << "...'"; TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(false) << "Expect error returned from " "TRITONREPOAGENT_ModelRepositoryUpdate()"; } // Check location shouldn't be changed err = TRITONREPOAGENT_ModelRepositoryLocation( agent, model, &artifact_type, &location); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelRepositoryLocation(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_EQ(artifact_type, current_type) << "Unexpected artifact type"; EXPECT_EQ(std::string(location), current_location) << "Unexpected location"; } }; model_action_fn_ = model_init_fn_; model_fini_fn_ = model_init_fn_; // Overriding the model action function in agent handle because the action // type needs to be checked here tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ModelActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { std::string new_location = current_location + "_new"; TRITONREPOAGENT_ArtifactType artifact_type = TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM; const char* location = new_location.c_str(); auto err = TRITONREPOAGENT_ModelRepositoryUpdate( agent, model, artifact_type, location); if (action_type == TRITONREPOAGENT_ACTION_LOAD) { if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelRepositoryUpdate(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { current_location = new_location; current_type = artifact_type; } } else { if (err != nullptr) { const std::string search_msg = "location can only be updated during TRITONREPOAGENT_ACTION_LOAD"; const std::string err_msg = TRITONSERVER_ErrorMessage(err); EXPECT_NE(err_msg.find(search_msg), std::string::npos) << "Unexpected error message: '" << err_msg << "', expect '" << search_msg << "...'"; TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(false) << "Expect error returned from " "TRITONREPOAGENT_ModelRepositoryUpdate()"; } } // Check location err = TRITONREPOAGENT_ModelRepositoryLocation( agent, model, &artifact_type, &location); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelRepositoryLocation(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_EQ(artifact_type, current_type) << "Unexpected artifact type"; EXPECT_EQ(std::string(location), current_location) << "Unexpected location"; } return nullptr; }; global_mock_agents["agent_path"].AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ModelActionFn)); const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Reset location and type current_location = original_location_; current_type = TRITONREPOAGENT_ARTIFACT_FILESYSTEM; // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, current_location, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_ModelParameter) { static tc::TritonRepoAgent::Parameters expected_params{ {"key_a", "value_a"}, {"key_b", "value_b"}}; model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { uint32_t count; auto err = TRITONREPOAGENT_ModelParameterCount(agent, model, &count); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelParameterCount(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_EQ(count, expected_params.size()); } const char* parameter_name = nullptr; const char* parameter_value = nullptr; for (size_t idx = 0; idx < count; ++idx) { err = TRITONREPOAGENT_ModelParameter( agent, model, idx, ¶meter_name, ¶meter_value); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelParameter(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_EQ(std::string(parameter_name), expected_params[idx].first); EXPECT_EQ( std::string(parameter_value), expected_params[idx].second); } } // out of range err = TRITONREPOAGENT_ModelParameter( agent, model, count, ¶meter_name, ¶meter_value); if (err != nullptr) { const std::string search_msg = "index out of range for model parameters"; const std::string err_msg = TRITONSERVER_ErrorMessage(err); EXPECT_NE(err_msg.find(search_msg), std::string::npos) << "Unexpected error message: '" << err_msg << "', expect '" << search_msg << "...'"; TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(false) << "Expect error returned from TRITONREPOAGENT_ModelParameter()"; } }; model_action_fn_ = model_init_fn_; model_fini_fn_ = model_init_fn_; const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, expected_params, &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_ModelConfig) { model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { TRITONSERVER_Message* config = nullptr; auto err = TRITONREPOAGENT_ModelConfig(agent, model, 1, &config); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelConfig(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } const char* base = nullptr; size_t byte_size = 0; err = TRITONSERVER_MessageSerializeToJson(config, &base, &byte_size); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONSERVER_MessageSerializeToJson(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { const std::string search_msg = "simple_config"; const std::string serialized_config(base, byte_size); EXPECT_NE(serialized_config.find(search_msg), std::string::npos) << "Expect finding '" << search_msg << "' in returned config: " << serialized_config; } // unsupported version err = TRITONREPOAGENT_ModelConfig(agent, model, 2, &config); if (err != nullptr) { const std::string search_msg = "model configuration version 2 not supported, supported versions " "are: 1"; const std::string err_msg = TRITONSERVER_ErrorMessage(err); EXPECT_NE(err_msg.find(search_msg), std::string::npos) << "Unexpected error message: '" << err_msg << "', expect '" << search_msg << "...'"; TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(false) << "Expect error returned from TRITONREPOAGENT_ModelConfig()"; } }; model_action_fn_ = model_init_fn_; model_fini_fn_ = model_init_fn_; const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_ModelState) { model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { size_t* state = nullptr; auto err = TRITONREPOAGENT_ModelState(model, reinterpret_cast(&state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(state == nullptr) << "Expect state is not set"; } state = new size_t(0); err = TRITONREPOAGENT_ModelSetState( model, reinterpret_cast(state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelSetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); delete state; } }; model_fini_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { size_t* state = nullptr; auto err = TRITONREPOAGENT_ModelState(model, reinterpret_cast(&state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(state != nullptr) << "Expect state is set"; EXPECT_EQ(*state, size_t(0)); } // Sanity check that set state works elsewhere size_t* new_state = new size_t(*state); delete state; err = TRITONREPOAGENT_ModelSetState( model, reinterpret_cast(new_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelSetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } // Delete state before end of model lifecycle delete new_state; }; // Overriding the model action function in agent handle because the action // type needs to be checked here tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ModelActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { size_t* state = nullptr; auto err = TRITONREPOAGENT_ModelState(model, reinterpret_cast(&state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } EXPECT_TRUE(state != nullptr) << "Expect state is set"; switch (action_type) { case TRITONREPOAGENT_ACTION_LOAD: { EXPECT_EQ(*state, size_t(0)); ++*state; break; } case TRITONREPOAGENT_ACTION_LOAD_COMPLETE: { EXPECT_EQ(*state, size_t(1)); ++*state; break; } case TRITONREPOAGENT_ACTION_LOAD_FAIL: { EXPECT_EQ(*state, size_t(1)); --*state; break; } case TRITONREPOAGENT_ACTION_UNLOAD: { EXPECT_EQ(*state, size_t(2)); --*state; break; } case TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: { EXPECT_EQ(*state, size_t(1)); --*state; break; } } // Sanity check that set state works elsewhere size_t* new_state = new size_t(*state); delete state; err = TRITONREPOAGENT_ModelSetState( model, reinterpret_cast(new_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelSetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); delete new_state; } return nullptr; }; global_mock_agents["agent_path"].AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ModelActionFn)); const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::unique_ptr model; status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); for (const auto action : lifecycle) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } TEST_F(TritonRepoAgentAPITest, TRITONREPOAGENT_AgentState) { // Two models share one agent, check if agent state is properly shared agent_init_fn_ = [](TRITONREPOAGENT_Agent* agent) { size_t* state = nullptr; auto err = TRITONREPOAGENT_State(agent, reinterpret_cast(&state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_State(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(state == nullptr) << "Expect state is not set"; } state = new size_t(0); err = TRITONREPOAGENT_SetState(agent, reinterpret_cast(state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_SetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); delete state; } }; agent_fini_fn_ = [](TRITONREPOAGENT_Agent* agent) { size_t* state = nullptr; auto err = TRITONREPOAGENT_State(agent, reinterpret_cast(&state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_State(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(state != nullptr) << "Expect state is set"; EXPECT_EQ(*state, size_t(0)); } // Sanity check that set state works elsewhere size_t* new_state = new size_t(*state); delete state; err = TRITONREPOAGENT_SetState(agent, reinterpret_cast(new_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_SetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } // Delete state before end of agent lifecycle delete new_state; }; model_init_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { size_t* state = nullptr; auto err = TRITONREPOAGENT_State(agent, reinterpret_cast(&state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_State(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(state != nullptr) << "Expect state is set"; } // Agent state maybe 0 or 1 depending on the order of model lifecycle, // record that in model state to keep track of the order if ((*state == 0) || (*state == 1)) { size_t* model_state = new size_t(*state); err = TRITONREPOAGENT_ModelSetState( model, reinterpret_cast(model_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelSetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } } else { EXPECT_TRUE(false) << "Expect agent state is either 0 or 1"; } // Sanity check that set state works elsewhere ++*state; size_t* new_state = new size_t(*state); delete state; err = TRITONREPOAGENT_SetState(agent, reinterpret_cast(new_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_SetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); delete new_state; } }; model_fini_fn_ = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { size_t* model_state = nullptr; auto err = TRITONREPOAGENT_ModelState( model, reinterpret_cast(&model_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(model_state != nullptr) << "Expect state is set"; } size_t* state = nullptr; err = TRITONREPOAGENT_State(agent, reinterpret_cast(&state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_State(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(state != nullptr) << "Expect state is set"; EXPECT_EQ(*state, size_t(2) - *model_state); } // Sanity check that set state works elsewhere --*state; size_t* new_state = new size_t(*state); delete state; err = TRITONREPOAGENT_SetState(agent, reinterpret_cast(new_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_SetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); delete new_state; } // Delete state before end of model lifecycle delete model_state; }; // Overriding the model action function in agent handle because the action // type needs to be checked here tc::TritonRepoAgent::TritonRepoAgentModelActionFn_t ModelActionFn = [](TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) -> TRITONSERVER_Error* { size_t* model_state = nullptr; auto err = TRITONREPOAGENT_ModelState( model, reinterpret_cast(&model_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_ModelState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } else { EXPECT_TRUE(model_state != nullptr) << "Expect state is set"; } size_t* state = nullptr; err = TRITONREPOAGENT_State(agent, reinterpret_cast(&state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_State(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } EXPECT_TRUE(state != nullptr) << "Expect state is set"; switch (action_type) { case TRITONREPOAGENT_ACTION_LOAD: { EXPECT_EQ(*state, size_t(2) + *model_state); ++*state; break; } case TRITONREPOAGENT_ACTION_LOAD_COMPLETE: { EXPECT_EQ(*state, size_t(4) + *model_state); ++*state; break; } case TRITONREPOAGENT_ACTION_LOAD_FAIL: { EXPECT_EQ(*state, size_t(4) - *model_state); --*state; break; } case TRITONREPOAGENT_ACTION_UNLOAD: { EXPECT_EQ(*state, size_t(6) - *model_state); --*state; break; } case TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: { EXPECT_EQ(*state, size_t(4) - *model_state); --*state; break; } } // Sanity check that set state works elsewhere size_t* new_state = new size_t(*state); delete state; err = TRITONREPOAGENT_SetState(agent, reinterpret_cast(new_state)); if (err != nullptr) { EXPECT_TRUE(false) << "Expect successful TRITONREPOAGENT_SetState(): " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); delete new_state; } return nullptr; }; global_mock_agents["agent_path"].AddEntryPoint( "TRITONREPOAGENT_ModelAction", reinterpret_cast(ModelActionFn)); const auto lifecycles = valid_lifecycles_; for (const auto& lifecycle : lifecycles) { // Create agent to be associated with the model std::shared_ptr agent; auto status = tc::TritonRepoAgent::Create("agent", "agent_path", &agent); ASSERT_TRUE(status.IsOk()) << "Expect successful agent creation: " << status.AsString(); std::vector> models(2); for (auto& model : models) { status = tc::TritonRepoAgentModel::Create( original_type_, original_location_, simple_config_, agent, tc::TritonRepoAgent::Parameters(), &model); ASSERT_TRUE(status.IsOk()) << "Expect successful model creation: " << status.AsString(); } for (const auto action : lifecycle) { for (auto& model : models) { status = model->InvokeAgent(action); ASSERT_TRUE(status.IsOk()) << "Expect successful agent invocation with " << tc::TRITONREPOAGENT_ActionTypeString(action) << ": " << status.AsString(); } } } } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/request_cancellation_test.cc000066400000000000000000000262341527052371200241140ustar00rootroot00000000000000// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "gtest/gtest.h" #include "triton/core/tritonbackend.h" #include "triton/core/tritonserver.h" #define FAIL_TEST_IF_ERR(X) \ do { \ std::shared_ptr err__((X), TRITONSERVER_ErrorDelete); \ ASSERT_TRUE((err__ == nullptr)) \ << TRITONSERVER_ErrorCodeString(err__.get()) << " - " \ << TRITONSERVER_ErrorMessage(err__.get()); \ } while (false) TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { *actual_memory_type = TRITONSERVER_MEMORY_CPU; *actual_memory_type_id = preferred_memory_type_id; if (byte_size == 0) { *buffer = nullptr; *buffer_userp = nullptr; } else { void* allocated_ptr = nullptr; allocated_ptr = malloc(byte_size); if (allocated_ptr != nullptr) { *buffer = allocated_ptr; *buffer_userp = new std::string(tensor_name); } } return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { return nullptr; // Success } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { if (response != nullptr) { // Notify that the completion. std::promise* p = reinterpret_cast*>(userp); p->set_value(response); delete p; } } class RequestCancellationTest : public ::testing::Test { protected: static void SetUpTestSuite() { // Create the server... TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_TEST_IF_ERR(TRITONSERVER_ServerOptionsNew(&server_options)); FAIL_TEST_IF_ERR(TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, "./models")); FAIL_TEST_IF_ERR(TRITONSERVER_ServerOptionsSetBackendDirectory( server_options, "/opt/tritonserver/backends")); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetLogVerbose(server_options, 1)); FAIL_TEST_IF_ERR(TRITONSERVER_ServerOptionsSetRepoAgentDirectory( server_options, "/opt/tritonserver/repoagents")); FAIL_TEST_IF_ERR( TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true)); FAIL_TEST_IF_ERR(TRITONSERVER_ServerNew(&server_, server_options)); FAIL_TEST_IF_ERR(TRITONSERVER_ServerOptionsDelete(server_options)); } static void TearDownTestSuite() { FAIL_TEST_IF_ERR(TRITONSERVER_ServerDelete(server_)); } void SetUp() override { ASSERT_TRUE(server_ != nullptr) << "Server has not created"; // Wait until the server is both live and ready. size_t health_iters = 0; while (true) { bool live, ready; FAIL_TEST_IF_ERR(TRITONSERVER_ServerIsLive(server_, &live)); FAIL_TEST_IF_ERR(TRITONSERVER_ServerIsReady(server_, &ready)); if (live && ready) { break; } if (++health_iters >= 10) { FAIL() << "failed to find healthy inference server"; } std::this_thread::sleep_for(std::chrono::milliseconds(500)); } // Create allocator with common callback FAIL_TEST_IF_ERR(TRITONSERVER_ResponseAllocatorNew( &allocator_, ResponseAlloc, ResponseRelease, nullptr /* start_fn */)); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestNew( &irequest_, server_, "model", -1 /* model_version */)); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestSetReleaseCallback( irequest_, InferRequestComplete, nullptr /* request_release_userp */)); std::vector input0_shape({1, 1000}); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestAddInput( irequest_, "INPUT0", TRITONSERVER_TYPE_INT32, input0_shape.data(), input0_shape.size())); const auto input0_byte_size = sizeof(input0_data_[0]) * input0_data_.size(); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( irequest_, "INPUT0", input0_data_.data(), input0_byte_size, TRITONSERVER_MEMORY_CPU, 0)); } void TearDown() override { FAIL_TEST_IF_ERR(TRITONSERVER_ResponseAllocatorDelete(allocator_)); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestDelete(irequest_)); } static TRITONSERVER_Server* server_; TRITONSERVER_ResponseAllocator* allocator_ = nullptr; static std::vector input0_data_; TRITONSERVER_InferenceRequest* irequest_ = nullptr; }; TRITONSERVER_Server* RequestCancellationTest::server_ = nullptr; std::vector RequestCancellationTest::input0_data_(1000, 1); TEST_F(RequestCancellationTest, Cancellation) { auto p = new std::promise(); std::future future = p->get_future(); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(p))); TRITONBACKEND_Request* backend_request = reinterpret_cast(irequest_); FAIL_TEST_IF_ERR( TRITONSERVER_ServerInferAsync(server_, irequest_, nullptr /* trace */)); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestCancel(irequest_)); TRITONBACKEND_ResponseFactory* response_factory; FAIL_TEST_IF_ERR( TRITONBACKEND_ResponseFactoryNew(&response_factory, backend_request)); bool is_cancelled = false; FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestIsCancelled(irequest_, &is_cancelled)); ASSERT_TRUE(is_cancelled); is_cancelled = false; FAIL_TEST_IF_ERR(TRITONBACKEND_ResponseFactoryIsCancelled( response_factory, &is_cancelled)); ASSERT_TRUE(is_cancelled); TRITONSERVER_InferenceResponse* response = future.get(); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceResponseDelete(response)); FAIL_TEST_IF_ERR(TRITONBACKEND_ResponseFactoryDelete(response_factory)); // FIXME: Looks like there is an issue with internal request state management. // If the backend send responses before releasing the requests the state may // not be set to "RELEASED" which is allowed for converting to "INITIALIZED". std::this_thread::sleep_for(std::chrono::seconds(2)); p = new std::promise(); future = p->get_future(); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(p))); // Sending another request and the request should not be cancelled. FAIL_TEST_IF_ERR(TRITONSERVER_ServerInferAsync( server_, irequest_, nullptr /* trace */)); FAIL_TEST_IF_ERR( TRITONBACKEND_ResponseFactoryNew(&response_factory, backend_request)); is_cancelled = true; FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestIsCancelled(irequest_, &is_cancelled)); ASSERT_FALSE(is_cancelled); is_cancelled = true; FAIL_TEST_IF_ERR(TRITONBACKEND_ResponseFactoryIsCancelled( response_factory, &is_cancelled)); ASSERT_FALSE(is_cancelled); response = future.get(); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceResponseDelete(response)); FAIL_TEST_IF_ERR(TRITONBACKEND_ResponseFactoryDelete(response_factory)); } TEST_F(RequestCancellationTest, CancellationAfterRelease) { auto p = new std::promise(); std::future future = p->get_future(); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestSetResponseCallback( irequest_, allocator_, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast(p))); FAIL_TEST_IF_ERR(TRITONSERVER_ServerInferAsync( server_, irequest_, nullptr /* trace */)); TRITONBACKEND_Request* backend_request = reinterpret_cast(irequest_); TRITONBACKEND_ResponseFactory* response_factory; FAIL_TEST_IF_ERR( TRITONBACKEND_ResponseFactoryNew(&response_factory, backend_request)); FAIL_TEST_IF_ERR(TRITONBACKEND_RequestRelease( backend_request, TRITONSERVER_REQUEST_RELEASE_ALL)); bool is_cancelled = true; FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestIsCancelled(irequest_, &is_cancelled)); ASSERT_FALSE(is_cancelled); is_cancelled = true; FAIL_TEST_IF_ERR(TRITONBACKEND_ResponseFactoryIsCancelled( response_factory, &is_cancelled)); ASSERT_FALSE(is_cancelled); is_cancelled = false; FAIL_TEST_IF_ERR(TRITONSERVER_InferenceRequestCancel(irequest_)); is_cancelled = false; FAIL_TEST_IF_ERR( TRITONSERVER_InferenceRequestIsCancelled(irequest_, &is_cancelled)); ASSERT_TRUE(is_cancelled); TRITONSERVER_InferenceResponse* response = future.get(); FAIL_TEST_IF_ERR(TRITONSERVER_InferenceResponseDelete(response)); is_cancelled = false; FAIL_TEST_IF_ERR(TRITONBACKEND_ResponseFactoryIsCancelled( response_factory, &is_cancelled)); ASSERT_TRUE(is_cancelled); FAIL_TEST_IF_ERR(TRITONBACKEND_ResponseFactoryDelete(response_factory)); } int main(int argc, char** argv) { #ifdef TRITON_ENABLE_LOGGING LOG_SET_VERBOSE(2); #endif // TRITON_ENABLE_LOGGING ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/test/response_cache_test.cc000066400000000000000000001206071527052371200226700ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "cache_manager.h" #include "gtest/gtest-spi.h" #include "gtest/gtest.h" #include "memory.h" #include "triton/common/logging.h" namespace tc = triton::core; /* Mock classes for Unit Testing */ namespace triton { namespace core { // // InferenceResponseFactory // Status InferenceResponseFactory::CreateResponse( std::unique_ptr* response) const { response->reset(new InferenceResponse( model_, id_, allocator_, alloc_userp_, response_fn_, response_userp_, response_delegator_)); return Status::Success; } // // InferenceRequest // InferenceRequest::InferenceRequest( Model* model, const int64_t requested_model_version) : needs_normalization_(true), model_raw_(model), requested_model_version_(requested_model_version), flags_(0), correlation_id_(0), batch_size_(0), timeout_us_(0), collect_stats_(true) { // Unit test doesn't need actual response factory logic // or other priority/request_counting logic, it just needs // a non-null response factory object. response_factory_.reset(new InferenceResponseFactory()); } InferenceRequest::Input::Input( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count) : name_(name), datatype_(datatype), original_shape_(shape, shape + dim_count), tensor_type_(TensorType::TENSOR), data_(new MemoryReference), has_host_policy_specific_data_(false) { } // Use const global var as locals can't be returned in ModelName(), // and we don't care about the model for the unit test const std::string MODEL = "model"; const std::string& InferenceRequest::ModelName() const { return MODEL; } int64_t InferenceRequest::ActualModelVersion() const { // Not using model in unit test mock return requested_model_version_; } Status InferenceRequest::PrepareForInference() { // Remove override inputs as those are added during any previous // inference execution. inputs_.clear(); override_inputs_.clear(); // Initially show the actual inputs to be only the original // inputs. If overrides are added later they will be added to // 'inputs_'. for (auto& pr : original_inputs_) { inputs_.emplace(std::make_pair(pr.first, std::addressof(pr.second))); } // Clear the timestamps queue_start_ns_ = 0; #ifdef TRITON_ENABLE_STATS request_start_ns_ = 0; #endif // TRITON_ENABLE_STATS return Status::Success; } Status InferenceRequest::Input::DataBuffer( const size_t idx, const void** base, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) const { *base = data_->BufferAt(idx, byte_size, memory_type, memory_type_id); return Status::Success; } Status InferenceRequest::AddOriginalInput( const std::string& name, const inference::DataType datatype, const int64_t* shape, const uint64_t dim_count, InferenceRequest::Input** input) { const auto& pr = original_inputs_.emplace( std::piecewise_construct, std::forward_as_tuple(name), std::forward_as_tuple(name, datatype, shape, dim_count)); if (!pr.second) { return Status( Status::Code::INVALID_ARG, "input '" + name + "' already exists in request"); } if (input != nullptr) { *input = std::addressof(pr.first->second); } needs_normalization_ = true; return Status::Success; } Status InferenceRequest::AddOriginalInput( const std::string& name, const inference::DataType datatype, const std::vector& shape, InferenceRequest::Input** input) { return AddOriginalInput(name, datatype, &shape[0], shape.size(), input); } Status InferenceRequest::Input::AppendData( const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { if (byte_size > 0) { std::static_pointer_cast(data_)->AddBuffer( static_cast(base), byte_size, memory_type, memory_type_id); } return Status::Success; } // // InferenceResponse // InferenceResponse::InferenceResponse( const std::shared_ptr& model, const std::string& id, const ResponseAllocator* allocator, void* alloc_userp, TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp, const std::function< void(std::unique_ptr&&, const uint32_t)>& delegator) : model_(model), id_(id), allocator_(allocator), alloc_userp_(alloc_userp), response_fn_(response_fn), response_userp_(response_userp), response_delegator_(delegator), null_response_(false) { // Skip allocator logic / references in unit test } std::ostream& operator<<(std::ostream& out, const InferenceResponse& response) { out << "[0x" << std::addressof(response) << "] " << "response id: " << response.Id() << std::endl; out << "status:" << response.ResponseStatus().AsString() << std::endl; return out; } InferenceResponse::Output::~Output() { Status status = ReleaseDataBuffer(); if (!status.IsOk()) { std::cerr << "[ERROR] failed to release buffer for output '" << name_ << "': " << status.AsString(); } } Status InferenceResponse::Output::ReleaseDataBuffer() { if (allocated_buffer_ != nullptr) { free(allocated_buffer_); } allocated_buffer_ = nullptr; buffer_attributes_.SetByteSize(0); buffer_attributes_.SetMemoryType(TRITONSERVER_MEMORY_CPU); buffer_attributes_.SetMemoryTypeId(0); allocated_userp_ = nullptr; return Status::Success; } // Same as defined in infer_response.cc Status InferenceResponse::Output::DataBuffer( const void** buffer, size_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id, void** userp) const { *buffer = allocated_buffer_; *buffer_byte_size = buffer_attributes_.ByteSize(); *memory_type = buffer_attributes_.MemoryType(); *memory_type_id = buffer_attributes_.MemoryTypeId(); *userp = allocated_userp_; return Status::Success; } // Simplified version of AllocateDataBuffer for CPU memory only Status InferenceResponse::Output::AllocateDataBuffer( void** buffer, size_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { if (allocated_buffer_ != nullptr) { return Status( Status::Code::ALREADY_EXISTS, "allocated buffer for output '" + name_ + "' already exists"); } // Simplifications - CPU memory only for now if (*memory_type != TRITONSERVER_MEMORY_CPU || *memory_type_id != 0) { return Status( Status::Code::INTERNAL, "Only standard CPU memory supported for now"); } // Allocate buffer to copy to *buffer = malloc(buffer_byte_size); if (buffer == nullptr || *buffer == nullptr) { return Status( Status::Code::INTERNAL, "buffer was nullptr in AllocateDataBuffer"); } // Set relevant member variables for DataBuffer() to return allocated_buffer_ = *buffer; buffer_attributes_.SetByteSize(buffer_byte_size); buffer_attributes_.SetMemoryType(*memory_type); buffer_attributes_.SetMemoryTypeId(*memory_type_id); allocated_userp_ = nullptr; return Status::Success; } Status InferenceResponse::AddOutput( const std::string& name, const inference::DataType datatype, const std::vector& shape, InferenceResponse::Output** output) { outputs_.emplace_back(name, datatype, shape, allocator_, alloc_userp_); if (output != nullptr) { *output = std::addressof(outputs_.back()); } return Status::Success; } InferenceRequest::SequenceId::SequenceId() : sequence_label_(""), sequence_index_(0), id_type_(InferenceRequest::SequenceId::DataType::UINT64) { } InferenceRequest::SequenceId::SequenceId(const std::string& sequence_label) : sequence_label_(sequence_label), sequence_index_(0), id_type_(InferenceRequest::SequenceId::DataType::STRING) { } InferenceRequest::SequenceId::SequenceId(uint64_t sequence_index) : sequence_label_(""), sequence_index_(sequence_index), id_type_(InferenceRequest::SequenceId::DataType::UINT64) { } }} // namespace triton::core namespace helpers { // Helpers void CheckStatus(tc::Status status) { ASSERT_TRUE(status.IsOk()) << "ERROR: " << status.Message(); } void InsertWrapper( std::shared_ptr cache, tc::InferenceResponse* r, std::string key) { CheckStatus(cache->Insert(r, key)); } void LookupWrapper( std::shared_ptr cache, tc::InferenceResponse* r, std::string key) { CheckStatus(cache->Lookup(r, key)); } void LookupWrapperMaybeMiss( std::shared_ptr cache, tc::InferenceResponse* r, std::string key) { auto status = cache->Lookup(r, key); // Success and Cache Miss OK auto ok = (status.IsOk() || status.StatusCode() == tc::Status::Code::NOT_FOUND); ASSERT_TRUE(ok) << "ERROR: " << status.Message(); } void reset_response( std::unique_ptr* response, tc::InferenceRequest* request) { helpers::CheckStatus(request->ResponseFactory()->CreateResponse(response)); } // Only support 1-Dimensional data to keep it simple struct Tensor { std::string name; std::vector data; }; // Only support 1-Dimensional data to keep it simple std::unique_ptr GenerateResponse( const tc::InferenceRequest* request, inference::DataType dtype, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, const std::vector& outputs) { std::cout << "Create response object" << std::endl; std::unique_ptr response; helpers::CheckStatus(request->ResponseFactory()->CreateResponse(&response)); std::cout << "Add output metadata to response object" << std::endl; for (const auto& tensor : outputs) { if (tensor.data.size() == 0) { std::cout << "[ERROR] Can't generate a request with no output data" << std::endl; return nullptr; } tc::InferenceResponse::Output* response_output = nullptr; std::vector shape{1, -1}; shape[1] = tensor.data.size(); uint64_t output_size = sizeof(tensor.data[0]) * tensor.data.size(); helpers::CheckStatus( response->AddOutput(tensor.name, dtype, shape, &response_output)); std::cout << "Allocate output data buffer for response object of size: " << output_size << std::endl; void* buffer; helpers::CheckStatus(response_output->AllocateDataBuffer( &buffer, output_size, &memory_type, &memory_type_id)); if (buffer == nullptr) { std::cout << "[ERROR] buffer was nullptr;" << std::endl; return nullptr; } // Copy data from output to response buffer std::memcpy(buffer, tensor.data.data(), output_size); } return response; } // Only support 1-Dimensional data to keep it simple tc::InferenceRequest* GenerateRequest( tc::Model* model, uint64_t model_version, inference::DataType dtype, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, const std::vector& inputs, const std::string& request_id) { auto request = new tc::InferenceRequest(model, model_version); for (const auto& tensor : inputs) { if (tensor.data.size() == 0) { std::cout << "[ERROR] Can't generate a request with no input data" << std::endl; return nullptr; } tc::InferenceRequest::Input* request_input = nullptr; std::vector shape{1, -1}; shape[1] = tensor.data.size(); request->AddOriginalInput(tensor.name, dtype, shape, &request_input); if (request_input == nullptr) { std::cout << "[ERROR] request_input was nullptr" << std::endl; return nullptr; } uint64_t input_size = sizeof(tensor.data[0]) * tensor.data.size(); request_input->AppendData( tensor.data.data(), input_size, memory_type, memory_type_id); } // PrepareForInference for use of ImmutableInputs() helpers::CheckStatus(request->PrepareForInference()); request->SetId(request_id); // for debugging purposes return request; } tc::Status InsertLookupCompare( std::shared_ptr cache, std::vector> expected_buffers, const std::string& key) { if (!cache) { return tc::Status(tc::Status::Code::INTERNAL, "cache was nullptr"); } else if (expected_buffers.empty()) { return tc::Status(tc::Status::Code::INTERNAL, "entry was empty"); } helpers::CheckStatus(cache->Insert(expected_buffers, key)); auto lookup_entry = std::make_unique(); auto status = cache->Lookup(key, lookup_entry.get()); if (!status.IsOk()) { return tc::Status( tc::Status::Code::INTERNAL, "Lookup failed: " + status.Message()); } auto lookup_buffers = lookup_entry->Buffers(); if (lookup_buffers.size() != expected_buffers.size()) { return tc::Status( tc::Status::Code::INTERNAL, "Expected " + std::to_string(expected_buffers.size()) + " got " + std::to_string(lookup_buffers.size())); } for (size_t b = 0; b < expected_buffers.size(); b++) { boost::span lookup = { static_cast(lookup_buffers[b].first), lookup_buffers[b].second}; boost::span expected = expected_buffers[b]; if (!std::equal( lookup.begin(), lookup.end(), expected.begin(), expected.end())) { return tc::Status( tc::Status::Code::INTERNAL, "Buffer bytes didn't match for test input"); } } return tc::Status::Success; } std::shared_ptr CreateLocalCache(uint64_t cache_size) { // Create TritonCacheManager std::shared_ptr cache_manager; auto cache_dir = "/opt/tritonserver/caches"; helpers::CheckStatus( tc::TritonCacheManager::Create(&cache_manager, cache_dir)); // Create TritonCache std::shared_ptr cache; auto cache_config = R"({"size": )" + std::to_string(cache_size) + "}"; std::cout << "Creating local cache with config: " << cache_config << std::endl; auto cache_name = "local"; helpers::CheckStatus( cache_manager->CreateCache(cache_name, cache_config, &cache)); return cache; } std::shared_ptr CreateRedisCache(std::string host, std::string port) { // Create TritonCacheManager std::shared_ptr cache_manager; auto cache_dir = "/opt/tritonserver/caches"; helpers::CheckStatus( tc::TritonCacheManager::Create(&cache_manager, cache_dir)); // Create TritonCache std::shared_ptr cache; std::ostringstream cache_config_json; auto cache_config = R"({"host": ")" + host + R"(", "port": ")" + port + R"("})"; std::cout << "Creating redis cache with config: " << cache_config << std::endl; auto cache_name = "redis"; helpers::CheckStatus( cache_manager->CreateCache(cache_name, cache_config, &cache)); return cache; } void CreateCacheExpectFail( const std::string& cache_name, const std::string& cache_config) { // Create TritonCacheManager std::shared_ptr cache_manager; auto cache_dir = "/opt/tritonserver/caches"; helpers::CheckStatus( tc::TritonCacheManager::Create(&cache_manager, cache_dir)); // Create TritonCache std::shared_ptr cache; auto status = cache_manager->CreateCache(cache_name, cache_config, &cache); ASSERT_FALSE(status.IsOk()) << "Creating cache with config: '" << cache_config << "' succeeded when it should fail."; ASSERT_EQ(cache, nullptr); } } // namespace helpers namespace { // Test Fixture class RequestResponseCacheTest : public ::testing::Test { protected: void SetUp() override { // Sample input data data0 = {1, 2, 3, 4}; data1 = {5, 6, 7, 8}; // Sample input vectors inputs0 = std::vector{{"input", data0}}; inputs1 = std::vector{{"input", data1}}; inputs2 = std::vector{{"input", data1}}; inputs3 = std::vector{{"input0", data0}, {"input1", data1}}; inputs4 = std::vector{{"input1", data1}, {"input0", data0}}; // Create three requests with same input name, two with same data, one with // different data request0 = helpers::GenerateRequest( model, model_version, dtype, memory_type, memory_type_id, inputs0, "request0"); request1 = helpers::GenerateRequest( model, model_version, dtype, memory_type, memory_type_id, inputs1, "request1"); request2 = helpers::GenerateRequest( model, model_version, dtype, memory_type, memory_type_id, inputs2, "request2"); // Create two requests with the same two inputs but inserted in different // order request3 = helpers::GenerateRequest( model, model_version, dtype, memory_type, memory_type_id, inputs3, "request3"); request4 = helpers::GenerateRequest( model, model_version, dtype, memory_type, memory_type_id, inputs4, "request4"); // Verify requests were created correctly ASSERT_NE(request0, nullptr); ASSERT_NE(request1, nullptr); ASSERT_NE(request2, nullptr); ASSERT_NE(request3, nullptr); ASSERT_NE(request4, nullptr); // Generate a set of unique requests to use for parallelism tests for (size_t idx = 0; idx < thread_count; idx++) { std::vector data(thread_count, static_cast(idx)); std::vector inputs{ helpers::Tensor{"input" + std::to_string(idx), data}}; std::string request_id = "unique" + std::to_string(idx); auto request = helpers::GenerateRequest( model, model_version, dtype, memory_type, memory_type_id, inputs, request_id); ASSERT_NE(request, nullptr); unique_requests.emplace_back(request); } ASSERT_EQ(unique_requests.size(), thread_count); // Sample outputs helpers::Tensor output_tensor0 = {"output", data0}; outputs0 = std::vector{output_tensor0}; // Response of 100 ints, taking ~400 bytes at a time data100 = std::vector(100, 0); std::iota(data100.begin(), data100.end(), 1); output100_size = sizeof(int) * data100.size(); helpers::Tensor output_tensor100 = {"output", data100}; outputs100 = std::vector{output_tensor100}; // Sample responses response0 = helpers::GenerateResponse( request0, dtype, memory_type, memory_type_id, outputs0); ASSERT_NE(response0, nullptr); response_400bytes = helpers::GenerateResponse( request0, dtype, memory_type, memory_type_id, outputs100); ASSERT_NE(response_400bytes, nullptr); // Redis cache config auto rh = std::getenv("TRITON_REDIS_HOST"); if (rh) { redis_host = rh; } auto rp = std::getenv("TRITON_REDIS_PORT"); if (rp) { redis_port = rp; } } void TearDown() override { delete request0; delete request1; delete request2; delete request3; delete request4; for (auto r : unique_requests) { delete r; } } public: tc::Model* model = nullptr; uint64_t model_version = 1; inference::DataType dtype = inference::DataType::TYPE_INT32; TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; int64_t memory_type_id = 0; size_t thread_count = 100; uint64_t output100_size; std::string redis_host = "localhost"; std::string redis_port = "6379"; std::vector data0, data1, data100; std::vector inputs0, inputs1, inputs2, inputs3, inputs4, inputs100; std::vector outputs0, outputs100; tc::InferenceRequest *request0, *request1, *request2, *request3, *request4; std::vector unique_requests; std::unique_ptr response0, response_400bytes; }; // Group common cache tests into namespace for testing multiple implementations namespace tests { void InsertLookupCompareBytes(std::shared_ptr cache) { // Setup byte buffers std::vector buffer1{1, tc::Byte{1}}; std::vector buffer2{2, tc::Byte{2}}; std::vector buffer3{4, tc::Byte{4}}; std::vector buffer4{8, tc::Byte{8}}; std::vector buffer5{16, tc::Byte{16}}; // Setup entry std::vector> entry; // Add buffers to entry entry.push_back(buffer1); entry.push_back(buffer2); entry.push_back(buffer3); entry.push_back(buffer4); entry.push_back(buffer5); helpers::CheckStatus( helpers::InsertLookupCompare(cache, entry, "TestCacheEntry")); } // Hash a collection of unique requests and assert no collisions occurred void HashUnique( std::shared_ptr cache, std::vector& unique_requests) { ASSERT_NE(unique_requests.size(), 0); std::vector hashes; for (const auto& request : unique_requests) { std::string hash = ""; helpers::CheckStatus(cache->Hash(*request, &hash)); ASSERT_NE(hash, ""); hashes.push_back(hash); } ASSERT_NE(hashes.size(), 0); // Verify no two hashes from the unique requests are the same for (size_t i = 0; i < hashes.size(); i++) { for (size_t j = 0; j < hashes.size(); j++) { if (i == j) { continue; } ASSERT_NE(hashes[i], hashes[j]); } } } // Hash specifically crafted requests to verify their hashes are as expected void HashLogic( std::shared_ptr cache, tc::InferenceRequest* request0, tc::InferenceRequest* request1, tc::InferenceRequest* request2, tc::InferenceRequest* request3, tc::InferenceRequest* request4) { std::string hash0, hash1, hash2, hash3, hash4; helpers::CheckStatus(cache->Hash(*request0, &hash0)); helpers::CheckStatus(cache->Hash(*request1, &hash1)); helpers::CheckStatus(cache->Hash(*request2, &hash2)); helpers::CheckStatus(cache->Hash(*request3, &hash3)); helpers::CheckStatus(cache->Hash(*request4, &hash4)); // Different input data should have different hashes ASSERT_NE(hash0, hash1); // Same input data should have same hashes ASSERT_EQ(hash1, hash2); // Two requests with same two inputs but added in different orders ASSERT_EQ(hash3, hash4); } void ParallelInsert( std::shared_ptr cache, size_t thread_count, std::unique_ptr& insert_response, size_t expected_cache_hits) { // Create threads std::vector threads; std::cout << "Insert responses into cache with [" << thread_count << "] threads in parallel" << std::endl; for (size_t idx = 0; idx < thread_count; idx++) { auto key = std::to_string(idx); threads.emplace_back(std::thread( &helpers::InsertWrapper, cache, insert_response.get(), key)); } // Join threads for (size_t idx = 0; idx < thread_count; idx++) { threads[idx].join(); } // Lookup each inserted key to verify that expected number remain in cache size_t cache_hits = 0; size_t cache_misses = 0; for (size_t idx = 0; idx < thread_count; idx++) { auto key = std::to_string(idx); auto entry = std::make_unique(); auto status = cache->Lookup(key, entry.get()); if (status.IsOk()) { cache_hits++; } else { cache_misses++; } } ASSERT_EQ(cache_hits, expected_cache_hits); ASSERT_EQ(cache_hits + cache_misses, thread_count); } void ParallelLookup( std::shared_ptr cache, size_t thread_count, std::unique_ptr& insert_response, std::vector& unique_requests, std::vector expected_outputs) { const size_t expected_cache_hits = thread_count; constexpr size_t expected_cache_misses = 0; // Create threads std::vector threads; std::vector> responses; // Insert [thread_count] entries into cache sequentially for (size_t idx = 0; idx < thread_count; idx++) { // Create response for each thread to fill from cache std::unique_ptr response; helpers::CheckStatus( unique_requests[idx]->ResponseFactory()->CreateResponse(&response)); responses.push_back(std::move(response)); // Insert response for each thread auto key = std::to_string(idx); cache->Insert(insert_response.get(), key); } // Assert all entries were put into cache and no evictions occurred yet size_t cache_hits = 0; size_t cache_misses = 0; for (size_t idx = 0; idx < thread_count; idx++) { auto key = std::to_string(idx); auto entry = std::make_unique(); auto status = cache->Lookup(key, entry.get()); if (status.IsOk()) { cache_hits++; } else { std::cout << "ERROR: " << status.Message() << std::endl; cache_misses++; } } ASSERT_EQ(cache_hits, expected_cache_hits); ASSERT_EQ(cache_misses, expected_cache_misses); ASSERT_EQ(cache_hits + cache_misses, thread_count); std::cout << "Lookup from cache with [" << thread_count << "] threads in parallel" << std::endl; for (size_t idx = 0; idx < thread_count; idx++) { auto key = std::to_string(idx); threads.emplace_back( std::thread(&helpers::LookupWrapper, cache, responses[idx].get(), key)); } // Join threads for (size_t idx = 0; idx < thread_count; idx++) { threads[idx].join(); } // Grab output from sample response for comparison const auto& response0_output = insert_response->Outputs()[0]; // Verify output results from cache for (size_t idx = 0; idx < thread_count; idx++) { // Fetch output buffer details const void* response_buffer = nullptr; size_t response_byte_size = 0; TRITONSERVER_MemoryType response_memory_type; int64_t response_memory_type_id; void* userp; // TODO: Handle multiple outputs more generically const auto& response_test = responses[idx]; for (const auto& response_test_output : response_test->Outputs()) { ASSERT_EQ(response_test_output.Name(), response0_output.Name()); ASSERT_EQ(response_test_output.DType(), response0_output.DType()); ASSERT_EQ(response_test_output.Shape(), response0_output.Shape()); helpers::CheckStatus(response_test_output.DataBuffer( &response_buffer, &response_byte_size, &response_memory_type, &response_memory_type_id, &userp)); // TODO: Use Triton DType to cast buffer and compare outputs generically const int* cache_output = static_cast(response_buffer); for (size_t i = 0; i < response_byte_size / sizeof(int); i++) { ASSERT_EQ(cache_output[i], expected_outputs[i]); } } } } // Run Inserts/Lookups in parallel to check for race conditions, deadlocks, etc void ParallelLookupInsert( std::shared_ptr cache, size_t thread_count, std::unique_ptr& insert_response, std::vector& unique_requests) { // Create threads std::vector insert_threads; std::vector lookup_threads; std::vector> responses; std::cout << "Create responses" << std::endl; for (size_t idx = 0; idx < thread_count; idx++) { // Create response for each thread to fill from cache std::unique_ptr response; helpers::CheckStatus( unique_requests[idx]->ResponseFactory()->CreateResponse(&response)); responses.push_back(std::move(response)); } // Insert then Lookup [thread_count] entries from cache in parallel std::cout << "Insert and Lookup responses into cache with [" << thread_count << "] threads in parallel" << std::endl; for (size_t idx = 0; idx < thread_count; idx++) { auto key = std::to_string(idx); insert_threads.emplace_back(std::thread( &helpers::InsertWrapper, cache, insert_response.get(), key)); lookup_threads.emplace_back(std::thread( &helpers::LookupWrapperMaybeMiss, cache, responses[idx].get(), key)); } // Join threads for (size_t idx = 0; idx < thread_count; idx++) { insert_threads[idx].join(); lookup_threads[idx].join(); } } void EndToEnd( std::shared_ptr cache, tc::InferenceRequest* request, std::unique_ptr& response, const std::vector& expected_outputs) { std::string key = ""; helpers::CheckStatus(cache->Hash(*request, &key)); ASSERT_NE(key, ""); std::cout << "Lookup request in empty cache" << std::endl; auto status = cache->Lookup(nullptr, key); // This hash not in cache yet ASSERT_FALSE(status.IsOk()) << "hash [" + key + "] should not be in cache"; // Insertion should succeed helpers::CheckStatus(cache->Insert(response.get(), key)); // Duplicate insertion should fail since request already exists in cache status = cache->Insert(response.get(), key); // Cache implementations may choose behavior for duplicate insertion if (cache->Name() == "redis") { ASSERT_TRUE(status.IsOk()) << "Inserting duplicate item in cache should succeed for redis cache"; } else { ASSERT_FALSE(status.IsOk()) << "Inserting duplicate item in cache should fail unless " "implementation " << "explicitly allows it and is specified here."; } // Create response to test cache lookup std::cout << "Create response object into fill from cache" << std::endl; std::unique_ptr response_test; helpers::CheckStatus( request->ResponseFactory()->CreateResponse(&response_test)); // Lookup should now succeed std::cout << "Lookup request in cache after insertion" << std::endl; helpers::CheckStatus(cache->Lookup(response_test.get(), key)); // Grab output from sample response for comparison const auto& response0_output = response->Outputs()[0]; // Fetch output buffer details const void* response_buffer = nullptr; size_t response_byte_size = 0; TRITONSERVER_MemoryType response_memory_type; int64_t response_memory_type_id; void* userp; // TODO: Handle multiple outputs and memory types more generically for (const auto& response_test_output : response_test->Outputs()) { ASSERT_EQ(response_test_output.Name(), response0_output.Name()); ASSERT_EQ(response_test_output.DType(), response0_output.DType()); ASSERT_EQ(response_test_output.Shape(), response0_output.Shape()); helpers::CheckStatus(response_test_output.DataBuffer( &response_buffer, &response_byte_size, &response_memory_type, &response_memory_type_id, &userp)); } // TODO: Use Triton DType to cast buffer and compare outputs generically const int* cache_output = static_cast(response_buffer); for (size_t i = 0; i < response_byte_size / sizeof(int); i++) { ASSERT_EQ(cache_output[i], expected_outputs[0].data[i]); } } } // namespace tests // // Local Cache Testing // // Currently, cache size and eviction related tests are specific to the local // cache implementation. // // Other tests related to hashing, insertion, and lookups are fairly agnostic // to the cache implementation. // // Test cache size too small to initialize. TEST_F(RequestResponseCacheTest, TestLocalCacheSizeTooSmall) { // Pick intentionally small cache size, expecting failure constexpr uint64_t cache_size = 1; auto cache_config = R"({"size": )" + std::to_string(cache_size) + "}"; std::cout << "Create cache of size: " << cache_size << std::endl; helpers::CreateCacheExpectFail("local", cache_config); } // Test cache size too large to initialize. TEST_F(RequestResponseCacheTest, TestLocalCacheSizeTooLarge) { // Pick intentionally large cache size, expecting failure constexpr uint64_t cache_size = ULLONG_MAX; auto cache_config = R"({"size": )" + std::to_string(cache_size) + "}"; std::cout << "Create cache of size: " << cache_size << std::endl; helpers::CreateCacheExpectFail("local", cache_config); } TEST_F(RequestResponseCacheTest, TestLocalCacheSizeSmallerThanEntryBytes) { constexpr uint64_t cache_size = 4 * 1024 * 1024; // 4 MB, arbitrary auto cache = helpers::CreateLocalCache(cache_size); ASSERT_NE(cache, nullptr); // Setup byte buffer larger than cache size std::vector large_data(cache_size + 1); // Setup entry std::vector> entry; entry.push_back(large_data); auto status = cache->Insert(entry, "large_bytes"); // We expect insertion to fail here since cache is too small ASSERT_FALSE(status.IsOk()) << "Inserting item larger than cache succeeded when it should fail"; } TEST_F(RequestResponseCacheTest, TestLocalCacheSizeSmallerThanEntryResponse) { constexpr uint64_t cache_size = 4 * 1024 * 1024; // 4 MB, arbitrary auto cache = helpers::CreateLocalCache(cache_size); ASSERT_NE(cache, nullptr); // Set output data to be larger than cache size // NOTE: This is not 1 byte larger than cache_size, the cache_size + 1 is to // be clear it will always be larger than cache even if the dtype is changed. std::vector large_data(cache_size + 1, 0); std::cout << "Create large_response (larger than cache) of size: " << large_data.size() << std::endl; std::vector large_outputs{ helpers::Tensor{"output", large_data}}; auto large_response = helpers::GenerateResponse( request0, dtype, memory_type, memory_type_id, large_outputs); std::cout << "Insert large_response into cache" << std::endl; auto status = cache->Insert(large_response.get(), "large_response"); // We expect insertion to fail here since cache is too small ASSERT_FALSE(status.IsOk()) << "Inserting item larger than cache succeeded when it should fail"; } TEST_F(RequestResponseCacheTest, TestLocalCacheEvictionLRU) { // Set size 1200 to hold exactly 2x (400byte + metadata) responses, not 3x auto cache = helpers::CreateLocalCache(1200); ASSERT_NE(cache, nullptr); // Insert 2 responses, expecting both to fit in cache helpers::CheckStatus(cache->Insert(response_400bytes.get(), "request0")); helpers::CheckStatus(cache->Insert(response_400bytes.get(), "request1")); // Validate both responses fit in cache by looking them up tc::CacheEntry entry0, entry1, entry2, entry3, entry4, entry5, entry6, entry7; auto status = cache->Lookup("request0", &entry0); ASSERT_TRUE(status.IsOk()) << status.Message(); ASSERT_TRUE(cache->Lookup("request1", &entry1).IsOk()); // Insert a 3rd response, expecting the 1st response to be evicted // in LRU order helpers::CheckStatus(cache->Insert(response_400bytes.get(), "request2")); ASSERT_TRUE(cache->Lookup("request2", &entry2).IsOk()); ASSERT_FALSE(cache->Lookup("request0", &entry3).IsOk()); // Lookup 2nd request to bump its LRU order over 3rd ASSERT_TRUE(cache->Lookup("request1", &entry4).IsOk()); // Insert a 4th response, expecting the 3rd to get evicted by LRU order // after looking up the 2nd helpers::CheckStatus(cache->Insert(response_400bytes.get(), "request3")); ASSERT_TRUE(cache->Lookup("request3", &entry5).IsOk()); ASSERT_TRUE(cache->Lookup("request1", &entry6).IsOk()); ASSERT_FALSE(cache->Lookup("request2", &entry7).IsOk()); } TEST_F(RequestResponseCacheTest, TestLocalCacheInsertLookupCompareBytes) { auto cache = helpers::CreateLocalCache(1024); ASSERT_NE(cache, nullptr); tests::InsertLookupCompareBytes(cache); } // This test isn't cache implementation specific since hashing is done // in Triton core internally for now, but hashing may be exposed to // implementations in the future. TEST_F(RequestResponseCacheTest, TestLocalCacheHashing) { auto cache = helpers::CreateLocalCache(1024); ASSERT_NE(cache, nullptr); tests::HashLogic(cache, request0, request1, request2, request3, request4); tests::HashUnique(cache, unique_requests); } TEST_F(RequestResponseCacheTest, TestLocalCacheParallelInsert) { // Set size 1200 to hold exactly 2x (400byte + metadata) responses, not 3x auto cache = helpers::CreateLocalCache(1200); ASSERT_NE(cache, nullptr); const size_t expected_cache_hits = 2; tests::ParallelInsert( cache, thread_count, response_400bytes, expected_cache_hits); } TEST_F(RequestResponseCacheTest, TestLocalCacheParallelLookup) { // Set size large enough to hold all responses auto cache = helpers::CreateLocalCache(2 * thread_count * output100_size); ASSERT_NE(cache, nullptr); tests::ParallelLookup( cache, thread_count, response_400bytes, unique_requests, data100); } TEST_F(RequestResponseCacheTest, TestLocalCacheParallelLookupInsert) { // Set size that can hold a few responses but will certainly // run into evictions auto cache = helpers::CreateLocalCache(1024); ASSERT_NE(cache, nullptr); tests::ParallelLookupInsert( cache, thread_count, response_400bytes, unique_requests); } TEST_F(RequestResponseCacheTest, TestLocalCacheEndToEnd) { auto cache = helpers::CreateLocalCache(8 * 1024 * 1024); ASSERT_NE(cache, nullptr); tests::EndToEnd(cache, request0, response0, outputs0); } // // Redis Cache Testing // // The following tests are fairly agnostic to cache implementation, // there are no tests around specific Redis settings or eviction // policies at this time, and instead tests Redis's default settings. // // NOTE: These tests require a Redis server to already be running and // accessible. There is an assumed host:port of localhost:6379 for testing // purposes, but these can be configured via TRITON_REDIS_HOST and // TRITON_REDIS_PORT env vars. // TEST_F(RequestResponseCacheTest, TestRedisCacheInsertLookupCompareBytes) { auto cache = helpers::CreateRedisCache(redis_host, redis_port); ASSERT_NE(cache, nullptr); tests::InsertLookupCompareBytes(cache); } // This test isn't cache implementation specific since hashing is done // in Triton core internally for now, but hashing may be exposed to // implementations in the future. TEST_F(RequestResponseCacheTest, TestRedisCacheHashing) { auto cache = helpers::CreateRedisCache(redis_host, redis_port); ASSERT_NE(cache, nullptr); tests::HashLogic(cache, request0, request1, request2, request3, request4); tests::HashUnique(cache, unique_requests); } TEST_F(RequestResponseCacheTest, TestRedisCacheParallelInsert) { auto cache = helpers::CreateRedisCache(redis_host, redis_port); ASSERT_NE(cache, nullptr); // Don't expect any cache misses from Redis by default. // Future tests can set a fixed size and eviction policy on Redis. // For now, no eviction policy testing is done on Redis cache. const size_t expected_cache_hits = thread_count; tests::ParallelInsert( cache, thread_count, response_400bytes, expected_cache_hits); } TEST_F(RequestResponseCacheTest, TestRedisCacheParallelLookup) { auto cache = helpers::CreateRedisCache(redis_host, redis_port); ASSERT_NE(cache, nullptr); tests::ParallelLookup( cache, thread_count, response_400bytes, unique_requests, data100); } TEST_F(RequestResponseCacheTest, TestRedisCacheParallelLookupInsert) { auto cache = helpers::CreateRedisCache(redis_host, redis_port); ASSERT_NE(cache, nullptr); tests::ParallelLookupInsert( cache, thread_count, response_400bytes, unique_requests); } TEST_F(RequestResponseCacheTest, TestRedisCacheEndToEnd) { auto cache = helpers::CreateRedisCache(redis_host, redis_port); ASSERT_NE(cache, nullptr); tests::EndToEnd(cache, request0, response0, outputs0); } } // namespace int main(int argc, char** argv) { #ifdef TRITON_ENABLE_LOGGING LOG_SET_VERBOSE(2); #endif // TRITON_ENABLE_LOGGING ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } kytensor-2.49.0.7/core/src/tritoncache.cc000066400000000000000000000172651527052371200202010ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include "cache_manager.h" // For unknown reason, windows will not export the TRITONCACHE_* // functions declared with dllexport in tritoncache.h. To get // those functions exported it is (also?) necessary to mark the // definitions in this file with dllexport as well. #if defined(_MSC_VER) #define TRITONAPI_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONAPI_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONAPI_DECLSPEC #endif namespace triton { namespace core { extern "C" { // // TRITONCACHE API Version // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONCACHE_ApiVersion(uint32_t* major, uint32_t* minor) { *major = TRITONCACHE_API_VERSION_MAJOR; *minor = TRITONCACHE_API_VERSION_MINOR; return nullptr; // success } // // CacheEntry Lifetime Management // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntryNew(TRITONCACHE_CacheEntry** entry) { if (entry == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "entry was nullptr"); } *entry = reinterpret_cast(new CacheEntry()); return nullptr; } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntryDelete(TRITONCACHE_CacheEntry* entry) { if (entry == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "entry was nullptr"); } delete reinterpret_cast(entry); return nullptr; } // // CacheEntry Field Management // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntryBufferCount(TRITONCACHE_CacheEntry* entry, size_t* count) { if (entry == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "entry was nullptr"); } const auto lentry = reinterpret_cast(entry); *count = lentry->BufferCount(); return nullptr; // success } // Adds buffer to entry TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntryAddBuffer( TRITONCACHE_CacheEntry* entry, void* base, TRITONSERVER_BufferAttributes* attrs) { if (!entry || !base || !attrs) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "entry, base, or attrs was nullptr"); } // Get buffer attributes set by caller size_t byte_size = 0; TRITONSERVER_BufferAttributesByteSize(attrs, &byte_size); if (!byte_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Buffer byte size was zero"); } TRITONSERVER_MemoryType memory_type; TRITONSERVER_BufferAttributesMemoryType(attrs, &memory_type); // DLIS-2673: Add better memory_type support if (memory_type != TRITONSERVER_MEMORY_CPU && memory_type != TRITONSERVER_MEMORY_CPU_PINNED) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Only buffers in CPU memory are allowed in cache currently"); } const auto lentry = reinterpret_cast(entry); // This will add a short-lived reference to the corresponding cache // buffer of this entry. It should be copied into the target buffer either // directly or through a callback. lentry->AddBuffer({static_cast(base), byte_size}); return nullptr; // success } // Gets buffer at index from entry TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntryGetBuffer( TRITONCACHE_CacheEntry* entry, size_t index, void** base, TRITONSERVER_BufferAttributes* attrs) { if (!entry || !base || !attrs) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "entry, base, or attrs was nullptr"); } const auto lentry = reinterpret_cast(entry); const auto& lbuffers = lentry->Buffers(); if (index >= lbuffers.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "index was greater than count"); } const auto& buffer = lbuffers[index].first; const auto& buffer_size = lbuffers[index].second; // No copy, this buffer needs to stay alive until it is copied into the cache *base = buffer; // Set buffer attributes TRITONSERVER_BufferAttributesSetByteSize(attrs, buffer_size); // DLIS-2673: Add better memory_type support, default to CPU memory for now TRITONSERVER_BufferAttributesSetMemoryType(attrs, TRITONSERVER_MEMORY_CPU); TRITONSERVER_BufferAttributesSetMemoryTypeId(attrs, 0); return nullptr; // success } // Sets buffer at index in entry TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONCACHE_CacheEntrySetBuffer( TRITONCACHE_CacheEntry* entry, size_t index, void* new_base, TRITONSERVER_BufferAttributes* attrs) { if (!entry) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "entry was nullptr"); } const auto lentry = reinterpret_cast(entry); auto& lbuffers = lentry->MutableBuffers(); if (index >= lbuffers.size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "index was greater than count"); } auto& base = lbuffers[index].first; auto& buffer_size = lbuffers[index].second; base = new_base; // Only overwrite attributes if provided, buffer may already have some and // not need to change if new buffer shares the same properties if (attrs) { size_t byte_size = 0; TRITONSERVER_BufferAttributesByteSize(attrs, &byte_size); // Overwrite corresponding buffer size if provided buffer_size = byte_size; TRITONSERVER_MemoryType memory_type; TRITONSERVER_BufferAttributesMemoryType(attrs, &memory_type); // DLIS-2673: Add better memory_type support if (memory_type != TRITONSERVER_MEMORY_CPU && memory_type != TRITONSERVER_MEMORY_CPU_PINNED) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Only buffers in CPU memory are allowed in cache currently"); } } return nullptr; // success } TRITONSERVER_Error* TRITONCACHE_Copy( TRITONCACHE_Allocator* allocator, TRITONCACHE_CacheEntry* entry) { if (!allocator || !entry) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "allocator or entry was nullptr"); } const auto lallocator = reinterpret_cast(allocator); RETURN_TRITONSERVER_ERROR_IF_ERROR(lallocator->Allocate(entry)); return nullptr; // success } } // extern C }} // namespace triton::core kytensor-2.49.0.7/core/src/tritonserver.cc000066400000000000000000003470521527052371200204440ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "buffer_attributes.h" #include "cuda_utils.h" #include "infer_parameter.h" #include "infer_request.h" #include "infer_response.h" #include "infer_stats.h" #include "metric_family.h" #include "metrics.h" #include "model.h" #include "model_config_utils.h" #include "model_repository_manager/model_repository_manager.h" #include "rate_limiter.h" #include "response_allocator.h" #include "server.h" #include "server_message.h" #include "status.h" #include "triton/common/logging.h" #include "triton/common/model_config.h" #include "triton/common/nvtx.h" #include "triton/common/table_printer.h" #include "triton/common/triton_json.h" #include "tritonserver_apis.h" // For unknown reason, windows will not export some functions declared // with dllexport in tritonrepoagent.h and tritonbackend.h. To get // those functions exported it is (also?) necessary to mark the // definitions in this file with dllexport as well. The TRITONSERVER_* // functions are getting exported but for consistency adding the // declspec to these definitions as well. #if defined(_MSC_VER) #define TRITONAPI_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONAPI_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONAPI_DECLSPEC #endif namespace tc = triton::core; namespace { std::string ResourceString(const std::string& name, const int count, const int device_id) { return std::string( "{\"name\":\"" + name + "\", \"count\":" + std::to_string(count) + " \"device\":" + std::to_string(device_id) + "}"); } std::string RateLimitModeToString(const tc::RateLimitMode rate_limit_mode) { std::string rl_mode_str(""); switch (rate_limit_mode) { case tc::RateLimitMode::RL_EXEC_COUNT: { rl_mode_str = "EXEC_COUNT"; break; } case tc::RateLimitMode::RL_OFF: { rl_mode_str = "OFF"; break; } } return rl_mode_str; } // // TritonServerError // // Implementation for TRITONSERVER_Error. // class TritonServerError { public: static TRITONSERVER_Error* Create( TRITONSERVER_Error_Code code, const char* msg); static TRITONSERVER_Error* Create( TRITONSERVER_Error_Code code, const std::string& msg); static TRITONSERVER_Error* Create(const tc::Status& status); TRITONSERVER_Error_Code Code() const { return code_; } const std::string& Message() const { return msg_; } private: TritonServerError(TRITONSERVER_Error_Code code, const std::string& msg) : code_(code), msg_(msg) { } TritonServerError(TRITONSERVER_Error_Code code, const char* msg) : code_(code), msg_(msg) { } TRITONSERVER_Error_Code code_; const std::string msg_; }; TRITONSERVER_Error* TritonServerError::Create(TRITONSERVER_Error_Code code, const char* msg) { return reinterpret_cast( new TritonServerError(code, msg)); } TRITONSERVER_Error* TritonServerError::Create(TRITONSERVER_Error_Code code, const std::string& msg) { return reinterpret_cast( new TritonServerError(code, msg)); } TRITONSERVER_Error* TritonServerError::Create(const tc::Status& status) { // If 'status' is success then return nullptr as that indicates // success if (status.IsOk()) { return nullptr; } return Create( tc::StatusCodeToTritonCode(status.StatusCode()), status.Message()); } #define RETURN_IF_STATUS_ERROR(S) \ do { \ const tc::Status& status__ = (S); \ if (!status__.IsOk()) { \ return TritonServerError::Create(status__); \ } \ } while (false) // // TritonServerMetrics // // Implementation for TRITONSERVER_Metrics. // class TritonServerMetrics { public: TritonServerMetrics() = default; TRITONSERVER_Error* Serialize(const char** base, size_t* byte_size); private: std::string serialized_; }; TRITONSERVER_Error* TritonServerMetrics::Serialize(const char** base, size_t* byte_size) { #ifdef TRITON_ENABLE_METRICS serialized_ = tc::Metrics::SerializedMetrics(); *base = serialized_.c_str(); *byte_size = serialized_.size(); return nullptr; // Success #else *base = nullptr; *byte_size = 0; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } // // TritonServerOptions // // Implementation for TRITONSERVER_ServerOptions. // class TritonServerOptions { public: TritonServerOptions(); const std::string& ServerId() const { return server_id_; } void SetServerId(const char* id) { server_id_ = id; } const std::set& ModelRepositoryPaths() const { return repo_paths_; } void SetModelRepositoryPath(const char* p) { repo_paths_.insert(p); } tc::ModelControlMode ModelControlMode() const { return model_control_mode_; } void SetModelControlMode(tc::ModelControlMode m) { model_control_mode_ = m; } const std::set& StartupModels() const { return models_; } void SetStartupModel(const char* m) { models_.insert(m); } const std::string& ModelConfigName() const { return model_config_name_; } void SetModelConfigName(const std::string& name) { model_config_name_ = name; } bool ExitOnError() const { return exit_on_error_; } void SetExitOnError(bool b) { exit_on_error_ = b; } bool StrictModelConfig() const { return strict_model_config_; } void SetStrictModelConfig(bool b) { strict_model_config_ = b; } tc::RateLimitMode RateLimiterMode() const { return rate_limit_mode_; } void SetRateLimiterMode(tc::RateLimitMode m) { rate_limit_mode_ = m; } TRITONSERVER_Error* AddRateLimiterResource( const std::string& resource, const size_t count, const int device); // The resource map is the map from device id to the map of // of resources with their respective counts for that device. const tc::RateLimiter::ResourceMap& RateLimiterResources() const { return rate_limit_resource_map_; } uint64_t PinnedMemoryPoolByteSize() const { return pinned_memory_pool_size_; } void SetPinnedMemoryPoolByteSize(uint64_t s) { pinned_memory_pool_size_ = s; } const std::map& CudaMemoryPoolByteSize() const { return cuda_memory_pool_size_; } void SetCudaMemoryPoolByteSize(int id, uint64_t s) { cuda_memory_pool_size_[id] = s; } double MinSupportedComputeCapability() const { return min_compute_capability_; } void SetMinSupportedComputeCapability(double c) { min_compute_capability_ = c; } const std::map& CudaVirtualAddressSpaceSize() const { return cuda_virtual_address_space_size_; } void SetCudaVirtualAddressSpaceSize(int id, size_t virtual_address_space_size) { cuda_virtual_address_space_size_[id] = virtual_address_space_size; } bool StrictReadiness() const { return strict_readiness_; } void SetStrictReadiness(bool b) { strict_readiness_ = b; } unsigned int ExitTimeout() const { return exit_timeout_; } void SetExitTimeout(unsigned int t) { exit_timeout_ = t; } unsigned int BufferManagerThreadCount() const { return buffer_manager_thread_count_; } void SetBufferManagerThreadCount(unsigned int c) { buffer_manager_thread_count_ = c; } unsigned int ModelLoadThreadCount() const { return model_load_thread_count_; } void SetModelLoadThreadCount(unsigned int c) { model_load_thread_count_ = c; } unsigned int ModelLoadRetryCount() const { return model_load_retry_count_; } void SetModelLoadRetryCount(unsigned int c) { model_load_retry_count_ = c; } bool ModelNamespacingEnabled() { return enable_model_namespacing_; } bool PeerAccessEnabled() { return enable_peer_access_; } void SetModelNamespacingEnabled(const bool e) { enable_model_namespacing_ = e; } void SetEnablePeerAccess(const bool e) { enable_peer_access_ = e; } bool Metrics() const { return metrics_; } void SetMetrics(bool b) { metrics_ = b; } bool GpuMetrics() const { return gpu_metrics_; } void SetGpuMetrics(bool b) { gpu_metrics_ = b; } bool CpuMetrics() const { return cpu_metrics_; } void SetCpuMetrics(bool b) { cpu_metrics_ = b; } uint64_t MetricsInterval() const { return metrics_interval_; } void SetMetricsInterval(uint64_t m) { metrics_interval_ = m; } #ifdef TRITON_ENABLE_METRICS const tc::MetricsConfigMap& MetricsConfigMap() { return metrics_config_map_; } TRITONSERVER_Error* AddMetricsConfig( const std::string& name, const std::string& setting, const std::string& value); #endif // TRITON_ENABLE_METRICS const std::string& BackendDir() const { return backend_dir_; } void SetBackendDir(const std::string& bd) { backend_dir_ = bd; } const std::string& RepoAgentDir() const { return repoagent_dir_; } void SetRepoAgentDir(const std::string& rad) { repoagent_dir_ = rad; } // The backend config map is a map from backend name to the // setting=value pairs for that backend. The empty backend name ("") // is used to communicate configuration information that is used // internally. const triton::common::BackendCmdlineConfigMap& BackendCmdlineConfigMap() const { return backend_cmdline_config_map_; } TRITONSERVER_Error* AddBackendConfig( const std::string& backend_name, const std::string& setting, const std::string& value); TRITONSERVER_Error* SetHostPolicy( const std::string& policy_name, const std::string& setting, const std::string& value); const triton::common::HostPolicyCmdlineConfigMap& HostPolicyCmdlineConfigMap() const { return host_policy_map_; } const tc::CacheConfigMap& CacheConfig() { return cache_config_map_; } TRITONSERVER_Error* AddCacheConfig( const std::string& cache_name, const std::string& config_json); const std::string& CacheDir() const { return cache_dir_; } void SetCacheDir(const std::string& dir) { cache_dir_ = dir; } private: std::string server_id_; std::set repo_paths_; tc::ModelControlMode model_control_mode_; std::set models_; bool exit_on_error_; bool strict_model_config_; std::string model_config_name_; bool strict_readiness_; tc::RateLimitMode rate_limit_mode_; tc::RateLimiter::ResourceMap rate_limit_resource_map_; bool metrics_; bool gpu_metrics_; bool cpu_metrics_; uint64_t metrics_interval_; unsigned int exit_timeout_; uint64_t pinned_memory_pool_size_; unsigned int buffer_manager_thread_count_; unsigned int model_load_thread_count_; unsigned int model_load_retry_count_; bool enable_model_namespacing_; bool enable_peer_access_; std::map cuda_memory_pool_size_; double min_compute_capability_; std::string backend_dir_; std::string repoagent_dir_; std::string cache_dir_; tc::CacheConfigMap cache_config_map_; triton::common::BackendCmdlineConfigMap backend_cmdline_config_map_; triton::common::HostPolicyCmdlineConfigMap host_policy_map_; std::map cuda_virtual_address_space_size_; #ifdef TRITON_ENABLE_METRICS tc::MetricsConfigMap metrics_config_map_; #endif // TRITON_ENABLE_METRICS }; TritonServerOptions::TritonServerOptions() : server_id_("triton"), model_control_mode_(tc::ModelControlMode::MODE_POLL), exit_on_error_(true), strict_model_config_(true), model_config_name_(""), strict_readiness_(true), rate_limit_mode_(tc::RateLimitMode::RL_OFF), metrics_(true), gpu_metrics_(true), cpu_metrics_(true), metrics_interval_(2000), exit_timeout_(30), pinned_memory_pool_size_(1 << 28), buffer_manager_thread_count_(0), model_load_thread_count_(4), enable_model_namespacing_(false), #ifdef TRITON_ENABLE_GPU min_compute_capability_(TRITON_MIN_COMPUTE_CAPABILITY), #else min_compute_capability_(0), #endif // TRITON_ENABLE_GPU backend_dir_("/opt/tritonserver/backends"), repoagent_dir_("/opt/tritonserver/repoagents"), cache_dir_("/opt/tritonserver/caches") { #ifndef TRITON_ENABLE_METRICS metrics_ = false; gpu_metrics_ = false; cpu_metrics_ = false; #endif // TRITON_ENABLE_METRICS #ifndef TRITON_ENABLE_METRICS_GPU gpu_metrics_ = false; #endif // TRITON_ENABLE_METRICS_GPU #ifndef TRITON_ENABLE_METRICS_CPU cpu_metrics_ = false; #endif // TRITON_ENABLE_METRICS_CPU } TRITONSERVER_Error* TritonServerOptions::AddRateLimiterResource( const std::string& name, const size_t count, const int device) { auto ditr = rate_limit_resource_map_.find(device); if (ditr == rate_limit_resource_map_.end()) { ditr = rate_limit_resource_map_ .emplace(device, std::map()) .first; } auto ritr = ditr->second.find(name); if (ritr == ditr->second.end()) { ditr->second.emplace(name, count).first; } else { // If already present then store the minimum of the two. if (ritr->second > count) { ritr->second = count; } } return nullptr; // success } TRITONSERVER_Error* TritonServerOptions::AddBackendConfig( const std::string& backend_name, const std::string& setting, const std::string& value) { triton::common::BackendCmdlineConfig& cc = backend_cmdline_config_map_[backend_name]; cc.push_back(std::make_pair(setting, value)); return nullptr; // success } TRITONSERVER_Error* TritonServerOptions::AddCacheConfig( const std::string& cache_name, const std::string& config_json) { cache_config_map_[cache_name] = config_json; return nullptr; // success } #ifdef TRITON_ENABLE_METRICS TRITONSERVER_Error* TritonServerOptions::AddMetricsConfig( const std::string& name, const std::string& setting, const std::string& value) { tc::MetricsConfig& mc = metrics_config_map_[name]; mc.push_back(std::make_pair(setting, value)); return nullptr; // success } #endif // TRITON_ENABLE_METRICS TRITONSERVER_Error* TritonServerOptions::SetHostPolicy( const std::string& policy_name, const std::string& setting, const std::string& value) { // Check if supported setting is passed if ((setting != "numa-node") && (setting != "cpu-cores")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string( "Unsupported host policy setting '" + setting + "' is specified, supported settings are 'numa-node', 'cpu-cores'") .c_str()); } triton::common::HostPolicyCmdlineConfig& hp = host_policy_map_[policy_name]; hp[setting] = value; return nullptr; // success } #define SetDurationStat(DOC, PARENT, STAT_NAME, COUNT, NS) \ do { \ triton::common::TritonJson::Value dstat( \ DOC, triton::common::TritonJson::ValueType::OBJECT); \ dstat.AddUInt("count", (COUNT)); \ dstat.AddUInt("ns", (NS)); \ PARENT.Add(STAT_NAME, std::move(dstat)); \ } while (false) } // namespace extern "C" { // // TRITONSERVER API Version // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ApiVersion(uint32_t* major, uint32_t* minor) { *major = TRITONSERVER_API_VERSION_MAJOR; *minor = TRITONSERVER_API_VERSION_MINOR; return nullptr; // success } // // TRITONSERVER_DataType // TRITONAPI_DECLSPEC const char* TRITONSERVER_DataTypeString(TRITONSERVER_DataType datatype) { switch (datatype) { case TRITONSERVER_TYPE_BOOL: return "BOOL"; case TRITONSERVER_TYPE_UINT8: return "UINT8"; case TRITONSERVER_TYPE_UINT16: return "UINT16"; case TRITONSERVER_TYPE_UINT32: return "UINT32"; case TRITONSERVER_TYPE_UINT64: return "UINT64"; case TRITONSERVER_TYPE_INT8: return "INT8"; case TRITONSERVER_TYPE_INT16: return "INT16"; case TRITONSERVER_TYPE_INT32: return "INT32"; case TRITONSERVER_TYPE_INT64: return "INT64"; case TRITONSERVER_TYPE_FP16: return "FP16"; case TRITONSERVER_TYPE_FP32: return "FP32"; case TRITONSERVER_TYPE_FP64: return "FP64"; case TRITONSERVER_TYPE_BYTES: return "BYTES"; case TRITONSERVER_TYPE_BF16: return "BF16"; default: break; } return ""; } TRITONAPI_DECLSPEC TRITONSERVER_DataType TRITONSERVER_StringToDataType(const char* dtype) { const size_t len = strlen(dtype); return tc::DataTypeToTriton( triton::common::ProtocolStringToDataType(dtype, len)); } TRITONAPI_DECLSPEC uint32_t TRITONSERVER_DataTypeByteSize(TRITONSERVER_DataType datatype) { switch (datatype) { case TRITONSERVER_TYPE_BOOL: case TRITONSERVER_TYPE_INT8: case TRITONSERVER_TYPE_UINT8: return 1; case TRITONSERVER_TYPE_INT16: case TRITONSERVER_TYPE_UINT16: case TRITONSERVER_TYPE_FP16: case TRITONSERVER_TYPE_BF16: return 2; case TRITONSERVER_TYPE_INT32: case TRITONSERVER_TYPE_UINT32: case TRITONSERVER_TYPE_FP32: return 4; case TRITONSERVER_TYPE_INT64: case TRITONSERVER_TYPE_UINT64: case TRITONSERVER_TYPE_FP64: return 8; case TRITONSERVER_TYPE_BYTES: return 0; default: break; } return 0; } // // TRITONSERVER_MemoryType // TRITONAPI_DECLSPEC const char* TRITONSERVER_MemoryTypeString(TRITONSERVER_MemoryType memtype) { switch (memtype) { case TRITONSERVER_MEMORY_CPU: return "CPU"; case TRITONSERVER_MEMORY_CPU_PINNED: return "CPU_PINNED"; case TRITONSERVER_MEMORY_GPU: return "GPU"; default: break; } return ""; } // // TRITONSERVER_Parameter // TRITONAPI_DECLSPEC const char* TRITONSERVER_ParameterTypeString(TRITONSERVER_ParameterType paramtype) { switch (paramtype) { case TRITONSERVER_PARAMETER_STRING: return "STRING"; case TRITONSERVER_PARAMETER_INT: return "INT"; case TRITONSERVER_PARAMETER_BOOL: return "BOOL"; case TRITONSERVER_PARAMETER_DOUBLE: return "DOUBLE"; case TRITONSERVER_PARAMETER_BYTES: return "BYTES"; default: break; } return ""; } TRITONAPI_DECLSPEC TRITONSERVER_Parameter* TRITONSERVER_ParameterNew( const char* name, const TRITONSERVER_ParameterType type, const void* value) { std::unique_ptr lparam; switch (type) { case TRITONSERVER_PARAMETER_STRING: lparam.reset(new tc::InferenceParameter( name, reinterpret_cast(value))); break; case TRITONSERVER_PARAMETER_INT: lparam.reset(new tc::InferenceParameter( name, *reinterpret_cast(value))); break; case TRITONSERVER_PARAMETER_DOUBLE: lparam.reset(new tc::InferenceParameter( name, *reinterpret_cast(value))); break; case TRITONSERVER_PARAMETER_BOOL: lparam.reset(new tc::InferenceParameter( name, *reinterpret_cast(value))); break; default: break; } return reinterpret_cast(lparam.release()); } TRITONAPI_DECLSPEC TRITONSERVER_Parameter* TRITONSERVER_ParameterBytesNew( const char* name, const void* byte_ptr, const uint64_t size) { std::unique_ptr lparam( new tc::InferenceParameter(name, byte_ptr, size)); return reinterpret_cast(lparam.release()); } TRITONAPI_DECLSPEC void TRITONSERVER_ParameterDelete(TRITONSERVER_Parameter* parameter) { delete reinterpret_cast(parameter); } // // TRITONSERVER_InstanceGroupKind // TRITONAPI_DECLSPEC const char* TRITONSERVER_InstanceGroupKindString(TRITONSERVER_InstanceGroupKind kind) { switch (kind) { case TRITONSERVER_INSTANCEGROUPKIND_AUTO: return "AUTO"; case TRITONSERVER_INSTANCEGROUPKIND_CPU: return "CPU"; case TRITONSERVER_INSTANCEGROUPKIND_GPU: return "GPU"; case TRITONSERVER_INSTANCEGROUPKIND_MODEL: return "MODEL"; default: break; } return ""; } // // TRITONSERVER_Log // TRITONAPI_DECLSPEC bool TRITONSERVER_LogIsEnabled(TRITONSERVER_LogLevel level) { switch (level) { case TRITONSERVER_LOG_INFO: return LOG_INFO_IS_ON; case TRITONSERVER_LOG_WARN: return LOG_WARNING_IS_ON; case TRITONSERVER_LOG_ERROR: return LOG_ERROR_IS_ON; case TRITONSERVER_LOG_VERBOSE: return LOG_VERBOSE_IS_ON(1); } return false; } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_LogMessage( TRITONSERVER_LogLevel level, const char* filename, const int line, const char* msg) { switch (level) { case TRITONSERVER_LOG_INFO: LOG_INFO_FL(filename, line) << msg; return nullptr; case TRITONSERVER_LOG_WARN: LOG_WARNING_FL(filename, line) << msg; return nullptr; case TRITONSERVER_LOG_ERROR: LOG_ERROR_FL(filename, line) << msg; return nullptr; case TRITONSERVER_LOG_VERBOSE: LOG_VERBOSE_FL(1, filename, line) << msg; return nullptr; default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("unknown logging level '" + std::to_string(level) + "'") .c_str()); } } // // TRITONSERVER_Error // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ErrorNew(TRITONSERVER_Error_Code code, const char* msg) { return reinterpret_cast( TritonServerError::Create(code, msg)); } TRITONAPI_DECLSPEC void TRITONSERVER_ErrorDelete(TRITONSERVER_Error* error) { TritonServerError* lerror = reinterpret_cast(error); delete lerror; } TRITONSERVER_Error_Code TRITONSERVER_ErrorCode(TRITONSERVER_Error* error) { TritonServerError* lerror = reinterpret_cast(error); return lerror->Code(); } TRITONAPI_DECLSPEC const char* TRITONSERVER_ErrorCodeString(TRITONSERVER_Error* error) { TritonServerError* lerror = reinterpret_cast(error); return tc::Status::CodeString(tc::TritonCodeToStatusCode(lerror->Code())); } TRITONAPI_DECLSPEC const char* TRITONSERVER_ErrorMessage(TRITONSERVER_Error* error) { TritonServerError* lerror = reinterpret_cast(error); return lerror->Message().c_str(); } // // TRITONSERVER_ResponseAllocator // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorNew( TRITONSERVER_ResponseAllocator** allocator, TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn, TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn, TRITONSERVER_ResponseAllocatorStartFn_t start_fn) { *allocator = reinterpret_cast( new tc::ResponseAllocator(alloc_fn, release_fn, start_fn)); return nullptr; // Success } TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorSetQueryFunction( TRITONSERVER_ResponseAllocator* allocator, TRITONSERVER_ResponseAllocatorQueryFn_t query_fn) { reinterpret_cast(allocator)->SetQueryFunction( query_fn); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( TRITONSERVER_ResponseAllocator* allocator, TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn) { reinterpret_cast(allocator) ->SetBufferAttributesFunction(buffer_attributes_fn); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorDelete(TRITONSERVER_ResponseAllocator* allocator) { tc::ResponseAllocator* lalloc = reinterpret_cast(allocator); delete lalloc; return nullptr; // Success } // // TRITONSERVER_Message // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MessageNewFromSerializedJson( TRITONSERVER_Message** message, const char* base, size_t byte_size) { *message = reinterpret_cast( new tc::TritonServerMessage({base, byte_size})); return nullptr; } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MessageDelete(TRITONSERVER_Message* message) { tc::TritonServerMessage* lmessage = reinterpret_cast(message); delete lmessage; return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MessageSerializeToJson( TRITONSERVER_Message* message, const char** base, size_t* byte_size) { tc::TritonServerMessage* lmessage = reinterpret_cast(message); lmessage->Serialize(base, byte_size); return nullptr; // Success } // // TRITONSERVER_Metrics // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricsDelete(TRITONSERVER_Metrics* metrics) { TritonServerMetrics* lmetrics = reinterpret_cast(metrics); delete lmetrics; return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricsFormatted( TRITONSERVER_Metrics* metrics, TRITONSERVER_MetricFormat format, const char** base, size_t* byte_size) { TritonServerMetrics* lmetrics = reinterpret_cast(metrics); switch (format) { case TRITONSERVER_METRIC_PROMETHEUS: { return lmetrics->Serialize(base, byte_size); } default: break; } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("unknown metrics format '" + std::to_string(format) + "'") .c_str()); } // // TRITONSERVER_InferenceTrace // TRITONAPI_DECLSPEC const char* TRITONSERVER_InferenceTraceLevelString(TRITONSERVER_InferenceTraceLevel level) { switch (level) { case TRITONSERVER_TRACE_LEVEL_DISABLED: return "DISABLED"; case TRITONSERVER_TRACE_LEVEL_MIN: return "MIN"; case TRITONSERVER_TRACE_LEVEL_MAX: return "MAX"; case TRITONSERVER_TRACE_LEVEL_TIMESTAMPS: return "TIMESTAMPS"; case TRITONSERVER_TRACE_LEVEL_TENSORS: return "TENSORS"; } return ""; } TRITONAPI_DECLSPEC const char* TRITONSERVER_InferenceTraceActivityString( TRITONSERVER_InferenceTraceActivity activity) { switch (activity) { case TRITONSERVER_TRACE_REQUEST_START: return "REQUEST_START"; case TRITONSERVER_TRACE_QUEUE_START: return "QUEUE_START"; case TRITONSERVER_TRACE_COMPUTE_START: return "COMPUTE_START"; case TRITONSERVER_TRACE_COMPUTE_INPUT_END: return "COMPUTE_INPUT_END"; case TRITONSERVER_TRACE_COMPUTE_OUTPUT_START: return "COMPUTE_OUTPUT_START"; case TRITONSERVER_TRACE_COMPUTE_END: return "COMPUTE_END"; case TRITONSERVER_TRACE_REQUEST_END: return "REQUEST_END"; case TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT: return "TENSOR_QUEUE_INPUT"; case TRITONSERVER_TRACE_TENSOR_BACKEND_INPUT: return "TENSOR_BACKEND_INPUT"; case TRITONSERVER_TRACE_TENSOR_BACKEND_OUTPUT: return "TENSOR_BACKEND_OUTPUT"; case TRITONSERVER_TRACE_CUSTOM_ACTIVITY: return "CUSTOM_ACTIVITY"; } return ""; } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceNew( TRITONSERVER_InferenceTrace** trace, TRITONSERVER_InferenceTraceLevel level, uint64_t parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, TRITONSERVER_InferenceTraceReleaseFn_t release_fn, void* trace_userp) { #ifdef TRITON_ENABLE_TRACING if ((level & TRITONSERVER_TRACE_LEVEL_MIN) > 0) { level = static_cast( (level ^ TRITONSERVER_TRACE_LEVEL_MIN) | TRITONSERVER_TRACE_LEVEL_TIMESTAMPS); } if ((level & TRITONSERVER_TRACE_LEVEL_MAX) > 0) { level = static_cast( (level ^ TRITONSERVER_TRACE_LEVEL_MAX) | TRITONSERVER_TRACE_LEVEL_TIMESTAMPS); } tc::InferenceTrace* ltrace = new tc::InferenceTrace( level, parent_id, activity_fn, nullptr, release_fn, trace_userp); *trace = reinterpret_cast(ltrace); return nullptr; // Success #else *trace = nullptr; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceTensorNew( TRITONSERVER_InferenceTrace** trace, TRITONSERVER_InferenceTraceLevel level, uint64_t parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn, TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn, TRITONSERVER_InferenceTraceReleaseFn_t release_fn, void* trace_userp) { #ifdef TRITON_ENABLE_TRACING if ((level & TRITONSERVER_TRACE_LEVEL_MIN) > 0) { level = static_cast( (level ^ TRITONSERVER_TRACE_LEVEL_MIN) | TRITONSERVER_TRACE_LEVEL_TIMESTAMPS); } if ((level & TRITONSERVER_TRACE_LEVEL_MAX) > 0) { level = static_cast( (level ^ TRITONSERVER_TRACE_LEVEL_MAX) | TRITONSERVER_TRACE_LEVEL_TIMESTAMPS); } tc::InferenceTrace* ltrace = new tc::InferenceTrace( level, parent_id, activity_fn, tensor_activity_fn, release_fn, trace_userp); *trace = reinterpret_cast(ltrace); return nullptr; // Success #else *trace = nullptr; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceDelete(TRITONSERVER_InferenceTrace* trace) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); delete ltrace; return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceId(TRITONSERVER_InferenceTrace* trace, uint64_t* id) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); *id = ltrace->Id(); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceParentId( TRITONSERVER_InferenceTrace* trace, uint64_t* parent_id) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); *parent_id = ltrace->ParentId(); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceModelName( TRITONSERVER_InferenceTrace* trace, const char** model_name) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); *model_name = ltrace->ModelName().c_str(); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceRequestId( TRITONSERVER_InferenceTrace* trace, const char** request_id) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); *request_id = ltrace->RequestId().c_str(); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceModelVersion( TRITONSERVER_InferenceTrace* trace, int64_t* model_version) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); *model_version = ltrace->ModelVersion(); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceSpawnChildTrace( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTrace** child_trace) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); if (trace != nullptr) { *child_trace = reinterpret_cast( ltrace->SpawnChildTrace()); } else { *child_trace = nullptr; } return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceReportActivity( TRITONSERVER_InferenceTrace* trace, uint64_t timestamp, const char* activity_name) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); if (trace != nullptr) { ltrace->Report( TRITONSERVER_TRACE_CUSTOM_ACTIVITY, timestamp, activity_name); } return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceSetContext( TRITONSERVER_InferenceTrace* trace, const char* trace_context) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); ltrace->SetContext(trace_context); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceContext( TRITONSERVER_InferenceTrace* trace, const char** trace_context) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); *trace_context = ltrace->Context().c_str(); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } // // TRITONSERVER_ServerOptions // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsNew(TRITONSERVER_ServerOptions** options) { *options = reinterpret_cast(new TritonServerOptions()); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsDelete(TRITONSERVER_ServerOptions* options) { TritonServerOptions* loptions = reinterpret_cast(options); delete loptions; return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetServerId( TRITONSERVER_ServerOptions* options, const char* server_id) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetServerId(server_id); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelRepositoryPath( TRITONSERVER_ServerOptions* options, const char* model_repository_path) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetModelRepositoryPath(model_repository_path); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelControlMode( TRITONSERVER_ServerOptions* options, TRITONSERVER_ModelControlMode mode) { TritonServerOptions* loptions = reinterpret_cast(options); // convert mode from TRITONSERVER_ to triton::core switch (mode) { case TRITONSERVER_MODEL_CONTROL_NONE: { loptions->SetModelControlMode(tc::ModelControlMode::MODE_NONE); break; } case TRITONSERVER_MODEL_CONTROL_POLL: { loptions->SetModelControlMode(tc::ModelControlMode::MODE_POLL); break; } case TRITONSERVER_MODEL_CONTROL_EXPLICIT: { loptions->SetModelControlMode(tc::ModelControlMode::MODE_EXPLICIT); break; } default: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("unknown control mode '" + std::to_string(mode) + "'") .c_str()); } } return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetStartupModel( TRITONSERVER_ServerOptions* options, const char* model_name) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetStartupModel(model_name); return nullptr; // Success } TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelConfigName( struct TRITONSERVER_ServerOptions* options, const char* model_config_name) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetModelConfigName(model_config_name); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetExitOnError( TRITONSERVER_ServerOptions* options, bool exit) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetExitOnError(exit); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetStrictModelConfig( TRITONSERVER_ServerOptions* options, bool strict) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetStrictModelConfig(strict); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetRateLimiterMode( TRITONSERVER_ServerOptions* options, TRITONSERVER_RateLimitMode mode) { TritonServerOptions* loptions = reinterpret_cast(options); // convert mode from TRITONSERVER_ to triton::core switch (mode) { case TRITONSERVER_RATE_LIMIT_EXEC_COUNT: { loptions->SetRateLimiterMode(tc::RateLimitMode::RL_EXEC_COUNT); break; } case TRITONSERVER_RATE_LIMIT_OFF: { loptions->SetRateLimiterMode(tc::RateLimitMode::RL_OFF); break; } default: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("unknown rate limit mode '" + std::to_string(mode) + "'") .c_str()); } } return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsAddRateLimiterResource( TRITONSERVER_ServerOptions* options, const char* name, const size_t count, const int device) { TritonServerOptions* loptions = reinterpret_cast(options); return loptions->AddRateLimiterResource(name, count, device); } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize( TRITONSERVER_ServerOptions* options, uint64_t size) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetPinnedMemoryPoolByteSize(size); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize( TRITONSERVER_ServerOptions* options, int gpu_device, uint64_t size) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetCudaMemoryPoolByteSize(gpu_device, size); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCudaVirtualAddressSize( TRITONSERVER_ServerOptions* options, int gpu_device, size_t cuda_virtual_address_size) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetCudaVirtualAddressSpaceSize( gpu_device, cuda_virtual_address_size); return nullptr; } // Deprecated. See TRITONSERVER_ServerOptionsSetCacheConfig instead. TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetResponseCacheByteSize( TRITONSERVER_ServerOptions* options, uint64_t size) { // Setting a cache config is equivalent to enabling the cache, so triton will // attempt to create it. To maintain old behavior, simply return and do // nothing for a cache size of zero. if (size == 0) { return nullptr; // success } // Otherwise, forward args to new CacheConfig API for backwards compatibility. std::string config_json = R"({"size": )" + std::to_string(size) + "}"; std::string default_cache = "local"; return TRITONSERVER_ServerOptionsSetCacheConfig( options, default_cache.c_str(), config_json.c_str()); } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCacheConfig( TRITONSERVER_ServerOptions* options, const char* cache_name, const char* config_json) { TritonServerOptions* loptions = reinterpret_cast(options); // NOTE: cache_name included for future extensibility, but not currently used. return loptions->AddCacheConfig(cache_name, config_json); } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCacheDirectory( TRITONSERVER_ServerOptions* options, const char* cache_dir) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetCacheDir(cache_dir); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( TRITONSERVER_ServerOptions* options, double cc) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetMinSupportedComputeCapability(cc); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetStrictReadiness( TRITONSERVER_ServerOptions* options, bool strict) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetStrictReadiness(strict); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetExitTimeout( TRITONSERVER_ServerOptions* options, unsigned int timeout) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetExitTimeout(timeout); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetBufferManagerThreadCount( TRITONSERVER_ServerOptions* options, unsigned int thread_count) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetBufferManagerThreadCount(thread_count); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelLoadThreadCount( TRITONSERVER_ServerOptions* options, unsigned int thread_count) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetModelLoadThreadCount(thread_count); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelLoadRetryCount( TRITONSERVER_ServerOptions* options, unsigned int retry_count) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetModelLoadRetryCount(retry_count); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelNamespacing( TRITONSERVER_ServerOptions* options, bool enable_namespace) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetModelNamespacingEnabled(enable_namespace); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetEnablePeerAccess( TRITONSERVER_ServerOptions* options, bool enable_peer_access) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetEnablePeerAccess(enable_peer_access); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogFile( TRITONSERVER_ServerOptions* options, const char* file) { #ifdef TRITON_ENABLE_LOGGING std::string out_file; if (file != nullptr) { out_file = std::string(file); } const std::string& error = LOG_SET_OUT_FILE(out_file); if (!error.empty()) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (error).c_str()); } return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "logging not supported"); #endif // TRITON_ENABLE_LOGGING } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogInfo( TRITONSERVER_ServerOptions* options, bool log) { #ifdef TRITON_ENABLE_LOGGING // Logging is global for now... LOG_ENABLE_INFO(log); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "logging not supported"); #endif // TRITON_ENABLE_LOGGING } // Enable or disable warning level logging. TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogWarn( TRITONSERVER_ServerOptions* options, bool log) { #ifdef TRITON_ENABLE_LOGGING // Logging is global for now... LOG_ENABLE_WARNING(log); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "logging not supported"); #endif // TRITON_ENABLE_LOGGING } // Enable or disable error level logging. TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogError( TRITONSERVER_ServerOptions* options, bool log) { #ifdef TRITON_ENABLE_LOGGING // Logging is global for now... LOG_ENABLE_ERROR(log); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "logging not supported"); #endif // TRITON_ENABLE_LOGGING } // Set verbose logging level. Level zero disables verbose logging. TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogVerbose( TRITONSERVER_ServerOptions* options, int level) { #ifdef TRITON_ENABLE_LOGGING // Logging is global for now... LOG_SET_VERBOSE(level); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "logging not supported"); #endif // TRITON_ENABLE_LOGGING return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogFormat( TRITONSERVER_ServerOptions* options, const TRITONSERVER_LogFormat format) { #ifdef TRITON_ENABLE_LOGGING // Logging is global for now... switch (format) { case TRITONSERVER_LOG_DEFAULT: LOG_SET_FORMAT(triton::common::Logger::Format::kDEFAULT); break; case TRITONSERVER_LOG_ISO8601: LOG_SET_FORMAT(triton::common::Logger::Format::kISO8601); break; } #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "logging not supported"); #endif // TRITON_ENABLE_LOGGING return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMetrics( TRITONSERVER_ServerOptions* options, bool metrics) { #ifdef TRITON_ENABLE_METRICS TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetMetrics(metrics); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetGpuMetrics( TRITONSERVER_ServerOptions* options, bool gpu_metrics) { #ifdef TRITON_ENABLE_METRICS TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetGpuMetrics(gpu_metrics); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetCpuMetrics( TRITONSERVER_ServerOptions* options, bool cpu_metrics) { #ifdef TRITON_ENABLE_METRICS TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetCpuMetrics(cpu_metrics); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMetricsInterval( TRITONSERVER_ServerOptions* options, uint64_t metrics_interval_ms) { #ifdef TRITON_ENABLE_METRICS TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetMetricsInterval(metrics_interval_ms); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetBackendDirectory( TRITONSERVER_ServerOptions* options, const char* backend_dir) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetBackendDir(backend_dir); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetRepoAgentDirectory( TRITONSERVER_ServerOptions* options, const char* repoagent_dir) { TritonServerOptions* loptions = reinterpret_cast(options); loptions->SetRepoAgentDir(repoagent_dir); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit( TRITONSERVER_ServerOptions* options, const TRITONSERVER_InstanceGroupKind kind, const int device_id, const double fraction) { if (device_id < 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expects device ID >= 0, got ") + std::to_string(device_id)) .c_str()); } else if ((fraction < 0.0) || (fraction > 1.0)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expects limit fraction to be in range [0.0, 1.0], got ") + std::to_string(fraction)) .c_str()); } TritonServerOptions* loptions = reinterpret_cast(options); switch (kind) { case TRITONSERVER_INSTANCEGROUPKIND_GPU: { static std::string key_prefix = "model-load-gpu-limit-device-"; return loptions->AddBackendConfig( "", key_prefix + std::to_string(device_id), std::to_string(fraction)); } default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("given device kind is not supported, got: ") + TRITONSERVER_InstanceGroupKindString(kind)) .c_str()); } } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetBackendConfig( TRITONSERVER_ServerOptions* options, const char* backend_name, const char* setting, const char* value) { TritonServerOptions* loptions = reinterpret_cast(options); return loptions->AddBackendConfig(backend_name, setting, value); } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetHostPolicy( TRITONSERVER_ServerOptions* options, const char* policy_name, const char* setting, const char* value) { TritonServerOptions* loptions = reinterpret_cast(options); return loptions->SetHostPolicy(policy_name, setting, value); } // // TRITONSERVER_InferenceRequest // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestNew( TRITONSERVER_InferenceRequest** inference_request, TRITONSERVER_Server* server, const char* model_name, const int64_t model_version) { tc::InferenceServer* lserver = reinterpret_cast(server); std::shared_ptr model; RETURN_IF_STATUS_ERROR(lserver->GetModel(model_name, model_version, &model)); *inference_request = reinterpret_cast( new tc::InferenceRequest(model, model_version)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestDelete( TRITONSERVER_InferenceRequest* inference_request) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); delete lrequest; return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestId( TRITONSERVER_InferenceRequest* inference_request, const char** id) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); *id = lrequest->Id().c_str(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetId( TRITONSERVER_InferenceRequest* inference_request, const char* id) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); lrequest->SetId(id); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestFlags( TRITONSERVER_InferenceRequest* inference_request, uint32_t* flags) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); *flags = lrequest->Flags(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestIsCancelled( struct TRITONSERVER_InferenceRequest* inference_request, bool* is_cancelled) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->IsCancelled(is_cancelled)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestCancel( struct TRITONSERVER_InferenceRequest* inference_request) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->Cancel()); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetFlags( TRITONSERVER_InferenceRequest* inference_request, uint32_t flags) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); lrequest->SetFlags(flags); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestCorrelationId( TRITONSERVER_InferenceRequest* inference_request, uint64_t* correlation_id) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); const tc::InferenceRequest::SequenceId& corr_id = lrequest->CorrelationId(); if (corr_id.Type() != tc::InferenceRequest::SequenceId::DataType::UINT64) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("given request's correlation id is not an unsigned int") .c_str()); } *correlation_id = corr_id.UnsignedIntValue(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestCorrelationIdString( TRITONSERVER_InferenceRequest* inference_request, const char** correlation_id) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); const tc::InferenceRequest::SequenceId& corr_id = lrequest->CorrelationId(); if (corr_id.Type() != tc::InferenceRequest::SequenceId::DataType::STRING) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("given request's correlation id is not a string").c_str()); } *correlation_id = corr_id.StringValue().c_str(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetCorrelationId( TRITONSERVER_InferenceRequest* inference_request, uint64_t correlation_id) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); lrequest->SetCorrelationId(tc::InferenceRequest::SequenceId(correlation_id)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetCorrelationIdString( TRITONSERVER_InferenceRequest* inference_request, const char* correlation_id) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); if (std::string(correlation_id).length() > 128) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string( "string correlation ID cannot be longer than 128 characters") .c_str()); } lrequest->SetCorrelationId(tc::InferenceRequest::SequenceId(correlation_id)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestPriority( TRITONSERVER_InferenceRequest* inference_request, uint32_t* priority) { uint64_t temp; auto error = TRITONSERVER_InferenceRequestPriorityUInt64(inference_request, &temp); if (error != nullptr) { return error; } if (temp > std::numeric_limits::max()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("request priority overflows uint32_t, use " "TRITONSERVER_InferenceRequestPriorityUInt64, priority=") + std::to_string(temp)) .c_str()); } *priority = temp; return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestPriorityUInt64( TRITONSERVER_InferenceRequest* inference_request, uint64_t* priority) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); *priority = lrequest->Priority(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetPriority( TRITONSERVER_InferenceRequest* inference_request, uint32_t priority) { return TRITONSERVER_InferenceRequestSetPriorityUInt64( inference_request, priority); } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetPriorityUInt64( TRITONSERVER_InferenceRequest* inference_request, uint64_t priority) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); lrequest->SetPriority(priority); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestTimeoutMicroseconds( TRITONSERVER_InferenceRequest* inference_request, uint64_t* timeout_us) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); *timeout_us = lrequest->TimeoutMicroseconds(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetTimeoutMicroseconds( TRITONSERVER_InferenceRequest* inference_request, uint64_t timeout_us) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); lrequest->SetTimeoutMicroseconds(timeout_us); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestAddInput( TRITONSERVER_InferenceRequest* inference_request, const char* name, const TRITONSERVER_DataType datatype, const int64_t* shape, uint64_t dim_count) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->AddOriginalInput( name, tc::TritonToDataType(datatype), shape, dim_count)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestAddRawInput( TRITONSERVER_InferenceRequest* inference_request, const char* name) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->AddRawInput(name)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveInput( TRITONSERVER_InferenceRequest* inference_request, const char* name) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->RemoveOriginalInput(name)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveAllInputs( TRITONSERVER_InferenceRequest* inference_request) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->RemoveAllOriginalInputs()); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestAppendInputData( TRITONSERVER_InferenceRequest* inference_request, const char* name, const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); tc::InferenceRequest::Input* input; RETURN_IF_STATUS_ERROR(lrequest->MutableOriginalInput(name, &input)); RETURN_IF_STATUS_ERROR( input->AppendData(base, byte_size, memory_type, memory_type_id)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( TRITONSERVER_InferenceRequest* inference_request, const char* name, const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, const char* host_policy_name) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); tc::InferenceRequest::Input* input; RETURN_IF_STATUS_ERROR(lrequest->MutableOriginalInput(name, &input)); RETURN_IF_STATUS_ERROR(input->AppendDataWithHostPolicy( base, byte_size, memory_type, memory_type_id, host_policy_name)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( TRITONSERVER_InferenceRequest* inference_request, const char* name, const void* base, TRITONSERVER_BufferAttributes* buffer_attributes) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); tc::InferenceRequest::Input* input; RETURN_IF_STATUS_ERROR(lrequest->MutableOriginalInput(name, &input)); RETURN_IF_STATUS_ERROR( input->AppendDataWithBufferAttributes(base, lbuffer_attributes)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveAllInputData( TRITONSERVER_InferenceRequest* inference_request, const char* name) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); tc::InferenceRequest::Input* input; RETURN_IF_STATUS_ERROR(lrequest->MutableOriginalInput(name, &input)); RETURN_IF_STATUS_ERROR(input->RemoveAllData()); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestAddRequestedOutput( TRITONSERVER_InferenceRequest* inference_request, const char* name) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->AddOriginalRequestedOutput(name)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveRequestedOutput( TRITONSERVER_InferenceRequest* inference_request, const char* name) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->RemoveOriginalRequestedOutput(name)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestRemoveAllRequestedOutputs( TRITONSERVER_InferenceRequest* inference_request) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->RemoveAllOriginalRequestedOutputs()); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetReleaseCallback( TRITONSERVER_InferenceRequest* inference_request, TRITONSERVER_InferenceRequestReleaseFn_t request_release_fn, void* request_release_userp) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR( lrequest->SetReleaseCallback(request_release_fn, request_release_userp)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetResponseCallback( TRITONSERVER_InferenceRequest* inference_request, TRITONSERVER_ResponseAllocator* response_allocator, void* response_allocator_userp, TRITONSERVER_InferenceResponseCompleteFn_t response_fn, void* response_userp) { tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); tc::ResponseAllocator* lallocator = reinterpret_cast(response_allocator); RETURN_IF_STATUS_ERROR(lrequest->SetResponseCallback( lallocator, response_allocator_userp, response_fn, response_userp)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetStringParameter( TRITONSERVER_InferenceRequest* request, const char* name, const char* value) { tc::InferenceRequest* tr = reinterpret_cast(request); RETURN_IF_STATUS_ERROR(tr->AddParameter(name, value)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetIntParameter( TRITONSERVER_InferenceRequest* request, const char* name, const int64_t value) { tc::InferenceRequest* tr = reinterpret_cast(request); RETURN_IF_STATUS_ERROR(tr->AddParameter(name, value)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetBoolParameter( TRITONSERVER_InferenceRequest* request, const char* name, const bool value) { tc::InferenceRequest* tr = reinterpret_cast(request); RETURN_IF_STATUS_ERROR(tr->AddParameter(name, value)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetDoubleParameter( TRITONSERVER_InferenceRequest* request, const char* name, const double value) { tc::InferenceRequest* tr = reinterpret_cast(request); RETURN_IF_STATUS_ERROR(tr->AddParameter(name, value)); return nullptr; // success } // // TRITONSERVER_InferenceResponse // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseDelete( TRITONSERVER_InferenceResponse* inference_response) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); delete lresponse; return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseError( TRITONSERVER_InferenceResponse* inference_response) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); RETURN_IF_STATUS_ERROR(lresponse->ResponseStatus()); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseModel( TRITONSERVER_InferenceResponse* inference_response, const char** model_name, int64_t* model_version) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); *model_name = lresponse->ModelName().c_str(); *model_version = lresponse->ActualModelVersion(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseId( TRITONSERVER_InferenceResponse* inference_response, const char** request_id) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); *request_id = lresponse->Id().c_str(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseParameterCount( TRITONSERVER_InferenceResponse* inference_response, uint32_t* count) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); const auto& parameters = lresponse->Parameters(); *count = parameters.size(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseParameter( TRITONSERVER_InferenceResponse* inference_response, const uint32_t index, const char** name, TRITONSERVER_ParameterType* type, const void** vvalue) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); const auto& parameters = lresponse->Parameters(); if (index >= parameters.size()) { return TritonServerError::Create( TRITONSERVER_ERROR_INVALID_ARG, "out of bounds index " + std::to_string(index) + std::string(": response has ") + std::to_string(parameters.size()) + " parameters"); } const tc::InferenceParameter& param = parameters[index]; *name = param.Name().c_str(); *type = param.Type(); *vvalue = param.ValuePointer(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseOutputCount( TRITONSERVER_InferenceResponse* inference_response, uint32_t* count) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); const auto& outputs = lresponse->Outputs(); *count = outputs.size(); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseOutput( TRITONSERVER_InferenceResponse* inference_response, const uint32_t index, const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape, uint64_t* dim_count, const void** base, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id, void** userp) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); const auto& outputs = lresponse->Outputs(); if (index >= outputs.size()) { return TritonServerError::Create( TRITONSERVER_ERROR_INVALID_ARG, "out of bounds index " + std::to_string(index) + std::string(": response has ") + std::to_string(outputs.size()) + " outputs"); } const tc::InferenceResponse::Output& output = outputs[index]; *name = output.Name().c_str(); *datatype = tc::DataTypeToTriton(output.DType()); const std::vector& oshape = output.Shape(); *shape = &oshape[0]; *dim_count = oshape.size(); RETURN_IF_STATUS_ERROR( output.DataBuffer(base, byte_size, memory_type, memory_type_id, userp)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseOutputClassificationLabel( TRITONSERVER_InferenceResponse* inference_response, const uint32_t index, const size_t class_index, const char** label) { tc::InferenceResponse* lresponse = reinterpret_cast(inference_response); const auto& outputs = lresponse->Outputs(); if (index >= outputs.size()) { return TritonServerError::Create( TRITONSERVER_ERROR_INVALID_ARG, "out of bounds index " + std::to_string(index) + std::string(": response has ") + std::to_string(outputs.size()) + " outputs"); } const tc::InferenceResponse::Output& output = outputs[index]; RETURN_IF_STATUS_ERROR( lresponse->ClassificationLabel(output, class_index, label)); return nullptr; // Success } // // TRITONSERVER_BufferAttributes // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesNew( TRITONSERVER_BufferAttributes** buffer_attributes) { tc::BufferAttributes* lbuffer_attributes = new tc::BufferAttributes(); *buffer_attributes = reinterpret_cast(lbuffer_attributes); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesDelete( TRITONSERVER_BufferAttributes* buffer_attributes) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); delete lbuffer_attributes; return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesSetMemoryTypeId( TRITONSERVER_BufferAttributes* buffer_attributes, int64_t memory_type_id) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); lbuffer_attributes->SetMemoryTypeId(memory_type_id); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesSetMemoryType( TRITONSERVER_BufferAttributes* buffer_attributes, TRITONSERVER_MemoryType memory_type) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); lbuffer_attributes->SetMemoryType(memory_type); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesSetCudaIpcHandle( TRITONSERVER_BufferAttributes* buffer_attributes, void* cuda_ipc_handle) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); lbuffer_attributes->SetCudaIpcHandle(cuda_ipc_handle); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesSetByteSize( TRITONSERVER_BufferAttributes* buffer_attributes, size_t byte_size) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); lbuffer_attributes->SetByteSize(byte_size); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesMemoryTypeId( TRITONSERVER_BufferAttributes* buffer_attributes, int64_t* memory_type_id) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); *memory_type_id = lbuffer_attributes->MemoryTypeId(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesMemoryType( TRITONSERVER_BufferAttributes* buffer_attributes, TRITONSERVER_MemoryType* memory_type) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); *memory_type = lbuffer_attributes->MemoryType(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesCudaIpcHandle( TRITONSERVER_BufferAttributes* buffer_attributes, void** cuda_ipc_handle) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); *cuda_ipc_handle = lbuffer_attributes->CudaIpcHandle(); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesByteSize( TRITONSERVER_BufferAttributes* buffer_attributes, size_t* byte_size) { tc::BufferAttributes* lbuffer_attributes = reinterpret_cast(buffer_attributes); *byte_size = lbuffer_attributes->ByteSize(); return nullptr; // success } // // TRITONSERVER_Server // TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerNew( TRITONSERVER_Server** server, TRITONSERVER_ServerOptions* options) { tc::InferenceServer* lserver = new tc::InferenceServer(); TritonServerOptions* loptions = reinterpret_cast(options); NVTX_INITIALIZE; #ifdef TRITON_ENABLE_METRICS // NOTE: Metrics must be enabled before backends are setup if (loptions->Metrics()) { tc::Metrics::EnableMetrics(); tc::Metrics::SetMetricsInterval(loptions->MetricsInterval()); tc::Metrics::SetConfigMap(loptions->MetricsConfigMap()); } #endif // TRITON_ENABLE_METRICS lserver->SetId(loptions->ServerId()); lserver->SetModelRepositoryPaths(loptions->ModelRepositoryPaths()); lserver->SetModelControlMode(loptions->ModelControlMode()); lserver->SetStartupModels(loptions->StartupModels()); bool strict_model_config = loptions->StrictModelConfig(); lserver->SetStrictModelConfigEnabled(strict_model_config); lserver->SetModelConfigName(loptions->ModelConfigName()); lserver->SetRateLimiterMode(loptions->RateLimiterMode()); lserver->SetRateLimiterResources(loptions->RateLimiterResources()); lserver->SetPinnedMemoryPoolByteSize(loptions->PinnedMemoryPoolByteSize()); lserver->SetCudaMemoryPoolByteSize(loptions->CudaMemoryPoolByteSize()); lserver->SetCudaVirtualAddressSpaceSize( loptions->CudaVirtualAddressSpaceSize()); bool cache_enabled = !loptions->CacheConfig().empty(); lserver->SetResponseCacheEnabled(cache_enabled); lserver->SetCacheConfig(loptions->CacheConfig()); lserver->SetCacheDir(loptions->CacheDir()); double min_compute_capability = loptions->MinSupportedComputeCapability(); lserver->SetMinSupportedComputeCapability(min_compute_capability); lserver->SetStrictReadinessEnabled(loptions->StrictReadiness()); lserver->SetExitTimeoutSeconds(loptions->ExitTimeout()); lserver->SetHostPolicyCmdlineConfig(loptions->HostPolicyCmdlineConfigMap()); lserver->SetRepoAgentDir(loptions->RepoAgentDir()); lserver->SetBufferManagerThreadCount(loptions->BufferManagerThreadCount()); lserver->SetModelLoadThreadCount(loptions->ModelLoadThreadCount()); lserver->SetModelLoadRetryCount(loptions->ModelLoadRetryCount()); lserver->SetModelNamespacingEnabled(loptions->ModelNamespacingEnabled()); lserver->SetEnablePeerAccess(loptions->PeerAccessEnabled()); // SetBackendCmdlineConfig must be called after all AddBackendConfig calls // have completed. // Note that the auto complete config condition is reverted // due to setting name being different loptions->AddBackendConfig( std::string(), "auto-complete-config", strict_model_config ? "false" : "true"); loptions->AddBackendConfig( std::string(), "min-compute-capability", std::to_string(min_compute_capability)); loptions->AddBackendConfig( std::string(), "backend-directory", loptions->BackendDir()); lserver->SetBackendCmdlineConfig(loptions->BackendCmdlineConfigMap()); // Initialize server tc::Status status = lserver->Init(); #ifdef TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_METRICS_GPU if (loptions->Metrics() && loptions->GpuMetrics()) { tc::Metrics::EnableGPUMetrics(); } #endif // TRITON_ENABLE_METRICS_GPU #ifdef TRITON_ENABLE_METRICS_CPU if (loptions->Metrics() && loptions->CpuMetrics()) { tc::Metrics::EnableCpuMetrics(); } #endif // TRITON_ENABLE_METRICS_CPU const bool poll_metrics = (loptions->GpuMetrics() || loptions->CpuMetrics()); if (loptions->Metrics() && poll_metrics) { // Start thread to poll enabled metrics periodically tc::Metrics::StartPollingThreadSingleton(); } #endif // TRITON_ENABLE_METRICS // Setup tritonserver options table std::vector options_headers; options_headers.emplace_back("Option"); options_headers.emplace_back("Value"); triton::common::TablePrinter options_table(options_headers); options_table.InsertRow(std::vector{"server_id", lserver->Id()}); options_table.InsertRow( std::vector{"server_version", lserver->Version()}); auto extensions = lserver->Extensions(); std::string exts; for (const auto& ext : extensions) { exts.append(ext); exts.append(" "); } // Remove the trailing space if (exts.size() > 0) exts.pop_back(); options_table.InsertRow(std::vector{"server_extensions", exts}); size_t i = 0; for (const auto& model_repository_path : lserver->ModelRepositoryPaths()) { options_table.InsertRow(std::vector{ "model_repository_path[" + std::to_string(i) + "]", model_repository_path}); ++i; } std::string model_control_mode; auto control_mode = lserver->GetModelControlMode(); switch (control_mode) { case tc::ModelControlMode::MODE_NONE: { model_control_mode = "MODE_NONE"; break; } case tc::ModelControlMode::MODE_POLL: { model_control_mode = "MODE_POLL"; break; } case tc::ModelControlMode::MODE_EXPLICIT: { model_control_mode = "MODE_EXPLICIT"; break; } default: { model_control_mode = ""; } } options_table.InsertRow( std::vector{"model_control_mode", model_control_mode}); i = 0; for (const auto& startup_model : lserver->StartupModels()) { options_table.InsertRow(std::vector{ "startup_models_" + std::to_string(i), startup_model}); ++i; } options_table.InsertRow(std::vector{ "strict_model_config", std::to_string(lserver->StrictModelConfigEnabled())}); options_table.InsertRow(std::vector{ "model_config_name", lserver->ModelConfigName()}); std::string rate_limit = RateLimitModeToString(lserver->RateLimiterMode()); options_table.InsertRow(std::vector{"rate_limit", rate_limit}); i = 0; for (const auto& device_resources : lserver->RateLimiterResources()) { for (const auto& resource : device_resources.second) { options_table.InsertRow(std::vector{ "rate_limit_resource[" + std::to_string(i) + "]", ResourceString( resource.first, resource.second, device_resources.first)}); ++i; } } options_table.InsertRow(std::vector{ "pinned_memory_pool_byte_size", std::to_string(lserver->PinnedMemoryPoolByteSize())}); for (const auto& cuda_memory_pool : lserver->CudaMemoryPoolByteSize()) { options_table.InsertRow(std::vector{ "cuda_memory_pool_byte_size{" + std::to_string(cuda_memory_pool.first) + "}", std::to_string(cuda_memory_pool.second)}); } std::stringstream compute_capability_ss; compute_capability_ss.setf(std::ios::fixed); compute_capability_ss.precision(1); compute_capability_ss << lserver->MinSupportedComputeCapability(); options_table.InsertRow(std::vector{ "min_supported_compute_capability", compute_capability_ss.str()}); options_table.InsertRow(std::vector{ "strict_readiness", std::to_string(lserver->StrictReadinessEnabled())}); options_table.InsertRow(std::vector{ "exit_timeout", std::to_string(lserver->ExitTimeoutSeconds())}); options_table.InsertRow(std::vector{ "cache_enabled", std::to_string(lserver->ResponseCacheEnabled())}); LOG_TABLE_INFO(options_table); if (!status.IsOk()) { if (loptions->ExitOnError()) { lserver->Stop(true /* force */); delete lserver; RETURN_IF_STATUS_ERROR(status); } LOG_ERROR << status.AsString(); } *server = reinterpret_cast(lserver); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerDelete(TRITONSERVER_Server* server) { tc::InferenceServer* lserver = reinterpret_cast(server); if (lserver != nullptr) { RETURN_IF_STATUS_ERROR(lserver->Stop()); } delete lserver; return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerStop(TRITONSERVER_Server* server) { tc::InferenceServer* lserver = reinterpret_cast(server); if (lserver != nullptr) { RETURN_IF_STATUS_ERROR(lserver->Stop()); } return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerSetExitTimeout( TRITONSERVER_Server* server, unsigned int timeout) { tc::InferenceServer* lserver = reinterpret_cast(server); if (lserver != nullptr) { lserver->SetExitTimeoutSeconds(timeout); } return nullptr; // Success } TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerRegisterModelRepository( TRITONSERVER_Server* server, const char* repository_path, const TRITONSERVER_Parameter** name_mapping, const uint32_t mapping_count) { tc::InferenceServer* lserver = reinterpret_cast(server); if ((name_mapping == nullptr) && (mapping_count != 0)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "model mappings are not provided while mapping count is non-zero"); } std::unordered_map model_mapping; for (size_t i = 0; i < mapping_count; ++i) { auto mapping = reinterpret_cast(name_mapping[i]); auto subdir = mapping->Name(); if (mapping->Type() != TRITONSERVER_PARAMETER_STRING) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "Mapped model name must be a string, found " "another type for " + subdir) .c_str()); } auto model_name = std::string(reinterpret_cast(mapping->ValuePointer())); if (!(model_mapping.emplace(model_name, subdir).second)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("failed to register '") + repository_path + "', there is a conflicting mapping for '" + std::string(model_name) + "'") .c_str()); } } RETURN_IF_STATUS_ERROR( lserver->RegisterModelRepository(repository_path, model_mapping)); return nullptr; // Success } TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerUnregisterModelRepository( TRITONSERVER_Server* server, const char* repository_path) { tc::InferenceServer* lserver = reinterpret_cast(server); RETURN_IF_STATUS_ERROR(lserver->UnregisterModelRepository(repository_path)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerPollModelRepository(TRITONSERVER_Server* server) { tc::InferenceServer* lserver = reinterpret_cast(server); RETURN_IF_STATUS_ERROR(lserver->PollModelRepository()); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerIsLive(TRITONSERVER_Server* server, bool* live) { tc::InferenceServer* lserver = reinterpret_cast(server); RETURN_IF_STATUS_ERROR(lserver->IsLive(live)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerIsReady(TRITONSERVER_Server* server, bool* ready) { tc::InferenceServer* lserver = reinterpret_cast(server); RETURN_IF_STATUS_ERROR(lserver->IsReady(ready)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelIsReady( TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, bool* ready) { tc::InferenceServer* lserver = reinterpret_cast(server); RETURN_IF_STATUS_ERROR( lserver->ModelIsReady(model_name, model_version, ready)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelBatchProperties( TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, uint32_t* flags, void** voidp) { tc::InferenceServer* lserver = reinterpret_cast(server); if (voidp != nullptr) { *voidp = nullptr; } std::shared_ptr model; RETURN_IF_STATUS_ERROR(lserver->GetModel(model_name, model_version, &model)); if (model->Config().max_batch_size() > 0) { *flags = TRITONSERVER_BATCH_FIRST_DIM; } else { *flags = TRITONSERVER_BATCH_UNKNOWN; } return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelTransactionProperties( TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, uint32_t* txn_flags, void** voidp) { tc::InferenceServer* lserver = reinterpret_cast(server); if (voidp != nullptr) { *voidp = nullptr; } *txn_flags = 0; std::shared_ptr model; RETURN_IF_STATUS_ERROR(lserver->GetModel(model_name, model_version, &model)); if (model->Config().model_transaction_policy().decoupled()) { *txn_flags = TRITONSERVER_TXN_DECOUPLED; } else { *txn_flags = TRITONSERVER_TXN_ONE_TO_ONE; } return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerMetadata( TRITONSERVER_Server* server, TRITONSERVER_Message** server_metadata) { tc::InferenceServer* lserver = reinterpret_cast(server); triton::common::TritonJson::Value metadata( triton::common::TritonJson::ValueType::OBJECT); // Just store string reference in JSON object since it will be // serialized to another buffer before lserver->Id() or // lserver->Version() lifetime ends. RETURN_IF_STATUS_ERROR(metadata.AddStringRef("name", lserver->Id().c_str())); RETURN_IF_STATUS_ERROR( metadata.AddStringRef("version", lserver->Version().c_str())); triton::common::TritonJson::Value extensions( metadata, triton::common::TritonJson::ValueType::ARRAY); const std::vector& exts = lserver->Extensions(); for (const auto ext : exts) { RETURN_IF_STATUS_ERROR(extensions.AppendStringRef(ext)); } RETURN_IF_STATUS_ERROR(metadata.Add("extensions", std::move(extensions))); *server_metadata = reinterpret_cast( new tc::TritonServerMessage(metadata)); return nullptr; // Success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelMetadata( TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, TRITONSERVER_Message** model_metadata) { tc::InferenceServer* lserver = reinterpret_cast(server); std::shared_ptr model; RETURN_IF_STATUS_ERROR(lserver->GetModel(model_name, model_version, &model)); std::vector ready_versions; RETURN_IF_STATUS_ERROR( lserver->ModelReadyVersions(model_name, &ready_versions)); triton::common::TritonJson::Value metadata( triton::common::TritonJson::ValueType::OBJECT); // Can use string ref in this function even though model can be // unloaded and config becomes invalid, because TritonServeMessage // serializes the json when it is constructed below. RETURN_IF_STATUS_ERROR(metadata.AddStringRef("name", model_name)); triton::common::TritonJson::Value versions( metadata, triton::common::TritonJson::ValueType::ARRAY); if (model_version != -1) { RETURN_IF_STATUS_ERROR( versions.AppendString(std::move(std::to_string(model_version)))); } else { for (const auto v : ready_versions) { RETURN_IF_STATUS_ERROR( versions.AppendString(std::move(std::to_string(v)))); } } RETURN_IF_STATUS_ERROR(metadata.Add("versions", std::move(versions))); const auto& model_config = model->Config(); if (!model_config.platform().empty()) { RETURN_IF_STATUS_ERROR( metadata.AddStringRef("platform", model_config.platform().c_str())); } else { RETURN_IF_STATUS_ERROR( metadata.AddStringRef("platform", model_config.backend().c_str())); } triton::common::TritonJson::Value inputs( metadata, triton::common::TritonJson::ValueType::ARRAY); for (const auto& io : model_config.input()) { triton::common::TritonJson::Value io_metadata( metadata, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_STATUS_ERROR(io_metadata.AddStringRef("name", io.name().c_str())); RETURN_IF_STATUS_ERROR(io_metadata.AddStringRef( "datatype", triton::common::DataTypeToProtocolString(io.data_type()))); // Input shape. If the model supports batching then must include // '-1' for the batch dimension. triton::common::TritonJson::Value io_metadata_shape( metadata, triton::common::TritonJson::ValueType::ARRAY); if (model_config.max_batch_size() >= 1) { RETURN_IF_STATUS_ERROR(io_metadata_shape.AppendInt(-1)); } for (const auto d : io.dims()) { RETURN_IF_STATUS_ERROR(io_metadata_shape.AppendInt(d)); } RETURN_IF_STATUS_ERROR( io_metadata.Add("shape", std::move(io_metadata_shape))); RETURN_IF_STATUS_ERROR(inputs.Append(std::move(io_metadata))); } RETURN_IF_STATUS_ERROR(metadata.Add("inputs", std::move(inputs))); triton::common::TritonJson::Value outputs( metadata, triton::common::TritonJson::ValueType::ARRAY); for (const auto& io : model_config.output()) { triton::common::TritonJson::Value io_metadata( metadata, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_STATUS_ERROR(io_metadata.AddStringRef("name", io.name().c_str())); RETURN_IF_STATUS_ERROR(io_metadata.AddStringRef( "datatype", triton::common::DataTypeToProtocolString(io.data_type()))); // Output shape. If the model supports batching then must include // '-1' for the batch dimension. triton::common::TritonJson::Value io_metadata_shape( metadata, triton::common::TritonJson::ValueType::ARRAY); if (model_config.max_batch_size() >= 1) { RETURN_IF_STATUS_ERROR(io_metadata_shape.AppendInt(-1)); } for (const auto d : io.dims()) { RETURN_IF_STATUS_ERROR(io_metadata_shape.AppendInt(d)); } RETURN_IF_STATUS_ERROR( io_metadata.Add("shape", std::move(io_metadata_shape))); RETURN_IF_STATUS_ERROR(outputs.Append(std::move(io_metadata))); } RETURN_IF_STATUS_ERROR(metadata.Add("outputs", std::move(outputs))); *model_metadata = reinterpret_cast( new tc::TritonServerMessage(metadata)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelStatistics( TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, TRITONSERVER_Message** model_stats) { #ifndef TRITON_ENABLE_STATS return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "statistics not supported"); #else tc::InferenceServer* lserver = reinterpret_cast(server); auto model_name_string = std::string(model_name); std::map> ready_model_versions; if (model_name_string.empty()) { RETURN_IF_STATUS_ERROR(lserver->ModelReadyVersions(&ready_model_versions)); } else { std::vector ready_versions; RETURN_IF_STATUS_ERROR( lserver->ModelReadyVersions(model_name_string, &ready_versions)); if (ready_versions.empty()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "requested model '" + model_name_string + "' is not available") .c_str()); } if (model_version == -1) { ready_model_versions.emplace( model_name_string, std::move(ready_versions)); } else { bool found = false; for (const auto v : ready_versions) { if (v == model_version) { found = true; break; } } if (found) { ready_model_versions.emplace( model_name_string, std::vector{model_version}); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "requested model version is not available for model '" + model_name_string + "'") .c_str()); } } } // Can use string ref in this function because TritonServeMessage // serializes the json when it is constructed below. triton::common::TritonJson::Value metadata( triton::common::TritonJson::ValueType::OBJECT); triton::common::TritonJson::Value model_stats_json( metadata, triton::common::TritonJson::ValueType::ARRAY); for (const auto& mv_pair : ready_model_versions) { for (const auto& version : mv_pair.second) { std::shared_ptr model; RETURN_IF_STATUS_ERROR(lserver->GetModel(mv_pair.first, version, &model)); // Add memory usage triton::common::TritonJson::Value memory_usage( metadata, triton::common::TritonJson::ValueType::ARRAY); const std::vector& usages = model->AccumulatedMemoryUsage(); for (const auto& usage : usages) { triton::common::TritonJson::Value usage_json( metadata, triton::common::TritonJson::ValueType::OBJECT); std::string type = TRITONSERVER_MemoryTypeString(usage.MemoryType()); RETURN_IF_STATUS_ERROR(usage_json.AddString("type", std::move(type))); RETURN_IF_STATUS_ERROR(usage_json.AddInt("id", usage.MemoryTypeId())); RETURN_IF_STATUS_ERROR( usage_json.AddUInt("byte_size", usage.ByteSize())); RETURN_IF_STATUS_ERROR(memory_usage.Append(std::move(usage_json))); } // Add infer statistic const auto& infer_stats = model->StatsAggregator().ImmutableInferStats(); const auto& infer_response_stats = model->StatsAggregator().ImmutableInferResponseStats(); const auto& infer_batch_stats = model->StatsAggregator().ImmutableInferBatchStats(); triton::common::TritonJson::Value inference_stats( metadata, triton::common::TritonJson::ValueType::OBJECT); // Compute figures only calculated when not going through cache, so // subtract cache_hit count from success count. Cache hit count will // simply be 0 when cache is disabled. uint64_t compute_count = infer_stats.success_count_ - infer_stats.cache_hit_count_; SetDurationStat( metadata, inference_stats, "success", infer_stats.success_count_, infer_stats.request_duration_ns_); SetDurationStat( metadata, inference_stats, "fail", infer_stats.failure_count_, infer_stats.failure_duration_ns_); SetDurationStat( metadata, inference_stats, "queue", infer_stats.success_count_, infer_stats.queue_duration_ns_); SetDurationStat( metadata, inference_stats, "compute_input", compute_count, infer_stats.compute_input_duration_ns_); SetDurationStat( metadata, inference_stats, "compute_infer", compute_count, infer_stats.compute_infer_duration_ns_); SetDurationStat( metadata, inference_stats, "compute_output", compute_count, infer_stats.compute_output_duration_ns_); SetDurationStat( metadata, inference_stats, "cache_hit", infer_stats.cache_hit_count_, infer_stats.cache_hit_duration_ns_); // NOTE: cache_miss_count_ should equal compute_count if non-zero SetDurationStat( metadata, inference_stats, "cache_miss", infer_stats.cache_miss_count_, infer_stats.cache_miss_duration_ns_); // Add response statistics triton::common::TritonJson::Value response_stats( metadata, triton::common::TritonJson::ValueType::OBJECT); for (const auto& res_pair : infer_response_stats) { triton::common::TritonJson::Value res_stat( metadata, triton::common::TritonJson::ValueType::OBJECT); SetDurationStat( metadata, res_stat, "compute_infer", res_pair.second.compute_infer_count, res_pair.second.compute_infer_duration_ns); SetDurationStat( metadata, res_stat, "compute_output", res_pair.second.compute_output_count, res_pair.second.compute_output_duration_ns); SetDurationStat( metadata, res_stat, "success", res_pair.second.success_count, res_pair.second.success_duration_ns); SetDurationStat( metadata, res_stat, "fail", res_pair.second.fail_count, res_pair.second.fail_duration_ns); SetDurationStat( metadata, res_stat, "empty_response", res_pair.second.empty_response_count, res_pair.second.empty_response_duration_ns); SetDurationStat( metadata, res_stat, "cancel", res_pair.second.cancel_count, res_pair.second.cancel_duration_ns); RETURN_IF_STATUS_ERROR( response_stats.Add(res_pair.first.c_str(), std::move(res_stat))); } // Add batch statistics triton::common::TritonJson::Value batch_stats( metadata, triton::common::TritonJson::ValueType::ARRAY); for (const auto& batch : infer_batch_stats) { triton::common::TritonJson::Value batch_stat( metadata, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_STATUS_ERROR(batch_stat.AddUInt("batch_size", batch.first)); SetDurationStat( metadata, batch_stat, "compute_input", batch.second.count_, batch.second.compute_input_duration_ns_); SetDurationStat( metadata, batch_stat, "compute_infer", batch.second.count_, batch.second.compute_infer_duration_ns_); SetDurationStat( metadata, batch_stat, "compute_output", batch.second.count_, batch.second.compute_output_duration_ns_); RETURN_IF_STATUS_ERROR(batch_stats.Append(std::move(batch_stat))); } triton::common::TritonJson::Value model_stat( metadata, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_STATUS_ERROR( model_stat.AddStringRef("name", mv_pair.first.c_str())); RETURN_IF_STATUS_ERROR( model_stat.AddString("version", std::move(std::to_string(version)))); RETURN_IF_STATUS_ERROR(model_stat.AddUInt( "last_inference", model->StatsAggregator().LastInferenceMs())); RETURN_IF_STATUS_ERROR(model_stat.AddUInt( "inference_count", model->StatsAggregator().InferenceCount())); RETURN_IF_STATUS_ERROR(model_stat.AddUInt( "execution_count", model->StatsAggregator().ExecutionCount())); RETURN_IF_STATUS_ERROR( model_stat.Add("inference_stats", std::move(inference_stats))); RETURN_IF_STATUS_ERROR( model_stat.Add("response_stats", std::move(response_stats))); RETURN_IF_STATUS_ERROR( model_stat.Add("batch_stats", std::move(batch_stats))); RETURN_IF_STATUS_ERROR( model_stat.Add("memory_usage", std::move(memory_usage))); RETURN_IF_STATUS_ERROR(model_stats_json.Append(std::move(model_stat))); } } RETURN_IF_STATUS_ERROR( metadata.Add("model_stats", std::move(model_stats_json))); *model_stats = reinterpret_cast( new tc::TritonServerMessage(metadata)); return nullptr; // success #endif // TRITON_ENABLE_STATS } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelConfig( TRITONSERVER_Server* server, const char* model_name, const int64_t model_version, const uint32_t config_version, TRITONSERVER_Message** model_config) { tc::InferenceServer* lserver = reinterpret_cast(server); std::shared_ptr model; RETURN_IF_STATUS_ERROR(lserver->GetModel(model_name, model_version, &model)); std::string model_config_json; RETURN_IF_STATUS_ERROR(tc::ModelConfigToJson( model->Config(), config_version, &model_config_json)); *model_config = reinterpret_cast( new tc::TritonServerMessage(std::move(model_config_json))); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelIndex( TRITONSERVER_Server* server, uint32_t flags, TRITONSERVER_Message** repository_index) { tc::InferenceServer* lserver = reinterpret_cast(server); const bool ready_only = ((flags & TRITONSERVER_INDEX_FLAG_READY) != 0); std::vector index; RETURN_IF_STATUS_ERROR(lserver->RepositoryIndex(ready_only, &index)); // Can use string ref in this function because TritonServerMessage // serializes the json when it is constructed below. triton::common::TritonJson::Value repository_index_json( triton::common::TritonJson::ValueType::ARRAY); for (const auto& in : index) { triton::common::TritonJson::Value model_index( repository_index_json, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_STATUS_ERROR(model_index.AddStringRef("name", in.name_.c_str())); if (!in.name_only_) { if (in.version_ >= 0) { RETURN_IF_STATUS_ERROR(model_index.AddString( "version", std::move(std::to_string(in.version_)))); } RETURN_IF_STATUS_ERROR(model_index.AddStringRef( "state", tc::ModelReadyStateString(in.state_).c_str())); if (!in.reason_.empty()) { RETURN_IF_STATUS_ERROR( model_index.AddStringRef("reason", in.reason_.c_str())); } } RETURN_IF_STATUS_ERROR( repository_index_json.Append(std::move(model_index))); } *repository_index = reinterpret_cast( new tc::TritonServerMessage(repository_index_json)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerLoadModel( TRITONSERVER_Server* server, const char* model_name) { tc::InferenceServer* lserver = reinterpret_cast(server); RETURN_IF_STATUS_ERROR(lserver->LoadModel({{std::string(model_name), {}}})); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerLoadModelWithParameters( TRITONSERVER_Server* server, const char* model_name, const TRITONSERVER_Parameter** parameters, const uint64_t parameter_count) { if ((parameters == nullptr) && (parameter_count != 0)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "load parameters are not provided while parameter count is non-zero"); } tc::InferenceServer* lserver = reinterpret_cast(server); std::unordered_map> models; std::vector mp; for (size_t i = 0; i < parameter_count; ++i) { mp.emplace_back( reinterpret_cast(parameters[i])); } models[model_name] = std::move(mp); RETURN_IF_STATUS_ERROR(lserver->LoadModel(models)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerUnloadModel( TRITONSERVER_Server* server, const char* model_name) { tc::InferenceServer* lserver = reinterpret_cast(server); RETURN_IF_STATUS_ERROR(lserver->UnloadModel( std::string(model_name), false /* unload_dependents */)); return nullptr; // success } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerUnloadModelAndDependents( TRITONSERVER_Server* server, const char* model_name) { { tc::InferenceServer* lserver = reinterpret_cast(server); RETURN_IF_STATUS_ERROR(lserver->UnloadModel( std::string(model_name), true /* unload_dependents */)); return nullptr; // success } } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerMetrics( TRITONSERVER_Server* server, TRITONSERVER_Metrics** metrics) { #ifdef TRITON_ENABLE_METRICS TritonServerMetrics* lmetrics = new TritonServerMetrics(); *metrics = reinterpret_cast(lmetrics); return nullptr; // Success #else *metrics = nullptr; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONAPI_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerInferAsync( TRITONSERVER_Server* server, TRITONSERVER_InferenceRequest* inference_request, TRITONSERVER_InferenceTrace* trace) { tc::InferenceServer* lserver = reinterpret_cast(server); tc::InferenceRequest* lrequest = reinterpret_cast(inference_request); RETURN_IF_STATUS_ERROR(lrequest->PrepareForInference()); // Set the trace object in the request so that activity associated // with the request can be recorded as the request flows through // Triton. if (trace != nullptr) { #ifdef TRITON_ENABLE_TRACING tc::InferenceTrace* ltrace = reinterpret_cast(trace); ltrace->SetModelName(lrequest->ModelName()); ltrace->SetModelVersion(lrequest->ActualModelVersion()); ltrace->SetRequestId(lrequest->Id()); lrequest->SetTrace(std::make_shared(ltrace)); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "inference tracing not supported"); #endif // TRITON_ENABLE_TRACING } // We wrap the request in a unique pointer to ensure that it flows // through inferencing with clear ownership. std::unique_ptr ureq(lrequest); // Run inference... tc::Status status = lserver->InferAsync(ureq); // If there is an error then must explicitly release any trace // object associated with the inference request above. #ifdef TRITON_ENABLE_TRACING if (!status.IsOk()) { ureq->ReleaseTrace(); } #endif // TRITON_ENABLE_TRACING // If there is an error then ureq will still have 'lrequest' and we // must release it from unique_ptr since the caller should retain // ownership when there is error. If there is not an error then ureq // == nullptr and so this release is a nop. ureq.release(); RETURN_IF_STATUS_ERROR(status); return nullptr; // Success } // // TRITONSERVER_MetricFamily // TRITONSERVER_Error* TRITONSERVER_MetricFamilyNew( TRITONSERVER_MetricFamily** family, TRITONSERVER_MetricKind kind, const char* name, const char* description) { #ifdef TRITON_ENABLE_METRICS try { *family = reinterpret_cast( new tc::MetricFamily(kind, name, description)); } catch (std::invalid_argument const& ex) { // Catch invalid kinds passed to constructor return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, ex.what()); } return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricFamilyDelete(TRITONSERVER_MetricFamily* family) { #ifdef TRITON_ENABLE_METRICS auto lfamily = reinterpret_cast(family); if (lfamily->NumMetrics() > 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Must call MetricDelete on all dependent metrics before calling " "MetricFamilyDelete."); } delete lfamily; return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_GetMetricFamilyKind( TRITONSERVER_MetricFamily* family, TRITONSERVER_MetricKind* kind) { #ifdef TRITON_ENABLE_METRICS *kind = reinterpret_cast(family)->Kind(); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricArgsNew(TRITONSERVER_MetricArgs** args) { #ifdef TRITON_ENABLE_METRICS tc::TritonServerMetricArgs* largs = new tc::TritonServerMetricArgs(); *args = reinterpret_cast(largs); return nullptr; // Success #else *args = nullptr; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricArgsSetHistogram( TRITONSERVER_MetricArgs* args, const double* buckets, const uint64_t buckets_count) { #ifdef TRITON_ENABLE_METRICS auto largs = reinterpret_cast(args); largs->SetHistogramArgs(buckets, buckets_count); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricArgsDelete(TRITONSERVER_MetricArgs* args) { #ifdef TRITON_ENABLE_METRICS auto largs = reinterpret_cast(args); delete largs; return nullptr; // success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } // // TRITONSERVER_Metric // TRITONSERVER_Error* TRITONSERVER_MetricNew( TRITONSERVER_Metric** metric, TRITONSERVER_MetricFamily* family, const TRITONSERVER_Parameter** labels, const uint64_t label_count) { #ifdef TRITON_ENABLE_METRICS return TRITONSERVER_MetricNewWithArgs( metric, family, labels, label_count, nullptr); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } // // TRITONSERVER_MetricGeneric // TRITONSERVER_Error* TRITONSERVER_MetricNewWithArgs( TRITONSERVER_Metric** metric, TRITONSERVER_MetricFamily* family, const TRITONSERVER_Parameter** labels, const uint64_t label_count, const TRITONSERVER_MetricArgs* args) { #ifdef TRITON_ENABLE_METRICS std::vector labels_vec; for (size_t i = 0; i < label_count; i++) { labels_vec.emplace_back( reinterpret_cast(labels[i])); } try { *metric = reinterpret_cast(new tc::Metric( family, labels_vec, reinterpret_cast(args))); } catch (std::invalid_argument const& ex) { // Catch invalid kinds passed to constructor return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, ex.what()); } return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricDelete(TRITONSERVER_Metric* metric) { #ifdef TRITON_ENABLE_METRICS auto lmetric = reinterpret_cast(metric); if (lmetric->Family() == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "MetricFamily reference was invalidated before Metric was deleted. " "Must call MetricDelete on all dependent metrics before calling " "MetricFamilyDelete."); } delete lmetric; return nullptr; // success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricValue(TRITONSERVER_Metric* metric, double* value) { #ifdef TRITON_ENABLE_METRICS return reinterpret_cast(metric)->Value(value); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricIncrement(TRITONSERVER_Metric* metric, double value) { #ifdef TRITON_ENABLE_METRICS return reinterpret_cast(metric)->Increment(value); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricSet(TRITONSERVER_Metric* metric, double value) { #ifdef TRITON_ENABLE_METRICS return reinterpret_cast(metric)->Set(value); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_MetricObserve(TRITONSERVER_Metric* metric, double value) { #ifdef TRITON_ENABLE_METRICS return reinterpret_cast(metric)->Observe(value); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_GetMetricKind( TRITONSERVER_Metric* metric, TRITONSERVER_MetricKind* kind) { #ifdef TRITON_ENABLE_METRICS *kind = reinterpret_cast(metric)->Kind(); return nullptr; // Success #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMetricsConfig( TRITONSERVER_ServerOptions* options, const char* name, const char* setting, const char* value) { #ifdef TRITON_ENABLE_METRICS TritonServerOptions* loptions = reinterpret_cast(options); return loptions->AddMetricsConfig(name, setting, value); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "metrics not supported"); #endif // TRITON_ENABLE_METRICS } } // extern C kytensor-2.49.0.7/core/src/tritonserver_apis.h000066400000000000000000000037311527052371200213130ustar00rootroot00000000000000// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #define _COMPILING_TRITONSERVER 1 #define _COMPILING_TRITONBACKEND 1 #define _COMPILING_TRITONREPOAGENT 1 #define _COMPILING_TRITONCACHE 1 #include "triton/core/tritonbackend.h" #include "triton/core/tritoncache.h" #include "triton/core/tritonrepoagent.h" #include "triton/core/tritonserver.h" #undef _COMPILING_TRITONSERVER #undef _COMPILING_TRITONBACKEND #undef _COMPILING_TRITONREPOAGENT #undef _COMPILING_TRITONCACHE kytensor-2.49.0.7/core/src/tritonserver_stub.cc000066400000000000000000000510001527052371200214620ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES{} LOSS OF USE, DATA, OR // PROFITS{} OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(_MSC_VER) #define TRITONAPI_DECLSPEC __declspec(dllexport) #elif defined(__GNUC__) #define TRITONAPI_DECLSPEC __attribute__((__visibility__("default"))) #else #define TRITONAPI_DECLSPEC #endif extern "C" { TRITONAPI_DECLSPEC void TRITONSERVER_ApiVersion() { } TRITONAPI_DECLSPEC void TRITONSERVER_DataTypeString() { } TRITONAPI_DECLSPEC void TRITONSERVER_StringToDataType() { } TRITONAPI_DECLSPEC void TRITONSERVER_DataTypeByteSize() { } TRITONAPI_DECLSPEC void TRITONSERVER_MemoryTypeString() { } TRITONAPI_DECLSPEC void TRITONSERVER_ParameterTypeString() { } TRITONAPI_DECLSPEC void TRITONSERVER_ParameterNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_ParameterBytesNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_ParameterDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_InstanceGroupKindString() { } TRITONAPI_DECLSPEC void TRITONSERVER_LogIsEnabled() { } TRITONAPI_DECLSPEC void TRITONSERVER_LogMessage() { } TRITONAPI_DECLSPEC void TRITONSERVER_ErrorNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_ErrorDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_ErrorCode() { } TRITONAPI_DECLSPEC void TRITONSERVER_ErrorCodeString() { } TRITONAPI_DECLSPEC void TRITONSERVER_ErrorMessage() { } TRITONAPI_DECLSPEC void TRITONSERVER_ResponseAllocatorNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_ResponseAllocatorSetQueryFunction() { } TRITONAPI_DECLSPEC void TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction() { } TRITONAPI_DECLSPEC void TRITONSERVER_ResponseAllocatorDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestIsCancelled() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestCancel() { } TRITONAPI_DECLSPEC void TRITONSERVER_MessageNewFromSerializedJson() { } TRITONAPI_DECLSPEC void TRITONSERVER_MessageDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_MessageSerializeToJson() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricsDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricsFormatted() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceLevelString() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceActivityString() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceTensorNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceId() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceParentId() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceModelName() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceModelVersion() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceRequestId() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceSpawnChildTrace() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceSetContext() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceContext() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestId() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetId() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestFlags() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetFlags() { } TRITONAPI_DECLSPEC void TRITONBACKEND_InferenceRequestTimeoutMicroseconds() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestCorrelationId() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestCorrelationIdString() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetCorrelationId() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetCorrelationIdString() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestPriority() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestPriorityUInt64() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetPriority() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetPriorityUInt64() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestTimeoutMicroseconds() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetTimeoutMicroseconds() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestAddInput() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestAddRawInput() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestRemoveInput() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestRemoveAllInputs() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestAppendInputData() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestRemoveAllInputData() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestAddRequestedOutput() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestRemoveRequestedOutput() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestRemoveAllRequestedOutputs() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetReleaseCallback() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetResponseCallback() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseError() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseModel() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseId() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseParameterCount() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseParameter() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseOutputCount() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseOutput() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceResponseOutputClassificationLabel() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetServerId() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetModelRepositoryPath() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetModelControlMode() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetStartupModel() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetStrictModelConfig() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetModelConfigName() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetRateLimiterMode() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsAddRateLimiterResource() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetCudaVirtualAddressSize() { } // Deprecated. See TRITONSERVER_ServerOptionsSetCacheConfig instead. TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetResponseCacheByteSize() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetCacheConfig() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetCacheDirectory() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetExitOnError() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetStrictReadiness() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetExitTimeout() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetBufferManagerThreadCount() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetModelLoadThreadCount() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetModelLoadRetryCount() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetModelNamespacing() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetEnablePeerAccess() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetLogFile() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetLogInfo() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetLogWarn() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetLogError() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetLogVerbose() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetLogFormat() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetMetrics() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetGpuMetrics() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetCpuMetrics() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetMetricsInterval() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetBackendDirectory() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetRepoAgentDirectory() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetBackendConfig() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetHostPolicy() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesSetMemoryTypeId() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesSetMemoryType() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesSetCudaIpcHandle() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesSetByteSize() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesMemoryTypeId() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesMemoryType() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesCudaIpcHandle() { } TRITONAPI_DECLSPEC void TRITONSERVER_BufferAttributesByteSize() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerStop() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerSetExitTimeout() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerPollModelRepository() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerIsLive() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerIsReady() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerModelIsReady() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerModelBatchProperties() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerModelTransactionProperties() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerMetadata() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerModelMetadata() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerModelStatistics() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerModelConfig() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerModelIndex() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerRegisterModelRepository() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerUnregisterModelRepository() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerLoadModel() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerLoadModelWithParameters() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerUnloadModel() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerUnloadModelAndDependents() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerMetrics() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerInferAsync() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ApiVersion() { } TRITONAPI_DECLSPEC void TRITONBACKEND_MemoryManagerAllocate() { } TRITONAPI_DECLSPEC void TRITONBACKEND_MemoryManagerFree() { } TRITONAPI_DECLSPEC void TRITONBACKEND_InputProperties() { } TRITONAPI_DECLSPEC void TRITONBACKEND_InputPropertiesForHostPolicy() { } TRITONAPI_DECLSPEC void TRITONBACKEND_InputBuffer() { } TRITONAPI_DECLSPEC void TRITONBACKEND_InputBufferForHostPolicy() { } TRITONAPI_DECLSPEC void TRITONBACKEND_InputBufferAttributes() { } TRITONAPI_DECLSPEC void TRITONBACKEND_OutputBuffer() { } TRITONAPI_DECLSPEC void TRITONBACKEND_OutputBufferAttributes() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestId() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseFactoryIsCancelled() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestIsCancelled() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestCorrelationId() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestCorrelationIdString() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestFlags() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestInputCount() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestInputName() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestInput() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestInputByIndex() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestOutputCount() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestOutputName() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestOutputBufferProperties() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestRelease() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestTrace() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetBoolParameter() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetDoubleParameter() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetIntParameter() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceRequestSetStringParameter() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestParameter() { } TRITONAPI_DECLSPEC void TRITONBACKEND_RequestParameterCount() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseFactoryNew() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseFactoryDelete() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseFactorySendFlags() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseNew() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseNewFromFactory() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseDelete() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseSetStringParameter() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseSetIntParameter() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseSetBoolParameter() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseSetDoubleParameter() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseOutput() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ResponseSend() { } TRITONAPI_DECLSPEC void TRITONBACKEND_StateNew() { } TRITONAPI_DECLSPEC void TRITONBACKEND_StateUpdate() { } TRITONAPI_DECLSPEC void TRITONBACKEND_StateBuffer() { } TRITONAPI_DECLSPEC void TRITONBACKEND_StateBufferAttributes() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendName() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendConfig() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendExecutionPolicy() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendSetExecutionPolicy() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendArtifacts() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendMemoryManager() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendState() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendSetState() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelName() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelVersion() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelRepository() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelConfig() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelAutoCompleteConfig() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelSetConfig() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelServer() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelBackend() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelState() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelSetState() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelReportMemoryUsage() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceName() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceKind() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceDeviceId() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceHostPolicy() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceIsPassive() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceProfileCount() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceProfileName() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceSecondaryDeviceCount() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceSecondaryDeviceProperties() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceModel() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceState() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceSetState() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceReportMemoryUsage() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceReportStatistics() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceResponseStatisticsNew() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceResponseStatisticsSetModelInstance() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseFactory() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseStart() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceResponseStatisticsSetComputeOutputStart() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceResponseStatisticsSetResponseEnd() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceResponseStatisticsSetError() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceResponseStatisticsDelete() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceReportResponseStatistics() { } TRITONAPI_DECLSPEC void TRITONBACKEND_ModelInstanceReportBatchStatistics() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ApiVersion() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelRepositoryLocation() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelRepositoryLocationAcquire() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelRepositoryLocationRelease() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelRepositoryUpdate() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelParameterCount() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelParameter() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelConfig() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelState() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_ModelSetState() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_State() { } TRITONAPI_DECLSPEC void TRITONREPOAGENT_SetState() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricFamilyNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricFamilyDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_GetMetricFamilyKind() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricArgsNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricArgsSetHistogram() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricArgsDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricNew() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricNewWithArgs() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricDelete() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricValue() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricIncrement() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricSet() { } TRITONAPI_DECLSPEC void TRITONSERVER_MetricObserve() { } TRITONAPI_DECLSPEC void TRITONSERVER_GetMetricKind() { } TRITONAPI_DECLSPEC void TRITONSERVER_ServerOptionsSetMetricsConfig() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup() { } TRITONAPI_DECLSPEC void TRITONSERVER_InferenceTraceReportActivity() { } TRITONAPI_DECLSPEC void TRITONBACKEND_BackendAttributeSetParallelModelInstanceLoading() { } TRITONAPI_DECLSPEC void TRITONCACHE_ApiVersion() { } TRITONAPI_DECLSPEC void TRITONCACHE_CacheEntryBufferCount() { } TRITONAPI_DECLSPEC void TRITONCACHE_CacheEntryAddBuffer() { } TRITONAPI_DECLSPEC void TRITONCACHE_CacheEntryGetBuffer() { } TRITONAPI_DECLSPEC void TRITONCACHE_CacheEntrySetBuffer() { } TRITONAPI_DECLSPEC void TRITONCACHE_CacheInitialize() { } TRITONAPI_DECLSPEC void TRITONCACHE_CacheFinalize() { } TRITONAPI_DECLSPEC void TRITONCACHE_CacheInsert() { } TRITONAPI_DECLSPEC void TRITONCACHE_CacheLookup() { } TRITONAPI_DECLSPEC void TRITONCACHE_Copy() { } } /* extern "C" */ kytensor-2.49.0.7/framework_backends/000077500000000000000000000000001527052371200174645ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/identity_backend/000077500000000000000000000000001527052371200227645ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/identity_backend/.clang-format000066400000000000000000000014261527052371200253420ustar00rootroot00000000000000--- BasedOnStyle: Google IndentWidth: 2 ColumnLimit: 80 ContinuationIndentWidth: 4 UseTab: Never MaxEmptyLinesToKeep: 2 SortIncludes: true CompactNamespaces: true ReflowComments: true DerivePointerAlignment: false PointerAlignment: Left AllowShortIfStatementsOnASingleLine: false AllowShortBlocksOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AlwaysBreakAfterReturnType: TopLevelDefinitions AlignAfterOpenBracket: AlwaysBreak BreakBeforeBraces: Custom BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: true AfterNamespace: false AfterStruct: false AfterUnion: false BeforeCatch: true BinPackArguments: true BinPackParameters: true ConstructorInitializerAllOnOneLineOrOnePerLine: false IndentCaseLabels: true kytensor-2.49.0.7/framework_backends/identity_backend/.github/000077500000000000000000000000001527052371200243245ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/identity_backend/.github/workflows/000077500000000000000000000000001527052371200263615ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/identity_backend/.github/workflows/pre-commit.yml000066400000000000000000000033201527052371200311560ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: pre-commit on: pull_request: jobs: pre-commit: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - uses: pre-commit/action@v3.0.0 kytensor-2.49.0.7/framework_backends/identity_backend/.pre-commit-config.yaml000066400000000000000000000056111527052371200272500ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black rev: 23.1.0 hooks: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] types_or: [python, cython] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.5 hooks: - id: clang-format types_or: [c, c++, cuda, proto, textproto, java] args: ["-fallback-style=none", "-style=file", "-i"] - repo: https://github.com/codespell-project/codespell rev: v2.2.4 hooks: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$) # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-json - id: check-toml - id: check-yaml - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace kytensor-2.49.0.7/framework_backends/identity_backend/CMakeLists.txt000066400000000000000000000134421527052371200255300ustar00rootroot00000000000000# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonidentitybackend LANGUAGES C CXX) # Use C++17 standard as Triton's minimum required. set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.") # # Options # # Must include options required for this project as well as any # projects included in this one by FetchContent. # # GPU support is disabled by default because identity backend doesn't # support GPUs. # option(TRITON_ENABLE_GPU "Enable GPU support in backend" OFF) option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON) option(TRITON_ENABLE_METRICS "Include metrics support in backend" ON) set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # # FetchContent's composability isn't very good. We must include the # transitive closure of all repos so that we can override the tag. # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-backend SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/backend ) FetchContent_MakeAvailable(repo-common repo-core repo-backend) # # Shared library implementing the Triton Backend API # configure_file(src/libtriton_identity.ldscript libtriton_identity.ldscript COPYONLY) add_library( triton-identity-backend SHARED src/identity.cc ) add_library( TritonIdentityBackend::triton-identity-backend ALIAS triton-identity-backend ) target_include_directories( triton-identity-backend PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_compile_features(triton-identity-backend PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-identity-backend PRIVATE $<$,$,$>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc> ) if(${TRITON_ENABLE_METRICS}) target_compile_definitions( triton-identity-backend PRIVATE TRITON_ENABLE_METRICS=1 ) endif() # TRITON_ENABLE_METRICS target_link_libraries( triton-identity-backend PRIVATE triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core triton-backend-utils # from repo-backend ) if(WIN32) set_target_properties( triton-identity-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_identity ) else() set_target_properties( triton-identity-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_identity LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_identity.ldscript LINK_FLAGS "-Wl,--version-script libtriton_identity.ldscript" ) endif() # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonIdentityBackend) install( TARGETS triton-identity-backend EXPORT triton-identity-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/identity RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/identity ) install( EXPORT triton-identity-backend-targets FILE TritonIdentityBackendTargets.cmake NAMESPACE TritonIdentityBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonIdentityBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonIdentityBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonIdentityBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-identity-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonIdentityBackendTargets.cmake NAMESPACE TritonIdentityBackend:: ) export(PACKAGE TritonIdentityBackend) kytensor-2.49.0.7/framework_backends/identity_backend/LICENSE000066400000000000000000000027221527052371200237740ustar00rootroot00000000000000Copyright (c) 2018-2020, NVIDIA CORPORATION. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/framework_backends/identity_backend/README.md000066400000000000000000000077351527052371200242570ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Triton Inference Server Identity Backend A simple Triton backend that copies input tensors to corresponding output tensors. This backend is used primarily for testing. To learn more about writing your own Triton backend including simple examples, see the documentation included in the [backend repo](https://github.com/triton-inference-server/backend). Ask questions or report problems with the Identity backend in the main Triton [issues page](https://github.com/triton-inference-server/server/issues). ## Build Use cmake to build and install in a local directory. ``` $ mkdir build $ cd build $ cmake -DTRITON_ENABLE_GPU=ON -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install .. $ make install ``` The following required Triton repositories will be pulled and used in the build. By default the "main" branch/tag will be used for each repo but the listed CMake argument can be used to override. * triton-inference-server/backend: -DTRITON_BACKEND_REPO_TAG=[tag] * triton-inference-server/core: -DTRITON_CORE_REPO_TAG=[tag] * triton-inference-server/common: -DTRITON_COMMON_REPO_TAG=[tag] If you are building on a release branch (or on a development branch that is based off of a release branch), then you must set these cmake arguments to point to that release branch as well. For example, if you are building the r23.04 identity_backend branch then you need to use the following additional cmake flags: ``` -DTRITON_BACKEND_REPO_TAG=r23.04 -DTRITON_CORE_REPO_TAG=r23.04 -DTRITON_COMMON_REPO_TAG=r23.04 ``` ## Custom Metric Example When `TRITON_ENABLE_METRICS` is enabled, this backend implements an example of registering a custom metric to Triton's existing metrics endpoint via the [Metrics API](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md#custom-metrics). This metric will track the cumulative `input_byte_size` of all requests to this backend per-model. Here's an example output of the custom metric from Triton's metrics endpoint after a few requests to each model: ``` # HELP input_byte_size_counter Cumulative input byte size of all requests received by the model # TYPE input_byte_size_counter counter input_byte_size_counter{model="identity_uint32",version="1"} 64.000000 input_byte_size_counter{model="identity_fp32",version="1"} 32.000000 ``` This example can be referenced to implement custom metrics for various use cases. kytensor-2.49.0.7/framework_backends/identity_backend/cmake/000077500000000000000000000000001527052371200240445ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/identity_backend/cmake/TritonIdentityBackendConfig.cmake.in000066400000000000000000000036611527052371200330500ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONIDENTITYBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONIDENTITYBACKEND_CMAKE_DIR}) if(NOT TARGET TritonIdentityBackend::triton-identity-backend) include("${TRITONIDENTITYBACKEND_CMAKE_DIR}/TritonIdentityBackendTargets.cmake") endif() set(TRITONIDENTITYBACKEND_LIBRARIES TritonIdentityBackend::triton-identity-backend) kytensor-2.49.0.7/framework_backends/identity_backend/pyproject.toml000066400000000000000000000043031527052371200257000ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [tool.codespell] # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - # this is only to allow you to run codespell interactively skip = "./.git,./.github" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" # use the 'clear' dictionary for unambiguous spelling mistakes builtin = "clear" # disable warnings about binary files and wrong encoding quiet-level = 3 [tool.isort] profile = "black" use_parentheses = true multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 ensure_newline_before_comments = true line_length = 88 balanced_wrapping = true indent = " " skip = ["build"] kytensor-2.49.0.7/framework_backends/identity_backend/src/000077500000000000000000000000001527052371200235535ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/identity_backend/src/identity.cc000066400000000000000000001470521527052371200257240ustar00rootroot00000000000000// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" #include "triton/core/tritonbackend.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace backend { namespace identity { // // Simple backend that demonstrates the TRITONBACKEND API for a blocking // backend. A blocking backend completes execution of the inference before // returning from TRITONBACKEND_ModelInstanceExecute. // // This backend supports any model that has same number of inputs and outputs. // The input and output must follow a naming convention i.e INPUT and // OUTPUT where 'index' is the input/output index. The datatype and // shape/reshaped shape must match. The backend simply responds with the output // tensors equal to the corresponding input tensors. // #define GUARDED_RESPOND_IF_ERROR(RESPONSES, IDX, X) \ do { \ if ((RESPONSES)[IDX] != nullptr) { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (RESPONSES)[IDX], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ err__), \ "failed to send error response"); \ (RESPONSES)[IDX] = nullptr; \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) // Custom object to store global state for this backend struct IdentityBackendState { TRITONSERVER_MetricFamily* metric_family_ = nullptr; std::string message_ = "backend state"; explicit IdentityBackendState() { #ifdef TRITON_ENABLE_METRICS // Create metric family THROW_IF_BACKEND_MODEL_ERROR(TRITONSERVER_MetricFamilyNew( &metric_family_, TRITONSERVER_METRIC_KIND_COUNTER, "input_byte_size_counter", "Cumulative input byte size of all requests received by the model")); #endif // TRITON_ENABLE_METRICS } ~IdentityBackendState() { #ifdef TRITON_ENABLE_METRICS if (metric_family_ != nullptr) { TRITONSERVER_MetricFamilyDelete(metric_family_); } #endif // TRITON_ENABLE_METRICS } }; // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each TRITONBACKEND_Model. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); ~ModelState(); // Get execution delay and delay multiplier uint64_t ExecDelay() const { return execute_delay_ms_; } uint64_t DelayMultiplier() const { return delay_multiplier_; } // Get the amount of nested custom trace spans to test bool EnableCustomTracing() const { return enable_custom_tracing_; } uint64_t NestedSpanCount() const { return nested_span_count_; } uint64_t SingleActivityFrequency() const { return single_activity_frequency_; } const std::map>>& OptionalInputs() { return optional_inputs_; } // Stores the instance count. Atomic to protect reads/writes by all instances. std::atomic instance_count_; // Validate that model configuration is supported by this backend. TRITONSERVER_Error* ValidateModelConfig(); // Block the thread for seconds specified in 'creation_delay_sec' parameter. // This function is used for testing. TRITONSERVER_Error* CreationDelay(); #ifdef TRITON_ENABLE_METRICS // Setup metrics for this backend. TRITONSERVER_Error* InitMetrics( TRITONSERVER_MetricFamily* family, std::string model_name, uint64_t model_version); // Update metrics for this backend. TRITONSERVER_Error* UpdateMetrics(uint64_t input_byte_size); #endif // TRITON_ENABLE_METRICS private: ModelState(TRITONBACKEND_Model* triton_model); // Delay time and multiplier to introduce into execution, in milliseconds. int execute_delay_ms_; int delay_multiplier_; // Amount of nested custom trace spans to test. bool enable_custom_tracing_{false}; int nested_span_count_{0}; int single_activity_frequency_{1}; // Store index that the corresponding inputs can be optional. Also store // the output metadata to use, if an input is marked optional and not provided // in inference while the output is requested std::map>> optional_inputs_; #ifdef TRITON_ENABLE_METRICS // Custom metrics associated with this model TRITONSERVER_Metric* input_byte_size_counter_ = nullptr; #endif // TRITON_ENABLE_METRICS }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } // Identity backend supports optional inputs intrinsically as it executes // requests at per input basis ModelState::ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model, true /* allow_optional */), instance_count_(0), execute_delay_ms_(0), delay_multiplier_(0) { } ModelState::~ModelState() { #ifdef TRITON_ENABLE_METRICS if (input_byte_size_counter_ != nullptr) { TRITONSERVER_MetricDelete(input_byte_size_counter_); } #endif // TRITON_ENABLE_METRICS } #ifdef TRITON_ENABLE_METRICS TRITONSERVER_Error* ModelState::InitMetrics( TRITONSERVER_MetricFamily* family, std::string model_name, uint64_t model_version) { // Create labels for model/version pair to breakdown backend metrics per-model std::vector labels; labels.emplace_back(TRITONSERVER_ParameterNew( "model", TRITONSERVER_PARAMETER_STRING, model_name.c_str())); labels.emplace_back(TRITONSERVER_ParameterNew( "version", TRITONSERVER_PARAMETER_STRING, std::to_string(model_version).c_str())); RETURN_IF_ERROR(TRITONSERVER_MetricNew( &input_byte_size_counter_, family, labels.data(), labels.size())); for (const auto label : labels) { TRITONSERVER_ParameterDelete(const_cast(label)); } return nullptr; // success } TRITONSERVER_Error* ModelState::UpdateMetrics(uint64_t input_byte_size) { RETURN_IF_ERROR( TRITONSERVER_MetricIncrement(input_byte_size_counter_, input_byte_size)); return nullptr; // success } #endif // TRITON_ENABLE_METRICS TRITONSERVER_Error* ModelState::CreationDelay() { // Feature for testing purpose... // look for parameter 'creation_delay_sec' in model config // and sleep for the value specified common::TritonJson::Value parameters; if (model_config_.Find("parameters", ¶meters)) { common::TritonJson::Value creation_delay_sec; if (parameters.Find("creation_delay_sec", &creation_delay_sec)) { std::string creation_delay_sec_str; RETURN_IF_ERROR(creation_delay_sec.MemberAsString( "string_value", &creation_delay_sec_str)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Creation delay is set to: ") + creation_delay_sec_str) .c_str()); std::this_thread::sleep_for( std::chrono::seconds(std::stoi(creation_delay_sec_str))); } } return nullptr; // success } TRITONSERVER_Error* ModelState::ValidateModelConfig() { // We have the json DOM for the model configuration... common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(model_config_.PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model configuration:\n") + buffer.Contents()).c_str()); common::TritonJson::Value inputs, outputs; RETURN_IF_ERROR(model_config_.MemberAsArray("input", &inputs)); RETURN_IF_ERROR(model_config_.MemberAsArray("output", &outputs)); // There must be equal number of inputs and outputs. RETURN_ERROR_IF_FALSE( inputs.ArraySize() == outputs.ArraySize(), TRITONSERVER_ERROR_INVALID_ARG, std::string("model configuration must have equal input/output pairs")); // Collect input/output names, shapes and datatypes std::map>> input_infos, output_infos; for (size_t io_index = 0; io_index < inputs.ArraySize(); io_index++) { common::TritonJson::Value input, output; RETURN_IF_ERROR(inputs.IndexAsObject(io_index, &input)); RETURN_IF_ERROR(outputs.IndexAsObject(io_index, &output)); // Input and output names must follow INPUT/OUTPUT pattern const char* input_name; size_t input_name_len; RETURN_IF_ERROR(input.MemberAsString("name", &input_name, &input_name_len)); const char* output_name; size_t output_name_len; RETURN_IF_ERROR( output.MemberAsString("name", &output_name, &output_name_len)); std::string input_name_str = std::string(input_name); RETURN_ERROR_IF_FALSE( input_name_str.rfind("INPUT", 0) == 0, TRITONSERVER_ERROR_INVALID_ARG, std::string( "expected input name to follow INPUT pattern, got '") + input_name + "'"); // Check if input is optional bool optional = false; RETURN_IF_ERROR(input.MemberAsBool("optional", &optional)); std::string output_name_str = std::string(output_name); RETURN_ERROR_IF_FALSE( output_name_str.rfind("OUTPUT", 0) == 0, TRITONSERVER_ERROR_INVALID_ARG, std::string( "expected output name to follow OUTPUT pattern, got '") + output_name + "'"); // Input and output must have same datatype std::string input_dtype, output_dtype; RETURN_IF_ERROR(input.MemberAsString("data_type", &input_dtype)); RETURN_IF_ERROR(output.MemberAsString("data_type", &output_dtype)); // Input and output must have same shape or reshaped shape std::vector input_shape, output_shape; triton::common::TritonJson::Value reshape; if (input.Find("reshape", &reshape)) { RETURN_IF_ERROR(backend::ParseShape(reshape, "shape", &input_shape)); } else { RETURN_IF_ERROR(backend::ParseShape(input, "dims", &input_shape)); } if (output.Find("reshape", &reshape)) { RETURN_IF_ERROR(backend::ParseShape(reshape, "shape", &output_shape)); } else { RETURN_IF_ERROR(backend::ParseShape(output, "dims", &output_shape)); } input_infos.insert(std::make_pair( input_name_str.substr(strlen("INPUT")), std::make_tuple(input_dtype, input_shape))); output_infos.insert(std::make_pair( output_name_str.substr(strlen("OUTPUT")), std::make_tuple(output_dtype, output_shape))); if (optional) { const int idx = std::stoi(std::string(input_name, 5, -1)); const auto dtype = ModelConfigDataTypeToTritonServerDataType(input_dtype); auto shape = input_shape; for (auto& dim : shape) { if (dim == -1) { dim = 1; } } optional_inputs_[idx] = std::make_tuple(dtype, shape); } } // Must validate name, shape and datatype with corresponding input for (auto it = output_infos.begin(); it != output_infos.end(); ++it) { std::string output_index = it->first; auto input_it = input_infos.find(output_index); RETURN_ERROR_IF_FALSE( input_it != input_infos.end(), TRITONSERVER_ERROR_INVALID_ARG, std::string("expected input and output indices to match")); RETURN_ERROR_IF_FALSE( std::get<0>(input_it->second) == std::get<0>(it->second), TRITONSERVER_ERROR_INVALID_ARG, std::string("expected input and output datatype to match, got ") + std::get<0>(input_it->second) + " and " + std::get<0>(it->second)); RETURN_ERROR_IF_FALSE( std::get<1>(input_it->second) == std::get<1>(it->second), TRITONSERVER_ERROR_INVALID_ARG, std::string("expected input and output shape to match, got ") + backend::ShapeToString(std::get<1>(input_it->second)) + " and " + backend::ShapeToString(std::get<1>(it->second))); } triton::common::TritonJson::Value params; if (model_config_.Find("parameters", ¶ms)) { common::TritonJson::Value exec_delay; if (params.Find("execute_delay_ms", &exec_delay)) { std::string exec_delay_str; RETURN_IF_ERROR( exec_delay.MemberAsString("string_value", &exec_delay_str)); execute_delay_ms_ = std::stoi(exec_delay_str); // Apply delay multiplier based on instance index, this is not taking // multiple devices into consideration, so the behavior is best controlled // in single device case. common::TritonJson::Value delay_multiplier; if (params.Find("instance_wise_delay_multiplier", &delay_multiplier)) { std::string delay_multiplier_str; RETURN_IF_ERROR(delay_multiplier.MemberAsString( "string_value", &delay_multiplier_str)); delay_multiplier_ = std::stoi(delay_multiplier_str); } } common::TritonJson::Value custom_tracing; if (params.Find("enable_custom_tracing", &custom_tracing)) { std::string enable_custom_tracing_str; RETURN_IF_ERROR(custom_tracing.MemberAsString( "string_value", &enable_custom_tracing_str)); enable_custom_tracing_ = enable_custom_tracing_str == "true"; common::TritonJson::Value nested_span_count; if (params.Find("nested_span_count", &nested_span_count)) { std::string nested_span_count_str; RETURN_IF_ERROR(nested_span_count.MemberAsString( "string_value", &nested_span_count_str)); nested_span_count_ = std::stoi(nested_span_count_str); common::TritonJson::Value single_activity_frequency; if (params.Find( "single_activity_frequency", &single_activity_frequency)) { std::string single_activity_frequency_str; RETURN_IF_ERROR(single_activity_frequency.MemberAsString( "string_value", &single_activity_frequency_str)); single_activity_frequency_ = std::stoi(single_activity_frequency_str); } } } } return nullptr; // success } // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each TRITONBACKEND_ModelInstance. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState() = default; // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } // Get the instance ID of the instance. size_t InstanceId() const { return instance_id_; } private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, const size_t instance_id); ModelState* model_state_; const size_t instance_id_; }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { // Must be initialized and incremented before passing to constructor // to guarantee unique_ids if parallel instance loading is supported. const auto instance_id = model_state->instance_count_++; try { *state = new ModelInstanceState(model_state, triton_model_instance, instance_id); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelInstanceState::ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, const size_t instance_id) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state), instance_id_(instance_id) { } ///////////// extern "C" { // Implementing TRITONBACKEND_Initialize is optional. The backend // should initialize any global state that is intended to be shared // across all models and model instances that use the backend. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); // We should check the backend API version that Triton supports // vs. what this backend was compiled against. uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Triton TRITONBACKEND API version: ") + std::to_string(api_version_major) + "." + std::to_string(api_version_minor)) .c_str()); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("'") + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "triton backend API version does not support this backend"); } // The backend configuration may contain information needed by the // backend, such a command-line arguments. This backend doesn't use // any such configuration but we print whatever is available. TRITONSERVER_Message* backend_config_message; RETURN_IF_ERROR( TRITONBACKEND_BackendConfig(backend, &backend_config_message)); const char* buffer; size_t byte_size; RETURN_IF_ERROR(TRITONSERVER_MessageSerializeToJson( backend_config_message, &buffer, &byte_size)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("backend configuration:\n") + buffer).c_str()); // If we have any global backend state we create and set it here. We // make use of the global backend state here to track a custom metric across // all models using this backend if metrics are enabled. try { IdentityBackendState* state = new IdentityBackendState(); RETURN_IF_ERROR( TRITONBACKEND_BackendSetState(backend, reinterpret_cast(state))); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } // Implementing TRITONBACKEND_Finalize is optional unless state is set // using TRITONBACKEND_BackendSetState. The backend must free this // state and perform any other global cleanup. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Finalize(TRITONBACKEND_Backend* backend) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_BackendState(backend, &vstate)); RETURN_ERROR_IF_TRUE( vstate == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr state in TRITONBACKEND_Finalize")); IdentityBackendState* state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Finalize: state is '") + state->message_ + "'") .c_str()); delete state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInitialize is optional. The backend // should initialize any state that is intended to be shared across // all instances of the model. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); // Can get location of the model artifacts. Normally we would need // to check the artifact type to make sure it was something we can // handle... but we are just going to log the location so we don't // need the check. We would use the location if we wanted to load // something from the model's repo. TRITONBACKEND_ArtifactType artifact_type; const char* clocation; RETURN_IF_ERROR( TRITONBACKEND_ModelRepository(model, &artifact_type, &clocation)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Repository location: ") + clocation).c_str()); // The model can access the backend as well... here we can access // the backend global state. We will use it to add per-model metrics // to the global metric family object stored in the state, if metrics // are enabled, TRITONBACKEND_Backend* backend; RETURN_IF_ERROR(TRITONBACKEND_ModelBackend(model, &backend)); void* vbackendstate; RETURN_IF_ERROR(TRITONBACKEND_BackendState(backend, &vbackendstate)); RETURN_ERROR_IF_TRUE( vbackendstate == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr state in TRITONBACKEND_ModelInitialize")); IdentityBackendState* backend_state = reinterpret_cast(vbackendstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("backend state is '") + backend_state->message_ + "'") .c_str()); // Create a ModelState object and associate it with the TRITONBACKEND_Model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast(model_state))); // One of the primary things to do in ModelInitialize is to examine // the model configuration to ensure that it is something that this // backend can support. If not, returning an error from this // function will prevent the model from loading. RETURN_IF_ERROR(model_state->ValidateModelConfig()); // For testing.. Block the thread for certain time period before returning. RETURN_IF_ERROR(model_state->CreationDelay()); #ifdef TRITON_ENABLE_METRICS // Create custom metric per model with metric family shared across backend RETURN_IF_ERROR( model_state->InitMetrics(backend_state->metric_family_, name, version)); #endif // TRITON_ENABLE_METRICS return nullptr; // success } // Implementing TRITONBACKEND_ModelFinalize is optional unless state // is set using TRITONBACKEND_ModelSetState. The backend must free // this state and perform any other cleanup. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); RETURN_ERROR_IF_TRUE( vstate == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr state in TRITONBACKEND_ModelFinalize")); ModelState* model_state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceInitialize is optional. The // backend should initialize any state that is required for a model // instance. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); #ifdef TRITON_ENABLE_GPU if (kind == TRITONSERVER_INSTANCEGROUPKIND_GPU) { cudaSetDevice(device_id); } #endif // TRITON_ENABLE_GPU LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); // The instance can access the corresponding model as well... here // we get the model and from that get the model's state. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); RETURN_ERROR_IF_TRUE( vmodelstate == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string( "unexpected nullptr state in TRITONBACKEND_ModelInstanceInitialize")); ModelState* model_state = reinterpret_cast(vmodelstate); // With each instance we create a ModelInstanceState object and // associate it with the TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast(instance_state))); #ifndef TRITON_ENABLE_GPU // Because this backend just copies IN -> OUT and requires that // input and output be in CPU memory, we fail if a GPU instances is // requested. RETURN_ERROR_IF_FALSE( instance_state->Kind() == TRITONSERVER_INSTANCEGROUPKIND_CPU, TRITONSERVER_ERROR_INVALID_ARG, std::string("'identity' backend only supports CPU instances")); #endif // TRITON_ENABLE_GPU return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceFinalize is optional unless // state is set using TRITONBACKEND_ModelInstanceSetState. The backend // must free this state and perform any other cleanup. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); RETURN_ERROR_IF_TRUE( vstate == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string( "unexpected nullptr state in TRITONBACKEND_ModelInstanceFinalize")); ModelInstanceState* instance_state = reinterpret_cast(vstate); #ifdef TRITON_ENABLE_GPU if (instance_state->Kind() == TRITONSERVER_INSTANCEGROUPKIND_GPU) { cudaSetDevice(instance_state->DeviceId()); } #endif // TRITON_ENABLE_GPU LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceExecute is required. TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Suggested practice for this is to use only // function-local and model-instance-specific state (obtained from // 'instance'), which is what we do here. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast(&instance_state))); RETURN_ERROR_IF_TRUE( instance_state == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string( "unexpected nullptr state in TRITONBACKEND_ModelInstanceExecute")); ModelState* model_state = instance_state->StateForModel(); #ifdef TRITON_ENABLE_GPU if (instance_state->Kind() == TRITONSERVER_INSTANCEGROUPKIND_GPU) { cudaSetDevice(instance_state->DeviceId()); } #endif // TRITON_ENABLE_GPU // This backend specifies BLOCKING execution policy. That means that // we should not return from this function until execution is // complete. Triton will automatically release 'instance' on return // from this function so that it is again available to be used for // another call to TRITONBACKEND_ModelInstanceExecute. LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("model ") + model_state->Name() + ", instance " + instance_state->Name() + ", executing " + std::to_string(request_count) + " requests") .c_str()); uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); bool supports_batching = false; RETURN_IF_ERROR(model_state->SupportsFirstDimBatching(&supports_batching)); // 'responses' is initialized with the response objects below and // if/when an error response is sent the corresponding entry in // 'responses' is set to nullptr to indicate that that response has // already been sent. std::vector responses; responses.reserve(request_count); // Create a single response object for each request. If something // goes wrong when attempting to create the response objects just // fail all of the requests by returning an error. for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); responses.push_back(response); } // After this point we take ownership of 'requests', which means // that a response must be sent for every request. If something does // go wrong in processing a particular request then we send an error // response just for the specific request. // The way we collect these batch timestamps is not entirely // accurate. Normally, in a performant backend you would execute all // the requests at the same time, and so there would be a single // compute-start / compute-end time-range. But here we execute each // request separately so there is no single range. As a result we // just show the entire execute time as being the compute time as // well. uint64_t compute_start_ns = 0; SET_TIMESTAMP(compute_start_ns); // Delay if requested... if (model_state->ExecDelay() > 0) { uint64_t multiplier = model_state->DelayMultiplier(); if (model_state->DelayMultiplier() > 0) { multiplier *= instance_state->InstanceId(); } uint64_t delay_ms = model_state->ExecDelay() * std::max(multiplier, uint64_t(1)); std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); } // For simplicity we just process each request separately... in // general a backend should try to operate on the entire batch of // requests at the same time for improved performance. uint64_t total_batch_size = 0; bool cuda_copy = false; for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; if (model_state->EnableCustomTracing()) { // Example for tracing a custom activity from the backend TRITONSERVER_InferenceTrace* trace; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestTrace(request, &trace)); auto nesting_count = model_state->NestedSpanCount(); auto singl_act_frequency = model_state->SingleActivityFrequency(); if (trace != nullptr) { uint64_t custom_activity_ns; const char* activity_name = "CUSTOM_ACTIVITY_START"; SET_TIMESTAMP(custom_activity_ns); GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_InferenceTraceReportActivity( trace, custom_activity_ns, activity_name)); std::this_thread::sleep_for(std::chrono::milliseconds(100)); activity_name = "CUSTOM_ACTIVITY_END"; SET_TIMESTAMP(custom_activity_ns); GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_InferenceTraceReportActivity( trace, custom_activity_ns, activity_name)); uint64_t custom_single_activity_ns; const char* single_activity_name = "CUSTOM_SINGLE_ACTIVITY"; SET_TIMESTAMP(custom_single_activity_ns); GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_InferenceTraceReportActivity( trace, custom_single_activity_ns, single_activity_name)); for (uint32_t count = 0; count < nesting_count; count++) { std::string start_span_str = std::string("CUSTOM_ACTIVITY" + std::to_string(count) + "_START"); SET_TIMESTAMP(custom_activity_ns); GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_InferenceTraceReportActivity( trace, custom_activity_ns, start_span_str.c_str())); std::this_thread::sleep_for(std::chrono::milliseconds(100)); if (count % singl_act_frequency == 0) { SET_TIMESTAMP(custom_single_activity_ns); GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_InferenceTraceReportActivity( trace, custom_single_activity_ns, single_activity_name)); } } for (uint32_t count = 0; count < nesting_count; count++) { std::string end_span_str = std::string( "CUSTOM_ACTIVITY" + std::to_string(nesting_count - count - 1) + "_END"); SET_TIMESTAMP(custom_activity_ns); GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_InferenceTraceReportActivity( trace, custom_activity_ns, end_span_str.c_str())); } } } const char* request_id = ""; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestId(request, &request_id)); uint64_t correlation_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestCorrelationId(request, &correlation_id)); uint32_t input_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInputCount(request, &input_count)); uint32_t requested_output_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestOutputCount(request, &requested_output_count)); // If an error response was sent for the above then display an error message // and move on to next request. if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read request input/output counts, error response sent") .c_str()); continue; } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("request ") + std::to_string(r) + ": id = \"" + request_id + "\", correlation_id = " + std::to_string(correlation_id) + ", input_count = " + std::to_string(input_count) + ", requested_output_count = " + std::to_string(requested_output_count)) .c_str()); // Collect all input indices std::set input_indices; for (uint32_t io_index = 0; io_index < input_count; io_index++) { const char* input_name; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInputName(request, io_index, &input_name)); input_indices.insert(std::stoi(std::string(input_name, 5, -1))); } // For statistics we need to collect the total batch size of all the // requests. If the model doesn't support batching then each request is // necessarily batch-size 1. If the model does support batching then the // first dimension of the shape is the batch size. We only the first input // for this. int64_t batch_dim = 0; if (supports_batching) { TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInputByIndex(request, 0 /* index */, &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input, error response sent") .c_str()); continue; } const int64_t* input_shape; uint32_t input_dims_count; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputProperties( input, nullptr, nullptr, &input_shape, &input_dims_count, nullptr, nullptr)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input properties, error response sent") .c_str()); continue; } if (input_dims_count > 0) { total_batch_size += input_shape[0]; batch_dim = input_shape[0]; } else { total_batch_size++; } } // We validated that the model configuration specifies N inputs, but the // request is not required to request any output at all so we only produce // outputs that are requested. for (uint32_t io_index = 0; io_index < requested_output_count; io_index++) { const char* output_name; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestOutputName(request, io_index, &output_name)); // If an error response was sent while getting the requested output name // then display an error message and move on to next request. if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read requested output name, error response sent") .c_str()); continue; } std::string index_str = std::string(output_name, 6, -1); std::string input_name = "INPUT" + index_str; TRITONBACKEND_Input* input = nullptr; if (input_indices.find(std::stoi(index_str)) != input_indices.end()) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, input_name.c_str(), &input)); // If an error response was sent while getting the input then display an // error message and move on to next request. if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input, error response sent") .c_str()); continue; } } else { const auto it = model_state->OptionalInputs().find(std::stoi(index_str)); if (it != model_state->OptionalInputs().end()) { // If the input is optional but the corresponding output is requested, // return zero tensor TRITONBACKEND_Output* output; const auto& dtype = std::get<0>(it->second); auto shape = std::get<1>(it->second); if (batch_dim != 0) { shape.insert(shape.begin(), batch_dim); } GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_ResponseOutput( responses[r], &output, output_name, dtype, shape.data(), shape.size())); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create response output, error response sent") .c_str()); continue; } // Get the output buffer. void* output_buffer; const auto byte_size = GetByteSize(dtype, shape); TRITONSERVER_MemoryType output_memory_type = TRITONSERVER_MEMORY_CPU; int64_t output_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_OutputBuffer( output, &output_buffer, byte_size, &output_memory_type, &output_memory_type_id)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create output buffer, error response sent") .c_str()); continue; } if (output_memory_type == TRITONSERVER_MEMORY_GPU) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create CPU output buffer")); continue; } // Set zero data memset(output_buffer, 0, byte_size); // Done with the output, continue to the next continue; } else { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, ("failed to get input '" + input_name + "'").c_str())); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input '" + input_name + "', error response sent") .c_str()); continue; } } TRITONSERVER_DataType input_datatype; const int64_t* input_shape; uint32_t input_dims_count; uint64_t input_byte_size; uint32_t input_buffer_count; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputProperties( input, nullptr /* input_name */, &input_datatype, &input_shape, &input_dims_count, &input_byte_size, &input_buffer_count)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input properties, error response sent") .c_str()); continue; } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("\tinput ") + input_name + ": datatype = " + TRITONSERVER_DataTypeString(input_datatype) + ", shape = " + backend::ShapeToString(input_shape, input_dims_count) + ", byte_size = " + std::to_string(input_byte_size) + ", buffer_count = " + std::to_string(input_buffer_count)) .c_str()); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("\trequested_output ") + output_name).c_str()); #ifdef TRITON_ENABLE_METRICS GUARDED_RESPOND_IF_ERROR( responses, r, model_state->UpdateMetrics(input_byte_size)); #endif // TRITON_ENABLE_METRICS // This backend simply copies the output tensors from the corresponding // input tensors. The input tensors contents are available in one or more // contiguous buffers. To do the copy we: // // 1. Create an output tensor in the response. // // 2. Allocate appropriately sized buffer in the output // tensor. // // 3. Iterate over the input tensor buffers and copy the contents into // the output buffer. TRITONBACKEND_Response* response = responses[r]; // Step 1. Create an output tensor. Input and output have same datatype // and shape/reshaped shape TRITONBACKEND_Output* output; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_ResponseOutput( response, &output, output_name, input_datatype, input_shape, input_dims_count)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create response output, error response sent") .c_str()); continue; } // Step 2. Get the output buffer. void* output_buffer; TRITONSERVER_MemoryType output_memory_type = TRITONSERVER_MEMORY_CPU; int64_t output_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_OutputBuffer( output, &output_buffer, input_byte_size, &output_memory_type, &output_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create output buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create output buffer, error response sent") .c_str()); continue; } // Step 3. Copy input -> output size_t output_buffer_offset = 0; for (uint32_t b = 0; b < input_buffer_count; ++b) { const void* input_buffer = nullptr; uint64_t buffer_byte_size = 0; TRITONSERVER_MemoryType input_memory_type = TRITONSERVER_MEMORY_CPU; int64_t input_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( input, b, &input_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error response sent") .c_str()); continue; } bool cuda_used = false; GUARDED_RESPOND_IF_ERROR( responses, r, CopyBuffer( input_name, input_memory_type, input_memory_type_id, output_memory_type, output_memory_type_id, buffer_byte_size, input_buffer, reinterpret_cast(output_buffer) + output_buffer_offset, instance_state->CudaStream(), &cuda_used)); cuda_copy |= cuda_used; if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get copy buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get copy buffer, error response sent") .c_str()); continue; } output_buffer_offset += buffer_byte_size; } } // To demonstrate response parameters we attach some here. Most responses do // not use parameters but they provide a way for backends to communicate // arbitrary information along with the response. LOG_IF_ERROR( TRITONBACKEND_ResponseSetStringParameter( responses[r], "param0", "an example string parameter"), "failed setting string parameter"); LOG_IF_ERROR( TRITONBACKEND_ResponseSetIntParameter(responses[r], "param1", 42), "failed setting integer parameter"); LOG_IF_ERROR( TRITONBACKEND_ResponseSetBoolParameter(responses[r], "param2", false), "failed setting boolean parameter"); LOG_IF_ERROR( TRITONBACKEND_ResponseSetDoubleParameter( responses[r], "param3", 123.123), "failed setting double parameter"); } uint64_t compute_end_ns = 0; SET_TIMESTAMP(compute_end_ns); #ifdef TRITON_ENABLE_GPU if (cuda_copy) { cudaStreamSynchronize(instance_state->CudaStream()); } #endif // TRITON_ENABLE_GPU uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); // If we get to this point then there hasn't been any error and the response // is complete and we can send it. This is the last (and only) response that // we are sending for the request so we must mark it FINAL. If there is an // error when sending all we can do is log it. for (auto& response : responses) { if (response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr), "failed sending response"); } } // There are two types of statistics that we can report... the statistics for // the entire batch of requests that we just executed and statistics for each // individual request. Here we report statistics for each request. for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_state->TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting request statistics"); LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } // Here we report statistics for the entire batch of requests. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( instance_state->TritonModelInstance(), total_batch_size, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting batch request statistics"); return nullptr; // success } TRITONSERVER_Error* TRITONBACKEND_GetBackendAttribute( TRITONBACKEND_Backend* backend, TRITONBACKEND_BackendAttribute* backend_attributes) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, "TRITONBACKEND_GetBackendAttribute: setting attributes"); // This backend can safely handle parallel calls to // TRITONBACKEND_ModelInstanceInitialize (thread-safe). RETURN_IF_ERROR(TRITONBACKEND_BackendAttributeSetParallelModelInstanceLoading( backend_attributes, true)); return nullptr; } } // extern "C" }}} // namespace triton::backend::identity kytensor-2.49.0.7/framework_backends/identity_backend/src/libtriton_identity.ldscript000066400000000000000000000030551527052371200312430ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; kytensor-2.49.0.7/framework_backends/onnxruntime_backend/000077500000000000000000000000001527052371200235215ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/CMakeLists.txt000066400000000000000000000244551527052371200262730ustar00rootroot00000000000000# Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritononnxruntimebackend LANGUAGES C CXX) # # Options # # To build the ONNX Runtime backend you must either: # # - Point to an already built ONNX Runtime using # TRITON_ONNXRUNTIME_INCLUDE_PATHS and # TRITON_ONNXRUNTIME_LIB_PATHS # # or: # # - Set TRITON_BUILD_ONNXRUNTIME_VERSION to the version of ONNX # Runtime that you want to be built for the backend. # # - Set TRITON_BUILD_CONTAINER to the Triton container to use as a # base for the build. On linux you can instead set # TRITON_BUILD_CONTAINER_VERSION to the Triton version that you # want to target with the build and the corresponding container # from NGC will be used. # # - Optionally set TRITON_BUILD_CUDA_VERSION and # TRITON_BUILD_CUDA_HOME. If not set these are automatically set # by using the standard cuda install location. For example on # windows these will be automatically set based on CUDA_PATH, for # example: # # TRITON_BUILD_CUDA_VERSION=11.1 # TRITON_BUILD_CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\v11.1" # # - If you want TensorRT support set # TRITON_ENABLE_ONNXRUNTIME_TENSORRT=ON and set TRITON_BUILD_TENSORRT_HOME. # # Optionally set TRITON_ONNX_TENSORRT_REPO_TAG to specify a branch in https://github.com/onnx/onnx-tensorrt repo # example: # TRITON_ONNX_TENSORRT_REPO_TAG=master # This enables using a version of tensorrt which is not yet supported in ONNXRuntime release branch. # By default we pick the default branch which comes with the requested version of onnxruntime. # # Optionally set TRT_VERSION to specify the version of TRT which is being used. # This along with TRITON_BUILD_ONNXRUNTIME_VERSION is used to pick the right onnx tensorrt parser version. # When TRITON_ONNX_TENSORRT_REPO_TAG is set TRT_VERSION is ignored. # When neither TRITON_ONNX_TENSORRT_REPO_TAG or TRT_VERSION are set # the default parser version which comes with ORT is picked. # # - If you want OpenVINO support set # TRITON_ENABLE_ONNXRUNTIME_OPENVINO=ON and set # TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION to the OpenVino # version that is compatible with the specified version of ONNX # Runtime. # # - Optionally set TRITON_BUILD_TARGET_PLATFORM to either linux, windows or # igpu. If not set, the current platform will be used. If building on # Jetpack, always set to igpu to avoid misdetection. # # - If you want to disable GPU usage, set TRITON_ENABLE_GPU=OFF. # This will make builds with CUDA and TensorRT flags to fail. # option(TRITON_ENABLE_GPU "Enable GPU support in backend" ON) option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON) option(TRITON_ENABLE_ONNXRUNTIME_TENSORRT "Enable TensorRT execution provider for ONNXRuntime backend in server" OFF) option(TRITON_ENABLE_ONNXRUNTIME_OPENVINO "Enable OpenVINO execution provider for ONNXRuntime backend in server" OFF) set(TRITON_BUILD_CONTAINER "" CACHE STRING "Triton container to use a base for build") set(TRITON_BUILD_CONTAINER_VERSION "" CACHE STRING "Triton container version to target") set(TRITON_BUILD_ONNXRUNTIME_VERSION "" CACHE STRING "ONNXRuntime version to build") set(TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION "" CACHE STRING "ONNXRuntime OpenVINO version to build") set(TRITON_BUILD_TARGET_PLATFORM "" CACHE STRING "Target platform for ONNXRuntime build") set(TRITON_BUILD_CUDA_VERSION "" CACHE STRING "Version of CUDA install") set(TRITON_BUILD_CUDA_HOME "" CACHE PATH "Path to CUDA install") set(TRITON_BUILD_CUDNN_HOME "" CACHE PATH "Path to CUDNN install") set(TRITON_BUILD_TENSORRT_HOME "" CACHE PATH "Path to TensorRT install") set(TRITON_ONNXRUNTIME_INCLUDE_PATHS "" CACHE PATH "Paths to ONNXRuntime includes") set(TRITON_ONNX_TENSORRT_REPO_TAG "" CACHE STRING "Tag for onnx-tensorrt repo") set(TRT_VERSION "" CACHE STRING "TRT version for this build.") set(TRITON_ONNXRUNTIME_LIB_PATHS "" CACHE PATH "Paths to ONNXRuntime libraries") set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") # Use C++17 standard as Triton's minimum required. set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.") if (WIN32) if(TRITON_ENABLE_ONNXRUNTIME_OPENVINO) message(FATAL_ERROR "TRITON_ENABLE_ONNXRUNTIME_OPENVINO=ON not supported for Windows") endif() endif() # WIN32 if (NOT TRITON_ENABLE_GPU) if (TRITON_ENABLE_ONNXRUNTIME_TENSORRT) message(FATAL_ERROR "TRITON_ENABLE_ONNXRUNTIME_TENSORRT=ON requires TRITON_ENABLE_GPU=ON") endif() # TRITON_ENABLE_ONNXRUNTIME_TENSORRT endif() # NOT TRITON_ENABLE_GPU if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() set(TRITON_ONNXRUNTIME_DOCKER_BUILD OFF) if(TRITON_ONNXRUNTIME_LIB_PATHS STREQUAL "") set(TRITON_ONNXRUNTIME_DOCKER_BUILD OFF) endif() message(STATUS "Using Onnxruntime docker: ${TRITON_ONNXRUNTIME_DOCKER_BUILD}") if(NOT TRITON_ONNXRUNTIME_DOCKER_BUILD) find_library(ONNXRUNTIME_LIBRARY NAMES onnxruntime PATHS ${TRITON_ONNXRUNTIME_LIB_PATHS}) if(${TRITON_ENABLE_ONNXRUNTIME_OPENVINO}) find_library(OV_LIBRARY NAMES openvino PATHS ${TRITON_ONNXRUNTIME_LIB_PATHS}) endif() # TRITON_ENABLE_ONNXRUNTIME_OPENVINO endif() # # Dependencies # # FetchContent's composability isn't very good. We must include the # transitive closure of all repos so that we can override the tag. # find_package(TritonCommon CONFIG REQUIRED) find_package(TritonCore CONFIG REQUIRED) find_package(TritonBackend CONFIG REQUIRED) find_package(onnxruntime CONFIG REQUIRED) # # CUDA # if(${TRITON_ENABLE_GPU}) find_package(CUDAToolkit REQUIRED) endif() # TRITON_ENABLE_GPU # # Shared library implementing the Triton Backend API # configure_file(src/libtriton_onnxruntime.ldscript libtriton_onnxruntime.ldscript COPYONLY) add_library( triton-onnxruntime-backend SHARED src/onnxruntime.cc src/onnxruntime_loader.cc src/onnxruntime_loader.h src/onnxruntime_utils.cc src/onnxruntime_utils.h ) add_library( TritonOnnxRuntimeBackend::triton-onnxruntime-backend ALIAS triton-onnxruntime-backend ) target_include_directories( triton-onnxruntime-backend PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ${TRITON_ONNXRUNTIME_INCLUDE_PATHS} ) target_compile_features(triton-onnxruntime-backend PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-onnxruntime-backend PRIVATE $<$,$,$>: -Wall -Wno-maybe-uninitialized -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) if(${TRITON_ENABLE_GPU}) target_compile_definitions( triton-onnxruntime-backend PRIVATE TRITON_ENABLE_GPU=1 ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_ONNXRUNTIME_TENSORRT}) target_compile_definitions( triton-onnxruntime-backend PRIVATE TRITON_ENABLE_ONNXRUNTIME_TENSORRT=1 ) endif() # TRITON_ENABLE_ONNXRUNTIME_TENSORRT if(${TRITON_ENABLE_ONNXRUNTIME_OPENVINO}) target_compile_definitions( triton-onnxruntime-backend PRIVATE TRITON_ENABLE_ONNXRUNTIME_OPENVINO=1 ) endif() # TRITON_ENABLE_ONNXRUNTIME_OPENVINO if (WIN32) set_target_properties( triton-onnxruntime-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_onnxruntime SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "$\{ORIGIN\}" ) else () set_target_properties( triton-onnxruntime-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_onnxruntime SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "$\{ORIGIN\}" LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_onnxruntime.ldscript LINK_FLAGS "-Wl,--version-script libtriton_onnxruntime.ldscript" ) endif() FOREACH(p ${TRITON_ONNXRUNTIME_LIB_PATHS}) target_link_directories( triton-onnxruntime-backend PRIVATE ${p} ) ENDFOREACH(p) target_link_libraries( triton-onnxruntime-backend PRIVATE TritonCore::triton-core-serverstub TritonBackend::triton-backend-utils TritonCommon::triton-common-async-work-queue onnxruntime ) if(${TRITON_ENABLE_GPU}) target_link_libraries( triton-onnxruntime-backend PRIVATE CUDA::cudart ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_ONNXRUNTIME_OPENVINO}) target_link_libraries( triton-onnxruntime-backend PRIVATE ${OV_LIBRARY} ) endif() # TRITON_ENABLE_ONNXRUNTIME_OPENVINO kytensor-2.49.0.7/framework_backends/onnxruntime_backend/LICENSE000066400000000000000000000027151527052371200245330ustar00rootroot00000000000000Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/framework_backends/onnxruntime_backend/README.md000066400000000000000000000441231527052371200250040ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # ONNX Runtime Backend The Triton backend for the [ONNX Runtime](https://github.com/microsoft/onnxruntime). You can learn more about Triton backends in the [backend repo](https://github.com/triton-inference-server/backend). Ask questions or report problems on the [issues page](https://github.com/triton-inference-server/onnxruntime_backend/issues). Use a recent cmake to build and install in a local directory. Typically you will want to build an appropriate ONNX Runtime implementation as part of the build. You do this by specifying a ONNX Runtime version and a Triton container version that you want to use with the backend. You can find the combination of versions used in a particular Triton release in the TRITON_VERSION_MAP at the top of build.py in the branch matching the Triton release you are interested in. For example, to build the ONNX Runtime backend for Triton 23.04, use the versions from TRITON_VERSION_MAP in the [r23.04 branch of build.py](https://github.com/triton-inference-server/server/blob/r23.04/build.py#L73). ``` $ mkdir build $ cd build $ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_ONNXRUNTIME_VERSION=1.14.1 -DTRITON_BUILD_CONTAINER_VERSION=23.04 .. $ make install ``` The resulting install/backends/onnxruntime directory can be added to a Triton installation as /opt/tritonserver/backends/onnxruntime. The following required Triton repositories will be pulled and used in the build. By default the "main" branch/tag will be used for each repo but the listed CMake argument can be used to override. * triton-inference-server/backend: -DTRITON_BACKEND_REPO_TAG=[tag] * triton-inference-server/core: -DTRITON_CORE_REPO_TAG=[tag] * triton-inference-server/common: -DTRITON_COMMON_REPO_TAG=[tag] You can add TensorRT support to the ONNX Runtime backend by using -DTRITON_ENABLE_ONNXRUNTIME_TENSORRT=ON. You can add OpenVino support by using -DTRITON_ENABLE_ONNXRUNTIME_OPENVINO=ON -DTRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION=\, where \ is the OpenVino version to use and should match the TRITON_VERSION_MAP entry as described above. So, to build with both TensorRT and OpenVino support: ``` $ mkdir build $ cd build $ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_ONNXRUNTIME_VERSION=1.14.1 -DTRITON_BUILD_CONTAINER_VERSION=23.04 -DTRITON_ENABLE_ONNXRUNTIME_TENSORRT=ON -DTRITON_ENABLE_ONNXRUNTIME_OPENVINO=ON -DTRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION=2021.2.200 .. $ make install ``` ## ONNX Runtime with TensorRT optimization TensorRT can be used in conjunction with an ONNX model to further optimize the performance. To enable TensorRT optimization you must set the model configuration appropriately. There are several optimizations available for TensorRT, like selection of the compute precision and workspace size. The optimization parameters and their description are as follows. * `precision_mode`: The precision used for optimization. Allowed values are "FP32", "FP16" and "INT8". Default value is "FP32". * `max_workspace_size_bytes`: The maximum GPU memory the model can use temporarily during execution. Default value is 1GB. * `int8_calibration_table_name`: Specify INT8 calibration table name. Applicable when precision_mode=="INT8" and the models do not contain Q/DQ nodes. If calibration table is provided for model with Q/DQ nodes then ORT session creation will fail. * `int8_use_native_calibration_table`: Calibration table to use. Allowed values are 1 (use native TensorRT generated calibration table) and 0 (use ORT generated calibration table). Default is 0. **Note: Latest calibration table file needs to be copied to trt_engine_cache_path before inference. Calibration table is specific to models and calibration data sets. Whenever new calibration table is generated, old file in the path should be cleaned up or be replaced. * `trt_engine_cache_enable`: Enable engine caching. * `trt_engine_cache_path`: Specify engine cache path. To explore the usage of more parameters, follow the mapping table below and check [ONNX Runtime doc](https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#execution-provider-options) for detail. > Please link to the latest ONNX Runtime binaries in CMake or build from [main branch of ONNX Runtime](https://github.com/microsoft/onnxruntime/tree/main) to enable latest options. ### Parameter mapping between ONNX Runtime and Triton ONNXRuntime Backend | Key in Triton model configuration | Value in Triton model config | Corresponding TensorRT EP option in ONNX Runtime | Type | | --------------------------------- | --------------------------------------------------- | :----------------------------------------------- | :----- | | max_workspace_size_bytes | e.g: "4294967296" | trt_max_workspace_size | int | | trt_max_partition_iterations | e.g: "1000" | trt_max_partition_iterations | int | | trt_min_subgraph_size | e.g: "1" | trt_min_subgraph_size | int | | precision_mode | "FP16" | trt_fp16_enable | bool | | precision_mode | "INT8" | trt_int8_enable | bool | | int8_calibration_table_name | | trt_int8_calibration_table_name | string | | int8_use_native_calibration_table | e.g: "1" or "true", "0" or "false" | trt_int8_use_native_calibration_table | bool | | trt_dla_enable | | trt_dla_enable | bool | | trt_dla_core | e.g: "0" | trt_dla_core | int | | trt_engine_cache_enable | e.g: "1" or "true", "0" or "false" | trt_engine_cache_enable | bool | | trt_engine_cache_path | | trt_engine_cache_path | string | | trt_engine_cache_prefix | | trt_engine_cache_prefix | string | | trt_dump_subgraphs | e.g: "1" or "true", "0" or "false" | trt_dump_subgraphs | bool | | trt_force_sequential_engine_build | e.g: "1" or "true", "0" or "false" | trt_force_sequential_engine_build | bool | | trt_context_memory_sharing_enable | e.g: "1" or "true", "0" or "false" | trt_context_memory_sharing_enable | bool | | trt_layer_norm_fp32_fallback | e.g: "1" or "true", "0" or "false" | trt_layer_norm_fp32_fallback | bool | | trt_timing_cache_enable | e.g: "1" or "true", "0" or "false" | trt_timing_cache_enable | bool | | trt_timing_cache_path | | trt_timing_cache_path | string | | trt_force_timing_cache | e.g: "1" or "true", "0" or "false" | trt_force_timing_cache | bool | | trt_detailed_build_log | e.g: "1" or "true", "0" or "false" | trt_detailed_build_log | bool | | trt_build_heuristics_enable | e.g: "1" or "true", "0" or "false" | trt_build_heuristics_enable | bool | | trt_sparsity_enable | e.g: "1" or "true", "0" or "false" | trt_sparsity_enable | bool | | trt_builder_optimization_level | e.g: "3" | trt_builder_optimization_level | int | | trt_auxiliary_streams | e.g: "-1" | trt_auxiliary_streams | int | | trt_tactic_sources | e.g: "-CUDNN,+CUBLAS"; | trt_tactic_sources | string | | trt_extra_plugin_lib_paths | | trt_extra_plugin_lib_paths | string | | trt_profile_min_shapes | e.g: "input1:dim1xdimd2...,input2:dim1xdim2...,..." | trt_profile_min_shapes | string | | trt_profile_max_shapes | e.g: "input1:dim1xdimd2...,input2:dim1xdim2...,..." | trt_profile_max_shapes | string | | trt_profile_opt_shapes | e.g: "input1:dim1xdimd2...,input2:dim1xdim2...,..." | trt_profile_opt_shapes | string | | trt_cuda_graph_enable | e.g: "1" or "true", "0" or "false" | trt_cuda_graph_enable | bool | | trt_dump_ep_context_model | e.g: "1" or "true", "0" or "false" | trt_dump_ep_context_model | bool | | trt_ep_context_file_path | | trt_ep_context_file_path | string | | trt_ep_context_embed_mode | e.g: "1" | trt_ep_context_embed_mode | int | The section of model config file specifying these parameters will look like: ``` . . . optimization { execution_accelerators { gpu_execution_accelerator : [ { name : "tensorrt" parameters { key: "precision_mode" value: "FP16" } parameters { key: "max_workspace_size_bytes" value: "1073741824" }} parameters { key: "trt_engine_cache_enable" value: "1" }} ] }} . . . ``` ## ONNX Runtime with CUDA Execution Provider optimization When GPU is enabled for ORT, CUDA execution provider is enabled. If TensorRT is also enabled then CUDA EP is treated as a fallback option (only comes into picture for nodes which TensorRT cannot execute). If TensorRT is not enabled then CUDA EP is the primary EP which executes the models. ORT enabled configuring options for CUDA EP to further optimize based on the specific model and user scenarios. There are several optimizations available, please refer to the [ONNX Runtime doc](https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#cuda-execution-provider) for more details. To enable CUDA EP optimization you must set the model configuration appropriately: ``` optimization { execution_accelerators { gpu_execution_accelerator : [ { name : "cuda" parameters { key: "cudnn_conv_use_max_workspace" value: "0" } parameters { key: "use_ep_level_unified_stream" value: "1" }} ] }} ``` ### Deprecated Parameters The way to specify these specific parameters as shown below is deprecated. For backward compatibility, these parameters are still supported. Please use the above method to specify the parameters. * `cudnn_conv_algo_search`: CUDA Convolution algorithm search configuration. Available options are 0 - EXHAUSTIVE (expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx). This is also the default option, 1 - HEURISTIC (lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7), 2 - DEFAULT (default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM) * `gpu_mem_limit`: CUDA memory limit. To use all possible memory pass in maximum size_t. Defaults to SIZE_MAX. * `arena_extend_strategy`: Strategy used to grow the memory arena. Available options are: 0 = kNextPowerOfTwo, 1 = kSameAsRequested. Defaults to 0. * `do_copy_in_default_stream`: Flag indicating if copying needs to take place on the same stream as the compute stream in the CUDA EP. Available options are: 0 = Use separate streams for copying and compute, 1 = Use the same stream for copying and compute. Defaults to 1. In the model config file, specifying these parameters will look like: ``` . . . parameters { key: "cudnn_conv_algo_search" value: { string_value: "0" } } parameters { key: "gpu_mem_limit" value: { string_value: "4294967200" } } . . . ``` ## ONNX Runtime with OpenVINO optimization [OpenVINO](https://docs.openvinotoolkit.org/latest/index.html) can be used in conjunction with an ONNX model to further optimize performance. To enable OpenVINO optimization you must set the model configuration as shown below. ``` . . . optimization { execution_accelerators { cpu_execution_accelerator : [ { name : "openvino" } ] }} . . . ``` ## Other Optimization Options with ONNX Runtime Details regarding when to use these options and what to expect from them can be found [here](https://onnxruntime.ai/docs/performance/tune-performance.html) ### Model Config Options * `intra_op_thread_count`: Sets the number of threads used to parallelize the execution within nodes. A value of 0 means ORT will pick a default which is number of cores. * `inter_op_thread_count`: Sets the number of threads used to parallelize the execution of the graph (across nodes). If sequential execution is enabled this value is ignored. A value of 0 means ORT will pick a default which is number of cores. * `execution_mode`: Controls whether operators in the graph are executed sequentially or in parallel. Usually when the model has many branches, setting this option to 1 .i.e. "parallel" will give you better performance. Default is 0 which is "sequential execution." * `level`: Refers to the graph optimization level. By default all optimizations are enabled. Allowed values are -1, 1 and 2. -1 refers to BASIC optimizations, 1 refers to basic plus extended optimizations like fusions and 2 refers to all optimizations being disabled. Please find the details [here](https://onnxruntime.ai/docs/performance/graph-optimizations.html). ``` optimization { graph : { level : 1 }} parameters { key: "intra_op_thread_count" value: { string_value: "0" } } parameters { key: "execution_mode" value: { string_value: "0" } } parameters { key: "inter_op_thread_count" value: { string_value: "0" } } ``` * `enable_mem_arena`: Use 1 to enable the arena and 0 to disable. See [this](https://onnxruntime.ai/docs/api/c/struct_ort_api.html#a0bbd62df2b3c119636fba89192240593) for more information. * `enable_mem_pattern`: Use 1 to enable memory pattern and 0 to disable. See [this](https://onnxruntime.ai/docs/api/c/struct_ort_api.html#ad13b711736956bf0565fea0f8d7a5d75) for more information. * `memory.enable_memory_arena_shrinkage`: See [this](https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h) for more information. ### Command line options #### Thread Pools When intra and inter op threads is set to 0 or a value higher than 1, by default ORT creates threadpool per session. This may not be ideal in every scenario, therefore ORT also supports global threadpools. When global threadpools are enabled ORT creates 1 global threadpool which is shared by every session. Use the backend config to enable global threadpool. When global threadpool is enabled, intra and inter op num threads config should also be provided via backend config. Config values provided in model config will be ignored. ``` --backend-config=onnxruntime,enable-global-threadpool=<0,1>, --backend-config=onnxruntime,intra_op_thread_count= , --backend-config=onnxruntime,inter_op_thread_count= ``` #### Default Max Batch Size The default-max-batch-size value is used for max_batch_size during [Autocomplete](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#auto-generated-model-configuration) when no other value is found. Assuming server was not launched with `--disable-auto-complete-config` command-line option, the onnxruntime backend will set the max_batch_size of the model to this default value under the following conditions: 1. Autocomplete has determined the model is capable of batching requests. 2. max_batch_size is 0 in the model configuration or max_batch_size is omitted from the model configuration. If max_batch_size > 1 and no [scheduler](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#scheduling-and-batching) is provided, the dynamic batch scheduler will be used. ``` --backend-config=onnxruntime,default-max-batch-size= ``` The default value of `default-max-batch-size` is 4. kytensor-2.49.0.7/framework_backends/onnxruntime_backend/cmake/000077500000000000000000000000001527052371200246015ustar00rootroot00000000000000TritonOnnxRuntimeBackendConfig.cmake.in000066400000000000000000000037141527052371200342220ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/cmake# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONONNXRUNTIMEBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONONNXRUNTIMEBACKEND_CMAKE_DIR}) if(NOT TARGET TritonOnnxRuntimeBackend::triton-onnxruntime-backend) include("${TRITONONNXRUNTIMEBACKEND_CMAKE_DIR}/TritonOnnxRuntimeBackendTargets.cmake") endif() set(TRITONONNXRUNTIMEBACKEND_LIBRARIES TritonOnnxRuntimeBackend::triton-onnxruntime-backend) kytensor-2.49.0.7/framework_backends/onnxruntime_backend/pyproject.toml000066400000000000000000000043031527052371200264350ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [tool.codespell] # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - # this is only to allow you to run codespell interactively skip = "./.git,./.github" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" # use the 'clear' dictionary for unambiguous spelling mistakes builtin = "clear" # disable warnings about binary files and wrong encoding quiet-level = 3 [tool.isort] profile = "black" use_parentheses = true multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 ensure_newline_before_comments = true line_length = 88 balanced_wrapping = true indent = " " skip = ["build"] kytensor-2.49.0.7/framework_backends/onnxruntime_backend/src/000077500000000000000000000000001527052371200243105ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/src/libtriton_onnxruntime.ldscript000066400000000000000000000030551527052371200325350ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; kytensor-2.49.0.7/framework_backends/onnxruntime_backend/src/onnxruntime.cc000066400000000000000000003664201527052371200272200ustar00rootroot00000000000000// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include "onnxruntime_loader.h" #include "onnxruntime_utils.h" #include "triton/backend/backend_common.h" #include "triton/backend/backend_input_collector.h" #include "triton/backend/backend_memory.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" #include "triton/backend/backend_output_responder.h" #include "triton/backend/device_memory_tracker.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU // // ONNX Runtime Backend that implements the TRITONBACKEND API. // namespace triton { namespace backend { namespace onnxruntime { /// Deleter for OrtSession. struct SessionDeleter { void operator()(OrtSession* f) { OnnxLoader::UnloadSession(f); } }; // BackendConfiguration struct BackendConfiguration { static const BackendConfiguration& RetrieveFrom( TRITONBACKEND_Backend* backend) { void* state = nullptr; THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_BackendState(backend, &state)); return *reinterpret_cast(state); } static const BackendConfiguration& RetrieveFrom(TRITONBACKEND_Model* model) { TRITONBACKEND_Backend* backend = nullptr; THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_ModelBackend(model, &backend)); return RetrieveFrom(backend); } static const BackendConfiguration& RetrieveFrom( TRITONBACKEND_ModelInstance* instance) { TRITONBACKEND_Model* model = nullptr; THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_ModelInstanceModel(instance, &model)); return RetrieveFrom(model); } bool enable_memory_tracker_{false}; int default_max_batch_size_{0}; }; // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; // Load an ONNX model using 'artifact_name' as the name for the ONNX // file/directory. If 'instance_group_kind' is not // TRITONSERVER_INSTANCEGROUPKIND_AUTO then use it and // 'instance_group_device_id' to initialize the appropriate // execution providers. Return in 'model_path' the full path to the // onnx file, return in 'session' and 'allocator' the ORT session // and allocator. TRITONSERVER_Error* LoadModel( const std::string& artifact_name, const TRITONSERVER_InstanceGroupKind instance_group_kind, const int32_t instance_group_device_id, std::string* model_path, OrtSession** session, OrtAllocator** default_allocator, cudaStream_t stream); const std::map>& ModelOutputs() { return model_outputs_; } private: ModelState(TRITONBACKEND_Model* triton_model); TRITONSERVER_Error* AutoCompleteConfig(); TRITONSERVER_Error* AutoCompleteMaxBatch( const OnnxTensorInfoMap& input_tensor_infos, const OnnxTensorInfoMap& output_tensor_infos); TRITONSERVER_Error* AutoCompleteIO( const char* key, const OnnxTensorInfoMap& io_infos); // Session options used when creating a ORT session. std::unique_ptr session_options_; // model_outputs is a map that contains unique outputs that the model must // provide. In the model configuration, the output in the state configuration // can have intersection with the outputs section of the model. If an output // is specified both in the output section and state section, it indicates // that the backend must return the output state to the client too. std::map> model_outputs_; }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } // Auto-complete the configuration if requested... bool auto_complete_config = false; RETURN_IF_ERROR(TRITONBACKEND_ModelAutoCompleteConfig( triton_model, &auto_complete_config)); if (auto_complete_config) { RETURN_IF_ERROR((*state)->AutoCompleteConfig()); RETURN_IF_ERROR((*state)->SetModelConfig()); } auto& model_outputs = (*state)->model_outputs_; // Parse the output states in the model configuration triton::common::TritonJson::Value sequence_batching; if ((*state)->ModelConfig().Find("sequence_batching", &sequence_batching)) { triton::common::TritonJson::Value states; if (sequence_batching.Find("state", &states)) { for (size_t i = 0; i < states.ArraySize(); i++) { triton::common::TritonJson::Value state; RETURN_IF_ERROR(states.IndexAsObject(i, &state)); std::string output_state_name; RETURN_IF_ERROR( state.MemberAsString("output_name", &output_state_name)); auto it = model_outputs.find(output_state_name); if (it == model_outputs.end()) { model_outputs.insert({output_state_name, std::make_pair(-1, i)}); } else { it->second.second = i; } } } } // Parse the output names in the model configuration triton::common::TritonJson::Value outputs; RETURN_IF_ERROR((*state)->ModelConfig().MemberAsArray("output", &outputs)); for (size_t i = 0; i < outputs.ArraySize(); i++) { triton::common::TritonJson::Value output; RETURN_IF_ERROR(outputs.IndexAsObject(i, &output)); std::string output_name_str; RETURN_IF_ERROR(output.MemberAsString("name", &output_name_str)); auto it = model_outputs.find(output_name_str); if (it == model_outputs.end()) { model_outputs.insert({output_name_str, {i, -1}}); } else { it->second.first = i; } } return nullptr; // success } ModelState::ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model, true /* allow_optional */) { // Create session options that will be cloned and used for each // instance when creating that instance's session. OrtSessionOptions* soptions; THROW_IF_BACKEND_MODEL_ORT_ERROR(ort_api->CreateSessionOptions(&soptions)); session_options_.reset(soptions); GraphOptimizationLevel optimization_level = GraphOptimizationLevel::ORT_ENABLE_ALL; { triton::common::TritonJson::Value optimization; if (ModelConfig().Find("optimization", &optimization)) { triton::common::TritonJson::Value graph; if (optimization.Find("graph", &graph)) { int64_t graph_level = 0; THROW_IF_BACKEND_MODEL_ERROR(graph.MemberAsInt("level", &graph_level)); if (graph_level == -1) { optimization_level = GraphOptimizationLevel::ORT_ENABLE_BASIC; } else if (graph_level == 1) { optimization_level = GraphOptimizationLevel::ORT_ENABLE_EXTENDED; } else if (graph_level == 2) { optimization_level = GraphOptimizationLevel::ORT_DISABLE_ALL; } } } } THROW_IF_BACKEND_MODEL_ORT_ERROR( ort_api->SetSessionGraphOptimizationLevel(soptions, optimization_level)); { // Controls whether you want to execute operators in your graph sequentially // or in parallel. Usually when the model has many branches, setting this // option to ExecutionMode::ORT_PARALLEL will give you better performance. int execution_mode = 0; triton::common::TritonJson::Value params; if (ModelConfig().Find("parameters", ¶ms)) { THROW_IF_BACKEND_MODEL_ERROR(TryParseModelStringParameter( params, "execution_mode", &execution_mode, 0)); } // 0 and 1 are the only valid values. if (execution_mode != 0 && execution_mode != 1) { throw BackendModelException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "Invalid configuration value provided. Expected values for " " execution_mode are 0 or 1 but got " + std::to_string(execution_mode) + " .") .c_str()))); } else { THROW_IF_BACKEND_MODEL_ORT_ERROR(ort_api->SetSessionExecutionMode( soptions, execution_mode == 0 ? ExecutionMode::ORT_SEQUENTIAL : ExecutionMode::ORT_PARALLEL)); } } // If global threadpool is enabled, disable per session threads. // If it is not enabled then try to read the configs for intra and inter // op num threads and set them for session. if (OnnxLoader::IsGlobalThreadPoolEnabled()) { THROW_IF_BACKEND_MODEL_ORT_ERROR( ort_api->DisablePerSessionThreads(soptions)); } else { // Enable CPU usage limit and do not set it repeatedly with // intra_op_thread_count, otherwise it will be overwritten { triton::common::TritonJson::Value params; if (ModelConfig().Find("parameters", ¶ms)) { triton::common::TritonJson::Value json_value; if (params.Find("limit_cpu_usage", &json_value)) { std::string string_value; THROW_IF_BACKEND_MODEL_ERROR( json_value.MemberAsString("string_value", &string_value)); bool limit_cpu_usage = false; THROW_IF_BACKEND_MODEL_ERROR( ParseBoolValue(string_value, &limit_cpu_usage)); if (limit_cpu_usage) { int core_count = GetCpuCoreCount(); int intra_threads = std::max(1, static_cast(std::round(core_count * 0.2))); THROW_IF_BACKEND_MODEL_ORT_ERROR( ort_api->SetIntraOpNumThreads(soptions, intra_threads)); } } } } { // Sets the number of threads used to parallelize the execution within // nodes A value of 0 means ORT will pick a default int intra_op_thread_count = 0; triton::common::TritonJson::Value params; if (ModelConfig().Find("parameters", ¶ms)) { THROW_IF_BACKEND_MODEL_ERROR(TryParseModelStringParameter( params, "intra_op_thread_count", &intra_op_thread_count, 0)); } if (intra_op_thread_count > 0) { THROW_IF_BACKEND_MODEL_ORT_ERROR( ort_api->SetIntraOpNumThreads(soptions, intra_op_thread_count)); } } { // Sets the number of threads used to parallelize the execution of the // graph (across nodes) If sequential execution is enabled this value is // ignored A value of 0 means ORT will pick a default int inter_op_thread_count = 0; triton::common::TritonJson::Value params; if (ModelConfig().Find("parameters", ¶ms)) { THROW_IF_BACKEND_MODEL_ERROR(TryParseModelStringParameter( params, "inter_op_thread_count", &inter_op_thread_count, 0)); } if (inter_op_thread_count > 0) { THROW_IF_BACKEND_MODEL_ORT_ERROR( ort_api->SetInterOpNumThreads(soptions, inter_op_thread_count)); } } } // memory configs // enable/disable mem arena { triton::common::TritonJson::Value params; std::string string_value = "false"; bool enable_cpu_mem_arena = false; if (ModelConfig().Find("parameters", ¶ms)) { triton::common::TritonJson::Value json_value; if (params.Find("enable_mem_arena", &json_value)) { std::string string_value; THROW_IF_BACKEND_MODEL_ERROR( json_value.MemberAsString("string_value", &string_value)); THROW_IF_BACKEND_MODEL_ERROR( ParseBoolValue(string_value, &enable_cpu_mem_arena)); } } OrtStatus* ort_status = nullptr; if (enable_cpu_mem_arena) { ort_status = ort_api->EnableCpuMemArena(soptions); } else { ort_status = ort_api->DisableCpuMemArena(soptions); } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Configuring enable_mem_arena to ") + string_value) .c_str()); THROW_IF_BACKEND_MODEL_ORT_ERROR(ort_status); } // enable/disable mem pattern { triton::common::TritonJson::Value params; if (ModelConfig().Find("parameters", ¶ms)) { triton::common::TritonJson::Value json_value; if (params.Find("enable_mem_pattern", &json_value)) { std::string string_value; THROW_IF_BACKEND_MODEL_ERROR( json_value.MemberAsString("string_value", &string_value)); bool enable_mem_pattern = false; auto err = ParseBoolValue(string_value, &enable_mem_pattern); THROW_IF_BACKEND_MODEL_ERROR(err); OrtStatus* ort_status = nullptr; if (enable_mem_pattern) { ort_status = ort_api->EnableMemPattern(soptions); } else { ort_status = ort_api->DisableMemPattern(soptions); } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Configuring enable_mem_pattern to ") + string_value) .c_str()); THROW_IF_BACKEND_MODEL_ORT_ERROR(ort_status); } } } // FIXME. Is it possible to share a single OrtSession across // multiple instances? If so then should move loading and validation // of the session to here instead of creating a session for each // instance in ModelStateInstance::Create(). } TRITONSERVER_Error* ModelState::LoadModel( const std::string& artifact_name, const TRITONSERVER_InstanceGroupKind instance_group_kind, const int32_t instance_group_device_id, std::string* model_path, OrtSession** session, OrtAllocator** default_allocator, cudaStream_t stream) { // Find the ONNX file that describes the model itself. If the model // configuration doesn't have an explicit model file specified then // use the default name ("model.onnx"). std::string cc_model_filename = artifact_name; if (cc_model_filename.empty()) { cc_model_filename = "model.onnx"; } *model_path = JoinPath( {RepositoryPath(), std::to_string(Version()), cc_model_filename}); // If the model path is a directory then the actual model is //

/model.onnx. { bool is_dir; RETURN_IF_ERROR(IsDirectory(*model_path, &is_dir)); if (is_dir) { *model_path = JoinPath({*model_path, "model.onnx"}); } } { bool exists; RETURN_IF_ERROR(FileExists(*model_path, &exists)); RETURN_ERROR_IF_FALSE( exists, TRITONSERVER_ERROR_UNAVAILABLE, std::string("unable to find '") + *model_path + "' for model instance '" + Name() + "'"); } // Make a clone for the session options for this instance... OrtSessionOptions* soptions; RETURN_IF_ORT_ERROR( ort_api->CloneSessionOptions(session_options_.get(), &soptions)); std::unique_ptr soptions_wrapper( soptions); bool need_lock = false; // Add execution providers if they are requested. // Don't need to ensure uniqueness of the providers, ONNX Runtime // will check it. // GPU execution providers #ifdef TRITON_ENABLE_GPU if ((instance_group_kind == TRITONSERVER_INSTANCEGROUPKIND_GPU) || (instance_group_kind == TRITONSERVER_INSTANCEGROUPKIND_AUTO)) { std::map cuda_options_map; triton::common::TritonJson::Value optimization; if (model_config_.Find("optimization", &optimization)) { triton::common::TritonJson::Value eas; if (optimization.Find("execution_accelerators", &eas)) { triton::common::TritonJson::Value gpu_eas; if (eas.Find("gpu_execution_accelerator", &gpu_eas)) { for (size_t ea_idx = 0; ea_idx < gpu_eas.ArraySize(); ea_idx++) { triton::common::TritonJson::Value ea; RETURN_IF_ERROR(gpu_eas.IndexAsObject(ea_idx, &ea)); std::string name; RETURN_IF_ERROR(ea.MemberAsString("name", &name)); #ifdef TRITON_ENABLE_ONNXRUNTIME_TENSORRT if (name == kTensorRTExecutionAccelerator) { // create tensorrt options with default values OrtTensorRTProviderOptionsV2* trt_options; THROW_IF_BACKEND_MODEL_ORT_ERROR( ort_api->CreateTensorRTProviderOptions(&trt_options)); std::unique_ptr< OrtTensorRTProviderOptionsV2, decltype(ort_api->ReleaseTensorRTProviderOptions)> rel_trt_options( trt_options, ort_api->ReleaseTensorRTProviderOptions); std::string int8_calibration_table_name; std::string trt_engine_cache_path; // Validate and set parameters triton::common::TritonJson::Value params; if (ea.Find("parameters", ¶ms)) { std::vector param_keys, keys, values; RETURN_IF_ERROR(params.Members(¶m_keys)); for (const auto& param_key : param_keys) { std::string value_string, key, value; if (param_key == "precision_mode") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); if (value_string == "FP16") { key = "trt_fp16_enable"; value = "1"; } else if (value_string == "INT8") { key = "trt_int8_enable"; value = "1"; } else if (value_string != "FP32") { RETURN_ERROR_IF_FALSE( false, TRITONSERVER_ERROR_INVALID_ARG, std::string("unsupported precision mode '") + value_string + "' is requested"); } } else if (param_key == "max_workspace_size_bytes") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); size_t max_workspace_size_bytes; RETURN_IF_ERROR(ParseUnsignedLongLongValue( value_string, &max_workspace_size_bytes)); key = "trt_max_workspace_size"; value = value_string; } else if (param_key == "trt_max_partition_iterations") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); int trt_max_partition_iterations; RETURN_IF_ERROR(ParseIntValue( value_string, &trt_max_partition_iterations)); key = "trt_max_partition_iterations"; value = value_string; } else if (param_key == "trt_min_subgraph_size") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); int trt_min_subgraph_size; RETURN_IF_ERROR( ParseIntValue(value_string, &trt_min_subgraph_size)); key = "trt_min_subgraph_size"; value = value_string; } else if (param_key == "int8_calibration_table_name") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_int8_calibration_table_name"; } else if (param_key == "int8_use_native_calibration_table") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool use_native_calibration_table; RETURN_IF_ERROR(ParseBoolValue( value_string, &use_native_calibration_table)); key = "trt_int8_use_native_calibration_table"; value = value_string; } else if (param_key == "trt_dla_enable") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_dla_enable; RETURN_IF_ERROR( ParseBoolValue(value_string, &trt_dla_enable)); key = "trt_dla_enable"; value = value_string; } else if (param_key == "trt_dla_core") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); int trt_dla_core; RETURN_IF_ERROR(ParseIntValue(value_string, &trt_dla_core)); key = "trt_dla_core"; value = value_string; } else if (param_key == "trt_engine_cache_enable") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool enable_cache; RETURN_IF_ERROR( ParseBoolValue(value_string, &enable_cache)); key = "trt_engine_cache_enable"; value = value_string; } else if (param_key == "trt_engine_cache_path") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_engine_cache_path"; } else if (param_key == "trt_engine_cache_prefix") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_engine_cache_prefix"; } else if (param_key == "trt_dump_subgraphs") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool dump_subgraphs; RETURN_IF_ERROR( ParseBoolValue(value_string, &dump_subgraphs)); key = "trt_dump_subgraphs"; value = value_string; } else if (param_key == "trt_force_sequential_engine_build") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_force_sequential_engine_build; RETURN_IF_ERROR(ParseBoolValue( value_string, &trt_force_sequential_engine_build)); key = "trt_force_sequential_engine_build"; value = value_string; } else if (param_key == "trt_context_memory_sharing_enable") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_context_memory_sharing_enable; RETURN_IF_ERROR(ParseBoolValue( value_string, &trt_context_memory_sharing_enable)); key = "trt_context_memory_sharing_enable"; value = value_string; } else if (param_key == "trt_layer_norm_fp32_fallback") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_layer_norm_fp32_fallback; RETURN_IF_ERROR(ParseBoolValue( value_string, &trt_layer_norm_fp32_fallback)); key = "trt_layer_norm_fp32_fallback"; value = value_string; } else if (param_key == "trt_timing_cache_enable") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_timing_cache_enable; RETURN_IF_ERROR( ParseBoolValue(value_string, &trt_timing_cache_enable)); key = "trt_timing_cache_enable"; value = value_string; } else if (param_key == "trt_timing_cache_path") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_timing_cache_path"; } else if (param_key == "trt_force_timing_cache") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_force_timing_cache; RETURN_IF_ERROR( ParseBoolValue(value_string, &trt_force_timing_cache)); key = "trt_force_timing_cache"; value = value_string; } else if (param_key == "trt_detailed_build_log") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_detailed_build_log; RETURN_IF_ERROR( ParseBoolValue(value_string, &trt_detailed_build_log)); key = "trt_detailed_build_log"; value = value_string; } else if (param_key == "trt_build_heuristics_enable") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_build_heuristics_enable; RETURN_IF_ERROR(ParseBoolValue( value_string, &trt_build_heuristics_enable)); key = "trt_build_heuristics_enable"; value = value_string; } else if (param_key == "trt_sparsity_enable") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_sparsity_enable; RETURN_IF_ERROR( ParseBoolValue(value_string, &trt_sparsity_enable)); key = "trt_sparsity_enable"; value = value_string; } else if (param_key == "trt_builder_optimization_level") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); int trt_builder_optimization_level; RETURN_IF_ERROR(ParseIntValue( value_string, &trt_builder_optimization_level)); key = "trt_builder_optimization_level"; value = value_string; } else if (param_key == "trt_auxiliary_streams") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); int trt_auxiliary_streams; RETURN_IF_ERROR( ParseIntValue(value_string, &trt_auxiliary_streams)); key = "trt_auxiliary_streams"; value = value_string; } else if (param_key == "trt_tactic_sources") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_tactic_sources"; } else if (param_key == "trt_extra_plugin_lib_paths") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_extra_plugin_lib_paths"; } else if (param_key == "trt_profile_min_shapes") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_profile_min_shapes"; } else if (param_key == "trt_profile_max_shapes") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_profile_max_shapes"; } else if (param_key == "trt_profile_opt_shapes") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_profile_opt_shapes"; } else if (param_key == "trt_cuda_graph_enable") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_cuda_graph_enable; RETURN_IF_ERROR( ParseBoolValue(value_string, &trt_cuda_graph_enable)); key = "trt_cuda_graph_enable"; value = value_string; } else if (param_key == "trt_dump_ep_context_model") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool trt_dump_ep_context_model; RETURN_IF_ERROR(ParseBoolValue( value_string, &trt_dump_ep_context_model)); key = "trt_dump_ep_context_model"; value = value_string; } else if (param_key == "trt_ep_context_file_path") { RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); key = "trt_ep_context_file_path"; } else if (param_key == "trt_ep_context_embed_mode") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); int trt_ep_context_embed_mode; RETURN_IF_ERROR(ParseIntValue( value_string, &trt_ep_context_embed_mode)); key = "trt_ep_context_embed_mode"; value = value_string; } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unknown parameter '" + param_key + "' is provided for TensorRT Execution " "Accelerator") .c_str()); } if (!key.empty() && !value.empty()) { keys.push_back(key); values.push_back(value); } } std::vector c_keys, c_values; if (!keys.empty() && !values.empty()) { for (size_t i = 0; i < keys.size(); ++i) { c_keys.push_back(keys[i].c_str()); c_values.push_back(values[i].c_str()); } RETURN_IF_ORT_ERROR(ort_api->UpdateTensorRTProviderOptions( rel_trt_options.get(), c_keys.data(), c_values.data(), keys.size())); } } RETURN_IF_ORT_ERROR( ort_api->SessionOptionsAppendExecutionProvider_TensorRT_V2( static_cast(soptions), rel_trt_options.get())); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("TensorRT Execution Accelerator is set for '") + Name() + "' on device " + std::to_string(instance_group_device_id)) .c_str()); continue; } #endif // TRITON_ENABLE_ONNXRUNTIME_TENSORRT if (name == "cuda") { // Parse CUDA EP configurations triton::common::TritonJson::Value params; if (ea.Find("parameters", ¶ms)) { std::vector param_keys; RETURN_IF_ERROR(params.Members(¶m_keys)); for (const auto& param_key : param_keys) { std::string value_string, key, value; // Special handling for boolean values if (param_key == "do_copy_in_default_stream" || param_key == "use_ep_level_unified_stream") { RETURN_IF_ERROR(params.MemberAsString( param_key.c_str(), &value_string)); bool bool_value; RETURN_IF_ERROR(ParseBoolValue(value_string, &bool_value)); key = param_key; value = value_string; } else { key = param_key; RETURN_IF_ERROR( params.MemberAsString(param_key.c_str(), &value)); } if (!key.empty() && !value.empty()) { cuda_options_map[key] = value; } } } } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unknown Execution Accelerator '") + name + "' is requested") .c_str()); } } } } } // Default GPU execution provider. // Using default values for everything other than device id and cuda // stream OrtCUDAProviderOptionsV2* cuda_options; RETURN_IF_ORT_ERROR(ort_api->CreateCUDAProviderOptions(&cuda_options)); std::unique_ptr< OrtCUDAProviderOptionsV2, decltype(ort_api->ReleaseCUDAProviderOptions)> rel_cuda_options(cuda_options, ort_api->ReleaseCUDAProviderOptions); cuda_options_map["device_id"] = std::to_string(instance_group_device_id); cuda_options_map["has_user_compute_stream"] = stream != nullptr ? "1" : "0"; RETURN_IF_ORT_ERROR(ort_api->UpdateCUDAProviderOptionsWithValue( rel_cuda_options.get(), "default_memory_arena_cfg", nullptr)); { // Parse CUDA EP configurations directly from the parameters field. // This is deprecated with adding support for CUDA EP in the // gpu_execution_accelerator field. Keeping this for backward // compatibility. triton::common::TritonJson::Value params; if (model_config_.Find("parameters", ¶ms)) { triton::common::TritonJson::Value json_value; if (params.Find("cudnn_conv_algo_search", &json_value)) { int cudnn_conv_algo_search = 0; RETURN_IF_ERROR(TryParseModelStringParameter( params, "cudnn_conv_algo_search", &cudnn_conv_algo_search, 0)); std::string string_value; switch (cudnn_conv_algo_search) { case 0: string_value = "EXHAUSTIVE"; break; case 1: string_value = "HEURISTIC"; break; case 2: string_value = "DEFAULT"; break; default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unsupported cudnn_conv_algo_search value '") + std::to_string(cudnn_conv_algo_search) + "' is requested") .c_str()); } cuda_options_map["cudnn_conv_algo_search"] = string_value; } else { cuda_options_map["cudnn_conv_algo_search"] = "EXHAUSTIVE"; } if (params.Find("gpu_mem_limit", &json_value)) { std::string string_value; RETURN_IF_ERROR( json_value.MemberAsString("string_value", &string_value)); cuda_options_map["gpu_mem_limit"] = string_value; } else { cuda_options_map["gpu_mem_limit"] = std::to_string(std::numeric_limits::max()); } if (params.Find("arena_extend_strategy", &json_value)) { int arena_extend_strategy = 0; RETURN_IF_ERROR(TryParseModelStringParameter( params, "arena_extend_strategy", &arena_extend_strategy, 0)); std::string string_value; switch (arena_extend_strategy) { case 0: string_value = "kNextPowerOfTwo"; break; case 1: string_value = "kSameAsRequested"; break; default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unsupported arena_extend_strategy value '") + std::to_string(arena_extend_strategy) + "' is requested") .c_str()); } cuda_options_map["arena_extend_strategy"] = string_value; } else { cuda_options_map["arena_extend_strategy"] = "kNextPowerOfTwo"; } if (params.Find("do_copy_in_default_stream", &json_value)) { std::string string_value; RETURN_IF_ERROR( json_value.MemberAsString("string_value", &string_value)); cuda_options_map["do_copy_in_default_stream"] = string_value; } else { cuda_options_map["do_copy_in_default_stream"] = "1"; } } } std::vector option_names, option_values; for (const auto& [key, value] : cuda_options_map) { option_names.push_back(key.c_str()); option_values.push_back(value.c_str()); } RETURN_IF_ORT_ERROR(ort_api->UpdateCUDAProviderOptions( rel_cuda_options.get(), option_names.data(), option_values.data(), option_values.size())); if (stream != nullptr) { RETURN_IF_ORT_ERROR(ort_api->UpdateCUDAProviderOptionsWithValue( rel_cuda_options.get(), "user_compute_stream", stream)); } RETURN_IF_ORT_ERROR(ort_api->SessionOptionsAppendExecutionProvider_CUDA_V2( soptions, cuda_options)); OrtAllocator* allocator; char* options; RETURN_IF_ORT_ERROR(ort_api->GetAllocatorWithDefaultOptions(&allocator)); RETURN_IF_ORT_ERROR(ort_api->GetCUDAProviderOptionsAsString( rel_cuda_options.get(), allocator, &options)); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("CUDA Execution Accelerator is set for '") + Name() + "' on device " + std::to_string(instance_group_device_id) + std::string(" with options: ") + std::string(options)) .c_str()); RETURN_IF_ORT_ERROR(ort_api->AllocatorFree(allocator, options)); } #endif // TRITON_ENABLE_GPU // CPU execution providers { triton::common::TritonJson::Value optimization; if (model_config_.Find("optimization", &optimization)) { triton::common::TritonJson::Value eas; if (optimization.Find("execution_accelerators", &eas)) { triton::common::TritonJson::Value cpu_eas; if (eas.Find("cpu_execution_accelerator", &cpu_eas)) { for (size_t ea_idx = 0; ea_idx < cpu_eas.ArraySize(); ea_idx++) { triton::common::TritonJson::Value ea; RETURN_IF_ERROR(cpu_eas.IndexAsObject(ea_idx, &ea)); std::string name; RETURN_IF_ERROR(ea.MemberAsString("name", &name)); #ifdef TRITON_ENABLE_ONNXRUNTIME_OPENVINO if (name == kOpenVINOExecutionAccelerator) { need_lock = true; OrtOpenVINOProviderOptions openvino_options; openvino_options.device_type = "CPU"; // device_type default is CPU RETURN_IF_ORT_ERROR( ort_api->SessionOptionsAppendExecutionProvider_OpenVINO( soptions, &openvino_options)); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("OpenVINO Execution Accelerator is set for '") + Name() + "' on CPU") .c_str()); continue; } #endif // TRITON_ENABLE_ONNXRUNTIME_OPENVINO return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unknown Execution Accelerator '") + name + "' is requested") .c_str()); } } } } } // Register all op libraries that contain custom operations. { triton::common::TritonJson::Value model_ops; if (model_config_.Find("model_operations", &model_ops)) { triton::common::TritonJson::Value op_library_filenames; if (model_ops.Find("op_library_filename", &op_library_filenames)) { for (size_t op_idx = 0; op_idx < op_library_filenames.ArraySize(); op_idx++) { std::string op_filename; RETURN_IF_ERROR( op_library_filenames.IndexAsString(op_idx, &op_filename)); void* library_handle = nullptr; RETURN_IF_ORT_ERROR(ort_api->RegisterCustomOpsLibrary( soptions, op_filename.c_str(), &library_handle)); } } } } // ONNX session creation with OpenVINO is not thread-safe, // so multiple creations are serialized with a global lock. static std::mutex global_context_mu; std::unique_lock glock(global_context_mu, std::defer_lock); if (need_lock) { glock.lock(); } RETURN_IF_ERROR(OnnxLoader::LoadSession( true /* is_path */, *model_path, soptions, session)); // get default cpu allocator RETURN_IF_ORT_ERROR( ort_api->GetAllocatorWithDefaultOptions(default_allocator)); return nullptr; // success } TRITONSERVER_Error* ModelState::AutoCompleteConfig() { // If the model configuration already specifies inputs and outputs // then don't perform any auto-completion. size_t input_cnt = 0; size_t output_cnt = 0; { triton::common::TritonJson::Value inputs; if (ModelConfig().Find("input", &inputs)) { input_cnt = inputs.ArraySize(); } triton::common::TritonJson::Value config_batch_inputs; if (ModelConfig().Find("batch_input", &config_batch_inputs)) { input_cnt += config_batch_inputs.ArraySize(); } triton::common::TritonJson::Value outputs; if (ModelConfig().Find("output", &outputs)) { output_cnt = outputs.ArraySize(); } } if ((input_cnt > 0) && (output_cnt > 0)) { LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("skipping model configuration auto-complete for '") + Name() + "': inputs and outputs already specified") .c_str()); return nullptr; // success } std::string artifact_name; RETURN_IF_ERROR( ModelConfig().MemberAsString("default_model_filename", &artifact_name)); // Must cleanup 'session'. 'allocator' is default allocator which // is managed by ONNX Runtime so don't need to free/release std::unique_ptr session; OrtAllocator* default_allocator; std::string model_path; { TRITONSERVER_InstanceGroupKind kind = TRITONSERVER_INSTANCEGROUPKIND_CPU; #ifdef TRITON_ENABLE_GPU triton::common::TritonJson::Value instance_group; ModelConfig().Find("instance_group", &instance_group); // Earlier in the model lifecycle, device checks for the instance group // have already occurred. If at least one instance group with // "kind" = "KIND_GPU" then allow model to use GPU else autocomplete to // "KIND_CPU" for (size_t i = 0; i < instance_group.ArraySize(); ++i) { triton::common::TritonJson::Value instance_obj; RETURN_IF_ERROR(instance_group.IndexAsObject(i, &instance_obj)); triton::common::TritonJson::Value instance_group_kind; instance_obj.Find("kind", &instance_group_kind); std::string kind_str; RETURN_IF_ERROR(instance_group_kind.AsString(&kind_str)); if (kind_str == "KIND_GPU") { kind = TRITONSERVER_INSTANCEGROUPKIND_GPU; break; } } #endif // TRITON_ENABLE_GPU OrtSession* sptr = nullptr; RETURN_IF_ERROR(LoadModel( artifact_name, kind, 0, &model_path, &sptr, &default_allocator, nullptr)); session.reset(sptr); } OnnxTensorInfoMap input_tensor_infos; RETURN_IF_ERROR( InputInfos(session.get(), default_allocator, input_tensor_infos)); OnnxTensorInfoMap output_tensor_infos; RETURN_IF_ERROR( OutputInfos(session.get(), default_allocator, output_tensor_infos)); RETURN_IF_ERROR( AutoCompleteMaxBatch(input_tensor_infos, output_tensor_infos)); if (input_cnt == 0) { RETURN_IF_ERROR(AutoCompleteIO("input", input_tensor_infos)); } if (output_cnt == 0) { RETURN_IF_ERROR(AutoCompleteIO("output", output_tensor_infos)); } if (TRITONSERVER_LogIsEnabled(TRITONSERVER_LOG_VERBOSE)) { triton::common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(ModelConfig().PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("post auto-complete:\n") + buffer.Contents()).c_str()); } return nullptr; // success } TRITONSERVER_Error* ModelState::AutoCompleteMaxBatch( const OnnxTensorInfoMap& input_tensor_infos, const OnnxTensorInfoMap& output_tensor_infos) { // Determine if the model can potentially support batching. All // input and output tensors must have a variable first dimension. bool can_support_batching = true; for (const auto& io_info : input_tensor_infos) { const auto& dims = io_info.second.dims_; if ((dims.size() == 0) || (dims[0] != -1)) { can_support_batching = false; } } for (const auto& io_info : output_tensor_infos) { const auto& dims = io_info.second.dims_; if ((dims.size() == 0) || (dims[0] != -1)) { can_support_batching = false; } } // Set max-batch-size to 1 if we have determined that batching is // supported and max-batch-size is not specified. We need to update // the configuration itself as well as the cached value we have already // initialized in the model state. if (can_support_batching) { if (MaxBatchSize() == 0) { int default_max_batch_size = 0; { TRITONBACKEND_Backend* backend; THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_ModelBackend(TritonModel(), &backend)); void* state; THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_BackendState(backend, &state)); default_max_batch_size = reinterpret_cast(state) ->default_max_batch_size_; } int max_batch_size = std::max(default_max_batch_size, 0); triton::common::TritonJson::Value mbs_value; ModelConfig().Find("max_batch_size", &mbs_value); mbs_value.SetInt(max_batch_size); SetMaxBatchSize(max_batch_size); LOG_MESSAGE( TRITONSERVER_LOG_WARN, (std::string( "autofilled max_batch_size to " + std::to_string(max_batch_size) + " for model '") + Name() + "' since batching is supporrted but no max_batch_size is " "specified " "in model configuration. Must specify max_batch_size to utilize " "autofill with a larger max batch size") .c_str()); } // Check to see if we need to turn on dynamic batching // since model supports batching if (MaxBatchSize() > 1) { triton::common::TritonJson::Value value; bool found_sequence_batching = ModelConfig().Find("sequence_batching", &value); bool found_dynamic_batching = ModelConfig().Find("dynamic_batching", &value); if (!found_sequence_batching && !found_dynamic_batching) { triton::common::TritonJson::Value dynamic_batching( ModelConfig(), triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_ERROR( ModelConfig().Add("dynamic_batching", std::move(dynamic_batching))); } } } else if (MaxBatchSize() != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("autofill failed for model '") + Name() + "': model does not support batching while non-zero max_batch_size" " is specified") .c_str()); } return nullptr; // success } TRITONSERVER_Error* ModelState::AutoCompleteIO(const char* key, const OnnxTensorInfoMap& io_infos) { triton::common::TritonJson::Value existing_ios; bool found_ios = ModelConfig().Find(key, &existing_ios); triton::common::TritonJson::Value ios( ModelConfig(), triton::common::TritonJson::ValueType::ARRAY); for (const auto& io_info : io_infos) { triton::common::TritonJson::Value io( ModelConfig(), triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_ERROR(io.AddString("name", io_info.first)); RETURN_IF_ERROR(io.AddString( "data_type", OnnxDataTypeToModelConfigDataType(io_info.second.type_))); // The model signature supports batching then the first dimension // is -1 and should not appear in the model configuration 'dims' // that we are creating. const auto& io_info_dims = io_info.second.dims_; triton::common::TritonJson::Value dims( ModelConfig(), triton::common::TritonJson::ValueType::ARRAY); for (size_t i = (MaxBatchSize() > 0) ? 1 : 0; i < io_info_dims.size(); ++i) { RETURN_IF_ERROR(dims.AppendInt(io_info_dims[i])); } // If dims are empty then must use a reshape... if (dims.ArraySize() == 0) { RETURN_IF_ERROR(dims.AppendInt(1)); triton::common::TritonJson::Value reshape( ModelConfig(), triton::common::TritonJson::ValueType::OBJECT); triton::common::TritonJson::Value reshape_dims( ModelConfig(), triton::common::TritonJson::ValueType::ARRAY); RETURN_IF_ERROR(reshape.Add("shape", std::move(reshape_dims))); // Empty reshape with `max_batch_size` indicates a scalar tensor in the // model configuration which is not a valid model configuration. if (MaxBatchSize() > 0) { RETURN_IF_ERROR(io.Add("reshape", std::move(reshape))); } } RETURN_IF_ERROR(io.Add("dims", std::move(dims))); RETURN_IF_ERROR(ios.Append(std::move(io))); } if (found_ios) { existing_ios.Swap(ios); } else { ModelConfig().Add(key, std::move(ios)); } return nullptr; // success } // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each TRITONBACKEND_ModelInstance. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState(); // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } // Execute... void ProcessRequests( TRITONBACKEND_Request** requests, const uint32_t request_count); private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance); void ReleaseOrtRunResources(); TRITONSERVER_Error* ValidateBooleanSequenceControl( triton::common::TritonJson::Value& sequence_batching, const std::string& control_kind, bool required, bool* have_control); TRITONSERVER_Error* ValidateTypedSequenceControl( triton::common::TritonJson::Value& sequence_batching, const std::string& control_kind, bool required, bool* have_control); TRITONSERVER_Error* ValidateInputs(const size_t expected_input_cnt); TRITONSERVER_Error* ValidateOutputs(); TRITONSERVER_Error* OrtRun( std::vector* responses, const uint32_t response_count); TRITONSERVER_Error* SetInputTensors( size_t total_batch_size, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, BackendInputCollector* collector, std::vector* input_names, bool* cuda_copy); TRITONSERVER_Error* SetStringInputTensor( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, const char* input_name, std::vector* string_ptrs, bool* cuda_copy); void SetStringInputBuffer( const std::string& name, const std::vector& expected_byte_sizes, const std::vector& expected_element_cnts, std::vector* responses, char* input_buffer, std::vector* string_ptrs); void FillStringData(std::vector* string_ptrs, size_t cnt); TRITONSERVER_Error* ReadOutputTensors( size_t total_batch_size, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses); TRITONSERVER_Error* ReadOutputTensor( std::vector& batchn_shape, TRITONSERVER_DataType& dtype, OrtValue* output_tensor, void** output_buffer, std::vector>& string_buffers, std::vector& offsets); bool SetStringOutputBuffer( const std::string& name, const char* content, const size_t* offsets, std::vector* batchn_shape, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses); bool SetStringStateBuffer( const std::string& name, const char* content, const size_t* offsets, std::vector* batchn_shape, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses); bool SetStringBuffer( const std::string& name, const char* content, const size_t* offsets, std::vector* batchn_shape, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, bool state); ModelState* model_state_; // The full path to the ONNX model file. std::string model_path_; // Onnx Runtime variables that are used across runs on this // instance. OrtSession* session_; OrtAllocator* default_allocator_; OrtMemoryInfo* cuda_allocator_info_; const OrtMemoryInfo* cpu_allocator_info_; OrtIoBinding* io_binding_; OrtRunOptions* runOptions_; // map of output name -> bound mem type and id std::unordered_map> output_device_info_; // map of output name -> tensor info OnnxTensorInfoMap output_tensor_infos_; // map of input name -> tensor info OnnxTensorInfoMap input_tensor_infos_; // A map from scalar output tensors to the dimension specified in model config std::unordered_map> scalar_outputs_; // Onnx Runtime variables that will be reset and used for every run // on this instance. std::vector input_tensors_; std::vector output_tensors_; OrtValue** output_buffer_; std::vector input_tensor_memories_; }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelInstanceState::ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state), session_(nullptr), default_allocator_(nullptr), cuda_allocator_info_(nullptr), cpu_allocator_info_(nullptr), io_binding_(nullptr), output_buffer_(nullptr) { THROW_IF_BACKEND_INSTANCE_ERROR(model_state->LoadModel( ArtifactFilename(), Kind(), DeviceId(), &model_path_, &session_, &default_allocator_, CudaStream())); if (Kind() == TRITONSERVER_INSTANCEGROUPKIND_GPU) { THROW_IF_BACKEND_INSTANCE_ORT_ERROR(ort_api->CreateMemoryInfo( "Cuda", OrtAllocatorType::OrtArenaAllocator, DeviceId(), OrtMemTypeDefault, &cuda_allocator_info_)); } THROW_IF_BACKEND_INSTANCE_ORT_ERROR( ort_api->AllocatorGetInfo(default_allocator_, &cpu_allocator_info_)); THROW_IF_BACKEND_INSTANCE_ORT_ERROR( ort_api->CreateIoBinding(session_, &io_binding_)); THROW_IF_BACKEND_INSTANCE_ORT_ERROR(ort_api->CreateRunOptions(&runOptions_)); // Read configs that needs to be set in RunOptions triton::common::TritonJson::Value params; if (model_state->ModelConfig().Find("parameters", ¶ms)) { triton::common::TritonJson::Value json_value; const char* enable_memory_arena_shrinkage_key = "memory.enable_memory_arena_shrinkage"; if (params.Find(enable_memory_arena_shrinkage_key, &json_value)) { std::string string_value; THROW_IF_BACKEND_MODEL_ERROR( json_value.MemberAsString("string_value", &string_value)); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Configuring ") + enable_memory_arena_shrinkage_key + " to " + string_value) .c_str()); THROW_IF_BACKEND_MODEL_ORT_ERROR(ort_api->AddRunConfigEntry( runOptions_, enable_memory_arena_shrinkage_key, string_value.c_str())); } } size_t expected_input_cnt = 0; { triton::common::TritonJson::Value inputs; if (model_state->ModelConfig().Find("input", &inputs)) { expected_input_cnt = inputs.ArraySize(); // Skip the optional inputs which are initializers for (size_t i = 0; i < inputs.ArraySize(); i++) { triton::common::TritonJson::Value input; THROW_IF_BACKEND_INSTANCE_ERROR(inputs.IndexAsObject(i, &input)); bool is_optional; THROW_IF_BACKEND_INSTANCE_ERROR( input.MemberAsBool("optional", &is_optional)); if (is_optional) { expected_input_cnt--; } } } triton::common::TritonJson::Value config_batch_inputs; if (model_state->ModelConfig().Find("batch_input", &config_batch_inputs)) { expected_input_cnt += config_batch_inputs.ArraySize(); } } // If this is a sequence model then make sure that the required // inputs are present in the model and have the correct shape and // datatype. triton::common::TritonJson::Value sequence_batching; if (model_state->ModelConfig().Find( "sequence_batching", &sequence_batching)) { bool have_start, have_end, have_ready, have_corrid; THROW_IF_BACKEND_INSTANCE_ERROR(ValidateBooleanSequenceControl( sequence_batching, "CONTROL_SEQUENCE_START", false /* required */, &have_start)); THROW_IF_BACKEND_INSTANCE_ERROR(ValidateBooleanSequenceControl( sequence_batching, "CONTROL_SEQUENCE_END", false /* required */, &have_end)); THROW_IF_BACKEND_INSTANCE_ERROR(ValidateBooleanSequenceControl( sequence_batching, "CONTROL_SEQUENCE_READY", false /* required */, &have_ready)); THROW_IF_BACKEND_INSTANCE_ERROR(ValidateTypedSequenceControl( sequence_batching, "CONTROL_SEQUENCE_CORRID", false /* required */, &have_corrid)); if (have_start) { expected_input_cnt += 1; } if (have_end) { expected_input_cnt += 1; } if (have_ready) { expected_input_cnt += 1; } if (have_corrid) { expected_input_cnt += 1; } // Add the state inputs to the expected count triton::common::TritonJson::Value states; if (sequence_batching.Find("state", &states)) { expected_input_cnt += states.ArraySize(); } } THROW_IF_BACKEND_INSTANCE_ERROR(ValidateInputs(expected_input_cnt)); THROW_IF_BACKEND_INSTANCE_ERROR(ValidateOutputs()); } ModelInstanceState::~ModelInstanceState() { ReleaseOrtRunResources(); ort_api->ReleaseRunOptions(runOptions_); ort_api->ReleaseIoBinding(io_binding_); ort_api->ReleaseMemoryInfo(cuda_allocator_info_); if (session_ != nullptr) { OnnxLoader::UnloadSession(session_); } // 'default_allocator_' is default allocator which is managed by ONNX // Runtime } void ModelInstanceState::ReleaseOrtRunResources() { ort_api->ClearBoundInputs(io_binding_); for (auto& tensor : input_tensors_) { if (tensor != nullptr) { ort_api->ReleaseValue(tensor); } } input_tensors_.clear(); // first release the Ortvalues ort_api->ClearBoundOutputs(io_binding_); for (auto& tensor : output_tensors_) { if (tensor != nullptr) { ort_api->ReleaseValue(tensor); } } output_tensors_.clear(); // next release the allocated buffer using the specified allocator if (output_buffer_) { auto free_status = ort_api->AllocatorFree(default_allocator_, output_buffer_); output_buffer_ = nullptr; if (free_status != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("onnx runtime allocator free error:") + std::to_string(ort_api->GetErrorCode(free_status)) + ort_api->GetErrorMessage(free_status)) .c_str()); ort_api->ReleaseStatus(free_status); } } for (BackendMemory* mem : input_tensor_memories_) { delete mem; } input_tensor_memories_.clear(); } TRITONSERVER_Error* ModelInstanceState::ValidateBooleanSequenceControl( triton::common::TritonJson::Value& sequence_batching, const std::string& control_kind, bool required, bool* have_control) { std::string tensor_name; std::string tensor_datatype; RETURN_IF_ERROR(GetBooleanSequenceControlProperties( sequence_batching, model_state_->Name(), control_kind, required, &tensor_name, &tensor_datatype, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)); *have_control = !tensor_name.empty(); if (*have_control) { OnnxTensorInfoMap input_tensor_infos; RETURN_IF_ERROR( InputInfos(session_, default_allocator_, input_tensor_infos)); const auto& iit = input_tensor_infos.find(tensor_name); if (iit == input_tensor_infos.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("configuration specified sequence control '") + tensor_name + "', but model does not provide that input") .c_str()); } // Control tensors must have shape [1]. const int nonbatch_start_idx = (model_state_->MaxBatchSize() > 0) ? 1 : 0; std::vector debatched_dims; for (size_t i = nonbatch_start_idx; i < iit->second.dims_.size(); i++) { debatched_dims.push_back(iit->second.dims_[i]); } if ((debatched_dims.size() != 1) || (debatched_dims[0] != 1)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', sequence control '" + tensor_name + "' in model has dims " + ShapeToString(debatched_dims) + " but dims [1] is expected") .c_str()); } if (ModelConfigDataTypeToOnnxDataType(tensor_datatype) != iit->second.type_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', sequence control '" + tensor_name + "', the model expects data-type " + OnnxDataTypeName(iit->second.type_) + " but the model configuration specifies data-type " + tensor_datatype) .c_str()); } } return nullptr; // success } TRITONSERVER_Error* ModelInstanceState::ValidateTypedSequenceControl( triton::common::TritonJson::Value& sequence_batching, const std::string& control_kind, bool required, bool* have_control) { std::string tensor_name; std::string tensor_datatype; RETURN_IF_ERROR(GetTypedSequenceControlProperties( sequence_batching, model_state_->Name(), control_kind, required, &tensor_name, &tensor_datatype)); *have_control = !tensor_name.empty(); if (*have_control) { OnnxTensorInfoMap input_tensor_infos; RETURN_IF_ERROR( InputInfos(session_, default_allocator_, input_tensor_infos)); const auto& iit = input_tensor_infos.find(tensor_name); if (iit == input_tensor_infos.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("configuration specified sequence control '") + tensor_name + "', but model does not provide that input") .c_str()); } // Control tensors must have shape [1]. const int nonbatch_start_idx = (model_state_->MaxBatchSize() > 0) ? 1 : 0; std::vector debatched_dims; for (size_t i = nonbatch_start_idx; i < iit->second.dims_.size(); i++) { debatched_dims.push_back(iit->second.dims_[i]); } if ((debatched_dims.size() != 1) || (debatched_dims[0] != 1)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', sequence control '" + tensor_name + "' in model has dims " + ShapeToString(debatched_dims) + " but dims [1] is expected") .c_str()); } if (ModelConfigDataTypeToOnnxDataType(tensor_datatype) != iit->second.type_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', sequence control '" + tensor_name + "', the model expects data-type " + OnnxDataTypeName(iit->second.type_) + " but the model configuration specifies data-type " + tensor_datatype) .c_str()); } } return nullptr; // success } TRITONSERVER_Error* ModelInstanceState::ValidateInputs(const size_t expected_input_cnt) { std::set input_tensor_names; RETURN_IF_ERROR(InputNames(session_, input_tensor_names)); RETURN_IF_ERROR( InputInfos(session_, default_allocator_, input_tensor_infos_)); std::set overridable_initializer_tensor_names; RETURN_IF_ERROR(OverridableInitializerNames( session_, overridable_initializer_tensor_names)); OnnxTensorInfoMap overridable_initializer_tensor_infos; RETURN_IF_ERROR(OverridableInitializerInfos( session_, default_allocator_, overridable_initializer_tensor_infos)); if (input_tensor_infos_.size() != expected_input_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', configuration expects " + std::to_string(expected_input_cnt) + " inputs, model provides " + std::to_string(input_tensor_infos_.size())) .c_str()); } // Merge 'overridable_xxx' into 'input_xxx' as they can be request inputs, // and all request inputs are checked against 'input_xxx' for (const auto& name : overridable_initializer_tensor_names) { input_tensor_names.emplace(name); } for (const auto& info : overridable_initializer_tensor_infos) { input_tensor_infos_[info.first] = info.second; } triton::common::TritonJson::Value ios; RETURN_IF_ERROR(model_state_->ModelConfig().MemberAsArray("input", &ios)); for (size_t i = 0; i < ios.ArraySize(); i++) { triton::common::TritonJson::Value io; RETURN_IF_ERROR(ios.IndexAsObject(i, &io)); std::string io_name; RETURN_IF_ERROR(io.MemberAsString("name", &io_name)); std::string io_dtype; RETURN_IF_ERROR(io.MemberAsString("data_type", &io_dtype)); bool io_optional; RETURN_IF_ERROR(io.MemberAsBool("optional", &io_optional)); if (io_optional && model_state_->MaxBatchSize() != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', optional input '" + io_name + "' is not supported for models that support batching") .c_str()); } const auto& tensor_names = input_tensor_names; const auto& tensor_infos = input_tensor_infos_; auto iit = tensor_infos.find(io_name); if (iit == tensor_infos.end()) { RETURN_IF_ERROR(CheckAllowedModelInput(io, tensor_names)); } auto onnx_data_type = ModelConfigDataTypeToOnnxDataType(io_dtype); if (onnx_data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("unsupported datatype ") + io_dtype + " for input '" + io_name + "' for model '" + model_state_->Name() + "'") .c_str()); } else if (onnx_data_type != iit->second.type_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', configuration expects datatype " + io_dtype + " for input '" + io_name + "', model provides TYPE_" + TRITONSERVER_DataTypeString( ConvertFromOnnxDataType(iit->second.type_))) .c_str()); } // If a reshape is provided for the input then use that when // validating that the model matches what is expected. std::vector dims; triton::common::TritonJson::Value reshape; if (io.Find("reshape", &reshape)) { RETURN_IF_ERROR(ParseShape(reshape, "shape", &dims)); } else { RETURN_IF_ERROR(ParseShape(io, "dims", &dims)); } triton::common::TritonJson::Value allow_ragged_batch_json; bool allow_ragged_batch = false; if (io.Find("allow_ragged_batch", &allow_ragged_batch_json)) { RETURN_IF_ERROR(allow_ragged_batch_json.AsBool(&allow_ragged_batch)); } if (io_optional && allow_ragged_batch) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', configuration expects model provides input with shape [-1] " "for ragged input '" + io_name + "', which is not supported for optional input") .c_str()); } if (allow_ragged_batch) { const std::vector& model_shape = iit->second.dims_; // Make sure the input has shape [-1] if ((model_shape.size() != 1) || (model_shape[0] != WILDCARD_DIM)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', configuration expects model provides input with shape [-1] " "for ragged input '" + io_name + "', model provides " + ShapeToString(model_shape)) .c_str()); } } else { // Only compare the dimensions if the tensor is not scalar if (iit->second.dims_.size() != 0) { RETURN_IF_ERROR(CompareDimsSupported( model_state_->Name(), io_name, iit->second.dims_, dims, model_state_->MaxBatchSize(), false /* compare_exact */)); } else { // if max_batch_size == 0 and is a scalar tensor all the // dimensions specified must be equal to 1 for (auto& dim : dims) { if (dim != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', scalar tensor '" + io_name + "', should only provide 1 in the model configuration when the " "model doesn't support batching. Model configuration " "provided: " + ShapeToString(dims) + ".") .c_str()); } } } } } return nullptr; // success } TRITONSERVER_Error* ModelInstanceState::ValidateOutputs() { std::set output_tensor_names; RETURN_IF_ERROR(OutputNames(session_, output_tensor_names)); RETURN_IF_ERROR( OutputInfos(session_, default_allocator_, output_tensor_infos_)); triton::common::TritonJson::Value ios; RETURN_IF_ERROR(model_state_->ModelConfig().MemberAsArray("output", &ios)); for (size_t i = 0; i < ios.ArraySize(); i++) { triton::common::TritonJson::Value io; RETURN_IF_ERROR(ios.IndexAsObject(i, &io)); std::string io_name; RETURN_IF_ERROR(io.MemberAsString("name", &io_name)); std::string io_dtype; RETURN_IF_ERROR(io.MemberAsString("data_type", &io_dtype)); auto iit = output_tensor_infos_.find(io_name); if (iit == output_tensor_infos_.end()) { RETURN_IF_ERROR(CheckAllowedModelOutput(io, output_tensor_names)); } auto onnx_data_type = ModelConfigDataTypeToOnnxDataType(io_dtype); if (onnx_data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("unsupported datatype ") + io_dtype + " for output '" + io_name + "' for model '" + model_state_->Name() + "'") .c_str()); } else if (onnx_data_type != iit->second.type_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', configuration expects datatype " + io_dtype + " for output '" + io_name + "', model provides TYPE_" + TRITONSERVER_DataTypeString( ConvertFromOnnxDataType(iit->second.type_))) .c_str()); } // If a reshape is provided for the input then use that when // validating that the model matches what is expected. std::vector dims; triton::common::TritonJson::Value reshape; if (io.Find("reshape", &reshape)) { RETURN_IF_ERROR(ParseShape(reshape, "shape", &dims)); } else { RETURN_IF_ERROR(ParseShape(io, "dims", &dims)); } // The batch output shape doesn't necessarily match the model if (model_state_->FindBatchOutput(io_name) == nullptr) { // Only compare the dimensions if the tensor is not scalar if (iit->second.dims_.size() != 0) { RETURN_IF_ERROR(CompareDimsSupported( model_state_->Name(), io_name, iit->second.dims_, dims, model_state_->MaxBatchSize(), true /* compare_exact */)); } else { for (auto& dim : dims) { if (dim != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', scalar tensor '" + io_name + "', should only provide 1 in the model configuration when the " "model doesn't support batching. Model configuration " "provided: " + ShapeToString(dims) + ".") .c_str()); } } // store the dimension for reference. scalar_outputs_[io_name] = dims; } } } triton::common::TritonJson::Value sequence_batching; if (model_state_->ModelConfig().Find( "sequence_batching", &sequence_batching)) { triton::common::TritonJson::Value states; if (sequence_batching.MemberAsArray("state", &states)) { for (size_t i = 0; i < states.ArraySize(); i++) { triton::common::TritonJson::Value state; RETURN_IF_ERROR(states.IndexAsObject(i, &state)); std::string state_name; RETURN_IF_ERROR(state.MemberAsString("name", &state_name)); std::string state_dtype; RETURN_IF_ERROR(state.MemberAsString("data_type", &state_dtype)); std::vector dims; RETURN_IF_ERROR(ParseShape(state, "dims", &dims)); auto iit = output_tensor_infos_.find(state_name); if (iit == output_tensor_infos_.end()) { RETURN_IF_ERROR(CheckAllowedModelOutput(state, output_tensor_names)); } auto onnx_data_type = ModelConfigDataTypeToOnnxDataType(state_dtype); if (onnx_data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("unsupported datatype ") + state_dtype + " for output state '" + state_name + "' for model '" + model_state_->Name() + "'") .c_str()); } else if (onnx_data_type != iit->second.type_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unable to load model '") + model_state_->Name() + "', configuration expects datatype " + state_dtype + " for output state '" + state_name + "', model provides TYPE_" + TRITONSERVER_DataTypeString( ConvertFromOnnxDataType(iit->second.type_))) .c_str()); } } } } return nullptr; // success } void ModelInstanceState::ProcessRequests( TRITONBACKEND_Request** requests, const uint32_t request_count) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("TRITONBACKEND_ModelExecute: Running ") + Name() + " with " + std::to_string(request_count) + " requests") .c_str()); uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); const int max_batch_size = model_state_->MaxBatchSize(); // For each request collect the total batch size for this inference // execution. The batch-size, number of inputs, and size of each // input has already been checked so don't need to do that here. size_t total_batch_size = 0; for (size_t i = 0; i < request_count; i++) { // If we get a nullptr request then something is badly wrong. Fail // and release all requests. if (requests[i] == nullptr) { RequestsRespondWithError( requests, request_count, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "null request given to ONNX Runtime backend for '" + Name() + "'") .c_str())); return; } if (max_batch_size > 0) { // Retrieve the batch size from one of the inputs, if the model // supports batching, the first dimension size is batch size TRITONBACKEND_Input* input; TRITONSERVER_Error* err = TRITONBACKEND_RequestInputByIndex(requests[i], 0 /* index */, &input); if (err == nullptr) { const int64_t* shape; err = TRITONBACKEND_InputProperties( input, nullptr, nullptr, &shape, nullptr, nullptr, nullptr); total_batch_size += shape[0]; } if (err != nullptr) { RequestsRespondWithError(requests, request_count, err); return; } } else { total_batch_size += 1; } } // If there are no valid payloads then no need to run the inference. if (total_batch_size == 0) { return; } // Make sure the maximum batch size is not exceeded. The // total_batch_size must be 1 for models that don't support batching // (i.e. max_batch_size == 0). If max_batch_size is exceeded then // scheduler has done something badly wrong so fail and release all // requests. if ((total_batch_size != 1) && (total_batch_size > (size_t)max_batch_size)) { RequestsRespondWithError( requests, request_count, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "batch size " + std::to_string(total_batch_size) + " for '" + Name() + "', max allowed is " + std::to_string(max_batch_size)) .c_str())); return; } // At this point we are committed to running inference with all // 'requests'. Create a response for each request. During input // processing if there is an error with any request that error will // be sent immediately with the corresponding response (and the // response unique_ptr will then be nullptr). The request object // itself will not be released until after all inferencing is done // (below) as we may need to access the request object when // determine how to process outputs (for example, even if we don't // need the outputs for a request that has an error, we do need to // know the size of those outputs associated with the request so we // can skip them in the output tensors). std::vector responses; responses.reserve(request_count); bool all_response_failed = false; for (size_t i = 0; i < request_count; i++) { TRITONBACKEND_Response* response; auto err = TRITONBACKEND_ResponseNew(&response, requests[i]); if (err == nullptr) { responses.emplace_back(response); } else { responses.emplace_back(nullptr); LOG_MESSAGE(TRITONSERVER_LOG_ERROR, "Fail to create response"); TRITONSERVER_ErrorDelete(err); } } // Use scoped class to clean up ORT tensors and other resources that // need to persist until ORT run completes. struct ScopedCleanup { ScopedCleanup(ModelInstanceState* ctx) : ctx_(ctx) {} ~ScopedCleanup() { if (ctx_ != nullptr) { ctx_->ReleaseOrtRunResources(); } } ModelInstanceState* ctx_; } io_tensor_wrapper(this); std::vector input_names; bool cuda_copy = false; BackendInputCollector collector( requests, request_count, &responses, model_state_->TritonMemoryManager(), model_state_->EnablePinnedInput(), CudaStream(), nullptr, nullptr, 0, HostPolicyName().c_str()); RESPOND_ALL_AND_SET_TRUE_IF_ERROR( responses, request_count, all_response_failed, SetInputTensors( total_batch_size, requests, request_count, &responses, &collector, &input_names, &cuda_copy)); if (!all_response_failed) { // Set preferred memory type and id. This will be used while querying // memory type to be used for output buffer. TRITONSERVER_MemoryType preferred_memory_type = TRITONSERVER_MEMORY_CPU; int64_t preferred_memory_type_id = 0; if (Kind() == TRITONSERVER_INSTANCEGROUPKIND_GPU) { preferred_memory_type = TRITONSERVER_MEMORY_GPU; preferred_memory_type_id = DeviceId(); } // Request to retrieve all model outputs. 'output_names' and // 'output_tensors_' are parallel vectors and so must be kept in // sync. [TODO] should collect only the outputs needed by some // request. for (auto& output_name : StateForModel()->ModelOutputs()) { output_tensors_.emplace_back(nullptr); TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; int64_t memory_type_id = 0; // Get data type for this output. If this is a string then // use CPU for binding output otherwise, query the preferred location // for this output and bind accordingly. In case of any errors we // fallback to binding the output to CPU. auto iit = output_tensor_infos_.find(output_name.first); if (iit == output_tensor_infos_.end()) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string( "Error while retrieving output data type. Using cpu " "as preferred location for output: " + output_name.first) .c_str())); } else if (iit->second.type_ != ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) { // Query the memory type of destination output buffer. Bind the // output to this destination memory type. The destination memory type // for an output for all requests should be same. So use any request // for this query. memory_type = preferred_memory_type; memory_type_id = preferred_memory_type_id; auto err = TRITONBACKEND_RequestOutputBufferProperties( requests[0], output_name.first.c_str(), /*byte_size*/ nullptr, &memory_type, &memory_type_id); if (err != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string( "Output Properties Unavailable. Using cpu as " "preferred location for output: " + output_name.first + " Error: " + TRITONSERVER_ErrorMessage(err)) .c_str())); memory_type = TRITONSERVER_MEMORY_CPU; memory_type_id = 0; } } // If the cuda allocator is not set, bind the output to CPU. if (cuda_allocator_info_ == nullptr) { memory_type = TRITONSERVER_MEMORY_CPU; memory_type_id = 0; } // finally save the derived mem type and device id as we need it for // reading the outputs. output_device_info_[output_name.first] = {memory_type, memory_type_id}; RESPOND_ALL_AND_SET_TRUE_IF_ORT_ERROR( responses, request_count, all_response_failed, ort_api->BindOutputToDevice( io_binding_, output_name.first.c_str(), memory_type == TRITONSERVER_MEMORY_GPU ? cuda_allocator_info_ : cpu_allocator_info_)); } } // Wait for any in-flight input tensor copies to complete. #ifdef TRITON_ENABLE_GPU if (cuda_copy) { cudaStreamSynchronize(CudaStream()); } #endif uint64_t compute_start_ns = 0; SET_TIMESTAMP(compute_start_ns); if (!all_response_failed) { RESPOND_ALL_AND_SET_TRUE_IF_ERROR( responses, request_count, all_response_failed, OrtRun(&responses, request_count)); } uint64_t compute_end_ns = 0; SET_TIMESTAMP(compute_end_ns); if (!all_response_failed) { RESPOND_ALL_AND_SET_TRUE_IF_ERROR( responses, request_count, all_response_failed, ReadOutputTensors( total_batch_size, requests, request_count, &responses)); } uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); // Send all the responses that haven't already been sent because of // an earlier error. Note that the responses are not set to nullptr // here as we need that indication below to determine if the request // we successful or not. for (auto& response : responses) { if (response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr), "failed to send onnxruntime backend response"); } } // Report statistics for each request. for (uint32_t r = 0; r < request_count; ++r) { auto& request = requests[r]; LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting request statistics"); LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } if (!all_response_failed) { // Report the entire batch statistics. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( TritonModelInstance(), total_batch_size, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting batch request statistics"); } } TRITONSERVER_Error* ModelInstanceState::OrtRun( std::vector* responses, const uint32_t response_count) { RETURN_IF_ORT_ERROR( ort_api->RunWithBinding(session_, runOptions_, io_binding_)); return nullptr; } TRITONSERVER_Error* ModelInstanceState::SetInputTensors( size_t total_batch_size, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, BackendInputCollector* collector, std::vector* input_names, bool* cuda_copy) { const int max_batch_size = model_state_->MaxBatchSize(); // All requests must have equally-sized input tensors so use any // request as the representative for the input tensors. uint32_t input_count; RETURN_IF_ERROR(TRITONBACKEND_RequestInputCount(requests[0], &input_count)); for (uint32_t input_idx = 0; input_idx < input_count; input_idx++) { TRITONBACKEND_Input* input; RETURN_IF_ERROR( TRITONBACKEND_RequestInputByIndex(requests[0], input_idx, &input)); const char* input_name; TRITONSERVER_DataType input_datatype; const int64_t* input_shape; uint32_t input_dims_count; RETURN_IF_ERROR(TRITONBACKEND_InputProperties( input, &input_name, &input_datatype, &input_shape, &input_dims_count, nullptr, nullptr)); input_names->emplace_back(input_name); input_tensors_.emplace_back(nullptr); std::vector batchn_shape; // For a ragged input tensor, the tensor shape should be // the flatten shape of the whole batch if (StateForModel()->IsInputRagged(input_name)) { batchn_shape = std::vector{0}; for (size_t idx = 0; idx < request_count; idx++) { TRITONBACKEND_Input* input; RESPOND_AND_SET_NULL_IF_ERROR( &((*responses)[idx]), TRITONBACKEND_RequestInput(requests[idx], input_name, &input)); const int64_t* input_shape; uint32_t input_dims_count; RESPOND_AND_SET_NULL_IF_ERROR( &((*responses)[idx]), TRITONBACKEND_InputProperties( input, nullptr, nullptr, &input_shape, &input_dims_count, nullptr, nullptr)); batchn_shape[0] += GetElementCount(input_shape, input_dims_count); } } // The shape for the entire input batch, [total_batch_size, ...] else { batchn_shape = std::vector(input_shape, input_shape + input_dims_count); if (max_batch_size != 0) { batchn_shape[0] = total_batch_size; } } if (input_datatype != TRITONSERVER_TYPE_BYTES) { // The input must be in contiguous CPU memory. Use appropriate // allocator info to bind inputs to the right device. .i.e bind inputs // to GPU if they are being provided on GPU. const char* input_buffer; size_t batchn_byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; std::vector> allowed_input_types; if (Kind() == TRITONSERVER_INSTANCEGROUPKIND_GPU) { allowed_input_types = { {TRITONSERVER_MEMORY_GPU, DeviceId()}, {TRITONSERVER_MEMORY_CPU_PINNED, 0}, {TRITONSERVER_MEMORY_CPU, 0}}; } else { allowed_input_types = { {TRITONSERVER_MEMORY_CPU_PINNED, 0}, {TRITONSERVER_MEMORY_CPU, 0}}; } RETURN_IF_ERROR(collector->ProcessTensor( input_name, nullptr, 0, allowed_input_types, &input_buffer, &batchn_byte_size, &memory_type, &memory_type_id)); auto iti = input_tensor_infos_.find(input_name); if (iti == input_tensor_infos_.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( std::string( "Failed to retrieve the ONNX input tensor info from '") + input_name + "'.") .c_str()); } // Create ORT Tensor if (iti->second.dims_.size() == 0) { // scalar tensor RETURN_IF_ORT_ERROR(ort_api->CreateTensorWithDataAsOrtValue( memory_type == TRITONSERVER_MEMORY_GPU ? cuda_allocator_info_ : cpu_allocator_info_, (void*)input_buffer, batchn_byte_size, nullptr /* scalar */, 0 /* number of dims */, ConvertToOnnxDataType(input_datatype), &input_tensors_.back())); } else { RETURN_IF_ORT_ERROR(ort_api->CreateTensorWithDataAsOrtValue( memory_type == TRITONSERVER_MEMORY_GPU ? cuda_allocator_info_ : cpu_allocator_info_, (void*)input_buffer, batchn_byte_size, batchn_shape.data(), batchn_shape.size(), ConvertToOnnxDataType(input_datatype), &input_tensors_.back())); } RETURN_IF_ORT_ERROR( ort_api->BindInput(io_binding_, input_name, input_tensors_.back())); } else { // For BYTES input, we need to convert the serialized string // representation into what is required for ORT. ORT expects a // vector of char*, one for each element. For each tensor we get // a copy of the data in a contiguous CPU buffer and then // in-place modify that from the Triton // ... serialization into a // ... serialization // and then initialize 'string_ptrs' to point to each . std::vector string_ptrs; SetStringInputTensor( requests, request_count, responses, input_name, &string_ptrs, cuda_copy); RETURN_IF_ORT_ERROR(ort_api->CreateTensorAsOrtValue( default_allocator_, batchn_shape.data(), batchn_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, &input_tensors_.back())); RETURN_IF_ORT_ERROR(ort_api->FillStringTensor( input_tensors_.back(), string_ptrs.data(), string_ptrs.size())); RETURN_IF_ORT_ERROR( ort_api->BindInput(io_binding_, input_name, input_tensors_.back())); } } // Process batch input if any for (const auto& batch_input : StateForModel()->BatchInputs()) { std::vector shape; collector->BatchInputShape(batch_input, &shape); for (const auto& input_name : batch_input.TargetNames()) { input_names->emplace_back(input_name.c_str()); input_tensors_.emplace_back(nullptr); const char* dst_buffer; size_t dst_buffer_byte_size; TRITONSERVER_MemoryType dst_memory_type; int64_t dst_memory_type_id; // Batch inputs are always created on CPU RESPOND_ALL_AND_SET_NULL_IF_ERROR( (*responses), responses->size(), collector->ProcessBatchInput( batch_input, nullptr, 0, {{TRITONSERVER_MEMORY_CPU, 0}}, &dst_buffer, &dst_buffer_byte_size, &dst_memory_type, &dst_memory_type_id)); // Create ORT Tensor RETURN_IF_ORT_ERROR(ort_api->CreateTensorWithDataAsOrtValue( cpu_allocator_info_, (void*)dst_buffer, dst_buffer_byte_size, shape.data(), shape.size(), ConvertToOnnxDataType(batch_input.DataType()), &input_tensors_.back())); RETURN_IF_ORT_ERROR(ort_api->BindInput( io_binding_, input_name.c_str(), input_tensors_.back())); } } // Finalize... *cuda_copy |= collector->Finalize(); return nullptr; } TRITONSERVER_Error* ModelInstanceState::SetStringInputTensor( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, const char* input_name, std::vector* string_ptrs, bool* cuda_copy) { size_t total_byte_size = 0; std::vector expected_byte_sizes; std::vector expected_element_cnts; expected_byte_sizes.reserve(request_count); expected_element_cnts.reserve(request_count); for (size_t ridx = 0; ridx < request_count; ++ridx) { TRITONBACKEND_Input* in; RESPOND_AND_SET_NULL_IF_ERROR( &((*responses)[ridx]), TRITONBACKEND_RequestInput(requests[ridx], input_name, &in)); const int64_t* input_shape; uint32_t input_dims_count; uint64_t input_byte_size; RETURN_IF_ERROR(TRITONBACKEND_InputProperties( in, nullptr, nullptr, &input_shape, &input_dims_count, &input_byte_size, nullptr)); // Skip input in this request if error response has already been sent. if ((*responses)[ridx] == nullptr) { expected_byte_sizes.push_back(0); expected_element_cnts.push_back(0); } else { expected_element_cnts.push_back( GetElementCount(input_shape, input_dims_count)); expected_byte_sizes.push_back(input_byte_size); } total_byte_size += expected_byte_sizes.back(); } // For string input, the copy to contiguous buffer is needed because ORT // expects elements to be C strings thus we need to modify input buffer. // Reserve one more byte at the end of input_buffer to ensure last // element of String data can become valid C string. BackendMemory* input_memory; RETURN_IF_ERROR(BackendMemory::Create( model_state_->TritonMemoryManager(), {BackendMemory::AllocationType::CPU_PINNED_POOL, BackendMemory::AllocationType::CPU}, 0 /* memory_type_id */, total_byte_size + 1, &input_memory)); input_tensor_memories_.push_back(input_memory); const TRITONSERVER_MemoryType mem_type = input_memory->MemoryType(); char* input_buffer = input_memory->MemoryPtr(); size_t buffer_offset = 0; for (size_t ridx = 0; ridx < request_count; ++ridx) { TRITONBACKEND_Input* in; TRITONSERVER_Error* err = TRITONBACKEND_RequestInput(requests[ridx], input_name, &in); if ((err == nullptr) && ((*responses)[ridx] != nullptr)) { uint32_t input_buffer_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( in, HostPolicyName().c_str(), nullptr, nullptr, nullptr, nullptr, nullptr, &input_buffer_count)); size_t input_offset = 0; for (size_t idx = 0; idx < input_buffer_count; ++idx) { const void* src_buffer; size_t src_byte_size; TRITONSERVER_MemoryType src_memory_type; int64_t src_memory_type_id; err = TRITONBACKEND_InputBufferForHostPolicy( in, HostPolicyName().c_str(), idx, &src_buffer, &src_byte_size, &src_memory_type, &src_memory_type_id); if (err == nullptr) { if ((input_offset + src_byte_size) > expected_byte_sizes[ridx]) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("buffer size for input '") + input_name + "' exceeds batch byte size " + std::to_string(expected_byte_sizes[ridx])) .c_str()); } else { bool cuda_used = false; err = CopyBuffer( input_name, src_memory_type, src_memory_type_id, mem_type, 0, src_byte_size, src_buffer, input_buffer + buffer_offset + input_offset, CudaStream(), &cuda_used); *cuda_copy |= cuda_used; } } if (err == nullptr) { input_offset += src_byte_size; } else { break; } } } if (err != nullptr) { if ((*responses)[ridx] != nullptr) { RESPOND_AND_SET_NULL_IF_ERROR(&((*responses)[ridx]), err); } TRITONSERVER_ErrorDelete(err); } buffer_offset += expected_byte_sizes[ridx]; } #ifdef TRITON_ENABLE_GPU // Synchronize to ensure the buffer is ready to be modified if (*cuda_copy) { cudaStreamSynchronize(CudaStream()); *cuda_copy = false; } #endif // TRITON_ENABLE_GPU // Modify input buffer and set string expected by ORT SetStringInputBuffer( input_name, expected_byte_sizes, expected_element_cnts, responses, input_buffer, string_ptrs); input_buffer[total_byte_size] = 0; return nullptr; } void ModelInstanceState::SetStringInputBuffer( const std::string& input_name, const std::vector& expected_byte_sizes, const std::vector& expected_element_cnts, std::vector* responses, char* input_buffer, std::vector* string_ptrs) { std::vector> str_list; // offset for each response size_t buffer_copy_offset = 0; for (size_t idx = 0; idx < expected_byte_sizes.size(); idx++) { const size_t expected_byte_size = expected_byte_sizes[idx]; const size_t expected_element_cnt = expected_element_cnts[idx]; if ((*responses)[idx] != nullptr) { char* data_content = input_buffer + buffer_copy_offset; TRITONSERVER_Error* err = ValidateStringBuffer( data_content, expected_byte_size, expected_element_cnt, input_name.c_str(), &str_list); // Set string values. for (const auto& [addr, len] : str_list) { // Make first byte of size info 0, so that if there is string data // in front of it, the data becomes valid C string. *const_cast(addr - sizeof(uint32_t)) = 0; string_ptrs->push_back(addr); } size_t element_cnt = str_list.size(); if (err != nullptr) { RESPOND_AND_SET_NULL_IF_ERROR(&((*responses)[idx]), err); FillStringData(string_ptrs, expected_element_cnt - element_cnt); } str_list.clear(); } buffer_copy_offset += expected_byte_size; } } void ModelInstanceState::FillStringData( std::vector* string_ptrs, size_t cnt) { static const char* empty = ""; for (size_t c = 0; c < cnt; c++) { string_ptrs->push_back(empty); } } TRITONSERVER_Error* ModelInstanceState::ReadOutputTensor( std::vector& batchn_shape, TRITONSERVER_DataType& dtype, OrtValue* output_tensor, void** output_buffer, std::vector>& string_buffers, std::vector& offsets) { // Get output type and shape OrtTypeInfo* typeinfo; RETURN_IF_ORT_ERROR(ort_api->GetTypeInfo(output_tensor, &typeinfo)); std::unique_ptr typeinfo_wrapper(typeinfo); const OrtTensorTypeAndShapeInfo* type_and_shape; RETURN_IF_ORT_ERROR( ort_api->CastTypeInfoToTensorInfo(typeinfo, &type_and_shape)); size_t num_dims; RETURN_IF_ORT_ERROR(ort_api->GetDimensionsCount(type_and_shape, &num_dims)); batchn_shape.resize(num_dims); RETURN_IF_ORT_ERROR(ort_api->GetDimensions( type_and_shape, batchn_shape.data(), batchn_shape.size())); ONNXTensorElementDataType type; RETURN_IF_ORT_ERROR(ort_api->GetTensorElementType(type_and_shape, &type)); if (type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) { const size_t element_count = GetElementCount(batchn_shape); size_t total_length = 0; RETURN_IF_ORT_ERROR( ort_api->GetStringTensorDataLength(output_tensor, &total_length)); string_buffers.emplace_back(std::vector(total_length)); auto content = string_buffers.back().data(); offsets.reserve(element_count + 1); RETURN_IF_ORT_ERROR(ort_api->GetStringTensorContent( output_tensor, content, total_length, offsets.data(), element_count)); // Mark "passed end byte offset" offsets[element_count] = total_length; } else { // Fixed size data type... RETURN_IF_ORT_ERROR( ort_api->GetTensorMutableData(output_tensor, output_buffer)); } dtype = ConvertFromOnnxDataType(type); return nullptr; // success } TRITONSERVER_Error* ModelInstanceState::ReadOutputTensors( size_t total_batch_size, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses) { BackendOutputResponder responder( requests, request_count, responses, model_state_->TritonMemoryManager(), model_state_->MaxBatchSize() > 0, model_state_->EnablePinnedInput(), CudaStream()); // Use to hold string output contents bool cuda_copy = false; auto& model_outputs = StateForModel()->ModelOutputs(); size_t output_count = 0; RETURN_IF_ORT_ERROR(ort_api->GetBoundOutputValues( io_binding_, default_allocator_, &output_buffer_, &output_count)); if (output_count != model_outputs.size()) { RETURN_IF_ERROR(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, ("Retrieved output count is not equal to expected count."))); } std::vector> string_buffers; auto model_outputs_it = model_outputs.begin(); for (size_t idx = 0; idx < model_outputs.size(); idx++, model_outputs_it++) { OrtValue* output_tensor = output_tensors_[idx] = output_buffer_[idx]; const std::string& name = model_outputs_it->first; auto& output_tensor_pair = model_outputs_it->second; auto output_device_info_iter = output_device_info_.find(name); if (output_device_info_iter == output_device_info_.end()) { RETURN_IF_ERROR(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("device info for output tensor '") + name + "' not found") .c_str())); } const auto& alloc_perference = output_device_info_iter->second; const BatchOutput* batch_output = StateForModel()->FindBatchOutput(name); if (batch_output == nullptr) { if (output_tensor == nullptr) { RETURN_IF_ERROR(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("output tensor '") + name + "' is not found") .c_str())); } std::vector batchn_shape; TRITONSERVER_DataType dtype; void* output_buffer; std::vector> string_buffers; std::vector offsets; RETURN_IF_ERROR(ReadOutputTensor( batchn_shape, dtype, output_tensor, &output_buffer, string_buffers, offsets)); // If the number of dimensions is equal to zero, it means that it is a // scalar and it would use the dimensions specified in the model // configuration. if (batchn_shape.size() == 0) { auto scalar_output_dims_it = scalar_outputs_.find(name); if (scalar_output_dims_it == scalar_outputs_.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "Failed to find the scalar output dimension for " + name + " in the model configuration.") .c_str()); } batchn_shape = scalar_output_dims_it->second; } if (output_tensor_pair.first != -1) { if (dtype == TRITONSERVER_TYPE_BYTES) { auto content = string_buffers.back().data(); cuda_copy |= SetStringOutputBuffer( name, content, offsets.data(), &batchn_shape, requests, request_count, responses); } else { responder.ProcessTensor( name, dtype, batchn_shape, reinterpret_cast(output_buffer), alloc_perference.first, alloc_perference.second); } } if (output_tensor_pair.second != -1) { std::vector states; if (dtype == TRITONSERVER_TYPE_BYTES) { auto content = string_buffers.back().data(); cuda_copy |= SetStringStateBuffer( name, content, offsets.data(), &batchn_shape, requests, request_count, responses); } else { states = responder.ProcessStateTensor( name, dtype, batchn_shape, reinterpret_cast(output_buffer), alloc_perference.first, alloc_perference.second); } // Update the states for (auto& state : states) { RETURN_IF_ERROR(TRITONBACKEND_StateUpdate(state)); } } } else { char* output_buffer = nullptr; RETURN_IF_ORT_ERROR( ort_api->GetTensorMutableData(output_tensor, (void**)&output_buffer)); responder.ProcessBatchOutput( name, *batch_output, output_buffer, alloc_perference.first, alloc_perference.second); } } // Finalize and wait for any pending buffer copies. cuda_copy |= responder.Finalize(); #ifdef TRITON_ENABLE_GPU if (cuda_copy) { cudaStreamSynchronize(stream_); } #endif // TRITON_ENABLE_GPU return nullptr; } bool ModelInstanceState::SetStringStateBuffer( const std::string& name, const char* content, const size_t* offsets, std::vector* batchn_shape, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses) { return SetStringBuffer( name, content, offsets, batchn_shape, requests, request_count, responses, true /* state */); } bool ModelInstanceState::SetStringOutputBuffer( const std::string& name, const char* content, const size_t* offsets, std::vector* batchn_shape, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses) { return SetStringBuffer( name, content, offsets, batchn_shape, requests, request_count, responses, false /* state */); } bool ModelInstanceState::SetStringBuffer( const std::string& name, const char* content, const size_t* offsets, std::vector* batchn_shape, TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector* responses, bool state) { size_t element_idx = 0; bool cuda_copy = false; for (size_t ridx = 0; ridx < request_count; ++ridx) { const auto& request = requests[ridx]; auto& response = (*responses)[ridx]; // batchn_shape holds the shape of the entire tensor batch. When // batching is enabled override the first batch dimension with each // requests batch size (reusing for efficiency). if (model_state_->MaxBatchSize() > 0) { TRITONBACKEND_Input* input; TRITONBACKEND_RequestInputByIndex(request, 0 /* index */, &input); const int64_t* shape; TRITONBACKEND_InputProperties( input, nullptr, nullptr, &shape, nullptr, nullptr, nullptr); (*batchn_shape)[0] = shape[0]; } const size_t expected_element_cnt = GetElementCount(*batchn_shape); // If 'request' requested this output then copy it from // 'content'. If it did not request this output then just skip it // in the 'content'. bool need_output = false; if (!state) { if (response != nullptr) { uint32_t output_count; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestOutputCount(request, &output_count)); if (response != nullptr) { for (uint32_t output_idx = 0; output_idx < output_count; output_idx++) { const char* req_output_name; RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_RequestOutputName( request, output_idx, &req_output_name)); if ((response != nullptr) && (req_output_name == name)) { need_output = true; break; } } } } } else { // need_output must be always set to true for state tensors. need_output = true; } if (need_output) { TRITONSERVER_Error* err; TRITONBACKEND_Output* response_output; TRITONBACKEND_State* response_state; if (!state) { err = TRITONBACKEND_ResponseOutput( response, &response_output, name.c_str(), TRITONSERVER_TYPE_BYTES, batchn_shape->data(), batchn_shape->size()); } else { err = TRITONBACKEND_StateNew( &response_state, request, name.c_str(), TRITONSERVER_TYPE_BYTES, batchn_shape->data(), batchn_shape->size()); } if (err == nullptr) { // Calculate expected byte size in advance using string offsets const size_t data_byte_size = offsets[element_idx + expected_element_cnt] - offsets[element_idx]; const size_t expected_byte_size = data_byte_size + sizeof(uint32_t) * expected_element_cnt; TRITONSERVER_MemoryType actual_memory_type = TRITONSERVER_MEMORY_CPU_PINNED; int64_t actual_memory_type_id = 0; void* buffer; if (!state) { err = TRITONBACKEND_OutputBuffer( response_output, &buffer, expected_byte_size, &actual_memory_type, &actual_memory_type_id); } else { err = TRITONBACKEND_StateBuffer( response_state, &buffer, expected_byte_size, &actual_memory_type, &actual_memory_type_id); } if (err == nullptr) { bool cuda_used = false; size_t copied_byte_size = 0; for (size_t e = 0; e < expected_element_cnt; ++e) { const uint32_t len = offsets[element_idx + e + 1] - offsets[element_idx + e]; // Prepend size of the string err = CopyBuffer( name, TRITONSERVER_MEMORY_CPU /* src_memory_type */, 0 /* src_memory_type_id */, actual_memory_type, actual_memory_type_id, sizeof(uint32_t), static_cast(&len), static_cast(buffer) + copied_byte_size, stream_, &cuda_used); if (err != nullptr) { break; } cuda_copy |= cuda_used; copied_byte_size += sizeof(uint32_t); // Copy raw string content err = CopyBuffer( name, TRITONSERVER_MEMORY_CPU /* src_memory_type */, 0 /* src_memory_type_id */, actual_memory_type, actual_memory_type_id, len, content + offsets[element_idx + e], static_cast(buffer) + copied_byte_size, stream_, &cuda_used); if (err != nullptr) { break; } cuda_copy |= cuda_used; copied_byte_size += len; } } } RESPOND_AND_SET_NULL_IF_ERROR(&response, err); if (state) { RESPOND_AND_SET_NULL_IF_ERROR( &response, TRITONBACKEND_StateUpdate(response_state)); } } element_idx += expected_element_cnt; } return cuda_copy; } ///////////// extern "C" { TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); // Check the backend API version that Triton supports vs. what this // backend was compiled against. uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Triton TRITONBACKEND API version: ") + std::to_string(api_version_major) + "." + std::to_string(api_version_minor)) .c_str()); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("'") + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, (std::string("Triton TRITONBACKEND API version: ") + std::to_string(api_version_major) + "." + std::to_string(api_version_minor) + " does not support '" + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); } // The backend configuration may contain information needed by the // ort backend, such as command-line arguments. TRITONSERVER_Message* backend_config_message; RETURN_IF_ERROR( TRITONBACKEND_BackendConfig(backend, &backend_config_message)); const char* buffer; size_t byte_size; RETURN_IF_ERROR(TRITONSERVER_MessageSerializeToJson( backend_config_message, &buffer, &byte_size)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("backend configuration:\n") + buffer).c_str()); triton::common::TritonJson::Value backend_config; TRITONSERVER_Error* err = nullptr; if (byte_size != 0) { err = backend_config.Parse(buffer, byte_size); } RETURN_IF_ERROR(err); // Onetime initialization for the onnxruntime loader. RETURN_IF_ERROR(OnnxLoader::Init(backend_config)); std::unique_ptr lconfig(new BackendConfiguration()); triton::common::TritonJson::Value cmdline; if (backend_config.Find("cmdline", &cmdline)) { triton::common::TritonJson::Value value; std::string value_str; if (cmdline.Find("default-max-batch-size", &value)) { RETURN_IF_ERROR(value.AsString(&value_str)); int lvalue; RETURN_IF_ERROR(ParseIntValue(value_str, &lvalue)); lconfig->default_max_batch_size_ = lvalue; } } // Check if device memory tracker is explicitly enabled if (DeviceMemoryTracker::EnableFromBackendConfig(backend_config)) { lconfig->enable_memory_tracker_ = DeviceMemoryTracker::Init(); } RETURN_IF_ERROR(TRITONBACKEND_BackendSetState( backend, reinterpret_cast(lconfig.get()))); lconfig.release(); return nullptr; // success } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Finalize(TRITONBACKEND_Backend* backend) { if (BackendConfiguration::RetrieveFrom(backend).enable_memory_tracker_) { DeviceMemoryTracker::Fini(); } LOG_IF_ERROR(OnnxLoader::Stop(), "failed to stop OnnxLoader"); void* state = nullptr; LOG_IF_ERROR( TRITONBACKEND_BackendState(backend, &state), "failed to get backend state"); if (state != nullptr) { delete reinterpret_cast(state); } return nullptr; // success } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); // Utilizing DeviceMemoryTracker behavior that function calls with // 'nullptr' for usage will be no-ops. std::unique_ptr lusage; if (BackendConfiguration::RetrieveFrom(model).enable_memory_tracker_) { lusage.reset(new DeviceMemoryTracker::MemoryUsage()); DeviceMemoryTracker::TrackThreadMemoryUsage(lusage.get()); } // Create a ModelState object and associate it with the // TRITONBACKEND_Model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast(model_state))); if (lusage) { DeviceMemoryTracker::UntrackThreadMemoryUsage(lusage.get()); TRITONSERVER_BufferAttributes** ba_array; uint32_t ba_len = 0; RETURN_IF_ERROR(lusage->SerializeToBufferAttributes(&ba_array, &ba_len)); RETURN_IF_ERROR( TRITONBACKEND_ModelReportMemoryUsage(model, ba_array, ba_len)); } return nullptr; // success } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return nullptr; // success } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { // NOTE: If the corresponding TRITONBACKEND_BackendAttribute is enabled by the // backend for parallel model instance loading, the // TRITONBACKEND_ModelInstanceInitialize may be called concurrently. // Therefore, this function should be thread-safe. const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); // Get the model state associated with this instance's model. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast(vmodelstate); // Utilizing DeviceMemoryTracker behavior that function calls with // 'nullptr' for usage will be no-ops. std::unique_ptr lusage; if (BackendConfiguration::RetrieveFrom(instance).enable_memory_tracker_) { lusage.reset(new DeviceMemoryTracker::MemoryUsage()); DeviceMemoryTracker::TrackThreadMemoryUsage(lusage.get()); } // Create a ModelInstanceState object and associate it with the // TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast(instance_state))); if (lusage) { DeviceMemoryTracker::UntrackThreadMemoryUsage(lusage.get()); TRITONSERVER_BufferAttributes** ba_array; uint32_t ba_len = 0; RETURN_IF_ERROR(lusage->SerializeToBufferAttributes(&ba_array, &ba_len)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceReportMemoryUsage( instance, ba_array, ba_len)); } return nullptr; // success } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; // success } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Suggested practice for this is to use only // function-local and model-instance-specific state (obtained from // 'instance'), which is what we do here. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast(&instance_state))); ModelState* model_state = instance_state->StateForModel(); // This backend specifies BLOCKING execution policy. That means that // we should not return from this function until execution is // complete. Triton will automatically release 'instance' on return // from this function so that it is again available to be used for // another call to TRITONBACKEND_ModelInstanceExecute. LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("model ") + model_state->Name() + ", instance " + instance_state->Name() + ", executing " + std::to_string(request_count) + " requests") .c_str()); // At this point we accept ownership of 'requests', which means that // even if something goes wrong we must still return success from // this function. If something does go wrong in processing a // particular request then we send an error response just for the // specific request. instance_state->ProcessRequests(requests, request_count); return nullptr; // success } TRITONSERVER_Error* TRITONBACKEND_GetBackendAttribute( TRITONBACKEND_Backend* backend, TRITONBACKEND_BackendAttribute* backend_attributes) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, "TRITONBACKEND_GetBackendAttribute: setting attributes"); // This backend can safely handle parallel calls to // TRITONBACKEND_ModelInstanceInitialize (thread-safe). RETURN_IF_ERROR(TRITONBACKEND_BackendAttributeSetParallelModelInstanceLoading( backend_attributes, true)); return nullptr; } } // extern "C" }}} // namespace triton::backend::onnxruntime kytensor-2.49.0.7/framework_backends/onnxruntime_backend/src/onnxruntime_loader.cc000066400000000000000000000170411527052371200305360ustar00rootroot00000000000000// Copyright (c) 2018-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "onnxruntime_loader.h" #include #include #include #include #include #include "onnxruntime_utils.h" namespace triton { namespace backend { namespace onnxruntime { std::unique_ptr OnnxLoader::loader = nullptr; OnnxLoader::~OnnxLoader() { if (env_ != nullptr) { ort_api->ReleaseEnv(env_); } } TRITONSERVER_Error* OnnxLoader::Init(common::TritonJson::Value& backend_config) { if (loader == nullptr) { OrtEnv* env; // If needed, provide custom logger with // ort_api->CreateEnvWithCustomLogger() OrtStatus* status; OrtLoggingLevel logging_level = TRITONSERVER_LogIsEnabled(TRITONSERVER_LOG_VERBOSE) ? ORT_LOGGING_LEVEL_VERBOSE : TRITONSERVER_LogIsEnabled(TRITONSERVER_LOG_WARN) ? ORT_LOGGING_LEVEL_WARNING : ORT_LOGGING_LEVEL_ERROR; // Controls whether to enable global threadpool which will be shared across // sessions. Use this in conjunction with DisablePerSessionThreads API or // else the session will use it's own thread pool. bool global_threadpool_enabled = false; OrtThreadingOptions* threading_options = nullptr; // Read backend config triton::common::TritonJson::Value cmdline; if (backend_config.Find("cmdline", &cmdline)) { triton::common::TritonJson::Value value; std::string value_str; if (cmdline.Find("enable-global-threadpool", &value)) { RETURN_IF_ERROR(value.AsString(&value_str)); RETURN_IF_ERROR(ParseBoolValue(value_str, &global_threadpool_enabled)); if (global_threadpool_enabled) { // If provided by user, read intra and inter op num thread // configuration and set ThreadingOptions accordingly. If not, we use // default 0 which means value equal to number of cores will be used. RETURN_IF_ORT_ERROR( ort_api->CreateThreadingOptions(&threading_options)); if (cmdline.Find("intra_op_thread_count", &value)) { int intra_op_num_threads = 0; RETURN_IF_ERROR(value.AsString(&value_str)); RETURN_IF_ERROR(ParseIntValue(value_str, &intra_op_num_threads)); if (intra_op_num_threads > 0) { RETURN_IF_ORT_ERROR(ort_api->SetGlobalIntraOpNumThreads( threading_options, intra_op_num_threads)); } } if (cmdline.Find("inter_op_thread_count", &value)) { int inter_op_num_threads = 0; RETURN_IF_ERROR(value.AsString(&value_str)); RETURN_IF_ERROR(ParseIntValue(value_str, &inter_op_num_threads)); if (inter_op_num_threads > 0) { RETURN_IF_ORT_ERROR(ort_api->SetGlobalInterOpNumThreads( threading_options, inter_op_num_threads)); } } } } } if (global_threadpool_enabled && threading_options != nullptr) { status = ort_api->CreateEnvWithGlobalThreadPools( logging_level, "log", threading_options, &env); ort_api->ReleaseThreadingOptions(threading_options); } else { status = ort_api->CreateEnv(logging_level, "log", &env); } loader.reset(new OnnxLoader(env, global_threadpool_enabled)); RETURN_IF_ORT_ERROR(status); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_ALREADY_EXISTS, "OnnxLoader singleton already initialized"); } return nullptr; // success } void OnnxLoader::TryRelease(bool decrement_session_cnt) { std::unique_ptr lloader; { std::lock_guard lk(loader->mu_); if (decrement_session_cnt) { loader->live_session_cnt_--; } if (loader->closing_ && (loader->live_session_cnt_ == 0)) { lloader.swap(loader); } } } TRITONSERVER_Error* OnnxLoader::Stop() { if (loader != nullptr) { loader->closing_ = true; TryRelease(false); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "OnnxLoader singleton has not been initialized"); } return nullptr; // success } bool OnnxLoader::IsGlobalThreadPoolEnabled() { if (loader != nullptr) { return loader->global_threadpool_enabled_; } return false; } TRITONSERVER_Error* OnnxLoader::LoadSession( const bool is_path, const std::string& model, const OrtSessionOptions* session_options, OrtSession** session) { #ifdef _WIN32 std::wstring_convert> converter; std::wstring ort_style_model_str = converter.from_bytes(model); #else const auto& ort_style_model_str = model; #endif if (loader != nullptr) { { std::lock_guard lk(loader->mu_); if (loader->closing_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "OnnxLoader has been stopped"); } else { loader->live_session_cnt_++; } } OrtStatus* status = nullptr; { // [FIXME] Remove lock when ORT create session is thread safe [DLIS-4663] static std::mutex ort_create_session_mu; std::lock_guard ort_lk(ort_create_session_mu); if (!is_path) { status = ort_api->CreateSessionFromArray( loader->env_, ort_style_model_str.c_str(), model.size(), session_options, session); } else { status = ort_api->CreateSession( loader->env_, ort_style_model_str.c_str(), session_options, session); } } if (status != nullptr) { TryRelease(true); } RETURN_IF_ORT_ERROR(status); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "OnnxLoader singleton has not been initialized"); } return nullptr; // success } TRITONSERVER_Error* OnnxLoader::UnloadSession(OrtSession* session) { if (loader != nullptr) { ort_api->ReleaseSession(session); TryRelease(true); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "OnnxLoader singleton has not been initialized"); } return nullptr; // success } }}} // namespace triton::backend::onnxruntime kytensor-2.49.0.7/framework_backends/onnxruntime_backend/src/onnxruntime_loader.h000066400000000000000000000074631527052371200304070ustar00rootroot00000000000000// Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include "triton/backend/backend_common.h" #include "triton/core/tritonbackend.h" namespace triton { namespace backend { namespace onnxruntime { /// A singleton to load Onnx model because loading models requires /// Onnx Runtime environment which is unique per process class OnnxLoader { public: ~OnnxLoader(); /// Initialize loader with default environment settings static TRITONSERVER_Error* Init(common::TritonJson::Value& backend_config); /// Stop loader, and once all Onnx sessions are unloaded via UnloadSession() /// the resource it allocated will be released static TRITONSERVER_Error* Stop(); /// Load a Onnx model from a path and return the corresponding /// OrtSession. /// /// \param bool is_path If true 'model' is a path to the model file, /// if false 'model' is the serialized model. /// \param model The Onnx model or path to the model. /// \param session_options The options to use when creating the session /// \param session Returns the Onnx model session /// \return Error status. static TRITONSERVER_Error* LoadSession( const bool is_path, const std::string& model, const OrtSessionOptions* session_options, OrtSession** session); /// Unload a Onnx model session /// /// \param session The Onnx model session to be unloaded static TRITONSERVER_Error* UnloadSession(OrtSession* session); /// Returns whether global thread pool is enabled. /// If the loader is not initialized it returns false. static bool IsGlobalThreadPoolEnabled(); private: OnnxLoader(OrtEnv* env, bool enable_global_threadpool = false) : env_(env), global_threadpool_enabled_(enable_global_threadpool), live_session_cnt_(0), closing_(false) { } /// Decrease 'live_session_cnt_' if 'decrement_session_cnt' is true, and then /// release Onnx Runtime environment if it is closing and no live sessions /// /// \param decrement_session_cnt Whether to decrease the 'live_session_cnt_' static void TryRelease(bool decrement_session_cnt); static std::unique_ptr loader; OrtEnv* env_; bool global_threadpool_enabled_; std::mutex mu_; size_t live_session_cnt_; bool closing_; }; }}} // namespace triton::backend::onnxruntime kytensor-2.49.0.7/framework_backends/onnxruntime_backend/src/onnxruntime_utils.cc000066400000000000000000000500121527052371200304230ustar00rootroot00000000000000// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "onnxruntime_utils.h" namespace triton { namespace backend { namespace onnxruntime { const OrtApi* ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION); namespace { std::string OnnxTypeName(ONNXType onnx_type) { switch (onnx_type) { case ONNX_TYPE_TENSOR: return "ONNX_TYPE_TENSOR"; case ONNX_TYPE_SEQUENCE: return "ONNX_TYPE_SEQUENCE"; case ONNX_TYPE_MAP: return "ONNX_TYPE_MAP"; case ONNX_TYPE_OPAQUE: return "ONNX_TYPE_OPAQUE"; case ONNX_TYPE_SPARSETENSOR: return "ONNX_TYPE_SPARSETENSOR"; case ONNX_TYPE_UNKNOWN: default: break; } return "ONNX_TYPE_UNKNOWN"; } enum class NameType { INPUT, OUTPUT, INITIALIZER, }; TRITONSERVER_Error* InputOutputInitializerNames( OrtSession* session, NameType type, std::set& names) { names.clear(); size_t num_nodes; switch (type) { case NameType::INPUT: RETURN_IF_ORT_ERROR(ort_api->SessionGetInputCount(session, &num_nodes)); break; case NameType::OUTPUT: RETURN_IF_ORT_ERROR(ort_api->SessionGetOutputCount(session, &num_nodes)); break; case NameType::INITIALIZER: RETURN_IF_ORT_ERROR( ort_api->SessionGetOverridableInitializerCount(session, &num_nodes)); break; } // iterate over all input / output nodes OrtAllocator* allocator; RETURN_IF_ORT_ERROR(ort_api->GetAllocatorWithDefaultOptions(&allocator)); OrtStatus* onnx_status = nullptr; for (size_t i = 0; i < num_nodes; i++) { char* node_name = nullptr; switch (type) { case NameType::INPUT: onnx_status = ort_api->SessionGetInputName(session, i, allocator, &node_name); break; case NameType::OUTPUT: onnx_status = ort_api->SessionGetOutputName(session, i, allocator, &node_name); break; case NameType::INITIALIZER: onnx_status = ort_api->SessionGetOverridableInitializerName( session, i, allocator, &node_name); break; } // Make a std::string copy of the name and then free 'node_name' // since the ORT API makes us responsible for doing that. std::string name(node_name); auto free_status = ort_api->AllocatorFree(allocator, node_name); if (free_status != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("onnx runtime allocator free error:") + std::to_string(ort_api->GetErrorCode(free_status)) + ort_api->GetErrorMessage(free_status)) .c_str()); ort_api->ReleaseStatus(free_status); } if (onnx_status != nullptr) { break; } names.emplace(std::move(name)); } RETURN_IF_ORT_ERROR(onnx_status); return nullptr; // success } TRITONSERVER_Error* InputOutputInitializerInfos( OrtSession* session, OrtAllocator* allocator, NameType type, OnnxTensorInfoMap& infos) { infos.clear(); size_t num_nodes; switch (type) { case NameType::INPUT: RETURN_IF_ORT_ERROR(ort_api->SessionGetInputCount(session, &num_nodes)); break; case NameType::OUTPUT: RETURN_IF_ORT_ERROR(ort_api->SessionGetOutputCount(session, &num_nodes)); break; case NameType::INITIALIZER: RETURN_IF_ORT_ERROR( ort_api->SessionGetOverridableInitializerCount(session, &num_nodes)); break; } // iterate over all nodes for (size_t i = 0; i < num_nodes; i++) { char* cname = nullptr; switch (type) { case NameType::INPUT: RETURN_IF_ORT_ERROR( ort_api->SessionGetInputName(session, i, allocator, &cname)); break; case NameType::OUTPUT: RETURN_IF_ORT_ERROR( ort_api->SessionGetOutputName(session, i, allocator, &cname)); break; case NameType::INITIALIZER: RETURN_IF_ORT_ERROR(ort_api->SessionGetOverridableInitializerName( session, i, allocator, &cname)); break; } // Make a std::string copy of the name and then free 'cname' since // the ORT API makes us responsible for doing that. std::string name(cname); auto free_status = ort_api->AllocatorFree(allocator, cname); if (free_status != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("onnx runtime allocator free error:") + std::to_string(ort_api->GetErrorCode(free_status)) + ort_api->GetErrorMessage(free_status)) .c_str()); ort_api->ReleaseStatus(free_status); } OrtTypeInfo* typeinfo; switch (type) { case NameType::INPUT: RETURN_IF_ORT_ERROR( ort_api->SessionGetInputTypeInfo(session, i, &typeinfo)); break; case NameType::OUTPUT: RETURN_IF_ORT_ERROR( ort_api->SessionGetOutputTypeInfo(session, i, &typeinfo)); break; case NameType::INITIALIZER: RETURN_IF_ORT_ERROR(ort_api->SessionGetOverridableInitializerTypeInfo( session, i, &typeinfo)); break; } std::unique_ptr typeinfo_wrapper(typeinfo); ONNXType onnx_type; RETURN_IF_ORT_ERROR(ort_api->GetOnnxTypeFromTypeInfo(typeinfo, &onnx_type)); RETURN_ERROR_IF_TRUE( onnx_type != ONNX_TYPE_TENSOR, TRITONSERVER_ERROR_UNSUPPORTED, std::string("Unsupported ONNX Type '") + OnnxTypeName(onnx_type) + "' for I/O '" + name + "', expected '" + OnnxTypeName(ONNX_TYPE_TENSOR) + "'."); const OrtTensorTypeAndShapeInfo* tensor_info; RETURN_IF_ORT_ERROR( ort_api->CastTypeInfoToTensorInfo(typeinfo, &tensor_info)); ONNXTensorElementDataType type; RETURN_IF_ORT_ERROR(ort_api->GetTensorElementType(tensor_info, &type)); size_t num_dims; RETURN_IF_ORT_ERROR(ort_api->GetDimensionsCount(tensor_info, &num_dims)); std::vector dims(num_dims); RETURN_IF_ORT_ERROR( ort_api->GetDimensions(tensor_info, (int64_t*)dims.data(), num_dims)); infos.emplace(std::move(name), OnnxTensorInfo(type, dims)); } return nullptr; // success } } // namespace int GetCpuCoreCount() { // Method 1 // Use the combination of physical_package_id and core_id // to get the cpu physical core from the topology. std::unordered_set physical_cores; for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) { std::ifstream physical_id_file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/topology/physical_package_id"); std::ifstream core_id_file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/topology/core_id"); if (!physical_id_file.is_open() || !core_id_file.is_open()) { break; // no more cpus } std::string physical_id, core_id; if (std::getline(physical_id_file, physical_id) && std::getline(core_id_file, core_id)) { std::string core_identifier = physical_id + ":" + core_id; physical_cores.insert(core_identifier); } } if (!physical_cores.empty()) { return static_cast(physical_cores.size()); } // Method 2 // If the Method 1 fails, it will be an extra guarantee // Get physical cores through thread_siblings std::unordered_set siblings; for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) { std::ifstream thread_siblings("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/topology/thread_siblings"); if (!thread_siblings.is_open()) { break; // no more cpus } std::string line; if (std::getline(thread_siblings, line)) { siblings.insert(line); } } if (!siblings.empty()) { return static_cast(siblings.size()); } return 0; } std::string OnnxDataTypeName(ONNXTensorElementDataType onnx_type) { switch (onnx_type) { case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return "FLOAT"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return "UINT8"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return "INT8"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: return "UINT16"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: return "INT16"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: return "INT32"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return "INT64"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: return "STRING"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return "BOOL"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return "FLOAT16"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: return "DOUBLE"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: return "UINT32"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: return "UINT64"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: return "COMPLEX64"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: return "COMPLEX64"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: return "BFLOAT16"; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED: default: break; } return "UNDEFINED"; } TRITONSERVER_DataType ConvertFromOnnxDataType(ONNXTensorElementDataType onnx_type) { switch (onnx_type) { case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: // maps to c type float (4 bytes) return TRITONSERVER_TYPE_FP32; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return TRITONSERVER_TYPE_UINT8; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return TRITONSERVER_TYPE_INT8; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: return TRITONSERVER_TYPE_UINT16; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: return TRITONSERVER_TYPE_INT16; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: return TRITONSERVER_TYPE_INT32; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return TRITONSERVER_TYPE_INT64; case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: return TRITONSERVER_TYPE_BYTES; case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return TRITONSERVER_TYPE_BOOL; case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return TRITONSERVER_TYPE_FP16; case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: // maps to c type double (8 bytes) return TRITONSERVER_TYPE_FP64; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: return TRITONSERVER_TYPE_UINT32; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: return TRITONSERVER_TYPE_UINT64; // The following types are not supported: // complex with float32 real and imaginary components case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: // complex with float64 real and imaginary components case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: // Non-IEEE floating-point format based on IEEE754 single-precision case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: case ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED: default: break; } return TRITONSERVER_TYPE_INVALID; } ONNXTensorElementDataType ConvertToOnnxDataType(TRITONSERVER_DataType data_type) { switch (data_type) { case TRITONSERVER_TYPE_UINT8: return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; case TRITONSERVER_TYPE_UINT16: return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; case TRITONSERVER_TYPE_UINT32: return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; case TRITONSERVER_TYPE_UINT64: return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; case TRITONSERVER_TYPE_INT8: return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; case TRITONSERVER_TYPE_INT16: return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; case TRITONSERVER_TYPE_INT32: return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; case TRITONSERVER_TYPE_INT64: return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; case TRITONSERVER_TYPE_FP16: return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; case TRITONSERVER_TYPE_FP32: return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; case TRITONSERVER_TYPE_FP64: return ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; case TRITONSERVER_TYPE_BYTES: return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING; case TRITONSERVER_TYPE_BOOL: return ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; default: break; } return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; } ONNXTensorElementDataType ConvertToOnnxDataType(const std::string& data_type_str) { TRITONSERVER_DataType data_type = TRITONSERVER_StringToDataType(data_type_str.c_str()); return ConvertToOnnxDataType(data_type); } ONNXTensorElementDataType ModelConfigDataTypeToOnnxDataType(const std::string& data_type_str) { // Must start with "TYPE_". if (data_type_str.rfind("TYPE_", 0) != 0) { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; } const std::string dtype = data_type_str.substr(strlen("TYPE_")); if (dtype == "BOOL") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; } else if (dtype == "UINT8") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; } else if (dtype == "UINT16") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; } else if (dtype == "UINT32") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; } else if (dtype == "UINT64") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; } else if (dtype == "INT8") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; } else if (dtype == "INT16") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; } else if (dtype == "INT32") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; } else if (dtype == "INT64") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; } else if (dtype == "FP16") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; } else if (dtype == "FP32") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; } else if (dtype == "FP64") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; } else if (dtype == "STRING") { return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING; } return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; } std::string OnnxDataTypeToModelConfigDataType(ONNXTensorElementDataType data_type) { if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL) { return "TYPE_BOOL"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8) { return "TYPE_UINT8"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16) { return "TYPE_UINT16"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32) { return "TYPE_UINT32"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64) { return "TYPE_UINT64"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) { return "TYPE_INT8"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16) { return "TYPE_INT16"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) { return "TYPE_INT32"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) { return "TYPE_INT64"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16) { return "TYPE_FP16"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { return "TYPE_FP32"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE) { return "TYPE_FP64"; } else if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) { return "TYPE_STRING"; } return "TYPE_INVALID"; } TRITONSERVER_Error* OverridableInitializerNames(OrtSession* session, std::set& names) { return InputOutputInitializerNames(session, NameType::INITIALIZER, names); } TRITONSERVER_Error* InputNames(OrtSession* session, std::set& names) { return InputOutputInitializerNames(session, NameType::INPUT, names); } TRITONSERVER_Error* OutputNames(OrtSession* session, std::set& names) { return InputOutputInitializerNames(session, NameType::OUTPUT, names); } TRITONSERVER_Error* InputInfos( OrtSession* session, OrtAllocator* allocator, OnnxTensorInfoMap& infos) { return InputOutputInitializerInfos( session, allocator, NameType::INPUT, infos); } TRITONSERVER_Error* OutputInfos( OrtSession* session, OrtAllocator* allocator, OnnxTensorInfoMap& infos) { return InputOutputInitializerInfos( session, allocator, NameType::OUTPUT, infos); } TRITONSERVER_Error* OverridableInitializerInfos( OrtSession* session, OrtAllocator* allocator, OnnxTensorInfoMap& infos) { return InputOutputInitializerInfos( session, allocator, NameType::INITIALIZER, infos); } TRITONSERVER_Error* CompareDimsSupported( const std::string& model_name, const std::string& tensor_name, const std::vector& model_shape, const std::vector& dims, const int max_batch_size, const bool compare_exact) { // If the model configuration expects batching support in the model, // then the onnx shape first dimension must be -1. const bool supports_batching = (max_batch_size > 0); if (supports_batching) { RETURN_ERROR_IF_TRUE( (model_shape.size() == 0) || (model_shape[0] != -1), TRITONSERVER_ERROR_INVALID_ARG, std::string("model '") + model_name + "', tensor '" + tensor_name + "': for the model to support batching the shape should have at " "least 1 dimension and the first dimension must be -1; but shape " "expected by the model is " + ShapeToString(model_shape)); std::vector full_dims; full_dims.reserve(1 + dims.size()); full_dims.push_back(-1); full_dims.insert(full_dims.end(), dims.begin(), dims.end()); bool succ = (model_shape.size() == (size_t)full_dims.size()); if (succ) { for (size_t i = 0; i < full_dims.size(); ++i) { const int64_t model_dim = model_shape[i]; if (compare_exact || (model_dim != -1)) { succ &= (model_dim == full_dims[i]); } } } RETURN_ERROR_IF_TRUE( !succ, TRITONSERVER_ERROR_INVALID_ARG, std::string("model '") + model_name + "', tensor '" + tensor_name + "': the model expects " + std::to_string(model_shape.size()) + " dimensions (shape " + ShapeToString(model_shape) + ") but the model configuration specifies " + std::to_string(full_dims.size()) + " dimensions (an initial batch dimension because max_batch_size " "> 0 followed by the explicit tensor shape, making complete " "shape " + ShapeToString(full_dims) + ")"); } else { // ! supports_batching bool succ = (model_shape.size() == dims.size()); if (succ) { for (size_t i = 0; i < dims.size(); ++i) { const int64_t model_dim = model_shape[i]; if (compare_exact || (model_dim != -1)) { succ &= (model_dim == dims[i]); } } } RETURN_ERROR_IF_TRUE( !succ, TRITONSERVER_ERROR_INVALID_ARG, std::string("model '") + model_name + "', tensor '" + tensor_name + "': the model expects " + std::to_string(model_shape.size()) + " dimensions (shape " + ShapeToString(model_shape) + ") but the model configuration specifies " + std::to_string(dims.size()) + " dimensions (shape " + ShapeToString(dims) + ")"); } return nullptr; // success } }}} // namespace triton::backend::onnxruntime kytensor-2.49.0.7/framework_backends/onnxruntime_backend/src/onnxruntime_utils.h000066400000000000000000000177151527052371200303020ustar00rootroot00000000000000// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include "triton/backend/backend_common.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { namespace onnxruntime { extern const OrtApi* ort_api; #define RESPOND_ALL_AND_SET_TRUE_IF_ORT_ERROR( \ RESPONSES, RESPONSES_COUNT, BOOL, S) \ do { \ OrtStatus* status__ = (S); \ if (status__ != nullptr) { \ OrtErrorCode code = ort_api->GetErrorCode(status__); \ std::string msg = std::string(ort_api->GetErrorMessage(status__)); \ ort_api->ReleaseStatus(status__); \ auto err__ = TRITONSERVER_ErrorNew( \ TRITONSERVER_ERROR_INTERNAL, (std::string("onnx runtime error ") + \ std::to_string(code) + ": " + msg) \ .c_str()); \ RESPOND_ALL_AND_SET_TRUE_IF_ERROR( \ RESPONSES, RESPONSES_COUNT, BOOL, err__); \ } \ } while (false) #define RETURN_IF_ORT_ERROR(S) \ do { \ OrtStatus* status__ = (S); \ if (status__ != nullptr) { \ OrtErrorCode code = ort_api->GetErrorCode(status__); \ std::string msg = std::string(ort_api->GetErrorMessage(status__)); \ ort_api->ReleaseStatus(status__); \ return TRITONSERVER_ErrorNew( \ TRITONSERVER_ERROR_INTERNAL, (std::string("onnx runtime error ") + \ std::to_string(code) + ": " + msg) \ .c_str()); \ } \ } while (false) #define THROW_IF_BACKEND_MODEL_ORT_ERROR(S) \ do { \ OrtStatus* status__ = (S); \ if (status__ != nullptr) { \ OrtErrorCode code = ort_api->GetErrorCode(status__); \ std::string msg = std::string(ort_api->GetErrorMessage(status__)); \ ort_api->ReleaseStatus(status__); \ throw BackendModelException(TRITONSERVER_ErrorNew( \ TRITONSERVER_ERROR_INTERNAL, (std::string("onnx runtime error ") + \ std::to_string(code) + ": " + msg) \ .c_str())); \ } \ } while (false) #define THROW_IF_BACKEND_INSTANCE_ORT_ERROR(S) \ do { \ OrtStatus* status__ = (S); \ if (status__ != nullptr) { \ OrtErrorCode code = ort_api->GetErrorCode(status__); \ std::string msg = std::string(ort_api->GetErrorMessage(status__)); \ ort_api->ReleaseStatus(status__); \ throw BackendModelInstanceException(TRITONSERVER_ErrorNew( \ TRITONSERVER_ERROR_INTERNAL, (std::string("onnx runtime error ") + \ std::to_string(code) + ": " + msg) \ .c_str())); \ } \ } while (false) struct OnnxTensorInfo { OnnxTensorInfo(ONNXTensorElementDataType type, std::vector dims) : type_(type), dims_(dims) { } OnnxTensorInfo() {} ONNXTensorElementDataType type_{ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED}; std::vector dims_; }; using OnnxTensorInfoMap = std::unordered_map; /// Deleter for OrtTypeInfo. struct TypeInfoDeleter { void operator()(OrtTypeInfo* f) { ort_api->ReleaseTypeInfo(f); } }; /// Deleter for OrtSessionOptions. struct SessionOptionsDeleter { void operator()(OrtSessionOptions* f) { ort_api->ReleaseSessionOptions(f); } }; int32_t GetCpuCoreCount(); std::string OnnxDataTypeName(ONNXTensorElementDataType onnx_type); TRITONSERVER_DataType ConvertFromOnnxDataType( ONNXTensorElementDataType onnx_type); ONNXTensorElementDataType ConvertToOnnxDataType( TRITONSERVER_DataType data_type); ONNXTensorElementDataType ConvertToOnnxDataType( const std::string& data_type_str); ONNXTensorElementDataType ModelConfigDataTypeToOnnxDataType( const std::string& data_type_str); std::string OnnxDataTypeToModelConfigDataType( ONNXTensorElementDataType data_type); TRITONSERVER_Error* OverridableInitializerNames( OrtSession* session, std::set& names); TRITONSERVER_Error* InputNames( OrtSession* session, std::set& names); TRITONSERVER_Error* OutputNames( OrtSession* session, std::set& names); TRITONSERVER_Error* OverridableInitializerInfos( OrtSession* session, OrtAllocator* allocator, OnnxTensorInfoMap& infos); TRITONSERVER_Error* InputInfos( OrtSession* session, OrtAllocator* allocator, OnnxTensorInfoMap& infos); TRITONSERVER_Error* OutputInfos( OrtSession* session, OrtAllocator* allocator, OnnxTensorInfoMap& infos); TRITONSERVER_Error* CompareDimsSupported( const std::string& model_name, const std::string& tensor_name, const std::vector& model_shape, const std::vector& dims, const int max_batch_size, const bool compare_exact); }}} // namespace triton::backend::onnxruntime kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/000077500000000000000000000000001527052371200245005ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/000077500000000000000000000000001527052371200307255ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/README.md000066400000000000000000000035421527052371200322100ustar00rootroot00000000000000 This test is originated in "onnxruntime_backend" repository to better represent the scope of the test, however, this test utilizes Triton utilities and assumes that the test is located under "qa" directory in "server" repository for accessing those utilities. Please make sure the test environment is properly set before running the test.generate_test_model.py000066400000000000000000000050261527052371200352340ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import numpy as np import onnx # Reference script on how the model used in this test is created if __name__ == "__main__": values = np.ones((5, 5)).astype(np.float32) onnx_dtype = onnx.TensorProto.FLOAT initialized_input = onnx.helper.make_tensor( name="INITIALIZER", data_type=onnx_dtype, dims=values.shape, vals=values.flatten().astype(float), ) add = onnx.helper.make_node("Add", ["INPUT", "INITIALIZER"], ["OUTPUT"]) input = onnx.helper.make_tensor_value_info("INPUT", onnx_dtype, values.shape) initializer = onnx.helper.make_tensor_value_info( "INITIALIZER", onnx_dtype, values.shape ) output = onnx.helper.make_tensor_value_info("OUTPUT", onnx_dtype, values.shape) graph_proto = onnx.helper.make_graph( [add], "init_input", [input, initializer], [output], initializer=[initialized_input], ) model_def = onnx.helper.make_model(graph_proto, producer_name="triton") onnx.save(model_def, "model.onnx") kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/models/000077500000000000000000000000001527052371200322105ustar00rootroot00000000000000add_with_initializer/000077500000000000000000000000001527052371200363175ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/models1/000077500000000000000000000000001527052371200364575ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/models/add_with_initializermodel.onnx000066400000000000000000000004151527052371200404630ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/models/add_with_initializer/1triton: ! INPUT INITIALIZEROUTPUT"Add init_input*y"d?????????????????????????B INITIALIZERZ INPUT   Z INITIALIZER   b OUTPUT   Bconfig.pbtxt000066400000000000000000000034621527052371200406540ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/models/add_with_initializer# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. platform: "onnxruntime_onnx" max_batch_size: 0 input [ { name: "INPUT" data_type: TYPE_FP32 dims: [5, 5] }, { name: "INITIALIZER" data_type: TYPE_FP32 dims: [5, 5] optional: true } ] output [ { name: "OUTPUT" data_type: TYPE_FP32 dims: [ 5, 5] } ]kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/test.py000077500000000000000000000057421527052371200322710ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest import numpy as np import tritonclient.http as httpclient class OptionalInputTest(unittest.TestCase): def setUp(self): self.client_ = httpclient.InferenceServerClient("localhost:8000") self.model_name_ = "add_with_initializer" self.input_data_ = np.zeros((5, 5)).astype(np.float32) self.input_ = httpclient.InferInput("INPUT", self.input_data_.shape, "FP32") self.input_.set_data_from_numpy(self.input_data_, binary_data=False) self.optional_input_ = httpclient.InferInput( "INITIALIZER", self.input_data_.shape, "FP32" ) self.optional_input_.set_data_from_numpy(self.input_data_, binary_data=False) def test_without_optional(self): # Send request without providing optional input, the ONNX model # should use stored initializer value (tensor of all 1s) results = self.client_.infer(self.model_name_, [self.input_]) np.testing.assert_allclose(results.as_numpy("OUTPUT"), (self.input_data_ + 1)) def test_with_optional(self): # Send request with optional input provided, the ONNX model # should use provided value for the initializer results = self.client_.infer( self.model_name_, [self.input_, self.optional_input_] ) np.testing.assert_allclose( results.as_numpy("OUTPUT"), (self.input_data_ + self.input_data_) ) if __name__ == "__main__": unittest.main() kytensor-2.49.0.7/framework_backends/onnxruntime_backend/test/initializer_as_input/test.sh000077500000000000000000000042461527052371200322510ustar00rootroot00000000000000#!/bin/bash # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. export CUDA_VISIBLE_DEVICES=0 SERVER=/opt/tritonserver/bin/tritonserver SERVER_ARGS="--model-repository=`pwd`/models" SERVER_LOG="./server.log" CLIENT_LOG="./test.log" source ../common/util.sh rm -f *.log run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" cat $SERVER_LOG exit 1 fi RET=0 set +e python test.py >>$CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Test Failed\n***" RET=1 fi set -e kill $SERVER_PID wait $SERVER_PID if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else echo -e "\n***\n*** Test FAILED\n***" fi exit $RET kytensor-2.49.0.7/framework_backends/onnxruntime_backend/tools/000077500000000000000000000000001527052371200246615ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/onnxruntime_backend/tools/gen_ort_dockerfile.py000077500000000000000000000607711527052371200310750ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os import platform import re FLAGS = None ORT_TO_TRTPARSER_VERSION_MAP = { "1.9.0": ( "8.2", # TensorRT version "release/8.2-GA", # ONNX-Tensorrt parser version ), "1.10.0": ( "8.2", # TensorRT version "release/8.2-GA", # ONNX-Tensorrt parser version ), } OPENVINO_VERSION_MAP = { "2024.0.0": ( "2024.0", # OpenVINO short version "2024.0.0.14509.34caeefd078", # OpenVINO version with build number ), "2024.1.0": ( "2024.1", # OpenVINO short version "2024.1.0.15008.f4afc983258", # OpenVINO version with build number ), } def target_platform(): if FLAGS.target_platform is not None: return FLAGS.target_platform return platform.system().lower() def dockerfile_common(): df = """ ARG BASE_IMAGE={} ARG ONNXRUNTIME_VERSION={} ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime ARG ONNXRUNTIME_BUILD_CONFIG={} """.format( FLAGS.triton_container, FLAGS.ort_version, FLAGS.ort_build_config ) if FLAGS.ort_openvino is not None: df += """ ARG ONNXRUNTIME_OPENVINO_VERSION={} """.format( FLAGS.ort_openvino ) df += """ FROM ${BASE_IMAGE} WORKDIR /workspace """ return df def dockerfile_for_linux(output_file): df = dockerfile_common() df += """ # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive # The Onnx Runtime dockerfile is the collection of steps in # https://github.com/microsoft/onnxruntime/tree/master/dockerfiles """ # Consider moving rhel logic to its own function e.g., dockerfile_for_rhel # if the changes become more substantial. if target_platform() == "rhel": df += """ # The manylinux container defaults to Python 3.7, but some feature installation # requires a higher version. ARG PYVER=3.10 ENV PYTHONPATH=/opt/python/v RUN ln -sf /opt/python/cp${PYVER/./}* ${PYTHONPATH} ENV PYBIN=${PYTHONPATH}/bin ENV PYTHON_BIN_PATH=${PYBIN}/python${PYVER} \ PATH=${PYBIN}:${PATH} RUN yum install -y \ wget \ zip \ ca-certificates \ curl \ patchelf \ python3-pip \ git \ gnupg \ gnupg1 \ openssl-devel """ else: df += """ RUN apt-get update && apt-get install -y --no-install-recommends \ software-properties-common \ wget \ zip \ ca-certificates \ build-essential \ curl \ libcurl4-openssl-dev \ libssl-dev \ patchelf \ python3-dev \ python3-pip \ git \ gnupg \ gnupg1 # Install dependencies from # onnxruntime/dockerfiles/scripts/install_common_deps.sh. RUN apt update -q=2 \\ && apt install -y gpg wget \\ && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \\ && . /etc/os-release \\ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \\ && apt-get update -q=2 \\ && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7* \\ && cmake --version """ if FLAGS.enable_gpu: df += """ # Allow configure to pick up cuDNN where it expects it. # (Note: $CUDNN_VERSION is defined by base image) RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \ mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/include && \ ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \ mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \ ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so """ if FLAGS.ort_openvino is not None: df += """ # Install OpenVINO ARG ONNXRUNTIME_OPENVINO_VERSION ENV INTEL_OPENVINO_DIR /opt/intel/openvino_${ONNXRUNTIME_OPENVINO_VERSION} """ df += """ ARG OPENVINO_SHORT_VERSION={} ARG OPENVINO_VERSION_WITH_BUILD_NUMBER={} """.format( OPENVINO_VERSION_MAP[FLAGS.ort_openvino][0], OPENVINO_VERSION_MAP[FLAGS.ort_openvino][1], ) df += """ # Step 1: Download and install core components # Ref: https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html#step-1-download-and-install-the-openvino-core-components RUN curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_SHORT_VERSION}/linux/l_openvino_toolkit_ubuntu22_${OPENVINO_VERSION_WITH_BUILD_NUMBER}_x86_64.tgz --output openvino_${ONNXRUNTIME_OPENVINO_VERSION}.tgz && \ tar -xf openvino_${ONNXRUNTIME_OPENVINO_VERSION}.tgz && \ mkdir -p ${INTEL_OPENVINO_DIR} && \ mv l_openvino_toolkit_ubuntu22_${OPENVINO_VERSION_WITH_BUILD_NUMBER}_x86_64/* ${INTEL_OPENVINO_DIR} && \ rm openvino_${ONNXRUNTIME_OPENVINO_VERSION}.tgz && \ (cd ${INTEL_OPENVINO_DIR}/install_dependencies && \ ./install_openvino_dependencies.sh -y) && \ ln -s ${INTEL_OPENVINO_DIR} ${INTEL_OPENVINO_DIR}/../openvino_`echo ${ONNXRUNTIME_OPENVINO_VERSION} | awk '{print substr($0,0,4)}'` # Step 2: Configure the environment # Ref: https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html#step-2-configure-the-environment ENV OpenVINO_DIR=$INTEL_OPENVINO_DIR/runtime/cmake ENV LD_LIBRARY_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64:$LD_LIBRARY_PATH ENV PKG_CONFIG_PATH=$INTEL_OPENVINO_DIR/runtime/lib/intel64/pkgconfig ENV PYTHONPATH $INTEL_OPENVINO_DIR/python/python3.10:$INTEL_OPENVINO_DIR/python/python3:$PYTHONPATH """ ## TEMPORARY: Using the tensorrt-8.0 branch until ORT 1.9 release to enable ORT backend with TRT 8.0 support. # For ORT versions 1.8.0 and below the behavior will remain same. For ORT version 1.8.1 we will # use tensorrt-8.0 branch instead of using rel-1.8.1 # From ORT 1.9 onwards we will switch back to using rel-* branches if FLAGS.ort_version == "1.8.1": df += """ # # ONNX Runtime build # ARG ONNXRUNTIME_VERSION ARG ONNXRUNTIME_REPO ARG ONNXRUNTIME_BUILD_CONFIG RUN git clone -b tensorrt-8.0 --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ (cd onnxruntime && git submodule update --init --recursive) """ # Use the tensorrt-8.5ea branch to use Tensor RT 8.5a to use the built-in tensorrt parser elif FLAGS.ort_version == "1.12.1": df += """ # # ONNX Runtime build # ARG ONNXRUNTIME_VERSION ARG ONNXRUNTIME_REPO ARG ONNXRUNTIME_BUILD_CONFIG RUN git clone -b tensorrt-8.5ea --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ (cd onnxruntime && git submodule update --init --recursive) """ else: df += """ # # ONNX Runtime build # ARG ONNXRUNTIME_VERSION ARG ONNXRUNTIME_REPO ARG ONNXRUNTIME_BUILD_CONFIG RUN git clone -b rel-${ONNXRUNTIME_VERSION} --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ (cd onnxruntime && git submodule update --init --recursive) """ if FLAGS.onnx_tensorrt_tag != "": df += """ RUN (cd /workspace/onnxruntime/cmake/external/onnx-tensorrt && git fetch origin {}:ortrefbranch && git checkout ortrefbranch) """.format( FLAGS.onnx_tensorrt_tag ) ep_flags = "" if FLAGS.enable_gpu: ep_flags = "--use_cuda" if FLAGS.cuda_version is not None: ep_flags += ' --cuda_version "{}"'.format(FLAGS.cuda_version) if FLAGS.cuda_home is not None: ep_flags += ' --cuda_home "{}"'.format(FLAGS.cuda_home) if FLAGS.cudnn_home is not None: ep_flags += ' --cudnn_home "{}"'.format(FLAGS.cudnn_home) elif target_platform() == "igpu": ep_flags += ' --cudnn_home "/usr/lib/aarch64-linux-gnu"' if FLAGS.ort_tensorrt: ep_flags += " --use_tensorrt" if FLAGS.ort_version >= "1.12.1": ep_flags += " --use_tensorrt_builtin_parser" if FLAGS.tensorrt_home is not None: ep_flags += ' --tensorrt_home "{}"'.format(FLAGS.tensorrt_home) if os.name == "posix": if os.getuid() == 0: ep_flags += " --allow_running_as_root" if FLAGS.ort_openvino is not None: ep_flags += " --use_openvino CPU" if target_platform() == "igpu": ep_flags += ( " --skip_tests --cmake_extra_defines 'onnxruntime_BUILD_UNIT_TESTS=OFF'" ) cuda_archs = "53;62;72;87" else: cuda_archs = "60;61;70;75;80;86;90" df += """ WORKDIR /workspace/onnxruntime ARG COMMON_BUILD_ARGS="--config ${{ONNXRUNTIME_BUILD_CONFIG}} --skip_submodule_sync --parallel --build_shared_lib \ --build_dir /workspace/build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES='{}' " """.format( cuda_archs ) df += """ RUN ./build.sh ${{COMMON_BUILD_ARGS}} --update --build {} """.format( ep_flags ) df += """ # # Copy all artifacts needed by the backend to /opt/onnxruntime # WORKDIR /opt/onnxruntime RUN mkdir -p /opt/onnxruntime && \ cp /workspace/onnxruntime/LICENSE /opt/onnxruntime && \ cat /workspace/onnxruntime/cmake/external/onnx/VERSION_NUMBER > /opt/onnxruntime/ort_onnx_version.txt # ONNX Runtime headers, libraries and binaries RUN mkdir -p /opt/onnxruntime/include && \ cp /workspace/onnxruntime/include/onnxruntime/core/session/onnxruntime_c_api.h \ /opt/onnxruntime/include && \ cp /workspace/onnxruntime/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h \ /opt/onnxruntime/include && \ cp /workspace/onnxruntime/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h \ /opt/onnxruntime/include RUN mkdir -p /opt/onnxruntime/lib && \ cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_shared.so \ /opt/onnxruntime/lib && \ cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime.so \ /opt/onnxruntime/lib """ if target_platform() == "igpu": df += """ RUN mkdir -p /opt/onnxruntime/bin """ else: df += """ RUN mkdir -p /opt/onnxruntime/bin && \ cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/onnxruntime_perf_test \ /opt/onnxruntime/bin && \ cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/onnx_test_runner \ /opt/onnxruntime/bin && \ (cd /opt/onnxruntime/bin && chmod a+x *) """ if FLAGS.enable_gpu: df += """ RUN cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_cuda.so \ /opt/onnxruntime/lib """ if FLAGS.ort_tensorrt: df += """ # TensorRT specific headers and libraries RUN cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_tensorrt.so \ /opt/onnxruntime/lib """ if FLAGS.ort_openvino is not None: df += """ # OpenVino specific headers and libraries RUN cp -r ${INTEL_OPENVINO_DIR}/docs/licensing /opt/onnxruntime/LICENSE.openvino RUN cp /workspace/onnxruntime/include/onnxruntime/core/providers/openvino/openvino_provider_factory.h \ /opt/onnxruntime/include RUN cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_openvino.so \ /opt/onnxruntime/lib && \ cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} \ /opt/onnxruntime/lib && \ cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} \ /opt/onnxruntime/lib && \ cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_intel_cpu_plugin.so \ /opt/onnxruntime/lib && \ cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} \ /opt/onnxruntime/lib && \ cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} \ /opt/onnxruntime/lib && \ cp /usr/lib/x86_64-linux-gnu/libtbb.so.12 /opt/onnxruntime/lib RUN OV_SHORT_VERSION=`echo ${ONNXRUNTIME_OPENVINO_VERSION} | awk '{ split($0,a,"."); print substr(a[1],3) a[2] a[3] }'` && \ (cd /opt/onnxruntime/lib && \ chmod a-x * && \ ln -s libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino.so.${OV_SHORT_VERSION} && \ ln -s libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino.so && \ ln -s libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_c.so.${OV_SHORT_VERSION} && \ ln -s libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_c.so && \ ln -s libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_ir_frontend.so.${OV_SHORT_VERSION} && \ ln -s libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_ir_frontend.so && \ ln -s libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_onnx_frontend.so.${OV_SHORT_VERSION} && \ ln -s libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_onnx_frontend.so) """ # Linking compiled ONNX Runtime libraries to their corresponding versioned libraries df += """ RUN cd /opt/onnxruntime/lib \ && ln -s libonnxruntime.so libonnxruntime.so.${ONNXRUNTIME_VERSION} """ df += """ RUN cd /opt/onnxruntime/lib && \ for i in `find . -mindepth 1 -maxdepth 1 -type f -name '*\.so*'`; do \ patchelf --set-rpath '$ORIGIN' $i; \ done # For testing copy ONNX custom op library and model """ if target_platform() == "igpu": df += """ RUN mkdir -p /opt/onnxruntime/test """ else: df += """ RUN mkdir -p /opt/onnxruntime/test && \ cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libcustom_op_library.so \ /opt/onnxruntime/test && \ cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/testdata/custom_op_library/custom_op_test.onnx \ /opt/onnxruntime/test """ with open(output_file, "w") as dfile: dfile.write(df) def dockerfile_for_windows(output_file): df = dockerfile_common() ## TEMPORARY: Using the tensorrt-8.0 branch until ORT 1.9 release to enable ORT backend with TRT 8.0 support. # For ORT versions 1.8.0 and below the behavior will remain same. For ORT version 1.8.1 we will # use tensorrt-8.0 branch instead of using rel-1.8.1 # From ORT 1.9 onwards we will switch back to using rel-* branches if FLAGS.ort_version == "1.8.1": df += """ SHELL ["cmd", "/S", "/C"] # # ONNX Runtime build # ARG ONNXRUNTIME_VERSION ARG ONNXRUNTIME_REPO RUN git clone -b tensorrt-8.0 --recursive %ONNXRUNTIME_REPO% onnxruntime && \ (cd onnxruntime && git submodule update --init --recursive) """ else: df += """ SHELL ["cmd", "/S", "/C"] # # ONNX Runtime build # ARG ONNXRUNTIME_VERSION ARG ONNXRUNTIME_REPO RUN git clone -b rel-%ONNXRUNTIME_VERSION% --recursive %ONNXRUNTIME_REPO% onnxruntime && \ (cd onnxruntime && git submodule update --init --recursive) """ if FLAGS.onnx_tensorrt_tag != "": df += """ RUN (cd \\workspace\\onnxruntime\\cmake\\external\\onnx-tensorrt && git fetch origin {}:ortrefbranch && git checkout ortrefbranch) """.format( FLAGS.onnx_tensorrt_tag ) ep_flags = "" if FLAGS.enable_gpu: ep_flags = "--use_cuda" if FLAGS.cuda_version is not None: ep_flags += ' --cuda_version "{}"'.format(FLAGS.cuda_version) if FLAGS.cuda_home is not None: ep_flags += ' --cuda_home "{}"'.format(FLAGS.cuda_home) if FLAGS.cudnn_home is not None: ep_flags += ' --cudnn_home "{}"'.format(FLAGS.cudnn_home) if FLAGS.ort_tensorrt: ep_flags += " --use_tensorrt" if FLAGS.tensorrt_home is not None: ep_flags += ' --tensorrt_home "{}"'.format(FLAGS.tensorrt_home) if FLAGS.ort_openvino is not None: ep_flags += " --use_openvino CPU" df += """ WORKDIR /workspace/onnxruntime ARG VS_DEVCMD_BAT="\BuildTools\VC\Auxiliary\Build\vcvars64.bat" RUN powershell Set-Content 'build.bat' -value 'call %VS_DEVCMD_BAT%',(Get-Content 'build.bat') RUN build.bat --cmake_generator "Visual Studio 17 2022" --config Release --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80;86;90" --skip_submodule_sync --parallel --build_shared_lib --compile_no_warning_as_error --skip_tests --update --build --build_dir /workspace/build {} """.format( ep_flags ) df += """ # # Copy all artifacts needed by the backend to /opt/onnxruntime # WORKDIR /opt/onnxruntime RUN copy \\workspace\\onnxruntime\\LICENSE \\opt\\onnxruntime RUN copy \\workspace\\onnxruntime\\cmake\\external\\onnx\\VERSION_NUMBER \\opt\\onnxruntime\\ort_onnx_version.txt # ONNX Runtime headers, libraries and binaries WORKDIR /opt/onnxruntime/include RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\session\\onnxruntime_c_api.h \\opt\\onnxruntime\\include RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\session\\onnxruntime_session_options_config_keys.h \\opt\\onnxruntime\\include RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\providers\\cpu\\cpu_provider_factory.h \\opt\\onnxruntime\\include WORKDIR /opt/onnxruntime/bin RUN copy \\workspace\\build\\Release\\Release\\onnxruntime.dll \\opt\\onnxruntime\\bin RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_shared.dll \\opt\\onnxruntime\\bin RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_perf_test.exe \\opt\\onnxruntime\\bin RUN copy \\workspace\\build\\Release\\Release\\onnx_test_runner.exe \\opt\\onnxruntime\\bin WORKDIR /opt/onnxruntime/lib RUN copy \\workspace\\build\\Release\\Release\\onnxruntime.lib \\opt\\onnxruntime\\lib RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_shared.lib \\opt\\onnxruntime\\lib """ if FLAGS.enable_gpu: df += """ WORKDIR /opt/onnxruntime/lib RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_cuda.lib \\opt\\onnxruntime\\lib WORKDIR /opt/onnxruntime/bin RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_cuda.dll \\opt\\onnxruntime\\bin """ if FLAGS.ort_tensorrt: df += """ # TensorRT specific headers and libraries WORKDIR /opt/onnxruntime/lib RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_tensorrt.dll \\opt\\onnxruntime\\bin WORKDIR /opt/onnxruntime/lib RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_tensorrt.lib \\opt\\onnxruntime\\lib """ with open(output_file, "w") as dfile: dfile.write(df) def preprocess_gpu_flags(): if target_platform() == "windows": # Default to CUDA based on CUDA_PATH envvar and TensorRT in # C:/tensorrt if "CUDA_PATH" in os.environ: if FLAGS.cuda_home is None: FLAGS.cuda_home = os.environ["CUDA_PATH"] elif FLAGS.cuda_home != os.environ["CUDA_PATH"]: print("warning: --cuda-home does not match CUDA_PATH envvar") if FLAGS.cudnn_home is None: FLAGS.cudnn_home = FLAGS.cuda_home version = None m = re.match(r".*v([1-9]?[0-9]+\.[0-9]+)$", FLAGS.cuda_home) if m: version = m.group(1) if FLAGS.cuda_version is None: FLAGS.cuda_version = version elif FLAGS.cuda_version != version: print("warning: --cuda-version does not match CUDA_PATH envvar") if (FLAGS.cuda_home is None) or (FLAGS.cuda_version is None): print("error: windows build requires --cuda-version and --cuda-home") if FLAGS.tensorrt_home is None: FLAGS.tensorrt_home = "/tensorrt" else: if "CUDNN_VERSION" in os.environ: version = None m = re.match(r"([0-9]\.[0-9])\.[0-9]\.[0-9]", os.environ["CUDNN_VERSION"]) if m: version = m.group(1) if FLAGS.cudnn_home is None: FLAGS.cudnn_home = "/usr/local/cudnn-{}/cuda".format(version) if FLAGS.cuda_home is None: FLAGS.cuda_home = "/usr/local/cuda" if (FLAGS.cuda_home is None) or (FLAGS.cudnn_home is None): print("error: linux build requires --cudnn-home and --cuda-home") if FLAGS.tensorrt_home is None: FLAGS.tensorrt_home = "/usr/src/tensorrt" if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--triton-container", type=str, required=True, help="Triton base container to use for ORT build.", ) parser.add_argument("--ort-version", type=str, required=True, help="ORT version.") parser.add_argument( "--output", type=str, required=True, help="File to write Dockerfile to." ) parser.add_argument( "--enable-gpu", action="store_true", required=False, help="Enable GPU support" ) parser.add_argument( "--ort-build-config", type=str, default="Release", choices=["Debug", "Release", "RelWithDebInfo"], help="ORT build configuration.", ) parser.add_argument( "--target-platform", required=False, default=None, help='Target for build, can be "linux", "windows" or "igpu". If not specified, build targets the current platform.', ) parser.add_argument( "--cuda-version", type=str, required=False, help="Version for CUDA." ) parser.add_argument( "--cuda-home", type=str, required=False, help="Home directory for CUDA." ) parser.add_argument( "--cudnn-home", type=str, required=False, help="Home directory for CUDNN." ) parser.add_argument( "--ort-openvino", type=str, required=False, help="Enable OpenVino execution provider using specified OpenVINO version.", ) parser.add_argument( "--ort-tensorrt", action="store_true", required=False, help="Enable TensorRT execution provider.", ) parser.add_argument( "--tensorrt-home", type=str, required=False, help="Home directory for TensorRT." ) parser.add_argument( "--onnx-tensorrt-tag", type=str, default="", help="onnx-tensorrt repo tag." ) parser.add_argument("--trt-version", type=str, default="", help="TRT version.") FLAGS = parser.parse_args() if FLAGS.enable_gpu: preprocess_gpu_flags() # if a tag is provided by the user, then simply use it # if the tag is empty - check whether there is an entry in the ORT_TO_TRTPARSER_VERSION_MAP # map corresponding to ort version + trt version combo. If yes then use it # otherwise we leave it empty and use the defaults from ort if ( FLAGS.onnx_tensorrt_tag == "" and FLAGS.ort_version in ORT_TO_TRTPARSER_VERSION_MAP.keys() ): trt_version = re.match(r"^[0-9]+\.[0-9]+", FLAGS.trt_version) if ( trt_version and trt_version.group(0) == ORT_TO_TRTPARSER_VERSION_MAP[FLAGS.ort_version][0] ): FLAGS.onnx_tensorrt_tag = ORT_TO_TRTPARSER_VERSION_MAP[FLAGS.ort_version][1] if target_platform() == "windows": # OpenVINO EP not yet supported for windows build if FLAGS.ort_openvino is not None: print("warning: OpenVINO not supported for windows, ignoring") FLAGS.ort_openvino = None dockerfile_for_windows(FLAGS.output) else: dockerfile_for_linux(FLAGS.output) kytensor-2.49.0.7/framework_backends/python_backend/000077500000000000000000000000001527052371200224545ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/.clang-format000066400000000000000000000014261527052371200250320ustar00rootroot00000000000000--- BasedOnStyle: Google IndentWidth: 2 ColumnLimit: 80 ContinuationIndentWidth: 4 UseTab: Never MaxEmptyLinesToKeep: 2 SortIncludes: true CompactNamespaces: true ReflowComments: true DerivePointerAlignment: false PointerAlignment: Left AllowShortIfStatementsOnASingleLine: false AllowShortBlocksOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AlwaysBreakAfterReturnType: TopLevelDefinitions AlignAfterOpenBracket: AlwaysBreak BreakBeforeBraces: Custom BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: true AfterNamespace: false AfterStruct: false AfterUnion: false BeforeCatch: true BinPackArguments: true BinPackParameters: true ConstructorInitializerAllOnOneLineOrOnePerLine: false IndentCaseLabels: true kytensor-2.49.0.7/framework_backends/python_backend/.devcontainer/000077500000000000000000000000001527052371200252135ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/.devcontainer/Dockerfile000066400000000000000000000037551527052371200272170ustar00rootroot00000000000000# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. FROM nvcr.io/nvidia/tritonserver:24.03-py3 ARG USERNAME=triton-server RUN apt-get update \ && apt-get install -y sudo RUN pip3 install transformers torch # Create the user RUN apt-get update \ && apt-get install -y sudo \ && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ && chmod 0440 /etc/sudoers.d/$USERNAME RUN pip3 install pre-commit ipdb RUN mkhomedir_helper triton-server RUN apt-get install -y cmake rapidjson-dev USER ${USERNAME} kytensor-2.49.0.7/framework_backends/python_backend/.devcontainer/devcontainer.json000066400000000000000000000014311527052371200305660ustar00rootroot00000000000000{ "name": "Python Backend", "build": { "dockerfile": "Dockerfile" }, "customizations": { "vscode": { "extensions": [ "ms-python.vscode-pylance", "ms-python.python", "ms-vscode.cpptools-extension-pack", "ms-vscode.cmake-tools", "github.vscode-pull-request-github" ] } }, "postCreateCommand": "sudo chown -R triton-server:triton-server ~/.cache", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined", "--gpus=all", "--shm-size=2g", "--ulimit", "stack=67108864" ], "mounts": [ "source=${localEnv:HOME}/.ssh,target=/home/triton-server/.ssh,type=bind,consistency=cached", "source=${localEnv:HOME}/.cache/huggingface,target=/home/triton-server/.cache/huggingface,type=bind,consistency=cached" ], "remoteUser": "triton-server" } kytensor-2.49.0.7/framework_backends/python_backend/.github/000077500000000000000000000000001527052371200240145ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/.github/workflows/000077500000000000000000000000001527052371200260515ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/.github/workflows/codeql.yml000066400000000000000000000071451527052371200300520ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "CodeQL" on: pull_request: jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to: # https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs queries: +security-and-quality # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 # Command-line programs to run using the OS shell. # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | # echo "Run, Build Application using script" # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: category: "/language:${{matrix.language}}" kytensor-2.49.0.7/framework_backends/python_backend/.github/workflows/pre-commit.yml000066400000000000000000000033201527052371200306460ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: pre-commit on: pull_request: jobs: pre-commit: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - uses: pre-commit/action@v3.0.0 kytensor-2.49.0.7/framework_backends/python_backend/.pre-commit-config.yaml000066400000000000000000000056111527052371200267400ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black rev: 23.1.0 hooks: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] types_or: [python, cython] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.5 hooks: - id: clang-format types_or: [c, c++, cuda, proto, textproto, java] args: ["-fallback-style=none", "-style=file", "-i"] - repo: https://github.com/codespell-project/codespell rev: v2.2.4 hooks: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$) # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-json - id: check-toml - id: check-yaml - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace kytensor-2.49.0.7/framework_backends/python_backend/.vscode/000077500000000000000000000000001527052371200240155ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/.vscode/tasks.json000066400000000000000000000046541527052371200260460ustar00rootroot00000000000000{ "version": "2.0.0", "tasks": [ { "label": "Configure", "type": "shell", "command": "cmake", "args": [ "-DCMAKE_INSTALL_PREFIX:STRING=/opt/tritonserver/", "-DTRITON_COMMON_REPO_TAG:STRING=main", "-DTRITON_BACKEND_REPO_TAG:STRING=main", "-DTRITON_CORE_REPO_TAG:STRING=main", "-DTRITON_ENABLE_GPU:STRING=ON", "-DTRITON_ENABLE_NVTX:STRING=ON", "-DCMAKE_INSTALL_PREFIX:STRING=${workspaceFolder}/build/install", "-DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=TRUE", "-DCMAKE_BUILD_TYPE:STRING=Debug", "-DCMAKE_C_COMPILER:FILEPATH=/usr/bin/gcc", "-DCMAKE_CXX_COMPILER:FILEPATH=/usr/bin/g++", "-S${workspaceFolder}", "-B${workspaceFolder}/build", "-G", "Unix Makefiles" ], "problemMatcher": [] }, { "label": "Build", "type": "shell", "command": "cmake", "args": [ "--build", "/${workspaceFolder}/build", "--config", "Debug", "--target", "all", "-j", "18", "--" ] }, { "label": "Install", "type": "shell", "command": "cmake", "args": [ "--build", "${workspaceFolder}/build", "--config", "Debug", "--target", "install", "-j", "18", "--" ] }, { "label": "Move", "type": "shell", "command": "sudo", "args": [ "cp", "-r", "${workspaceFolder}/build/install/backends/python/*", "/opt/tritonserver/backends/python" ] }, { "label": "Build Python Backend", "dependsOrder": "sequence", "dependsOn": [ "Configure", "Build", "Install", "Move" ], "group": { "kind": "build", "isDefault": true } } ] } kytensor-2.49.0.7/framework_backends/python_backend/CMakeLists.txt000066400000000000000000000262041527052371200252200ustar00rootroot00000000000000# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonpythonbackend LANGUAGES C CXX) # Use C++17 standard as Triton's minimum required. set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.") # # Options # # Must include options required for this project as well as any # projects included in this one by FetchContent. # option(TRITON_ENABLE_GPU "Enable GPU support in backend" ON) option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON) option(TRITON_ENABLE_NVTX "Include nvtx markers collection in backend." OFF) # FIXME: CI needs to enable the GPU flag. Python for window currently does not # support GPU tensors. For simplicity, we will override this option here. if(WIN32) set(TRITON_ENABLE_GPU OFF CACHE BOOL "GPU disabled" FORCE) endif() set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() # # Dependencies # # FetchContent's composability isn't very good. We must include the # transitive closure of all repos so that we can override the tag. # include(FetchContent) # We need to use ExternalProject because we want to use Boost 1.76 which is not # available in Ubuntu 20.04 include(ExternalProject) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-backend SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/backend ) FetchContent_MakeAvailable(repo-common repo-core repo-backend) # FetchContent_Declare( # pybind11 # SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/third_party/pybind11 # #GIT_REPOSITORY "https://github.com/pybind/pybind11" # # COMMIT ID for v2.10.0 # #GIT_TAG "aa304c9c7d725ffb9d10af08a3b34cb372307020" # #GIT_SHALLOW ON # ) # FetchContent_MakeAvailable(pybind11) # # DLPack # FetchContent_Declare( dlpack SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/third_party/dlpack #GIT_REPOSITORY "https://github.com/dmlc/dlpack" #GIT_TAG "v0.8" #GIT_SHALLOW ON ) # Option must be set off so WIN32 build does not break set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) set(BUILD_MOCK OFF) FetchContent_MakeAvailable(dlpack) # # Boost # # ExternalProject_Add( # boostorg # URL https://archives.boost.io/release/1.79.0/source/boost_1_79_0.tar.gz # URL_HASH SHA256=273f1be93238a068aba4f9735a4a2b003019af067b9c183ed227780b8f36062c # PREFIX "boost-src" # CONFIGURE_COMMAND ${CMAKE_COMMAND} -E copy_directory # /boost/ ${CMAKE_BINARY_DIR}/boost # INSTALL_COMMAND "" # BUILD_COMMAND "" # ) # set(boostorg_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/boost/") # set(BOOST_ROOT ${Boost_DIR}) # set(Boost_NO_SYSTEM_PATHS ON) # find_package(Boost REQUIRED) # include_directories(${Boost_INCLUDE_DIRS}) find_package(pybind11 CONFIG REQUIRED) message(STATUS "Using pybind11 ${pybind11_VERSION}") # # CUDA # if(${TRITON_ENABLE_GPU}) find_package(CUDAToolkit REQUIRED) message(STATUS "Using CUDA ${CUDA_VERSION}") set(CUDA_NVCC_FLAGS -std=c++${TRITON_MIN_CXX_STANDARD}) elseif() message(WARNING "TRITON_ENABLE_GPU is OFF, GPU Tensor support will be disabled") endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_NVTX}) add_definitions(-DTRITON_ENABLE_NVTX=1) endif() # TRITON_ENABLE_NVTX find_package(ZLIB REQUIRED) if(NOT WIN32) find_package(Threads REQUIRED) endif() include_directories(${CMAKE_BINARY_DIR}) configure_file(src/libtriton_python.ldscript libtriton_python.ldscript COPYONLY) set( COMMON_SRCS src/correlation_id.cc src/correlation_id.h src/infer_response.cc src/infer_response.h src/infer_request.cc src/infer_request.h src/infer_trace.cc src/infer_trace.h src/message_queue.h src/ipc_message.cc src/ipc_message.h src/pb_string.cc src/pb_string.h src/pb_map.cc src/pb_map.h src/scoped_defer.cc src/scoped_defer.h src/pb_error.cc src/pb_error.h src/pb_log.cc src/pb_log.h src/pb_memory.cc src/pb_memory.h src/pb_tensor.cc src/pb_tensor.h src/pb_utils.cc src/pb_utils.h src/shm_manager.cc src/shm_manager.h src/pb_exception.h src/pb_preferred_memory.h src/metric.h src/metric.cc src/metric_family.h src/metric_family.cc src/gpu_buffers.cc src/gpu_buffers.h src/model_loader.h src/model_loader.cc ) set( PYTHON_BACKEND_SRCS src/python_be.cc src/python_be.h src/pb_env.cc src/pb_env.h src/pb_metric_reporter.cc src/pb_metric_reporter.h src/memory_manager.cc src/memory_manager.h src/request_executor.cc src/request_executor.h src/stub_launcher.h src/stub_launcher.cc src/infer_payload.h src/infer_payload.cc ) list(APPEND PYTHON_BACKEND_SRCS ${COMMON_SRCS} ) add_library( triton-python-backend SHARED ${PYTHON_BACKEND_SRCS} ) set( PYTHON_BACKEND_STUB_SRCS src/pb_stub_utils.h src/pb_stub_utils.cc src/response_sender.cc src/response_sender.h src/pb_stub.h src/pb_stub.cc src/pb_response_iterator.h src/pb_response_iterator.cc src/pb_cancel.cc src/pb_cancel.h ) list(APPEND PYTHON_BACKEND_STUB_SRCS ${COMMON_SRCS} ) add_executable( triton-python-backend-stub ${PYTHON_BACKEND_STUB_SRCS} ) #add_dependencies(triton-python-backend boostorg) #add_dependencies(triton-python-backend-stub boostorg) set_property(TARGET triton-python-backend-stub PROPERTY OUTPUT_NAME triton_python_backend_stub) add_library( TritonPythonBackend::triton-python-backend ALIAS triton-python-backend ) target_compile_features(triton-python-backend PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-python-backend PRIVATE $<$,$,$>: -Wall -Wno-deprecated-declarations -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) target_compile_features(triton-python-backend-stub PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) target_compile_options( triton-python-backend-stub PRIVATE $<$,$,$>: -fvisibility=hidden -Wall -Wno-deprecated-declarations -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) target_compile_definitions(triton-python-backend-stub PRIVATE TRITON_PB_STUB) # For WIN32 do not link Threads and DL_LIBS if(WIN32) target_link_libraries( triton-python-backend PRIVATE dlpack triton-backend-utils # from repo-backend -lrt # shared memory triton-core-serverstub # from repo-core ZLIB::ZLIB -larchive ) target_link_libraries( triton-python-backend-stub PRIVATE dlpack triton-backend-utils # from repo-backend pybind11::embed -lrt # shared memory -larchive # libarchive ) else() target_link_libraries( triton-python-backend PRIVATE dlpack Threads::Threads triton-backend-utils # from repo-backend ${CMAKE_DL_LIBS} # dlopen and dlclose -lrt # shared memory triton-core-serverstub # from repo-core ZLIB::ZLIB -larchive ) target_link_libraries( triton-python-backend-stub PRIVATE dlpack Threads::Threads triton-backend-utils # from repo-backend ${CMAKE_DL_LIBS} # dlopen and dlclose pybind11::embed -lrt # shared memory -larchive # libarchive ) endif() if(WIN32) set_target_properties( triton-python-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_python ) else() set_target_properties( triton-python-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_python LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_python.ldscript LINK_FLAGS "-Wl,--version-script libtriton_python.ldscript" ) endif() add_subdirectory(./src/shm_monitor) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonPythonBackend) install( TARGETS triton-python-backend triton-python-backend-stub EXPORT triton-python-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/python ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/python RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/python ) install( EXPORT triton-python-backend-targets FILE TritonPythonBackendTargets.cmake NAMESPACE TritonPythonBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES src/resources/triton_python_backend_utils.py DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/python ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonPythonBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonPythonBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonPythonBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-python-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonPythonBackendTargets.cmake NAMESPACE TritonPythonBackend:: ) export(PACKAGE TritonPythonBackend) kytensor-2.49.0.7/framework_backends/python_backend/LICENSE000066400000000000000000000027151527052371200234660ustar00rootroot00000000000000Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. kytensor-2.49.0.7/framework_backends/python_backend/README.md000066400000000000000000002265551527052371200237520ustar00rootroot00000000000000 [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) # Python Backend The Triton backend for Python. The goal of Python backend is to let you serve models written in Python by Triton Inference Server without having to write any C++ code. ## User Documentation - [Python Backend](#python-backend) - [User Documentation](#user-documentation) - [Quick Start](#quick-start) - [Building from Source](#building-from-source) - [Usage](#usage) - [`auto_complete_config`](#auto_complete_config) - [`initialize`](#initialize) - [`execute`](#execute) - [Default Mode](#default-mode) - [Error Handling](#error-handling) - [Request Cancellation Handling](#request-cancellation-handling) - [Decoupled mode](#decoupled-mode) - [Use Cases](#use-cases) - [Async Execute](#async-execute) - [Request Rescheduling](#request-rescheduling) - [`finalize`](#finalize) - [Model Config File](#model-config-file) - [Inference Request Parameters](#inference-request-parameters) - [Managing Python Runtime and Libraries](#managing-python-runtime-and-libraries) - [Building Custom Python Backend Stub](#building-custom-python-backend-stub) - [Creating Custom Execution Environments](#creating-custom-execution-environments) - [Important Notes](#important-notes) - [Error Handling](#error-handling-1) - [Managing Shared Memory](#managing-shared-memory) - [Multiple Model Instance Support](#multiple-model-instance-support) - [Running Multiple Instances of Triton Server](#running-multiple-instances-of-triton-server) - [Business Logic Scripting](#business-logic-scripting) - [Using BLS with Decoupled Models](#using-bls-with-decoupled-models) - [Model Loading API](#model-loading-api) - [Using BLS with Stateful Models](#using-bls-with-stateful-models) - [Limitation](#limitation) - [Interoperability and GPU Support](#interoperability-and-gpu-support) - [`pb_utils.Tensor.to_dlpack() -> PyCapsule`](#pb_utilstensorto_dlpack---pycapsule) - [`pb_utils.Tensor.from_dlpack() -> Tensor`](#pb_utilstensorfrom_dlpack---tensor) - [`pb_utils.Tensor.is_cpu() -> bool`](#pb_utilstensoris_cpu---bool) - [Input Tensor Device Placement](#input-tensor-device-placement) - [Frameworks](#frameworks) - [PyTorch](#pytorch) - [PyTorch Determinism](#pytorch-determinism) - [TensorFlow](#tensorflow) - [TensorFlow Determinism](#tensorflow-determinism) - [Custom Metrics](#custom-metrics) - [Examples](#examples) - [AddSub in NumPy](#addsub-in-numpy) - [AddSubNet in PyTorch](#addsubnet-in-pytorch) - [AddSub in JAX](#addsub-in-jax) - [Business Logic Scripting](#business-logic-scripting-1) - [Preprocessing](#preprocessing) - [Decoupled Models](#decoupled-models) - [Model Instance Kind](#model-instance-kind) - [Auto-complete config](#auto-complete-config) - [Custom Metrics](#custom-metrics-1) - [Running with Inferentia](#running-with-inferentia) - [Logging](#logging) - [Development with VSCode](#development-with-vscode) - [Reporting problems, asking questions](#reporting-problems-asking-questions) ## Quick Start 1. Run the Triton Inference Server container. ``` docker run --shm-size=1g --ulimit memlock=-1 -p 8000:8000 -p 8001:8001 -p 8002:8002 --ulimit stack=67108864 -ti nvcr.io/nvidia/tritonserver:-py3 ``` Replace \ with the Triton version (e.g. 21.05). 2. Inside the container, clone the Python backend repository. ``` git clone https://github.com/triton-inference-server/python_backend -b r ``` 3. Install example model. ``` cd python_backend mkdir -p models/add_sub/1/ cp examples/add_sub/model.py models/add_sub/1/model.py cp examples/add_sub/config.pbtxt models/add_sub/config.pbtxt ``` 4. Start the Triton server. ``` tritonserver --model-repository `pwd`/models ``` 5. In the host machine, start the client container. ``` docker run -ti --net host nvcr.io/nvidia/tritonserver:-py3-sdk /bin/bash ``` 6. In the client container, clone the Python backend repository. ``` git clone https://github.com/triton-inference-server/python_backend -b r ``` 7. Run the example client. ``` python3 python_backend/examples/add_sub/client.py ``` ## Building from Source 1. Requirements * cmake >= 3.17 * numpy * rapidjson-dev * libarchive-dev * zlib1g-dev ``` pip3 install numpy ``` On Ubuntu or Debian you can use the command below to install `rapidjson`, `libarchive`, and `zlib`: ``` sudo apt-get install rapidjson-dev libarchive-dev zlib1g-dev ``` 2. Build Python backend. Replace \ with the GitHub branch that you want to compile. For release branches it should be r\ (e.g. r21.06). ``` mkdir build cd build cmake -DTRITON_ENABLE_GPU=ON -DTRITON_BACKEND_REPO_TAG= -DTRITON_COMMON_REPO_TAG= -DTRITON_CORE_REPO_TAG= -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install .. make install ``` The following required Triton repositories will be pulled and used in the build. If the CMake variables below are not specified, "main" branch of those repositories will be used. \ should be the same as the Python backend repository branch that you are trying to compile. * triton-inference-server/backend: `-DTRITON_BACKEND_REPO_TAG=` * triton-inference-server/common: `-DTRITON_COMMON_REPO_TAG=` * triton-inference-server/core: `-DTRITON_CORE_REPO_TAG=` Set `-DCMAKE_INSTALL_PREFIX` to the location where the Triton Server is installed. In the released containers, this location is `/opt/tritonserver`. 3. Copy example model and configuration ``` mkdir -p models/add_sub/1/ cp examples/add_sub/model.py models/add_sub/1/model.py cp examples/add_sub/config.pbtxt models/add_sub/config.pbtxt ``` 4. Start the Triton Server ``` /opt/tritonserver/bin/tritonserver --model-repository=`pwd`/models ``` 5. Use the client app to perform inference ``` python3 examples/add_sub/client.py ``` ## Usage In order to use the Python backend, you need to create a Python file that has a structure similar to below: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ @staticmethod def auto_complete_config(auto_complete_model_config): """`auto_complete_config` is called only once when loading the model assuming the server was not started with `--disable-auto-complete-config`. Implementing this function is optional. No implementation of `auto_complete_config` will do nothing. This function can be used to set `max_batch_size`, `input` and `output` properties of the model using `set_max_batch_size`, `add_input`, and `add_output`. These properties will allow Triton to load the model with minimal model configuration in absence of a configuration file. This function returns the `pb_utils.ModelConfig` object with these properties. You can use the `as_dict` function to gain read-only access to the `pb_utils.ModelConfig` object. The `pb_utils.ModelConfig` object being returned from here will be used as the final configuration for the model. Note: The Python interpreter used to invoke this function will be destroyed upon returning from this function and as a result none of the objects created here will be available in the `initialize`, `execute`, or `finalize` functions. Parameters ---------- auto_complete_model_config : pb_utils.ModelConfig An object containing the existing model configuration. You can build upon the configuration given by this object when setting the properties for this model. Returns ------- pb_utils.ModelConfig An object containing the auto-completed model configuration """ inputs = [{ 'name': 'INPUT0', 'data_type': 'TYPE_FP32', 'dims': [4], # this parameter will set `INPUT0 as an optional input` 'optional': True }, { 'name': 'INPUT1', 'data_type': 'TYPE_FP32', 'dims': [4] }] outputs = [{ 'name': 'OUTPUT0', 'data_type': 'TYPE_FP32', 'dims': [4] }, { 'name': 'OUTPUT1', 'data_type': 'TYPE_FP32', 'dims': [4] }] # Demonstrate the usage of `as_dict`, `add_input`, `add_output`, # `set_max_batch_size`, and `set_dynamic_batching` functions. # Store the model configuration as a dictionary. config = auto_complete_model_config.as_dict() input_names = [] output_names = [] for input in config['input']: input_names.append(input['name']) for output in config['output']: output_names.append(output['name']) for input in inputs: # The name checking here is only for demonstrating the usage of # `as_dict` function. `add_input` will check for conflicts and # raise errors if an input with the same name already exists in # the configuration but has different data_type or dims property. if input['name'] not in input_names: auto_complete_model_config.add_input(input) for output in outputs: # The name checking here is only for demonstrating the usage of # `as_dict` function. `add_output` will check for conflicts and # raise errors if an output with the same name already exists in # the configuration but has different data_type or dims property. if output['name'] not in output_names: auto_complete_model_config.add_output(output) auto_complete_model_config.set_max_batch_size(0) # To enable a dynamic batcher with default settings, you can use # auto_complete_model_config set_dynamic_batching() function. It is # commented in this example because the max_batch_size is zero. # # auto_complete_model_config.set_dynamic_batching() return auto_complete_model_config def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ print('Initialized...') def execute(self, requests): """`execute` must be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference is requested for this model. Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ responses = [] # Every Python backend must iterate through list of requests and create # an instance of pb_utils.InferenceResponse class for each of them. # Reusing the same pb_utils.InferenceResponse object for multiple # requests may result in segmentation faults. You should avoid storing # any of the input Tensors in the class attributes as they will be # overridden in subsequent inference requests. You can make a copy of # the underlying NumPy array and store it if it is required. for request in requests: # Perform inference on the request and append it to responses # list... # You must return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is optional. This function allows the model to perform any necessary clean ups before exit. """ print('Cleaning up...') ``` Every Python backend can implement four main functions: ### `auto_complete_config` `auto_complete_config` is called only once when loading the model assuming the server was not started with [`--disable-auto-complete-config`](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#auto-generated-model-configuration). Implementing this function is optional. No implementation of `auto_complete_config` will do nothing. This function can be used to set [`max_batch_size`]( https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#maximum-batch-size), [dynamic_batching]( https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher), [`input`]( https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#inputs-and-outputs) and [`output`]( https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#inputs-and-outputs) properties of the model using `set_max_batch_size`, `set_dynamic_batching`, `add_input`, and `add_output`. These properties will allow Triton to load the model with [minimal model configuration]( https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#minimal-model-configuration) in absence of a configuration file. This function returns the `pb_utils.ModelConfig` object with these properties. You can use the `as_dict` function to gain read-only access to the `pb_utils.ModelConfig` object. The `pb_utils.ModelConfig` object being returned from here will be used as the final configuration for the model. In addition to minimal properties, you can also set [model_transaction_policy]( https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#model-transaction-policy) through `auto_complete_config` using `set_model_transaction_policy`. For example, ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: @staticmethod def auto_complete_config(auto_complete_model_config): ... transaction_policy = {"decoupled": True} auto_complete_model_config.set_model_transaction_policy(transaction_policy) ... ``` Note: The Python interpreter used to invoke this function will be destroyed upon returning from this function and as a result none of the objects created here will be available in the `initialize`, `execute`, or `finalize` functions. ### `initialize` `initialize` is called once the model is being loaded. Implementing `initialize` is optional. `initialize` allows you to do any necessary initializations before execution. In the `initialize` function, you are given an `args` variable. `args` is a Python dictionary. Both keys and values for this Python dictionary are strings. You can find the available keys in the `args` dictionary along with their description in the table below: | key | description | | ------------------------ | ------------------------------------------------ | | model_config | A JSON string containing the model configuration | | model_instance_kind | A string containing model instance kind | | model_instance_device_id | A string containing model instance device ID | | model_repository | Model repository path | | model_version | Model version | | model_name | Model name | ### `execute` `execute` function is called whenever an inference request is made. Every Python model must implement `execute` function. In the `execute` function you are given a list of `InferenceRequest` objects. There are two modes of implementing this function. The mode you choose should depend on your use case. That is whether or not you want to return decoupled responses from this model or not. #### Default Mode This is the most generic way you would like to implement your model and requires the `execute` function to return exactly one response per request. This entails that in this mode, your `execute` function must return a list of `InferenceResponse` objects that has the same length as `requests`. The work flow in this mode is: * `execute` function receives a batch of pb_utils.InferenceRequest as a length N array. * Perform inference on the pb_utils.InferenceRequest and append the corresponding pb_utils.InferenceResponse to a response list. * Return back the response list. * The length of response list being returned must be N. * Each element in the list should be the response for the corresponding element in the request array. * Each element must contain a response (a response can be either output tensors or an error); an element cannot be None. Triton checks to ensure that these requirements on response list are satisfied and if not returns an error response for all inference requests. Upon return from the execute function all tensor data associated with the InferenceRequest objects passed to the function are deleted, and so InferenceRequest objects should not be retained by the Python model. Starting from 24.06, models may choose to send the response using the `InferenceResponseSender` as illustrated on [Decoupled mode](#decoupled-mode). Since the model is in default mode, it must send exactly one response per request. The `pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL` flag must be sent either with the response or as a flag only response afterward. #### Error Handling In case one of the requests has an error, you can use the `TritonError` object to set the error message for that specific request. Below is an example of setting errors for an `InferenceResponse` object: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: ... def execute(self, requests): responses = [] for request in requests: if an_error_occurred: # If there is an error, there is no need to pass the # "output_tensors" to the InferenceResponse. The "output_tensors" # that are passed in this case will be ignored. responses.append(pb_utils.InferenceResponse( error=pb_utils.TritonError("An Error Occurred"))) return responses ``` Starting from 23.09, `pb_utils.TritonError` may be constructed with an optional Triton error code on the second parameter. For example: ```python pb_utils.TritonError("The file is not found", pb_utils.TritonError.NOT_FOUND) ``` If no code is specified, `pb_utils.TritonError.INTERNAL` will be used by default. Supported error codes: * `pb_utils.TritonError.UNKNOWN` * `pb_utils.TritonError.INTERNAL` * `pb_utils.TritonError.NOT_FOUND` * `pb_utils.TritonError.INVALID_ARG` * `pb_utils.TritonError.UNAVAILABLE` * `pb_utils.TritonError.UNSUPPORTED` * `pb_utils.TritonError.ALREADY_EXISTS` * `pb_utils.TritonError.CANCELLED` (since 23.10) #### Request Cancellation Handling One or more requests may be cancelled by the client during execution. Starting from 23.10, `request.is_cancelled()` returns whether the request is cancelled or not. For example: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: ... def execute(self, requests): responses = [] for request in requests: if request.is_cancelled(): responses.append(pb_utils.InferenceResponse( error=pb_utils.TritonError("Message", pb_utils.TritonError.CANCELLED))) else: ... return responses ``` Although checking for request cancellation is optional, it is recommended to check for cancellation at strategic request execution stages that can early terminate the execution in the event of its response is no longer needed. #### Decoupled mode This mode allows user to send multiple responses for a request or not send any responses for a request. A model may also send responses out-of-order relative to the order that the request batches are executed. Such models are called *decoupled* models. In order to use this mode, the [transaction policy](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#model-transaction-policy) in the model configuration must be set to decoupled. In decoupled mode, model must use `InferenceResponseSender` object per request to keep creating and sending any number of responses for the request. The workflow in this mode may look like: * `execute` function receives a batch of pb_utils.InferenceRequest as a length N array. * Iterate through each pb_utils.InferenceRequest and perform for the following steps for each pb_utils.InferenceRequest object: 1. Get `InferenceResponseSender` object for the InferenceRequest using InferenceRequest.get_response_sender(). 2. Create and populate pb_utils.InferenceResponse to be sent back. 3. Use InferenceResponseSender.send() to send the above response. If this is the last request then pass pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL as a flag with InferenceResponseSender.send(). Otherwise continue with Step 1 for sending next request. * The return value for `execute` function in this mode should be None. Similar to above, in case one of the requests has an error, you can use the `TritonError` object to set the error message for that specific request. After setting errors for an pb_utils.InferenceResponse object, use InferenceResponseSender.send() to send response with the error back to the user. Starting from 23.10, request cancellation can be checked directly on the `InferenceResponseSender` object using `response_sender.is_cancelled()`. Sending the TRITONSERVER_RESPONSE_COMPLETE_FINAL flag at the end of response is still needed even the request is cancelled. ##### Use Cases The decoupled mode is powerful and supports various other use cases: * If the model should not send any response for the request, then call InferenceResponseSender.send() with no response but flag parameter set to pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL. * The model can also send responses out-of-order in which it received requests. * The request data and `InferenceResponseSender` object can be passed to a separate thread in the model. This means main caller thread can exit from `execute` function and the model can still continue generating responses as long as it holds `InferenceResponseSender` object. The [decoupled examples](examples/decoupled/README.md) demonstrate full power of what can be achieved from decoupled API. Read [Decoupled Backends and Models](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/decoupled_models.md) for more details on how to host a decoupled model. ##### Async Execute Starting from 24.04, `async def execute(self, requests):` is supported for decoupled Python models. Its coroutine will be executed by an AsyncIO event loop shared with requests executing in the same model instance. The next request for the model instance can start executing while the current request is waiting. This is useful for minimizing the number of model instances for models that spend the majority of its time waiting, given requests can be executed concurrently by AsyncIO. To take full advantage of the concurrency, it is vital for the async execute function to not block the event loop from making progress while it is waiting, i.e. downloading over the network. Notes: * The model should not modify the running event loop, as this might cause unexpected issues. * The server/backend do not control how many requests are added to the event loop by a model instance. #### Request Rescheduling Starting from 23.11, Python backend supports request rescheduling. By calling the `set_release_flags` function on the request object with the flag `pb_utils.TRITONSERVER_REQUEST_RELEASE_RESCHEDULE`, you can reschedule the request for further execution in a future batch. This feature is useful for handling iterative sequences. The model config must be configured to enable iterative sequence batching in order to use the request rescheduling API: ``` sequence_batching { iterative_sequence : true } ``` For non-decoupled models, there can only be one response for each request. Since the rescheduled request is the same as the original, you must append a `None` object to the response list for the rescheduled request. For example: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: ... def execute(self, requests): responses = [] for request in requests: # Explicitly reschedule the first request if self.idx == 0: request.set_release_flags( pb_utils.TRITONSERVER_REQUEST_RELEASE_RESCHEDULE ) responses.append(None) self.idx += 1 else: responses.append(inference_response) return responses ``` For decoupled models, it is required to reschedule a request *before* returning from the `execute` function. Below is an example of a decoupled model using request rescheduling. This model takes 1 input tensor, an INT32 [ 1 ] input named "IN", and produces an output tensor "OUT" with the same shape as the input tensor. The input value indicates the total number of responses to be generated and the output value indicates the number of remaining responses. For example, if the request input has value 2, the model will: - Send a response with value 1. - Release request with RESCHEDULE flag. - When execute on the same request, send the last response with value 0. - Release request with ALL flag. ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: ... def execute(self, requests): responses = [] for request in requests: in_input = pb_utils.get_input_tensor_by_name(request, "IN").as_numpy() if self.reset_flag: self.remaining_response = in_input[0] self.reset_flag = False response_sender = request.get_response_sender() self.remaining_response -= 1 out_output = pb_utils.Tensor( "OUT", np.array([self.remaining_response], np.int32) ) response = pb_utils.InferenceResponse(output_tensors=[out_output]) if self.remaining_response <= 0: response_sender.send( response, flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL ) self.reset_flag = True else: request.set_release_flags( pb_utils.TRITONSERVER_REQUEST_RELEASE_RESCHEDULE ) response_sender.send(response) return None ``` ### `finalize` Implementing `finalize` is optional. This function allows you to do any clean ups necessary before the model is unloaded from Triton server. You can look at the [add_sub example](examples/add_sub/model.py) which contains a complete example of implementing all these functions for a Python model that adds and subtracts the inputs given to it. After implementing all the necessary functions, you should save this file as `model.py`. ## Model Config File Every Python Triton model must provide a `config.pbtxt` file describing the model configuration. In order to use this backend you must set the `backend` field of your model `config.pbtxt` file to `python`. You shouldn't set `platform` field of the configuration. Your models directory should look like below: ``` models └── add_sub ├── 1 │ └── model.py └── config.pbtxt ``` ## Inference Request Parameters You can retrieve the parameters associated with an inference request using the `inference_request.parameters()` function. This function returns a JSON string where the keys are the keys of the parameters object and the values are the values for the parameters field. Note that you need to parse this string using `json.loads` to convert it to a dictionary. Starting from 23.11 release, parameters may be provided to the `InferenceRequest` object during construction. The parameters should be a dictionary of key value pairs, where keys are `str` and values are `bool`, `int` or `str`. ```python request = pb_utils.InferenceRequest(parameters={"key": "value"}, ...) ``` You can read more about the inference request parameters in the [parameters extension](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_parameters.md) documentation. ## Managing Python Runtime and Libraries Python backend shipped in the [NVIDIA GPU Cloud](https://ngc.nvidia.com/) containers uses Python 3.10. Python backend is able to use the libraries that exist in the current Python environment. These libraries can be installed in a virtualenv, conda environment, or the global system Python. These libraries will only be used if the Python version matches the Python version of the Python backend's stub executable. For example, if you install a set of libraries in a Python 3.9 environment and your Python backend stub is compiled with Python 3.10 these libraries will NOT be available in your Python model served using Triton. You would need to compile the stub executable with Python 3.9 using the instructions in [Building Custom Python Backend Stub](#building-custom-python-backend-stub) section. ### Building Custom Python Backend Stub **Important Note: You only need to compile a custom Python backend stub if the Python version is different from Python 3.10 which is shipped by default in the Triton containers.** Python backend uses a *stub* process to connect your `model.py` file to the Triton C++ core. This stub process dynamically links to a specific `libpython..so` version. If you intend to use a Python interpreter with different version from the default Python backend stub, you need to compile your own Python backend stub by following the steps below: 1. Install the software packages below: * [cmake](https://cmake.org) * rapidjson and libarchive (instructions for installing these packages in Ubuntu or Debian are included in [Building from Source Section](#building-from-source)) 2. Make sure that the expected Python version is available in your environment. If you are using `conda`, you should make sure to activate the environment by `conda activate `. Note that you don't have to use `conda` and can install Python however you wish. Python backend relies on [pybind11](https://github.com/pybind/pybind11) to find the correct Python version. If you noticed that the correct Python version is not picked up, you can read more on how [pybind11 decides which Python to use](https://pybind11.readthedocs.io/en/stable/faq.html?highlight=cmake#cmake-doesn-t-detect-the-right-python-version). 3. Clone the Python backend repository and compile the Python backend stub (replace \ with the branch name that you want to use, for release branches it should be r\): ```bash git clone https://github.com/triton-inference-server/python_backend -b cd python_backend mkdir build && cd build cmake -DTRITON_ENABLE_GPU=ON -DTRITON_BACKEND_REPO_TAG= -DTRITON_COMMON_REPO_TAG= -DTRITON_CORE_REPO_TAG= -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install .. make triton-python-backend-stub ``` Now, you have access to a Python backend stub with your Python version. You can verify that using `ldd`: ``` ldd triton_python_backend_stub ... libpython3.6m.so.1.0 => /home/ubuntu/envs/miniconda3/envs/python-3-6/lib/libpython3.6m.so.1.0 (0x00007fbb69cf3000) ... ``` There are many other shared libraries printed in addition to the library posted above. However, it is important to see `libpython.m.so.1.0` in the list of linked shared libraries. If you use a different Python version, you should see that version instead. You need to copy the `triton_python_backend_stub` to the model directory of the models that want to use the custom Python backend stub. For example, if you have `model_a` in your [model repository](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md), the folder structure should look like below: ``` models |-- model_a |-- 1 | |-- model.py |-- config.pbtxt `-- triton_python_backend_stub ``` Note the location of `triton_python_backend_stub` in the directory structure above. ### Creating Custom Execution Environments If you want to create a tar file that contains all your Python dependencies or you want to use different Python environments for each Python model you need to create a *Custom Execution Environment* in Python backend. Currently, Python backend supports [conda-pack](https://conda.github.io/conda-pack/) for this purpose. [conda-pack](https://conda.github.io/conda-pack/) ensures that your conda environment is portable. You can create a tar file for your conda environment using `conda-pack` command: ``` conda-pack Collecting packages... Packing environment at '/home/iman/miniconda3/envs/python-3-6' to 'python-3-6.tar.gz' [########################################] | 100% Completed | 4.5s ``` **Important Note:** Before installing the packages in your conda environment, make sure that you have exported [`PYTHONNOUSERSITE`](https://docs.python.org/3/using/cmdline.html#envvar-PYTHONNOUSERSITE) environment variable: ``` export PYTHONNOUSERSITE=True ``` If this variable is not exported and similar packages are installed outside your conda environment, your tar file may not contain all the dependencies required for an isolated Python environment. Alternatively, Python backend also supports unpacked conda execution environments, given it points to an activation script to setup the conda environment. To do this, the execution environment can be first packed using [conda-pack](https://conda.github.io/conda-pack/) and then unpacked, or created using [conda create -p](https://docs.conda.io/projects/conda/en/latest/commands/create.html). In this case, the conda activation script is located in: ```$path_to_conda_pack/lib/python/site-packages/conda_pack/scripts/posix/activate``` This speeds up the server loading time for models. After creating the packed file from the conda environment or creating a conda environment with a custom activation script, you need to tell Python backend to use that environment for your model. You can do this by adding the lines below to the `config.pbtxt` file: ``` name: "model_a" backend: "python" ... parameters: { key: "EXECUTION_ENV_PATH", value: {string_value: "/home/iman/miniconda3/envs/python-3-6/python3.6.tar.gz"} } ``` It is also possible to provide the execution environment path relative to the model folder in model repository: ``` name: "model_a" backend: "python" ... parameters: { key: "EXECUTION_ENV_PATH", value: {string_value: "$$TRITON_MODEL_DIRECTORY/python3.6.tar.gz"} } ``` In this case, `python3.tar.gz` should be placed in the model folder and the model repository should look like below: ``` models |-- model_a | |-- 1 | | `-- model.py | |-- config.pbtxt | |-- python3.6.tar.gz | `-- triton_python_backend_stub ``` In the example above, `$$TRITON_MODEL_DIRECTORY` is resolved to `$pwd/models/model_a`. To accelerate the loading time of `model_a`, you can follow the steps below to unpack the conda environment in the model folder: ```bash mkdir -p $pwd/models/model_a/python3.6 tar -xvf $pwd/models/model_a/python3.6.tar.gz -C $pwd/models/model_a/python3.6 ``` Then you can change the `EXECUTION_ENV_PATH` to point to the unpacked directory: ``` parameters: { key: "EXECUTION_ENV_PATH", value: {string_value: "$$TRITON_MODEL_DIRECTORY/python3.6"} } ``` This is useful if you want to use S3, GCS, or Azure and you do not have access to the absolute path of the execution env that is stored in the cloud object storage service. ### Important Notes 1. The version of the Python interpreter in the execution environment must match the version of triton_python_backend_stub. 2. If you don't want to use a different Python interpreter, you can skip [Building Custom Python Backend Stub](#building-custom-python-backend-stub). In this case you only need to pack your environment using `conda-pack` and provide the path to tar file in the model config. However, the previous note still applies here and the version of the Python interpreter inside the conda environment must match the Python version of stub used by Python backend. The default version of the stub is Python 3.10. 3. You can share a single execution environment across multiple models. You need to provide the path to the tar file in the `EXECUTION_ENV_PATH` in the `config.pbtxt` of all the models that want to use the execution environment. 4. If `$$TRITON_MODEL_DIRECTORY` is used in the `EXECUTION_ENV_PATH`, the final `EXECUTION_ENV_PATH` **must not** escape from the `$$TRITON_MODEL_DIRECTORY`, as the behavior of accessing anywhere outside the `$$TRITON_MODEL_DIRECTORY` is **undefined**. 5. If a non-`$$TRITON_MODEL_DIRECTORY` `EXECUTION_ENV_PATH` is used, only local file system paths are currently supported. The behavior of using cloud paths is **undefined**. 6. If you need to compile the Python backend stub, it is recommended that you compile it in the official Triton NGC containers. Otherwise, your compiled stub may use dependencies that are not available in the Triton container that you are using for deployment. For example, compiling the Python backend stub on an OS other than Ubuntu 22.04 can lead to unexpected errors. 7. If you encounter the "GLIBCXX_3.4.30 not found" error during runtime, we recommend upgrading your conda version and installing `libstdcxx-ng=12` by running `conda install -c conda-forge libstdcxx-ng=12 -y`. If this solution does not resolve the issue, please feel free to open an issue on the [GitHub issue page](https://github.com/triton-inference-server/server/issues) following the provided [instructions](https://github.com/triton-inference-server/server#reporting-problems-asking-questions). ## Error Handling If there is an error that affects the `initialize`, `execute`, or `finalize` function of the Python model you can use `TritonInferenceException`. Example below shows how you can do error handling in `finalize`: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: ... def finalize(self): if error_during_finalize: raise pb_utils.TritonModelException( "An error occurred during finalize.") ``` ## Managing Shared Memory Starting from 21.04 release, Python backend uses shared memory to connect user's code to Triton. Note that this change is completely transparent and does not require any change to the existing user's model code. Python backend, by default, allocates 1 MB for each model instance. Then, it will grow the shared memory region by 1 MB chunks whenever an increase is required. You can configure the default shared memory used by each model instance using the `shm-default-byte-size` flag. The amount of shared memory growth can be configured using the `shm-growth-byte-size`. You can also configure the timeout used for connecting Triton main process to the Python backend stubs using the `stub-timeout-seconds`. The default value is 30 seconds. The config values described above can be passed to Triton using `--backend-config` flag: ``` /opt/tritonserver/bin/tritonserver --model-repository=`pwd`/models --backend-config=python,= ``` Also, if you are running Triton inside a Docker container you need to properly set the `--shm-size` flag depending on the size of your inputs and outputs. The default value for docker run command is `64MB` which is very small. ## Multiple Model Instance Support Python interpreter uses a global lock known as [GIL](https://docs.python.org/3/c-api/init.html#thread-state-and-the-global-interpreter-lock). Because of GIL, it is not possible have multiple threads running in the same Python interpreter simultaneously as each thread requires to acquire the GIL when accessing Python objects which will serialize all the operations. In order to work around this issue, Python backend spawns a separate process for each [model instance](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#multiple-model-instances). This is in contrast with how other Triton backends such as [ONNXRuntime](https://github.com/triton-inference-server/onnxruntime_backend), [TensorFlow](https://github.com/triton-inference-server/tensorflow_backend), and [PyTorch](https://github.com/triton-inference-server/pytorch_backend) handle multiple instances. Increasing the instance count for these backends will create additional threads instead of spawning separate processes. ## Running Multiple Instances of Triton Server Starting from 24.04 release, Python backend uses UUID to generate unique names for Python backend shared memory regions so that multiple instances of the server can run at the same time without any conflicts. If you're using a Python backend released before the 24.04 release, you need to specify different `shm-region-prefix-name` using the `--backend-config` flag to avoid conflicts between the shared memory regions. For example: ``` # Triton instance 1 tritonserver --model-repository=/models --backend-config=python,shm-region-prefix-name=prefix1 # Triton instance 2 tritonserver --model-repository=/models --backend-config=python,shm-region-prefix-name=prefix2 ``` Note that the hangs would only occur if the `/dev/shm` is shared between the two instances of the server. If you run the servers in different containers that don't share this location, you don't need to specify `shm-region-prefix-name`. # Business Logic Scripting Triton's [ensemble](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/architecture.md#ensemble-models) feature supports many use cases where multiple models are composed into a pipeline (or more generally a DAG, directed acyclic graph). However, there are many other use cases that are not supported because as part of the model pipeline they require loops, conditionals (if-then-else), data-dependent control-flow and other custom logic to be intermixed with model execution. We call this combination of custom logic and model executions *Business Logic Scripting (BLS)*. Starting from 21.08, you can implement BLS in your Python model. A new set of utility functions allows you to execute inference requests on other models being served by Triton as a part of executing your Python model. Note that BLS should only be used inside the `execute` function and is not supported in the `initialize` or `finalize` methods. Example below shows how to use this feature: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: ... def execute(self, requests): ... # Create an InferenceRequest object. `model_name`, # `requested_output_names`, and `inputs` are the required arguments and # must be provided when constructing an InferenceRequest object. Make # sure to replace `inputs` argument with a list of `pb_utils.Tensor` # objects. inference_request = pb_utils.InferenceRequest( model_name='model_name', requested_output_names=['REQUESTED_OUTPUT_1', 'REQUESTED_OUTPUT_2'], inputs=[]) # `pb_utils.InferenceRequest` supports request_id, correlation_id, # model version, timeout and preferred_memory in addition to the # arguments described above. # Note: Starting from the 24.03 release, the `correlation_id` parameter # supports both string and unsigned integer values. # These arguments are optional. An example containing all the arguments: # inference_request = pb_utils.InferenceRequest(model_name='model_name', # requested_output_names=['REQUESTED_OUTPUT_1', 'REQUESTED_OUTPUT_2'], # inputs=[], # request_id="1", correlation_id=4, model_version=1, flags=0, timeout=5, # preferred_memory=pb_utils.PreferredMemory( # pb_utils.TRITONSERVER_MEMORY_GPU, # or pb_utils.TRITONSERVER_MEMORY_CPU # 0)) # Execute the inference_request and wait for the response inference_response = inference_request.exec() # Check if the inference response has an error if inference_response.has_error(): raise pb_utils.TritonModelException( inference_response.error().message()) else: # Extract the output tensors from the inference response. output1 = pb_utils.get_output_tensor_by_name( inference_response, 'REQUESTED_OUTPUT_1') output2 = pb_utils.get_output_tensor_by_name( inference_response, 'REQUESTED_OUTPUT_2') # Decide the next steps for model execution based on the received # output tensors. It is possible to use the same output tensors # to for the final inference response too. ``` In addition to the `inference_request.exec` function that allows you to execute blocking inference requests, `inference_request.async_exec` allows you to perform async inference requests. This can be useful when you do not need the result of the inference immediately. Using `async_exec` function, it is possible to have multiple inflight inference requests and wait for the responses only when needed. Example below shows how to use `async_exec`: ```python import triton_python_backend_utils as pb_utils import asyncio class TritonPythonModel: ... # You must add the Python 'async' keyword to the beginning of `execute` # function if you want to use `async_exec` function. async def execute(self, requests): ... # Create an InferenceRequest object. `model_name`, # `requested_output_names`, and `inputs` are the required arguments and # must be provided when constructing an InferenceRequest object. Make # sure to replace `inputs` argument with a list of `pb_utils.Tensor` # objects. inference_request = pb_utils.InferenceRequest( model_name='model_name', requested_output_names=['REQUESTED_OUTPUT_1', 'REQUESTED_OUTPUT_2'], inputs=[]) infer_response_awaits = [] for i in range(4): # async_exec function returns an # [Awaitable](https://docs.python.org/3/library/asyncio-task.html#awaitables) # object. infer_response_awaits.append(inference_request.async_exec()) # Wait for all of the inference requests to complete. infer_responses = await asyncio.gather(*infer_response_awaits) for infer_response in infer_responses: # Check if the inference response has an error if inference_response.has_error(): raise pb_utils.TritonModelException( inference_response.error().message()) else: # Extract the output tensors from the inference response. output1 = pb_utils.get_output_tensor_by_name( inference_response, 'REQUESTED_OUTPUT_1') output2 = pb_utils.get_output_tensor_by_name( inference_response, 'REQUESTED_OUTPUT_2') # Decide the next steps for model execution based on the received # output tensors. ``` A complete example for sync and async BLS in Python backend is included in the [Examples](#examples) section. ## Using BLS with Decoupled Models Starting from 23.03 release, you can execute inference requests on decoupled models in both [default mode](#default-mode) and [decoupled mode](#decoupled-mode). By setting the `decoupled` parameter to `True`, the `exec` and `async_exec` function will return an [iterator](https://docs.python.org/3/glossary.html#term-iterator) of inference responses returned by a decoupled model. If the `decoupled` parameter is set to `False`, the `exec` and `async_exec` function will return a single response as shown in the example above. Besides, you can set the timeout via the parameter 'timeout' in microseconds within the constructor of `InferenceRequest`. If the request times out, the request will respond with an error. The default of 'timeout' is 0 which indicates that the request has no timeout. Additionally, starting from the 23.04 release, you have the flexibility to select a specific device to receive output tensors from BLS calls. This can be achieved by setting the optional `preferred_memory` parameter within the `InferenceRequest` constructor. To do this, you can create a `PreferredMemory` object and specify the `preferred_memory_type` as either `TRITONSERVER_MEMORY_GPU` or `TRITONSERVER_MEMORY_CPU`, as well as the `preferred_device_id` as an integer to indicate the memory type and device ID on which you wish to receive output tensors. If you do not specify the `preferred_memory` parameter, the output tensors will be allocated on the same device where the output tensors were received from the model to which the BLS call is made. Example below shows how to use this feature: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: ... def execute(self, requests): ... # Create an InferenceRequest object. `model_name`, # `requested_output_names`, and `inputs` are the required arguments and # must be provided when constructing an InferenceRequest object. Make # sure to replace `inputs` argument with a list of `pb_utils.Tensor` # objects. inference_request = pb_utils.InferenceRequest( model_name='model_name', requested_output_names=['REQUESTED_OUTPUT_1', 'REQUESTED_OUTPUT_2'], inputs=[]) # `pb_utils.InferenceRequest` supports request_id, correlation_id, # model version, timeout and preferred_memory in addition to the # arguments described above. # Note: Starting from the 24.03 release, the `correlation_id` parameter # supports both string and unsigned integer values. # These arguments are optional. An example containing all the arguments: # inference_request = pb_utils.InferenceRequest(model_name='model_name', # requested_output_names=['REQUESTED_OUTPUT_1', 'REQUESTED_OUTPUT_2'], # inputs=[], # request_id="1", correlation_id="ex-4", model_version=1, flags=0, timeout=5, # preferred_memory=pb_utils.PreferredMemory( # pb_utils.TRITONSERVER_MEMORY_GPU, # or pb_utils.TRITONSERVER_MEMORY_CPU # 0)) # Execute the inference_request and wait for the response. Here we are # running a BLS request on a decoupled model, hence setting the parameter # 'decoupled' to 'True'. inference_responses = inference_request.exec(decoupled=True) for inference_response in inference_responses: # Check if the inference response has an error if inference_response.has_error(): raise pb_utils.TritonModelException( inference_response.error().message()) # For some models, it is possible that the last response is empty if len(infer_response.output_tensors()) > 0: # Extract the output tensors from the inference response. output1 = pb_utils.get_output_tensor_by_name( inference_response, 'REQUESTED_OUTPUT_1') output2 = pb_utils.get_output_tensor_by_name( inference_response, 'REQUESTED_OUTPUT_2') # Decide the next steps for model execution based on the received # output tensors. It is possible to use the same output tensors to # for the final inference response too. ``` In addition to the `inference_request.exec(decoupled=True)` function that allows you to execute blocking inference requests on decoupled models, `inference_request.async_exec(decoupled=True)` allows you to perform async inference requests. This can be useful when you do not need the result of the inference immediately. Using `async_exec` function, it is possible to have multiple inflight inference requests and wait for the responses only when needed. Example below shows how to use `async_exec`: ```python import triton_python_backend_utils as pb_utils import asyncio class TritonPythonModel: ... # You must add the Python 'async' keyword to the beginning of `execute` # function if you want to use `async_exec` function. async def execute(self, requests): ... # Create an InferenceRequest object. `model_name`, # `requested_output_names`, and `inputs` are the required arguments and # must be provided when constructing an InferenceRequest object. Make # sure to replace `inputs` argument with a list of `pb_utils.Tensor` # objects. inference_request = pb_utils.InferenceRequest( model_name='model_name', requested_output_names=['REQUESTED_OUTPUT_1', 'REQUESTED_OUTPUT_2'], inputs=[]) infer_response_awaits = [] for i in range(4): # async_exec function returns an # [Awaitable](https://docs.python.org/3/library/asyncio-task.html#awaitables) # object. infer_response_awaits.append( inference_request.async_exec(decoupled=True)) # Wait for all of the inference requests to complete. async_responses = await asyncio.gather(*infer_response_awaits) for infer_responses in async_responses: for infer_response in infer_responses: # Check if the inference response has an error if inference_response.has_error(): raise pb_utils.TritonModelException( inference_response.error().message()) # For some models, it is possible that the last response is empty if len(infer_response.output_tensors()) > 0: # Extract the output tensors from the inference response. output1 = pb_utils.get_output_tensor_by_name( inference_response, 'REQUESTED_OUTPUT_1') output2 = pb_utils.get_output_tensor_by_name( inference_response, 'REQUESTED_OUTPUT_2') # Decide the next steps for model execution based on the received # output tensors. ``` A complete example for sync and async BLS for decoupled models is included in the [Examples](#examples) section. Starting from the 22.04 release, the lifetime of the BLS output tensors have been improved such that if a tensor is no longer needed in your Python model it will be automatically deallocated. This can increase the number of BLS requests that you can execute in your model without running into the out of GPU or shared memory error. Note: Async BLS is not supported on Python 3.6 or lower due to the `async` keyword and `asyncio.run` being introduced in Python 3.7. ## Model Loading API Starting from 23.07 release, you can use the model loading API to load models required by your BLS model. The model loading API is equivalent to the Triton C API for loading models which are documented in [tritonserver.h](https://github.com/triton-inference-server/core/blob/main/include/triton/core/tritonserver.h). Below is an example of how to use the model loading API: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: def initialize(self, args): self.model_name="onnx_model" # Check if the model is ready, and load the model if it is not ready. # You can specify the model version in string format. The version is # optional, and if not provided, the server will choose a version based # on the model and internal policy. if not pb_utils.is_model_ready(model_name=self.model_name, model_version="1"): # Load the model from the model repository pb_utils.load_model(model_name=self.model_name) # Load the model with an optional override model config in JSON # representation. If provided, this config will be used for # loading the model. config = "{\"backend\":\"onnxruntime\", \"version_policy\":{\"specific\":{\"versions\":[1]}}}" pb_utils.load_model(model_name=self.model_name, config=config) # Load the mode with optional override files. The override files are # specified as a dictionary where the key is the file path (with # "file:" prefix) and the value is the file content as bytes. The # files will form the model directory that the model will be loaded # from. If specified, 'config' must be provided to be the model # configuration of the override model directory. with open('models/onnx_int32_int32_int32/1/model.onnx', 'rb') as file: data = file.read() files = {"file:1/model.onnx": data} pb_utils.load_model(model_name=self.model_name, config=config, files=files) def execute(self, requests): # Execute the model ... # If the model is no longer needed, you can unload it. You can also # specify whether the dependents of the model should also be unloaded by # setting the 'unload_dependents' parameter to True. The default value # is False. Need to be careful when unloading the model as it can affect # other model instances or other models that depend on it. pb_utils.unload_model(model_name=self.model_name, unload_dependents=True) ``` Note that the model loading API is only supported if the server is running in [explicit model control mode](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_management.md#model-control-mode-explicit). Additionally, the model loading API should only be used after the server has been running, which means that the BLS model should not be loaded during server startup. You can use different [client endpoints](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_model_repository.md) to load the model after the server has been started. The model loading API is currently not supported during the `auto_complete_config` and `finalize` functions. ## Using BLS with Stateful Models [Stateful models](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/architecture.md#stateful-models) require setting additional flags in the inference request to indicate the start and end of a sequence. The `flags` argument in the `pb_utils.InferenceRequest` object can be used to indicate whether the request is the first or last request in the sequence. An example indicating that the request is starting the sequence: ```python inference_request = pb_utils.InferenceRequest(model_name='model_name', requested_output_names=['REQUESTED_OUTPUT_1', 'REQUESTED_OUTPUT_2'], inputs=[], request_id="1", correlation_id=4, flags=pb_utils.TRITONSERVER_REQUEST_FLAG_SEQUENCE_START) ``` For indicating the ending of the sequence you can use the `pb_utils.TRITONSERVER_REQUEST_FLAG_SEQUENCE_END` flag. If the request is both starting and ending a sequence at the same time (i.e. the sequence has only a single request), you can use the bitwise OR operator to enable both of the flags: ``` flags = pb_utils.TRITONSERVER_REQUEST_FLAG_SEQUENCE_START | pb_utils.TRITONSERVER_REQUEST_FLAG_SEQUENCE_END ``` ## Limitation - You need to make sure that the inference requests performed as a part of your model do not create a circular dependency. For example, if model A performs an inference request on itself and there are no more model instances ready to execute the inference request, the model will block on the inference execution forever. - Async BLS is not supported when running a Python model in decoupled mode. # Interoperability and GPU Support Starting from 21.09 release, Python backend supports [DLPack](https://github.com/dmlc/dlpack) for zero-copy transfer of Python backend tensors to other frameworks. The methods below are added to the `pb_utils.Tensor` object to facilitate the same: ## `pb_utils.Tensor.to_dlpack() -> PyCapsule` This method can be called on existing instantiated tensors to convert a Tensor to DLPack. The code snippet below shows how this works with PyTorch: ```python from torch.utils.dlpack import from_dlpack import triton_python_backend_utils as pb_utils class TritonPythonModel: def execute(self, requests): ... input0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # We have converted a Python backend tensor to a PyTorch tensor without # making any copies. pytorch_tensor = from_dlpack(input0.to_dlpack()) ``` ## `pb_utils.Tensor.from_dlpack() -> Tensor` This static method can be used for creating a `Tensor` object from the DLPack encoding of the tensor. For example: ```python from torch.utils.dlpack import to_dlpack import torch import triton_python_backend_utils as pb_utils class TritonPythonModel: def execute(self, requests): ... pytorch_tensor = torch.tensor([1, 2, 3], device='cuda') # Create a Python backend tensor from the DLPack encoding of a PyTorch # tensor. input0 = pb_utils.Tensor.from_dlpack("INPUT0", to_dlpack(pytorch_tensor)) ``` Python backend allows tensors implementing [`__dlpack__`](https://data-apis.org/array-api/2022.12/API_specification/generated/array_api.array.__dlpack__.html) and [`__dlpack_device__`](https://data-apis.org/array-api/2022.12/API_specification/generated/array_api.array.__dlpack_device__.html) [interface](https://dmlc.github.io/dlpack/latest/python_spec.html) to be converted to Python backend tensors. For instance: ```python input0 = pb_utils.Tensor.from_dlpack("INPUT0", pytorch_tensor) ``` This method only supports contiguous Tensors that are in C-order. If the tensor is not C-order contiguous an exception will be raised. For python models with input or output tensors of type BFloat16 (BF16), the `as_numpy()` method is not supported, and the `from_dlpack` and `to_dlpack` methods must be used instead. ## `pb_utils.Tensor.is_cpu() -> bool` This function can be used to check whether a tensor is placed in CPU or not. ## Input Tensor Device Placement By default, the Python backend moves all input tensors to CPU before providing them to the Python model. Starting from 21.09, you can change this default behavior. By setting `FORCE_CPU_ONLY_INPUT_TENSORS` to "no", Triton will not move input tensors to CPU for the Python model. Instead, Triton will provide the input tensors to the Python model in either CPU or GPU memory, depending on how those tensors were last used. You cannot predict which memory will be used for each input tensor so your Python model must be able to handle tensors in both CPU and GPU memory. To enable this setting, you need to add this setting to the `parameters` section of model configuration: ``` parameters: { key: "FORCE_CPU_ONLY_INPUT_TENSORS" value: {string_value:"no"}} ``` # Frameworks Since Python Backend models can support most python packages, it is a common workflow for users to use Deep Learning Frameworks like PyTorch in their `model.py` implementation. This section will document some notes and FAQ about this workflow. > **Note** > > Using a deep learning framework/package in a Python Backend model is > not necessarily the same as using the corresponding Triton Backend > implementation. For example, the > [PyTorch Backend](https://github.com/triton-inference-server/pytorch_backend) > is different from using a Python Backend model that uses `import torch`. > If you are seeing significantly different results from a model executed by > the framework (ex: PyTorch) compared to the Python Backend model running the > same framework, some of the first things you should check is that the > framework versions being used and the input/output preparation are the same. ## PyTorch For a simple example of using PyTorch in a Python Backend model, see the [AddSubNet PyTorch example](#addsubnet-in-pytorch). ### PyTorch Determinism When running PyTorch code, you may notice slight differences in output values across runs or across servers depending on hardware, system load, driver, or even batch size. These differences are generally related to the selection of CUDA kernels used to execute the operations, based on the factors mentioned. For most intents and purposes, these differences aren't large enough to affect a model's final prediction. However, to understand where these differences come from, see this [doc](https://pytorch.org/docs/stable/notes/randomness.html). On Ampere devices and later, there is an optimization related to FP32 operations called [TensorFloat32 (TF32)](https://blogs.nvidia.com/blog/2020/05/14/tensorfloat-32-precision-format/). Typically this optimization will improve overall performance at the cost of minor precision loss, but similarly this precision loss is acceptable for most model predictions. For more info on TF32 in PyTorch and how to enable/disable it as needed, see [here](https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices). ## TensorFlow ### TensorFlow Determinism Similar to the PyTorch determinism section above, TensorFlow can have slight differences in outputs based on various factors like hardware, system configurations, or batch sizes due to the library's internal CUDA kernel selection process. For more information on improving the determinism of outputs in TensorFlow, see [here](https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_op_determinism). # Custom Metrics Starting from 23.05, you can utlize Custom Metrics API to register and collect custom metrics in the `initialize`, `execute`, and `finalize` functions of your Python model. The Custom Metrics API is the Python equivalent of the [TRITON C API custom metrics](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/metrics.md#custom-metrics) support. You will need to take the ownership of the custom metrics created through the APIs and must manage their lifetime. Note that a `MetricFamily` object should be deleted only after all the `Metric` objects under it are deleted if you'd like to explicitly delete the custom metrics objects. Example below shows how to use this feature: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: def initialize(self, args): # Create a MetricFamily object to report the latency of the model # execution. The 'kind' parameter must be either 'COUNTER', # 'GAUGE' or 'HISTOGRAM'. self.metric_family = pb_utils.MetricFamily( name="preprocess_latency_ns", description="Cumulative time spent pre-processing requests", kind=pb_utils.MetricFamily.COUNTER ) # Create a Metric object under the MetricFamily object. The 'labels' # is a dictionary of key-value pairs. self.metric = self.metric_family.Metric( labels={"model" : "model_name", "version" : "1"} ) def execute(self, requests): responses = [] for request in requests: # Pre-processing - time it to capture latency start_ns = time.time_ns() self.preprocess(request) end_ns = time.time_ns() # Update metric to track cumulative pre-processing latency self.metric.increment(end_ns - start_ns) ... print("Cumulative pre-processing latency:", self.metric.value()) return responses ``` You can look at the [custom_metrics example](examples/custom_metrics/model.py) which contains a complete example of demonstrating the Custom Metrics API for a Python model. # Examples For using the Triton Python client in these examples you need to install the [Triton Python Client Library](https://github.com/triton-inference-server/client#getting-the-client-libraries-and-examples). The Python client for each of the examples is in the `client.py` file. ## AddSub in NumPy There is no dependencies required for the AddSub NumPy example. Instructions on how to use this model is explained in the quick start section. You can find the files in [examples/add_sub](examples/add_sub). ## AddSubNet in PyTorch In order to use this model, you need to install PyTorch. We recommend using `pip` method mentioned in the [PyTorch website](https://pytorch.org/get-started/locally/). Make sure that PyTorch is available in the same Python environment as other dependencies. Alternatively, you can create a [Python Execution Environment](#creating-custom-execution-environments). You can find the files for this example in [examples/pytorch](examples/pytorch). ## AddSub in JAX The JAX example shows how to serve JAX in Triton using Python Backend. You can find the complete example instructions in [examples/jax](examples/jax/README.md). ## Business Logic Scripting The BLS example needs the dependencies required for both of the above examples. You can find the complete example instructions in [examples/bls](examples/bls/README.md) and [examples/bls_decoupled](examples/bls_decoupled/README.md). ## Preprocessing The Preprocessing example shows how to use Python Backend to do model preprocessing. You can find the complete example instructions in [examples/preprocessing](examples/preprocessing/README.md). ## Decoupled Models The examples of decoupled models shows how to develop and serve [decoupled models](#decoupled-mode) in Triton using Python backend. You can find the complete example instructions in [examples/decoupled](examples/decoupled/README.md). ## Model Instance Kind Triton model configuration allows users to provide kind to [instance group settings.](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#instance-groups) A python backend model can be written to respect the kind setting to control the execution of a model instance either on CPU or GPU. In the [model instance kind example](examples/instance_kind/README.md) we demonstrate how this can be achieved for your python model. ## Auto-complete config The auto-complete config example demonstrates how to use the `auto_complete_config` function to define [minimal model configuration](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#minimal-model-configuration) when a configuration file is not available. You can find the complete example instructions in [examples/auto_complete](examples/auto_complete/README.md). ## Custom Metrics The example shows how to use custom metrics API in Python Backend. You can find the complete example instructions in [examples/custom_metrics](examples/custom_metrics/README.md). # Running with Inferentia Please see the [README.md](https://github.com/triton-inference-server/python_backend/tree/main/inferentia/README.md) located in the python_backend/inferentia sub folder. # Logging Starting from 22.09 release, your Python model can log information using the following methods: ```python import triton_python_backend_utils as pb_utils class TritonPythonModel: def execute(self, requests): ... logger = pb_utils.Logger logger.log_info("Info Msg!") logger.log_warn("Warning Msg!") logger.log_error("Error Msg!") logger.log_verbose("Verbose Msg!") ``` *Note:* The logger can be defined and used in following class methods: * initialize * execute * finalize Log messages can also be sent with their log-level explcitiy specified: ```python # log-level options: INFO, WARNING, ERROR, VERBOSE logger.log("Specific Msg!", logger.INFO) ``` If no log-level is specified, this method will log INFO level messages. Note that the Triton server's settings determine which log messages appear within the server log. For example, if a model attempts to log a verbose-level message, but Triton is not set to log verbose-level messages, it will not appear in the server log. For more information on Triton's log settings and how to adjust them dynamically, please see Triton's [logging extension](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_logging.md) documentation. # Adding Custom Parameters in the Model Configuration If your model requires custom parameters in the configuration, you can specify that in the `parameters` section of the model config. For example: ``` parameters { key: "custom_key" value: { string_value: "custom_value" } } ``` Now you can access this parameter in the `args` argument of the `initialize` function: ```python def initialize(self, args): print(json.loads(args['model_config'])['parameters']) # Should print {'custom_key': {'string_value': 'custom_value'}} ``` # Development with VSCode The repository includes a `.devcontainer` folder that contains a `Dockerfile` and `devcontainer.json` file to help you develop the Python backend using [Visual Studio Code](https://code.visualstudio.com/docs/devcontainers/containers). In order to build the backend, you can execute the "Build Python Backend" task in the [VSCode tasks](https://code.visualstudio.com/docs/editor/tasks). This will build the Python backend and install the artifacts in `/opt/tritonserver/backends/python`. # Reporting problems, asking questions We appreciate any feedback, questions or bug reporting regarding this project. When help with code is needed, follow the process outlined in the Stack Overflow (https://stackoverflow.com/help/mcve) document. Ensure posted examples are: * minimal – use as little code as possible that still produces the same problem * complete – provide all parts needed to reproduce the problem. Check if you can strip external dependency and still show the problem. The less time we spend on reproducing problems the more time we have to fix it * verifiable – test the code you're about to provide to make sure it reproduces the problem. Remove all other problems that are not related to your request/question. kytensor-2.49.0.7/framework_backends/python_backend/cmake/000077500000000000000000000000001527052371200235345ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/cmake/TritonPythonBackendConfig.cmake.in000066400000000000000000000036371527052371200322330ustar00rootroot00000000000000 # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONPYTHONBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONPYTHONBACKEND_CMAKE_DIR}) if(NOT TARGET TritonPythonBackend::triton-python-backend) include("${TRITONPYTHONBACKEND_CMAKE_DIR}/TritonPythonBackendTargets.cmake") endif() set(TRITONPYTHONBACKEND_LIBRARIES TritonPythonBackend::triton-python-backend)kytensor-2.49.0.7/framework_backends/python_backend/examples/000077500000000000000000000000001527052371200242725ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/add_sub/000077500000000000000000000000001527052371200256735ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/add_sub/client.py000066400000000000000000000061611527052371200275270ustar00rootroot00000000000000# Copyright 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import tritonclient.http as httpclient from tritonclient.utils import * model_name = "add_sub" shape = [4] with httpclient.InferenceServerClient("localhost:8000") as client: input0_data = np.random.rand(*shape).astype(np.float32) input1_data = np.random.rand(*shape).astype(np.float32) inputs = [ httpclient.InferInput( "INPUT0", input0_data.shape, np_to_triton_dtype(input0_data.dtype) ), httpclient.InferInput( "INPUT1", input1_data.shape, np_to_triton_dtype(input1_data.dtype) ), ] inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs = [ httpclient.InferRequestedOutput("OUTPUT0"), httpclient.InferRequestedOutput("OUTPUT1"), ] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() output0_data = response.as_numpy("OUTPUT0") output1_data = response.as_numpy("OUTPUT1") print( "INPUT0 ({}) + INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output0_data ) ) print( "INPUT0 ({}) - INPUT1 ({}) = OUTPUT1 ({})".format( input0_data, input1_data, output1_data ) ) if not np.allclose(input0_data + input1_data, output0_data): print("add_sub example error: incorrect sum") sys.exit(1) if not np.allclose(input0_data - input1_data, output1_data): print("add_sub example error: incorrect difference") sys.exit(1) print("PASS: add_sub") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/add_sub/config.pbtxt000066400000000000000000000036021527052371200302240ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "add_sub" backend: "python" input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] input [ { name: "INPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/add_sub/model.py000066400000000000000000000142721527052371200273530ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args["model_config"]) # Get OUTPUT0 configuration output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") # Get OUTPUT1 configuration output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") # Convert Triton types to numpy types self.output0_dtype = pb_utils.triton_string_to_numpy( output0_config["data_type"] ) self.output1_dtype = pb_utils.triton_string_to_numpy( output1_config["data_type"] ) def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ output0_dtype = self.output0_dtype output1_dtype = self.output1_dtype responses = [] # Every Python backend must iterate over everyone of the requests # and create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # Get INPUT1 in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") out_0, out_1 = ( in_0.as_numpy() + in_1.as_numpy(), in_0.as_numpy() - in_1.as_numpy(), ) # Create output tensors. You need pb_utils.Tensor # objects to create pb_utils.InferenceResponse. out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(output0_dtype)) out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(output1_dtype)) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) inference_response = pb_utils.InferenceResponse( output_tensors=[out_tensor_0, out_tensor_1] ) responses.append(inference_response) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/auto_complete/000077500000000000000000000000001527052371200271325ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/auto_complete/README.md000066400000000000000000000114001527052371200304050ustar00rootroot00000000000000 # Auto-Complete Example This example shows how to implement [`auto_complete_config`](https://github.com/triton-inference-server/python_backend/#auto_complete_config) function in Python backend to provide [`max_batch_size`](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#maximum-batch-size), [`input`](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#inputs-and-outputs) and [`output`](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#inputs-and-outputs) properties. These properties will allow Triton to load the Python model with [Minimal Model Configuration](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#minimal-model-configuration) in absence of a configuration file. The [model repository](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md) should contain [nobatch_auto_complete](./nobatch_model.py), and [batch_auto_complete](./batch_model.py) models. The max_batch_size of [nobatch_auto_complete](./nobatch_model.py) model is set to zero, whereas the max_batch_size of [batch_auto_complete](./batch_model.py) model is set to 4. For models with a non-zero value of max_batch_size, the configuration can specify a different value of max_batch_size as long as it does not exceed the value set in the model file. The [nobatch_auto_complete](./nobatch_model.py) and [batch_auto_complete](./batch_model.py) models calculate the sum and difference of the `INPUT0` and `INPUT1` and put the results in `OUTPUT0` and `OUTPUT1` respectively. ## Deploying the Auto-Complete Models 1. Create the model repository: ```console mkdir -p models/nobatch_auto_complete/1/ mkdir -p models/batch_auto_complete/1/ # Copy the Python models cp examples/auto_complete/nobatch_model.py models/nobatch_auto_complete/1/model.py cp examples/auto_complete/batch_model.py models/batch_auto_complete/1/model.py ``` **Note that we don't need a model configuration file since Triton will use the auto-complete model configuration provided in the Python model.** 2. Start the tritonserver: ``` tritonserver --model-repository `pwd`/models ``` ## Running inferences on Nobatch and Batch models: Send inference requests using [client.py](./client.py). ``` python3 examples/auto_complete/client.py ``` You should see an output similar to the output below: ``` 'nobatch_auto_complete' configuration matches the expected auto complete configuration 'batch_auto_complete' configuration matches the expected auto complete configuration PASS: auto_complete ``` The [nobatch_model.py](./nobatch_model.py) and [batch_model.py](./batch_model.py) model files are heavily commented with explanations about how to utilize `set_max_batch_size`, `add_input`, and `add_output`functions to set `max_batch_size`, `input` and `output` properties of the model. ### Explanation of the Client Output For each model, the [client.py](./client.py) first requests the model configuration from Triton to validate if the model configuration has been registered as expected. The client then sends an inference request to verify whether the inference has run properly and the result is correct. kytensor-2.49.0.7/framework_backends/python_backend/examples/auto_complete/batch_model.py000066400000000000000000000232111527052371200317440ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ @staticmethod def auto_complete_config(auto_complete_model_config): """`auto_complete_config` is called only once when loading the model assuming the server was not started with `--disable-auto-complete-config`. Implementing this function is optional. No implementation of `auto_complete_config` will do nothing. This function can be used to set `max_batch_size`, `input` and `output` properties of the model using `set_max_batch_size`, `add_input`, and `add_output`. These properties will allow Triton to load the model with minimal model configuration in absence of a configuration file. This function returns the `pb_utils.ModelConfig` object with these properties. You can use the `as_dict` function to gain read-only access to the `pb_utils.ModelConfig` object. The `pb_utils.ModelConfig` object being returned from here will be used as the final configuration for the model. Note: The Python interpreter used to invoke this function will be destroyed upon returning from this function and as a result none of the objects created here will be available in the `initialize`, `execute`, or `finalize` functions. Parameters ---------- auto_complete_model_config : pb_utils.ModelConfig An object containing the existing model configuration. You can build upon the configuration given by this object when setting the properties for this model. Returns ------- pb_utils.ModelConfig An object containing the auto-completed model configuration """ inputs = [ {"name": "INPUT0", "data_type": "TYPE_FP32", "dims": [4]}, {"name": "INPUT1", "data_type": "TYPE_FP32", "dims": [4]}, ] outputs = [ {"name": "OUTPUT0", "data_type": "TYPE_FP32", "dims": [4]}, {"name": "OUTPUT1", "data_type": "TYPE_FP32", "dims": [4]}, ] # Demonstrate the usage of `as_dict`, `add_input`, `add_output`, # and `set_max_batch_size` functions. # Store the model configuration as a dictionary. config = auto_complete_model_config.as_dict() input_names = [] output_names = [] for input in config["input"]: input_names.append(input["name"]) for output in config["output"]: output_names.append(output["name"]) for input in inputs: # The name checking here is only for demonstrating the usage of # `as_dict` function. `add_input` will check for conflicts and # raise errors if an input with the same name already exists in # the configuration but has different data_type or dims property. if input["name"] not in input_names: auto_complete_model_config.add_input(input) for output in outputs: # The name checking here is only for demonstrating the usage of # `as_dict` function. `add_output` will check for conflicts and # raise errors if an output with the same name already exists in # the configuration but has different data_type or dims property. if output["name"] not in output_names: auto_complete_model_config.add_output(output) auto_complete_model_config.set_max_batch_size(4) auto_complete_model_config.set_dynamic_batching() return auto_complete_model_config def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args["model_config"]) # Get OUTPUT0 configuration output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") # Get OUTPUT1 configuration output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") # Convert Triton types to numpy types self.output0_dtype = pb_utils.triton_string_to_numpy( output0_config["data_type"] ) self.output1_dtype = pb_utils.triton_string_to_numpy( output1_config["data_type"] ) def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ output0_dtype = self.output0_dtype output1_dtype = self.output1_dtype responses = [] # Every Python backend must iterate over everyone of the requests # and create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # Get INPUT1 in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") out_0, out_1 = ( in_0.as_numpy() + in_1.as_numpy(), in_0.as_numpy() - in_1.as_numpy(), ) # Create output tensors. You need pb_utils.Tensor # objects to create pb_utils.InferenceResponse. out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(output0_dtype)) out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(output1_dtype)) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) inference_response = pb_utils.InferenceResponse( output_tensors=[out_tensor_0, out_tensor_1] ) responses.append(inference_response) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/auto_complete/client.py000066400000000000000000000070651527052371200307720ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import tritonclient.http as httpclient from tritonclient.utils import * nobatch_model_name = "nobatch_auto_complete" batch_model_name = "batch_auto_complete" def validate_ios(config, expected_ios, model_name): for io in config: for expected_io in expected_ios: if io["name"] == expected_io["name"]: if io["data_type"] != expected_io["data_type"]: print("model '" + model_name + "' has unexpected data_type") sys.exit(1) elif io["dims"] != expected_io["dims"]: print("model '" + model_name + "' has unexpected dims") sys.exit(1) if __name__ == "__main__": with httpclient.InferenceServerClient("localhost:8000") as client: expected_max_batch_size = { "nobatch_auto_complete": 0, "batch_auto_complete": 4, } expected_inputs = [ {"name": "INPUT0", "data_type": "TYPE_FP32", "dims": [4]}, {"name": "INPUT1", "data_type": "TYPE_FP32", "dims": [4]}, ] expected_outputs = [ {"name": "OUTPUT0", "data_type": "TYPE_FP32", "dims": [4]}, {"name": "OUTPUT1", "data_type": "TYPE_FP32", "dims": [4]}, ] models = [nobatch_model_name, batch_model_name] for model_name in models: # Validate the auto-complete model configuration model_config = client.get_model_config(model_name) if model_config["max_batch_size"] != expected_max_batch_size[model_name]: print("model '" + model_name + "' has unexpected max_batch_size") sys.exit(1) validate_ios(model_config["input"], expected_inputs, model_name) validate_ios(model_config["output"], expected_outputs, model_name) print( "'" + model_name + "' configuration matches the expected " + "auto complete configuration\n" ) print("PASS: auto_complete") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/auto_complete/nobatch_model.py000066400000000000000000000231171527052371200323060ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ @staticmethod def auto_complete_config(auto_complete_model_config): """`auto_complete_config` is called only once when loading the model assuming the server was not started with `--disable-auto-complete-config`. Implementing this function is optional. No implementation of `auto_complete_config` will do nothing. This function can be used to set `max_batch_size`, `input` and `output` properties of the model using `set_max_batch_size`, `add_input`, and `add_output`. These properties will allow Triton to load the model with minimal model configuration in absence of a configuration file. This function returns the `pb_utils.ModelConfig` object with these properties. You can use the `as_dict` function to gain read-only access to the `pb_utils.ModelConfig` object. The `pb_utils.ModelConfig` object being returned from here will be used as the final configuration for the model. Note: The Python interpreter used to invoke this function will be destroyed upon returning from this function and as a result none of the objects created here will be available in the `initialize`, `execute`, or `finalize` functions. Parameters ---------- auto_complete_model_config : pb_utils.ModelConfig An object containing the existing model configuration. You can build upon the configuration given by this object when setting the properties for this model. Returns ------- pb_utils.ModelConfig An object containing the auto-completed model configuration """ inputs = [ {"name": "INPUT0", "data_type": "TYPE_FP32", "dims": [4]}, {"name": "INPUT1", "data_type": "TYPE_FP32", "dims": [4]}, ] outputs = [ {"name": "OUTPUT0", "data_type": "TYPE_FP32", "dims": [4]}, {"name": "OUTPUT1", "data_type": "TYPE_FP32", "dims": [4]}, ] # Demonstrate the usage of `as_dict`, `add_input`, `add_output`, # and `set_max_batch_size` functions. # Store the model configuration as a dictionary. config = auto_complete_model_config.as_dict() input_names = [] output_names = [] for input in config["input"]: input_names.append(input["name"]) for output in config["output"]: output_names.append(output["name"]) for input in inputs: # The name checking here is only for demonstrating the usage of # `as_dict` function. `add_input` will check for conflicts and # raise errors if an input with the same name already exists in # the configuration but has different data_type or dims property. if input["name"] not in input_names: auto_complete_model_config.add_input(input) for output in outputs: # The name checking here is only for demonstrating the usage of # `as_dict` function. `add_output` will check for conflicts and # raise errors if an output with the same name already exists in # the configuration but has different data_type or dims property. if output["name"] not in output_names: auto_complete_model_config.add_output(output) auto_complete_model_config.set_max_batch_size(0) return auto_complete_model_config def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args["model_config"]) # Get OUTPUT0 configuration output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") # Get OUTPUT1 configuration output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") # Convert Triton types to numpy types self.output0_dtype = pb_utils.triton_string_to_numpy( output0_config["data_type"] ) self.output1_dtype = pb_utils.triton_string_to_numpy( output1_config["data_type"] ) def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ output0_dtype = self.output0_dtype output1_dtype = self.output1_dtype responses = [] # Every Python backend must iterate over everyone of the requests # and create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # Get INPUT1 in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") out_0, out_1 = ( in_0.as_numpy() + in_1.as_numpy(), in_0.as_numpy() - in_1.as_numpy(), ) # Create output tensors. You need pb_utils.Tensor # objects to create pb_utils.InferenceResponse. out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(output0_dtype)) out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(output1_dtype)) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) inference_response = pb_utils.InferenceResponse( output_tensors=[out_tensor_0, out_tensor_1] ) responses.append(inference_response) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/bls/000077500000000000000000000000001527052371200250525ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/bls/README.md000066400000000000000000000152521527052371200263360ustar00rootroot00000000000000 # BLS Example In this section we demonstrate an end-to-end example for [BLS](../../README.md#business-logic-scripting) in Python backend. The [model repository](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md) should contain [pytorch](../pytorch), [addsub](../add_sub). The [pytorch](../pytorch) and [addsub](../add_sub) models calculate the sum and difference of the `INPUT0` and `INPUT1` and put the results in `OUTPUT0` and `OUTPUT1` respectively. This example is broken into two sections. The first section demonstrates how to perform synchronous BLS requests and the second section shows how to execute asynchronous BLS requests. ## Synchronous BLS Requests The goal of sync BLS model is the same as [pytorch](../pytorch) and [addsub](../add_sub) models but the difference is that the BLS model will not calculate the sum and difference by itself. The sync BLS model will pass the input tensors to the [pytorch](../pytorch) or [addsub](../add_sub) models and return the responses of that model as the final response. The additional parameter `MODEL_NAME` determines which model will be used for calculating the final outputs. 1. Create the model repository: ```console mkdir -p models/add_sub/1 mkdir -p models/bls_sync/1 mkdir -p models/pytorch/1 # Copy the Python models cp examples/add_sub/model.py models/add_sub/1/ cp examples/add_sub/config.pbtxt models/add_sub/config.pbtxt cp examples/bls/sync_model.py models/bls_sync/1/model.py cp examples/bls/sync_config.pbtxt models/bls_sync/config.pbtxt cp examples/pytorch/model.py models/pytorch/1/ cp examples/pytorch/config.pbtxt models/pytorch/ ``` 2. Start the tritonserver: ``` tritonserver --model-repository `pwd`/models ``` 3. Send inference requests to server: ``` python3 examples/bls/sync_client.py ``` You should see an output similar to the output below: ``` =========='add_sub' model result========== INPUT0 ([0.34984654 0.6808792 0.6509772 0.6211422 ]) + INPUT1 ([0.37917137 0.9080451 0.60789365 0.33425143]) = OUTPUT0 ([0.7290179 1.5889243 1.2588708 0.9553937]) INPUT0 ([0.34984654 0.6808792 0.6509772 0.6211422 ]) - INPUT1 ([0.37917137 0.9080451 0.60789365 0.33425143]) = OUTPUT1 ([-0.02932483 -0.22716594 0.04308355 0.28689077]) =========='pytorch' model result========== INPUT0 ([0.34984654 0.6808792 0.6509772 0.6211422 ]) + INPUT1 ([0.37917137 0.9080451 0.60789365 0.33425143]) = OUTPUT0 ([0.7290179 1.5889243 1.2588708 0.9553937]) INPUT0 ([0.34984654 0.6808792 0.6509772 0.6211422 ]) - INPUT1 ([0.37917137 0.9080451 0.60789365 0.33425143]) = OUTPUT1 ([-0.02932483 -0.22716594 0.04308355 0.28689077]) =========='undefined' model result========== Failed to process the request(s) for model instance 'bls_0', message: TritonModelException: Failed for execute the inference request. Model 'undefined_model' is not ready. At: /tmp/python_backend/models/bls/1/model.py(110): execute ``` The [sync_model.py](./sync_model.py) model file is heavily commented with explanations about each of the function calls. ### Explanation of the Client Output The [client.py](./sync_client.py) sends three inference requests to the 'bls_sync' model with different values for the "MODEL_NAME" input. As explained earlier, "MODEL_NAME" determines the model name that the "bls" model will use for calculating the final outputs. In the first request, it will use the "add_sub" model and in the second request it will use the "pytorch" model. The third request uses an incorrect model name to demonstrate error handling during the inference request execution. ## Asynchronous BLS Requests In this section we explain how to send multiple BLS requests without waiting for their response. Asynchronous execution of BLS requests will not block your model execution and can lead to speedups under certain conditions. The `bls_async` model will perform two async BLS requests on the [pytorch](../pytorch) and [addsub](../add_sub) models. Then, it will wait until the inference requests on these models is completed. It will extract `OUTPUT0` from the [pytorch](../pytorch) and `OUTPUT1` from the [addsub](../add_sub) model to construct the final inference response object using these tensors. 1. Create the model repository: ```console mkdir -p models/add_sub/1 mkdir -p models/bls_async/1 mkdir -p models/pytorch/1 # Copy the Python models cp examples/add_sub/model.py models/add_sub/1/ cp examples/add_sub/config.pbtxt models/add_sub/ cp examples/bls/async_model.py models/bls_async/1/model.py cp examples/bls/async_config.pbtxt models/bls_async/config.pbtxt cp examples/pytorch/model.py models/pytorch/1/ cp examples/pytorch/config.pbtxt models/pytorch/ ``` 2. Start the tritonserver: ``` tritonserver --model-repository `pwd`/models ``` 3. Send inference requests to server: ``` python3 examples/bls/async_client.py ``` You should see an output similar to the output below: ``` INPUT0 ([0.72394824 0.45873794 0.4307444 0.07681174]) + INPUT1 ([0.34224355 0.8271524 0.5831284 0.904624 ]) = OUTPUT0 ([1.0661918 1.2858903 1.0138729 0.9814357]) INPUT0 ([0.72394824 0.45873794 0.4307444 0.07681174]) - INPUT1 ([0.34224355 0.8271524 0.5831284 0.904624 ]) = OUTPUT1 ([ 0.3817047 -0.36841443 -0.15238398 -0.82781225]) ``` kytensor-2.49.0.7/framework_backends/python_backend/examples/bls/async_client.py000066400000000000000000000061711527052371200301040ustar00rootroot00000000000000# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import tritonclient.http as httpclient from tritonclient.utils import * model_name = "bls_async" shape = [4] with httpclient.InferenceServerClient("localhost:8000") as client: input0_data = np.random.rand(*shape).astype(np.float32) input1_data = np.random.rand(*shape).astype(np.float32) inputs = [ httpclient.InferInput( "INPUT0", input0_data.shape, np_to_triton_dtype(input0_data.dtype) ), httpclient.InferInput( "INPUT1", input1_data.shape, np_to_triton_dtype(input1_data.dtype) ), ] inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs = [ httpclient.InferRequestedOutput("OUTPUT0"), httpclient.InferRequestedOutput("OUTPUT1"), ] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() output0_data = response.as_numpy("OUTPUT0") output1_data = response.as_numpy("OUTPUT1") print( "INPUT0 ({}) + INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output0_data ) ) print( "INPUT0 ({}) - INPUT1 ({}) = OUTPUT1 ({})".format( input0_data, input1_data, output1_data ) ) if not np.allclose(input0_data + input1_data, output0_data): print("BLS async example error: incorrect sum") sys.exit(1) if not np.allclose(input0_data - input1_data, output1_data): print("BLS async example error: incorrect difference") sys.exit(1) print("PASS: BLS Async") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/bls/async_config.pbtxt000066400000000000000000000036151527052371200306040ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "bls_async" backend: "python" input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] input [ { name: "INPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/bls/async_model.py000066400000000000000000000165271527052371200277340ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio import json # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = json.loads(args["model_config"]) # You must add the Python 'async' keyword to the beginning of `execute` # function if you want to use `async_exec` function. async def execute(self, requests): """`execute` must be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ responses = [] # Every Python backend must iterate over everyone of the requests # and create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # Get INPUT1 in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") # List of awaitables containing inflight inference responses. inference_response_awaits = [] for model_name in ["pytorch", "add_sub"]: # Create inference request object infer_request = pb_utils.InferenceRequest( model_name=model_name, requested_output_names=["OUTPUT0", "OUTPUT1"], inputs=[in_0, in_1], ) # Store the awaitable inside the array. We don't need # the inference response immediately so we do not `await` # here. inference_response_awaits.append(infer_request.async_exec()) # Wait for all the inference requests to finish. The execution # of the Python script will be blocked until all the awaitables # are resolved. inference_responses = await asyncio.gather(*inference_response_awaits) for infer_response in inference_responses: # Make sure that the inference response doesn't have an error. # If it has an error and you can't proceed with your model # execution you can raise an exception. if infer_response.has_error(): raise pb_utils.TritonModelException( infer_response.error().message() ) # Get the OUTPUT0 from the "pytorch" model inference response pytorch_output0_tensor = pb_utils.get_output_tensor_by_name( inference_responses[0], "OUTPUT0" ) # Get the OUTPUT1 from the "addsub" model inference response addsub_output1_tensor = pb_utils.get_output_tensor_by_name( inference_responses[1], "OUTPUT1" ) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) # # Because the infer_response of the models contains the final # outputs with correct output names, we can just pass the list # of outputs to the InferenceResponse object. inference_response = pb_utils.InferenceResponse( output_tensors=[pytorch_output0_tensor, addsub_output1_tensor] ) responses.append(inference_response) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/bls/sync_client.py000066400000000000000000000114201527052371200277340ustar00rootroot00000000000000# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import tritonclient.http as httpclient from tritonclient.utils import * model_name = "bls_sync" shape = [4] with httpclient.InferenceServerClient("localhost:8000") as client: input0_data = np.random.rand(*shape).astype(np.float32) input1_data = np.random.rand(*shape).astype(np.float32) inputs = [ httpclient.InferInput( "INPUT0", input0_data.shape, np_to_triton_dtype(input0_data.dtype) ), httpclient.InferInput( "INPUT1", input1_data.shape, np_to_triton_dtype(input1_data.dtype) ), httpclient.InferInput("MODEL_NAME", [1], np_to_triton_dtype(np.object_)), ] inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) # Will perform the inference request on the 'add_sub' model. inputs[2].set_data_from_numpy(np.array(["add_sub"], dtype=np.object_)) outputs = [ httpclient.InferRequestedOutput("OUTPUT0"), httpclient.InferRequestedOutput("OUTPUT1"), ] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() output0_data = response.as_numpy("OUTPUT0") output1_data = response.as_numpy("OUTPUT1") print("=========='add_sub' model result==========") print( "INPUT0 ({}) + INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output0_data ) ) print( "INPUT0 ({}) - INPUT1 ({}) = OUTPUT1 ({})".format( input0_data, input1_data, output1_data ) ) if not np.allclose(input0_data + input1_data, output0_data): print("BLS sync example error: incorrect sum") sys.exit(1) if not np.allclose(input0_data - input1_data, output1_data): print("BLS sync example error: incorrect difference") sys.exit(1) # Will perform the inference request on the pytorch model: inputs[2].set_data_from_numpy(np.array(["pytorch"], dtype=np.object_)) response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() output0_data = response.as_numpy("OUTPUT0") output1_data = response.as_numpy("OUTPUT1") print("\n") print("=========='pytorch' model result==========") print( "INPUT0 ({}) + INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output0_data ) ) print( "INPUT0 ({}) - INPUT1 ({}) = OUTPUT1 ({})".format( input0_data, input1_data, output1_data ) ) if not np.allclose(input0_data + input1_data, output0_data): print("BLS sync example error: incorrect sum") sys.exit(1) if not np.allclose(input0_data - input1_data, output1_data): print("BLS sync example error: incorrect difference") sys.exit(1) # Will perform the same inference request on an undefined model. This leads # to an exception: print("\n") print("=========='undefined' model result==========") try: inputs[2].set_data_from_numpy(np.array(["undefined_model"], dtype=np.object_)) _ = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) except InferenceServerException as e: print(e.message()) print("PASS: BLS Sync") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/bls/sync_config.pbtxt000066400000000000000000000037401527052371200304420ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "bls_sync" backend: "python" input [ { name: "MODEL_NAME" data_type: TYPE_STRING dims: [ 1 ] } ] input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] input [ { name: "INPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/bls/sync_model.py000066400000000000000000000144171527052371200275670ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = json.loads(args["model_config"]) def execute(self, requests): """`execute` must be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ responses = [] # Every Python backend must iterate over everyone of the requests # and create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # Get INPUT1 in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") # Get Model Name model_name = pb_utils.get_input_tensor_by_name(request, "MODEL_NAME") # Model Name string model_name_string = model_name.as_numpy()[0] # Create inference request object infer_request = pb_utils.InferenceRequest( model_name=model_name_string, requested_output_names=["OUTPUT0", "OUTPUT1"], inputs=[in_0, in_1], ) # Perform synchronous blocking inference request infer_response = infer_request.exec() # Make sure that the inference response doesn't have an error. If # it has an error and you can't proceed with your model execution # you can raise an exception. if infer_response.has_error(): raise pb_utils.TritonModelException(infer_response.error().message()) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) # # Because the infer_response of the models contains the final # outputs with correct output names, we can just pass the list # of outputs to the InferenceResponse object. inference_response = pb_utils.InferenceResponse( output_tensors=infer_response.output_tensors() ) responses.append(inference_response) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/bls_decoupled/000077500000000000000000000000001527052371200270765ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/bls_decoupled/README.md000066400000000000000000000137371527052371200303700ustar00rootroot00000000000000 # Example of using BLS with decoupled models In this section we demonstrate an end-to-end example for [BLS](../../README.md#business-logic-scripting) in Python backend. The [model repository](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md) should contain [square](../decoupled) model. The [square](../decoupled) model will send 'n' responses where 'n' is the value of input `IN`. For each response, output `OUT` will equal the value of `IN`. This example is broken into two sections. The first section demonstrates how to perform synchronous BLS requests and the second section shows how to execute asynchronous BLS requests. ## Synchronous BLS Requests with Decoupled Models The goal of `bls_decoupled_sync` model is to calculate the sum of the responses returned from the [square](../decoupled) model and return the summation as the final response. The value of input 'IN' will be passed as an input to the [square](../decoupled) model which determines how many responses the [square](../decoupled) model will generate. 1. Create the model repository: ```console mkdir -p models/bls_decoupled_sync/1 mkdir -p models/square_int32/1 # Copy the Python models cp examples/bls_decoupled/sync_model.py models/bls_decoupled_sync/1/model.py cp examples/bls_decoupled/sync_config.pbtxt models/bls_decoupled_sync/config.pbtxt cp examples/decoupled/square_model.py models/square_int32/1/model.py cp examples/decoupled/square_config.pbtxt models/square_int32/config.pbtxt ``` 2. Start the tritonserver: ``` tritonserver --model-repository `pwd`/models ``` 3. Send inference requests to server: ``` python3 examples/bls_decoupled/sync_client.py ``` You should see an output similar to the output below: ``` ==========model result========== The square value of [4] is [16] ==========model result========== The square value of [2] is [4] ==========model result========== The square value of [0] is [0] ==========model result========== The square value of [1] is [1] PASS: BLS Decoupled Sync ``` The [sync_model.py](./sync_model.py) model file is heavily commented with explanations about each of the function calls. ### Explanation of the Client Output The [client.py](./sync_client.py) sends 4 inference requests to the `bls_decoupled_sync` model with the input as: [4], [2], [0] and [1] respectively. In compliance with the behavior of the sync BLS model, it will expect the output to be the square value of the input. ## Asynchronous BLS Requests with Decoupled Models In this section we explain how to send multiple BLS requests without waiting for their response. Asynchronous execution of BLS requests will not block your model execution and can lead to speedups under certain conditions. The `bls_decoupled_async` model will perform two async BLS requests on the [square](../decoupled) model. Then, it will wait until the inference requests are completed. It will calculate the sum of the output `OUT` from the [square](../decoupled) model in both two requests to construct the final inference response object using these tensors. 1. Create the model repository: ```console mkdir -p models/bls_decoupled_async/1 mkdir -p models/square_int32/1 # Copy the Python models cp examples/bls_decoupled/async_model.py models/bls_decoupled_async/1/model.py cp examples/bls_decoupled/async_config.pbtxt models/bls_decoupled_async/config.pbtxt cp examples/decoupled/square_model.py models/square_int32/1/model.py cp examples/decoupled/square_config.pbtxt models/square_int32/config.pbtxt ``` 2. Start the tritonserver: ``` tritonserver --model-repository `pwd`/models ``` 3. Send inference requests to server: ``` python3 examples/bls_decoupled/async_client.py ``` You should see an output similar to the output below: ``` ==========model result========== Two times the square value of [4] is [32] ==========model result========== Two times the square value of [2] is [8] ==========model result========== Two times the square value of [0] is [0] ==========model result========== Two times the square value of [1] is [2] PASS: BLS Decoupled Async ``` The [async_model.py](./async_model.py) model file is heavily commented with explanations about each of the function calls. ### Explanation of the Client Output The [client.py](./async_client.py) sends 4 inference requests to the 'bls_decoupled_sync' model with the input as: [4], [2], [0] and [1] respectively. In compliance with the behavior of sync BLS model model, it will expect the output to be two time the square value of the input. kytensor-2.49.0.7/framework_backends/python_backend/examples/bls_decoupled/async_client.py000066400000000000000000000055731527052371200321350ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import tritonclient.http as httpclient from tritonclient.utils import * model_name = "bls_decoupled_async" shape = [1] with httpclient.InferenceServerClient("localhost:8000") as client: in_values = [4, 2, 0, 1] for in_value in in_values: input_data = np.array([in_value], dtype=np.int32) inputs = [ httpclient.InferInput( "IN", input_data.shape, np_to_triton_dtype(input_data.dtype) ) ] inputs[0].set_data_from_numpy(input_data) outputs = [httpclient.InferRequestedOutput("SUM")] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() # output_data contains two times of the square value of the input value. output_data = response.as_numpy("SUM") print("==========model result==========") print( "Two times the square value of {} is {}\n".format(input_data, output_data) ) if not np.allclose((2 * input_data * input_data), output_data): print( "BLS Decoupled Async example error: incorrect output value. Expected {}, got {}.".format( (2 * input_data * input_data), output_data ) ) sys.exit(1) print("PASS: BLS Decoupled Async") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/bls_decoupled/async_config.pbtxt000066400000000000000000000033631527052371200326300ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "bls_decoupled_async" backend: "python" input [ { name: "IN" data_type: TYPE_INT32 dims: [ 1 ] } ] output [ { name: "SUM" data_type: TYPE_INT32 dims: [ 1 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/bls_decoupled/async_model.py000066400000000000000000000167731527052371200317630ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio import json import numpy as np # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. This model demonstrates how to use BLS with decoupled models. This model has a single input and a single output. The model does not support batching. - Input 'IN' shape must be equal to [1], datatype must be INT32. - For each response, output 'SUM' shape must be equal to [1], datatype must be INT32. For every request, the model will send a single response that contains an output named 'SUM'. We will send two BLS requests to the square model and the 'SUM' will contain the summation of the 'OUT' response output returned by the square model in the two BLS requests. The input 'IN' determines how many responses the square model will generate. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = json.loads(args["model_config"]) # You must add the Python 'async' keyword to the beginning of `execute` # function if you want to use `async_exec` function. async def execute(self, requests): """`execute` must be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ # This model does not support batching, so 'request_count' should # always be 1. if len(requests) != 1: raise pb_utils.TritonModelException( "unsupported batch size " + len(requests) ) response_num = pb_utils.get_input_tensor_by_name(requests[0], "IN") # List of awaitables containing inflight inference responses. inference_response_awaits = [] # For detailed explanation about the inputs of the repeat model, refer # to the example below: # https://github.com/triton-inference-server/python_backend/blob/r22.12/examples/decoupled/square_model.py # Construct two BLS requests for _ in range(2): infer_request = pb_utils.InferenceRequest( model_name="square_int32", inputs=[response_num], requested_output_names=["OUT"], ) # Store the awaitable inside the array. We don't need # the inference response immediately so we do not `await` # here. inference_response_awaits.append(infer_request.async_exec(decoupled=True)) # Wait for all the inference requests to finish. The execution # of the Python script will be blocked until all the awaitables # are resolved. async_responses = await asyncio.gather(*inference_response_awaits) # The variable that will store the sum of the responses. response_sum = np.array([0]) # Iterate over the list of generators of responses returned by the BLS # request. This interface can support zero, one, and many inference # responses per request. for infer_responses in async_responses: for infer_response in infer_responses: # If inference response has an error, raise an exception if infer_response.has_error(): raise pb_utils.TritonModelException( infer_response.error().message() ) # Check for the last empty response. if len(infer_response.output_tensors()) > 0: response_sum += pb_utils.get_output_tensor_by_name( infer_response, "OUT" ).as_numpy() response = [ pb_utils.InferenceResponse( output_tensors=[pb_utils.Tensor("SUM", response_sum)] ) ] # Since the model is using the default mode in this example, we # will be returning a single response. return response def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/bls_decoupled/sync_client.py000066400000000000000000000053251527052371200317670ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import tritonclient.http as httpclient from tritonclient.utils import * model_name = "bls_decoupled_sync" shape = [1] with httpclient.InferenceServerClient("localhost:8000") as client: in_values = [4, 2, 0, 1] for in_value in in_values: input_data = np.array([in_value], dtype=np.int32) inputs = [ httpclient.InferInput( "IN", input_data.shape, np_to_triton_dtype(input_data.dtype) ) ] inputs[0].set_data_from_numpy(input_data) outputs = [httpclient.InferRequestedOutput("SUM")] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() output_data = response.as_numpy("SUM") print("==========model result==========") print("The square value of {} is {}\n".format(input_data, output_data)) if not np.allclose(input_data * input_data, output_data): print( "BLS Decoupled Sync example error: incorrect output value. Expected {}, got {}." ).format(input_data * input_data, output_data) sys.exit(1) print("PASS: BLS Decoupled Sync") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/bls_decoupled/sync_config.pbtxt000066400000000000000000000033621527052371200324660ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "bls_decoupled_sync" backend: "python" input [ { name: "IN" data_type: TYPE_INT32 dims: [ 1 ] } ] output [ { name: "SUM" data_type: TYPE_INT32 dims: [ 1 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/bls_decoupled/sync_model.py000066400000000000000000000150761527052371200316150ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import numpy as np # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. This model demonstrates how to use BLS with decoupled models. This model has a single input and a single output. The model does not support batching. - Input 'IN' shape must be equal to [1], datatype must be INT32. - For each response, output 'SUM' shape must be equal to [1], datatype must be INT32. For every request, the model will send a single response that contains an output named 'SUM'. The 'SUM' will contain the summation of the 'OUT' response output returned by the square model. The input 'IN' determines how many responses the square model will generate. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = json.loads(args["model_config"]) def execute(self, requests): """`execute` must be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ # This model does not support batching, so 'request_count' should # always be 1. if len(requests) != 1: raise pb_utils.TritonModelException( "unsupported batch size " + len(requests) ) response_num = pb_utils.get_input_tensor_by_name(requests[0], "IN") # For detailed explanation about the inputs of the repeat model, refer # to the example below: # https://github.com/triton-inference-server/python_backend/blob/r22.12/examples/decoupled/square_model.py # Construct the BLS request infer_request = pb_utils.InferenceRequest( model_name="square_int32", inputs=[response_num], requested_output_names=["OUT"], ) # The variable that will store the sum of the responses. response_sum = np.array([0]) # Iterate over the generator of responses returned by the BLS request. # This interface can support zero, one, and many inference responses # per request. infer_responses = infer_request.exec(decoupled=True) for infer_response in infer_responses: # If inference response has an error, raise an exception if infer_response.has_error(): raise pb_utils.TritonModelException(infer_response.error().message()) # Check for the last empty response. if len(infer_response.output_tensors()) > 0: response_sum += pb_utils.get_output_tensor_by_name( infer_response, "OUT" ).as_numpy() response = [ pb_utils.InferenceResponse( output_tensors=[pb_utils.Tensor("SUM", response_sum)] ) ] # Since the model is using the default mode in this example, we # will be returning a single response. return response def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/custom_metrics/000077500000000000000000000000001527052371200273325ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/custom_metrics/README.md000066400000000000000000000066001527052371200306130ustar00rootroot00000000000000 # Custom Metrics Example In this section we demonstrate an end-to-end example for [Custom Metrics API](../../README.md#custom-metrics) in Python backend. The [model repository](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md) should contain [custom_metrics](./model.py) model. The [custom_metrics](./model.py) model uses [Custom Metrics API](../../README.md#custom-metrics) to register and collect custom metrics. ## Deploying the Custom Metrics Models 1. Create the model repository: ```console mkdir -p models/custom_metrics/1/ # Copy the Python models cp examples/custom_metrics/model.py models/custom_metrics/1/model.py cp examples/custom_metrics/config.pbtxt models/custom_metrics/config.pbtxt ``` 2. Start the tritonserver: ``` tritonserver --model-repository `pwd`/models ``` 3. Send inference requests to server: ``` python3 examples/custom_metrics/client.py ``` You should see an output similar to the output below in the client terminal: ``` custom_metrics example: found pattern '# HELP requests_process_latency_ns Cumulative time spent processing requests' in metrics custom_metrics example: found pattern '# TYPE requests_process_latency_ns counter' in metrics custom_metrics example: found pattern 'requests_process_latency_ns{model="custom_metrics",version="1"}' in metrics PASS: custom_metrics ``` In the terminal that runs Triton Server, you should see an output similar to the output below: ``` Cumulative requests processing latency: 223406.0 ``` The [model.py](./model.py) model file is heavily commented with explanations about each of the function calls. ### Explanation of the Client Output The [client.py](./client.py) sends a HTTP request with url `http://localhost:8002/metrics` to fetch the metrics from Triton server. The client then verifies if the custom metrics added in the model file are correctly reported. kytensor-2.49.0.7/framework_backends/python_backend/examples/custom_metrics/client.py000066400000000000000000000071631527052371200311710ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION& AFFILIATES.All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and / or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import requests import tritonclient.http as httpclient from tritonclient.utils import * model_name = "custom_metrics" shape = [4] def get_metrics(): metrics_url = "http://localhost:8002/metrics" r = requests.get(metrics_url) r.raise_for_status() return r.text with httpclient.InferenceServerClient("localhost:8000") as client: input0_data = np.random.rand(*shape).astype(np.float32) input1_data = np.random.rand(*shape).astype(np.float32) inputs = [ httpclient.InferInput( "INPUT0", input0_data.shape, np_to_triton_dtype(input0_data.dtype) ), httpclient.InferInput( "INPUT1", input1_data.shape, np_to_triton_dtype(input1_data.dtype) ), ] inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs = [ httpclient.InferRequestedOutput("OUTPUT0"), httpclient.InferRequestedOutput("OUTPUT1"), ] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) output0_data = response.as_numpy("OUTPUT0") output1_data = response.as_numpy("OUTPUT1") if not np.allclose(input0_data + input1_data, output0_data): print("custom_metrics example error: incorrect sum") sys.exit(1) if not np.allclose(input0_data - input1_data, output1_data): print("custom_metrics example error: incorrect difference") sys.exit(1) metrics = get_metrics() patterns = [ "# HELP requests_process_latency_ns Cumulative time spent processing requests", "# TYPE requests_process_latency_ns counter", 'requests_process_latency_ns{model="custom_metrics",version="1"}', ] for pattern in patterns: if pattern not in metrics: print( "custom_metrics example error: missing pattern '{}' in metrics".format( pattern ) ) sys.exit(1) else: print( "custom_metrics example: found pattern '{}' in metrics".format(pattern) ) print("PASS: custom_metrics") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/custom_metrics/config.pbtxt000066400000000000000000000036521527052371200316700ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "custom_metrics" backend: "python" input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] input [ { name: "INPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] instance_group [ { count: 3 kind: KIND_CPU } ] kytensor-2.49.0.7/framework_backends/python_backend/examples/custom_metrics/model.py000066400000000000000000000203621527052371200310070ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import time # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # Parse model_config and extract OUTPUT0 and OUTPUT1 configuration self.model_config = model_config = json.loads(args["model_config"]) output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") # Convert Triton types to numpy types self.out0_dtype = pb_utils.triton_string_to_numpy(output0_config["data_type"]) self.out1_dtype = pb_utils.triton_string_to_numpy(output1_config["data_type"]) # Create a MetricFamily object to report the latency of the model # execution. The 'kind' parameter must be either 'COUNTER' or # 'GAUGE'. # If duplicate name is used, both MetricFamily objects # will reference to the same underlying MetricFamily. If there are two # MetricFamily objects with the same name and same kind but different # description, the original description will be used. Note that # Duplicate name with different kind is not allowed. self.metric_family = pb_utils.MetricFamily( name="requests_process_latency_ns", description="Cumulative time spent processing requests", kind=pb_utils.MetricFamily.COUNTER, # or pb_utils.MetricFamily.GAUGE ) # Create a Metric object under the MetricFamily object. The 'labels' # is a dictionary of key-value pairs. You can create multiple Metric # objects under the same MetricFamily object with unique labels. Empty # labels is allowed. The 'labels' parameter is optional. If you don't # specify the 'labels' parameter, empty labels will be used. self.metric = self.metric_family.Metric( labels={"model": "custom_metrics", "version": "1"} ) def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ responses = [] # Record the start time of processing the requests start_ns = time.time_ns() # Every Python backend must iterate over everyone of the requests # and create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # Get INPUT1 in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") out_0, out_1 = ( in_0.as_numpy() + in_1.as_numpy(), in_0.as_numpy() - in_1.as_numpy(), ) # Create output tensors. You need pb_utils.Tensor # objects to create pb_utils.InferenceResponse. out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(self.out0_dtype)) out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(self.out1_dtype)) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) inference_response = pb_utils.InferenceResponse( output_tensors=[out_tensor_0, out_tensor_1] ) responses.append(inference_response) # Record the end time of processing the requests end_ns = time.time_ns() # Update metric to track cumulative requests processing latency. # There are three operations you can do with the Metric object: # - Metric.increment(value): Increment the value of the metric by # the given value. The type of the value is double. The 'COUNTER' # kind does not support negative value. # - Metric.set(value): Set the value of the metric to the given # value. This operation is only supported in 'GAUGE' kind. The # type of the value is double. # - Metric.value(): Get the current value of the metric. self.metric.increment(end_ns - start_ns) logger = pb_utils.Logger logger.log_info( "Cumulative requests processing latency: {}".format(self.metric.value()) ) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/decoupled/000077500000000000000000000000001527052371200262365ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/decoupled/README.md000066400000000000000000000170451527052371200275240ustar00rootroot00000000000000 # Decoupled Model Examples In this section we demonstrate an end-to-end examples for developing and serving [decoupled models](../../README.md#decoupled-mode) in Python backend. [repeat_model.py](repeat_model.py) and [square_model.py](square_model.py) demonstrate how to write a decoupled model where each request can generate 0 to many responses. These files are heavily commented to describe each function call. These example models are designed to show the flexibility available to decoupled models and in no way should be used in production. These examples circumvents the restriction placed by the [instance count](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#instance-groups) and allows multiple requests to be in process even for single instance. In real deployment, the model should not allow the caller thread to return from `execute` until that instance is ready to handle another set of requests. ## Deploying the Decoupled Models 1. Create the model repository: ```console mkdir -p models/repeat_int32/1 mkdir -p models/square_int32/1 # Copy the Python models cp examples/decoupled/repeat_model.py models/repeat_int32/1/model.py cp examples/decoupled/repeat_config.pbtxt models/repeat_int32/config.pbtxt cp examples/decoupled/square_model.py models/square_int32/1/model.py cp examples/decoupled/square_config.pbtxt models/square_int32/config.pbtxt ``` 2. Start the tritonserver: ``` tritonserver --model-repository `pwd`/models ``` ## Running inference on Repeat model: Send inference requests to repeat model using [repeat_client.py](repeat_client.py). ``` python3 examples/decoupled/repeat_client.py ``` You should see an output similar to the output below: ``` stream started... async_stream_infer model_name: "repeat_int32" id: "0" inputs { name: "IN" datatype: "INT32" shape: 4 } inputs { name: "DELAY" datatype: "UINT32" shape: 4 } inputs { name: "WAIT" datatype: "UINT32" shape: 1 } outputs { name: "OUT" } outputs { name: "IDX" } raw_input_contents: "\004\000\000\000\002\000\000\000\000\000\000\000\001\000\000\000" raw_input_contents: "\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000" raw_input_contents: "\005\000\000\000" enqueued request 0 to stream... infer_response { model_name: "repeat_int32" model_version: "1" id: "0" outputs { name: "IDX" datatype: "UINT32" shape: 1 } outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\000\000\000\000" raw_output_contents: "\004\000\000\000" } infer_response { model_name: "repeat_int32" model_version: "1" id: "0" outputs { name: "IDX" datatype: "UINT32" shape: 1 } outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\001\000\000\000" raw_output_contents: "\002\000\000\000" } infer_response { model_name: "repeat_int32" model_version: "1" id: "0" outputs { name: "IDX" datatype: "UINT32" shape: 1 } outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\002\000\000\000" raw_output_contents: "\000\000\000\000" } infer_response { model_name: "repeat_int32" model_version: "1" id: "0" outputs { name: "IDX" datatype: "UINT32" shape: 1 } outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\003\000\000\000" raw_output_contents: "\001\000\000\000" } PASS: repeat_int32 stream stopped... ``` Look how a single request generated 4 responses. ## Running inference on Square model: Send inference requests to square model using [square_client.py](square_client.py). ``` python3 examples/decoupled/square_client.py ``` You should see an output similar to the output below: ``` stream started... async_stream_infer model_name: "square_int32" id: "0" inputs { name: "IN" datatype: "INT32" shape: 1 } outputs { name: "OUT" } raw_input_contents: "\004\000\000\000" enqueued request 0 to stream... async_stream_infer model_name: "square_int32" id: "1" inputs { name: "IN" datatype: "INT32" shape: 1 } outputs { name: "OUT" } raw_input_contents: "\002\000\000\000" enqueued request 1 to stream... async_stream_infer model_name: "square_int32" id: "2" inputs { name: "IN" datatype: "INT32" shape: 1 } outputs { name: "OUT" } raw_input_contents: "\000\000\000\000" enqueued request 2 to stream... async_stream_infer model_name: "square_int32" id: "3" inputs { name: "IN" datatype: "INT32" shape: 1 } outputs { name: "OUT" } raw_input_contents: "\001\000\000\000" enqueued request 3 to stream... infer_response { model_name: "square_int32" model_version: "1" id: "0" outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\004\000\000\000" } infer_response { model_name: "square_int32" model_version: "1" id: "1" outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\002\000\000\000" } infer_response { model_name: "square_int32" model_version: "1" id: "0" outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\004\000\000\000" } infer_response { model_name: "square_int32" model_version: "1" id: "3" outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\001\000\000\000" } infer_response { model_name: "square_int32" model_version: "1" id: "1" outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\002\000\000\000" } infer_response { model_name: "square_int32" model_version: "1" id: "0" outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\004\000\000\000" } infer_response { model_name: "square_int32" model_version: "1" id: "0" outputs { name: "OUT" datatype: "INT32" shape: 1 } raw_output_contents: "\004\000\000\000" } PASS: square_int32 stream stopped... ``` Look how responses were delivered out-of-order of requests. The generated responses can be tracked to their request using the `id` field. kytensor-2.49.0.7/framework_backends/python_backend/examples/decoupled/repeat_client.py000066400000000000000000000107151527052371200314320ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import queue import sys from functools import partial import numpy as np import tritonclient.grpc as grpcclient from tritonclient.utils import * class UserData: def __init__(self): self._completed_requests = queue.Queue() def callback(user_data, result, error): if error: user_data._completed_requests.put(error) else: user_data._completed_requests.put(result) # This client sends a single request to the model with the # following tensor data. In compliance with the behavior # of repeat_int32 model, it will expect the 4 responses # with output: [4], [2], [0] and [1] respectively. model_name = "repeat_int32" in_value = [4, 2, 0, 1] delay_value = [1, 2, 3, 4] wait_value = 5 inputs = [] inputs.append(grpcclient.InferInput("IN", [len(in_value)], "INT32")) inputs.append(grpcclient.InferInput("DELAY", [len(delay_value)], "UINT32")) inputs.append(grpcclient.InferInput("WAIT", [1], "UINT32")) outputs = [] outputs.append(grpcclient.InferRequestedOutput("OUT")) outputs.append(grpcclient.InferRequestedOutput("IDX")) user_data = UserData() with grpcclient.InferenceServerClient( url="localhost:8001", verbose=True ) as triton_client: # Establish stream triton_client.start_stream(callback=partial(callback, user_data)) in_data = np.array(in_value, dtype=np.int32) inputs[0].set_data_from_numpy(in_data) delay_data = np.array(delay_value, dtype=np.uint32) inputs[1].set_data_from_numpy(delay_data) wait_data = np.array([wait_value], dtype=np.uint32) inputs[2].set_data_from_numpy(wait_data) request_id = "0" triton_client.async_stream_infer( model_name=model_name, inputs=inputs, request_id=request_id, outputs=outputs, ) # Retrieve results... recv_count = 0 expected_count = len(in_value) result_dict = {} while recv_count < expected_count: data_item = user_data._completed_requests.get() if type(data_item) == InferenceServerException: raise data_item else: this_id = data_item.get_response().id if this_id not in result_dict.keys(): result_dict[this_id] = [] result_dict[this_id].append((recv_count, data_item)) recv_count += 1 # Validate results... if len(result_dict[request_id]) != len(in_value): print( "expected {} many responses for request id {}, got {}".format( len(in_value), request_id, len(result_dict[request_id]) ) ) sys.exit(1) result_list = result_dict[request_id] for i in range(len(result_list)): expected_data = np.array([in_value[i]], dtype=np.int32) this_data = result_list[i][1].as_numpy("OUT") if not np.array_equal(expected_data, this_data): print( "incorrect data: expected {}, got {}".format(expected_data, this_data) ) sys.exit(1) print("PASS: repeat_int32") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/decoupled/repeat_config.pbtxt000066400000000000000000000040001527052371200321200ustar00rootroot00000000000000# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "repeat_int32" backend: "python" max_batch_size: 0 model_transaction_policy { decoupled: True } input [ { name: "IN" data_type: TYPE_INT32 dims: [ -1 ] }, { name: "DELAY" data_type: TYPE_UINT32 dims: [ -1 ] }, { name: "WAIT" data_type: TYPE_UINT32 dims: [ 1 ] } ] output [ { name: "OUT" data_type: TYPE_INT32 dims: [ 1 ] }, { name: "IDX" data_type: TYPE_UINT32 dims: [ 1 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/decoupled/repeat_model.py000066400000000000000000000267421527052371200312630ustar00rootroot00000000000000# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import threading import time import numpy # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. This model demonstrates how to write a decoupled model where each request can generate 0 to many responses. This model has three inputs and two outputs. The model does not support batching. - Input 'IN' can have any vector shape (e.g. [4] or [12]), datatype must be INT32. - Input 'DELAY' must have the same shape as IN, datatype must be UINT32. - Input 'WAIT' must have shape [1] and datatype UINT32. - For each response, output 'OUT' must have shape [1] and datatype INT32. - For each response, output 'IDX' must have shape [1] and datatype UINT32. For a request, the model will send 'n' responses where 'n' is the number of elements in IN. For the i'th response, OUT will equal the i'th element of IN and IDX will equal the zero-based count of this response for the request. For example, the first response for a request will have IDX = 0 and OUT = IN[0], the second will have IDX = 1 and OUT = IN[1], etc. The model will wait the i'th DELAY, in milliseconds, before sending the i'th response. If IN shape is [0] then no responses will be sent. After WAIT milliseconds the model will return from the execute function so that Triton can call execute again with another request. WAIT can be less than the sum of DELAY so that execute returns before all responses are sent. Thus, even if there is only one instance of the model, multiple requests can be processed at the same time, and the responses for multiple requests can be intermixed, depending on the values of DELAY and WAIT. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args["model_config"]) using_decoupled = pb_utils.using_decoupled_model_transaction_policy( model_config ) if not using_decoupled: raise pb_utils.TritonModelException( """the model `{}` can generate any number of responses per request, enable decoupled transaction policy in model configuration to serve this model""".format( args["model_name"] ) ) # Get OUT configuration out_config = pb_utils.get_output_config_by_name(model_config, "OUT") # Get IDX configuration idx_config = pb_utils.get_output_config_by_name(model_config, "IDX") # Convert Triton types to numpy types self.out_dtype = pb_utils.triton_string_to_numpy(out_config["data_type"]) self.idx_dtype = pb_utils.triton_string_to_numpy(idx_config["data_type"]) # To keep track of response threads so that we can delay # the finalizing the model until all response threads # have completed. self.inflight_thread_count = 0 self.inflight_thread_count_lck = threading.Lock() def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. The request.get_response_sender() must be used to get an InferenceResponseSender object associated with the request. Use the InferenceResponseSender.send(response=, flags=) to send responses. In the final response sent using the response sender object, you must set the flags argument to TRITONSERVER_RESPONSE_COMPLETE_FINAL to indicate no responses will be sent for the corresponding request. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse. Setting the flags argument is optional and defaults to zero. When the flags argument is set to TRITONSERVER_RESPONSE_COMPLETE_FINAL providing the response argument is optional. Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- None """ # This model does not support batching, so 'request_count' should always # be 1. if len(requests) != 1: raise pb_utils.TritonModelException( "unsupported batch size " + len(requests) ) in_input = pb_utils.get_input_tensor_by_name(requests[0], "IN").as_numpy() delay_input = pb_utils.get_input_tensor_by_name(requests[0], "DELAY").as_numpy() if in_input.shape != delay_input.shape: raise pb_utils.TritonModelException( f"expected IN and DELAY shape to match, got {list(in_input.shape)} and {list(delay_input.shape)}." ) # Start a separate thread to send the responses for the request. The # sending back the responses is delegated to this thread. thread = threading.Thread( target=self.response_thread, args=(requests[0].get_response_sender(), in_input, delay_input), ) # A model using decoupled transaction policy is not required to send all # responses for the current request before returning from the execute. # To demonstrate the flexibility of the decoupled API, we are running # response thread entirely independent of the execute thread. thread.daemon = True with self.inflight_thread_count_lck: self.inflight_thread_count += 1 thread.start() # Read WAIT input for wait time, then return so that Triton can call # execute again with another request. wait_input = pb_utils.get_input_tensor_by_name(requests[0], "WAIT").as_numpy() time.sleep(wait_input[0] / 1000) # Unlike in non-decoupled model transaction policy, execute function # here returns no response. A return from this function only notifies # Triton that the model instance is ready to receive another request. As # we are not waiting for the response thread to complete here, it is # possible that at any give time the model may be processing multiple # requests. Depending upon the request workload, this may lead to a lot # of requests being processed by a single model instance at a time. In # real-world models, the developer should be mindful of when to return # from execute and be willing to accept next request. return None def response_thread(self, response_sender, in_input, delay_input): # The response_sender is used to send response(s) associated with the # corresponding request. Iterate over input/delay pairs. Wait for DELAY # milliseconds and then create and send a response. idx_dtype = self.idx_dtype out_dtype = self.out_dtype for idx in range(in_input.size): in_value = in_input[idx] delay_value = delay_input[idx] time.sleep(delay_value / 1000) idx_output = pb_utils.Tensor("IDX", numpy.array([idx], idx_dtype)) out_output = pb_utils.Tensor("OUT", numpy.array([in_value], out_dtype)) response = pb_utils.InferenceResponse( output_tensors=[idx_output, out_output] ) response_sender.send(response) # We must close the response sender to indicate to Triton that we are # done sending responses for the corresponding request. We can't use the # response sender after closing it. The response sender is closed by # setting the TRITONSERVER_RESPONSE_COMPLETE_FINAL. response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) with self.inflight_thread_count_lck: self.inflight_thread_count -= 1 def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. Here we will wait for all response threads to complete sending responses. """ print("Finalize invoked") inflight_threads = True cycles = 0 logging_time_sec = 5 sleep_time_sec = 0.1 cycle_to_log = logging_time_sec / sleep_time_sec while inflight_threads: with self.inflight_thread_count_lck: inflight_threads = self.inflight_thread_count != 0 if cycles % cycle_to_log == 0: print( f"Waiting for {self.inflight_thread_count} response threads to complete..." ) if inflight_threads: time.sleep(sleep_time_sec) cycles += 1 print("Finalize complete...") kytensor-2.49.0.7/framework_backends/python_backend/examples/decoupled/square_client.py000066400000000000000000000115621527052371200314530ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import queue import sys from functools import partial import numpy as np import tritonclient.grpc as grpcclient from tritonclient.utils import * class UserData: def __init__(self): self._completed_requests = queue.Queue() def callback(user_data, result, error): if error: user_data._completed_requests.put(error) else: user_data._completed_requests.put(result) # This client sends a 4 requests to the model with the # input as: [4], [2], [0] and [1] respectively. In # compliance with the behavior of square_int32 model, # it will expect the 4 responses for the 1st request # each with output [4], 2 responses for 2nd request # each with output [2], no response for the 3rd request # and finally 1 response for the 4th request with output # [1] model_name = "square_int32" in_values = [4, 2, 0, 1] inputs = [grpcclient.InferInput("IN", [1], np_to_triton_dtype(np.int32))] outputs = [grpcclient.InferRequestedOutput("OUT")] user_data = UserData() with grpcclient.InferenceServerClient( url="localhost:8001", verbose=True ) as triton_client: # Establish stream triton_client.start_stream(callback=partial(callback, user_data)) # Send specified many requests in parallel for i in range(len(in_values)): in_data = np.array([in_values[i]], dtype=np.int32) inputs[0].set_data_from_numpy(in_data) triton_client.async_stream_infer( model_name=model_name, inputs=inputs, request_id=str(i), outputs=outputs, ) # Retrieve results... recv_count = 0 expected_count = sum(in_values) result_dict = {} while recv_count < expected_count: data_item = user_data._completed_requests.get() if type(data_item) == InferenceServerException: raise data_item else: this_id = data_item.get_response().id if this_id not in result_dict.keys(): result_dict[this_id] = [] result_dict[this_id].append((recv_count, data_item)) recv_count += 1 # Validate results... for i in range(len(in_values)): this_id = str(i) if in_values[i] != 0 and this_id not in result_dict.keys(): print("response for request id {} not received".format(this_id)) sys.exit(1) elif in_values[i] == 0 and this_id in result_dict.keys(): print("received unexpected response for request id {}".format(this_id)) sys.exit(1) if in_values[i] != 0: if len(result_dict[this_id]) != in_values[i]: print( "expected {} many responses for request id {}, got {}".format( in_values[i], this_id, result_dict[this_id] ) ) sys.exit(1) if in_values[i] != 0: result_list = result_dict[this_id] expected_data = np.array([in_values[i]], dtype=np.int32) for j in range(len(result_list)): this_data = result_list[j][1].as_numpy("OUT") if not np.array_equal(expected_data, this_data): print( "incorrect data: expected {}, got {}".format( expected_data, this_data ) ) sys.exit(1) print("PASS: square_int32") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/decoupled/square_config.pbtxt000066400000000000000000000034601527052371200321510ustar00rootroot00000000000000# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "square_int32" backend: "python" max_batch_size: 0 model_transaction_policy { decoupled: True } input [ { name: "IN" data_type: TYPE_INT32 dims: [ 1 ] } ] output [ { name: "OUT" data_type: TYPE_INT32 dims: [ 1 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/decoupled/square_model.py000066400000000000000000000252101527052371200312700ustar00rootroot00000000000000# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import threading import time import numpy as np # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. This model demonstrates how to write a decoupled model where each request can generate 0 to many responses. This model has one input and one output. The model can support batching, with constraint that each request must be batch-1 request, but the shapes described here refer to the non-batch portion of the shape. - Input 'IN' must have shape [1] and datatype INT32. - Output 'OUT' must have shape [1] and datatype INT32. For a request, the backend will sent 'n' responses where 'n' is the element in IN. For each response, OUT will equal the element of IN. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args["model_config"]) using_decoupled = pb_utils.using_decoupled_model_transaction_policy( model_config ) if not using_decoupled: raise pb_utils.TritonModelException( """the model `{}` can generate any number of responses per request, enable decoupled transaction policy in model configuration to serve this model""".format( args["model_name"] ) ) # Get IN configuration in_config = pb_utils.get_input_config_by_name(model_config, "IN") # Validate the shape and data type of IN in_shape = in_config["dims"] if (len(in_shape) != 1) or (in_shape[0] != 1): raise pb_utils.TritonModelException( """the model `{}` requires the shape of 'IN' to be [1], got {}""".format( args["model_name"], in_shape ) ) if in_config["data_type"] != "TYPE_INT32": raise pb_utils.TritonModelException( """the model `{}` requires the data_type of 'IN' to be 'TYPE_INT32', got {}""".format( args["model_name"], in_config["data_type"] ) ) # Get OUT configuration out_config = pb_utils.get_output_config_by_name(model_config, "OUT") # Validate the shape and data type of OUT out_shape = out_config["dims"] if (len(out_shape) != 1) or (out_shape[0] != 1): raise pb_utils.TritonModelException( """the model `{}` requires the shape of 'OUT' to be [1], got {}""".format( args["model_name"], out_shape ) ) if out_config["data_type"] != "TYPE_INT32": raise pb_utils.TritonModelException( """the model `{}` requires the data_type of 'OUT' to be 'TYPE_INT32', got {}""".format( args["model_name"], out_config["data_type"] ) ) self.inflight_thread_count = 0 self.inflight_thread_count_lck = threading.Lock() def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. The request.get_response_sender() must be used to get an InferenceResponseSender object associated with the request. Use the InferenceResponseSender.send(response=, flags=) to send responses. In the final response sent using the response sender object, you must set the flags argument to TRITONSERVER_RESPONSE_COMPLETE_FINAL to indicate no responses will be sent for the corresponding request. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse. Setting the flags argument is optional and defaults to zero. When the flags argument is set to TRITONSERVER_RESPONSE_COMPLETE_FINAL providing the response argument is optional. Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- None """ # Visit individual request to start processing them. Note that execute # function is not required to wait for all the requests of the current # batch to be processed before returning. for request in requests: self.process_request(request) # Unlike in non-decoupled model transaction policy, execute function # here returns no response. A return from this function only notifies # Triton that the model instance is ready to receive another batch of # requests. As we are not waiting for the response thread to complete # here, it is possible that at any give time the model may be processing # multiple batches of requests. Depending upon the request workload, # this may lead to a lot of requests being processed by a single model # instance at a time. In real-world models, the developer should be # mindful of when to return from execute and be willing to accept next # request batch. return None def process_request(self, request): # Start a separate thread to send the responses for the request. The # sending back the responses is delegated to this thread. thread = threading.Thread( target=self.response_thread, args=( request.get_response_sender(), pb_utils.get_input_tensor_by_name(request, "IN").as_numpy(), ), ) # A model using decoupled transaction policy is not required to send all # responses for the current request before returning from the execute. # To demonstrate the flexibility of the decoupled API, we are running # response thread entirely independent of the execute thread. thread.daemon = True with self.inflight_thread_count_lck: self.inflight_thread_count += 1 thread.start() def response_thread(self, response_sender, in_input): # The response_sender is used to send response(s) associated with the # corresponding request. for idx in range(in_input[0]): out_output = pb_utils.Tensor("OUT", np.array([in_input[0]], np.int32)) response = pb_utils.InferenceResponse(output_tensors=[out_output]) response_sender.send(response) # We must close the response sender to indicate to Triton that we are # done sending responses for the corresponding request. We can't use the # response sender after closing it. The response sender is closed by # setting the TRITONSERVER_RESPONSE_COMPLETE_FINAL. response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) with self.inflight_thread_count_lck: self.inflight_thread_count -= 1 def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. Here we will wait for all response threads to complete sending responses. """ print("Finalize invoked") inflight_threads = True cycles = 0 logging_time_sec = 5 sleep_time_sec = 0.1 cycle_to_log = logging_time_sec / sleep_time_sec while inflight_threads: with self.inflight_thread_count_lck: inflight_threads = self.inflight_thread_count != 0 if cycles % cycle_to_log == 0: print( f"Waiting for {self.inflight_thread_count} response threads to complete..." ) if inflight_threads: time.sleep(sleep_time_sec) cycles += 1 print("Finalize complete...") kytensor-2.49.0.7/framework_backends/python_backend/examples/instance_kind/000077500000000000000000000000001527052371200271035ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/instance_kind/README.md000066400000000000000000000160361527052371200303700ustar00rootroot00000000000000 # Model Instance Kind Example Triton model configuration allows users to provide kind to [instance group settings.](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#instance-groups) A python backend model can be written to respect the kind setting to control the execution of a model instance either on CPU or GPU. In this example, we demonstrate how this can be achieved for your python model. We will use a `ResNet50` model as our base model for this example. ## Create a ResNet50 model repository We will use the files that come with this example to create the model repository. First, download the [client.py](client.py), [config.pbtxt](config.pbtxt), [resnet50_labels.txt](resnet50_labels.txt), and [model.py](model.py) to your local machine. Next, in the same directory with the four aforementioned files, create the model repository with the following commands: ``` mkdir -p models/resnet50/1 && mv model.py models/resnet50/1/ && mv config.pbtxt models/resnet50/ ``` ## Pull the Triton Docker images We need to install Docker and NVIDIA Container Toolkit before proceeding, refer to the [installation steps](https://github.com/triton-inference-server/server/tree/main/docs#installation). To pull the latest containers, run the following commands: ``` docker pull nvcr.io/nvidia/tritonserver:-py3 docker pull nvcr.io/nvidia/tritonserver:-py3-sdk ``` See the installation steps above for the `` version. For example, if the latest version is `23.01`, the above commands translate to the following: ``` docker pull nvcr.io/nvidia/tritonserver:23.01-py3 docker pull nvcr.io/nvidia/tritonserver:23.01-py3-sdk ``` Be sure to replace the `` with the version pulled for all the remaining parts of this example. ## Start the Triton Server At the directory where we copied our resnet50 model (at where the "models" folder is located), run the following command: ``` docker run --gpus all --shm-size 1G -it --rm -p 8000:8000 -v `pwd`:/instance_kind nvcr.io/nvidia/tritonserver:-py3 /bin/bash ``` Inside the container, we need to install `torch`, `torchvision` and `pillow` to run this example. We recommend to use `pip` method for the installation: ``` pip3 install torch==1.13.0+cu117 -f https://download.pytorch.org/whl/torch_stable.html torchvision==0.14.0+cu117 pillow ``` Finally, we need to start the Triton Server: ``` tritonserver --model-repository /instance_kind/models ``` To leave the container for the next step, press: `CTRL + P + Q`. ## Start the Triton SDK Container and Test Inference To start the sdk container, run the following command: ``` docker run --gpus all --network=host --pid=host --ipc=host -v `pwd`:/instance_kind -ti nvcr.io/nvidia/tritonserver:-py3-sdk /bin/bash ``` The `client.py` requires the following packages to be installed: `torch`, `torchvision`, `pillow` and `validators`. Similarly, we recommend to use `pip` method for the installation: ``` pip3 install torch==1.13.0+cu117 -f https://download.pytorch.org/whl/torch_stable.html torchvision==0.14.0+cu117 pillow validators ``` Finally, let's test an inference call with the following command: ``` python client.py ``` On a first run, a successful inference will print the following at the end: ``` Downloading: "https://github.com/NVIDIA/DeepLearningExamples/zipball/torchhub" to /root/.cache/torch/hub/torchhub.zip Results is class: TABBY PASS: ResNet50 ``` It may take some time due to `torchhub` downloads, but any future calls will be quicker, since the client will use already downloaded artifacts. ## Test Instance Kind Provided `config.pbtxt` sets the instance group setting to `KIND_CPU`, which enables the execution of a model on the CPU. To test that your model is actually loaded onto CPU, run the following: ``` python client.py -v ``` The `-v` argument asks the client to request model's confiuration from the server and prints it in your console: ``` { ..., "instance_group": [ { "name": "resnet50_0", "kind": "KIND_CPU", "count": 1, "gpus": [], "secondary_devices": [], "profile": [], "passive": false, "host_policy": "" } ], ... } Results is class: TABBY PASS: ResNet50 instance kind ``` Based on the printed model config, we can see that `instance_group` field has `kind` entry, which is set to `KIND_CPU`. To change an `instance_group` parameter to `KIND_GPU`, a user can simply replace `KIND_CPU` with `KIND_GPU` in the `config.pbtxt`. After restarting the server with an updated config file, a successful inference request with `-v` argument will result into the similar output, but with an updated `instance_group` entry: ``` { ..., "instance_group": [ { "name": "resnet50_0", "kind": "KIND_GPU", "count": 1, "gpus": [ 0 ], "secondary_devices": [], "profile": [], "passive": false, "host_policy": "" } ], ... } Results is class: TABBY PASS: ResNet50 instance kind ``` It is also possible to load multiple model instances on CPU and GPU if necessary. Below the instance group setting will create two model instances, one on CPU and other on GPU. ``` instance_group [{ kind: KIND_CPU }, { kind: KIND_GPU}] ``` For more information on possible model configurations, check out the Triton Server documentation [here](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#model-configuration)kytensor-2.49.0.7/framework_backends/python_backend/examples/instance_kind/client.py000066400000000000000000000077201527052371200307410ustar00rootroot00000000000000# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import json import sys import warnings import numpy as np import torch import tritonclient.http as httpclient from tritonclient.utils import * warnings.filterwarnings("ignore") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--model_name", type=str, required=False, default="resnet50", help="Model name", ) parser.add_argument( "--image_url", type=str, required=False, default="http://images.cocodataset.org/test2017/000000557146.jpg", help="Image URL. Default is:\ http://images.cocodataset.org/test2017/000000557146.jpg", ) parser.add_argument( "--url", type=str, required=False, default="localhost:8000", help="Inference server URL. Default is localhost:8000.", ) parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "--label_file", type=str, required=False, default="./resnet50_labels.txt", help="Path to the file with text representation \ of available labels", ) args = parser.parse_args() utils = torch.hub.load( "NVIDIA/DeepLearningExamples:torchhub", "nvidia_convnets_processing_utils", skip_validation=True, ) try: triton_client = httpclient.InferenceServerClient(args.url) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) with open(args.label_file) as f: labels_dict = {idx: line.strip() for idx, line in enumerate(f)} if args.verbose: print(json.dumps(triton_client.get_model_config(args.model_name), indent=4)) input_name = "INPUT" output_name = "OUTPUT" batch = np.asarray(utils.prepare_input_from_uri(args.image_url)) input = httpclient.InferInput(input_name, batch.shape, "FP32") output = httpclient.InferRequestedOutput(output_name) input.set_data_from_numpy(batch) results = triton_client.infer( model_name=args.model_name, inputs=[input], outputs=[output] ) output_data = results.as_numpy(output_name) max_id = np.argmax(output_data, axis=1)[0] print("Results is class: {}".format(labels_dict[max_id])) print("PASS: ResNet50 instance kind") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/instance_kind/config.pbtxt000066400000000000000000000034171527052371200314400ustar00rootroot00000000000000# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "resnet50" backend: "python" max_batch_size: 128 input { name: "INPUT" data_type: TYPE_FP32 format: FORMAT_NCHW dims: [ 3, 224, 224 ] } output { name: "OUTPUT" data_type: TYPE_FP32 dims: [ 1000 ] } instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/instance_kind/model.py000066400000000000000000000074041527052371200305620ustar00rootroot00000000000000# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import numpy as np import torch import triton_python_backend_utils as pb_utils from torch.utils.dlpack import to_dlpack class TritonPythonModel: def initialize(self, args): """ This function initializes pre-trained ResNet50 model, depending on the value specified by an `instance_group` parameter in `config.pbtxt`. Depending on what `instance_group` was specified in the config.pbtxt file (KIND_CPU or KIND_GPU), the model instance will be initialised on a cpu, a gpu, or both. If `instance_group` was not specified in the config file, then models will be loaded onto the default device of the framework. """ # Here we set up the device onto which our model will beloaded, # based on specified `model_instance_kind` and `model_instance_device_id` # fields. device = "cuda" if args["model_instance_kind"] == "GPU" else "cpu" device_id = args["model_instance_device_id"] self.device = f"{device}:{device_id}" # This example is configured to work with torch=1.13 # and torchvision=0.14. Thus, we need to provide a proper tag `0.14.1` # to make sure loaded Resnet50 is compatible with # installed `torchvision`. # Refer to README for installation instructions. self.model = ( torch.hub.load( "pytorch/vision:v0.14.1", "resnet50", weights="IMAGENET1K_V2", skip_validation=True, ) .to(self.device) .eval() ) def execute(self, requests): """ This function receives a list of requests (`pb_utils.InferenceRequest`), performs inference on every request and appends it to responses. """ responses = [] for request in requests: input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT") with torch.no_grad(): result = self.model( torch.as_tensor(input_tensor.as_numpy(), device=self.device) ) out_tensor = pb_utils.Tensor.from_dlpack("OUTPUT", to_dlpack(result)) responses.append(pb_utils.InferenceResponse([out_tensor])) return responses kytensor-2.49.0.7/framework_backends/python_backend/examples/instance_kind/resnet50_labels.txt000066400000000000000000000241101527052371200326310ustar00rootroot00000000000000TENCH GOLDFISH WHITE SHARK TIGER SHARK HAMMERHEAD SHARK ELECTRIC RAY STINGRAY ROOSTER HEN OSTRICH BRAMBLING GOLDFINCH HOUSE FINCH SNOWBIRD INDIGO FINCH ROBIN BULBUL JAY MAGPIE CHICKADEE WATER OUZEL KITE BALD EAGLE VULTURE GREAT GREY OWL FIRE SALAMANDER NEWT EFT SPOTTED SALAMANDER AXOLOTL BULL FROG TREE FROG TAILED FROG LOGGERHEAD LEATHERBACK TURTLE MUD TURTLE TERRAPIN BOX TURTLE BANDED GECKO COMMON IGUANA AMERICAN CHAMELEON WHIPTAIL AGAMA FRILLED LIZARD ALLIGATOR LIZARD GILA MONSTER GREEN LIZARD AFRICAN CHAMELEON KOMODO DRAGON AFRICAN CROCODILE AMERICAN ALLIGATOR TRICERATOPS THUNDER SNAKE RINGNECK SNAKE HOGNOSE SNAKE GREEN SNAKE KING SNAKE GARTER SNAKE WATER SNAKE VINE SNAKE NIGHT SNAKE BOA ROCK PYTHON COBRA GREEN MAMBA SEA SNAKE HORNED VIPER DIAMONDBACK SIDEWINDER TRILOBITE HARVESTMAN SCORPION GARDEN SPIDER BARN SPIDER GARDEN SPIDER BLACK WIDOW TARANTULA WOLF SPIDER TICK CENTIPEDE GROUSE PTARMIGAN RUFFED GROUSE PRAIRIE CHICKEN PEACOCK QUAIL PARTRIDGE AFRICAN GREY MACAW COCKATOO LORIKEET COUCAL BEE EATER HORNBILL HUMMINGBIRD JACAMAR TOUCAN DRAKE MERGANSER GOOSE BLACK SWAN TUSKER ECHIDNA PLATYPUS WALLABY KOALA WOMBAT JELLYFISH SEA ANEMONE BRAIN CORAL FLATWORM NEMATODE CONCH SNAIL SLUG SEA SLUG CHITON CHAMBERED NAUTILUS DUNGENESS CRAB ROCK CRAB FIDDLER CRAB KING CRAB AMERICAN LOBSTER SPINY LOBSTER CRAYFISH HERMIT CRAB ISOPOD WHITE STORK BLACK STORK SPOONBILL FLAMINGO LITTLE BLUE HERON AMERICAN EGRET BITTERN CRANE LIMPKIN EUROPEAN GALLINULE AMERICAN COOT BUSTARD RUDDY TURNSTONE RED-BACKED SANDPIPER REDSHANK DOWITCHER OYSTERCATCHER PELICAN KING PENGUIN ALBATROSS GREY WHALE KILLER WHALE DUGONG SEA LION CHIHUAHUA JAPANESE SPANIEL MALTESE DOG PEKINESE SHIH-TZU BLENHEIM SPANIEL PAPILLON TOY TERRIER RHODESIAN RIDGEBACK AFGHAN HOUND BASSET BEAGLE BLOODHOUND BLUETICK COONHOUND WALKER HOUND ENGLISH FOXHOUND REDBONE BORZOI IRISH WOLFHOUND ITALIAN GREYHOUND WHIPPET IBIZAN HOUND NORWEGIAN ELKHOUND OTTERHOUND SALUKI SCOTTISH DEERHOUND WEIMARANER STAFFORDSHIRE BULLTERRIER STAFFORDSHIRE TERRIER BEDLINGTON TERRIER BORDER TERRIER KERRY BLUE TERRIER IRISH TERRIER NORFOLK TERRIER NORWICH TERRIER YORKSHIRE TERRIER WIRE-HAIRED FOX TERRIER LAKELAND TERRIER SEALYHAM TERRIER AIREDALE CAIRN AUSTRALIAN TERRIER DANDIE DINMONT BOSTON BULL MINIATURE SCHNAUZER GIANT SCHNAUZER STANDARD SCHNAUZER SCOTCH TERRIER TIBETAN TERRIER SILKY TERRIER WHEATEN TERRIER WHITE TERRIER LHASA RETRIEVER CURLY-COATED RETRIEVER GOLDEN RETRIEVER LABRADOR RETRIEVER CHESAPEAKE BAY RETRIEVER SHORT-HAIRED POINTER VISLA ENGLISH SETTER IRISH SETTER GORDON SETTER BRITTANY SPANIEL CLUMBER ENGLISH SPRINGER WELSH SPRINGER SPANIEL COCKER SPANIEL SUSSEX SPANIEL IRISH WATERSPANIEL KUVASZ SCHIPPERKE GROENENDAEL MALINOIS BRIARD KELPIE KOMONDOR OLD ENGLISH SHEEPDOG SHETLAND SHEEPDOG COLLIE BORDER COLLIE BOUVIER DES FLANDRES ROTTWEILER GERMAN SHEPHERD DOBERMAN MINIATURE PINSCHER GREATER SWISS MOUNTAIN DOG BERNESE MOUNTAIN DOG APPENZELLER ENTLEBUCHER BOXER BULL MASTIFF TIBETAN MASTIFF FRENCH BULLDOG GREAT DANE SAINT BERNARD ESKIMO DOG MALAMUTE SIBERIAN HUSKY DALMATIAN AFFENPINSCHER BASENJI PUG LEONBERG NEWFOUNDLAND GREAT PYRENEES SAMOYED POMERANIAN CHOW KEESHOND BRABANCON GRIFFON PEMBROKE CARDIGAN TOY POODLE MINIATURE POODLE STANDARD POODLE MEXICAN HAIRLESS TIMBER WOLF WHITE WOLF RED WOLF COYOTE DINGO DHOLE AFRICAN HUNTING DOG HYENA RED FOX KIT FOX ARCTIC FOX GREY FOX TABBY TIGER CAT PERSIAN CAT SIAMESE CAT EGYPTIAN CAT COUGAR LYNX LEOPARD SNOW LEOPARD JAGUAR LION TIGER CHEETAH BROWN BEAR AMERICAN BLACK BEAR ICE BEAR SLOTH BEAR MONGOOSE MEERKAT TIGER BEETLE LADYBUG GROUND BEETLE LONG-HORNED BEETLE LEAF BEETLE DUNG BEETLE RHINOCEROS BEETLE WEEVIL FLY BEE ANT GRASSHOPPER CRICKET WALKING STICK COCKROACH MANTIS CICADA LEAFHOPPER LACEWING DRAGONFLY DAMSELFLY ADMIRAL RINGLET MONARCH CABBAGE BUTTERFLY SULPHUR BUTTERFLY LYCAENID STARFISH SEA URCHIN SEA CUCUMBER WOOD RABBIT HARE ANGORA HAMSTER PORCUPINE FOX SQUIRREL MARMOT BEAVER GUINEA PIG SORREL ZEBRA HOG WILD BOAR WARTHOG HIPPOPOTAMUS OX WATER BUFFALO BISON RAM BIGHORN IBEX HARTEBEEST IMPALA GAZELLE ARABIAN CAMEL LLAMA WEASEL MINK POLECAT BLACK-FOOTED FERRET OTTER SKUNK BADGER ARMADILLO THREE-TOED SLOTH ORANGUTAN GORILLA CHIMPANZEE GIBBON SIAMANG GUENON PATAS BABOON MACAQUE LANGUR COLOBUS PROBOSCIS MONKEY MARMOSET CAPUCHIN HOWLER MONKEY TITI SPIDER MONKEY SQUIRREL MONKEY MADAGASCAR CAT INDRI INDIAN ELEPHANT AFRICAN ELEPHANT LESSER PANDA GIANT PANDA BARRACOUTA EEL COHO ROCK BEAUTY ANEMONE FISH STURGEON GAR LIONFISH PUFFER ABACUS ABAYA ACADEMIC GOWN ACCORDION ACOUSTIC GUITAR AIRCRAFT CARRIER AIRLINER AIRSHIP ALTAR AMBULANCE AMPHIBIAN ANALOG CLOCK APIARY APRON ASHCAN ASSAULT RIFLE BACKPACK BAKERY BALANCE BEAM BALLOON BALLPOINT BAND AID BANJO BANNISTER BARBELL BARBER CHAIR BARBERSHOP BARN BAROMETER BARREL BARROW BASEBALL BASKETBALL BASSINET BASSOON BATHING CAP BATH TOWEL BATHTUB BEACH WAGON BEACON BEAKER BEARSKIN BEER BOTTLE BEER GLASS BELL COTE BIB BICYCLE-BUILT-FOR-TWO BIKINI BINDER BINOCULARS BIRDHOUSE BOATHOUSE BOBSLED BOLO TIE BONNET BOOKCASE BOOKSHOP BOTTLECAP BOW BOW TIE BRASS BRASSIERE BREAKWATER BREASTPLATE BROOM BUCKET BUCKLE BULLETPROOF VEST BULLET TRAIN BUTCHER SHOP CAB CALDRON CANDLE CANNON CANOE CAN OPENER CARDIGAN CAR MIRROR CAROUSEL CARPENTERS KIT CARTON CAR WHEEL CASH MACHINE CASSETTE CASSETTE PLAYER CASTLE CATAMARAN CD PLAYER CELLO CELLULAR TELEPHONE CHAIN CHAINLINK FENCE CHAIN MAIL CHAIN SAW CHEST CHIFFONIER CHIME CHINA CABINET CHRISTMAS STOCKING CHURCH CINEMA CLEAVER CLIFF DWELLING CLOAK CLOG COCKTAIL SHAKER COFFEE MUG COFFEEPOT COIL COMBINATION LOCK COMPUTER KEYBOARD CONFECTIONERY CONTAINER SHIP CONVERTIBLE CORKSCREW CORNET COWBOY BOOT COWBOY HAT CRADLE CRANE CRASH HELMET CREATE CRIB CROCK POT CROQUET BALL CRUTCH CUIRASS DAM DESK DESKTOP COMPUTER DIAL TELEPHONE DIAPER DIGITAL CLOCK DIGITAL WATCH DINING TABLE DISHRAG DISHWASHER DISK BRAKE DOCK DOGSLED DOME DOORMAT DRILLING PLATFORM DRUM DRUMSTICK DUMBBELL DUTCH OVEN ELECTRIC FAN ELECTRIC GUITAR ELECTRIC LOCOMOTIVE ENTERTAINMENT CENTER ENVELOPE ESPRESSO MAKER FACE POWDER FEATHER BOA FILE FIREBOAT FIRE ENGINE FIRE SCREEN FLAGPOLE FLUTE FOLDING CHAIR FOOTBALL HELMET FORKLIFT FOUNTAIN FOUNTAIN PEN FOUR-POSTER FREIGHT CAR FRENCH HORN FRYING PAN FUR COAT GARBAGE TRUCK GASMASK GAS PUMP GOBLET GO-KART GOLF BALL GOLFCART GONDOLA GONG GOWN GRAND PIANO GREENHOUSE GRILLE GROCERY STORE GUILLOTINE HAIR SLIDE HAIR SPRAY HALF TRACK HAMMER HAMPER HAND BLOWER HAND-HELD COMPUTER HANDKERCHIEF HARD DISC HARMONICA HARP HARVESTER HATCHET HOLSTER HOME THEATER HONEYCOMB HOOK HOOPSKIRT HORIZONTAL BAR HORSE CART HOURGLASS IPOD IRON JACK-O-LANTERN JEAN JEEP JERSEY JIGSAW PUZZLE JINRIKISHA JOYSTICK KIMONO KNEE PAD KNOT LAB COAT LADLE LAMPSHADE LAPTOP LAWN MOWER LENS CAP LETTER OPENER LIBRARY LIFEBOAT LIGHTER LIMOUSINE LINER LIPSTICK LOAFER LOTION LOUDSPEAKER LOUPE LUMBERMILL MAGNETIC COMPASS MAILBAG MAILBOX MAILLOT MAILLOT MANHOLE COVER MARACA MARIMBA MASK MATCHSTICK MAYPOLE MAZE MEASURING CUP MEDICINE CHEST MEGALITH MICROPHONE MICROWAVE MILITARY UNIFORM MILK CAN MINIBUS MINISKIRT MINIVAN MISSILE MITTEN MIXING BOWL MOBILE HOME MODEL T MODEM MONASTERY MONITOR MOPED MORTAR MORTARBOARD MOSQUE MOSQUITO NET MOTOR SCOOTER MOUNTAIN BIKE MOUNTAIN TENT MOUSE MOUSETRAP MOVING VAN MUZZLE NAIL NECK BRACE NECKLACE NIPPLE NOTEBOOK OBELISK OBOE OCARINA ODOMETER OIL FILTER ORGAN OSCILLOSCOPE OVERSKIRT OXCART OXYGEN MASK PACKET PADDLE PADDLEWHEEL PADLOCK PAINTBRUSH PAJAMA PALACE PANPIPE PAPER TOWEL PARACHUTE PARALLEL BARS PARK BENCH PARKING METER PASSENGER CAR PATIO PAY-PHONE PEDESTAL PENCIL BOX PENCIL SHARPENER PERFUME PETRI DISH PHOTOCOPIER PICK PICKELHAUBE PICKET FENCE PICKUP PIER PIGGY BANK PILL BOTTLE PILLOW PING-PONG BALL PINWHEEL PIRATE PITCHER PLANE PLANETARIUM PLASTIC BAG PLATE RACK PLOW PLUNGER POLAROID CAMERA POLE POLICE VAN PONCHO POOL TABLE POP BOTTLE POT POTTERS WHEEL POWER DRILL PRAYER RUG PRINTER PRISON PROJECTILE PROJECTOR PUCK PUNCHING BAG PURSE QUILL QUILT RACER RACKET RADIATOR RADIO RADIO TELESCOPE RAIN BARREL RECREATIONAL VEHICLE REEL REFLEX CAMERA REFRIGERATOR REMOTE CONTROL RESTAURANT REVOLVER RIFLE ROCKING CHAIR ROTISSERIE RUBBER ERASER RUGBY BALL RULE RUNNING SHOE SAFE SAFETY PIN SALTSHAKER SANDAL SARONG SAX SCABBARD SCALE SCHOOL BUS SCHOONER SCOREBOARD SCREEN SCREW SCREWDRIVER SEAT BELT SEWING MACHINE SHIELD SHOE SHOP SHOJI SHOPPING BASKET SHOPPING CART SHOVEL SHOWER CAP SHOWER CURTAIN SKI SKI MASK SLEEPING BAG SLIDE RULE SLIDING DOOR SLOT SNORKEL SNOWMOBILE SNOWPLOW SOAP DISPENSER SOCCER BALL SOCK SOLAR DISH SOMBRERO SOUP BOWL SPACE BAR SPACE HEATER SPACE SHUTTLE SPATULA SPEEDBOAT SPIDER WEB SPINDLE SPORTS CAR SPOTLIGHT STAGE STEAM LOCOMOTIVE STEEL ARCH BRIDGE STEEL DRUM STETHOSCOPE STOLE STONE WALL STOPWATCH STOVE STRAINER STREETCAR STRETCHER STUDIO COUCH STUPA SUBMARINE SUIT SUNDIAL SUNGLASS SUNGLASSES SUNSCREEN SUSPENSION BRIDGE SWAB SWEATSHIRT SWIMMING TRUNKS SWING SWITCH SYRINGE TABLE LAMP TANK TAPE PLAYER TEAPOT TEDDY TELEVISION TENNIS BALL THATCH THEATER CURTAIN THIMBLE THRESHER THRONE TILE ROOF TOASTER TOBACCO SHOP TOILET SEAT TORCH TOTEM POLE TOW TRUCK TOYSHOP TRACTOR TRAILER TRUCK TRAY TRENCH COAT TRICYCLE TRIMARAN TRIPOD TRIUMPHAL ARCH TROLLEYBUS TROMBONE TUB TURNSTILE TYPEWRITER KEYBOARD UMBRELLA UNICYCLE UPRIGHT VACUUM VASE VAULT VELVET VENDING MACHINE VESTMENT VIADUCT VIOLIN VOLLEYBALL WAFFLE IRON WALL CLOCK WALLET WARDROBE WARPLANE WASHBASIN WASHER WATER BOTTLE WATER JUG WATER TOWER WHISKEY JUG WHISTLE WIG WINDOW SCREEN WINDOW SHADE WINDSOR TIE WINE BOTTLE WING WOK WOODEN SPOON WOOL WORM FENCE WRECK YAWL YURT WEB SITE COMIC BOOK CROSSWORD PUZZLE STREET SIGN TRAFFIC LIGHT BOOK JACKET MENU PLATE GUACAMOLE CONSOMME HOT POT TRIFLE ICE CREAM ICE LOLLY FRENCH LOAF BAGEL PRETZEL CHEESEBURGER HOTDOG MASHED POTATO HEAD CABBAGE BROCCOLI CAULIFLOWER ZUCCHINI SPAGHETTI SQUASH ACORN SQUASH BUTTERNUT SQUASH CUCUMBER ARTICHOKE BELL PEPPER CARDOON MUSHROOM GRANNY SMITH STRAWBERRY ORANGE LEMON FIG PINEAPPLE BANANA JACKFRUIT CUSTARD APPLE POMEGRANATE HAY CARBONARA CHOCOLATE SAUCE DOUGH MEAT LOAF PIZZA POTPIE BURRITO RED WINE ESPRESSO CUP EGGNOG ALP BUBBLE CLIFF CORAL REEF GEYSER LAKESIDE PROMONTORY SANDBAR SEASHORE VALLEY VOLCANO BALLPLAYER GROOM SCUBA DIVER RAPESEED DAISY LADY SLIPPER CORN ACORN HIP BUCKEYE CORAL FUNGUS AGARIC GYROMITRA STINKHORN EARTHSTAR HEN-OF-THE-WOODS BOLETE EAR TOILET TISSUE kytensor-2.49.0.7/framework_backends/python_backend/examples/jax/000077500000000000000000000000001527052371200250545ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/jax/README.md000066400000000000000000000107261527052371200263410ustar00rootroot00000000000000 # JAX Example In this section, we demonstrate an end-to-end example for using [JAX](https://jax.readthedocs.io/en/latest/) in Python Backend. ## Create a JAX AddSub model repository We will use the files that come with this example to create the model repository. First, download the [client.py](client.py), [config.pbtxt](config.pbtxt) and [model.py](model.py) to your local machine. Next, at the directory where the three files located, create the model repository with the following commands: ``` mkdir -p models/jax/1 mv model.py models/jax/1 mv config.pbtxt models/jax ``` ## Pull the Triton Docker images We need to install Docker and NVIDIA Container Toolkit before proceeding, refer to the [installation steps](https://github.com/triton-inference-server/server/tree/main/docs#installation). To pull the latest containers, run the following commands: ``` docker pull nvcr.io/nvidia/tritonserver:-py3 docker pull nvcr.io/nvidia/tritonserver:-py3-sdk ``` See the installation steps above for the `` version. At the time of writing, the latest version is `23.04`, which translates to the following commands: ``` docker pull nvcr.io/nvidia/tritonserver:23.04-py3 docker pull nvcr.io/nvidia/tritonserver:23.04-py3-sdk ``` Be sure to replace the `` with the version pulled for all the remaining parts of this example. ## Start the Triton Server At the directory where we created the JAX models (at where the "models" folder is located), run the following command: ``` docker run --gpus all -it --rm -p 8000:8000 -v `pwd`:/jax nvcr.io/nvidia/tritonserver:-py3 /bin/bash ``` Inside the container, we need to install JAX to run this example. We recommend using the `pip` method mentioned in the [JAX documentation](https://github.com/google/jax#pip-installation-gpu-cuda). Make sure that JAX is available in the same Python environment as other dependencies. To install for this example, run the following command: ``` pip3 install --upgrade "jax[cuda12_local]" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html ``` Finally, we need to start the Triton Server, run the following command: ``` tritonserver --model-repository=/jax/models ``` To leave the container for the next step, press: `CTRL + P + Q`. ## Test inference At the directory where the client.py is located, run the following command: ``` docker run --rm --net=host -v `pwd`:/jax nvcr.io/nvidia/tritonserver:-py3-sdk python3 /jax/client.py ``` A successful inference will print the following at the end: ``` INPUT0 ([0.89262384 0.645457 0.18913145 0.17099917]) + INPUT1 ([0.5703733 0.21917151 0.22854741 0.97336507]) = OUTPUT0 ([1.4629972 0.86462855 0.41767886 1.1443642 ]) INPUT0 ([0.89262384 0.645457 0.18913145 0.17099917]) - INPUT1 ([0.5703733 0.21917151 0.22854741 0.97336507]) = OUTPUT0 ([ 0.32225055 0.4262855 -0.03941596 -0.8023659 ]) PASS: jax ``` Note: You inputs can be different from the above, but the outputs always correspond to its inputs. kytensor-2.49.0.7/framework_backends/python_backend/examples/jax/client.py000066400000000000000000000061341527052371200267100ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import tritonclient.http as httpclient from tritonclient.utils import * model_name = "jax" shape = [4] with httpclient.InferenceServerClient("localhost:8000") as client: input0_data = np.random.rand(*shape).astype(np.float32) input1_data = np.random.rand(*shape).astype(np.float32) inputs = [ httpclient.InferInput( "INPUT0", input0_data.shape, np_to_triton_dtype(input0_data.dtype) ), httpclient.InferInput( "INPUT1", input1_data.shape, np_to_triton_dtype(input1_data.dtype) ), ] inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs = [ httpclient.InferRequestedOutput("OUTPUT0"), httpclient.InferRequestedOutput("OUTPUT1"), ] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() output0_data = response.as_numpy("OUTPUT0") output1_data = response.as_numpy("OUTPUT1") print( "INPUT0 ({}) + INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output0_data ) ) print( "INPUT0 ({}) - INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output1_data ) ) if not np.allclose(input0_data + input1_data, output0_data): print("jax example error: incorrect sum") sys.exit(1) if not np.allclose(input0_data - input1_data, output1_data): print("jax example error: incorrect difference") sys.exit(1) print("PASS: jax") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/jax/config.pbtxt000066400000000000000000000036071527052371200274120ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "jax" backend: "python" input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] input [ { name: "INPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/jax/model.py000066400000000000000000000150711527052371200265320ustar00rootroot00000000000000# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import jax.numpy as jnp import numpy as np # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils def AddSub(input_0, input_1): """ Simple AddSub operations in JAX. This outputs the sum and subtraction of the inputs. JAX API: https://jax.readthedocs.io/en/latest/jax.html """ output_0 = jnp.add(input_0, input_1) output_1 = jnp.subtract(input_0, input_1) return [output_0, output_1] class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Absolute model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args["model_config"]) # Get OUTPUT0 configuration output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") # Get OUTPUT1 configuration output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") # Convert Triton types to numpy types self.output0_dtype = pb_utils.triton_string_to_numpy( output0_config["data_type"] ) self.output1_dtype = pb_utils.triton_string_to_numpy( output1_config["data_type"] ) def execute(self, requests): """`execute` must be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference is requested for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse. Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ output0_dtype = self.output0_dtype output1_dtype = self.output1_dtype responses = [] # Every Python backend must iterate over every one of the requests and # create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # Get INPUT1 in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") out_0, out_1 = AddSub(in_0.as_numpy(), in_1.as_numpy()) # Create output tensors. You need pb_utils.Tensor # objects to create pb_utils.InferenceResponse. out_tensor_0 = pb_utils.Tensor( "OUTPUT0", np.array(out_0).astype(output0_dtype) ) out_tensor_1 = pb_utils.Tensor( "OUTPUT1", np.array(out_1).astype(output1_dtype) ) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) inference_response = pb_utils.InferenceResponse( output_tensors=[out_tensor_0, out_tensor_1] ) responses.append(inference_response) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is optional. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/000077500000000000000000000000001527052371200271555ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/README.md000066400000000000000000000075431527052371200304450ustar00rootroot00000000000000 # Preprocessing Using Python Backend Example This example shows how to preprocess your inputs using Python backend before it is passed to the TensorRT model for inference. This ensemble model includes an image preprocessing model (preprocess) and a TensorRT model (resnet50_trt) to do inference. **1. Converting PyTorch Model to ONNX format:** Run onnx_exporter.py to convert ResNet50 PyTorch model to ONNX format. Width and height dims are fixed at 224 but dynamic axes arguments for dynamic batching are used. Commands from the 2. and 3. subsections shall be executed within this Docker container. docker run -it --gpus=all -v $(pwd):/workspace nvcr.io/nvidia/pytorch:xx.yy-py3 bash pip install numpy pillow torchvision python onnx_exporter.py --save model.onnx **2. Create the model repository:** mkdir -p model_repository/ensemble_python_resnet50/1 mkdir -p model_repository/preprocess/1 mkdir -p model_repository/resnet50_trt/1 # Copy the Python model cp model.py model_repository/preprocess/1 **3. Build a TensorRT engine for the ONNX model** Set the arguments for enabling fp16 precision --fp16. To enable dynamic shapes use --minShapes, --optShapes, and maxShapes with --explicitBatch: trtexec --onnx=model.onnx --saveEngine=./model_repository/resnet50_trt/1/model.plan --explicitBatch --minShapes=input:1x3x224x224 --optShapes=input:1x3x224x224 --maxShapes=input:256x3x224x224 --fp16 **4. Run the command below to start the server container:** Under python_backend/examples/preprocessing, run this command to start the server docker container: docker run --gpus=all -it --rm -p8000:8000 -p8001:8001 -p8002:8002 -v$(pwd):/workspace/ -v/$(pwd)/model_repository:/models nvcr.io/nvidia/tritonserver:xx.yy-py3 bash pip install numpy pillow torchvision tritonserver --model-repository=/models **5. Start the client to test:** Under python_backend/examples/preprocessing, run the commands below to start the client Docker container: wget https://raw.githubusercontent.com/triton-inference-server/server/main/qa/images/mug.jpg -O "mug.jpg" docker run --rm --net=host -v $(pwd):/workspace/ nvcr.io/nvidia/tritonserver:xx.yy-py3-sdk python client.py --image mug.jpg The result of classification is:COFFEE MUG Here, since we input an image of "mug" and the inference result is "COFFEE MUG" which is correct. kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/client.py000066400000000000000000000072601527052371200310120ustar00rootroot00000000000000# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import json import sys import numpy as np import tritonclient.grpc as tritongrpcclient def load_image(img_path: str): """ Loads an encoded image as an array of bytes. """ return np.fromfile(img_path, dtype="uint8") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--model_name", type=str, required=False, default="ensemble_python_resnet50", help="Model name", ) parser.add_argument("--image", type=str, required=True, help="Path to the image") parser.add_argument( "--url", type=str, required=False, default="localhost:8001", help="Inference server URL. Default is localhost:8001.", ) parser.add_argument( "-v", "--verbose", action="store_true", required=False, default=False, help="Enable verbose output", ) parser.add_argument( "--label_file", type=str, default="./model_repository/resnet50_trt/labels.txt", help="Path to the file with text representation of available labels", ) args = parser.parse_args() try: triton_client = tritongrpcclient.InferenceServerClient( url=args.url, verbose=args.verbose ) except Exception as e: print("channel creation failed: " + str(e)) sys.exit(1) with open(args.label_file) as f: labels_dict = {idx: line.strip() for idx, line in enumerate(f)} inputs = [] outputs = [] input_name = "INPUT" output_name = "OUTPUT" image_data = load_image(args.image) image_data = np.expand_dims(image_data, axis=0) inputs.append(tritongrpcclient.InferInput(input_name, image_data.shape, "UINT8")) outputs.append(tritongrpcclient.InferRequestedOutput(output_name)) inputs[0].set_data_from_numpy(image_data) results = triton_client.infer( model_name=args.model_name, inputs=inputs, outputs=outputs ) output0_data = results.as_numpy(output_name) print(output0_data) maxs = np.argmax(output0_data, axis=1) print(maxs) print("Result is class: {}".format(labels_dict[maxs[0]])) kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model.py000066400000000000000000000145231527052371200306340ustar00rootroot00000000000000# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import io import json import numpy as np import torchvision.transforms as transforms # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils from PIL import Image class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args["model_config"]) # Get OUTPUT0 configuration output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT_0") # Convert Triton types to numpy types self.output0_dtype = pb_utils.triton_string_to_numpy( output0_config["data_type"] ) def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ output0_dtype = self.output0_dtype responses = [] # Every Python backend must iterate over everyone of the requests # and create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT_0") normalize = transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ) loader = transforms.Compose( [ transforms.Resize([224, 224]), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ] ) def image_loader(image_name): image = loader(image_name) # expand the dimension to nchw image = image.unsqueeze(0) return image img = in_0.as_numpy() image = Image.open(io.BytesIO(img.tobytes())) img_out = image_loader(image) img_out = np.array(img_out) out_tensor_0 = pb_utils.Tensor("OUTPUT_0", img_out.astype(output0_dtype)) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) inference_response = pb_utils.InferenceResponse( output_tensors=[out_tensor_0] ) responses.append(inference_response) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model_repository/000077500000000000000000000000001527052371200325545ustar00rootroot00000000000000ensemble_python_resnet50/000077500000000000000000000000001527052371200374155ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model_repositoryconfig.pbtxt000066400000000000000000000043221527052371200417460ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model_repository/ensemble_python_resnet50# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "ensemble_python_resnet50" platform: "ensemble" max_batch_size: 256 input [ { name: "INPUT" data_type: TYPE_UINT8 dims: [ -1 ] } ] output [ { name: "OUTPUT" data_type: TYPE_FP32 dims: [ 1000 ] } ] ensemble_scheduling { step [ { model_name: "preprocess" model_version: -1 input_map { key: "INPUT_0" value: "INPUT" } output_map { key: "OUTPUT_0" value: "preprocessed_image" } }, { model_name: "resnet50_trt" model_version: -1 input_map { key: "input" value: "preprocessed_image" } output_map { key: "output" value: "OUTPUT" } } ] } preprocess/000077500000000000000000000000001527052371200346625ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model_repositoryconfig.pbtxt000066400000000000000000000034021527052371200372110ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model_repository/preprocess# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "preprocess" backend: "python" max_batch_size: 256 input [ { name: "INPUT_0" data_type: TYPE_UINT8 dims: [ -1 ] } ] output [ { name: "OUTPUT_0" data_type: TYPE_FP32 dims: [ 3, 224, 224 ] } ] instance_group [{ kind: KIND_CPU }] resnet50_trt/000077500000000000000000000000001527052371200350335ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model_repositoryconfig.pbtxt000066400000000000000000000034001527052371200373600ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model_repository/resnet50_trt# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "resnet50_trt" platform: "tensorrt_plan" max_batch_size: 256 input [ { name: "input" data_type: TYPE_FP32 dims: [3, -1, -1 ] } ] output[ { name: "output" data_type: TYPE_FP32 dims: [ 1000 ] label_filename: "labels.txt" } ] labels.txt000066400000000000000000000241101527052371200370340ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/model_repository/resnet50_trtTENCH GOLDFISH WHITE SHARK TIGER SHARK HAMMERHEAD SHARK ELECTRIC RAY STINGRAY ROOSTER HEN OSTRICH BRAMBLING GOLDFINCH HOUSE FINCH SNOWBIRD INDIGO FINCH ROBIN BULBUL JAY MAGPIE CHICKADEE WATER OUZEL KITE BALD EAGLE VULTURE GREAT GREY OWL FIRE SALAMANDER NEWT EFT SPOTTED SALAMANDER AXOLOTL BULL FROG TREE FROG TAILED FROG LOGGERHEAD LEATHERBACK TURTLE MUD TURTLE TERRAPIN BOX TURTLE BANDED GECKO COMMON IGUANA AMERICAN CHAMELEON WHIPTAIL AGAMA FRILLED LIZARD ALLIGATOR LIZARD GILA MONSTER GREEN LIZARD AFRICAN CHAMELEON KOMODO DRAGON AFRICAN CROCODILE AMERICAN ALLIGATOR TRICERATOPS THUNDER SNAKE RINGNECK SNAKE HOGNOSE SNAKE GREEN SNAKE KING SNAKE GARTER SNAKE WATER SNAKE VINE SNAKE NIGHT SNAKE BOA ROCK PYTHON COBRA GREEN MAMBA SEA SNAKE HORNED VIPER DIAMONDBACK SIDEWINDER TRILOBITE HARVESTMAN SCORPION GARDEN SPIDER BARN SPIDER GARDEN SPIDER BLACK WIDOW TARANTULA WOLF SPIDER TICK CENTIPEDE GROUSE PTARMIGAN RUFFED GROUSE PRAIRIE CHICKEN PEACOCK QUAIL PARTRIDGE AFRICAN GREY MACAW COCKATOO LORIKEET COUCAL BEE EATER HORNBILL HUMMINGBIRD JACAMAR TOUCAN DRAKE MERGANSER GOOSE BLACK SWAN TUSKER ECHIDNA PLATYPUS WALLABY KOALA WOMBAT JELLYFISH SEA ANEMONE BRAIN CORAL FLATWORM NEMATODE CONCH SNAIL SLUG SEA SLUG CHITON CHAMBERED NAUTILUS DUNGENESS CRAB ROCK CRAB FIDDLER CRAB KING CRAB AMERICAN LOBSTER SPINY LOBSTER CRAYFISH HERMIT CRAB ISOPOD WHITE STORK BLACK STORK SPOONBILL FLAMINGO LITTLE BLUE HERON AMERICAN EGRET BITTERN CRANE LIMPKIN EUROPEAN GALLINULE AMERICAN COOT BUSTARD RUDDY TURNSTONE RED-BACKED SANDPIPER REDSHANK DOWITCHER OYSTERCATCHER PELICAN KING PENGUIN ALBATROSS GREY WHALE KILLER WHALE DUGONG SEA LION CHIHUAHUA JAPANESE SPANIEL MALTESE DOG PEKINESE SHIH-TZU BLENHEIM SPANIEL PAPILLON TOY TERRIER RHODESIAN RIDGEBACK AFGHAN HOUND BASSET BEAGLE BLOODHOUND BLUETICK COONHOUND WALKER HOUND ENGLISH FOXHOUND REDBONE BORZOI IRISH WOLFHOUND ITALIAN GREYHOUND WHIPPET IBIZAN HOUND NORWEGIAN ELKHOUND OTTERHOUND SALUKI SCOTTISH DEERHOUND WEIMARANER STAFFORDSHIRE BULLTERRIER STAFFORDSHIRE TERRIER BEDLINGTON TERRIER BORDER TERRIER KERRY BLUE TERRIER IRISH TERRIER NORFOLK TERRIER NORWICH TERRIER YORKSHIRE TERRIER WIRE-HAIRED FOX TERRIER LAKELAND TERRIER SEALYHAM TERRIER AIREDALE CAIRN AUSTRALIAN TERRIER DANDIE DINMONT BOSTON BULL MINIATURE SCHNAUZER GIANT SCHNAUZER STANDARD SCHNAUZER SCOTCH TERRIER TIBETAN TERRIER SILKY TERRIER WHEATEN TERRIER WHITE TERRIER LHASA RETRIEVER CURLY-COATED RETRIEVER GOLDEN RETRIEVER LABRADOR RETRIEVER CHESAPEAKE BAY RETRIEVER SHORT-HAIRED POINTER VISLA ENGLISH SETTER IRISH SETTER GORDON SETTER BRITTANY SPANIEL CLUMBER ENGLISH SPRINGER WELSH SPRINGER SPANIEL COCKER SPANIEL SUSSEX SPANIEL IRISH WATERSPANIEL KUVASZ SCHIPPERKE GROENENDAEL MALINOIS BRIARD KELPIE KOMONDOR OLD ENGLISH SHEEPDOG SHETLAND SHEEPDOG COLLIE BORDER COLLIE BOUVIER DES FLANDRES ROTTWEILER GERMAN SHEPHERD DOBERMAN MINIATURE PINSCHER GREATER SWISS MOUNTAIN DOG BERNESE MOUNTAIN DOG APPENZELLER ENTLEBUCHER BOXER BULL MASTIFF TIBETAN MASTIFF FRENCH BULLDOG GREAT DANE SAINT BERNARD ESKIMO DOG MALAMUTE SIBERIAN HUSKY DALMATIAN AFFENPINSCHER BASENJI PUG LEONBERG NEWFOUNDLAND GREAT PYRENEES SAMOYED POMERANIAN CHOW KEESHOND BRABANCON GRIFFON PEMBROKE CARDIGAN TOY POODLE MINIATURE POODLE STANDARD POODLE MEXICAN HAIRLESS TIMBER WOLF WHITE WOLF RED WOLF COYOTE DINGO DHOLE AFRICAN HUNTING DOG HYENA RED FOX KIT FOX ARCTIC FOX GREY FOX TABBY TIGER CAT PERSIAN CAT SIAMESE CAT EGYPTIAN CAT COUGAR LYNX LEOPARD SNOW LEOPARD JAGUAR LION TIGER CHEETAH BROWN BEAR AMERICAN BLACK BEAR ICE BEAR SLOTH BEAR MONGOOSE MEERKAT TIGER BEETLE LADYBUG GROUND BEETLE LONG-HORNED BEETLE LEAF BEETLE DUNG BEETLE RHINOCEROS BEETLE WEEVIL FLY BEE ANT GRASSHOPPER CRICKET WALKING STICK COCKROACH MANTIS CICADA LEAFHOPPER LACEWING DRAGONFLY DAMSELFLY ADMIRAL RINGLET MONARCH CABBAGE BUTTERFLY SULPHUR BUTTERFLY LYCAENID STARFISH SEA URCHIN SEA CUCUMBER WOOD RABBIT HARE ANGORA HAMSTER PORCUPINE FOX SQUIRREL MARMOT BEAVER GUINEA PIG SORREL ZEBRA HOG WILD BOAR WARTHOG HIPPOPOTAMUS OX WATER BUFFALO BISON RAM BIGHORN IBEX HARTEBEEST IMPALA GAZELLE ARABIAN CAMEL LLAMA WEASEL MINK POLECAT BLACK-FOOTED FERRET OTTER SKUNK BADGER ARMADILLO THREE-TOED SLOTH ORANGUTAN GORILLA CHIMPANZEE GIBBON SIAMANG GUENON PATAS BABOON MACAQUE LANGUR COLOBUS PROBOSCIS MONKEY MARMOSET CAPUCHIN HOWLER MONKEY TITI SPIDER MONKEY SQUIRREL MONKEY MADAGASCAR CAT INDRI INDIAN ELEPHANT AFRICAN ELEPHANT LESSER PANDA GIANT PANDA BARRACOUTA EEL COHO ROCK BEAUTY ANEMONE FISH STURGEON GAR LIONFISH PUFFER ABACUS ABAYA ACADEMIC GOWN ACCORDION ACOUSTIC GUITAR AIRCRAFT CARRIER AIRLINER AIRSHIP ALTAR AMBULANCE AMPHIBIAN ANALOG CLOCK APIARY APRON ASHCAN ASSAULT RIFLE BACKPACK BAKERY BALANCE BEAM BALLOON BALLPOINT BAND AID BANJO BANNISTER BARBELL BARBER CHAIR BARBERSHOP BARN BAROMETER BARREL BARROW BASEBALL BASKETBALL BASSINET BASSOON BATHING CAP BATH TOWEL BATHTUB BEACH WAGON BEACON BEAKER BEARSKIN BEER BOTTLE BEER GLASS BELL COTE BIB BICYCLE-BUILT-FOR-TWO BIKINI BINDER BINOCULARS BIRDHOUSE BOATHOUSE BOBSLED BOLO TIE BONNET BOOKCASE BOOKSHOP BOTTLECAP BOW BOW TIE BRASS BRASSIERE BREAKWATER BREASTPLATE BROOM BUCKET BUCKLE BULLETPROOF VEST BULLET TRAIN BUTCHER SHOP CAB CALDRON CANDLE CANNON CANOE CAN OPENER CARDIGAN CAR MIRROR CAROUSEL CARPENTERS KIT CARTON CAR WHEEL CASH MACHINE CASSETTE CASSETTE PLAYER CASTLE CATAMARAN CD PLAYER CELLO CELLULAR TELEPHONE CHAIN CHAINLINK FENCE CHAIN MAIL CHAIN SAW CHEST CHIFFONIER CHIME CHINA CABINET CHRISTMAS STOCKING CHURCH CINEMA CLEAVER CLIFF DWELLING CLOAK CLOG COCKTAIL SHAKER COFFEE MUG COFFEEPOT COIL COMBINATION LOCK COMPUTER KEYBOARD CONFECTIONERY CONTAINER SHIP CONVERTIBLE CORKSCREW CORNET COWBOY BOOT COWBOY HAT CRADLE CRANE CRASH HELMET CREATE CRIB CROCK POT CROQUET BALL CRUTCH CUIRASS DAM DESK DESKTOP COMPUTER DIAL TELEPHONE DIAPER DIGITAL CLOCK DIGITAL WATCH DINING TABLE DISHRAG DISHWASHER DISK BRAKE DOCK DOGSLED DOME DOORMAT DRILLING PLATFORM DRUM DRUMSTICK DUMBBELL DUTCH OVEN ELECTRIC FAN ELECTRIC GUITAR ELECTRIC LOCOMOTIVE ENTERTAINMENT CENTER ENVELOPE ESPRESSO MAKER FACE POWDER FEATHER BOA FILE FIREBOAT FIRE ENGINE FIRE SCREEN FLAGPOLE FLUTE FOLDING CHAIR FOOTBALL HELMET FORKLIFT FOUNTAIN FOUNTAIN PEN FOUR-POSTER FREIGHT CAR FRENCH HORN FRYING PAN FUR COAT GARBAGE TRUCK GASMASK GAS PUMP GOBLET GO-KART GOLF BALL GOLFCART GONDOLA GONG GOWN GRAND PIANO GREENHOUSE GRILLE GROCERY STORE GUILLOTINE HAIR SLIDE HAIR SPRAY HALF TRACK HAMMER HAMPER HAND BLOWER HAND-HELD COMPUTER HANDKERCHIEF HARD DISC HARMONICA HARP HARVESTER HATCHET HOLSTER HOME THEATER HONEYCOMB HOOK HOOPSKIRT HORIZONTAL BAR HORSE CART HOURGLASS IPOD IRON JACK-O-LANTERN JEAN JEEP JERSEY JIGSAW PUZZLE JINRIKISHA JOYSTICK KIMONO KNEE PAD KNOT LAB COAT LADLE LAMPSHADE LAPTOP LAWN MOWER LENS CAP LETTER OPENER LIBRARY LIFEBOAT LIGHTER LIMOUSINE LINER LIPSTICK LOAFER LOTION LOUDSPEAKER LOUPE LUMBERMILL MAGNETIC COMPASS MAILBAG MAILBOX MAILLOT MAILLOT MANHOLE COVER MARACA MARIMBA MASK MATCHSTICK MAYPOLE MAZE MEASURING CUP MEDICINE CHEST MEGALITH MICROPHONE MICROWAVE MILITARY UNIFORM MILK CAN MINIBUS MINISKIRT MINIVAN MISSILE MITTEN MIXING BOWL MOBILE HOME MODEL T MODEM MONASTERY MONITOR MOPED MORTAR MORTARBOARD MOSQUE MOSQUITO NET MOTOR SCOOTER MOUNTAIN BIKE MOUNTAIN TENT MOUSE MOUSETRAP MOVING VAN MUZZLE NAIL NECK BRACE NECKLACE NIPPLE NOTEBOOK OBELISK OBOE OCARINA ODOMETER OIL FILTER ORGAN OSCILLOSCOPE OVERSKIRT OXCART OXYGEN MASK PACKET PADDLE PADDLEWHEEL PADLOCK PAINTBRUSH PAJAMA PALACE PANPIPE PAPER TOWEL PARACHUTE PARALLEL BARS PARK BENCH PARKING METER PASSENGER CAR PATIO PAY-PHONE PEDESTAL PENCIL BOX PENCIL SHARPENER PERFUME PETRI DISH PHOTOCOPIER PICK PICKELHAUBE PICKET FENCE PICKUP PIER PIGGY BANK PILL BOTTLE PILLOW PING-PONG BALL PINWHEEL PIRATE PITCHER PLANE PLANETARIUM PLASTIC BAG PLATE RACK PLOW PLUNGER POLAROID CAMERA POLE POLICE VAN PONCHO POOL TABLE POP BOTTLE POT POTTERS WHEEL POWER DRILL PRAYER RUG PRINTER PRISON PROJECTILE PROJECTOR PUCK PUNCHING BAG PURSE QUILL QUILT RACER RACKET RADIATOR RADIO RADIO TELESCOPE RAIN BARREL RECREATIONAL VEHICLE REEL REFLEX CAMERA REFRIGERATOR REMOTE CONTROL RESTAURANT REVOLVER RIFLE ROCKING CHAIR ROTISSERIE RUBBER ERASER RUGBY BALL RULE RUNNING SHOE SAFE SAFETY PIN SALTSHAKER SANDAL SARONG SAX SCABBARD SCALE SCHOOL BUS SCHOONER SCOREBOARD SCREEN SCREW SCREWDRIVER SEAT BELT SEWING MACHINE SHIELD SHOE SHOP SHOJI SHOPPING BASKET SHOPPING CART SHOVEL SHOWER CAP SHOWER CURTAIN SKI SKI MASK SLEEPING BAG SLIDE RULE SLIDING DOOR SLOT SNORKEL SNOWMOBILE SNOWPLOW SOAP DISPENSER SOCCER BALL SOCK SOLAR DISH SOMBRERO SOUP BOWL SPACE BAR SPACE HEATER SPACE SHUTTLE SPATULA SPEEDBOAT SPIDER WEB SPINDLE SPORTS CAR SPOTLIGHT STAGE STEAM LOCOMOTIVE STEEL ARCH BRIDGE STEEL DRUM STETHOSCOPE STOLE STONE WALL STOPWATCH STOVE STRAINER STREETCAR STRETCHER STUDIO COUCH STUPA SUBMARINE SUIT SUNDIAL SUNGLASS SUNGLASSES SUNSCREEN SUSPENSION BRIDGE SWAB SWEATSHIRT SWIMMING TRUNKS SWING SWITCH SYRINGE TABLE LAMP TANK TAPE PLAYER TEAPOT TEDDY TELEVISION TENNIS BALL THATCH THEATER CURTAIN THIMBLE THRESHER THRONE TILE ROOF TOASTER TOBACCO SHOP TOILET SEAT TORCH TOTEM POLE TOW TRUCK TOYSHOP TRACTOR TRAILER TRUCK TRAY TRENCH COAT TRICYCLE TRIMARAN TRIPOD TRIUMPHAL ARCH TROLLEYBUS TROMBONE TUB TURNSTILE TYPEWRITER KEYBOARD UMBRELLA UNICYCLE UPRIGHT VACUUM VASE VAULT VELVET VENDING MACHINE VESTMENT VIADUCT VIOLIN VOLLEYBALL WAFFLE IRON WALL CLOCK WALLET WARDROBE WARPLANE WASHBASIN WASHER WATER BOTTLE WATER JUG WATER TOWER WHISKEY JUG WHISTLE WIG WINDOW SCREEN WINDOW SHADE WINDSOR TIE WINE BOTTLE WING WOK WOODEN SPOON WOOL WORM FENCE WRECK YAWL YURT WEB SITE COMIC BOOK CROSSWORD PUZZLE STREET SIGN TRAFFIC LIGHT BOOK JACKET MENU PLATE GUACAMOLE CONSOMME HOT POT TRIFLE ICE CREAM ICE LOLLY FRENCH LOAF BAGEL PRETZEL CHEESEBURGER HOTDOG MASHED POTATO HEAD CABBAGE BROCCOLI CAULIFLOWER ZUCCHINI SPAGHETTI SQUASH ACORN SQUASH BUTTERNUT SQUASH CUCUMBER ARTICHOKE BELL PEPPER CARDOON MUSHROOM GRANNY SMITH STRAWBERRY ORANGE LEMON FIG PINEAPPLE BANANA JACKFRUIT CUSTARD APPLE POMEGRANATE HAY CARBONARA CHOCOLATE SAUCE DOUGH MEAT LOAF PIZZA POTPIE BURRITO RED WINE ESPRESSO CUP EGGNOG ALP BUBBLE CLIFF CORAL REEF GEYSER LAKESIDE PROMONTORY SANDBAR SEASHORE VALLEY VOLCANO BALLPLAYER GROOM SCUBA DIVER RAPESEED DAISY LADY SLIPPER CORN ACORN HIP BUCKEYE CORAL FUNGUS AGARIC GYROMITRA STINKHORN EARTHSTAR HEN-OF-THE-WOODS BOLETE EAR TOILET TISSUE kytensor-2.49.0.7/framework_backends/python_backend/examples/preprocessing/onnx_exporter.py000066400000000000000000000044071527052371200324460ustar00rootroot00000000000000# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os import torch import torchvision.models as models if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--save", default="model.onnx") args = parser.parse_args() resnet50 = models.resnet50(pretrained=True) dummy_input = torch.randn(1, 3, 224, 224) resnet50 = resnet50.eval() torch.onnx.export( resnet50, dummy_input, args.save, export_params=True, opset_version=10, do_constant_folding=True, input_names=["input"], output_names=["output"], dynamic_axes={ "input": {0: "batch_size", 2: "height", 3: "width"}, "output": {0: "batch_size"}, }, ) print("Saved {}".format(args.save)) kytensor-2.49.0.7/framework_backends/python_backend/examples/pytorch/000077500000000000000000000000001527052371200257625ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/examples/pytorch/client.py000066400000000000000000000061611527052371200276160ustar00rootroot00000000000000# Copyright 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import numpy as np import tritonclient.http as httpclient from tritonclient.utils import * model_name = "pytorch" shape = [4] with httpclient.InferenceServerClient("localhost:8000") as client: input0_data = np.random.rand(*shape).astype(np.float32) input1_data = np.random.rand(*shape).astype(np.float32) inputs = [ httpclient.InferInput( "INPUT0", input0_data.shape, np_to_triton_dtype(input0_data.dtype) ), httpclient.InferInput( "INPUT1", input1_data.shape, np_to_triton_dtype(input1_data.dtype) ), ] inputs[0].set_data_from_numpy(input0_data) inputs[1].set_data_from_numpy(input1_data) outputs = [ httpclient.InferRequestedOutput("OUTPUT0"), httpclient.InferRequestedOutput("OUTPUT1"), ] response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs) result = response.get_response() output0_data = response.as_numpy("OUTPUT0") output1_data = response.as_numpy("OUTPUT1") print( "INPUT0 ({}) + INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output0_data ) ) print( "INPUT0 ({}) - INPUT1 ({}) = OUTPUT0 ({})".format( input0_data, input1_data, output1_data ) ) if not np.allclose(input0_data + input1_data, output0_data): print("pytorch example error: incorrect sum") sys.exit(1) if not np.allclose(input0_data - input1_data, output1_data): print("pytorch example error: incorrect difference") sys.exit(1) print("PASS: pytorch") sys.exit(0) kytensor-2.49.0.7/framework_backends/python_backend/examples/pytorch/config.pbtxt000066400000000000000000000036021527052371200303130ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "pytorch" backend: "python" input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] input [ { name: "INPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ 4 ] } ] output [ { name: "OUTPUT1" data_type: TYPE_FP32 dims: [ 4 ] } ] instance_group [{ kind: KIND_CPU }] kytensor-2.49.0.7/framework_backends/python_backend/examples/pytorch/model.py000066400000000000000000000150211527052371200274330ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json # triton_python_backend_utils is available in every Triton Python model. You # need to use this module to create inference requests and responses. It also # contains some utility functions for extracting information from model_config # and converting Triton input/output types to numpy types. import triton_python_backend_utils as pb_utils from torch import nn class AddSubNet(nn.Module): """ Simple AddSub network in PyTorch. This network outputs the sum and subtraction of the inputs. """ def __init__(self): super(AddSubNet, self).__init__() def forward(self, input0, input1): return (input0 + input1), (input0 - input1) class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args["model_config"]) # Get OUTPUT0 configuration output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") # Get OUTPUT1 configuration output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") # Convert Triton types to numpy types self.output0_dtype = pb_utils.triton_string_to_numpy( output0_config["data_type"] ) self.output1_dtype = pb_utils.triton_string_to_numpy( output1_config["data_type"] ) # Instantiate the PyTorch model self.add_sub_model = AddSubNet() def execute(self, requests): """`execute` must be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference is requested for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse. Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ output0_dtype = self.output0_dtype output1_dtype = self.output1_dtype responses = [] # Every Python backend must iterate over everyone of the requests # and create a pb_utils.InferenceResponse for each of them. for request in requests: # Get INPUT0 in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") # Get INPUT1 in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") out_0, out_1 = self.add_sub_model(in_0.as_numpy(), in_1.as_numpy()) # Create output tensors. You need pb_utils.Tensor # objects to create pb_utils.InferenceResponse. out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(output0_dtype)) out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(output1_dtype)) # Create InferenceResponse. You can set an error here in case # there was a problem with handling this inference request. # Below is an example of how you can set errors in inference # response: # # pb_utils.InferenceResponse( # output_tensors=..., TritonError("An error occurred")) inference_response = pb_utils.InferenceResponse( output_tensors=[out_tensor_0, out_tensor_1] ) responses.append(inference_response) # You should return a list of pb_utils.InferenceResponse. Length # of this list must match the length of `requests` list. return responses def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is optional. This function allows the model to perform any necessary clean ups before exit. """ print("Cleaning up...") kytensor-2.49.0.7/framework_backends/python_backend/inferentia/000077500000000000000000000000001527052371200246005ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/inferentia/README.md000066400000000000000000000447401527052371200260700ustar00rootroot00000000000000 # Using Triton with Inferentia 1 Starting from 21.11 release, Triton supports [AWS Inferentia](https://aws.amazon.com/machine-learning/inferentia/) and the [Neuron Runtime](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-intro/get-started.html). ## Table of Contents - [Using Triton with Inferentia 1](#using-triton-with-inferentia-1) - [Table of Contents](#table-of-contents) - [Inferentia setup](#inferentia-setup) - [Setting up the Inferentia model](#setting-up-the-inferentia-model) - [PyTorch](#pytorch) - [TensorFlow](#tensorflow) - [Serving Inferentia model in Triton](#serving-inferentia-model-in-triton) - [Using Triton's Dynamic Batching](#using-tritons-dynamic-batching) - [Testing Inferentia Setup for Accuracy](#testing-inferentia-setup-for-accuracy) ## Inferentia setup First step of running Triton with Inferentia is to create an AWS Inferentia instance with Deep Learning AMI (tested with Ubuntu 18.04). `ssh -i .pem ubuntu@` Note: It is recommended to set your storage space to greater than default value of 110 GiB. The current version of Triton has been tested with storage of 500 GiB. After logging into the inf1* instance, you will need to clone [this current Github repo](https://github.com/triton-inference-server/python_backend). Follow [steps on Github to set up ssh access](https://docs.github.com/en/authentication/connecting-to-github-with-ssh) or simply clone with https. Clone this repo with Github to home repo `/home/ubuntu`. ``` chmod 777 /home/ubuntu/python_backend/inferentia/scripts/setup-pre-container.sh sudo /home/ubuntu/python_backend/inferentia/scripts/setup-pre-container.sh ``` Then, start the Triton instance with: ``` docker run --device /dev/neuron0 -v /home/ubuntu/python_backend:/home/ubuntu/python_backend -v /lib/udev:/mylib/udev --shm-size=1g --ulimit memlock=-1 -p 8000:8000 -p 8001:8001 -p 8002:8002 --ulimit stack=67108864 -ti nvcr.io/nvidia/tritonserver:-py3 ``` Note 1: The user would need to list any neuron device to run during container initialization. For example, to use 4 neuron devices on an instance, the user would need to run with: ``` docker run --device /dev/neuron0 --device /dev/neuron1 --device /dev/neuron2 --device /dev/neuron3 ...` ``` Note 2: `/mylib/udev` is used for Neuron parameter passing. Note 3: For Triton container version xx.yy, please refer to [Triton Inference Server Container Release Notes](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/index.html). The current build script has been tested with container version `21.10`. After starting the Triton container, go into the `python_backend` folder and run the setup script. ``` source /home/ubuntu/python_backend/inferentia/scripts/setup.sh ``` This script will: 1. Install necessary dependencies 2. Install [neuron-cc](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-cc/index.html), the Neuron compiler. 3. Install neuron framework packages as per your preference e.g., either pytorch, or tensorflow or both. There are user configurable options available for the script as well. Please use the `-h` or `--help` options to learn about more configurable options. ## Setting up the Inferentia model Currently, we only support [PyTorch](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/pytorch-neuron/index.html) and [TensorFlow](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/tensorflow-neuron/index.html) workflows for execution on inferentia. The user is required to create their own `*.pt` (for pytorch) or `*.savedmodels` (for tensorflow) models. This is a critical step since Inferentia will need the underlying `.NEFF` graph to execute the inference request. Please refer to: - [Neuron compiler CLI Reference Guide](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-cc/command-line-reference.html) - [PyTorch-Neuron trace python API](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/pytorch-neuron/api-compilation-python-api.html) - [PyTorch Tutorials](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/pytorch-neuron/tutorials/index.html) - [TensorFlow Tutorials](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/tensorflow-neuron/tutorials/index.html) for guidance on how to compile models. ### PyTorch For PyTorch, we support models traced by [PyTorch-Neuron trace python API](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/pytorch-neuron/api-compilation-python-api.html) for execution on Inferentia. Once the TorchScript model supporting Inferentia is obtained, use the [gen_triton_model.py](scripts/gen_triton_model.py) script to generate triton python model directory. An example invocation for the `gen_triton_model.py` for PyTorch model can look like: ``` python3 inferentia/scripts/gen_triton_model.py --model_type pytorch --triton_input INPUT__0,INT64,4x384 INPUT__1,INT64,4x384 INPUT__2,INT64,4x384 --triton_output OUTPUT__0,INT64,4x384 OUTPUT__1,INT64,4x384 --compiled_model /home/ubuntu/bert_large_mlperf_neuron_hack_bs1_dynamic.pt --neuron_core_range 0:3 --triton_model_dir bert-large-mlperf-bs1x4 ``` In order for the script to treat the compiled model as TorchScript model, `--model_type pytorch` needs to be provided. NOTE: Due to the absence of metadata for inputs and outputs in a TorchScript model - name, datatype and shape of tensor of both the inputs and outputs must be provided to the above script and the name must follow a specific naming convention i.e. `__`. Where `` can be any string and `` refers to the position of the corresponding input/output. This means if there are two inputs and two outputs they must be named as: "INPUT__0", "INPUT__1" and "OUTPUT__0", "OUTPUT__1" such that "INPUT__0" refers to first input and INPUT__1 refers to the second input, etc. Additionally, `--neuron_core_range` specifies the neuron cores to be used while serving this models. Currently, only `torch.neuron.DataParallel()` mode is supported. See [Data Parallel Inference](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/appnotes/perf/torch-neuron-dataparallel-app-note.html) for more information. Triton model instance count can be specified by using `--triton_model_instance_count` option. The neuron cores will be equally distributed among all instances. For example, in case of two triton model instances and 4 neuron cores, the first instance will be loaded on on cores 0-1 and second instance will be loaded on cores 2-3. To best engage inferentia device, try setting the number of neuron cores to be a proper multiple of the instance count. ### TensorFlow For TensorFlow, the model must be compiled for AWS Neuron. See [AWS Neuron TensorFlow](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/tensorflow-neuron/tutorials/index.html) tutorials to learn how to get a compiled model that uses Neuron cores. Currently, the code is tested only on `tensorflow==1.15`. Once the compiled model is obtained use [gen_triton_model.py](scripts/gen_triton_model.py) script to generate triton python model directory. An example invocation for the `gen_triton_model.py` for TensorFlow model can look like: ``` python3 gen_triton_model.py --model_type tensorflow --compiled_model /home/ubuntu/inferentia-poc-2.0/scripts-rn50-tf-native/resnet50_mlperf_opt_fp16_compiled_b5_nc1/1 --neuron_core_range 0:3 --triton_model_dir rn50-1neuroncores-bs1x1 ``` NOTE: Unlike TorchScript model, TensorFlow SavedModel stores sufficient metadata to detect the name, datatype and shape of the input and output tensors for the model. By default, the script will assume the compiled model to be torchscript. In order for it to treat the compiled model as TF savedmodel, `--model_type tensorflow` needs to be provided. The input and output details are read from the model itself. The user must have [`tensorflow`](https://www.tensorflow.org/install/pip) python module installed in order to use this script for tensorflow models. Similar to PyTorch, `--neuron_core_range` and `--triton_model_instance_count` can be used to specify the neuron core range and number of triton model instances. However, the neuron core indices don't point to a specific neuron core in the chip. For TensorFlow, we use deprecated feature of `NEURONCORE_GROUP_SIZES` to load model. The model in this case will be loaded on next available Neuron cores and not specific ones. See [Parallel Execution using NEURONCORE_GROUP_SIZES](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/appnotes/perf/parallel-ncgs.html?highlight=NEURONCORE_GROUP_SIZES) for more information. Another note, since Neuron-Tensorflow(unlike Neuron-Python) does not have built-in functions for running a model for multiple cores, `model.py` will distribute the workload by splitting the input tensor across available cores. It is recommended the first dimension for the inputs be `None` if the user enables processing across multiple cores. Please use the `-h` or `--help` options in `gen_triton_model.py` to learn about more configurable options. ## Serving Inferentia model in Triton The `gen_triton_model.py` should create a triton model directory with following structutre: ``` bert-large-mlperf-bs1x4 | |- 1 | |- model.py | |- config.pbtxt ``` Look at the usage message of the script to understand each option. The script will generate a model directory with the user-provided name. Move that model directory to Triton's model repository. Ensure the compiled model path provided to the script points to a valid torchscript file or tensorflow savedmodel. Now, the server can be launched with the model as below: ``` tritonserver --model-repository ``` Note: 1. The `config.pbtxt` and `model.py` should be treated as starting point. The users can customize these files as per their need. 2. Triton Inferentia is currently tested with a **single** model. ### Using Triton's Dynamic Batching To enable dynamic batching, `--enable_dynamic_batching` flag needs to be specified. `gen_triton_model.py` supports following three options for configuring [Triton's dynamic batching](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md): 1. `--preferred_batch_size`: Please refer to [model configuration documentation](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#preferred-batch-sizes) for details on preferred batch size. To optimize performance, this is recommended to be multiples of engaged neuron cores. For example, if each instance is using 2 neuron cores, `preferred_batch_size` could be 2, 4 or 6. 2. `--max_queue_delay_microseconds`: Please refer to [model configuration documentation](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#delayed-batching) for details. 3. `--disable_batch_requests_to_neuron`: Enable the non-default way for Triton to handle batched requests. Triton backend will send each request to neuron separately, irrespective of if the Triton server requests are batched. This flag is recommended when users want to optimize performance with models that do not perform well with batching without the flag. Additionally, `--max_batch_size` will affect the maximum batching limit. Please refer to the [model configuration documentation](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#maximum-batch-size) for details. ## Testing Inferentia Setup for Accuracy The [qa folder](https://github.com/triton-inference-server/python_backend/tree/main/inferentia/qa) contains the necessary files to set up testing with a simple add_sub model. The test requires an instance with more than 8 inferentia cores to run, eg:`inf1.6xlarge`. start the test, run ``` source /python_backend/inferentia/qa/setup_test_enviroment_and_test.sh ``` where `` is usually `/home/ubuntu`/. This script will pull the [server repo](https://github.com/triton-inference-server/server) that contains the tests for inferentia. It will then build the most recent Triton Server and Triton SDK. Note: If you would need to change some of the tests in the server repo, you would need to run ``` export TRITON_SERVER_REPO_TAG= ``` before running the script. # Using Triton with Inferentia 2, or Trn1 ## pytorch-neuronx and tensorflow-neuronx 1. Similar to the steps for inf1, change the argument to the pre-container and on-container setup scripts to include the `-inf2` or `-trn1`flags e.g., ``` chmod 777 /home/ubuntu/python_backend/inferentia/scripts/setup-pre-container.sh sudo /home/ubuntu/python_backend/inferentia/scripts/setup-pre-container.sh -inf2 ``` 2. On the container, followed by the `docker run` command, you can pass similar argument to the setup.sh script For Pytorch: ``` source /home/ubuntu/python_backend/inferentia/scripts/setup.sh -inf2 -p ``` For Tensorflow: ``` source /home/ubuntu/python_backend/inferentia/scripts/setup.sh -inf2 -t ``` 3. Following the above steps, when using the `gen_triton_model.py` script, you can pass similar argument `--inf2` to the setup.sh script e.g., for Pytorch ``` python3 inferentia/scripts/gen_triton_model.py --inf2 --model_type pytorch --triton_input INPUT__0,INT64,4x384 INPUT__1,INT64,4x384 INPUT__2,INT64,4x384 --triton_output OUTPUT__0,INT64,4x384 OUTPUT__1,INT64,4x384 --compiled_model bert_large_mlperf_neuron_hack_bs1_dynamic.pt --neuron_core_range 0:3 --triton_model_dir bert-large-mlperf-bs1x4 ``` 4. **Note**: When using the `--inf2` option, the `--compiled_model` path should be provided relative to the triton model directory. The `initialize()` function in model.py will derive the full path by concatenating the model path within the repository and the relative `--compiled_model` path. ## transformers-neuronx To use inf2/trn1 instances with transformers-neuronx packages for serving models, generate a `pytorch` model as per above instructions. The transformers-neuronx currently supports the models listed [here](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/frameworks/torch/torch-neuronx/transformers-neuronx/readme.html#currently-supported-models). As prescribed on the neuronx documentation page, while the neuronx load API differs per model, it follows the same pattern. 1. To serve transformers-neuronx models, first trace the model using `save_pretrained_split()` API on an inf2 instance (recommend inf2.24xl for Large Language Models). Following that, package the folder as the '--compiled_model' when using `gen_triton_model.py` file. 2. The following tree shows a sample model structure for OPT model: ``` opt/ ├── 1 │   └── model.py ├── opt-125m-model │   └── pytorch_model.bin └── opt-125m-tp12 ├── FullyUnrolled.1814.1 │   ├── penguin-sg0000 │   └── sg00 ├── FullyUnrolled.1814.2 │   ├── penguin-sg0000 │   └── sg00 ├── FullyUnrolled.1814.3 │   ├── penguin-sg0000 │   └── sg00 ├── FullyUnrolled.1814.4 │   ├── penguin-sg0000 │   └── sg00 └── FullyUnrolled.1814.5 ├── penguin-sg0000 └── sg00 ├── config.pbtxt ``` 3. Add the following imports (e.g., for OPT model). The import will differ as per the model you're trying to run. ``` from transformers_neuronx.opt.model import OPTForSampling ``` 4. Add the following lines in `initialize()` function. Set the `batch_size`, `tp_degree`, `n_positions`, `amp` and `unroll` args as per your requirement. `tp_degree` should typically match the number of neuron cores available on inf2 instance. ``` batch_size = 1 tp_degree = 12 n_positions = 2048 amp = 'bf16' unroll = None self.model_neuron = OPTForSampling.from_pretrained(compiled_model, batch_size=batch_size, amp=amp, tp_degree=tp_degree, n_positions=n_positions, unroll=unroll) self.model_neuron.to_neuron() self.model_neuron.num_workers = num_threads ``` You may also chose to add the `batch_size` etc. arguments to config.pbtxt as parameters and read them in the `initialize()` function similar to `--compiled-model`. 5. Finally, in the `execute()` function, use the following API to run the inference: ``` batched_results = self.model_neuron.sample(batched_tensor, 2048) ``` Above, `2048` is a sufficiently-long output token. It may also be passed in as one of the inputs if you wanto specify it as part of the payload. 6. Proceed to load the model, and submit the inference payload similar to any other triton model.kytensor-2.49.0.7/framework_backends/python_backend/inferentia/qa/000077500000000000000000000000001527052371200252015ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/inferentia/qa/Dockerfile.QA000066400000000000000000000071531527052371200275010ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Multistage build. # ARG BASE_IMAGE=tritonserver ARG BUILD_IMAGE=tritonserver_build ARG SDK_IMAGE=tritonserver_sdk ARG TRITON_PATH=/home/ubuntu FROM ${SDK_IMAGE} AS sdk FROM $BASE_IMAGE # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive # install platform specific packages RUN if [ $(cat /etc/os-release | grep 'VERSION_ID="20.04"' | wc -l) -ne 0 ]; then \ apt-get update && \ apt-get install -y --no-install-recommends \ libpng-dev; \ elif [ $(cat /etc/os-release | grep 'VERSION_ID="18.04"' | wc -l) -ne 0 ]; then \ apt-get update && \ apt-get install -y --no-install-recommends \ libpng-dev; \ else \ echo "Ubuntu version must be either 18.04 or 20.04" && \ exit 1; \ fi RUN apt-get update && apt-get install -y --no-install-recommends \ python3-dev \ python3-pip \ build-essential \ wget && \ rm -rf /var/lib/apt/lists/* RUN rm -f /usr/bin/python && \ ln -s /usr/bin/python3 /usr/bin/python RUN pip3 install --upgrade wheel setuptools && \ pip3 install --upgrade numpy pillow attrdict future grpcio requests gsutil awscli six grpcio-channelz WORKDIR /opt/tritonserver # Copy the entire qa repo to the /opt/tritonserver/qa repo COPY --from=tritonserver_build /workspace/qa qa COPY --chown=1000:1000 --from=sdk /workspace/install client_tmp RUN mkdir -p qa/clients && mkdir -p qa/pkgs && \ cp -a client_tmp/bin/* qa/clients/. && \ cp client_tmp/lib/libgrpcclient.so qa/clients/. && \ cp client_tmp/lib/libhttpclient.so qa/clients/. && \ cp client_tmp/python/*.py qa/clients/. && \ cp client_tmp/python/triton*.whl qa/pkgs/. && \ cp client_tmp/java/examples/*.jar qa/clients/. && \ rm -rf client_tmp # Create mount paths for lib RUN mkdir /mylib && mkdir /home/ubuntu ENV TRITON_PATH ${TRITON_PATH} ENV LD_LIBRARY_PATH /opt/tritonserver/qa/clients:${LD_LIBRARY_PATH} kytensor-2.49.0.7/framework_backends/python_backend/inferentia/qa/setup_test_enviroment_and_test.sh000077500000000000000000000240071527052371200340710ustar00rootroot00000000000000#!/bin/bash # Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. export TRITON_PATH="/home/ubuntu" export DEFAULT_REPO_TAG=${DEFAULT_REPO_TAG:="main"} export TRITON_COMMON_REPO_TAG=${DEFAULT_REPO_TAG} export TRITON_CORE_REPO_TAG=${DEFAULT_REPO_TAG} export TRITON_BACKEND_REPO_TAG=${DEFAULT_REPO_TAG} export TRITON_THIRD_PARTY_REPO_TAG=${DEFAULT_REPO_TAG} export IDENTITY_BACKEND_REPO_TAG=${DEFAULT_REPO_TAG} export PYTHON_BACKEND_REPO_TAG=${DEFAULT_REPO_TAG} export CHECKSUM_REPOAGENT_REPO_TAG=${DEFAULT_REPO_TAG} export TRITON_SERVER_REPO_TAG=${TRITON_SERVER_REPO_TAG:=${DEFAULT_REPO_TAG}} export TRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG:=${DEFAULT_REPO_TAG}} export BASE_IMAGE=tritonserver export SDK_IMAGE=tritonserver_sdk export BUILD_IMAGE=tritonserver_build export QA_IMAGE=tritonserver_qa export TEST_JSON_REPO=/opt/tritonserver/qa/common/inferentia_perf_analyzer_input_data_json export TEST_REPO=/opt/tritonserver/qa/L0_inferentia_perf_analyzer export TEST_SCRIPT="test.sh" CONTAINER_NAME="qa_container" CONTAINER_VERSION="" UPSTREAM_CONTAINER_VERSION="" USAGE=" usage: setup_test_enviroment_and_test.sh [options]. These setting will override exported variables Setup environment for testing on Inferentia chips and run perf analyzer tests. -h|--help Shows usage -d|--default-repo-tag DEFAULT_REPO_TAG for building the test container. Default is main -s|--server-repo-tag TRITON_SERVER_REPO_TAG for building test container. Default same DEFAULT_REPO_TAG -c|--client-repo-tag TRITON_CLIENT_REPO_TAG for building test container. Default same DEFAULT_REPO_TAG -v|--container-version Container version used in build.py. Default is container version used in build.py -u|--upstream-container-version Upstream container version for test container. Default is container version used in build.py -p|--triton-path The path where python backend is located and where server repo will be cloned to. Default is /home/ubuntu " # Get all options: OPTS=$(getopt -o hd:s:c:v:u:p: --long help,default-repo-tag:,server-repo-tag:,client-repo-tag:,container-version:,upstream-container-version:,triton-path -- "$@") for OPTS; do case "$OPTS" in -h|--help) printf "%s\\n" "$USAGE" return 0 ;; -d|--default-repo-tag) export DEFAULT_REPO_TAG=$2 echo "Default repo tag set to: ${DEFAULT_REPO_TAG}" shift 2 ;; -s|--server-repo-tag) export TRITON_SERVER_REPO_TAG=$2 shift 2 echo "Server repo tag set to: ${TRITON_SERVER_REPO_TAG}" ;; -c|--client-repo-tag) export TRITON_CLIENT_REPO_TAG=$2 echo "Client repo tag set to: ${TRITON_CLIENT_REPO_TAG}" shift 2 ;; -v|--container-version) export CONTAINER_VERSION=$2 echo "Container version set to: ${CONTAINER_VERSION}" shift 2 ;; -u|--upstream-container-version) export UPSTREAM_CONTAINER_VERSION=$2 echo "Upstream container version set to: ${UPSTREAM_CONTAINER_VERSION}" shift 2 ;; -p|--triton-path) export TRITON_PATH=$2 echo "Triton path set to: ${TRITON_PATH}" shift 2 ;; esac done cd ${TRITON_PATH} echo "Using server repo tag: $TRITON_SERVER_REPO_TAG" # Clone necessary branches rm -rf ${TRITON_PATH}/server git clone --single-branch --depth=1 -b ${TRITON_SERVER_REPO_TAG} \ https://github.com/triton-inference-server/server.git cd ${TRITON_PATH}/server git clone --single-branch --depth=1 -b ${TRITON_CLIENT_REPO_TAG} \ https://github.com/triton-inference-server/client.git clientrepo # First set up inferentia and run in detached mode cd ${TRITON_PATH}/python_backend chmod 777 ${TRITON_PATH}/python_backend/inferentia/scripts/setup-pre-container.sh sudo ${TRITON_PATH}/python_backend/inferentia/scripts/setup-pre-container.sh # If container version is not known, look up container version and upstream container version from build.py cd ${TRITON_PATH}/server if [ "${CONTAINER_VERSION}" = "" ]; then QUERY_STRING="import build; container_version,_= build.container_versions('$(cat TRITON_VERSION)', None, None); print(container_version)" CONTAINER_VERSION=$(python3 -c "${QUERY_STRING}") echo "found container version: ${CONTAINER_VERSION} from build.py" fi if [ "${UPSTREAM_CONTAINER_VERSION}" = "" ]; then QUERY_STRING="import build; _,upstream_container_version = build.container_versions('$(cat TRITON_VERSION)', None, None); print(upstream_container_version)" UPSTREAM_CONTAINER_VERSION=$(python3 -c "${QUERY_STRING}") echo "found upstream container version: ${UPSTREAM_CONTAINER_VERSION} from build.py" fi # Build container with only python backend cd ${TRITON_PATH}/server pip3 install docker ./build.py --container-version=${CONTAINER_VERSION} \ --upstream-container-version=${UPSTREAM_CONTAINER_VERSION} \ --enable-logging --enable-stats --enable-tracing \ --enable-metrics --enable-gpu-metrics --enable-gpu \ --filesystem=gcs --filesystem=azure_storage --filesystem=s3 \ --endpoint=http --endpoint=grpc \ --repo-tag=common:${TRITON_COMMON_REPO_TAG} \ --repo-tag=core:${TRITON_CORE_REPO_TAG} \ --repo-tag=backend:${TRITON_BACKEND_REPO_TAG} \ --repo-tag=thirdparty:${TRITON_THIRD_PARTY_REPO_TAG} \ --backend=identity:${IDENTITY_BACKEND_REPO_TAG} \ --backend=python:${PYTHON_BACKEND_REPO_TAG} \ --repoagent=checksum:${CHECKSUM_REPOAGENT_REPO_TAG} docker tag tritonserver_buildbase "${BUILD_IMAGE}" docker tag tritonserver "${BASE_IMAGE}" # Build docker container for SDK docker build -t ${SDK_IMAGE} \ -f ${TRITON_PATH}/server/Dockerfile.sdk \ --build-arg "BASE_IMAGE=nvcr.io/nvidia/tritonserver:${UPSTREAM_CONTAINER_VERSION}-py3-min" \ --build-arg "TRITON_CLIENT_REPO_SUBDIR=clientrepo" \ --build-arg "TRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG}" \ --build-arg "TRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG}" \ --build-arg "TRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG}" \ --build-arg "TRITON_THIRD_PARTY_REPO_TAG=${TRITON_THIRD_PARTY_REPO_TAG}" \ --build-arg "NVIDIA_TRITON_SERVER_SDK_VERSION=${CONTAINER_VERSION}" . # Build QA container docker build -t ${QA_IMAGE} \ -f ${TRITON_PATH}/python_backend/inferentia/qa/Dockerfile.QA \ --build-arg "TRITON_PATH=${TRITON_PATH}" \ --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ --build-arg "BUILD_IMAGE=${BUILD_IMAGE}" \ --build-arg "SDK_IMAGE=${SDK_IMAGE}" . # Run pytorch instance test docker stop ${CONTAINER_NAME} && docker rm ${CONTAINER_NAME} docker create --name ${CONTAINER_NAME} \ --device /dev/neuron0 \ --device /dev/neuron1 \ --shm-size=1g --ulimit memlock=-1 \ -p 8000:8000 -p 8001:8001 -p 8002:8002 \ --ulimit stack=67108864 \ -e TEST_REPO=${TEST_REPO} \ -e TEST_JSON_REPO=${TEST_JSON_REPO} \ -e TRITON_PATH=${TRITON_PATH} \ -e USE_PYTORCH="1" \ --net host -ti ${QA_IMAGE} \ /bin/bash -c "bash -ex ${TEST_REPO}/${TEST_SCRIPT}" && \ docker cp /lib/udev ${CONTAINER_NAME}:/mylib/udev && \ docker cp /home/ubuntu/python_backend ${CONTAINER_NAME}:${TRITON_PATH}/python_backend && \ docker start -a ${CONTAINER_NAME} || RV=$?; # Run tensorflow instance tests docker stop ${CONTAINER_NAME} && docker rm ${CONTAINER_NAME} docker create --name ${CONTAINER_NAME} \ --device /dev/neuron0 \ --device /dev/neuron1 \ --shm-size=1g --ulimit memlock=-1 \ -p 8000:8000 -p 8001:8001 -p 8002:8002 \ --ulimit stack=67108864 \ -e TEST_REPO=${TEST_REPO} \ -e TEST_JSON_REPO=${TEST_JSON_REPO} \ -e TRITON_PATH=${TRITON_PATH} \ -e USE_TENSORFLOW="1" \ --net host -ti ${QA_IMAGE} \ /bin/bash -c "bash -ex ${TEST_REPO}/${TEST_SCRIPT}" && \ docker cp /lib/udev ${CONTAINER_NAME}:/mylib/udev && \ docker cp /home/ubuntu/python_backend ${CONTAINER_NAME}:${TRITON_PATH}/python_backend && \ docker start -a ${CONTAINER_NAME} || RV=$?; kytensor-2.49.0.7/framework_backends/python_backend/inferentia/scripts/000077500000000000000000000000001527052371200262675ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/inferentia/scripts/gen_triton_model.py000066400000000000000000001024661527052371200322020ustar00rootroot00000000000000# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os def tf_to_triton_dtype(dtype): import tensorflow as tf if dtype == tf.float16: return "FP16" elif dtype == tf.float32: return "FP32" elif dtype == tf.float64: return "FP64" elif dtype == tf.int8: return "INT8" elif dtype == tf.uint8: return "UINT8" elif dtype == tf.uint16: return "UINT16" elif dtype == tf.uint32: return "UINT32" elif dtype == tf.uint64: return "UINT64" elif dtype == tf.int16: return "INT16" elif dtype == tf.int32: return "INT32" elif dtype == tf.int64: return "INT64" elif dtype == tf.bool: return "BOOL" elif dtype == tf.string: return "STRING" raise Exception("The data type in the TF model is not supported") def parse_tf_tensors(saved_model_dir, tag_set, signature_def_key): from tensorflow.python.tools import saved_model_utils meta_graph_def = saved_model_utils.get_meta_graph_def(saved_model_dir, tag_set) input_dict = {} input_signatures = list( meta_graph_def.signature_def[signature_def_key].inputs.values() ) for input_signature in input_signatures: datatype = tf_to_triton_dtype(input_signature.dtype) shape = [] for dim in input_signature.tensor_shape.dim: shape.append(dim.size) input_dict[input_signature.name] = [datatype, shape] output_dict = {} output_signatures = list( meta_graph_def.signature_def[signature_def_key].outputs.values() ) for output_signature in output_signatures: datatype = tf_to_triton_dtype(output_signature.dtype) shape = [] for dim in output_signature.tensor_shape.dim: shape.append(dim.size) output_dict[output_signature.name] = [datatype, shape] return input_dict, output_dict def parse_io_tensors(tensors): tensors_dict = {} for t in [t for tensor in tensors for t in tensor]: name, datatype, shape_str = t.split(",") shape = [int(i) for i in shape_str.split("x")] tensors_dict[name] = [datatype, shape] return tensors_dict def get_parameter_spec(key1, value): param_spec = 'parameters: {{key: "{}", value: {{string_value: "{}"}}}} \n'.format( key1, value ) return param_spec def create_modelconfig( model_name, max_batch_size, inputs, outputs, compiled_model_path, nc_start_idx, nc_end_idx, threads_per_core, instance_count, enable_dynamic_batching, preferred_batch_size, max_queue_delay_microseconds, ): config = 'name: "{}"\n'.format(model_name) config += 'backend: "python"\n' config += "max_batch_size: {}\n".format(max_batch_size) if enable_dynamic_batching: config += """ dynamic_batching { """ if preferred_batch_size is not None: config += """ preferred_batch_size: {} """.format( preferred_batch_size ) if max_queue_delay_microseconds is not None: config += """ max_queue_delay_microseconds: {} """.format( max_queue_delay_microseconds ) config += """ }\n""" for input_name in inputs.keys(): data_type, shape = inputs[input_name] config += """ input [ {{ name: \"{}\" data_type: {} dims: {} }} ]\n""".format( input_name, "TYPE_" + data_type, shape ) for output_name in outputs.keys(): data_type, shape = outputs[output_name] config += """ output [ {{ name: \"{}\" data_type: {} dims: {} }} ]\n""".format( output_name, "TYPE_" + data_type, shape ) config += """ instance_group [ {{ kind: KIND_MODEL count: {} }} ]\n""".format( instance_count ) config += get_parameter_spec("COMPILED_MODEL", compiled_model_path) config += get_parameter_spec("NEURON_CORE_START_INDEX", nc_start_idx) config += get_parameter_spec("NEURON_CORE_END_INDEX", nc_end_idx) config += get_parameter_spec("NUM_THREADS_PER_CORE", threads_per_core) return config def get_model_license(): lic = """# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ return lic def get_common_initialize_impl(): init_impl = ''' def initialize(self, args): """`initialize` is called only once when the model is being loaded. Implementing `initialize` function is optional. This function allows the model to initialize any state associated with this model. Parameters ---------- args : dict Both keys and values are strings. The dictionary keys and values are: * model_config: A JSON string containing the model configuration * model_instance_kind: A string containing model instance kind * model_instance_device_id: A string containing model instance device ID * model_repository: Model repository path * model_version: Model version * model_name: Model name """ # You must parse model_config. JSON string is not parsed here self.model_config = model_config = json.loads(args['model_config']) if (len(model_config['instance_group']) != 1): raise pb_utils.TritonModelException( "this model supports only a single instance group, got {}". format(len(model_config['instance_group']))) instance_group_config = model_config['instance_group'][0] instance_count = instance_group_config['count'] instance_idx = 0 if instance_count > 1: instance_name_parts = args['model_instance_name'].split("_") if not instance_name_parts[-1].isnumeric(): raise pb_utils.TritonModelException( "internal error: the model instance name should end with '_', got {}" .format(args['model_instance_name'])) instance_idx = int(instance_name_parts[-1]) params = model_config['parameters'] compiled_model = params['COMPILED_MODEL']['string_value'] nc_start_idx = int(params['NEURON_CORE_START_INDEX']['string_value']) nc_end_idx = int(params['NEURON_CORE_END_INDEX']['string_value']) if nc_end_idx < nc_start_idx: raise pb_utils.TritonModelException( "the neuron core end index should be greater than or equal to the start index" ) threads_per_core = int(params['NUM_THREADS_PER_CORE']['string_value']) if threads_per_core < 1: raise pb_utils.TritonModelException( "the number of threads per core should be greater than or equal to 1" ) num_threads = (nc_end_idx - nc_start_idx + 1) * threads_per_core total_core_count = nc_end_idx - nc_start_idx + 1 if (instance_count > total_core_count): raise pb_utils.TritonModelException( "can not distribute {} triton model instances to {} neuron cores" .format(instance_count, total_core_count)) cores_per_instance = total_core_count // instance_count ''' return init_impl def get_tensorflow_initialize_impl(is_inf2=False): init_impl = get_common_initialize_impl() init_impl += """ self.input_list = [] for config_input in model_config['input']: self.input_list.append( (config_input['name'], config_input['data_type'], config_input['dims'])) self.output_list = [] for config_output in model_config['output']: self.output_list.append( (config_output['name'], config_output['data_type'], config_output['dims'])) os.environ["NEURON_RT_NUM_CORES"] = str(cores_per_instance) """ if is_inf2: init_impl += """ compiled_model = os.path.join(args['model_repository'], compiled_model) self.pred_list = [ tf.keras.models.load_model(compiled_model) for _ in range(cores_per_instance) ] * threads_per_core """ else: init_impl += """ self.pred_list = [ tf.contrib.predictor.from_saved_model(compiled_model) for _ in range(cores_per_instance) ] * threads_per_core """ return init_impl def get_pytorch_initialize_impl(is_inf2=False): init_impl = """ def _validate_and_get_index(self, name): parts = name.split('__') if len(parts) != 2: raise pb_utils.TritonModelException( "tensor names are expected to be in format __, got {}" .format(name)) if not parts[1].isnumeric(): raise pb_utils.TritonModelException( "tensor names are expected to be in format __ where should be numeric, got {}" .format(name)) return int(parts[1]) def _validate_input_dict(self, expected_count): for i in range(expected_count): if i not in self.input_dict: raise pb_utils.TritonModelException( "input corresponding to index {} not found".format(i)) def _validate_output_dict(self, expected_count): for i in range(expected_count): if i not in self.output_dict: raise pb_utils.TritonModelException( "output corresponding to index {} not found".format(i)) """ init_impl += get_common_initialize_impl() init_impl += """ self.input_dict = {} expected_input_count = 0 for config_input in model_config['input']: index = self._validate_and_get_index(config_input['name']) self.input_dict[index] = [ config_input['name'], config_input['data_type'], config_input['dims'] ] expected_input_count += 1 self._validate_input_dict(expected_input_count) self.output_dict = {} for config_output in model_config['output']: index = self._validate_and_get_index(config_output['name']) self.output_dict[index] = [ config_output['name'], config_output['data_type'], config_output['dims'] ] adjusted_nc_start_idx = (instance_idx * cores_per_instance) + nc_start_idx cores_range = '{}-{}'.format( adjusted_nc_start_idx, (adjusted_nc_start_idx + cores_per_instance - 1)) os.environ["NEURON_RT_VISIBLE_CORES"] = cores_range consumed_cores_list = [i for i in range(cores_per_instance)] """ if is_inf2: init_impl += """ compiled_model = os.path.join(args['model_repository'], compiled_model) self.model_neuron = torch.jit.load(compiled_model) """ else: init_impl += """ self.model_neuron = torch.neuron.DataParallel( torch.jit.load(compiled_model), device_ids=consumed_cores_list) """ init_impl += """ self.model_neuron.num_workers = num_threads """ return init_impl def get_tensorflow_execute_impl(disable_batch_requests_to_neuron): exec_impl = ''' def _one_thread(self, pred, model_feed_dict): result = pred(model_feed_dict) return result def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ ''' if disable_batch_requests_to_neuron: exec_impl += """ responses = [] num_threads = len(self.pred_list) model_feed_dict_list = [{} for _ in range(num_threads)] for request in requests: inputs = [] for i in range(len(self.input_list)): name, dt, shape = self.input_list[i] tensor = pb_utils.get_input_tensor_by_name(request, name).as_numpy() split_tensor = [None] * num_threads for split_index in range(num_threads): model_feed_dict_list[split_index][name] = np.array_split( tensor, num_threads, axis=0)[split_index] executor = futures.ThreadPoolExecutor(max_workers=num_threads) running = { executor.submit(self._one_thread, self.pred_list[idx], model_feed_dict_list[idx]): idx for idx in range(num_threads) } results = [None] * num_threads for future in futures.as_completed(running): idx = running[future] results[idx] = future.result() output_tensors = [] for i in range(len(self.output_list)): name, dt, shape = self.output_list[i] out_list = [None] * num_threads for idx in range(num_threads): out_list[idx] = results[idx][name] full_tensor = out_list[0] for idx in range(num_threads - 1): full_tensor = np.concatenate( (full_tensor, out_list[idx + 1]), axis=0) output_tensor = pb_utils.Tensor( name, full_tensor.astype(pb_utils.triton_string_to_numpy(dt))) output_tensors.append(output_tensor) inference_response = pb_utils.InferenceResponse( output_tensors=output_tensors) responses.append(inference_response) return responses """ else: exec_impl += """ responses = [] num_threads = len(self.pred_list) model_feed_dict_list = [{} for _ in range(num_threads)] num_requests = len(requests) request_batch_sizes = [] inputs = [] for i in range(len(self.input_list)): name, dt, shape = self.input_list[i] first_tensor = pb_utils.get_input_tensor_by_name(requests[0], name).as_numpy() request_batch_sizes.append(np.size(first_tensor, axis=0)) batched_tensor = first_tensor for j in range(1, num_requests): tensor = pb_utils.get_input_tensor_by_name(requests[j], name).as_numpy() request_batch_sizes.append(request_batch_sizes[-1] + np.size(tensor, axis=0)) batched_tensor = np.concatenate((batched_tensor, tensor), axis=0) split_tensor = [None] * num_threads for split_index in range(num_threads): model_feed_dict_list[split_index][name] = np.array_split( batched_tensor, num_threads, axis=0)[split_index] executor = futures.ThreadPoolExecutor(max_workers=num_threads) running = { executor.submit(self._one_thread, self.pred_list[idx], model_feed_dict_list[idx]): idx for idx in range(num_threads) } results = [None] * num_threads for future in futures.as_completed(running): idx = running[future] results[idx] = future.result() chuncky_tensors = [] for i in range(len(self.output_list)): name, dt, shape = self.output_list[i] out_list = [None] * num_threads for idx in range(num_threads): out_list[idx] = results[idx][name] full_tensor = out_list[0] for idx in range(num_threads - 1): full_tensor = np.concatenate( (full_tensor, out_list[idx + 1]), axis=0) chuncky_tensors.append(np.split(full_tensor, request_batch_sizes, axis=0)) for i in range(num_requests): output_tensors = [] for j in range(len(self.output_list)): name, dt, shape = self.output_list[j] tensor = chuncky_tensors[j][i] output_tensor = pb_utils.Tensor( name, tensor.astype(pb_utils.triton_string_to_numpy(dt))) output_tensors.append(output_tensor) inference_response = pb_utils.InferenceResponse(output_tensors=output_tensors) responses.append(inference_response) return responses """ return exec_impl def get_pytorch_execute_impl(disable_batch_requests_to_neuron): exec_impl = ''' def execute(self, requests): """`execute` MUST be implemented in every Python model. `execute` function receives a list of pb_utils.InferenceRequest as the only argument. This function is called when an inference request is made for this model. Depending on the batching configuration (e.g. Dynamic Batching) used, `requests` may contain multiple requests. Every Python model, must create one pb_utils.InferenceResponse for every pb_utils.InferenceRequest in `requests`. If there is an error, you can set the error argument when creating a pb_utils.InferenceResponse Parameters ---------- requests : list A list of pb_utils.InferenceRequest Returns ------- list A list of pb_utils.InferenceResponse. The length of this list must be the same as `requests` """ ''' if disable_batch_requests_to_neuron: exec_impl += """ responses = [] for request in requests: inputs = [] for i in range(len(self.input_dict)): name, dt, shape = self.input_dict[i] tensor = torch.as_tensor(pb_utils.get_input_tensor_by_name(request, name).as_numpy()) inputs.append(tensor) results = self.model_neuron(*inputs) output_tensors = [] for i in self.output_dict.keys(): name, dt, shape = self.output_dict[i] result = results[i] if isinstance(results, tuple) else results output_tensor = pb_utils.Tensor( name, result.numpy().astype( pb_utils.triton_string_to_numpy(dt))) output_tensors.append(output_tensor) inference_response = pb_utils.InferenceResponse( output_tensors=output_tensors) responses.append(inference_response) return responses """ else: exec_impl += """ responses = [] inputs = [] num_requests = len(requests) request_batch_sizes = [] for i in self.input_dict.keys(): name, dt, shape = self.input_dict[i] first_tensor = torch.as_tensor(pb_utils.get_input_tensor_by_name(requests[0], name).as_numpy()) request_batch_sizes.append(first_tensor.size(dim=0)) batched_tensor = first_tensor for j in range(1, num_requests): tensor = torch.as_tensor(pb_utils.get_input_tensor_by_name(requests[j], name).as_numpy()) request_batch_sizes.append(request_batch_sizes[-1] + tensor.size(dim=0)) batched_tensor = torch.cat((batched_tensor, tensor), dim=0) inputs.append(batched_tensor) batched_results = self.model_neuron(*inputs) chunky_batched_results = [] for i in self.output_dict.keys(): batch = batched_results[i] if isinstance(batched_results, tuple) else batched_results chunky_batched_results.append(torch.tensor_split(batch, request_batch_sizes, dim=0)) for i in range(num_requests): output_tensors = [] for j in self.output_dict.keys(): name, dt, shape = self.output_dict[j] result = chunky_batched_results[j][i] output_tensor = pb_utils.Tensor( name, result.numpy().astype( pb_utils.triton_string_to_numpy(dt))) output_tensors.append(output_tensor) inference_response = pb_utils.InferenceResponse( output_tensors=output_tensors) responses.append(inference_response) return responses """ return exec_impl def get_finalize_impl(): finalize_impl = ''' def finalize(self): """`finalize` is called only once when the model is being unloaded. Implementing `finalize` function is OPTIONAL. This function allows the model to perform any necessary clean ups before exit. """ print('Cleaning up...') ''' return finalize_impl def get_triton_python_model_impl( using_tensorflow_model, disable_batch_requests_to_neuron, is_inf2=False ): triton_pmi = ''' class TritonPythonModel: """Your Python model must use the same class name. Every Python model that is created must have "TritonPythonModel" as the class name. """ ''' if using_tensorflow_model: triton_pmi += get_tensorflow_initialize_impl(is_inf2) triton_pmi += get_tensorflow_execute_impl(disable_batch_requests_to_neuron) else: triton_pmi += get_pytorch_initialize_impl(is_inf2) triton_pmi += get_pytorch_execute_impl(disable_batch_requests_to_neuron) triton_pmi += get_finalize_impl() return triton_pmi def create_model_file( using_tensorflow_model, disable_batch_requests_to_neuron, is_inf2=False ): triton_model = get_model_license() triton_model += """ import json import numpy as np import os import sys import triton_python_backend_utils as pb_utils """ if using_tensorflow_model: triton_model += """ import tensorflow as tf from concurrent import futures """ else: triton_model += """ import torch """ if not is_inf2: triton_model += """ import torch.neuron """ else: triton_model += """ import torch_neuronx """ triton_model += get_triton_python_model_impl( using_tensorflow_model, disable_batch_requests_to_neuron, is_inf2 ) return triton_model if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--inf2", required=False, default=False, action="store_true", help="Specify whether the model should be generate for inf2 or inf1, default is inf1", ) parser.add_argument( "--model_type", type=str, required=True, choices=["pytorch", "tensorflow"], help="""The type of the compiled model. Currently, only supports \"pytorch\" and \"tensorflow\".""", ) parser.add_argument( "--model_version", type=int, default=1, help="The version of the model" ) parser.add_argument( "--enable_dynamic_batching", action="store_true", help="""Enable dynamic batching. Please see model configuration documentation for details: https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md#dynamic-batcher""", ) parser.add_argument( "--max_batch_size", type=int, default=0, help="""The maximum batch size for the model being generated. Please see model configuration documentation for details: https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md#maximum-batch-size""", ) parser.add_argument( "--preferred_batch_size", type=int, help="""The preferred batch size. Should be multiples of cores available to ensure proper utilization of neuron cores. This flag is ignored if --enable_dynamic_batching is not specified. Please see model configuration documentation for details: https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md#preferred-batch-sizes""", ) parser.add_argument( "--max_queue_delay_microseconds", type=int, help="""Max queue delay time(ms) for dynamic batching. This flag is ignored if --enable_dynamic_batching is not specified. Please see model configuration documentation for details: https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md#delayed-batching""", ) parser.add_argument( "--disable_batch_requests_to_neuron", action="store_true", help="""Send each request separately to neuron if enabled. If not specified, then requests are combined and sent to neuron as a single batch""", ) parser.add_argument( "--tag_set", type=str, default="serve", help="""The tag set to use for the TF model. This option is ignored if `--model_type` is not \"tensorflow\". Default value is \'serve\'.""", ) parser.add_argument( "--signature_def_key", type=str, default="serving_default", help="""The signature def key to use for the TF model. This option is ignored if `--model_type` is not \"tensorflow\". Default value is \'serving_default\'.""", ) parser.add_argument( "--compiled_model", type=str, required=True, help="Fullpath to the compiled model", ) parser.add_argument( "--triton_input", type=str, action="append", nargs="*", help="""The name, datatype and shape of the model input in format ,,. This option can be provided multiple times for multiple inputs. For example, to provide a FP16 input with shape [1,384] specify the following: INPUT0,FP16,1x384. This option is not required when using tensorflow model""", ) parser.add_argument( "--triton_output", type=str, action="append", nargs="*", help="""The name, datatype and shape of the model output in format ,,. This option can be provided multiple times for multiple outputs. For example, to provide a FP16 output with shape [1,384] specify the following: OUTPUT0,FP16,1x384. This option is not required when using tensorflow model""", ) parser.add_argument( "--neuron_core_range", type=str, required=True, help="""The range of neuron core indices where the model needs to be loaded. The range should be specified in format :. For example to load model on neuron cores (0-7), specify the following: 0:7. NOTE: when using multiple triton model instances the neuron cores will get equally distributed. Assuming the instance count is 4, Instance0 will get loaded on cores 0:1, Instance1 will get loaded on cores 2:3, Instance2 will get loaded on cores 4:5 and Instance 3 will get loaded on cores 6:7""", ) parser.add_argument( "--threads_per_core", type=int, default=1, help="The number of threads per neuron core.", ) parser.add_argument( "--triton_model_instance_count", type=int, default=1, help="The number of triton model instances.", ) parser.add_argument( "--triton_model_dir", type=str, required=True, help="""Path to the triton model directory where script will generate config.pbtxt and model.py""", ) FLAGS, unparsed = parser.parse_known_args() if len(unparsed) > 0: raise Exception("Unrecognized options: {}".format(unparsed)) if FLAGS.model_type == "tensorflow": is_tensorflow_model = True elif FLAGS.model_type == "pytorch": is_tensorflow_model = False print( """Triton Dynamic Batching is enabled: {}, preferred_batch_size: {} and max_batch_size: {} with max_queue_delay_microseconds: {}. Batch requests to neruon are disabled: {}""".format( FLAGS.enable_dynamic_batching, FLAGS.preferred_batch_size, FLAGS.max_batch_size, FLAGS.max_queue_delay_microseconds, FLAGS.disable_batch_requests_to_neuron, ) ) if not is_tensorflow_model or ( FLAGS.triton_input != None and FLAGS.triton_output != None ): inputs = parse_io_tensors(FLAGS.triton_input) outputs = parse_io_tensors(FLAGS.triton_output) else: inputs, outputs = parse_tf_tensors( FLAGS.compiled_model, FLAGS.tag_set, FLAGS.signature_def_key ) nc_start_idx, nc_end_idx = [int(i) for i in FLAGS.neuron_core_range.split(":")] model_version_dir = FLAGS.triton_model_dir + "/" + str(FLAGS.model_version) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir model_name = os.path.basename(FLAGS.triton_model_dir) mc = create_modelconfig( model_name, FLAGS.max_batch_size, inputs, outputs, FLAGS.compiled_model, nc_start_idx, nc_end_idx, FLAGS.threads_per_core, FLAGS.triton_model_instance_count, FLAGS.enable_dynamic_batching, FLAGS.preferred_batch_size, FLAGS.max_queue_delay_microseconds, ) with open(FLAGS.triton_model_dir + "/config.pbtxt", "w") as config_file: config_file.write(mc) is_inf2 = FLAGS.inf2 mf = create_model_file( is_tensorflow_model, FLAGS.disable_batch_requests_to_neuron, is_inf2 ) with open(FLAGS.triton_model_dir + "/1/model.py", "w") as model_file: model_file.write(mf) kytensor-2.49.0.7/framework_backends/python_backend/inferentia/scripts/setup-pre-container.sh000077500000000000000000000112031527052371200325270ustar00rootroot00000000000000#!/bin/bash # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #! /bin/sh USAGE=" usage: setup.sh [options] Sets up runtime and tools for execution on Inferentia chips. -h|--help Shows usage -inf1|--inf1-setup Installs runtime and tools for inf1/neuron, inf1 is default -inf2|--inf2-setup Installs runtime and tools for inf2/neuronx -trn1|--trn1-setup Installs runtime, tools for inf2, and installs EFA for trn1 " # Get all options: OPTS=$(getopt -o hb:v:i:tp --long help,python-backend-path:,python-version:,inferentia-path:,use-tensorflow,use-pytorch,tensorflow-version: -- "$@") export INSTALL_INF2=0 export INSTALL_INF1=1 export INSTALL_TRN1=0 export CWD=`pwd` cd /home/ubuntu for OPTS; do case "$OPTS" in -h|--help) printf "%s\\n" "$USAGE" return 0 ;; -inf1|--inf1-setup) INSTALL_INF1=1 echo "Script will install runtime and tools for inf1/neuron" shift 1 ;; -inf2|--inf2-setup) INSTALL_INF2=1 shift 1 echo "Script will install runtime and tools for inf2/neruonx" ;; -trn1|--trn1-setup) INSTALL_TRN1=1 echo "Script will install runtime and tools for trn1" shift 1 ;; esac done if [ ${INSTALL_INF1} -ne 1 ] && [ ${INSTALL_INF2} -ne 1 ]; then echo "Error: need to specify either -inf1, -inf2 or -trn1." printf "source %s\\n" "$USAGE" return 1 fi if [ ${INSTALL_INF1} -eq 1 ] && [ ${INSTALL_INF2} -eq 1] then echo "Error: cannot install both inf1 and inf2 dependencies. Please select either -inf1 or -inf2." return 1 fi if [ ${INSTALL_INF1} -eq 1 ] && [ ${INSTALL_TRN1} -eq 1 ] then echo "Error: cannot install both inf1 and trn1 dependencies. Selecting -trn1 will install inf2 dependencies and EFA." fi # First stop and remove old neuron 1.X runtime sudo systemctl stop neuron-rtd || true sudo apt remove aws-neuron-runtime -y || true # Then install new neuron libraries . /etc/os-release sudo tee /etc/apt/sources.list.d/neuron.list > /dev/null < /dev/null <& correlation_id_shm) { id_uint_ = correlation_id_shm->id_uint_; id_type_ = correlation_id_shm->id_type_; id_string_ = correlation_id_shm->id_string_; } CorrelationId& CorrelationId::operator=(const CorrelationId& rhs) { id_uint_ = rhs.id_uint_; id_type_ = rhs.id_type_; id_string_ = rhs.id_string_; return *this; } void CorrelationId::SaveToSharedMemory( std::unique_ptr& shm_pool) { AllocatedSharedMemory correlation_id_shm = shm_pool->Construct(); correlation_id_shm_ptr_ = correlation_id_shm.data_.get(); std::unique_ptr id_string_shm = PbString::Create(shm_pool, id_string_); correlation_id_shm_ptr_->id_uint = id_uint_; correlation_id_shm_ptr_->id_string_shm_handle = id_string_shm->ShmHandle(); correlation_id_shm_ptr_->id_type = id_type_; // Save the references to shared memory. correlation_id_shm_ = std::move(correlation_id_shm); id_string_shm_ = std::move(id_string_shm); shm_handle_ = correlation_id_shm_.handle_; } std::unique_ptr CorrelationId::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle) { AllocatedSharedMemory correlation_id_shm = shm_pool->Load(handle); CorrelationIdShm* correlation_id_shm_ptr = correlation_id_shm.data_.get(); std::unique_ptr id_string_shm = PbString::LoadFromSharedMemory( shm_pool, correlation_id_shm_ptr->id_string_shm_handle); return std::unique_ptr( new CorrelationId(correlation_id_shm, id_string_shm)); } CorrelationId::CorrelationId( AllocatedSharedMemory& correlation_id_shm, std::unique_ptr& id_string_shm) : correlation_id_shm_(std::move(correlation_id_shm)), id_string_shm_(std::move(id_string_shm)) { correlation_id_shm_ptr_ = correlation_id_shm_.data_.get(); shm_handle_ = correlation_id_shm_.handle_; id_string_ = id_string_shm_->String(); id_uint_ = correlation_id_shm_ptr_->id_uint; id_type_ = correlation_id_shm_ptr_->id_type; } }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/correlation_id.h000066400000000000000000000073251527052371200264200ustar00rootroot00000000000000// Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "pb_string.h" #include "pb_utils.h" namespace triton { namespace backend { namespace python { enum class CorrelationIdDataType { UINT64, STRING }; struct CorrelationIdShm { bi::managed_external_buffer::handle_t id_string_shm_handle; uint64_t id_uint; CorrelationIdDataType id_type; }; class CorrelationId { public: CorrelationId(); CorrelationId(const std::string& id_string); CorrelationId(uint64_t id_uint); CorrelationId(const CorrelationId& rhs); CorrelationId(std::unique_ptr& correlation_id_shm); CorrelationId& operator=(const CorrelationId& rhs); /// Save CorrelationId object to shared memory. /// \param shm_pool Shared memory pool to save the CorrelationId object. void SaveToSharedMemory(std::unique_ptr& shm_pool); /// Create a CorrelationId object from shared memory. /// \param shm_pool Shared memory pool /// \param handle Shared memory handle of the CorrelationId. /// \return Returns the CorrelationId in the specified handle /// location. static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); // Function that help determine exact type of Correlation Id CorrelationIdDataType Type() const { return id_type_; } // Get the value of the CorrelationId based on the type const std::string& StringValue() const { return id_string_; } uint64_t UnsignedIntValue() const { return id_uint_; } bi::managed_external_buffer::handle_t ShmHandle() { return shm_handle_; } private: // The private constructor for creating a CorrelationId object from shared // memory. CorrelationId( AllocatedSharedMemory& correlation_id_shm, std::unique_ptr& id_string_shm); std::string id_string_; uint64_t id_uint_; CorrelationIdDataType id_type_; // Shared Memory Data Structures AllocatedSharedMemory correlation_id_shm_; CorrelationIdShm* correlation_id_shm_ptr_; bi::managed_external_buffer::handle_t shm_handle_; std::unique_ptr id_string_shm_; }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/gpu_buffers.cc000066400000000000000000000062241527052371200260650ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "gpu_buffers.h" #include "pb_string.h" namespace triton { namespace backend { namespace python { GPUBuffersHelper::GPUBuffersHelper() { completed_ = false; } void GPUBuffersHelper::AddBuffer(const bi::managed_external_buffer::handle_t& handle) { if (completed_) { throw PythonBackendException( "It is not possible to add buffers after 'Complete' has been called on " "a GPUBuffersHelper."); } buffers_.emplace_back(handle); } void GPUBuffersHelper::SetError( std::unique_ptr& shm_pool, const std::string& error) { error_shm_ = PbString::Create(shm_pool, error); } void GPUBuffersHelper::Complete(std::unique_ptr& shm_pool) { if (completed_) { throw PythonBackendException( "Complete has already been called. Complete should only be called " "once."); } gpu_buffers_shm_ = shm_pool->Construct(); if (!error_shm_) { buffers_handle_shm_ = shm_pool->Construct( buffers_.size()); gpu_buffers_shm_.data_->buffer_count = buffers_.size(); gpu_buffers_shm_.data_->success = true; gpu_buffers_shm_.data_->buffers = buffers_handle_shm_.handle_; for (size_t i = 0; i < buffers_.size(); ++i) { buffers_handle_shm_.data_.get()[i] = buffers_[i]; } } else { gpu_buffers_shm_.data_->success = false; gpu_buffers_shm_.data_->error = error_shm_->ShmHandle(); } completed_ = true; } bi::managed_external_buffer::handle_t GPUBuffersHelper::ShmHandle() { return gpu_buffers_shm_.handle_; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/gpu_buffers.h000066400000000000000000000055431527052371200257320ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "pb_string.h" #include "pb_utils.h" #include "scoped_defer.h" namespace triton { namespace backend { namespace python { /// \param success indicating whether the process of fetching the GPU buffers /// was successful. /// \param error if success is equal to false, the error object will be set. /// \param buffers list of buffers elements. /// \param buffer_count the number of buffers. struct GPUBuffersShm { bool success; bi::managed_external_buffer::handle_t error; bi::managed_external_buffer::handle_t buffers; uint32_t buffer_count; }; /// Helper class to facilitate transfer of metadata associated /// the GPU buffers in shared memory. class GPUBuffersHelper { public: GPUBuffersHelper(); void AddBuffer(const bi::managed_external_buffer::handle_t& handle); void Complete(std::unique_ptr& shm_pool); void SetError( std::unique_ptr& shm_pool, const std::string& error); bi::managed_external_buffer::handle_t ShmHandle(); private: AllocatedSharedMemory gpu_buffers_shm_; std::vector buffers_; AllocatedSharedMemory buffers_handle_shm_; std::unique_ptr error_shm_; bool completed_; }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/infer_payload.cc000066400000000000000000000060651527052371200263750ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_payload.h" namespace triton { namespace backend { namespace python { InferPayload::InferPayload( const bool is_decoupled, std::function)> callback) : is_decoupled_(is_decoupled), is_promise_set_(false), callback_(callback) { promise_.reset(new std::promise>()); } void InferPayload::SetValue(std::unique_ptr infer_response) { { // Only set value to the promise with the first response. Call the callback // function to send decoupled response to the stub. std::lock_guard lock(mutex_); if (!is_promise_set_) { is_promise_set_ = true; promise_->set_value(std::move(infer_response)); return; } } Callback(std::move(infer_response)); } void InferPayload::SetFuture( std::future>& response_future) { response_future = promise_->get_future(); } bool InferPayload::IsDecoupled() { return is_decoupled_; } bool InferPayload::IsPromiseSet() { return is_promise_set_; } void InferPayload::Callback(std::unique_ptr infer_response) { return callback_(std::move(infer_response)); } void InferPayload::SetResponseAllocUserp( const ResponseAllocatorUserp& response_alloc_userp) { response_alloc_userp_ = std::make_shared(response_alloc_userp); } std::shared_ptr InferPayload::ResponseAllocUserp() { return response_alloc_userp_; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/infer_payload.h000066400000000000000000000061741527052371200262400ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "infer_response.h" #include "pb_preferred_memory.h" namespace triton { namespace backend { namespace python { struct ResponseAllocatorUserp { ResponseAllocatorUserp( void* shm_pool, const PreferredMemory& preferred_memory) : shm_pool(shm_pool), preferred_memory(preferred_memory) { } void* shm_pool; PreferredMemory preferred_memory; }; class InferPayload : public std::enable_shared_from_this { public: InferPayload( const bool is_decouple, std::function)> callback); /// GetPtr should be only called when the InferPayload object is constructed /// using a shared pointer. Calling this function in any other circumstance /// is undefined behaviour until C++17. std::shared_ptr GetPtr() { return shared_from_this(); } void SetValue(std::unique_ptr infer_response); void SetFuture(std::future>& response_future); bool IsDecoupled(); bool IsPromiseSet(); void Callback(std::unique_ptr infer_response); void SetResponseAllocUserp( const ResponseAllocatorUserp& response_alloc_userp); std::shared_ptr ResponseAllocUserp(); private: std::unique_ptr>> promise_; bool is_decoupled_; std::mutex mutex_; bool is_promise_set_; std::function)> callback_; std::shared_ptr response_alloc_userp_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/infer_request.cc000066400000000000000000000525241527052371200264350ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_request.h" #include #include "gpu_buffers.h" #include "pb_utils.h" #include "scoped_defer.h" #ifdef TRITON_PB_STUB #include "pb_stub.h" #endif namespace triton { namespace backend { namespace python { InferRequest::InferRequest( const std::string& request_id, const CorrelationId& correlation_id, const std::vector>& inputs, const std::set& requested_output_names, const std::string& model_name, const int64_t model_version, const std::string& parameters, const uint32_t flags, const uint64_t timeout, const intptr_t response_factory_address, const intptr_t request_address, const PreferredMemory& preferred_memory, const InferenceTrace& trace) : request_id_(request_id), correlation_id_(correlation_id), inputs_(inputs), requested_output_names_(requested_output_names), model_name_(model_name), model_version_(model_version), parameters_(parameters), flags_(flags), timeout_(timeout), response_factory_address_(response_factory_address), request_address_(request_address), preferred_memory_(preferred_memory), trace_(trace), request_release_flags_(TRITONSERVER_REQUEST_RELEASE_ALL) { for (auto& input : inputs) { if (!input) { throw PythonBackendException( "Input tensor for request with id '" + request_id + "' and model name '" + model_name + "' should not be empty."); } } for (auto& requested_output_name : requested_output_names) { if (requested_output_name == "") { throw PythonBackendException( "Requested output name for request with id '" + request_id + "' and model name '" + model_name + "' should not be empty."); } } #ifdef TRITON_PB_STUB pb_cancel_ = std::make_shared(response_factory_address_, request_address_); response_sender_ = std::make_shared( request_address_, response_factory_address_, nullptr /* is_decoupled */, RequestedOutputNames(), Stub::GetOrCreateInstance()->SharedMemory(), pb_cancel_); #endif } const std::vector>& InferRequest::Inputs() { return inputs_; } const std::string& InferRequest::Parameters() { return parameters_; } const std::string& InferRequest::RequestId() { return request_id_; } CorrelationId& InferRequest::GetCorrelationId() { return correlation_id_; } const std::set& InferRequest::RequestedOutputNames() { return requested_output_names_; } const std::string& InferRequest::ModelName() { return model_name_; } int64_t InferRequest::ModelVersion() { return model_version_; } uint32_t InferRequest::Flags() { return flags_; } intptr_t InferRequest::RequestAddress() { return request_address_; } void InferRequest::SetFlags(uint32_t flags) { flags_ = flags; } bi::managed_external_buffer::handle_t InferRequest::ShmHandle() { return shm_handle_; } uint64_t InferRequest::Timeout() { return timeout_; } void InferRequest::SetIsDecoupled(const bool is_decoupled) { is_decoupled_ = is_decoupled; } bool InferRequest::IsDecoupled() { return is_decoupled_; } PreferredMemory& InferRequest::GetPreferredMemory() { return preferred_memory_; } InferenceTrace& InferRequest::GetTrace() { return trace_; } uint32_t InferRequest::ReleaseFlags() { request_release_flags_ = infer_request_shm_ptr_->request_release_flags; return request_release_flags_; } void InferRequest::SetReleaseFlags(const uint32_t& flags) { request_release_flags_ = flags; infer_request_shm_ptr_->request_release_flags = request_release_flags_; } void InferRequest::SaveToSharedMemory(std::unique_ptr& shm_pool) { AllocatedSharedMemory infer_request_shm = shm_pool->Construct( sizeof(InferRequestShm) + (RequestedOutputNames().size() * sizeof(bi::managed_external_buffer::handle_t)) + (Inputs().size() * sizeof(bi::managed_external_buffer::handle_t))); infer_request_shm_ptr_ = reinterpret_cast(infer_request_shm.data_.get()); infer_request_shm_ptr_->input_count = Inputs().size(); infer_request_shm_ptr_->model_version = model_version_; infer_request_shm_ptr_->requested_output_count = RequestedOutputNames().size(); infer_request_shm_ptr_->flags = Flags(); infer_request_shm_ptr_->address = request_address_; infer_request_shm_ptr_->response_factory_address = response_factory_address_; infer_request_shm_ptr_->is_decoupled = is_decoupled_; infer_request_shm_ptr_->timeout = timeout_; infer_request_shm_ptr_->preferred_memory = preferred_memory_; infer_request_shm_ptr_->request_release_flags = request_release_flags_; output_names_handle_shm_ptr_ = reinterpret_cast( reinterpret_cast(infer_request_shm_ptr_) + sizeof(InferRequestShm)); // [FIXME] This could also be a part of the single allocated memory for this // object. size_t i = 0; std::vector> requested_output_names_shm; for (auto& requested_output_name : requested_output_names_) { std::unique_ptr requested_output_name_shm = PbString::Create(shm_pool, requested_output_name); output_names_handle_shm_ptr_[i] = requested_output_name_shm->ShmHandle(); requested_output_names_shm.emplace_back( std::move(requested_output_name_shm)); i++; } input_tensors_handle_ptr_ = reinterpret_cast( reinterpret_cast(output_names_handle_shm_ptr_) + sizeof(bi::managed_external_buffer::handle_t) * RequestedOutputNames().size()); i = 0; for (auto& input : Inputs()) { input_tensors_handle_ptr_[i] = input->ShmHandle(); i++; } correlation_id_.SaveToSharedMemory(shm_pool); infer_request_shm_ptr_->correlation_id_shm_handle = correlation_id_.ShmHandle(); std::unique_ptr model_name_shm = PbString::Create(shm_pool, ModelName()); infer_request_shm_ptr_->model_name_shm_handle = model_name_shm->ShmHandle(); std::unique_ptr request_id_shm = PbString::Create(shm_pool, RequestId()); infer_request_shm_ptr_->request_id_shm_handle = request_id_shm->ShmHandle(); std::unique_ptr parameters_shm = PbString::Create(shm_pool, Parameters()); infer_request_shm_ptr_->parameters_shm_handle = parameters_shm->ShmHandle(); trace_.SaveToSharedMemory(shm_pool); infer_request_shm_ptr_->trace_shm_handle = trace_.ShmHandle(); // Save the references to shared memory. infer_request_shm_ = std::move(infer_request_shm); request_id_shm_ = std::move(request_id_shm); model_name_shm_ = std::move(model_name_shm); parameters_shm_ = std::move(parameters_shm); shm_handle_ = infer_request_shm_.handle_; requested_output_names_shm_ = std::move(requested_output_names_shm); } std::unique_ptr InferRequest::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t request_handle, bool open_cuda_handle, bool const* is_model_decoupled) { AllocatedSharedMemory infer_request_shm = shm_pool->Load(request_handle); InferRequestShm* infer_request_shm_ptr = reinterpret_cast(infer_request_shm.data_.get()); std::vector> requested_output_names_shm; uint32_t requested_output_count = infer_request_shm_ptr->requested_output_count; bi::managed_external_buffer::handle_t* output_names_handle_shm_ptr = reinterpret_cast( (reinterpret_cast(infer_request_shm_ptr) + sizeof(InferRequestShm))); for (size_t output_idx = 0; output_idx < requested_output_count; ++output_idx) { std::unique_ptr pb_string = PbString::LoadFromSharedMemory( shm_pool, output_names_handle_shm_ptr[output_idx]); requested_output_names_shm.emplace_back(std::move(pb_string)); } bi::managed_external_buffer::handle_t* input_names_handle_shm_ptr = reinterpret_cast( (reinterpret_cast(infer_request_shm_ptr) + sizeof(InferRequestShm) + (infer_request_shm_ptr->requested_output_count * sizeof(bi::managed_external_buffer::handle_t)))); std::vector> input_tensors; for (size_t input_idx = 0; input_idx < infer_request_shm_ptr->input_count; ++input_idx) { std::shared_ptr input_tensor = PbTensor::LoadFromSharedMemory( shm_pool, input_names_handle_shm_ptr[input_idx], open_cuda_handle); input_tensors.emplace_back(std::move(input_tensor)); } std::unique_ptr correlation_id_shm = CorrelationId::LoadFromSharedMemory( shm_pool, infer_request_shm_ptr->correlation_id_shm_handle); std::unique_ptr infer_trace_shm = InferenceTrace::LoadFromSharedMemory( shm_pool, infer_request_shm_ptr->trace_shm_handle); std::unique_ptr model_name_shm = PbString::LoadFromSharedMemory( shm_pool, infer_request_shm_ptr->model_name_shm_handle); std::unique_ptr request_id_shm = PbString::LoadFromSharedMemory( shm_pool, infer_request_shm_ptr->request_id_shm_handle); std::unique_ptr parameters_shm = PbString::LoadFromSharedMemory( shm_pool, infer_request_shm_ptr->parameters_shm_handle); return std::unique_ptr(new InferRequest( infer_request_shm, request_id_shm, correlation_id_shm, requested_output_names_shm, model_name_shm, input_tensors, parameters_shm, infer_trace_shm, is_model_decoupled)); } InferRequest::InferRequest( AllocatedSharedMemory& infer_request_shm, std::unique_ptr& request_id_shm, std::unique_ptr& correlation_id_shm, std::vector>& requested_output_names_shm, std::unique_ptr& model_name_shm, std::vector>& input_tensors, std::unique_ptr& parameters_shm, std::unique_ptr& infer_trace_shm, bool const* is_model_decoupled) : infer_request_shm_(std::move(infer_request_shm)), request_id_shm_(std::move(request_id_shm)), requested_output_names_shm_(std::move(requested_output_names_shm)), model_name_shm_(std::move(model_name_shm)), parameters_shm_(std::move(parameters_shm)) { infer_request_shm_ptr_ = reinterpret_cast(infer_request_shm_.data_.get()); output_names_handle_shm_ptr_ = reinterpret_cast( reinterpret_cast(infer_request_shm_ptr_) + sizeof(InferRequestShm)); input_tensors_handle_ptr_ = reinterpret_cast( reinterpret_cast(infer_request_shm_ptr_) + sizeof(InferRequestShm) + sizeof(bi::managed_external_buffer::handle_t) * infer_request_shm_ptr_->requested_output_count); inputs_ = std::move(input_tensors); std::set requested_output_names; for (size_t output_idx = 0; output_idx < infer_request_shm_ptr_->requested_output_count; ++output_idx) { auto& pb_string = requested_output_names_shm_[output_idx]; requested_output_names.emplace(pb_string->String()); } correlation_id_ = CorrelationId(correlation_id_shm); request_id_ = request_id_shm_->String(); parameters_ = parameters_shm_->String(); requested_output_names_ = std::move(requested_output_names); model_name_ = model_name_shm_->String(); flags_ = infer_request_shm_ptr_->flags; model_version_ = infer_request_shm_ptr_->model_version; request_address_ = infer_request_shm_ptr_->address; response_factory_address_ = infer_request_shm_ptr_->response_factory_address; is_decoupled_ = infer_request_shm_ptr_->is_decoupled; timeout_ = infer_request_shm_ptr_->timeout; preferred_memory_ = infer_request_shm_ptr_->preferred_memory; trace_ = InferenceTrace(infer_trace_shm); request_release_flags_ = infer_request_shm_ptr_->request_release_flags; #ifdef TRITON_PB_STUB pb_cancel_ = std::make_shared(response_factory_address_, request_address_); response_sender_ = std::make_shared( request_address_, response_factory_address_, is_model_decoupled, RequestedOutputNames(), Stub::GetOrCreateInstance()->SharedMemory(), pb_cancel_); #endif } #ifdef TRITON_PB_STUB bool InferRequest::IsCancelled() { return pb_cancel_->IsCancelled(); } std::shared_ptr InferRequest::GetResponseSender() { return response_sender_; } std::shared_ptr InferRequest::Exec(const bool is_decoupled) { // Release the GIL. This avoids a potential deadlock situation in the parent // process, where every thread in the thread pool is indirectly waiting for a // function in the stub process that acquires the GIL. Meanwhile, the current // thread, which holds the GIL, is also waiting for the parent side to have // the next available thread to pick up the job during resource contention. py::gil_scoped_release release; // BLS should not be used in "initialize" or "finalize" function. std::unique_ptr& stub = Stub::GetOrCreateInstance(); if (!stub->IsInitialized() || stub->IsFinalizing()) { throw PythonBackendException( "BLS is only supported during the 'execute' function."); } ResponseBatch* response_batch = nullptr; bool responses_is_set = false; std::unique_ptr& shm_pool = stub->SharedMemory(); bi::managed_external_buffer::handle_t* response_handle = nullptr; PythonBackendException pb_exception(std::string{}); std::unique_ptr ipc_message; AllocatedSharedMemory request_batch; ScopedDefer data_load_complete([&ipc_message] { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; ipc_message->ResponseCondition()->notify_all(); }); try { ipc_message = IPCMessage::Create(shm_pool, true /* inline_response */); bool has_exception = false; PythonBackendException pb_exception(std::string{}); if (is_decoupled) { ipc_message->Command() = PYTHONSTUB_CommandType::PYTHONSTUB_InferStreamExecRequest; } else { ipc_message->Command() = PYTHONSTUB_CommandType::PYTHONSTUB_InferExecRequest; } request_batch = shm_pool->Construct( sizeof(RequestBatch) + sizeof(bi::managed_external_buffer::handle_t)); RequestBatch* request_batch_shm_ptr = reinterpret_cast(request_batch.data_.get()); request_batch_shm_ptr->batch_size = 1; ipc_message->Args() = request_batch.handle_; bi::managed_external_buffer::handle_t* requests_shm = reinterpret_cast( request_batch.data_.get() + sizeof(RequestBatch)); request_batch_shm_ptr->batch_size = 1; bool has_gpu_tensor = false; for (auto& input_tensor : inputs_) { input_tensor->SaveToSharedMemory(shm_pool, false /* copy_gpu */); if (!input_tensor->IsCPU()) { has_gpu_tensor = true; } } SaveToSharedMemory(shm_pool); // Save the shared memory offset of the request. *requests_shm = ShmHandle(); // Send the BLS request to the parent process and wait for the response. { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; stub->SendIPCUtilsMessage(ipc_message); ipc_message->ResponseCondition()->wait(lock); } // Additional round trip required for asking the stub process // to fill in the GPU tensor buffers if (has_gpu_tensor) { AllocatedSharedMemory gpu_buffers_shm = shm_pool->Load( request_batch_shm_ptr->gpu_buffers_handle); AllocatedSharedMemory gpu_buffers_handle = shm_pool->Load( gpu_buffers_shm.data_->buffers); try { if (!gpu_buffers_shm.data_->success) { std::unique_ptr error = PbString::LoadFromSharedMemory( shm_pool, gpu_buffers_shm.data_->error); throw PythonBackendException(error->String()); } #ifdef TRITON_ENABLE_GPU size_t i = 0; for (auto& input_tensor : this->Inputs()) { if (!input_tensor->IsCPU()) { std::unique_ptr dst_buffer = PbMemory::LoadFromSharedMemory( shm_pool, (gpu_buffers_handle.data_.get())[i], true /* open cuda handle */); PbMemory::CopyBuffer(dst_buffer, input_tensor->Memory()); ++i; } } #endif // TRITON_ENABLE_GPU } catch (const PythonBackendException& exception) { // We need to catch the exception here. Otherwise, we will not notify // the main process and it will wait for the response forever. pb_exception = exception; has_exception = true; } { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; ipc_message->ResponseCondition()->notify_all(); ipc_message->ResponseCondition()->wait(lock); } } // The exception will be thrown after the message was sent to the main // process. if (has_exception) { throw pb_exception; } // Get the response for the current message. std::unique_ptr bls_response = IPCMessage::LoadFromSharedMemory( shm_pool, ipc_message->ResponseHandle()); AllocatedSharedMemory response_batch_shm = shm_pool->Load(bls_response->Args()); response_batch = reinterpret_cast(response_batch_shm.data_.get()); response_handle = reinterpret_cast( response_batch_shm.data_.get() + sizeof(ResponseBatch)); responses_is_set = true; if (response_batch->has_error) { if (response_batch->is_error_set) { std::unique_ptr pb_string = PbString::LoadFromSharedMemory(shm_pool, response_batch->error); auto error_response = std::make_unique( std::vector>{}, std::make_shared(pb_string->String())); return error_response; } else { auto error_response = std::make_unique( std::vector>{}, std::make_shared( "An error occurred while performing BLS request.")); return error_response; } } } catch (const PythonBackendException& pb_exception) { auto error_response = std::make_unique( std::vector>{}, std::make_shared(pb_exception.what())); return error_response; } if (responses_is_set) { auto& memory_manager_message_queue = stub->MemoryManagerQueue(); std::unique_ptr return_response = InferResponse::LoadFromSharedMemory( shm_pool, *response_handle, true /* open cuda handle */); for (auto& output_tensor : return_response->OutputTensors()) { if (!output_tensor->IsCPU()) { uint64_t memory_release_id = output_tensor->Memory()->MemoryReleaseId(); output_tensor->Memory()->SetMemoryReleaseCallback( [&memory_manager_message_queue, memory_release_id, &shm_pool]() { memory_manager_message_queue->Push(memory_release_id); }); } } return return_response; } else { auto error_response = std::make_unique( std::vector>{}, std::make_shared( "An error occurred while performing BLS request.")); return error_response; } } #endif }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/infer_request.h000066400000000000000000000151511527052371200262720ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "correlation_id.h" #include "infer_response.h" #include "infer_trace.h" #include "pb_preferred_memory.h" #include "pb_tensor.h" #ifdef TRITON_PB_STUB #include "pb_cancel.h" #include "response_sender.h" #endif namespace triton { namespace backend { namespace python { class Stub; // // Inference Request // struct InferRequestShm { uint32_t input_count; uint32_t requested_output_count; int64_t model_version; uint32_t flags; intptr_t address; intptr_t response_factory_address; bool is_decoupled; uint64_t timeout; PreferredMemory preferred_memory; bi::managed_external_buffer::handle_t trace_shm_handle; uint32_t request_release_flags; bi::managed_external_buffer::handle_t correlation_id_shm_handle; bi::managed_external_buffer::handle_t model_name_shm_handle; bi::managed_external_buffer::handle_t request_id_shm_handle; bi::managed_external_buffer::handle_t parameters_shm_handle; }; class InferRequest { public: InferRequest( const std::string& request_id, const CorrelationId& correlation_id, const std::vector>& inputs, const std::set& requested_output_names, const std::string& model_name, const int64_t model_version, const std::string& parameters, const uint32_t flags = 0, const uint64_t timeout = 0, const intptr_t response_factory_address = 0, const intptr_t request_address = 0, const PreferredMemory& preferred_memory = PreferredMemory(PreferredMemory::kDefault, 0), const InferenceTrace& trace = InferenceTrace()); const std::vector>& Inputs(); const std::string& RequestId(); const std::string& Parameters(); CorrelationId& GetCorrelationId(); const std::string& ModelName(); int64_t ModelVersion(); uint32_t Flags(); void SetFlags(uint32_t flags); const std::set& RequestedOutputNames(); bi::managed_external_buffer::handle_t ShmHandle(); uint64_t Timeout(); bool IsDecoupled(); void SetIsDecoupled(const bool is_decoupled); PreferredMemory& GetPreferredMemory(); InferenceTrace& GetTrace(); uint32_t ReleaseFlags(); void SetReleaseFlags(const uint32_t& flags); intptr_t GetResponseFactoryAddress() { return response_factory_address_; } #ifdef TRITON_PB_STUB std::shared_ptr Exec(const bool is_decoupled); std::shared_ptr GetResponseSender(); bool IsCancelled(); #endif /// Save an Inference Request to shared memory. /// \param shm_pool Shared memory pool to save the inference request. void SaveToSharedMemory(std::unique_ptr& shm_pool); /// Create an Inference Request object from shared memory. /// \param shm_pool Shared memory pool /// \param request_handle Shared memory handle of the request. /// \param open_cuda_handle Determines if the tensor in the infer request /// object is a GPU tensor, to call the cudaIpcOpenMemHandle to obtain the /// tensor or not. /// \return Returns the infer request in the specified request_handle /// location. static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t request_handle, bool open_cuda_handle, bool const* is_model_decoupled); /// Disallow copying the inference request object. DISALLOW_COPY_AND_ASSIGN(InferRequest); intptr_t RequestAddress(); ~InferRequest() {} private: InferRequest( AllocatedSharedMemory& infer_request_shm, std::unique_ptr& request_id_shm, std::unique_ptr& correlation_id, std::vector>& requested_output_names_shm, std::unique_ptr& model_name_shm, std::vector>& input_tensors, std::unique_ptr& parameters_shm, std::unique_ptr& infer_trace_shm, bool const* is_model_decoupled); std::string request_id_; CorrelationId correlation_id_; std::vector> inputs_; std::set requested_output_names_; std::string model_name_; int64_t model_version_; std::string parameters_; uint32_t flags_; uint64_t timeout_; intptr_t response_factory_address_; intptr_t request_address_; bool is_decoupled_; PreferredMemory preferred_memory_; InferenceTrace trace_; uint32_t request_release_flags_; // Shared Memory Data Structures AllocatedSharedMemory infer_request_shm_; InferRequestShm* infer_request_shm_ptr_; std::unique_ptr request_id_shm_; std::vector> requested_output_names_shm_; std::unique_ptr model_name_shm_; bi::managed_external_buffer::handle_t* output_names_handle_shm_ptr_; bi::managed_external_buffer::handle_t* input_tensors_handle_ptr_; bi::managed_external_buffer::handle_t shm_handle_; std::unique_ptr parameters_shm_; #ifdef TRITON_PB_STUB std::shared_ptr pb_cancel_; std::shared_ptr response_sender_; #endif }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/infer_response.cc000066400000000000000000000343411527052371200266000ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_response.h" #ifdef TRITON_PB_STUB #include namespace py = pybind11; #endif #include #include "scoped_defer.h" namespace triton { namespace backend { namespace python { InferResponse::InferResponse( const std::vector>& output_tensors, std::shared_ptr error, const bool is_last_response, void* id) : error_(error), is_last_response_(is_last_response), id_(id) { for (auto& output : output_tensors) { if (!output) { throw PythonBackendException( "Output tensor for inference response should not be empty."); } } output_tensors_ = output_tensors; } std::vector>& InferResponse::OutputTensors() { return output_tensors_; } bool InferResponse::HasError() { return error_.get() != nullptr; } void InferResponse::SaveToSharedMemory( std::unique_ptr& shm_pool, bool copy_gpu) { size_t output_tensor_length = output_tensors_.size(); if (HasError()) { response_shm_ = shm_pool->Construct(sizeof(ResponseShm)); } else { response_shm_ = shm_pool->Construct( sizeof(ResponseShm) + output_tensor_length * sizeof(bi::managed_external_buffer::handle_t)); } ResponseShm* response_shm_ptr = reinterpret_cast(response_shm_.data_.get()); response_shm_ptr->has_error = false; response_shm_ptr->is_error_set = false; shm_handle_ = response_shm_.handle_; response_shm_ptr->is_last_response = is_last_response_; // Only save the output tensors to shared memory when the inference response // doesn't have error. if (HasError()) { response_shm_ptr->has_error = true; Error()->SaveToSharedMemory(shm_pool); response_shm_ptr->is_error_set = true; response_shm_ptr->error = Error()->ShmHandle(); response_shm_ptr->outputs_size = 0; } else { bi::managed_external_buffer::handle_t* tensor_handle_shm_ptr = reinterpret_cast( response_shm_.data_.get() + sizeof(ResponseShm)); response_shm_ptr->outputs_size = output_tensor_length; size_t j = 0; for (auto& output_tensor : output_tensors_) { output_tensor->SaveToSharedMemory(shm_pool, copy_gpu); tensor_handle_shm_ptr[j] = output_tensor->ShmHandle(); j++; } response_shm_ptr->id = id_; } } bi::managed_external_buffer::handle_t InferResponse::ShmHandle() { return shm_handle_; } void InferResponse::PruneOutputTensors( const std::set& requested_output_names) { for (auto it = output_tensors_.begin(); it != output_tensors_.end();) { if (requested_output_names.find((*it)->Name()) == requested_output_names.end()) { it = output_tensors_.erase(it); } else { it++; } } } std::unique_ptr InferResponse::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t response_handle, bool open_cuda_handle) { AllocatedSharedMemory response_shm = shm_pool->Load(response_handle); ResponseShm* response_shm_ptr = reinterpret_cast(response_shm.data_.get()); uint32_t requested_output_count = response_shm_ptr->outputs_size; std::shared_ptr pb_error; std::vector> output_tensors; // If the error field is set, do not load output tensors from shared memory. if (response_shm_ptr->has_error && response_shm_ptr->is_error_set) { pb_error = PbError::LoadFromSharedMemory(shm_pool, response_shm_ptr->error); } else if (response_shm_ptr->has_error && !response_shm_ptr->is_error_set) { pb_error = std::make_shared("Failed to retrieve the response error."); } else { bi::managed_external_buffer::handle_t* tensor_handle_shm = reinterpret_cast( response_shm.data_.get() + sizeof(ResponseShm)); #ifdef TRITON_PB_STUB // Need to acquire the GIL to avoid hangs. py::gil_scoped_acquire acquire; #endif for (size_t idx = 0; idx < requested_output_count; ++idx) { std::shared_ptr pb_tensor = PbTensor::LoadFromSharedMemory( shm_pool, tensor_handle_shm[idx], open_cuda_handle); output_tensors.emplace_back(std::move(pb_tensor)); } } return std::unique_ptr(new InferResponse( response_shm, output_tensors, pb_error, response_shm_ptr->is_last_response, response_shm_ptr->id)); } InferResponse::InferResponse( AllocatedSharedMemory& response_shm, std::vector>& output_tensors, std::shared_ptr& pb_error, const bool is_last_response, void* id) { response_shm_ = std::move(response_shm); output_tensors_ = std::move(output_tensors); error_ = std::move(pb_error); shm_handle_ = response_shm_.handle_; id_ = id; is_last_response_ = is_last_response; } std::shared_ptr& InferResponse::Error() { return error_; } void* InferResponse::Id() { return id_; } bool InferResponse::IsLastResponse() { return is_last_response_; } #ifndef TRITON_PB_STUB void InferResponse::Send( TRITONBACKEND_Response* response, void* cuda_stream, bool& requires_deferred_callback, const uint32_t flags, std::unique_ptr& shm_pool, GPUBuffersHelper& gpu_buffer_helper, std::vector, void*>>& output_buffers, const std::set& requested_output_names) { #ifdef TRITON_ENABLE_GPU static bool log_warning = true; #endif // TRITON_ENABLE_GPU std::shared_ptr response_error = WrapTritonErrorInSharedPtr(nullptr); std::unique_ptr response_error_handling; requires_deferred_callback = false; // This lambda expression will be called when this function exits, if the // inference response doesn't have any GPU tensors. Otherwise, it will be // called when the object is destructed or DeferredSendCallback is called. response_error_handling = std::make_unique([response, response_error, flags] { if (response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend(response, flags, *response_error), "failed to send the response."); } }); // Moves the response sending callback so that it is not called until the stub // process fills in the GPU buffers. ScopedDefer deferred_task([this, &requires_deferred_callback, &response_error_handling, &gpu_buffer_helper, response_error, &shm_pool] { if (*response_error != nullptr) { gpu_buffer_helper.SetError( shm_pool, TRITONSERVER_ErrorMessage(*response_error)); } if (requires_deferred_callback) { deferred_send_callback_ = std::move(response_error_handling); } }); if (HasError()) { *response_error = TRITONSERVER_ErrorNew(Error()->Code(), Error()->Message().c_str()); return; } bool cuda_copy = false; for (auto& output_tensor : OutputTensors()) { // FIXME: for decoupled models we will skip the requested output names. TRITONSERVER_MemoryType src_memory_type = output_tensor->MemoryType(); int64_t src_memory_type_id = output_tensor->MemoryTypeId(); TRITONSERVER_MemoryType actual_memory_type = src_memory_type; int64_t actual_memory_type_id = src_memory_type_id; if (actual_memory_type == TRITONSERVER_MEMORY_GPU) { requires_deferred_callback = true; } TRITONBACKEND_Output* response_output; SET_ERROR_AND_RETURN( response_error, TRITONBACKEND_ResponseOutput( response, &response_output, output_tensor->Name().c_str(), static_cast(output_tensor->TritonDtype()), output_tensor->Dims().data(), output_tensor->Dims().size())); void* triton_output_buffer; SET_ERROR_AND_RETURN( response_error, TRITONBACKEND_OutputBuffer( response_output, &triton_output_buffer, output_tensor->ByteSize(), &actual_memory_type, &actual_memory_type_id)); bool cuda_used = false; TRITONSERVER_BufferAttributes* output_buffer_attributes; SET_ERROR_AND_RETURN( response_error, TRITONBACKEND_OutputBufferAttributes( response_output, &output_buffer_attributes)); std::unique_ptr output_buffer; if (src_memory_type == TRITONSERVER_MEMORY_GPU && actual_memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU // Check if the triton-provided output buffer is using CUDA shared memory // pool. If not, try to allocate a new buffer from the pool. void* buffer = triton_output_buffer; BackendMemory* backend_memory; std::unique_ptr lbackend_memory; std::unique_ptr& cuda_pool = shm_pool->GetCUDAMemoryPoolManager(); if (cuda_pool->UseCudaSharedPool(src_memory_type_id)) { try { if (!IsUsingCUDAPool( cuda_pool, actual_memory_type_id, triton_output_buffer)) { THROW_IF_TRITON_ERROR(BackendMemory::Create( reinterpret_cast( shm_pool->GetCUDAMemoryPoolManager() ->TritonMemoryManager()), BackendMemory::AllocationType::GPU_POOL, actual_memory_type_id, output_tensor->ByteSize(), &backend_memory)); lbackend_memory.reset(backend_memory); buffer = lbackend_memory->MemoryPtr(); } } catch (const PythonBackendException& pb_exception) { if (log_warning) { LOG_MESSAGE( TRITONSERVER_LOG_WARN, (std::string("Failed to allocate memory from CUDA memory pool " "for output tensor: ") + pb_exception.what() + std::string(", will use CUDA IPC for GPU output transfer.")) .c_str()); } log_warning = false; } } cudaIpcMemHandle_t* cuda_ipc_mem_handle_p; SET_ERROR_AND_RETURN( response_error, TRITONSERVER_BufferAttributesCudaIpcHandle( output_buffer_attributes, reinterpret_cast(&cuda_ipc_mem_handle_p))); if (cuda_ipc_mem_handle_p != nullptr) { SET_ERROR_AND_RETURN_IF_EXCEPTION( response_error, output_buffer = PbMemory::Create( shm_pool, actual_memory_type, actual_memory_type_id, output_tensor->ByteSize(), reinterpret_cast(buffer), false /* copy_gpu */)); output_buffer->SetCudaIpcHandle(cuda_ipc_mem_handle_p); } else { SET_ERROR_AND_RETURN_IF_EXCEPTION( response_error, output_buffer = PbMemory::Create( shm_pool, actual_memory_type, actual_memory_type_id, output_tensor->ByteSize(), reinterpret_cast(buffer), true /* copy_gpu */)); } if (lbackend_memory != nullptr) { output_buffer->SetBackendMemory(std::move(lbackend_memory)); } gpu_buffer_helper.AddBuffer(output_buffer->ShmHandle()); output_buffers.push_back( {std::move(output_buffer), triton_output_buffer}); #endif } // When we requested a GPU buffer but received a CPU buffer. if (src_memory_type == TRITONSERVER_MEMORY_GPU && (actual_memory_type == TRITONSERVER_MEMORY_CPU || actual_memory_type == TRITONSERVER_MEMORY_CPU_PINNED)) { SET_ERROR_AND_RETURN_IF_EXCEPTION( response_error, output_buffer = PbMemory::Create( shm_pool, actual_memory_type, actual_memory_type_id, output_tensor->ByteSize(), nullptr /* data ptr */)); gpu_buffer_helper.AddBuffer(output_buffer->ShmHandle()); output_buffers.push_back( {std::move(output_buffer), triton_output_buffer}); } if (src_memory_type != TRITONSERVER_MEMORY_GPU) { SET_ERROR_AND_RETURN( response_error, CopyBuffer( "Failed to copy the output tensor to buffer.", src_memory_type, src_memory_type_id, actual_memory_type, actual_memory_type_id, output_tensor->ByteSize(), output_tensor->DataPtr(), triton_output_buffer, reinterpret_cast(cuda_stream), &cuda_used)); } cuda_copy |= cuda_used; } #ifdef TRITON_ENABLE_GPU if (cuda_copy) { cudaStreamSynchronize(reinterpret_cast(cuda_stream)); } #endif // TRITON_ENABLE_GPU } #endif #ifndef TRITON_PB_STUB void InferResponse::DeferredSendCallback() { deferred_send_callback_.reset(); } #endif }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/infer_response.h000066400000000000000000000130311527052371200264330ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "gpu_buffers.h" #include "pb_error.h" #include "pb_tensor.h" #include "pb_utils.h" #include "scoped_defer.h" namespace triton { namespace backend { namespace python { struct ResponseShm { uint32_t outputs_size; bi::managed_external_buffer::handle_t error; bool has_error; // Indicates whether this error has a message or not. bool is_error_set; void* id; bool is_last_response; }; #define SET_ERROR_AND_RETURN(E, X) \ do { \ TRITONSERVER_Error* raasnie_err__ = (X); \ if (raasnie_err__ != nullptr) { \ *E = raasnie_err__; \ return; \ } \ } while (false) #define SET_ERROR_AND_RETURN_IF_EXCEPTION(E, X) \ do { \ try { \ (X); \ } \ catch (const PythonBackendException& pb_exception) { \ TRITONSERVER_Error* rarie_err__ = TRITONSERVER_ErrorNew( \ TRITONSERVER_ERROR_INTERNAL, pb_exception.what()); \ *E = rarie_err__; \ return; \ } \ } while (false) class InferResponse { public: InferResponse( const std::vector>& output_tensors, std::shared_ptr error = nullptr, const bool is_last_response = true, void* id = nullptr); std::vector>& OutputTensors(); void SaveToSharedMemory( std::unique_ptr& shm_pool, bool copy_gpu = true); static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t response_handle, bool open_cuda_handle); bool HasError(); std::shared_ptr& Error(); bi::managed_external_buffer::handle_t ShmHandle(); void PruneOutputTensors(const std::set& requested_output_names); std::unique_ptr>> GetNextResponse(); void SetNextResponseHandle( bi::managed_external_buffer::handle_t next_response_handle); bi::managed_external_buffer::handle_t NextResponseHandle(); void* Id(); bool IsLastResponse(); #ifndef TRITON_PB_STUB /// Send an inference response. If the response has a GPU tensor, sending the /// response needs to be done in two step. The boolean /// 'requires_deferred_callback' indicates whether DeferredSendCallback method /// should be called or not. void Send( TRITONBACKEND_Response* response, void* cuda_stream, bool& requires_deferred_callback, const uint32_t flags, std::unique_ptr& shm_pool, GPUBuffersHelper& gpu_buffer_helper, std::vector, void*>>& output_buffers, const std::set& requested_output_names = {}); void DeferredSendCallback(); #endif // Disallow copying the inference response object. DISALLOW_COPY_AND_ASSIGN(InferResponse); private: InferResponse( AllocatedSharedMemory& response_shm, std::vector>& output_tensors, std::shared_ptr& pb_error, const bool is_last_response, void* id); std::vector> output_tensors_; std::shared_ptr error_; bi::managed_external_buffer::handle_t shm_handle_; AllocatedSharedMemory response_shm_; std::vector, void*>> gpu_output_buffers_; std::unique_ptr deferred_send_callback_; bool is_last_response_; // Representing the request id that the response was created from. void* id_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/infer_trace.cc000066400000000000000000000075331527052371200260430ustar00rootroot00000000000000// Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_trace.h" namespace triton { namespace backend { namespace python { InferenceTrace::InferenceTrace(const InferenceTrace& rhs) { triton_trace_ = rhs.triton_trace_; trace_context_ = rhs.trace_context_; } InferenceTrace& InferenceTrace::operator=(const InferenceTrace& rhs) { triton_trace_ = rhs.triton_trace_; trace_context_ = rhs.trace_context_; return *this; } InferenceTrace::InferenceTrace(std::unique_ptr& trace_shm) { triton_trace_ = trace_shm->triton_trace_; trace_context_ = trace_shm->trace_context_; } void InferenceTrace::SaveToSharedMemory( std::unique_ptr& shm_pool) { AllocatedSharedMemory infer_trace_shm = shm_pool->Construct(); infer_trace_shm_ptr_ = infer_trace_shm.data_.get(); infer_trace_shm_ptr_->triton_trace = triton_trace_; std::unique_ptr trace_context_shm = PbString::Create(shm_pool, trace_context_); infer_trace_shm_ptr_->trace_context_shm_handle = trace_context_shm->ShmHandle(); // Save the references to shared memory. trace_context_shm_ = std::move(trace_context_shm); infer_trace_shm_ = std::move(infer_trace_shm); shm_handle_ = infer_trace_shm_.handle_; } std::unique_ptr InferenceTrace::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle) { AllocatedSharedMemory infer_trace_shm = shm_pool->Load(handle); InferenceTraceShm* infer_trace_shm_ptr = infer_trace_shm.data_.get(); std::unique_ptr trace_context_shm = PbString::LoadFromSharedMemory( shm_pool, infer_trace_shm_ptr->trace_context_shm_handle); return std::unique_ptr( new InferenceTrace(infer_trace_shm, trace_context_shm)); } InferenceTrace::InferenceTrace( AllocatedSharedMemory& infer_trace_shm, std::unique_ptr& trace_context_shm) : infer_trace_shm_(std::move(infer_trace_shm)), trace_context_shm_(std::move(trace_context_shm)) { infer_trace_shm_ptr_ = infer_trace_shm_.data_.get(); shm_handle_ = infer_trace_shm_.handle_; triton_trace_ = infer_trace_shm_ptr_->triton_trace; trace_context_ = trace_context_shm_->String(); } }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/infer_trace.h000066400000000000000000000071441527052371200257030ustar00rootroot00000000000000// Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "pb_string.h" #include "pb_utils.h" namespace triton { namespace backend { namespace python { struct InferenceTraceShm { bi::managed_external_buffer::handle_t trace_context_shm_handle; // The address of the 'TRITONSERVER_InferTrace' object. void* triton_trace; }; // // Inference Trace // class InferenceTrace { public: InferenceTrace(void* triton_trace, const std::string& ctxt) : triton_trace_(triton_trace), trace_context_(ctxt) { } InferenceTrace() : triton_trace_(nullptr), trace_context_("") {} InferenceTrace(const InferenceTrace& rhs); InferenceTrace(std::unique_ptr& trace_shm); InferenceTrace& operator=(const InferenceTrace& rhs); /// Save InferenceTrace object to shared memory. /// \param shm_pool Shared memory pool to save the InferenceTrace object. void SaveToSharedMemory(std::unique_ptr& shm_pool); /// Create a InferenceTrace object from shared memory. /// \param shm_pool Shared memory pool /// \param handle Shared memory handle of the InferenceTrace. /// \return Returns the InferenceTrace in the specified handle /// location. static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); void* TritonTrace() { return triton_trace_; } const std::string& Context() const { return trace_context_; } bi::managed_external_buffer::handle_t ShmHandle() { return shm_handle_; } private: // The private constructor for creating a InferenceTrace object from shared // memory. InferenceTrace( AllocatedSharedMemory& infer_trace_shm, std::unique_ptr& trace_context_shm); void* triton_trace_; std::string trace_context_; // Shared Memory Data Structures AllocatedSharedMemory infer_trace_shm_; InferenceTraceShm* infer_trace_shm_ptr_; bi::managed_external_buffer::handle_t shm_handle_; std::unique_ptr trace_context_shm_; }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/ipc_message.cc000066400000000000000000000126201527052371200260320ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ipc_message.h" #include namespace triton { namespace backend { namespace python { std::unique_ptr IPCMessage::Create( const std::unique_ptr& shm_pool, bool inline_response) { AllocatedSharedMemory ipc_message_shm = shm_pool->Construct(); ipc_message_shm.data_->inline_response = inline_response; AllocatedSharedMemory response_mutex_shm; AllocatedSharedMemory response_cond_shm; if (inline_response) { response_mutex_shm = std::move(shm_pool->Construct( 1 /* count */, true /* aligned */)); response_cond_shm = std::move(shm_pool->Construct( 1 /* count */, true /* aligned */)); ipc_message_shm.data_->response_mutex = response_mutex_shm.handle_; ipc_message_shm.data_->response_cond = response_cond_shm.handle_; new (response_mutex_shm.data_.get()) bi::interprocess_mutex{}; new (response_cond_shm.data_.get()) bi::interprocess_condition{}; } return std::unique_ptr( new IPCMessage(ipc_message_shm, response_mutex_shm, response_cond_shm)); } std::unique_ptr IPCMessage::Create( IPCMessageShm* ipc_message_shm, bi::managed_external_buffer::handle_t& message_handle) { return std::unique_ptr( new IPCMessage(ipc_message_shm, message_handle)); } AllocatedSharedMemory& IPCMessage::GetAllocatedSharedMemory() { return ipc_message_shm_; } std::unique_ptr IPCMessage::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t message_handle) { AllocatedSharedMemory ipc_message_shm = shm_pool->Load(message_handle); AllocatedSharedMemory response_mutex_shm; AllocatedSharedMemory response_cond_shm; if (ipc_message_shm.data_->inline_response) { response_mutex_shm = shm_pool->Load( ipc_message_shm.data_->response_mutex); response_cond_shm = shm_pool->Load( ipc_message_shm.data_->response_cond); } return std::unique_ptr( new IPCMessage(ipc_message_shm, response_mutex_shm, response_cond_shm)); } PYTHONSTUB_CommandType& IPCMessage::Command() { return ipc_message_shm_ptr_->command; } bi::managed_external_buffer::handle_t& IPCMessage::Args() { return ipc_message_shm_ptr_->args; } bool& IPCMessage::InlineResponse() { return ipc_message_shm_ptr_->inline_response; } bi::interprocess_condition* IPCMessage::ResponseCondition() { return response_cond_shm_ptr_; } bi::interprocess_mutex* IPCMessage::ResponseMutex() { return response_mutex_shm_ptr_; } bi::managed_external_buffer::handle_t& IPCMessage::ResponseHandle() { return ipc_message_shm_ptr_->response_handle; } bi::managed_external_buffer::handle_t IPCMessage::ShmHandle() { return ipc_message_handle_; } IPCMessage::IPCMessage( AllocatedSharedMemory& ipc_message_shm, AllocatedSharedMemory& response_mutex_shm, AllocatedSharedMemory& response_cond_shm) : ipc_message_shm_(std::move(ipc_message_shm)), response_mutex_shm_(std::move(response_mutex_shm)), response_cond_shm_(std::move(response_cond_shm)) { ipc_message_shm_ptr_ = ipc_message_shm_.data_.get(); response_mutex_shm_ptr_ = response_mutex_shm_.data_.get(); response_cond_shm_ptr_ = response_cond_shm_.data_.get(); ipc_message_handle_ = ipc_message_shm_.handle_; } IPCMessage::IPCMessage( IPCMessageShm* ipc_message_shm, bi::managed_external_buffer::handle_t& handle) { ipc_message_handle_ = handle; ipc_message_shm_ptr_ = ipc_message_shm; } }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/ipc_message.h000066400000000000000000000132421527052371200256750ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "shm_manager.h" namespace triton { namespace backend { namespace python { namespace bi = boost::interprocess; typedef enum PYTHONSTUB_commandtype_enum { PYTHONSTUB_ExecuteRequest, PYTHONSTUB_ExecuteResponse, PYTHONSTUB_InitializeRequest, PYTHONSTUB_InitializeResponse, PYTHONSTUB_CUDAPoolInitializeRequest, PYTHONSTUB_FinalizeRequest, PYTHONSTUB_FinalizeResponse, PYTHONSTUB_LoadGPUBuffers, PYTHONSTUB_InferExecRequest, PYTHONSTUB_InferStreamExecRequest, PYTHONSTUB_InferExecResponse, PYTHONSTUB_InferStreamExecResponse, PYTHONSTUB_ResponseSend, PYTHONSTUB_ResponseClose, PYTHONSTUB_AutoCompleteRequest, PYTHONSTUB_AutoCompleteResponse, PYTHONSTUB_LogRequest, PYTHONSTUB_BLSDecoupledInferPayloadCleanup, PYTHONSTUB_DecoupledResponseFactoryCleanup, PYTHONSTUB_MetricFamilyRequestNew, PYTHONSTUB_MetricFamilyRequestDelete, PYTHONSTUB_MetricRequestNew, PYTHONSTUB_MetricRequestDelete, PYTHONSTUB_MetricRequestValue, PYTHONSTUB_MetricRequestIncrement, PYTHONSTUB_MetricRequestSet, PYTHONSTUB_MetricRequestObserve, PYTHONSTUB_LoadModelRequest, PYTHONSTUB_UnloadModelRequest, PYTHONSTUB_ModelReadinessRequest, PYTHONSTUB_IsRequestCancelled } PYTHONSTUB_CommandType; /// /// Shared memory representation of IPCMessage /// /// \param command determines the IPC command that is going to be passed. /// \param args determines the shared memory handle for the input parameters. /// \param inline_response determines whether this is a response of another IPC /// message. If this parameter is set, it must provide the handle of the /// corresponding request in \param response_handle. /// \param response_handle determines the request handle. /// \param response_mutex stores the handle for the mutex for the response /// object. /// \param response_cond stores the handle for the condition variable /// for the response object. struct IPCMessageShm { PYTHONSTUB_CommandType command; bi::managed_external_buffer::handle_t args; bool inline_response = false; bi::managed_external_buffer::handle_t response_handle; bi::managed_external_buffer::handle_t response_mutex; bi::managed_external_buffer::handle_t response_cond; }; class IPCMessage { public: static std::unique_ptr Create( const std::unique_ptr& shm_pool, bool inline_response); static std::unique_ptr Create( IPCMessageShm* ipc_message_shm, bi::managed_external_buffer::handle_t& message_handle); static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t message_handle); PYTHONSTUB_CommandType& Command(); bool& InlineResponse(); bi::managed_external_buffer::handle_t& ResponseHandle(); bi::interprocess_condition* ResponseCondition(); bi::interprocess_mutex* ResponseMutex(); bi::managed_external_buffer::handle_t& Args(); bi::managed_external_buffer::handle_t ShmHandle(); AllocatedSharedMemory& GetAllocatedSharedMemory(); private: AllocatedSharedMemory ipc_message_shm_; IPCMessageShm* ipc_message_shm_ptr_; AllocatedSharedMemory response_mutex_shm_; bi::interprocess_mutex* response_mutex_shm_ptr_; AllocatedSharedMemory response_cond_shm_; bi::interprocess_condition* response_cond_shm_ptr_; bi::managed_external_buffer::handle_t ipc_message_handle_; /// Create/load a IPCMessage shm object. /// \param ipc_message_shm IPCMessage representation in shared memory. /// \param response_mutex_shm response mutex. /// \param response_condition_shm response condition. IPCMessage( AllocatedSharedMemory& ipc_message_shm, AllocatedSharedMemory& response_mutex_shm, AllocatedSharedMemory& response_cond_shm); IPCMessage( IPCMessageShm* ipc_message_shm, bi::managed_external_buffer::handle_t& handle); }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/libtriton_python.ldscript000066400000000000000000000030551527052371200304230ustar00rootroot00000000000000# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; kytensor-2.49.0.7/framework_backends/python_backend/src/memory_manager.cc000066400000000000000000000067331527052371200265650ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "memory_manager.h" #include "pb_utils.h" namespace triton { namespace backend { namespace python { #ifdef TRITON_ENABLE_GPU BackendMemoryRecord::BackendMemoryRecord( std::unique_ptr backend_memory) : backend_memory_(std::move(backend_memory)) { release_callback_ = [](void* ptr) { // Do nothing. The backend_memory_ will be destroyed in the destructor. }; } void* BackendMemoryRecord::MemoryId() { return reinterpret_cast(backend_memory_->MemoryPtr()); } const std::function& BackendMemoryRecord::ReleaseCallback() { return release_callback_; } #endif MemoryManager::MemoryManager( std::unique_ptr>&& memory_message_queue) { message_queue_ = std::move(memory_message_queue); thread_ = std::thread(&MemoryManager::QueueMonitorThread, this); } intptr_t MemoryManager::AddRecord(std::unique_ptr&& memory_record) { std::lock_guard lock{mu_}; intptr_t memory_record_id = reinterpret_cast(memory_record->MemoryId()); records_.emplace(memory_record_id, std::move(memory_record)); return memory_record_id; } void MemoryManager::QueueMonitorThread() { while (true) { intptr_t memory = message_queue_->Pop(); if (memory == 0) { return; } { std::lock_guard lock{mu_}; auto it = records_.find(memory); if (it == records_.end()) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, "Unexpected memory index received for deallocation."); continue; } // Call the release callback. it->second->ReleaseCallback()(it->second->MemoryId()); // it->second.reset(); records_.erase(it); } } } MemoryManager::~MemoryManager() { // Push a dummy message that will trigger the destruction of the background // thread. message_queue_->Push(DUMMY_MESSAGE); thread_.join(); } }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/memory_manager.h000066400000000000000000000065611527052371200264260ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "message_queue.h" #include "triton/backend/backend_common.h" #include "triton/backend/backend_memory.h" #include "triton/core/tritonserver.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace backend { namespace python { class MemoryRecord { public: virtual const std::function& ReleaseCallback() = 0; virtual void* MemoryId() = 0; virtual ~MemoryRecord() = default; }; #ifdef TRITON_ENABLE_GPU class BackendMemoryRecord : public MemoryRecord { public: BackendMemoryRecord(std::unique_ptr backend_memory); const std::function& ReleaseCallback() override; void* MemoryId() override; ~BackendMemoryRecord() { backend_memory_.reset(); } private: std::unique_ptr backend_memory_; std::function release_callback_; }; #endif /// Memory manager class is used primarily for managing the lifetime of GPU /// tensors in BLS. It mainly consists of a background thread that monitors a /// message queue in shared memory. Whenever a GPU tensor is created, it will /// be pushed to the memory manager. The stub process must send a message to the /// message queue asking the memory manager to deallocate the GPU tensor. class MemoryManager { public: MemoryManager(std::unique_ptr>&& memory_message_queue); intptr_t AddRecord(std::unique_ptr&& memory_record); TRITONSERVER_Error* ResetCounter(); ~MemoryManager(); private: std::thread thread_; std::unordered_map> records_; std::unique_ptr> message_queue_; void QueueMonitorThread(); std::mutex mu_; }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/message_queue.h000066400000000000000000000217341527052371200262530ustar00rootroot00000000000000// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include "shm_manager.h" namespace triton { namespace backend { namespace python { namespace bi = boost::interprocess; /// Struct holding the representation of a message queue inside the shared /// memory. /// \param size Total size of the message queue. /// \param mutex Handle of the mutex variable protecting index. /// \param index Used element index. /// \param sem_empty Semaphore object counting the number of empty buffer slots. /// \param sem_full Semaphore object counting the number of used buffer slots. struct MessageQueueShm { bi::interprocess_semaphore sem_empty{0}; bi::interprocess_semaphore sem_full{0}; bi::interprocess_mutex mutex; std::size_t size; bi::managed_external_buffer::handle_t buffer; int head; int tail; }; template class MessageQueue { public: /// Create a new MessageQueue in the shared memory. static std::unique_ptr> Create( std::unique_ptr& shm_pool, uint32_t message_queue_size) { AllocatedSharedMemory mq_shm = shm_pool->Construct(); mq_shm.data_->size = message_queue_size; AllocatedSharedMemory mq_buffer_shm = shm_pool->Construct(message_queue_size /* count */); mq_shm.data_->buffer = mq_buffer_shm.handle_; mq_shm.data_->head = 0; mq_shm.data_->tail = 0; new (&(mq_shm.data_->mutex)) bi::interprocess_mutex{}; new (&(mq_shm.data_->sem_empty)) bi::interprocess_semaphore{message_queue_size}; new (&(mq_shm.data_->sem_full)) bi::interprocess_semaphore{0}; return std::unique_ptr>( new MessageQueue(mq_shm, mq_buffer_shm)); } /// Load an already existing message queue from the shared memory. static std::unique_ptr> LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t message_queue_handle) { AllocatedSharedMemory mq_shm = shm_pool->Load(message_queue_handle); AllocatedSharedMemory mq_shm_buffer = shm_pool->Load(mq_shm.data_->buffer); return std::unique_ptr>( new MessageQueue(mq_shm, mq_shm_buffer)); } /// Push a message inside the message queue. /// \param message The shared memory handle of the message. void Push(T message) { while (true) { try { SemEmptyMutable()->wait(); break; } catch (bi::interprocess_exception& ex) { } } { bi::scoped_lock lock{*MutexMutable()}; Buffer()[Head()] = message; HeadIncrement(); } SemFullMutable()->post(); } void Push(T message, int const& duration, bool& success) { boost::system_time timeout = boost::get_system_time() + boost::posix_time::milliseconds(duration); while (true) { try { if (!SemEmptyMutable()->timed_wait(timeout)) { success = false; return; } else { break; } } catch (bi::interprocess_exception& ex) { } } { timeout = boost::get_system_time() + boost::posix_time::milliseconds(duration); bi::scoped_lock lock{*MutexMutable(), timeout}; if (!lock) { SemEmptyMutable()->post(); success = false; return; } success = true; Buffer()[Head()] = message; HeadIncrement(); } SemFullMutable()->post(); } /// Pop a message from the message queue. This call will block until there /// is a message inside the message queue to return. /// \return the handle of the new message. T Pop() { T message; while (true) { try { SemFullMutable()->wait(); break; } catch (bi::interprocess_exception& ex) { } } { bi::scoped_lock lock{*MutexMutable()}; message = Buffer()[Tail()]; TailIncrement(); } SemEmptyMutable()->post(); return message; } T Pop(int const& duration, bool& success) { T message = 0; boost::system_time timeout = boost::get_system_time() + boost::posix_time::milliseconds(duration); while (true) { try { if (!SemFullMutable()->timed_wait(timeout)) { success = false; return message; } else { break; } } catch (bi::interprocess_exception& ex) { } } { timeout = boost::get_system_time() + boost::posix_time::milliseconds(duration); bi::scoped_lock lock{*MutexMutable(), timeout}; if (!lock) { SemFullMutable()->post(); success = false; return message; } success = true; message = Buffer()[Tail()]; TailIncrement(); } SemEmptyMutable()->post(); return message; } /// Resets the semaphores for the message queue. This function is useful for /// when the stub process may have exited unexpectedly and the semaphores need /// to be restarted so that the message queue is in a proper state. void ResetSemaphores() { new (SemFullMutable()) bi::interprocess_semaphore(0); new (SemEmptyMutable()) bi::interprocess_semaphore(Size()); new (MutexMutable()) bi::interprocess_mutex; mq_shm_ptr_->tail = 0; mq_shm_ptr_->head = 0; } /// Get the shared memory handle of MessageQueue bi::managed_external_buffer::handle_t ShmHandle() { return mq_handle_; } /// Release the ownership of this object in shared memory. void Release() { if (mq_shm_.data_ != nullptr) { mq_shm_.data_.release(); } if (mq_buffer_shm_.data_ != nullptr) { mq_buffer_shm_.data_.release(); } } private: std::size_t& Size() { return mq_shm_ptr_->size; } const bi::interprocess_mutex& Mutex() { return mq_shm_ptr_->mutex; } bi::interprocess_mutex* MutexMutable() { return &(mq_shm_ptr_->mutex); } int& Head() { return mq_shm_ptr_->head; } int& Tail() { return mq_shm_ptr_->tail; } T* Buffer() { return mq_buffer_shm_ptr_; } const bi::interprocess_semaphore& SemEmpty() { return mq_shm_ptr_->sem_empty; } bi::interprocess_semaphore* SemEmptyMutable() { return &(mq_shm_ptr_->sem_empty); } const bi::interprocess_semaphore& SemFull() { return mq_shm_ptr_->sem_full; } bi::interprocess_semaphore* SemFullMutable() { return &(mq_shm_ptr_->sem_full); } void HeadIncrement() { mq_shm_ptr_->head = (mq_shm_ptr_->head + 1) % Size(); } void TailIncrement() { mq_shm_ptr_->tail = (mq_shm_ptr_->tail + 1) % Size(); } AllocatedSharedMemory mq_shm_; AllocatedSharedMemory mq_buffer_shm_; MessageQueueShm* mq_shm_ptr_; T* mq_buffer_shm_ptr_; bi::managed_external_buffer::handle_t mq_handle_; /// Create/load a Message queue. /// \param mq_shm Message queue representation in shared memory. MessageQueue( AllocatedSharedMemory& mq_shm, AllocatedSharedMemory& mq_buffer_shm) : mq_shm_(std::move(mq_shm)), mq_buffer_shm_(std::move(mq_buffer_shm)) { mq_buffer_shm_ptr_ = mq_buffer_shm_.data_.get(); mq_shm_ptr_ = mq_shm_.data_.get(); mq_handle_ = mq_shm_.handle_; } }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/metric.cc000066400000000000000000000321361527052371200250420ustar00rootroot00000000000000// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "metric.h" #ifdef TRITON_PB_STUB #include "pb_stub.h" #endif namespace triton { namespace backend { namespace python { Metric::Metric( const std::string& labels, std::optional> buckets, void* metric_family_address) : labels_(labels), buckets_(buckets), operation_value_(0), metric_address_(nullptr), metric_family_address_(metric_family_address), is_cleared_(false) { #ifdef TRITON_PB_STUB SendCreateMetricRequest(); #endif } Metric::~Metric() { #ifdef TRITON_PB_STUB Clear(); #endif } void Metric::SaveToSharedMemory(std::unique_ptr& shm_pool) { AllocatedSharedMemory custom_metric_shm = shm_pool->Construct(); custom_metric_shm_ptr_ = custom_metric_shm.data_.get(); std::unique_ptr labels_shm = PbString::Create(shm_pool, labels_); custom_metric_shm_ptr_->operation_value = operation_value_; custom_metric_shm_ptr_->labels_shm_handle = labels_shm->ShmHandle(); custom_metric_shm_ptr_->metric_family_address = metric_family_address_; custom_metric_shm_ptr_->metric_address = metric_address_; // Histogram specific case if (buckets_.has_value()) { auto buckets_size = buckets_.value().size() * sizeof(double); std::unique_ptr buckets_shm = PbMemory::Create( shm_pool, TRITONSERVER_MemoryType::TRITONSERVER_MEMORY_CPU, 0, buckets_size, reinterpret_cast(buckets_.value().data()), false /* copy_gpu */); custom_metric_shm_ptr_->buckets_shm_handle = buckets_shm->ShmHandle(); buckets_shm_ = std::move(buckets_shm); } else { custom_metric_shm_ptr_->buckets_shm_handle = 0; buckets_shm_ = nullptr; } // Save the references to shared memory. custom_metric_shm_ = std::move(custom_metric_shm); labels_shm_ = std::move(labels_shm); shm_handle_ = custom_metric_shm_.handle_; } std::unique_ptr Metric::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle) { AllocatedSharedMemory custom_metric_shm = shm_pool->Load(handle); MetricShm* custom_metric_shm_ptr = custom_metric_shm.data_.get(); std::unique_ptr labels_shm = PbString::LoadFromSharedMemory( shm_pool, custom_metric_shm_ptr->labels_shm_handle); std::unique_ptr buckets_shm = nullptr; if (custom_metric_shm_ptr->buckets_shm_handle != 0) { buckets_shm = PbMemory::LoadFromSharedMemory( shm_pool, custom_metric_shm_ptr->buckets_shm_handle, false /* open_cuda_handle */); } return std::unique_ptr( new Metric(custom_metric_shm, labels_shm, buckets_shm)); } Metric::Metric( AllocatedSharedMemory& custom_metric_shm, std::unique_ptr& labels_shm, std::unique_ptr& buckets_shm) : custom_metric_shm_(std::move(custom_metric_shm)), labels_shm_(std::move(labels_shm)), buckets_shm_(std::move(buckets_shm)) { custom_metric_shm_ptr_ = custom_metric_shm_.data_.get(); // FIXME: This constructor is called during each // set/increment/observe/get_value call. It only needs the pointers. labels_ = labels_shm_->String(); if (buckets_shm_ != nullptr) { // Histogram size_t bucket_size = buckets_shm_->ByteSize() / sizeof(double); std::vector buckets; buckets.reserve(bucket_size); for (size_t i = 0; i < bucket_size; ++i) { buckets.emplace_back( reinterpret_cast(buckets_shm_->DataPtr())[i]); } buckets_ = std::move(buckets); } operation_value_ = custom_metric_shm_ptr_->operation_value; metric_family_address_ = custom_metric_shm_ptr_->metric_family_address; metric_address_ = custom_metric_shm_ptr_->metric_address; } void* Metric::MetricAddress() { return metric_address_; } #ifdef TRITON_PB_STUB void Metric::SendCreateMetricRequest() { // Send the request to create the Metric to the parent process std::unique_ptr& stub = Stub::GetOrCreateInstance(); SaveToSharedMemory(stub->ShmPool()); CustomMetricsMessage* custom_metrics_msg = nullptr; AllocatedSharedMemory custom_metrics_shm; try { stub->SendMessage( custom_metrics_shm, PYTHONSTUB_MetricRequestNew, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Error when creating Metric: " + std::string(pb_exception.what())); } custom_metrics_msg = custom_metrics_shm.data_.get(); metric_address_ = custom_metrics_msg->address; } void Metric::SendIncrementRequest(const double& value) { py::gil_scoped_release release; try { CheckIfCleared(); std::unique_ptr& stub = Stub::GetOrCreateInstance(); operation_value_ = value; SaveToSharedMemory(stub->ShmPool()); AllocatedSharedMemory custom_metrics_shm; stub->SendMessage( custom_metrics_shm, PYTHONSTUB_MetricRequestIncrement, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Failed to increment metric value: " + std::string(pb_exception.what())); } } void Metric::SendSetValueRequest(const double& value) { try { CheckIfCleared(); std::unique_ptr& stub = Stub::GetOrCreateInstance(); operation_value_ = value; SaveToSharedMemory(stub->ShmPool()); AllocatedSharedMemory custom_metrics_shm; stub->SendMessage( custom_metrics_shm, PYTHONSTUB_MetricRequestSet, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Failed to set metric value: " + std::string(pb_exception.what())); } } void Metric::SendObserveRequest(const double& value) { py::gil_scoped_release release; try { CheckIfCleared(); std::unique_ptr& stub = Stub::GetOrCreateInstance(); operation_value_ = value; SaveToSharedMemory(stub->ShmPool()); AllocatedSharedMemory custom_metrics_shm; stub->SendMessage( custom_metrics_shm, PYTHONSTUB_MetricRequestObserve, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Failed to observe metric value: " + std::string(pb_exception.what())); } } double Metric::SendGetValueRequest() { CustomMetricsMessage* custom_metrics_msg = nullptr; AllocatedSharedMemory custom_metrics_shm; try { CheckIfCleared(); std::unique_ptr& stub = Stub::GetOrCreateInstance(); SaveToSharedMemory(stub->ShmPool()); stub->SendMessage( custom_metrics_shm, PYTHONSTUB_MetricRequestValue, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Failed to get metric value: " + std::string(pb_exception.what())); } custom_metrics_msg = custom_metrics_shm.data_.get(); return custom_metrics_msg->value; } void Metric::Clear() { // Need to check if the metric has been cleared before as the Clear()' // function can be called from two different locations: when the metric family // clears the 'metric_map_' and when the 'Metric' object goes out of // scope/being deleted. if (!is_cleared_) { is_cleared_ = true; std::unique_ptr& stub = Stub::GetOrCreateInstance(); SaveToSharedMemory(stub->ShmPool()); AllocatedSharedMemory custom_metrics_shm; try { stub->SendMessage( custom_metrics_shm, PYTHONSTUB_MetricRequestDelete, shm_handle_); } catch (const PythonBackendException& pb_exception) { std::cerr << "Error when deleting Metric: " << pb_exception.what() << "\n"; } } } void Metric::CheckIfCleared() { if (is_cleared_) { throw PythonBackendException( "Invalid metric operation as the corresponding 'MetricFamily' has been " "deleted. The 'MetricFamily' object should be deleted AFTER its " "corresponding 'Metric' objects have been deleted."); } } #else void* Metric::InitializeTritonMetric() { std::vector labels_params; ParseLabels(labels_params, labels_); TRITONSERVER_MetricKind kind; THROW_IF_TRITON_ERROR(TRITONSERVER_GetMetricFamilyKind( reinterpret_cast(metric_family_address_), &kind)); TRITONSERVER_MetricArgs* args = nullptr; switch (kind) { case TRITONSERVER_METRIC_KIND_COUNTER: case TRITONSERVER_METRIC_KIND_GAUGE: break; case TRITONSERVER_METRIC_KIND_HISTOGRAM: { const std::vector& buckets = buckets_.value(); THROW_IF_TRITON_ERROR(TRITONSERVER_MetricArgsNew(&args)); THROW_IF_TRITON_ERROR(TRITONSERVER_MetricArgsSetHistogram( args, buckets.data(), buckets.size())); break; } default: break; } TRITONSERVER_Metric* triton_metric = nullptr; THROW_IF_TRITON_ERROR(TRITONSERVER_MetricNewWithArgs( &triton_metric, reinterpret_cast(metric_family_address_), labels_params.data(), labels_params.size(), args)); for (const auto label : labels_params) { TRITONSERVER_ParameterDelete(const_cast(label)); } THROW_IF_TRITON_ERROR(TRITONSERVER_MetricArgsDelete(args)); return reinterpret_cast(triton_metric); } void Metric::ParseLabels( std::vector& labels_params, const std::string& labels) { triton::common::TritonJson::Value labels_json; THROW_IF_TRITON_ERROR(labels_json.Parse(labels)); std::vector members; labels_json.Members(&members); for (const auto& member : members) { std::string value; THROW_IF_TRITON_ERROR(labels_json.MemberAsString(member.c_str(), &value)); labels_params.emplace_back(TRITONSERVER_ParameterNew( member.c_str(), TRITONSERVER_PARAMETER_STRING, value.c_str())); } } void Metric::HandleMetricOperation( CustomMetricsMessage* metrics_message_ptr, const PYTHONSTUB_CommandType& command_type) { if (command_type == PYTHONSTUB_MetricRequestValue) { metrics_message_ptr->value = GetValue(); } else if (command_type == PYTHONSTUB_MetricRequestIncrement) { Increment(operation_value_); } else if (command_type == PYTHONSTUB_MetricRequestSet) { SetValue(operation_value_); } else if (command_type == PYTHONSTUB_MetricRequestObserve) { Observe(operation_value_); } else { throw PythonBackendException("Unknown metric operation"); } } void Metric::Increment(const double& value) { auto triton_metric = reinterpret_cast(metric_address_); THROW_IF_TRITON_ERROR(TRITONSERVER_MetricIncrement(triton_metric, value)); } void Metric::SetValue(const double& value) { auto triton_metric = reinterpret_cast(metric_address_); THROW_IF_TRITON_ERROR(TRITONSERVER_MetricSet(triton_metric, value)); } void Metric::Observe(const double& value) { auto triton_metric = reinterpret_cast(metric_address_); THROW_IF_TRITON_ERROR(TRITONSERVER_MetricObserve(triton_metric, value)); } double Metric::GetValue() { double value; auto triton_metric = reinterpret_cast(metric_address_); THROW_IF_TRITON_ERROR(TRITONSERVER_MetricValue(triton_metric, &value)); return value; } void Metric::ClearTritonMetric() { auto triton_metric = reinterpret_cast(metric_address_); if (triton_metric != nullptr) { LOG_IF_ERROR(TRITONSERVER_MetricDelete(triton_metric), "deleting metric"); } } #endif }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/metric.h000066400000000000000000000170111527052371200246770ustar00rootroot00000000000000// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "ipc_message.h" #include "pb_memory.h" #include "pb_string.h" #include "pb_utils.h" #ifdef TRITON_PB_STUB #include namespace py = pybind11; #else #include "triton/core/tritonserver.h" #endif namespace triton { namespace backend { namespace python { // The 'MetricShm' struct is utilized by the 'Metric' class for saving the // essential data to shared memory and for loading the data from shared memory // in order to reconstruct the 'Metric' object. struct MetricShm { // The shared memory handle of the labels in PbString format. bi::managed_external_buffer::handle_t labels_shm_handle; // The shared memory handle of the buckets in PbMemory format. bi::managed_external_buffer::handle_t buckets_shm_handle; // The value used for incrementing or setting the metric. double operation_value; // The address of the TRITONSERVER_Metric object. void* metric_address; // The address corresponds to the TRITONSERVER_MetricFamily object that this // metric belongs to. void* metric_family_address; }; class Metric { public: Metric( const std::string& labels, std::optional> buckets, void* metric_family_address); ~Metric(); /// Save Custom Metric object to shared memory. /// \param shm_pool Shared memory pool to save the custom metric object. void SaveToSharedMemory(std::unique_ptr& shm_pool); /// Create a Custom Metric object from shared memory. /// \param shm_pool Shared memory pool /// \param handle Shared memory handle of the custom metric. /// \return Returns the custom metrics in the specified request_handle /// location. static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); /// Get the address of the TRITONSERVER_Metric object. /// \return Returns the address of the TRITONSERVER_Metric object. void* MetricAddress(); /// Send the request to the parent process to delete the Metric object. void Clear(); #ifdef TRITON_PB_STUB /// Send a request to register a new 'TRITONSERVER_Metric' object to the /// parent process. void SendCreateMetricRequest(); /// Send the request to the parent process to increment the metric by the /// specified value. /// \param value The value to increment the metric by. void SendIncrementRequest(const double& value); /// Send the request to the parent process to set the metric to the specified /// value. /// \param value The value to set the metric to. void SendSetValueRequest(const double& value); /// Send the request to the parent process to observe the value to the metric. /// \param value The value to set the metric to. void SendObserveRequest(const double& value); /// Send the request to the parent process to get the value of the metric. /// \return Returns the value of the metric. double SendGetValueRequest(); /// Throws an exception if the metric has been cleared. This check is to avoid /// the user error where the corresponding metric family has been deleted /// before the metric is deleted. void CheckIfCleared(); #else // Initialize the TRITONSERVER_Metric object. /// \return Returns the address of the TRITONSERVER_Metric object. void* InitializeTritonMetric(); /// Parse the labels string into a vector of TRITONSERVER_Parameter. /// \param labels_params The vector of TRITONSERVER_Parameter to store the /// parsed labels. /// \param labels The labels string to parse. void ParseLabels( std::vector& labels_params, const std::string& labels); /// Handle the metric operation. /// \param metrics_message_ptr The pointer to the CustomMetricsMessage object. void HandleMetricOperation( CustomMetricsMessage* metrics_message_ptr, const PYTHONSTUB_CommandType& command_type); /// Use Triton C API to increment the value of the metric by the given value. /// \param value The value to increment the metric by. void Increment(const double& value); /// Use Triton C API to set the value of the metric to the given value. /// \param value The value to set the metric to. void SetValue(const double& value); /// Use Triton C API to sample the observation to the metric. /// \param value The value to sample observation to the metric. void Observe(const double& value); /// Use Triton C API to get the value of the metric. double GetValue(); /// Clear the TRITONSERVER_Metric object. void ClearTritonMetric(); #endif /// Disallow copying the custom metric object. DISALLOW_COPY_AND_ASSIGN(Metric); private: // The private constructor for creating a Metric object from shared memory. Metric( AllocatedSharedMemory& custom_metric_shm, std::unique_ptr& labels_shm, std::unique_ptr& buckets); // The labels of the metric, which is the identifier of the metric. std::string labels_; // Monotonically increasing values representing bucket boundaries for creating // histogram metric. std::optional> buckets_; // The value used for incrementing or setting the metric. double operation_value_; // The address of the TRITONSERVER_Metric object. void* metric_address_; // The address corresponds to the TRITONSERVER_MetricFamily object that this // metric belongs to. void* metric_family_address_; // Indicates whether the metric has been cleared. It is needed as the Clear()' // function can be called from two different locations: when the metric family // clears the 'metric_map_' and when the 'Metric' object goes out of // scope/being deleted. bool is_cleared_; // Shared Memory Data Structures AllocatedSharedMemory custom_metric_shm_; MetricShm* custom_metric_shm_ptr_; bi::managed_external_buffer::handle_t shm_handle_; std::unique_ptr labels_shm_; std::unique_ptr buckets_shm_; }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/metric_family.cc000066400000000000000000000210001527052371200263670ustar00rootroot00000000000000// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "metric_family.h" #ifdef TRITON_PB_STUB #include "pb_stub.h" #endif namespace triton { namespace backend { namespace python { MetricFamily::MetricFamily( const std::string& name, const std::string& description, const MetricKind& kind) : name_(name), description_(description), kind_(kind), metric_family_address_(nullptr) { #ifdef TRITON_PB_STUB SendCreateMetricFamilyRequest(); #endif } MetricFamily::~MetricFamily() { #ifdef TRITON_PB_STUB // Clear all the metrics first { std::lock_guard lock(metric_map_mu_); for (auto& m : metric_map_) { m.second->Clear(); } } // Send the request to delete the MetricFamily to the parent process std::unique_ptr& stub = Stub::GetOrCreateInstance(); SaveToSharedMemory(stub->ShmPool()); AllocatedSharedMemory custom_metrics_shm; try { stub->SendMessage( custom_metrics_shm, PYTHONSTUB_MetricFamilyRequestDelete, shm_handle_); } catch (const PythonBackendException& pb_exception) { std::cerr << "Error when deleting MetricFamily: " << pb_exception.what() << "\n"; } #endif }; void MetricFamily::SaveToSharedMemory(std::unique_ptr& shm_pool) { AllocatedSharedMemory custom_metric_family_shm = shm_pool->Construct(); custom_metric_family_shm_ptr_ = custom_metric_family_shm.data_.get(); std::unique_ptr name_shm = PbString::Create(shm_pool, name_); std::unique_ptr description_shm = PbString::Create(shm_pool, description_); custom_metric_family_shm_ptr_->kind = kind_; custom_metric_family_shm_ptr_->name_shm_handle = name_shm->ShmHandle(); custom_metric_family_shm_ptr_->description_shm_handle = description_shm->ShmHandle(); custom_metric_family_shm_ptr_->metric_family_address = metric_family_address_; // Save the references to shared memory. custom_metric_family_shm_ = std::move(custom_metric_family_shm); name_shm_ = std::move(name_shm); description_shm_ = std::move(description_shm); shm_handle_ = custom_metric_family_shm_.handle_; } std::unique_ptr MetricFamily::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle) { AllocatedSharedMemory custom_metric_family_shm = shm_pool->Load(handle); MetricFamilyShm* custom_metric_family_shm_ptr = custom_metric_family_shm.data_.get(); std::unique_ptr name_shm = PbString::LoadFromSharedMemory( shm_pool, custom_metric_family_shm_ptr->name_shm_handle); std::unique_ptr description_shm = PbString::LoadFromSharedMemory( shm_pool, custom_metric_family_shm_ptr->description_shm_handle); return std::unique_ptr( new MetricFamily(custom_metric_family_shm, name_shm, description_shm)); } MetricFamily::MetricFamily( AllocatedSharedMemory& custom_metric_family_shm, std::unique_ptr& name_shm, std::unique_ptr& description_shm) : custom_metric_family_shm_(std::move(custom_metric_family_shm)), name_shm_(std::move(name_shm)), description_shm_(std::move(description_shm)) { custom_metric_family_shm_ptr_ = custom_metric_family_shm_.data_.get(); name_ = name_shm_->String(); description_ = description_shm_->String(); kind_ = custom_metric_family_shm_ptr_->kind; metric_family_address_ = custom_metric_family_shm_ptr_->metric_family_address; } void* MetricFamily::MetricFamilyAddress() { return metric_family_address_; } #ifdef TRITON_PB_STUB std::shared_ptr MetricFamily::CreateMetricFamily( const std::string& name, const std::string& description, const MetricKind& kind) { std::shared_ptr metric_family = std::make_shared(name, description, kind); metric_family->SendCreateMetricFamilyRequest(); return metric_family; } void MetricFamily::SendCreateMetricFamilyRequest() { std::unique_ptr& stub = Stub::GetOrCreateInstance(); SaveToSharedMemory(stub->ShmPool()); CustomMetricsMessage* custom_metrics_msg = nullptr; AllocatedSharedMemory custom_metrics_shm; try { stub->SendMessage( custom_metrics_shm, PYTHONSTUB_MetricFamilyRequestNew, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Error when creating MetricFamily: " + std::string(pb_exception.what())); } custom_metrics_msg = custom_metrics_shm.data_.get(); metric_family_address_ = custom_metrics_msg->address; } std::shared_ptr MetricFamily::CreateMetric(const py::object& labels, const py::object& buckets) { if (!labels.is_none()) { if (!py::isinstance(labels)) { throw PythonBackendException( "Failed to create metric. Labels must be a dictionary."); } } py::module json = py::module_::import("json"); std::string labels_str = std::string(py::str(json.attr("dumps")(labels))); std::optional> buckets_vec; if (!buckets.is_none()) { if (!py::isinstance(buckets)) { throw PythonBackendException( "Failed to create metric. Buckets must be a list."); } if (kind_ == kCounter || kind_ == kGauge) { throw PythonBackendException( "Failed to create metric. Unexpected buckets found."); } buckets_vec = buckets.cast>(); } else { if (kind_ == kHistogram) { throw PythonBackendException( "Failed to create metric. Missing required buckets."); } buckets_vec = std::nullopt; } auto metric = std::make_shared(labels_str, buckets_vec, metric_family_address_); { std::lock_guard lock(metric_map_mu_); metric_map_.insert({metric->MetricAddress(), metric}); } return metric; } #else void* MetricFamily::InitializeTritonMetricFamily() { TRITONSERVER_MetricKind triton_kind = ToTritonServerMetricKind(kind_); TRITONSERVER_MetricFamily* triton_metric_family = nullptr; THROW_IF_TRITON_ERROR(TRITONSERVER_MetricFamilyNew( &triton_metric_family, triton_kind, name_.c_str(), description_.c_str())); return reinterpret_cast(triton_metric_family); } TRITONSERVER_MetricKind MetricFamily::ToTritonServerMetricKind(const MetricKind& kind) { switch (kind) { case kCounter: return TRITONSERVER_METRIC_KIND_COUNTER; case kGauge: return TRITONSERVER_METRIC_KIND_GAUGE; case kHistogram: return TRITONSERVER_METRIC_KIND_HISTOGRAM; default: throw PythonBackendException("Unknown metric kind"); } } void MetricFamily::ClearTritonMetricFamily() { auto metric_family = reinterpret_cast(metric_family_address_); if (metric_family != nullptr) { LOG_IF_ERROR( TRITONSERVER_MetricFamilyDelete(metric_family), "deleting metric family"); } } #endif }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/metric_family.h000066400000000000000000000144311527052371200262430ustar00rootroot00000000000000// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "ipc_message.h" #include "metric.h" #include "pb_string.h" #include "pb_utils.h" #ifdef TRITON_PB_STUB #include namespace py = pybind11; #else #include "triton/core/tritonserver.h" #endif namespace triton { namespace backend { namespace python { // The 'MetricFamilyShm' struct is utilized by the 'MetricFamily' class for // saving the essential data to shared memory and for loading the data from // shared memory in order to reconstruct the 'MetricFamily' object. struct MetricFamilyShm { // The shared memory handle of the name in PbString format. bi::managed_external_buffer::handle_t name_shm_handle; // The shared memory handle of the description in PbString format. bi::managed_external_buffer::handle_t description_shm_handle; // The metric kind of the 'MetricFamily'. MetricKind kind; // The address of the 'TRITONSERVER_MetricFamily' object. void* metric_family_address; }; class MetricFamily { public: MetricFamily( const std::string& name, const std::string& description, const MetricKind& kind); ~MetricFamily(); /// Save a custom metric family to shared memory. /// \param shm_pool Shared memory pool to save the custom metric family. void SaveToSharedMemory(std::unique_ptr& shm_pool); /// Create a Custom Metric Family object from shared memory. /// \param shm_pool Shared memory pool /// \param handle Shared memory handle of the custom metric family. /// \return Returns the custom metric family in the specified handle /// location. static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); /// Get the address of the TRITONSERVER_MetricFamily object. /// \return Returns the address of the TRITONSERVER_MetricFamily object. void* MetricFamilyAddress(); #ifdef TRITON_PB_STUB /// Create a metric family object and returned as a shared pointer. /// \param name The name of the metric family. /// \param description The description of the metric family. /// \param kind The metric kind of the metric family. /// \return Returns the shared pointer to the created metric family. static std::shared_ptr CreateMetricFamily( const std::string& name, const std::string& description, const MetricKind& kind); /// Send a request to register a new 'TRITONSERVER_MetricFamily' object to the /// parent process. void SendCreateMetricFamilyRequest(); /// Create a metric from the metric family and store it in the metric map. /// \param labels The labels of the metric. /// \param buckets Monotonically increasing values representing bucket /// boundaries for creating histogram metric. /// \return Returns the shared pointer to the created metric. std::shared_ptr CreateMetric( const py::object& labels, const py::object& buckets); #else /// Initialize the TRITONSERVER_MetricFamily object. /// \return Returns the address of the TRITONSERVER_MetricFamily object. void* InitializeTritonMetricFamily(); /// Helper function to convert the MetricKind enum to TRITONSERVER_MetricKind /// \param kind The MetricKind enum to be converted. /// \return Returns the TRITONSERVER_MetricKind enum. TRITONSERVER_MetricKind ToTritonServerMetricKind(const MetricKind& kind); /// Clear the TRITONSERVER_MetricFamily object. void ClearTritonMetricFamily(); #endif /// Disallow copying the metric family object. DISALLOW_COPY_AND_ASSIGN(MetricFamily); private: // The private constructor for creating a MetricFamily object from shared // memory. MetricFamily( AllocatedSharedMemory& custom_metric_family_shm, std::unique_ptr& name_shm, std::unique_ptr& description_shm); // The name of the metric family. std::string name_; // The description of the metric family. std::string description_; // The metric kind of the metric family. Currently only supports GAUGE, // COUNTER and HISTOGRAM. MetricKind kind_; // The address of the TRITONSERVER_MetricFamily object. void* metric_family_address_; // The mutex to protect the 'metric_map_'. std::mutex metric_map_mu_; // Need to keep track of the metrics associated with the metric family to make // sure the metrics are cleaned up before the metric family is deleted. std::unordered_map> metric_map_; // Shared Memory Data Structures AllocatedSharedMemory custom_metric_family_shm_; MetricFamilyShm* custom_metric_family_shm_ptr_; bi::managed_external_buffer::handle_t shm_handle_; std::unique_ptr name_shm_; std::unique_ptr description_shm_; }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/model_loader.cc000066400000000000000000000230541527052371200262040ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "model_loader.h" #ifdef TRITON_PB_STUB #include "pb_stub.h" #endif namespace triton { namespace backend { namespace python { void ModelLoader::SaveToSharedMemory(std::unique_ptr& shm_pool) { AllocatedSharedMemory model_loader_req_shm = shm_pool->Construct(); model_loader_req_shm_ptr_ = model_loader_req_shm.data_.get(); std::unique_ptr name_shm = PbString::Create(shm_pool, name_); std::unique_ptr version_shm = PbString::Create(shm_pool, version_); std::unique_ptr config_shm = PbString::Create(shm_pool, config_); std::unique_ptr files_shm = PbMap::Create(shm_pool, files_); model_loader_req_shm_ptr_->name_shm_handle = name_shm->ShmHandle(); model_loader_req_shm_ptr_->version_shm_handle = version_shm->ShmHandle(); model_loader_req_shm_ptr_->config_shm_handle = config_shm->ShmHandle(); model_loader_req_shm_ptr_->files_shm_handle = files_shm->ShmHandle(); model_loader_req_shm_ptr_->unload_dependents = unload_dependents_; // Save the references to shared memory. model_loader_req_shm_ = std::move(model_loader_req_shm); name_shm_ = std::move(name_shm); version_shm_ = std::move(version_shm); config_shm_ = std::move(config_shm); files_shm_ = std::move(files_shm); shm_handle_ = model_loader_req_shm_.handle_; } std::unique_ptr ModelLoader::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle) { AllocatedSharedMemory model_loader_req_shm = shm_pool->Load(handle); ModelLoaderRequestShm* model_loader_req_shm_ptr = model_loader_req_shm.data_.get(); std::unique_ptr name_shm = PbString::LoadFromSharedMemory( shm_pool, model_loader_req_shm_ptr->name_shm_handle); std::unique_ptr version_shm = PbString::LoadFromSharedMemory( shm_pool, model_loader_req_shm_ptr->version_shm_handle); std::unique_ptr config_shm = PbString::LoadFromSharedMemory( shm_pool, model_loader_req_shm_ptr->config_shm_handle); std::unique_ptr files_shm = PbMap::LoadFromSharedMemory( shm_pool, model_loader_req_shm_ptr->files_shm_handle); return std::unique_ptr(new ModelLoader( model_loader_req_shm, name_shm, version_shm, config_shm, files_shm)); } ModelLoader::ModelLoader( AllocatedSharedMemory& model_loader_req_shm, std::unique_ptr& name_shm, std::unique_ptr& version_shm, std::unique_ptr& config_shm, std::unique_ptr& files_shm) : model_loader_req_shm_(std::move(model_loader_req_shm)), name_shm_(std::move(name_shm)), version_shm_(std::move(version_shm)), config_shm_(std::move(config_shm)), files_shm_(std::move(files_shm)) { model_loader_req_shm_ptr_ = model_loader_req_shm_.data_.get(); name_ = name_shm_->String(); version_ = version_shm_->String(); config_ = config_shm_->String(); files_ = files_shm_->UnorderedMap(); unload_dependents_ = model_loader_req_shm_ptr_->unload_dependents; } #ifdef TRITON_PB_STUB void ModelLoader::SendLoadModelRequest() { std::unique_ptr& stub = Stub::GetOrCreateInstance(); SaveToSharedMemory(stub->ShmPool()); AllocatedSharedMemory model_loader_msg_shm; try { stub->SendMessage( model_loader_msg_shm, PYTHONSTUB_LoadModelRequest, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Failed to load model: " + std::string(pb_exception.what())); } } void ModelLoader::SendUnloadModelRequest() { std::unique_ptr& stub = Stub::GetOrCreateInstance(); SaveToSharedMemory(stub->ShmPool()); AllocatedSharedMemory model_loader_msg_shm; try { stub->SendMessage( model_loader_msg_shm, PYTHONSTUB_UnloadModelRequest, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Failed to unload model: " + std::string(pb_exception.what())); } } bool ModelLoader::SendModelReadinessRequest() { std::unique_ptr& stub = Stub::GetOrCreateInstance(); SaveToSharedMemory(stub->ShmPool()); ModelLoaderMessage* model_loader_msg = nullptr; AllocatedSharedMemory model_loader_msg_shm; try { stub->SendMessage( model_loader_msg_shm, PYTHONSTUB_ModelReadinessRequest, shm_handle_); } catch (const PythonBackendException& pb_exception) { throw PythonBackendException( "Failed to check model readiness: " + std::string(pb_exception.what())); } model_loader_msg = model_loader_msg_shm.data_.get(); return model_loader_msg->is_model_ready; } void LoadModel( const std::string& name, const std::string& config, const py::object& files) { std::unordered_map files_map; if (!files.is_none()) { if (!py::isinstance(files)) { throw PythonBackendException( "failed to load model '" + name + "', files should be a dictionary of file paths and file contents"); } py::dict files_dict = py::cast(files); for (const auto& item : files_dict) { std::string key = py::cast(item.first); py::bytes value = py::cast(item.second); std::string content(value); files_map[key] = content; } } ModelLoader model_loader(name, config, files_map); model_loader.SendLoadModelRequest(); } void UnloadModel(const std::string& name, const bool unload_dependents) { ModelLoader model_loader(name, unload_dependents); model_loader.SendUnloadModelRequest(); } bool IsModelReady(const std::string& name, const std::string& version) { ModelLoader model_loader(name, version); return model_loader.SendModelReadinessRequest(); } #else void ModelLoader::LoadModel(TRITONSERVER_Server* server) { std::string path = ""; std::string file_content = ""; std::vector const_params; if (!config_.empty()) { const_params.emplace_back(TRITONSERVER_ParameterNew( "config", TRITONSERVER_PARAMETER_STRING, config_.c_str())); } if (!files_.empty()) { for (auto& file : files_) { path = file.first; file_content = file.second; const_params.emplace_back(TRITONSERVER_ParameterBytesNew( path.c_str(), file_content.data(), file_content.size())); } } THROW_IF_TRITON_ERROR(TRITONSERVER_ServerLoadModelWithParameters( server, name_.c_str(), const_params.data(), const_params.size())); for (const auto param : const_params) { TRITONSERVER_ParameterDelete(const_cast(param)); } } void ModelLoader::UnloadModel(TRITONSERVER_Server* server) { if (unload_dependents_) { THROW_IF_TRITON_ERROR( TRITONSERVER_ServerUnloadModelAndDependents(server, name_.c_str())); } else { THROW_IF_TRITON_ERROR( TRITONSERVER_ServerUnloadModel(server, name_.c_str())); } } bool ModelLoader::IsModelReady(TRITONSERVER_Server* server) { bool is_ready = false; int64_t model_version = GetModelVersionFromString(version_); THROW_IF_TRITON_ERROR(TRITONSERVER_ServerModelIsReady( server, name_.c_str(), model_version, &is_ready)); return is_ready; } int64_t ModelLoader::GetModelVersionFromString(const std::string& version_string) { int64_t version = -1; if (!version_string.empty()) { try { version = std::stol(version_string); } catch (std::exception& e) { throw PythonBackendException( "failed to get model version from specified version string '" + version_string + "' (details: " + e.what() + "), version should be an integral value > 0"); } if (version < 0) { throw PythonBackendException( "failed to get model version from specified version string '" + version_string + "', version should be an integral value > 0"); } } return version; } #endif }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/model_loader.h000066400000000000000000000145001527052371200260420ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "ipc_message.h" #include "pb_map.h" #include "pb_string.h" #include "pb_utils.h" #ifdef TRITON_PB_STUB #include namespace py = pybind11; #else #include "triton/core/tritonserver.h" #endif namespace triton { namespace backend { namespace python { // The 'ModelLoaderRequestShm' struct is utilized by the 'ModelLoader' class for // saving the essential data to shared memory and for loading the data from // shared memory in order to reconstruct the 'ModelLoader' object. struct ModelLoaderRequestShm { // The shared memory handle of the model name in PbString format. bi::managed_external_buffer::handle_t name_shm_handle; // The shared memory handle of the model version in PbString format. bi::managed_external_buffer::handle_t version_shm_handle; // The flag to unload the dependent models. bool unload_dependents; // The shared memory handle of the config in PbString format. bi::managed_external_buffer::handle_t config_shm_handle; // The shared memory handle of the files in PbMap format. bi::managed_external_buffer::handle_t files_shm_handle; }; class ModelLoader { public: ModelLoader( const std::string& name, const std::string& config, const std::unordered_map& files) : name_(name), version_(""), config_(config), files_(files), unload_dependents_(false) { } ModelLoader(const std::string& name, const bool unload_dependents) : name_(name), version_(""), config_(""), files_({}), unload_dependents_(unload_dependents) { } ModelLoader(const std::string& name, const std::string& version) : name_(name), version_(version), config_(""), files_({}), unload_dependents_(false) { } /// Save ModelLoader object to shared memory. /// \param shm_pool Shared memory pool to save the ModelLoader object. void SaveToSharedMemory(std::unique_ptr& shm_pool); /// Create a ModelLoader object from shared memory. /// \param shm_pool Shared memory pool /// \param handle Shared memory handle of the ModelLoader. /// \return Returns the ModelLoaders in the specified request_handle /// location. static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); #ifdef TRITON_PB_STUB /// Send a request to load the model. void SendLoadModelRequest(); /// Send a request to unload the model. void SendUnloadModelRequest(); /// Send a request to check if the model is ready. bool SendModelReadinessRequest(); #else /// Use Triton C API to load the model. /// \param server The Triton server object. void LoadModel(TRITONSERVER_Server* server); /// Use Triton C API to unload the model. /// \param server The Triton server object. void UnloadModel(TRITONSERVER_Server* server); /// Use Triton C API to check if the model is ready. /// \param server The Triton server object. /// \return Returns true if the model is ready. bool IsModelReady(TRITONSERVER_Server* server); /// Get the model version from the version string. /// \param version_string The version string. /// \return Returns the model version in uint64_t. int64_t GetModelVersionFromString(const std::string& version_string); #endif /// Disallow copying the ModelLoader object. DISALLOW_COPY_AND_ASSIGN(ModelLoader); private: // The private constructor for creating a Metric object from shared memory. ModelLoader( AllocatedSharedMemory& model_loader_req_shm, std::unique_ptr& name_shm, std::unique_ptr& version_shm, std::unique_ptr& config_shm, std::unique_ptr& files_shm); // The name of the model. std::string name_; // The version of the model. std::string version_; // The configuration of the model. std::string config_; // The files of the model. std::unordered_map files_; // The flag to unload the dependent models. bool unload_dependents_; // // Shared Memory Data Structures AllocatedSharedMemory model_loader_req_shm_; ModelLoaderRequestShm* model_loader_req_shm_ptr_; bi::managed_external_buffer::handle_t shm_handle_; std::unique_ptr name_shm_; std::unique_ptr version_shm_; std::unique_ptr config_shm_; std::unique_ptr files_shm_; }; #ifdef TRITON_PB_STUB // The binding functions for the Python stub. void LoadModel( const std::string& name, const std::string& config, const py::object& files = py::none()); void UnloadModel(const std::string& name, const bool unload_dependents); bool IsModelReady(const std::string& name, const std::string& version); #endif }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_cancel.cc000066400000000000000000000063651527052371200254720ustar00rootroot00000000000000// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_cancel.h" #include "pb_stub.h" namespace triton { namespace backend { namespace python { void PbCancel::SaveToSharedMemory(std::unique_ptr& shm_pool) { cancel_shm_ = shm_pool->Construct(); new (&(cancel_shm_.data_->mu)) bi::interprocess_mutex; new (&(cancel_shm_.data_->cv)) bi::interprocess_condition; cancel_shm_.data_->waiting_on_stub = false; cancel_shm_.data_->response_factory_address = response_factory_address_; cancel_shm_.data_->request_address = request_address_; cancel_shm_.data_->is_cancelled = is_cancelled_; } bi::managed_external_buffer::handle_t PbCancel::ShmHandle() { return cancel_shm_.handle_; } IsCancelledMessage* PbCancel::ShmPayload() { return cancel_shm_.data_.get(); } bool PbCancel::IsCancelled() { // Release the GIL. Python objects are not accessed during the check. py::gil_scoped_release gil_release; std::unique_lock lk(mu_); // The cancelled flag can only move from false to true, not the other way, so // it is checked on each query until cancelled and then implicitly cached. if (is_cancelled_) { return is_cancelled_; } if (!updating_) { std::unique_ptr& stub = Stub::GetOrCreateInstance(); if (!stub->StubToParentServiceActive()) { LOG_ERROR << "Cannot communicate with parent service"; return false; } stub->EnqueueIsCancelled(this); updating_ = true; } cv_.wait(lk, [this] { return !updating_; }); return is_cancelled_; } void PbCancel::ReportIsCancelled(bool is_cancelled) { { std::lock_guard lk(mu_); is_cancelled_ = is_cancelled; updating_ = false; } cv_.notify_all(); } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_cancel.h000066400000000000000000000047111527052371200253250ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "pb_utils.h" namespace triton { namespace backend { namespace python { class PbCancel { public: PbCancel(intptr_t response_factory_address, intptr_t request_address) : updating_(false), response_factory_address_(response_factory_address), request_address_(request_address), is_cancelled_(false) { } DISALLOW_COPY_AND_ASSIGN(PbCancel); void SaveToSharedMemory(std::unique_ptr& shm_pool); bi::managed_external_buffer::handle_t ShmHandle(); IsCancelledMessage* ShmPayload(); bool IsCancelled(); void ReportIsCancelled(bool is_cancelled); private: AllocatedSharedMemory cancel_shm_; std::mutex mu_; std::condition_variable cv_; bool updating_; intptr_t response_factory_address_; intptr_t request_address_; bool is_cancelled_; }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_env.cc000066400000000000000000000245311527052371200250300ustar00rootroot00000000000000// Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_env.h" #ifndef _WIN32 #include #include #include #endif #include #include #include #include #include "pb_utils.h" namespace triton { namespace backend { namespace python { bool FileExists(std::string& path) { struct stat buffer; return stat(path.c_str(), &buffer) == 0; } void LastModifiedTime(const std::string& path, time_t* last_modified_time) { struct stat result; if (stat(path.c_str(), &result) == 0) { *last_modified_time = result.st_mtime; } else { throw PythonBackendException(std::string( "LastModifiedTime() failed as file \'" + path + std::string("\' does not exists."))); } } // FIXME: [DLIS-5969]: Develop platforom-agnostic functions // to support custom python environments. #ifndef _WIN32 void CopySingleArchiveEntry(archive* input_archive, archive* output_archive) { const void* buff; size_t size; #if ARCHIVE_VERSION_NUMBER >= 3000000 int64_t offset; #else off_t offset; #endif for (;;) { int return_status; return_status = archive_read_data_block(input_archive, &buff, &size, &offset); if (return_status == ARCHIVE_EOF) break; if (return_status != ARCHIVE_OK) throw PythonBackendException( "archive_read_data_block() failed with error code = " + std::to_string(return_status)); return_status = archive_write_data_block(output_archive, buff, size, offset); if (return_status != ARCHIVE_OK) { throw PythonBackendException( "archive_write_data_block() failed with error code = " + std::to_string(return_status) + ", error message is " + archive_error_string(output_archive)); } } } void ExtractTarFile(std::string& archive_path, std::string& dst_path) { char current_directory[PATH_MAX]; if (getcwd(current_directory, PATH_MAX) == nullptr) { throw PythonBackendException( (std::string("Failed to get the current working directory. Error: ") + std::strerror(errno))); } if (chdir(dst_path.c_str()) == -1) { throw PythonBackendException( (std::string("Failed to change the directory to ") + dst_path + " Error: " + std::strerror(errno)) .c_str()); } struct archive_entry* entry; int flags = ARCHIVE_EXTRACT_TIME; struct archive* input_archive = archive_read_new(); struct archive* output_archive = archive_write_disk_new(); archive_write_disk_set_options(output_archive, flags); archive_read_support_filter_gzip(input_archive); archive_read_support_format_tar(input_archive); if (archive_path.size() == 0) { throw PythonBackendException("The archive path is empty."); } THROW_IF_ERROR( "archive_read_open_filename() failed.", archive_read_open_filename( input_archive, archive_path.c_str(), 10240 /* block_size */)); while (true) { int read_status = archive_read_next_header(input_archive, &entry); if (read_status == ARCHIVE_EOF) break; if (read_status != ARCHIVE_OK) { throw PythonBackendException( std::string("archive_read_next_header() failed with error code = ") + std::to_string(read_status) + std::string(" error message is ") + archive_error_string(input_archive)); } read_status = archive_write_header(output_archive, entry); if (read_status != ARCHIVE_OK) { throw PythonBackendException(std::string( "archive_write_header() failed with error code = " + std::to_string(read_status) + std::string(" error message is ") + archive_error_string(output_archive))); } CopySingleArchiveEntry(input_archive, output_archive); read_status = archive_write_finish_entry(output_archive); if (read_status != ARCHIVE_OK) { throw PythonBackendException(std::string( "archive_write_finish_entry() failed with error code = " + std::to_string(read_status) + std::string(" error message is ") + archive_error_string(output_archive))); } } archive_read_close(input_archive); archive_read_free(input_archive); archive_write_close(output_archive); archive_write_free(output_archive); // Revert the directory change. if (chdir(current_directory) == -1) { throw PythonBackendException( (std::string("Failed to change the directory to ") + current_directory) .c_str()); } } void RecursiveDirectoryDelete(const char* dir) { FTS* ftsp = NULL; FTSENT* curr; char* files[] = {(char*)dir, NULL}; ftsp = fts_open(files, FTS_NOCHDIR | FTS_PHYSICAL | FTS_XDEV, NULL); if (!ftsp) { } while ((curr = fts_read(ftsp))) { switch (curr->fts_info) { case FTS_NS: case FTS_DNR: case FTS_ERR: throw PythonBackendException( std::string("fts_read error: ") + curr->fts_accpath + " error: " + strerror(curr->fts_errno)); break; case FTS_DC: case FTS_DOT: case FTS_NSOK: break; case FTS_D: // Do nothing. Directories are deleted in FTS_DP break; case FTS_DP: case FTS_F: case FTS_SL: case FTS_SLNONE: case FTS_DEFAULT: if (remove(curr->fts_accpath) < 0) { fts_close(ftsp); throw PythonBackendException( std::string("Failed to remove ") + curr->fts_path + " error: " + strerror(curr->fts_errno)); } break; } } fts_close(ftsp); } EnvironmentManager::EnvironmentManager() { char tmp_dir_template[PATH_MAX + 1]; strcpy(tmp_dir_template, "/tmp/python_env_XXXXXX"); char* env_path = mkdtemp(tmp_dir_template); if (env_path == nullptr) { throw PythonBackendException( "Failed to create temporary directory for Python environments."); } strcpy(base_path_, tmp_dir_template); } std::string EnvironmentManager::ExtractIfNotExtracted(std::string env_path) { // Lock the mutex. Only a single thread should modify the map. std::lock_guard lk(mutex_); char canonical_env_path[PATH_MAX + 1]; char* err = realpath(env_path.c_str(), canonical_env_path); if (err == nullptr) { throw PythonBackendException( std::string("Failed to get the canonical path for ") + env_path + "."); } time_t last_modified_time; LastModifiedTime(canonical_env_path, &last_modified_time); bool env_extracted = false; bool re_extraction = false; // If the path is not a conda-packed file, then bypass the extraction process struct stat info; if (stat(canonical_env_path, &info) != 0) { throw PythonBackendException( std::string("stat() of : ") + canonical_env_path + " returned error."); } else if (S_ISDIR(info.st_mode)) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Returning canonical path since EXECUTION_ENV_PATH does " "not contain compressed path. Path: ") + canonical_env_path) .c_str()); return canonical_env_path; } const auto env_itr = env_map_.find(canonical_env_path); if (env_itr != env_map_.end()) { // Check if the environment has been modified and would // need to be extracted again. if (env_itr->second.second == last_modified_time) { env_extracted = true; } else { // Environment file has been updated. Need to clear // the previously extracted environment and extract // the environment to the same destination directory. RecursiveDirectoryDelete(env_itr->second.first.c_str()); re_extraction = true; } } // Extract only if the env has not been extracted yet. if (!env_extracted) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Extracting Python execution env ") + canonical_env_path) .c_str()); std::string dst_env_path; if (re_extraction) { dst_env_path = env_map_[canonical_env_path].first; } else { dst_env_path = std::string(base_path_) + "/" + std::to_string(env_map_.size()); } std::string canonical_env_path_str(canonical_env_path); int status = mkdir(dst_env_path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); if (status == 0) { ExtractTarFile(canonical_env_path_str, dst_env_path); } else { throw PythonBackendException( std::string("Failed to create environment directory for '") + dst_env_path.c_str() + "'."); } if (re_extraction) { // Just update the last modified timestamp env_map_[canonical_env_path].second = last_modified_time; } else { // Add the path to the list of environments env_map_.insert({canonical_env_path, {dst_env_path, last_modified_time}}); } return dst_env_path; } else { return env_map_.find(canonical_env_path)->second.first; } } EnvironmentManager::~EnvironmentManager() { RecursiveDirectoryDelete(base_path_); } #endif }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_env.h000066400000000000000000000045051527052371200246710ustar00rootroot00000000000000// Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #ifdef WIN32 #include #undef PATH_MAX #define PATH_MAX MAX_PATH #endif namespace triton { namespace backend { namespace python { void ExtractTarFile(std::string& archive_path, std::string& dst_path); bool FileExists(std::string& path); // // A class that manages Python environments // #ifndef _WIN32 class EnvironmentManager { std::map> env_map_; char base_path_[PATH_MAX + 1]; std::mutex mutex_; public: EnvironmentManager(); // Extracts the tar.gz file in the 'env_path' if it has not been // already extracted. std::string ExtractIfNotExtracted(std::string env_path); ~EnvironmentManager(); }; #endif }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_error.cc000066400000000000000000000061261527052371200253710ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_error.h" namespace triton { namespace backend { namespace python { TRITONSERVER_Error_Code PbError::Code() { return code_; } const std::string& PbError::Message() { return message_; } bi::managed_external_buffer::handle_t PbError::ShmHandle() { return shm_handle_; } void PbError::SaveToSharedMemory(std::unique_ptr& shm_pool) { message_shm_ = PbString::Create(shm_pool, message_); error_shm_ = shm_pool->Construct(); error_shm_.data_->code = code_; error_shm_.data_->message_shm_handle = message_shm_->ShmHandle(); shm_handle_ = error_shm_.handle_; } std::shared_ptr PbError::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t shm_handle) { AllocatedSharedMemory error_shm = shm_pool->Load(shm_handle); std::unique_ptr message_shm = PbString::LoadFromSharedMemory( shm_pool, error_shm.data_->message_shm_handle); TRITONSERVER_Error_Code code = error_shm.data_->code; std::string message = message_shm->String(); return std::shared_ptr(new PbError( std::move(message_shm), std::move(error_shm), code, std::move(message))); } PbError::PbError( std::shared_ptr&& message_shm, AllocatedSharedMemory&& error_shm, TRITONSERVER_Error_Code code, std::string&& message) : message_shm_(std::move(message_shm)), error_shm_(std::move(error_shm)), code_(code), message_(std::move(message)) { } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_error.h000066400000000000000000000054261527052371200252350ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "pb_string.h" #include "pb_utils.h" namespace triton { namespace backend { namespace python { struct PbErrorShm { TRITONSERVER_Error_Code code; bi::managed_external_buffer::handle_t message_shm_handle; }; class PbError { public: PbError( const std::string& message, TRITONSERVER_Error_Code code = TRITONSERVER_ERROR_INTERNAL) : code_(code), message_(message) { } DISALLOW_COPY_AND_ASSIGN(PbError); TRITONSERVER_Error_Code Code(); const std::string& Message(); void SaveToSharedMemory(std::unique_ptr& shm_pool); bi::managed_external_buffer::handle_t ShmHandle(); static std::shared_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); private: PbError( std::shared_ptr&& message_shm, AllocatedSharedMemory&& error_shm, TRITONSERVER_Error_Code code, std::string&& message); std::shared_ptr message_shm_; AllocatedSharedMemory error_shm_; bi::managed_external_buffer::handle_t shm_handle_; TRITONSERVER_Error_Code code_; std::string message_; }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_exception.h000066400000000000000000000037411527052371200261000ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include namespace triton { namespace backend { namespace python { // // PythonBackendException // // Exception thrown if error occurs in PythonBackend. // struct PythonBackendException : std::exception { PythonBackendException(const std::string& message) : message_(message) {} const char* what() const throw() { return message_.c_str(); } std::string message_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_log.cc000066400000000000000000000105631527052371200250210ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_log.h" namespace triton { namespace backend { namespace python { PbLog::PbLog( const std::string& filename, uint32_t line, const std::string& message, LogLevel level) : filename_(filename), line_(line), message_(message), level_(level) { } const std::string& PbLog::Filename() { return filename_; } const std::string& PbLog::Message() { return message_; } const LogLevel& PbLog::Level() { return level_; } const uint32_t& PbLog::Line() { return line_; } PbLogShm::PbLogShm( AllocatedSharedMemory& log_container_shm, std::unique_ptr& filename, std::unique_ptr& message) : log_container_shm_(std::move(log_container_shm)), filename_pb_string_(std::move(filename)), message_pb_string_(std::move(message)) { log_container_shm_ptr_ = log_container_shm_.data_.get(); log_container_shm_ptr_->filename = filename_pb_string_->ShmHandle(); log_container_shm_ptr_->log_message = message_pb_string_->ShmHandle(); } std::unique_ptr PbLogShm::Create( std::unique_ptr& shm_pool, const std::string& filename, const uint32_t& line, const std::string& message, const LogLevel& level) { std::unique_ptr file_name = PbString::Create(shm_pool, filename); std::unique_ptr log_message = PbString::Create(shm_pool, message); AllocatedSharedMemory log_send_message = shm_pool->Construct(); LogSendMessage* send_message_payload = log_send_message.data_.get(); new (&(send_message_payload->mu)) bi::interprocess_mutex; new (&(send_message_payload->cv)) bi::interprocess_condition; send_message_payload->line = line; send_message_payload->level = level; return std::unique_ptr( new PbLogShm(log_send_message, file_name, log_message)); } std::unique_ptr PbLogShm::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle) { AllocatedSharedMemory log_container_shm = shm_pool->Load(handle); std::unique_ptr pb_string_filename = PbString::LoadFromSharedMemory( shm_pool, log_container_shm.data_->filename); const std::string& filename = pb_string_filename->String(); uint32_t line = log_container_shm.data_->line; std::unique_ptr pb_string_msg = PbString::LoadFromSharedMemory( shm_pool, log_container_shm.data_->log_message); const std::string& message = pb_string_msg->String(); LogLevel level = log_container_shm.data_->level; return std::unique_ptr(new PbLog(filename, line, message, level)); } bi::managed_external_buffer::handle_t PbLogShm::ShmHandle() { return log_container_shm_.handle_; } LogSendMessage* PbLogShm::LogMessage() { return log_container_shm_ptr_; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_log.h000066400000000000000000000064131527052371200246620ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "pb_string.h" #include "pb_utils.h" namespace triton { namespace backend { namespace python { class PbLog { public: /// Create a PbLog instance PbLog( const std::string& filename, uint32_t line, const std::string& message, LogLevel level); /// Get the filename where the log was recorded const std::string& Filename(); /// Get the log message const std::string& Message(); /// Get the log level of the message const LogLevel& Level(); /// Get the line number of the log message const uint32_t& Line(); private: std::string filename_; uint32_t line_; std::string message_; LogLevel level_; }; class PbLogShm { public: /// Save PbLog object to shared memory static std::unique_ptr Create( std::unique_ptr& shm_pool, const std::string& filename, const uint32_t& line, const std::string& message, const LogLevel& level); /// Load PbLog object to shared memory static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); /// Get the shared memory handle of the saved log message bi::managed_external_buffer::handle_t ShmHandle(); /// Get a pointer to the saved log message LogSendMessage* LogMessage(); private: AllocatedSharedMemory log_container_shm_; std::unique_ptr filename_pb_string_; std::unique_ptr message_pb_string_; LogSendMessage* log_container_shm_ptr_; PbLogShm( AllocatedSharedMemory& log_container_shm, std::unique_ptr& filename, std::unique_ptr& message); }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_map.cc000066400000000000000000000100521527052371200250060ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_map.h" namespace triton { namespace backend { namespace python { std::unique_ptr PbMap::Create( std::unique_ptr& shm_pool, std::unordered_map& map) { std::vector> strings; AllocatedSharedMemory dict_shm = shm_pool->Construct(); dict_shm.data_->length = map.size(); AllocatedSharedMemory pair_shms = shm_pool->Construct(map.size()); dict_shm.data_->values = pair_shms.handle_; size_t i = 0; for (auto& pair : map) { auto key = PbString::Create(shm_pool, pair.first); auto value = PbString::Create(shm_pool, pair.second); (pair_shms.data_.get())[i].key = key->ShmHandle(); (pair_shms.data_.get())[i].value = value->ShmHandle(); strings.emplace_back(std::move(key)); strings.emplace_back(std::move(value)); i++; } return std::unique_ptr(new PbMap(strings, dict_shm, pair_shms, map)); } const std::unordered_map& PbMap::UnorderedMap() { return map_; } bi::managed_external_buffer::handle_t PbMap::ShmHandle() { return dict_handle_; } std::unique_ptr PbMap::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle) { AllocatedSharedMemory dict_shm = shm_pool->Load(handle); AllocatedSharedMemory pair_shms = shm_pool->Load(dict_shm.data_->values); std::vector> pb_strings; std::unordered_map map; for (size_t i = 0; i < dict_shm.data_->length; i++) { std::unique_ptr key = PbString::LoadFromSharedMemory( shm_pool, (pair_shms.data_.get())[i].key); std::unique_ptr value = PbString::LoadFromSharedMemory( shm_pool, (pair_shms.data_.get())[i].value); map.insert({key->String(), value->String()}); pb_strings.emplace_back(std::move(key)); pb_strings.emplace_back(std::move(value)); } return std::unique_ptr( new PbMap(pb_strings, dict_shm, pair_shms, map)); } PbMap::PbMap( std::vector>& strings, AllocatedSharedMemory& dict_shm, AllocatedSharedMemory& pair_shms, std::unordered_map& map) : strings_(std::move(strings)), dict_shm_(std::move(dict_shm)), pair_shms_(std::move(pair_shms)), map_(std::move(map)) { dict_handle_ = dict_shm.handle_; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_map.h000066400000000000000000000056421527052371200246610ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "pb_string.h" #include "shm_manager.h" namespace triton { namespace backend { namespace python { struct PairShm { bi::managed_external_buffer::handle_t key; bi::managed_external_buffer::handle_t value; }; struct DictShm { uint32_t length; // `values` point to the location where there are `length` of Pair objects. bi::managed_external_buffer::handle_t values; }; class PbMap { public: static std::unique_ptr Create( std::unique_ptr& shm_pool, std::unordered_map& map); static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); const std::unordered_map& UnorderedMap(); bi::managed_external_buffer::handle_t ShmHandle(); private: PbMap( std::vector>& strings, AllocatedSharedMemory& dict_shm, AllocatedSharedMemory& pair_shms, std::unordered_map& map); std::vector> strings_; AllocatedSharedMemory dict_shm_; AllocatedSharedMemory pair_shms_; bi::managed_external_buffer::handle_t dict_handle_; std::unordered_map map_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_memory.cc000066400000000000000000000356601527052371200255550ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_memory.h" namespace triton { namespace backend { namespace python { std::unique_ptr PbMemory::Create( std::unique_ptr& shm_pool, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, uint64_t byte_size, char* data, bool copy_gpu) { size_t requested_byte_size = sizeof(MemoryShm); if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU requested_byte_size += sizeof(cudaIpcMemHandle_t); #endif } else { requested_byte_size += byte_size; } AllocatedSharedMemory memory_shm = shm_pool->Construct(requested_byte_size); PbMemory::FillShmData( shm_pool->GetCUDAMemoryPoolManager(), memory_type, memory_type_id, byte_size, data, memory_shm.data_.get(), memory_shm.handle_, copy_gpu); if (memory_type == TRITONSERVER_MEMORY_CPU) { data = memory_shm.data_.get() + sizeof(MemoryShm); } std::unique_ptr pb_memory( new PbMemory(memory_shm, data, false /* opened_cuda_ipc_handle */)); #ifdef TRITON_ENABLE_GPU if (memory_type == TRITONSERVER_MEMORY_GPU) { pb_memory->memory_shm_ptr_->gpu_pointer_offset = pb_memory->GetGPUPointerOffset(); } #endif return pb_memory; } #ifndef TRITON_PB_STUB std::unique_ptr PbMemory::Create( std::unique_ptr& shm_pool, std::unique_ptr&& backend_memory, bool copy_gpu) { std::unique_ptr pb_memory = PbMemory::Create( shm_pool, backend_memory->MemoryType(), backend_memory->MemoryTypeId(), backend_memory->ByteSize(), backend_memory->MemoryPtr(), copy_gpu); pb_memory->backend_memory_ = std::move(backend_memory); return pb_memory; } #endif std::unique_ptr PbMemory::Create( std::unique_ptr& shm_pool, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, uint64_t byte_size, char* data, char* data_shm, bi::managed_external_buffer::handle_t handle, bool copy_gpu) { PbMemory::FillShmData( shm_pool->GetCUDAMemoryPoolManager(), memory_type, memory_type_id, byte_size, data, data_shm, handle, copy_gpu); if (memory_type == TRITONSERVER_MEMORY_CPU) { data = data_shm + sizeof(MemoryShm); } std::unique_ptr pb_memory( new PbMemory(data_shm, data, handle, false /* opened_cuda_ipc_handle */)); #ifdef TRITON_ENABLE_GPU if (memory_type == TRITONSERVER_MEMORY_GPU) { pb_memory->memory_shm_ptr_->gpu_pointer_offset = pb_memory->GetGPUPointerOffset(); } #endif return pb_memory; } void PbMemory::CopyBuffer( std::unique_ptr& dst, std::unique_ptr& src) { if (src->ByteSize() != dst->ByteSize()) { throw PythonBackendException( "Failed to copy memory buffers. Source and destination byte size do " "not match: " + std::to_string(dst->ByteSize()) + " != " + std::to_string(src->ByteSize())); } if (src->MemoryType() == TRITONSERVER_MEMORY_CPU && dst->MemoryType() == TRITONSERVER_MEMORY_CPU) { std::memcpy(dst->DataPtr(), src->DataPtr(), dst->ByteSize()); return; } #ifdef TRITON_ENABLE_GPU cudaMemcpyKind kind = cudaMemcpyHostToDevice; if (src->MemoryType() == TRITONSERVER_MEMORY_CPU && dst->MemoryType() == TRITONSERVER_MEMORY_GPU) { kind = cudaMemcpyHostToDevice; } else if ( src->MemoryType() == TRITONSERVER_MEMORY_GPU && dst->MemoryType() == TRITONSERVER_MEMORY_CPU) { kind = cudaMemcpyDeviceToHost; } else if ( src->MemoryType() == TRITONSERVER_MEMORY_GPU && dst->MemoryType() == TRITONSERVER_MEMORY_GPU) { kind = cudaMemcpyDeviceToDevice; } cudaError_t err; if ((kind == cudaMemcpyDeviceToDevice) && (src->MemoryTypeId() != dst->MemoryTypeId())) { err = cudaMemcpyPeer( dst->DataPtr(), dst->MemoryTypeId(), src->DataPtr(), src->MemoryTypeId(), src->ByteSize()); } else { err = cudaMemcpy(dst->DataPtr(), src->DataPtr(), src->ByteSize(), kind); } if (err != cudaSuccess) { throw PythonBackendException( std::string( "failed to copy data: " + std::string(cudaGetErrorString(err))) .c_str()); } if (kind == cudaMemcpyDeviceToDevice) { // Synchronize the default stream for d2d copies. // https://docs.nvidia.com/cuda/cuda-runtime-api/api-sync-behavior.html#api-sync-behavior__memcpy-sync err = cudaStreamSynchronize(0); if (err != cudaSuccess) { throw PythonBackendException( std::string( "failed to synchronize the default CUDA stream. error: " + std::string(cudaGetErrorString(err))) .c_str()); } } #endif } void PbMemory::FillShmData( std::unique_ptr& cuda_pool, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, uint64_t byte_size, char* data, char* data_shm, bi::managed_external_buffer::handle_t handle, bool copy_gpu) { char* memory_data_shm = data_shm + sizeof(MemoryShm); MemoryShm* memory_shm_ptr = reinterpret_cast(data_shm); memory_shm_ptr->memory_release_id = 0; bool use_cuda_shared_pool = false; if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU if (data != nullptr) { if (copy_gpu) { ScopedSetDevice scoped_set_device(memory_type_id); THROW_IF_CUDA_ERROR(cudaIpcGetMemHandle( reinterpret_cast(memory_data_shm), data)); } if (cuda_pool->UseCudaSharedPool(memory_type_id) && IsUsingCUDAPool(cuda_pool, memory_type_id, data)) { use_cuda_shared_pool = true; memory_shm_ptr->cuda_pool_offset = data - reinterpret_cast(cuda_pool->CUDAPoolAddress(memory_type_id)); } } #endif // TRITON_ENABLE_GPU } else { if (data != nullptr) { std::copy(data, data + byte_size, memory_data_shm); } } memory_shm_ptr->byte_size = byte_size; memory_shm_ptr->memory_type_id = memory_type_id; memory_shm_ptr->memory_type = memory_type; memory_shm_ptr->use_cuda_shared_pool = use_cuda_shared_pool; } std::unique_ptr PbMemory::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle, char* data_shm, bool open_cuda_handle) { MemoryShm* memory_shm_ptr = reinterpret_cast(data_shm); char* memory_data_shm = data_shm + sizeof(MemoryShm); char* data_ptr = nullptr; bool opened_cuda_ipc_handle = false; if (memory_shm_ptr->memory_type == TRITONSERVER_MEMORY_GPU && open_cuda_handle) { #ifdef TRITON_ENABLE_GPU if (memory_shm_ptr->use_cuda_shared_pool) { // When CUDA shared memory pool is used, the stub will retrieve the // data pointer using the offset. data_ptr = (reinterpret_cast( shm_pool->GetCUDAMemoryPoolManager()->CUDAPoolAddress( memory_shm_ptr->memory_type_id)) + memory_shm_ptr->cuda_pool_offset); } else { cudaIpcMemHandle_t* cuda_handle = reinterpret_cast(memory_data_shm); // The pointer opened by the cudaIpcOpenMemHandle will refer to the base // address. We need to manually correct the offset. void* data_ptr_base; CUDAHandler& cuda_handler = CUDAHandler::getInstance(); cuda_handler.OpenCudaHandle( memory_shm_ptr->memory_type_id, cuda_handle, &data_ptr_base); data_ptr = (reinterpret_cast(data_ptr_base) + memory_shm_ptr->gpu_pointer_offset); opened_cuda_ipc_handle = true; } #endif // TRITON_ENABLE_GPU } else { data_ptr = memory_data_shm; } return std::unique_ptr(new PbMemory( data_shm, data_ptr, handle, opened_cuda_ipc_handle /* opened_cuda_ipc_handle */)); } std::unique_ptr PbMemory::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle, bool open_cuda_handle) { AllocatedSharedMemory memory_shm = shm_pool->Load(handle); MemoryShm* memory_shm_ptr = reinterpret_cast(memory_shm.data_.get()); char* memory_data_shm = memory_shm.data_.get() + sizeof(MemoryShm); char* data_ptr = nullptr; bool opened_cuda_ipc_handle = false; if (memory_shm_ptr->memory_type == TRITONSERVER_MEMORY_GPU) { if (memory_shm_ptr->byte_size > 0 && open_cuda_handle) { #ifdef TRITON_ENABLE_GPU if (memory_shm_ptr->use_cuda_shared_pool) { // When CUDA shared memory pool is used, the stub will retrieve the // data pointer using the offset. data_ptr = (reinterpret_cast( shm_pool->GetCUDAMemoryPoolManager()->CUDAPoolAddress( memory_shm_ptr->memory_type_id)) + memory_shm_ptr->cuda_pool_offset); } else { cudaIpcMemHandle_t* cuda_handle = reinterpret_cast(memory_data_shm); // The pointer opened by the cudaIpcOpenMemHandle will refer to the base // address. We need to manually correct the offset. void* data_ptr_base; CUDAHandler& cuda_handler = CUDAHandler::getInstance(); cuda_handler.OpenCudaHandle( memory_shm_ptr->memory_type_id, cuda_handle, &data_ptr_base); data_ptr = (reinterpret_cast(data_ptr_base) + memory_shm_ptr->gpu_pointer_offset); opened_cuda_ipc_handle = true; } #endif } } else { data_ptr = memory_data_shm; } return std::unique_ptr(new PbMemory( memory_shm, data_ptr, opened_cuda_ipc_handle /* opened_cuda_ipc_handle */)); } PbMemory::PbMemory( AllocatedSharedMemory& memory_shm, char* data, bool opened_cuda_ipc_handle) : memory_shm_(std::move(memory_shm)), data_ptr_(data), opened_cuda_ipc_handle_(opened_cuda_ipc_handle) { memory_shm_ptr_ = reinterpret_cast(memory_shm_.data_.get()); memory_shm_handle_ = memory_shm_.handle_; } PbMemory::PbMemory( char* memory_shm, char* data, bi::managed_external_buffer::handle_t handle, bool opened_cuda_ipc_handle) { memory_shm_ptr_ = reinterpret_cast(memory_shm); data_ptr_ = data; opened_cuda_ipc_handle_ = opened_cuda_ipc_handle; memory_shm_handle_ = handle; } bi::managed_external_buffer::handle_t PbMemory::ShmHandle() { return memory_shm_handle_; } #ifdef TRITON_ENABLE_GPU void* PbMemory::GetGPUStartAddress() { if (memory_shm_ptr_->memory_type == TRITONSERVER_MEMORY_GPU) { CUDAHandler& cuda_api = CUDAHandler::getInstance(); CUdeviceptr start_address = 0; // Skip this step for empty tensor as the CUDA API 'cuPointerGetAttribute' // we use in this function does not accept nullptr. if (data_ptr_) { cuda_api.PointerGetAttribute( &start_address, CU_POINTER_ATTRIBUTE_RANGE_START_ADDR, reinterpret_cast(data_ptr_)); } return reinterpret_cast(start_address); } throw PythonBackendException( "Calling GetGPUStartAddress function on CPU memory."); } uint64_t PbMemory::GetGPUPointerOffset() { uint64_t offset; if (memory_shm_ptr_->memory_type == TRITONSERVER_MEMORY_GPU) { offset = data_ptr_ - reinterpret_cast(GetGPUStartAddress()); } else { throw PythonBackendException( "Calling GetGPUPointerOffset function on CPU tensor."); } return offset; } #endif TRITONSERVER_MemoryType PbMemory::MemoryType() const { return memory_shm_ptr_->memory_type; } void PbMemory::SetMemoryReleaseId(uint64_t memory_release_id) { memory_shm_ptr_->memory_release_id = memory_release_id; } int64_t PbMemory::MemoryTypeId() const { return memory_shm_ptr_->memory_type_id; } uint64_t PbMemory::ByteSize() const { return memory_shm_ptr_->byte_size; } char* PbMemory::ShmData() const { return reinterpret_cast(memory_shm_ptr_) + sizeof(MemoryShm); } char* PbMemory::DataPtr() const { return data_ptr_; } uint64_t PbMemory::ShmStructSize(TRITONSERVER_MemoryType memory_type, uint64_t byte_size) { uint64_t total_memory_size = sizeof(MemoryShm); if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU total_memory_size += sizeof(cudaIpcMemHandle_t); #endif } else { total_memory_size += byte_size; } return total_memory_size; } #ifdef TRITON_ENABLE_GPU void PbMemory::SetCudaIpcHandle(cudaIpcMemHandle_t* cuda_ipc_handle) { *(reinterpret_cast(ShmData())) = *(cuda_ipc_handle); } void PbMemory::UpdateCUDAOffset(std::unique_ptr& cuda_pool) { if (cuda_pool->UseCudaSharedPool(MemoryTypeId()) && IsUsingCUDAPool(cuda_pool, MemoryTypeId(), DataPtr())) { memory_shm_ptr_->cuda_pool_offset = DataPtr() - reinterpret_cast(cuda_pool->CUDAPoolAddress(MemoryTypeId())); memory_shm_ptr_->use_cuda_shared_pool = true; } } #endif PbMemory::~PbMemory() { if (opened_cuda_ipc_handle_) { #ifdef TRITON_ENABLE_GPU CUDAHandler& cuda_handler = CUDAHandler::getInstance(); cuda_handler.CloseCudaHandle( memory_shm_ptr_->memory_type_id, GetGPUStartAddress()); #endif } if (release_callback_) { release_callback_(); } } void PbMemory::SetMemoryReleaseCallback(std::function release_callback) { if (!release_callback_) { release_callback_ = release_callback; } else { throw PythonBackendException("Release callback is already set."); } } uint64_t PbMemory::MemoryReleaseId() { return memory_shm_ptr_->memory_release_id; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_memory.h000066400000000000000000000150441527052371200254110ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "pb_utils.h" #include "shm_manager.h" #include "triton/backend/backend_common.h" #include "triton/backend/backend_memory.h" #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace triton { namespace backend { namespace python { // // Represents a memory object in shared memory. // struct MemoryShm { // If the memory type is a GPU pointer, the offset of the GPU pointer from the // base address. For CPU memory type this field contains garbage data. This // field will only be used when the memory is not allocated from the CUDA // shared memory pool. uint64_t gpu_pointer_offset; bool use_cuda_shared_pool; // The offset of the memory from the base address of the CUDA shared memory // pool. uint64_t cuda_pool_offset; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; uint64_t byte_size; uint64_t memory_release_id; }; class PbMemory { public: static std::unique_ptr Create( std::unique_ptr& shm_pool, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, uint64_t byte_size, char* data, bool copy_gpu = true); static std::unique_ptr Create( std::unique_ptr& shm_pool, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, uint64_t byte_size, char* data, char* data_shm, bi::managed_external_buffer::handle_t handle, bool copy_gpu = true); #ifndef TRITON_PB_STUB static std::unique_ptr Create( std::unique_ptr& shm_pool, std::unique_ptr&& backend_memory, bool copy_gpu = true); #endif #ifdef TRITON_ENABLE_GPU void SetCudaIpcHandle(cudaIpcMemHandle_t* cuda_ipc_handle); void UpdateCUDAOffset(std::unique_ptr& cuda_pool); #endif // Copy the destination buffer to the source buffer. static void CopyBuffer( std::unique_ptr& dst, std::unique_ptr& src); static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t memory_handle, bool open_cuda_handle); static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle, char* data_shm, bool open_cuda_handle); static uint64_t ShmStructSize( TRITONSERVER_MemoryType memory_type, uint64_t byte_size); bi::managed_external_buffer::handle_t ShmHandle(); /// Get the total byte size of the tensor. uint64_t ByteSize() const; /// Get the triton memory type. /// \return the memory type of the tensor. TRITONSERVER_MemoryType MemoryType() const; /// Get the pointer. /// \return The location to the memory where the data is stored. char* DataPtr() const; /// Get the memory type id. /// \return The memory type id of the tensor. int64_t MemoryTypeId() const; /// Get the shm data /// \return The memory type id of the tensor. char* ShmData() const; /// Set the memory release id void SetMemoryReleaseId(uint64_t memory_release_id); /// Memory Release ID uint64_t MemoryReleaseId(); void SetMemoryReleaseCallback(std::function release_callback); bool UseCUDASharedPool() const { return memory_shm_ptr_->use_cuda_shared_pool; } ~PbMemory(); #ifndef TRITON_PB_STUB void SetBackendMemory(std::unique_ptr&& backend_memory) { backend_memory_ = std::move(backend_memory); }; std::unique_ptr GetBackendMemory() { return std::move(backend_memory_); }; #endif private: AllocatedSharedMemory memory_shm_; MemoryShm* memory_shm_ptr_; #ifndef TRITON_PB_STUB std::unique_ptr backend_memory_; #endif std::function release_callback_; // Refers to the pointer that can hold the data. For CPU pointers this will be // the same as memory_data_shm_ptr_. char* data_ptr_; bi::managed_external_buffer::handle_t memory_shm_handle_; bool opened_cuda_ipc_handle_; #ifdef TRITON_ENABLE_GPU /// Calculate the pointer offset from the base address. /// \return The offset of a device pointer. /// \throws PythonBackendException if the tensor is stored in CPU. uint64_t GetGPUPointerOffset(); /// Get the GPU start address. /// \return The start address of a device pointer. /// \throws PythonBackendException if the tensor is stored in CPU. void* GetGPUStartAddress(); #endif static void FillShmData( std::unique_ptr& cuda_pool, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, uint64_t byte_size, char* data, char* data_shm, bi::managed_external_buffer::handle_t handle, bool copy_gpu = true); PbMemory( AllocatedSharedMemory& memory_shm, char* data, bool opened_cuda_ipc_handle); PbMemory( char* memory_shm, char* data, bi::managed_external_buffer::handle_t handle, bool opened_cuda_ipc_handle); }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_metric_reporter.cc000066400000000000000000000104321527052371200274400ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_metric_reporter.h" #include "triton/backend/backend_common.h" namespace triton { namespace backend { namespace python { PbMetricReporter::PbMetricReporter( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count, std::shared_ptr> responses) : instance_(instance), requests_(requests), request_count_(request_count), responses_(responses), total_batch_size_(0), exec_start_ns_(0), compute_start_ns_(0), compute_end_ns_(0), exec_end_ns_(0), success_status_(true) { } PbMetricReporter::~PbMetricReporter() { for (uint32_t r = 0; r < request_count_; ++r) { TRITONBACKEND_Request* request = requests_[r]; // Report statistics for the request. Note that there could // still be responses that have not yet been sent but those // cannot be captured in the statistics as they reflect only the // request object. We use the execution start/end time for // compute also so that the entire execution time is associated // with the inference computation. if (responses_) { LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_, request, ((*responses_)[r] != nullptr) /* success */, exec_start_ns_, compute_start_ns_, compute_end_ns_, exec_end_ns_), "failed reporting request statistics"); } else { LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_, request, success_status_, exec_start_ns_, compute_start_ns_, compute_end_ns_, exec_end_ns_), "failed reporting request statistics"); } } // Report the entire batch statistics. This backend does not support // batching so the total batch size is always 1. if (total_batch_size_ != 0) { LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( instance_, total_batch_size_, exec_start_ns_, compute_start_ns_, compute_end_ns_, exec_end_ns_), "failed reporting batch request statistics"); } } void PbMetricReporter::SetBatchStatistics(size_t total_batch_size) { total_batch_size_ = total_batch_size; } void PbMetricReporter::SetExecStartNs(const uint64_t exec_start_ns) { exec_start_ns_ = exec_start_ns; } void PbMetricReporter::SetComputeStartNs(const uint64_t compute_start_ns) { compute_start_ns_ = compute_start_ns; } void PbMetricReporter::SetComputeEndNs(const uint64_t compute_end_ns) { compute_end_ns_ = compute_end_ns; } void PbMetricReporter::SetExecEndNs(const uint64_t exec_end_ns) { exec_end_ns_ = exec_end_ns; } void PbMetricReporter::SetSuccessStatus(const bool success_status) { success_status_ = success_status; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_metric_reporter.h000066400000000000000000000052221527052371200273030ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include "triton/core/tritonbackend.h" namespace triton { namespace backend { namespace python { class PbMetricReporter { TRITONBACKEND_ModelInstance* instance_; TRITONBACKEND_Request** requests_; uint32_t request_count_; std::shared_ptr> responses_; size_t total_batch_size_; uint64_t exec_start_ns_; uint64_t compute_start_ns_; uint64_t compute_end_ns_; uint64_t exec_end_ns_; bool success_status_; public: PbMetricReporter( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count, std::shared_ptr> responses); ~PbMetricReporter(); void SetBatchStatistics(size_t total_batch_size); void SetExecStartNs(const uint64_t exec_start_ns); void SetComputeStartNs(const uint64_t compute_start_ns); void SetComputeEndNs(const uint64_t compute_end_ns); void SetExecEndNs(const uint64_t exec_end_ns); void SetSuccessStatus(const bool success_status); }; }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_preferred_memory.h000066400000000000000000000044041527052371200274450ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once namespace triton { namespace backend { namespace python { class PreferredMemory { public: enum MemoryType { kGPU, kCPU, kDefault }; PreferredMemory() : preferred_memory_type_(MemoryType::kDefault), preferred_device_id_(0) { } PreferredMemory( const MemoryType& preferred_memory_type, const int64_t& preferred_device_id) : preferred_memory_type_(preferred_memory_type), preferred_device_id_(preferred_device_id) { } MemoryType PreferredMemoryType() { return preferred_memory_type_; } int64_t PreferredDeviceId() { return preferred_device_id_; } private: MemoryType preferred_memory_type_; int64_t preferred_device_id_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_response_iterator.cc000066400000000000000000000106471527052371200300120ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_response_iterator.h" #include #include #include "pb_stub.h" namespace py = pybind11; namespace triton { namespace backend { namespace python { ResponseIterator::ResponseIterator( const std::shared_ptr& response) : id_(response->Id()), is_finished_(false), is_cleared_(false), idx_(0) { response_buffer_.push(response); } ResponseIterator::~ResponseIterator() { // Fetch all the remaining responses if not finished yet. if (!is_finished_) { bool done = false; while (!done) { try { Next(); } catch (const py::stop_iteration& exception) { done = true; } } } if (!is_cleared_) { Clear(); } responses_.clear(); } std::shared_ptr ResponseIterator::Next() { if (is_finished_) { if (!is_cleared_) { Clear(); } if (idx_ < responses_.size()) { return responses_[idx_++]; } else { throw py::stop_iteration("Iteration is done for the responses."); } } else { std::shared_ptr response; { { std::unique_lock lock{mu_}; while (response_buffer_.empty()) { py::gil_scoped_release release; cv_.wait(lock); } response = response_buffer_.front(); response_buffer_.pop(); is_finished_ = response->IsLastResponse(); responses_.push_back(response); } } if (is_finished_) { idx_ = responses_.size(); Clear(); } return response; } } void ResponseIterator::Iter() { if (is_finished_) { // If the previous iteration is finished, reset the index so that it will // iterator from the beginning of the responses. Otherwise just resume the // iteration from the previous index. if (idx_ >= responses_.size()) { idx_ = 0; } } } void ResponseIterator::EnqueueResponse(std::shared_ptr infer_response) { { std::lock_guard lock{mu_}; response_buffer_.push(infer_response); } cv_.notify_one(); } void* ResponseIterator::Id() { return id_; } void ResponseIterator::Clear() { std::unique_ptr& stub = Stub::GetOrCreateInstance(); stub->EnqueueCleanupId(id_, PYTHONSTUB_BLSDecoupledInferPayloadCleanup); { std::lock_guard lock{mu_}; response_buffer_.push(DUMMY_MESSAGE); } cv_.notify_all(); std::queue> empty; std::swap(response_buffer_, empty); is_cleared_ = true; } std::vector> ResponseIterator::GetExistingResponses() { std::vector> responses; std::unique_lock lock{mu_}; while (!response_buffer_.empty()) { responses.push_back(response_buffer_.front()); response_buffer_.pop(); } is_finished_ = true; is_cleared_ = true; return responses; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_response_iterator.h000066400000000000000000000044411527052371200276470ustar00rootroot00000000000000// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "infer_response.h" namespace triton { namespace backend { namespace python { class ResponseIterator { public: ResponseIterator(const std::shared_ptr& response); ~ResponseIterator(); std::shared_ptr Next(); void Iter(); void EnqueueResponse(std::shared_ptr infer_response); void* Id(); void Clear(); std::vector> GetExistingResponses(); private: std::vector> responses_; std::queue> response_buffer_; std::mutex mu_; std::condition_variable cv_; void* id_; bool is_finished_; bool is_cleared_; size_t idx_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_string.cc000066400000000000000000000106411527052371200255430ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_string.h" namespace triton { namespace backend { namespace python { std::unique_ptr PbString::Create( std::unique_ptr& shm_pool, const std::string& string) { AllocatedSharedMemory string_container_shm = shm_pool->Construct(); string_container_shm.data_->length = string.size(); AllocatedSharedMemory string_shm = shm_pool->Construct(string.size()); std::memcpy(string_shm.data_.get(), string.data(), string.size()); string_container_shm.data_->data = string_shm.handle_; return std::unique_ptr( new PbString(string_container_shm, string_shm)); } std::unique_ptr PbString::Create( const std::string& string, char* data_shm, bi::managed_external_buffer::handle_t handle) { StringShm* string_container_shm = reinterpret_cast(data_shm); string_container_shm->length = string.size(); char* string_shm = data_shm + sizeof(StringShm); std::memcpy(string_shm, string.data(), string.size()); return std::unique_ptr( new PbString(string_container_shm, string_shm, handle)); } std::unique_ptr PbString::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle) { AllocatedSharedMemory string_container_shm = shm_pool->Load(handle); AllocatedSharedMemory string_shm = shm_pool->Load(string_container_shm.data_->data); return std::unique_ptr( new PbString(string_container_shm, string_shm)); } std::unique_ptr PbString::LoadFromSharedMemory( bi::managed_external_buffer::handle_t handle, char* data_shm) { StringShm* string_container_shm = reinterpret_cast(data_shm); char* string_shm = data_shm + sizeof(StringShm); return std::unique_ptr( new PbString(string_container_shm, string_shm, handle)); } PbString::PbString( AllocatedSharedMemory& string_container_shm, AllocatedSharedMemory& string_shm) : string_container_shm_(std::move(string_container_shm)), string_shm_(std::move(string_shm)) { string_shm_ptr_ = string_shm_.data_.get(); string_container_shm_ptr_ = string_container_shm_.data_.get(); string_handle_ = string_container_shm_.handle_; } PbString::PbString( StringShm* string_container_shm, char* string_shm, bi::managed_external_buffer::handle_t handle) { string_shm_ptr_ = string_shm; string_container_shm_ptr_ = string_container_shm; string_handle_ = handle; } bi::managed_external_buffer::handle_t PbString::ShmHandle() { return string_handle_; } std::size_t PbString::ShmStructSize(const std::string& string) { return string.size() + sizeof(StringShm); } std::size_t PbString::Size() { return string_container_shm_ptr_->length + sizeof(StringShm); } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_string.h000066400000000000000000000061751527052371200254140ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "shm_manager.h" namespace triton { namespace backend { namespace python { struct StringShm { bi::managed_external_buffer::handle_t data; size_t length; }; class PbString { public: static std::unique_ptr Create( std::unique_ptr& shm_pool, const std::string& string); static std::unique_ptr Create( const std::string& string, char* data_shm, bi::managed_external_buffer::handle_t handle); static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t handle); static std::unique_ptr LoadFromSharedMemory( bi::managed_external_buffer::handle_t handle, char* data_shm); static std::size_t ShmStructSize(const std::string& string); char* MutableString() { return string_shm_ptr_; } std::string String() { return std::string( string_shm_ptr_, string_shm_ptr_ + string_container_shm_ptr_->length); } bi::managed_external_buffer::handle_t ShmHandle(); std::size_t Size(); private: AllocatedSharedMemory string_container_shm_; StringShm* string_container_shm_ptr_; AllocatedSharedMemory string_shm_; char* string_shm_ptr_; bi::managed_external_buffer::handle_t string_handle_; PbString( AllocatedSharedMemory& string_container_shm, AllocatedSharedMemory& string_shm); PbString( StringShm* string_container_shm, char* string_shm, bi::managed_external_buffer::handle_t handle); }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_stub.cc000066400000000000000000002315001527052371200252110ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_stub.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "correlation_id.h" #include "model_loader.h" #include "pb_error.h" #include "pb_map.h" #include "pb_preferred_memory.h" #include "pb_response_iterator.h" #include "pb_string.h" #include "pb_utils.h" #include "response_sender.h" #include "scoped_defer.h" #include "shm_manager.h" #include "triton/common/nvtx.h" #ifdef _WIN32 #include // SIGINT & SIGTERM #include #else #include #endif #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU namespace py = pybind11; using namespace pybind11::literals; namespace bi = boost::interprocess; #ifndef TRITON_ENABLE_GPU using cudaStream_t = void*; #endif namespace triton { namespace backend { namespace python { std::atomic non_graceful_exit = {false}; void SignalHandler(int signum) { // Skip the SIGINT and SIGTERM } template PYTYPE PyDefaultArgumentToMutableType(const py::object& argument) { // The default argument on Python functions always reference the same copy, // meaning if the default argument is changed by the function, then it is // changed for all subsequent calls to the function. Thus, default arguments // should be limited to basic types (i.e. None). This helper function returns // an empty expected type, if the argument is None (i.e. default initialized). // If the argument is neither None nor expected type, an exception is thrown. if (py::isinstance(argument)) { return PYTYPE(); } if (py::isinstance(argument)) { return argument; } throw PythonBackendException( std::string("Expect ") + typeid(PYTYPE).name() + ", got " + std::string(py::str(argument.get_type()))); } void AsyncEventFutureDoneCallback(const py::object& py_future) { std::unique_ptr& stub = Stub::GetOrCreateInstance(); stub->BackgroundFutureDone(py_future); } void Stub::Instantiate( int64_t shm_growth_size, int64_t shm_default_size, const std::string& shm_region_name, const std::string& model_path, const std::string& model_version, const std::string& triton_install_path, bi::managed_external_buffer::handle_t ipc_control_handle, const std::string& name, const std::string& python_runtime_model) { model_context_.Init( model_path, python_runtime_model, triton_install_path, model_version); name_ = name; health_mutex_ = nullptr; initialized_ = false; finalizing_ = false; stub_to_parent_thread_ = false; parent_to_stub_thread_ = false; try { shm_pool_ = std::make_unique( shm_region_name, shm_default_size, shm_growth_size, false /* create */); AllocatedSharedMemory ipc_control = shm_pool_->Load(ipc_control_handle); ipc_control_ = ipc_control.data_.get(); health_mutex_ = &(ipc_control_->stub_health_mutex); stub_message_queue_ = MessageQueue:: LoadFromSharedMemory(shm_pool_, ipc_control_->stub_message_queue); parent_message_queue_ = MessageQueue:: LoadFromSharedMemory(shm_pool_, ipc_control_->parent_message_queue); stub_to_parent_mq_ = MessageQueue:: LoadFromSharedMemory(shm_pool_, ipc_control_->stub_to_parent_mq); parent_to_stub_mq_ = MessageQueue:: LoadFromSharedMemory(shm_pool_, ipc_control_->parent_to_stub_mq); memory_manager_message_queue_ = MessageQueue::LoadFromSharedMemory( shm_pool_, ipc_control_->memory_manager_message_queue); // If the Python model is using an execution environment, we need to // remove the first part of the LD_LIBRARY_PATH before the colon (i.e. // :$OLD_LD_LIBRARY_PATH). The // section was added before launching the stub process and it may // interfere with the shared library resolution of other executable and // binaries. if (ipc_control_->uses_env) { #ifndef _WIN32 char* ld_library_path = std::getenv("LD_LIBRARY_PATH"); if (ld_library_path != nullptr) { std::string ld_library_path_str = ld_library_path; // If we use an Execute Environment, the path must contain a colon. size_t find_pos = ld_library_path_str.find(':'); if (find_pos == std::string::npos) { throw PythonBackendException( "LD_LIBRARY_PATH must contain a colon when passing an " "execution environment."); } ld_library_path_str = ld_library_path_str.substr(find_pos + 1); int status = setenv( "LD_LIBRARY_PATH", const_cast(ld_library_path_str.c_str()), 1 /* overwrite */); if (status != 0) { throw PythonBackendException( "Failed to correct the LD_LIBRARY_PATH environment in the " "Python backend stub."); } } else { throw PythonBackendException( "When using an execution environment, LD_LIBRARY_PATH variable " "cannot be empty."); } #else throw PythonBackendException( "Custom execution environments are not currently supported on " "Windows."); #endif } } catch (const PythonBackendException& pb_exception) { LOG_INFO << pb_exception.what() << std::endl; exit(1); } } std::unique_ptr>& Stub::MemoryManagerQueue() { return memory_manager_message_queue_; } bool& Stub::Health() { return ipc_control_->stub_health; } std::unique_ptr& Stub::SharedMemory() { return shm_pool_; } std::unique_ptr Stub::PopMessage() { bool success = false; std::unique_ptr ipc_message; bi::managed_external_buffer::handle_t message; while (!success) { message = stub_message_queue_->Pop(1000, success); } ipc_message = IPCMessage::LoadFromSharedMemory(shm_pool_, message); return ipc_message; } bool Stub::IsDecoupled() { return ipc_control_->decoupled; } bool Stub::RunCommand() { NVTX_RANGE(nvtx_, "RunCommand " + name_); std::unique_ptr ipc_message; { // Release the GIL lock when waiting for new message. Without this line, the // other threads in the user's Python model cannot make progress if they // give up GIL. py::gil_scoped_release release; ipc_message = this->PopMessage(); } switch (ipc_message->Command()) { case PYTHONSTUB_CommandType::PYTHONSTUB_AutoCompleteRequest: { // Only run this case when auto complete was requested by // Triton core. bool has_exception = false; std::string error_string; std::string auto_complete_config; std::unique_ptr auto_complete_response_msg = IPCMessage::Create(shm_pool_, false /* inline_response */); auto_complete_response_msg->Command() = PYTHONSTUB_AutoCompleteResponse; std::unique_ptr error_string_shm; std::unique_ptr auto_complete_config_shm; AllocatedSharedMemory auto_complete_response = shm_pool_->Construct(); ScopedDefer receive_autocomplete_finalize( [this] { stub_message_queue_->Pop(); }); ScopedDefer _([this, &auto_complete_response_msg] { SendIPCMessage(auto_complete_response_msg); }); auto_complete_response.data_->response_has_error = false; auto_complete_response.data_->response_is_error_set = false; auto_complete_response.data_->response_has_model_config = false; auto_complete_response_msg->Args() = auto_complete_response.handle_; try { AutoCompleteModelConfig(ipc_message->Args(), &auto_complete_config); } catch (const PythonBackendException& pb_exception) { has_exception = true; error_string = pb_exception.what(); } catch (const py::error_already_set& error) { has_exception = true; error_string = error.what(); } if (has_exception) { // Do not delete the region. The region will be deleted by the parent // process. shm_pool_->SetDeleteRegion(false); LOG_INFO << "Failed to initialize Python stub for auto-complete: " << error_string; auto_complete_response.data_->response_has_error = true; auto_complete_response.data_->response_is_error_set = false; LOG_IF_EXCEPTION( error_string_shm = PbString::Create(shm_pool_, error_string)); if (error_string_shm != nullptr) { auto_complete_response.data_->response_is_error_set = true; auto_complete_response.data_->response_error = error_string_shm->ShmHandle(); } return true; // Terminate the stub process. } else { LOG_IF_EXCEPTION( auto_complete_config_shm = PbString::Create(shm_pool_, auto_complete_config)); if (auto_complete_config_shm != nullptr) { auto_complete_response.data_->response_has_model_config = true; auto_complete_response.data_->response_model_config = auto_complete_config_shm->ShmHandle(); } } } break; case PYTHONSTUB_CommandType::PYTHONSTUB_InitializeRequest: { bool has_exception = false; std::string error_string; std::unique_ptr initialize_response_msg = IPCMessage::Create(shm_pool_, false /* inline_response */); initialize_response_msg->Command() = PYTHONSTUB_InitializeResponse; std::unique_ptr error_string_shm; AllocatedSharedMemory initialize_response = shm_pool_->Construct(); // The initialization is done in three steps. First the main process sends // a message to the stub process asking to begin to initialize the Python // model. After that is finished stub process sends a message to the // parent process that the initialization is finished. Finally, the // parent process sends a message to the stub process asking the stub // process to release any objects it has held in shared memory. ScopedDefer receive_initialize_finalize( [this] { stub_message_queue_->Pop(); }); ScopedDefer _([this, &initialize_response_msg] { SendIPCMessage(initialize_response_msg); }); initialize_response.data_->response_has_error = false; initialize_response.data_->response_is_error_set = false; initialize_response_msg->Args() = initialize_response.handle_; try { Initialize(ipc_message->Args()); } catch (const PythonBackendException& pb_exception) { has_exception = true; error_string = pb_exception.what(); } catch (const py::error_already_set& error) { has_exception = true; error_string = error.what(); } if (has_exception) { // Do not delete the region. The region will be deleted by the parent // process. shm_pool_->SetDeleteRegion(false); LOG_INFO << "Failed to initialize Python stub: " << error_string; initialize_response.data_->response_has_error = true; initialize_response.data_->response_is_error_set = false; LOG_IF_EXCEPTION( error_string_shm = PbString::Create(shm_pool_, error_string)); if (error_string_shm != nullptr) { initialize_response.data_->response_is_error_set = true; initialize_response.data_->response_error = error_string_shm->ShmHandle(); } return true; // Terminate the stub process. } } break; case PYTHONSTUB_CommandType::PYTHONSTUB_ExecuteRequest: { AllocatedSharedMemory request_batch = shm_pool_->Load(ipc_message->Args()); RequestBatch* request_batch_shm_ptr = reinterpret_cast(request_batch.data_.get()); ProcessRequests(request_batch_shm_ptr); } break; case PYTHONSTUB_CommandType::PYTHONSTUB_FinalizeRequest: ipc_message->Command() = PYTHONSTUB_FinalizeResponse; // Clean up response_iterator_map_ before sending sending message back to // the parent process to make sure that the clean up message can be // processed before the message queue is destroyed. { std::lock_guard lock(response_iterator_map_mu_); std::unordered_map>().swap( response_iterator_map_); } SendIPCMessage(ipc_message); return true; // Terminate the stub process case PYTHONSTUB_CommandType::PYTHONSTUB_LoadGPUBuffers: try { LoadGPUBuffers(ipc_message); } catch (const PythonBackendException& pb_exception) { LOG_ERROR << "An error occurred while trying to load GPU buffers in the " "Python backend stub: " << pb_exception.what() << std::endl; } break; default: break; } return false; } py::module Stub::StubSetup() { py::module sys = py::module_::import("sys"); model_context_.StubSetup(sys); py::module python_backend_utils = py::module_::import("triton_python_backend_utils"); py::module c_python_backend_utils = py::module_::import("c_python_backend_utils"); py::setattr( python_backend_utils, "TritonError", c_python_backend_utils.attr("TritonError")); py::setattr( python_backend_utils, "TritonModelException", c_python_backend_utils.attr("TritonModelException")); py::setattr( python_backend_utils, "Tensor", c_python_backend_utils.attr("Tensor")); py::setattr( python_backend_utils, "InferenceRequest", c_python_backend_utils.attr("InferenceRequest")); py::setattr( python_backend_utils, "InferenceResponse", c_python_backend_utils.attr("InferenceResponse")); py::setattr( python_backend_utils, "Logger", c_python_backend_utils.attr("Logger")); py::setattr( python_backend_utils, "PreferredMemory", c_python_backend_utils.attr("PreferredMemory")); py::setattr( python_backend_utils, "TRITONSERVER_MEMORY_GPU", c_python_backend_utils.attr("TRITONSERVER_MEMORY_GPU")); py::setattr( python_backend_utils, "TRITONSERVER_MEMORY_CPU", c_python_backend_utils.attr("TRITONSERVER_MEMORY_CPU")); py::setattr( python_backend_utils, "MetricFamily", c_python_backend_utils.attr("MetricFamily")); py::setattr( python_backend_utils, "load_model", c_python_backend_utils.attr("load_model")); py::setattr( python_backend_utils, "unload_model", c_python_backend_utils.attr("unload_model")); py::setattr( python_backend_utils, "is_model_ready", c_python_backend_utils.attr("is_model_ready")); c_python_backend_utils.attr("shared_memory") = py::cast(shm_pool_.get()); deserialize_bytes_ = python_backend_utils.attr("deserialize_bytes_tensor"); serialize_bytes_ = python_backend_utils.attr("serialize_byte_tensor"); return sys; } void Stub::AutoCompleteModelConfig( bi::managed_external_buffer::handle_t string_handle, std::string* auto_complete_config) { py::module sys = StubSetup(); std::unique_ptr pb_string_shm = PbString::LoadFromSharedMemory(shm_pool_, string_handle); py::module python_backend_utils = py::module_::import("triton_python_backend_utils"); py::object model_config = python_backend_utils.attr("ModelConfig")(pb_string_shm->String()); python_backend_utils.def( "get_model_dir", []() { std::unique_ptr& stub = Stub::GetOrCreateInstance(); return stub->GetModelDir(); }, py::return_value_policy::reference); if (py::hasattr(sys.attr("TritonPythonModel"), "auto_complete_config")) { model_config = sys.attr("TritonPythonModel") .attr("auto_complete_config")(model_config); } if (!py::isinstance(model_config, python_backend_utils.attr("ModelConfig"))) { throw PythonBackendException( "auto_complete_config function in model '" + name_ + "' must return a valid pb.ModelConfig object."); } py::module json = py::module_::import("json"); (*auto_complete_config) = std::string( py::str(json.attr("dumps")(model_config.attr("_model_config")))); } void Stub::Initialize(bi::managed_external_buffer::handle_t map_handle) { py::module sys = StubSetup(); py::module python_backend_utils = py::module_::import("triton_python_backend_utils"); py::module c_python_backend_utils = py::module_::import("c_python_backend_utils"); py::setattr( python_backend_utils, "TritonError", c_python_backend_utils.attr("TritonError")); py::setattr( python_backend_utils, "TritonModelException", c_python_backend_utils.attr("TritonModelException")); py::setattr( python_backend_utils, "Tensor", c_python_backend_utils.attr("Tensor")); py::setattr( python_backend_utils, "InferenceRequest", c_python_backend_utils.attr("InferenceRequest")); py::setattr( python_backend_utils, "InferenceResponse", c_python_backend_utils.attr("InferenceResponse")); c_python_backend_utils.attr("shared_memory") = py::cast(shm_pool_.get()); async_event_loop_ = py::none(); background_futures_ = py::set(); py::object TritonPythonModel = sys.attr("TritonPythonModel"); deserialize_bytes_ = python_backend_utils.attr("deserialize_bytes_tensor"); serialize_bytes_ = python_backend_utils.attr("serialize_byte_tensor"); python_backend_utils.def( "get_model_dir", []() { std::unique_ptr& stub = Stub::GetOrCreateInstance(); return stub->GetModelDir(); }, py::return_value_policy::reference); model_instance_ = TritonPythonModel(); std::unordered_map map; std::unique_ptr pb_map_shm = PbMap::LoadFromSharedMemory(shm_pool_, map_handle); // Get the unordered_map representation of the map in shared memory. map = pb_map_shm->UnorderedMap(); py::dict model_config_params; for (const auto& pair : map) { model_config_params[pair.first.c_str()] = pair.second; } LaunchStubToParentQueueMonitor(); LaunchParentToStubQueueMonitor(); // Call initialize if exists. if (py::hasattr(model_instance_, "initialize")) { model_instance_.attr("initialize")(model_config_params); } initialized_ = true; } void Stub::LoadGPUBuffers(std::unique_ptr& ipc_message) { ScopedDefer load_gpu_buffer_response([this] { // LoadGPUBuffers must let the parent process know when loading the // buffers have been finished. parent_message_queue_->Push(DUMMY_MESSAGE); gpu_tensors_.clear(); }); AllocatedSharedMemory gpu_buffers_handle = shm_pool_->Load(ipc_message->Args()); if (!gpu_buffers_handle.data_->success) { std::unique_ptr error = PbString::LoadFromSharedMemory( shm_pool_, gpu_buffers_handle.data_->error); throw PythonBackendException( "Failed to load GPU buffers: " + error->String()); } uint64_t gpu_buffer_count = gpu_buffers_handle.data_->buffer_count; AllocatedSharedMemory gpu_buffers_handle_shm = shm_pool_->Load( gpu_buffers_handle.data_->buffers); if (gpu_tensors_.size() != gpu_buffer_count) { throw PythonBackendException( std::string("GPU buffers size does not match the provided buffers: ") + std::to_string(gpu_tensors_.size()) + " != " + std::to_string(gpu_buffer_count)); } std::vector> dst_buffers; for (size_t i = 0; i < gpu_tensors_.size(); i++) { std::unique_ptr dst_buffer = PbMemory::LoadFromSharedMemory( shm_pool_, gpu_buffers_handle_shm.data_.get()[i], true /* open_cuda_handle */); dst_buffers.emplace_back(std::move(dst_buffer)); } for (size_t i = 0; i < gpu_tensors_.size(); i++) { std::shared_ptr& src_buffer = gpu_tensors_[i]; PbMemory::CopyBuffer(dst_buffers[i], src_buffer->Memory()); } } py::list Stub::LoadRequestsFromSharedMemory(RequestBatch* request_batch_shm_ptr) { uint32_t batch_size = request_batch_shm_ptr->batch_size; py::list py_request_list; if (batch_size == 0) { return py_request_list; } bi::managed_external_buffer::handle_t* request_shm_handle = reinterpret_cast( reinterpret_cast(request_batch_shm_ptr) + sizeof(RequestBatch)); for (size_t i = 0; i < batch_size; i++) { std::shared_ptr infer_request = InferRequest::LoadFromSharedMemory( shm_pool_, request_shm_handle[i], true /* open_cuda_handle */, &ipc_control_->decoupled /* is_model_decoupled */); py_request_list.append(infer_request); } return py_request_list; } void Stub::ProcessRequests(RequestBatch* request_batch_shm_ptr) { py::list py_request_list = LoadRequestsFromSharedMemory(request_batch_shm_ptr); std::unique_ptr execute_response; std::optional> response_batch; bool has_exception = false; std::string error_string; std::unique_ptr error_string_shm; std::string err_message; ScopedDefer execute_finalize([this] { stub_message_queue_->Pop(); }); ScopedDefer _( [this, &execute_response] { SendIPCMessage(execute_response); }); py::object execute_return; py::object coroutine_return; try { if (!py::hasattr(model_instance_, "execute")) { std::string message = "Python model " + model_context_.PythonModelPath() + " does not implement `execute` method."; throw PythonBackendException(message); } { NVTX_RANGE(nvtx_, "PyExecute " + name_); execute_return = model_instance_.attr("execute")(py_request_list); bool is_coroutine = py::module::import("asyncio") .attr("iscoroutine")(execute_return) .cast(); if (is_coroutine) { if (IsDecoupled()) { // Do not wait for async decoupled execute to return. RunCoroutine(execute_return, true /* in_background */); } else { coroutine_return = RunCoroutine(execute_return, false /* in_background */); ProcessReturnedResponses( py_request_list, coroutine_return, response_batch); } } else { ProcessReturnedResponses( py_request_list, execute_return, response_batch); } } } catch (const PythonBackendException& pb_exception) { has_exception = true; error_string = pb_exception.what(); } catch (const py::error_already_set& error) { has_exception = true; error_string = error.what(); } if (has_exception) { err_message = std::string( "Failed to process the request(s) for model '" + name_ + "', message: ") + error_string; LOG_ERROR << err_message.c_str(); if (!response_batch) { response_batch = shm_pool_->Construct( sizeof(ResponseBatch) + sizeof(IPCMessageShm)); } ResponseBatch* response_batch_shm_ptr = reinterpret_cast( response_batch.value().data_.get() + sizeof(IPCMessageShm)); // The backend will clean up the response factory if there is an error in // the response batch. For decoupled mode, it is necessary to handle cases // where the response sender should have already cleaned up, ensuring the // backend does not delete the response factory again during error handling. if (IsDecoupled()) { for (py::handle py_request : py_request_list) { InferRequest* request = py_request.cast(); if (request->GetResponseSender()->IsClosed()) { response_batch_shm_ptr->is_response_factory_deleted = true; } } } response_batch_shm_ptr->has_error = true; error_string_shm = PbString::Create(shm_pool_, err_message); response_batch_shm_ptr->error = error_string_shm->ShmHandle(); response_batch_shm_ptr->is_error_set = true; response_batch_shm_ptr->batch_size = 0; // Once the error is sent to the backend, the backend is supposed to close // all response factories if not already closed, so closing all response // senders if not already closed to prevent the model from sending more // responses after the factories are closed. for (py::handle py_request : py_request_list) { InferRequest* request = py_request.cast(); request->GetResponseSender()->Close(); } } else { if (!response_batch) { response_batch = shm_pool_->Construct( sizeof(ResponseBatch) + sizeof(IPCMessageShm)); ResponseBatch* response_batch_shm_ptr = reinterpret_cast( response_batch.value().data_.get() + sizeof(IPCMessageShm)); response_batch_shm_ptr->batch_size = 0; } ResponseBatch* response_batch_shm_ptr = reinterpret_cast( response_batch.value().data_.get() + sizeof(IPCMessageShm)); response_batch_shm_ptr->has_error = false; response_batch_shm_ptr->is_error_set = false; } execute_response = IPCMessage::Create( reinterpret_cast(response_batch.value().data_.get()), response_batch.value().handle_); execute_response->Args() = response_batch.value().handle_ + sizeof(IPCMessageShm); execute_response->InlineResponse() = false; execute_response->Command() = PYTHONSTUB_ExecuteResponse; _.Complete(); execute_finalize.Complete(); } void Stub::ProcessResponse(InferResponse* response) { response->SaveToSharedMemory(shm_pool_, false /* copy_gpu */); for (auto& output_tensor : response->OutputTensors()) { if (!output_tensor->IsCPU()) { gpu_tensors_.push_back(output_tensor); } } } void Stub::ProcessReturnedResponses( py::list py_requests, py::object py_responses_obj, std::optional>& response_batch) { // Return if there is nothing to process. if (py::isinstance(py_responses_obj)) { return; } // Only non-decoupled may return responses. if (IsDecoupled()) { throw PythonBackendException( "Python model '" + name_ + "' is using the decoupled mode and the execute function must return " "None."); } // Check responses is a list. if (!py::isinstance(py_responses_obj)) { throw PythonBackendException( "Expected a list in the execute return, found type '" + std::string(py::str(py_responses_obj.get_type())) + "'."); } py::list py_responses = py_responses_obj; // Responses and requests length must match. size_t requests_size = py::len(py_requests); size_t responses_size = py::len(py_responses); if (requests_size != responses_size) { throw PythonBackendException( "Number of InferenceResponse objects do not match the number of " "InferenceRequest objects. InferenceRequest(s) size is:" + std::to_string(requests_size) + ", and InferenceResponse(s) size is:" + std::to_string(responses_size) + "\n"); } for (size_t i = 0; i < responses_size; i++) { if (!py::isinstance(py_responses[i])) { InferRequest* request = py_requests[i].cast(); // Response must be None if rescheduled. if (request->ReleaseFlags() == TRITONSERVER_REQUEST_RELEASE_RESCHEDULE) { throw PythonBackendException( "Expected a None object in the execute function return list for " "reschduled request, found type '" + std::string(py::str(py_responses[i].get_type())) + "'."); } // Send the response. if (!py::isinstance(py_responses[i])) { throw PythonBackendException( "Expected an 'InferenceResponse' object in the execute function " "return list, found type '" + std::string(py::str(py_responses[i].get_type())) + "'."); } InferResponse* response = py_responses[i].cast(); try { request->GetResponseSender()->UpdateStateAndCounters( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL); } catch (const PythonBackendException& pb_exception) { // Handle the exception here to catch the error when there's a response // returned from `execute()`. if (request->GetResponseSender()->IsClosed()) { response_batch = std::move(shm_pool_->Construct( sizeof(ResponseBatch) + sizeof(IPCMessageShm))); ResponseBatch* response_batch_shm_ptr = reinterpret_cast( response_batch.value().data_.get() + sizeof(IPCMessageShm)); response_batch_shm_ptr->batch_size = 0; response_batch_shm_ptr->is_response_factory_deleted = true; } throw pb_exception; } } } // Return all the created responses using response_batch. The reason // that both of the paths are available is that sending the responses // using response_batch is faster than using `response_sender`. response_batch = std::move(shm_pool_->Construct( sizeof(IPCMessageShm) + requests_size * sizeof(bi::managed_external_buffer::handle_t) + sizeof(ResponseBatch))); ResponseBatch* response_batch_shm_ptr = reinterpret_cast( response_batch.value().data_.get() + sizeof(IPCMessageShm)); bi::managed_external_buffer::handle_t* responses_shm_handle = reinterpret_cast( response_batch.value().data_.get() + sizeof(ResponseBatch) + sizeof(IPCMessageShm)); for (size_t i = 0; i < responses_size; i++) { // Check the return type of execute function. InferRequest* infer_request = py_requests[i].cast(); InferResponse* infer_response = py_responses[i].cast(); if (!py::isinstance(py_responses[i])) { infer_response->PruneOutputTensors(infer_request->RequestedOutputNames()); ProcessResponse(infer_response); responses_shm_handle[i] = infer_response->ShmHandle(); } else { responses_shm_handle[i] = 0; } } response_batch_shm_ptr->batch_size = requests_size; } py::object Stub::GetAsyncEventLoop() { if (py::isinstance(async_event_loop_)) { // Create the event loop if not already. py::module asyncio = py::module_::import("asyncio"); async_event_loop_ = asyncio.attr("new_event_loop")(); asyncio.attr("set_event_loop")(async_event_loop_); py::object py_thread = py::module_::import("threading") .attr("Thread")( "target"_a = async_event_loop_.attr("run_forever"), "daemon"_a = true); py_thread.attr("start")(); } return async_event_loop_; } py::object Stub::RunCoroutine(py::object coroutine, bool in_background) { py::object loop = GetAsyncEventLoop(); py::object py_future = py::module_::import("asyncio").attr( "run_coroutine_threadsafe")(coroutine, loop); if (in_background) { py_future.attr("add_done_callback")( py::module_::import("c_python_backend_utils") .attr("async_event_future_done_callback")); background_futures_.attr("add")(py_future); return py::none(); } return py_future.attr("result")(); } void Stub::BackgroundFutureDone(const py::object& py_future) { ScopedDefer _([this, &py_future] { // Remove future from background try { background_futures_.attr("remove")(py_future); } catch (const py::error_already_set& error) { LOG_ERROR << "Cannot remove future from background; " << error.what(); } }); // TODO: Why using `py_future.result()` with error hangs on exit? try { py::object exception = py_future.attr("exception")(); if (!py::isinstance(exception)) { std::string err_msg = ""; py::object traceback = py::module_::import("traceback") .attr("TracebackException") .attr("from_exception")(exception) .attr("format")(); for (py::handle line : traceback) { err_msg += py::str(line); } LOG_ERROR << err_msg; } } catch (const PythonBackendException& pb_exception) { LOG_ERROR << pb_exception.what(); } catch (const py::error_already_set& error) { LOG_ERROR << error.what(); } } void Stub::UpdateHealth() { bi::scoped_lock lock(*health_mutex_); ipc_control_->stub_health = true; } void Stub::Finalize() { finalizing_ = true; if (initialized_) { // Stop async event loop if created. if (!py::isinstance(async_event_loop_)) { async_event_loop_.attr("stop")(); } // Call finalize if exists. if (py::hasattr(model_instance_, "finalize")) { try { model_instance_.attr("finalize")(); } catch (const py::error_already_set& e) { LOG_INFO << e.what(); } } } #ifdef TRITON_ENABLE_GPU // We also need to destroy created proxy CUDA streams for dlpack, if any std::lock_guard lock(dlpack_proxy_stream_pool_mu_); for (auto& entry : dlpack_proxy_stream_pool_) { // We don't need to switch device to destroy a stream // https://stackoverflow.com/questions/64663943/how-to-destroy-a-stream-that-was-created-on-a-specific-device cudaError_t err = cudaStreamDestroy(entry.second); if (err != cudaSuccess) { LOG_ERROR << "Failed to destroy dlpack CUDA proxy stream on device with id " + std::to_string(entry.first); } } #endif } void Stub::SendIPCMessage(std::unique_ptr& ipc_message) { bool success = false; while (!success) { parent_message_queue_->Push(ipc_message->ShmHandle(), 1000, success); } } void Stub::SendIPCUtilsMessage(std::unique_ptr& ipc_message) { bool success = false; while (!success) { stub_to_parent_mq_->Push(ipc_message->ShmHandle(), 1000, success); } } Stub::~Stub() { #ifdef TRITON_ENABLE_GPU try { CUDAHandler& cuda_api = CUDAHandler::getInstance(); for (auto& m : shm_pool_->GetCUDAMemoryPoolManager()->CUDAPoolAddressMap()) { if (m.second != nullptr) { cuda_api.CloseCudaHandle(m.first, m.second); } } } catch (const PythonBackendException& pb_exception) { std::cerr << "Error when closing CUDA handle: " << pb_exception.what(); } #endif { py::gil_scoped_acquire acquire; async_event_loop_ = py::none(); background_futures_ = py::none(); model_instance_ = py::none(); } stub_instance_.reset(); stub_message_queue_.reset(); parent_message_queue_.reset(); stub_to_parent_mq_.reset(); memory_manager_message_queue_.reset(); } std::unique_ptr Stub::stub_instance_; std::unique_ptr& Stub::GetOrCreateInstance() { if (Stub::stub_instance_.get() == nullptr) { Stub::stub_instance_ = std::make_unique(); } return Stub::stub_instance_; } void Stub::LaunchStubToParentQueueMonitor() { stub_to_parent_thread_ = true; stub_to_parent_queue_monitor_ = std::thread(&Stub::ServiceStubToParentRequests, this); Logger::GetOrCreateInstance()->SetBackendLoggingActive(true); } void Stub::TerminateStubToParentQueueMonitor() { Logger::GetOrCreateInstance()->SetBackendLoggingActive(false); { std::lock_guard guard{stub_to_parent_message_mu_}; // Push a dummy message to signal the thread to terminate. stub_to_parent_buffer_.push(DUMMY_MESSAGE); } stub_to_parent_message_cv_.notify_one(); stub_to_parent_queue_monitor_.join(); } void Stub::EnqueueLogRequest(std::unique_ptr& log_ptr) { std::unique_ptr utils_msg_payload = std::make_unique( PYTHONSTUB_LogRequest, reinterpret_cast(log_ptr.release())); EnqueueUtilsMessage(std::move(utils_msg_payload)); } void Stub::ServiceStubToParentRequests() { while (stub_to_parent_thread_) { std::unique_lock guard{stub_to_parent_message_mu_}; while (stub_to_parent_buffer_.empty()) { stub_to_parent_message_cv_.wait(guard); } // On exit, will send messages to the parent process until // DUMMY_MESSAGE is reached std::unique_ptr utils_msg_payload = std::move(stub_to_parent_buffer_.front()); if (utils_msg_payload == DUMMY_MESSAGE) { stub_to_parent_buffer_.pop(); break; } else { stub_to_parent_buffer_.pop(); if (utils_msg_payload->command_type == PYTHONSTUB_LogRequest) { SendLogMessage(utils_msg_payload); } else if ( (utils_msg_payload->command_type == PYTHONSTUB_BLSDecoupledInferPayloadCleanup) || (utils_msg_payload->command_type == PYTHONSTUB_DecoupledResponseFactoryCleanup)) { SendCleanupId(utils_msg_payload, utils_msg_payload->command_type); } else if ( utils_msg_payload->command_type == PYTHONSTUB_IsRequestCancelled) { SendIsCancelled(utils_msg_payload); } else { std::cerr << "Error when sending message via stub_to_parent message " "buffer - unknown command\n"; } } } } void Stub::SendLogMessage(std::unique_ptr& utils_msg_payload) { std::unique_ptr log_send_message = std::unique_ptr( reinterpret_cast(utils_msg_payload->utils_message_ptr)); std::unique_ptr log_request_shm = PbLogShm::Create( shm_pool_, log_send_message->Filename(), log_send_message->Line(), log_send_message->Message(), log_send_message->Level()); LogSendMessage* send_message_payload = log_request_shm->LogMessage(); send_message_payload->waiting_on_stub = false; std::unique_ptr log_request_msg = IPCMessage::Create(shm_pool_, false /* inline_response */); log_request_msg->Args() = log_request_shm->ShmHandle(); log_request_msg->Command() = PYTHONSTUB_LogRequest; ScopedDefer _([send_message_payload] { { bi::scoped_lock guard{send_message_payload->mu}; send_message_payload->waiting_on_stub = false; send_message_payload->cv.notify_all(); } }); { // Send a message to be caught by the log monitor thread in python_be.cc bi::scoped_lock guard{send_message_payload->mu}; SendIPCUtilsMessage(log_request_msg); while (!send_message_payload->waiting_on_stub) { send_message_payload->cv.wait(guard); } } } void Stub::SendCleanupId( std::unique_ptr& utils_msg_payload, const PYTHONSTUB_CommandType& command_type) { void* id = utils_msg_payload->utils_message_ptr; if (command_type == PYTHONSTUB_BLSDecoupledInferPayloadCleanup) { std::lock_guard lock(response_iterator_map_mu_); response_iterator_map_.erase(id); } std::unique_ptr ipc_message = IPCMessage::Create(shm_pool_, true /* inline_response */); ipc_message->Command() = command_type; AllocatedSharedMemory cleanup_request_message = shm_pool_->Construct( sizeof(CleanupMessage) + sizeof(bi::managed_external_buffer::handle_t)); CleanupMessage* cleanup_message_ptr = reinterpret_cast(cleanup_request_message.data_.get()); cleanup_message_ptr->id = id; cleanup_message_ptr->waiting_on_stub = false; ipc_message->Args() = cleanup_request_message.handle_; { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; SendIPCUtilsMessage(ipc_message); while (!cleanup_message_ptr->waiting_on_stub) { ipc_message->ResponseCondition()->wait(lock); } } } void Stub::EnqueueCleanupId(void* id, const PYTHONSTUB_CommandType& command_type) { if (id != nullptr) { std::unique_ptr utils_msg_payload = std::make_unique(command_type, id); EnqueueUtilsMessage(std::move(utils_msg_payload)); } } void Stub::EnqueueIsCancelled(PbCancel* pb_cancel) { std::unique_ptr utils_msg_payload = std::make_unique( PYTHONSTUB_IsRequestCancelled, reinterpret_cast(pb_cancel)); EnqueueUtilsMessage(std::move(utils_msg_payload)); } void Stub::SendIsCancelled(std::unique_ptr& utils_msg_payload) { PbCancel* pb_cancel = reinterpret_cast(utils_msg_payload->utils_message_ptr); pb_cancel->SaveToSharedMemory(shm_pool_); IsCancelledMessage* message_payload = pb_cancel->ShmPayload(); std::unique_ptr ipc_message = IPCMessage::Create(shm_pool_, false /* inline_response */); ipc_message->Command() = utils_msg_payload->command_type; ipc_message->Args() = pb_cancel->ShmHandle(); bool is_cancelled = false; { bi::scoped_lock lk(message_payload->mu); SendIPCUtilsMessage(ipc_message); while (!message_payload->waiting_on_stub) { message_payload->cv.wait(lk); } is_cancelled = message_payload->is_cancelled; message_payload->waiting_on_stub = false; message_payload->cv.notify_all(); } pb_cancel->ReportIsCancelled(is_cancelled); } bool Stub::StubToParentServiceActive() { return stub_to_parent_thread_; } void Stub::LaunchParentToStubQueueMonitor() { parent_to_stub_thread_ = true; parent_to_stub_queue_monitor_ = std::thread(&Stub::ParentToStubMQMonitor, this); } void Stub::TerminateParentToStubQueueMonitor() { if (parent_to_stub_thread_) { parent_to_stub_thread_ = false; // Push a dummy message to signal the thread to terminate. parent_to_stub_mq_->Push(DUMMY_MESSAGE); parent_to_stub_queue_monitor_.join(); } } void Stub::ParentToStubMQMonitor() { while (parent_to_stub_thread_) { bi::managed_external_buffer::handle_t handle = parent_to_stub_mq_->Pop(); if (handle == DUMMY_MESSAGE) { break; } std::unique_ptr ipc_message = IPCMessage::LoadFromSharedMemory(shm_pool_, handle); switch (ipc_message->Command()) { case PYTHONSTUB_CommandType::PYTHONSTUB_CUDAPoolInitializeRequest: { GetCUDAMemoryPoolAddress(ipc_message); } break; case PYTHONSTUB_CommandType::PYTHONSTUB_InferStreamExecResponse: { ProcessBLSResponseDecoupled(ipc_message); } break; default: break; } } } bool Stub::ParentToStubServiceActive() { return parent_to_stub_thread_; } std::shared_ptr Stub::GetResponseIterator(std::shared_ptr infer_response) { std::lock_guard lock(response_iterator_map_mu_); if (response_iterator_map_.find(infer_response->Id()) != response_iterator_map_.end()) { // Need to re-construct the 'ResponseIterator' and update the // 'response_iterator_map_' to make sure the 'ResponseIterator' object has // the correct first response. auto response_iterator = std::make_shared(infer_response); std::vector> existing_responses = response_iterator_map_[infer_response->Id()]->GetExistingResponses(); for (auto& response : existing_responses) { response_iterator->EnqueueResponse(response); } response_iterator_map_[infer_response->Id()] = response_iterator; } else { auto response_iterator = std::make_shared(infer_response); response_iterator_map_.insert( std::pair>( response_iterator->Id(), response_iterator)); } return response_iterator_map_[infer_response->Id()]; } bool Stub::IsInitialized() { return initialized_; } bool Stub::IsFinalizing() { return finalizing_; } void Stub::EnqueueUtilsMessage( std::unique_ptr utils_msg_payload) { { std::lock_guard guard{stub_to_parent_message_mu_}; stub_to_parent_buffer_.push(std::move(utils_msg_payload)); } stub_to_parent_message_cv_.notify_one(); } cudaStream_t Stub::GetProxyStream(const int& device_id) { #ifdef TRITON_ENABLE_GPU std::lock_guard lock(dlpack_proxy_stream_pool_mu_); if (dlpack_proxy_stream_pool_.find(device_id) == dlpack_proxy_stream_pool_.end()) { cudaStream_t new_proxy_stream; cudaError_t err = cudaStreamCreate(&new_proxy_stream); if (err == cudaSuccess) { dlpack_proxy_stream_pool_.emplace(device_id, new_proxy_stream); return new_proxy_stream; } else { throw PythonBackendException( "Failed to create a CUDA stream for a DLPack call."); } } return dlpack_proxy_stream_pool_[device_id]; #else return nullptr; #endif } void Stub::GetCUDAMemoryPoolAddress(std::unique_ptr& ipc_message) { #ifdef TRITON_ENABLE_GPU bool has_exception = false; std::string error_string; std::unique_ptr error_string_shm; CUDAMemPoolMessage* cuda_pool_message_ptr = nullptr; try { AllocatedSharedMemory cuda_handle_shm = shm_pool_->Load(ipc_message->Args()); cuda_pool_message_ptr = cuda_handle_shm.data_.get(); CUDAHandler& cuda_api = CUDAHandler::getInstance(); void* cuda_pool_address; cuda_api.OpenCudaHandle( cuda_pool_message_ptr->device_id, &cuda_pool_message_ptr->cuda_handle, &cuda_pool_address); shm_pool_->GetCUDAMemoryPoolManager()->SetCUDAPoolAddress( cuda_pool_message_ptr->device_id, cuda_pool_address); } catch (const PythonBackendException& pb_exception) { has_exception = true; error_string = pb_exception.what(); shm_pool_->GetCUDAMemoryPoolManager()->SetCUDAPoolAddress( cuda_pool_message_ptr->device_id, nullptr); } if (has_exception) { LOG_INFO << "Failed to initialize CUDA shared memory pool in Python stub: " << error_string; cuda_pool_message_ptr->has_error = true; cuda_pool_message_ptr->is_error_set = false; LOG_IF_EXCEPTION( error_string_shm = PbString::Create(shm_pool_, error_string)); if (error_string_shm != nullptr) { cuda_pool_message_ptr->is_error_set = true; cuda_pool_message_ptr->error = error_string_shm->ShmHandle(); } } { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; cuda_pool_message_ptr->waiting_on_stub = true; ipc_message->ResponseCondition()->notify_all(); while (cuda_pool_message_ptr->waiting_on_stub) { ipc_message->ResponseCondition()->wait(lock); } } #endif } void Stub::ProcessBLSResponseDecoupled(std::unique_ptr& ipc_message) { ResponseBatch* response_batch = nullptr; bi::managed_external_buffer::handle_t* response_handle = nullptr; std::unique_ptr infer_response; bool responses_is_set = false; PythonBackendException pb_exception(std::string{}); try { AllocatedSharedMemory response_batch_shm = shm_pool_->Load(ipc_message->Args()); response_batch = reinterpret_cast(response_batch_shm.data_.get()); response_handle = reinterpret_cast( response_batch_shm.data_.get() + sizeof(ResponseBatch)); responses_is_set = true; if (response_batch->has_error) { if (response_batch->is_error_set) { std::unique_ptr pb_string = PbString::LoadFromSharedMemory(shm_pool_, response_batch->error); infer_response = std::make_unique( std::vector>{}, std::make_shared(pb_string->String())); } else { infer_response = std::make_unique( std::vector>{}, std::make_shared( "An error occurred while performing BLS request.")); } } if (responses_is_set) { infer_response = InferResponse::LoadFromSharedMemory( shm_pool_, *response_handle, true /* open cuda handle */); for (auto& output_tensor : infer_response->OutputTensors()) { if (!output_tensor->IsCPU()) { uint64_t memory_release_id = output_tensor->Memory()->MemoryReleaseId(); output_tensor->Memory()->SetMemoryReleaseCallback( [this, memory_release_id]() { this->MemoryManagerQueue()->Push(memory_release_id); }); } } } else { infer_response = std::make_unique( std::vector>{}, std::make_shared( "An error occurred while performing BLS request.")); } } catch (const PythonBackendException& pb_exception) { infer_response = std::make_unique( std::vector>{}, std::make_shared(pb_exception.what())); } { std::lock_guard lock(response_iterator_map_mu_); if (response_iterator_map_.find(infer_response->Id()) != response_iterator_map_.end()) { response_iterator_map_[infer_response->Id()]->EnqueueResponse( std::move(infer_response)); } else { auto response_iterator = std::make_shared(std::move(infer_response)); response_iterator_map_.insert( std::pair>( response_iterator->Id(), response_iterator)); } } { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; response_batch->waiting_on_stub = true; ipc_message->ResponseCondition()->notify_all(); } } std::unique_ptr Logger::log_instance_; std::unique_ptr& Logger::GetOrCreateInstance() { if (Logger::log_instance_.get() == nullptr) { Logger::log_instance_ = std::make_unique(); } return Logger::log_instance_; } // Bound function, called from the python client void Logger::Log(const std::string& message, LogLevel level) { std::unique_ptr& stub = Stub::GetOrCreateInstance(); py::object frame = py::module_::import("inspect").attr("currentframe"); py::object caller_frame = frame(); py::object info = py::module_::import("inspect").attr("getframeinfo"); py::object caller_info = info(caller_frame); py::object filename_python = caller_info.attr("filename"); std::string filename = filename_python.cast(); py::object lineno = caller_info.attr("lineno"); uint32_t line = lineno.cast(); if (!stub->StubToParentServiceActive()) { Logger::GetOrCreateInstance()->Log(filename, line, level, message); } else { std::unique_ptr log_msg(new PbLog(filename, line, message, level)); stub->EnqueueLogRequest(log_msg); } } // Called internally (.e.g. LOG_ERROR << "Error"; ) void Logger::Log( const std::string& filename, uint32_t lineno, LogLevel level, const std::string& message) { // If the log monitor service is not active yet, format // and pass messages to cerr if (!BackendLoggingActive()) { std::string path(filename); size_t pos = path.rfind(std::filesystem::path::preferred_separator); if (pos != std::string::npos) { path = path.substr(pos + 1, std::string::npos); } #ifdef _WIN32 std::stringstream ss; SYSTEMTIME system_time; GetSystemTime(&system_time); ss << LeadingLogChar(level) << std::setfill('0') << std::setw(2) << system_time.wMonth << std::setw(2) << system_time.wDay << ' ' << std::setw(2) << system_time.wHour << ':' << std::setw(2) << system_time.wMinute << ':' << std::setw(2) << system_time.wSecond << '.' << std::setw(6) << system_time.wMilliseconds * 1000 << ' ' << static_cast(GetCurrentProcessId()) << ' ' << path << ':' << lineno << "] "; #else std::stringstream ss; struct timeval tv; gettimeofday(&tv, NULL); struct tm tm_time; gmtime_r(((time_t*)&(tv.tv_sec)), &tm_time); ss << LeadingLogChar(level) << std::setfill('0') << std::setw(2) << (tm_time.tm_mon + 1) << std::setw(2) << tm_time.tm_mday << " " << std::setw(2) << tm_time.tm_hour << ':' << std::setw(2) << tm_time.tm_min << ':' << std::setw(2) << tm_time.tm_sec << "." << std::setw(6) << tv.tv_usec << ' ' << static_cast(getpid()) << ' ' << path << ':' << lineno << "] "; std::cerr << ss.str() << " " << message << std::endl; #endif } else { // Ensure we do not create a stub instance before it has initialized std::unique_ptr& stub = Stub::GetOrCreateInstance(); std::unique_ptr log_msg(new PbLog(filename, lineno, message, level)); stub->EnqueueLogRequest(log_msg); } } void Logger::LogInfo(const std::string& message) { Logger::Log(message, LogLevel::kInfo); } void Logger::LogWarn(const std::string& message) { Logger::Log(message, LogLevel::kWarning); } void Logger::LogError(const std::string& message) { Logger::Log(message, LogLevel::kError); } void Logger::LogVerbose(const std::string& message) { Logger::Log(message, LogLevel::kVerbose); } const std::string Logger::LeadingLogChar(const LogLevel& level) { switch (level) { case LogLevel::kWarning: return "W"; case LogLevel::kError: return "E"; case LogLevel::kInfo: case LogLevel::kVerbose: default: return "I"; } } void Logger::SetBackendLoggingActive(bool status) { backend_logging_active_ = status; } bool Logger::BackendLoggingActive() { return backend_logging_active_; } PYBIND11_EMBEDDED_MODULE(c_python_backend_utils, module) { py::class_> triton_error( module, "TritonError"); py::enum_(triton_error, "__ErrorCode") .value("UNKNOWN", TRITONSERVER_Error_Code::TRITONSERVER_ERROR_UNKNOWN) .value("INTERNAL", TRITONSERVER_Error_Code::TRITONSERVER_ERROR_INTERNAL) .value("NOT_FOUND", TRITONSERVER_Error_Code::TRITONSERVER_ERROR_NOT_FOUND) .value( "INVALID_ARG", TRITONSERVER_Error_Code::TRITONSERVER_ERROR_INVALID_ARG) .value( "UNAVAILABLE", TRITONSERVER_Error_Code::TRITONSERVER_ERROR_UNAVAILABLE) .value( "UNSUPPORTED", TRITONSERVER_Error_Code::TRITONSERVER_ERROR_UNSUPPORTED) .value( "ALREADY_EXISTS", TRITONSERVER_Error_Code::TRITONSERVER_ERROR_ALREADY_EXISTS) .value("CANCELLED", TRITONSERVER_Error_Code::TRITONSERVER_ERROR_CANCELLED) .export_values(); triton_error.def_property_readonly_static( "UNKNOWN", [](py::object /* self */) { return TRITONSERVER_ERROR_UNKNOWN; }); triton_error.def_property_readonly_static( "INTERNAL", [](py::object /* self */) { return TRITONSERVER_ERROR_INTERNAL; }); triton_error.def_property_readonly_static( "NOT_FOUND", [](py::object /* self */) { return TRITONSERVER_ERROR_NOT_FOUND; }); triton_error.def_property_readonly_static( "INVALID_ARG", [](py::object /* self */) { return TRITONSERVER_ERROR_INVALID_ARG; }); triton_error.def_property_readonly_static( "UNAVAILABLE", [](py::object /* self */) { return TRITONSERVER_ERROR_UNAVAILABLE; }); triton_error.def_property_readonly_static( "UNSUPPORTED", [](py::object /* self */) { return TRITONSERVER_ERROR_UNSUPPORTED; }); triton_error.def_property_readonly_static( "ALREADY_EXISTS", [](py::object /* self */) { return TRITONSERVER_ERROR_ALREADY_EXISTS; }); triton_error.def_property_readonly_static( "CANCELLED", [](py::object /* self */) { return TRITONSERVER_ERROR_CANCELLED; }); triton_error.def( py::init(), py::arg("message").none(false), py::arg("code").none(false) = TRITONSERVER_ERROR_INTERNAL); triton_error.def("code", &PbError::Code); triton_error.def("message", &PbError::Message); py::class_>( module, "PreferredMemory") .def( py::init(), py::arg("preferred_memory_type").none(false), py::arg("preferred_device_id").none(false) = 0); py::enum_(module, "MemoryType") .value("TRITONSERVER_MEMORY_GPU", PreferredMemory::MemoryType::kGPU) .value("TRITONSERVER_MEMORY_CPU", PreferredMemory::MemoryType::kCPU) .export_values(); py::class_>( module, "InferenceTrace") .def("get_context", [](InferenceTrace& self) -> py::object { auto context = self.Context(); if (context != "") { return py::str(context); } return py::none(); }); py::class_>( module, "InferenceRequest") .def( py::init([](const std::string& request_id, const py::object& correlation_id, const std::vector>& inputs, const std::vector& requested_output_names, const std::string& model_name, const int64_t model_version, const uint32_t flags, const uint64_t timeout, const PreferredMemory& preferred_memory, const InferenceTrace& trace, const py::object& parameters_) { py::dict parameters = PyDefaultArgumentToMutableType(parameters_); std::set requested_outputs; for (auto& requested_output_name : requested_output_names) { requested_outputs.emplace(requested_output_name); } for (const auto& pair : parameters) { if (!py::isinstance(pair.first)) { throw PythonBackendException( "Expect parameters keys to have type str, found type " + std::string(py::str(pair.first.get_type()))); } if (!py::isinstance(pair.second) && !py::isinstance(pair.second) && !py::isinstance(pair.second)) { throw PythonBackendException( "Expect parameters values to have type bool/int/str, found " "type " + std::string(py::str(pair.second.get_type()))); } } py::module_ py_json = py::module_::import("json"); std::string parameters_str = py::str(py_json.attr("dumps")(parameters)); CorrelationId correlation_id_obj; if (py::isinstance(correlation_id)) { correlation_id_obj = CorrelationId(py::cast(correlation_id)); } else if (py::isinstance(correlation_id)) { correlation_id_obj = CorrelationId(py::cast(correlation_id)); } else { throw PythonBackendException( "Correlation ID must be integer or string"); } return std::make_shared( request_id, correlation_id_obj, inputs, requested_outputs, model_name, model_version, parameters_str, flags, timeout, 0 /*response_factory_address*/, 0 /*request_address*/, preferred_memory, trace); }), py::arg("request_id").none(false) = "", py::arg("correlation_id").none(false) = 0, py::arg("inputs").none(false), py::arg("requested_output_names").none(false), py::arg("model_name").none(false), py::arg("model_version").none(false) = -1, py::arg("flags").none(false) = 0, py::arg("timeout").none(false) = 0, py::arg("preferred_memory").none(false) = PreferredMemory(PreferredMemory::kDefault, 0), py::arg("trace").none(false) = InferenceTrace(), py::arg("parameters").none(true) = py::none()) .def( "inputs", &InferRequest::Inputs, py::return_value_policy::reference_internal) .def("request_id", &InferRequest::RequestId) .def( "correlation_id", [](InferRequest& self) -> py::object { CorrelationId correlation_id = self.GetCorrelationId(); if (correlation_id.Type() == CorrelationIdDataType::STRING) { return py::cast(correlation_id.StringValue()); } else { return py::cast(correlation_id.UnsignedIntValue()); } }) .def("flags", &InferRequest::Flags) .def("set_flags", &InferRequest::SetFlags) .def("timeout", &InferRequest::Timeout) .def("parameters", &InferRequest::Parameters) .def("trace", &InferRequest::GetTrace) .def( "exec", [](std::shared_ptr& infer_request, const bool decoupled) { std::unique_ptr& stub = Stub::GetOrCreateInstance(); std::shared_ptr response = infer_request->Exec(decoupled); py::object response_object; if (decoupled) { auto response_iterator = stub->GetResponseIterator(response); response_object = py::cast(response_iterator); } else { response_object = py::cast(response); } return response_object; }, py::arg("decoupled").none(false) = false) .def( "async_exec", [](std::shared_ptr& infer_request, const bool decoupled) { std::unique_ptr& stub = Stub::GetOrCreateInstance(); py::object loop = py::module_::import("asyncio").attr("get_running_loop")(); py::cpp_function callback = [&stub, infer_request, decoupled]() { std::shared_ptr response = infer_request->Exec(decoupled); py::object response_object; if (decoupled) { auto response_iterator = stub->GetResponseIterator(response); response_object = py::cast(response_iterator); } else { response_object = py::cast(response); } return response_object; }; py::object future = loop.attr("run_in_executor")(py::none(), callback); return future; }, py::arg("decoupled").none(false) = false) .def( "requested_output_names", &InferRequest::RequestedOutputNames, py::return_value_policy::reference_internal) .def("get_response_sender", &InferRequest::GetResponseSender) .def("is_cancelled", &InferRequest::IsCancelled) .def("set_release_flags", &InferRequest::SetReleaseFlags), py::arg("flags").none(false); py::class_>(module, "Tensor") .def(py::init(&PbTensor::FromNumpy)) .def("name", &PbTensor::Name) // The reference_internal is added to make sure that the NumPy object has // the same lifetime as the tensor object. This means even when the NumPy // object is only in scope, the tensor object is not deallocated from // shared memory to make sure the NumPy object is still valid. .def( "as_numpy", &PbTensor::AsNumpy, py::return_value_policy::reference_internal) .def("triton_dtype", &PbTensor::TritonDtype) .def("to_dlpack", &PbTensor::ToDLPack) .def("is_cpu", &PbTensor::IsCPU) .def("shape", &PbTensor::Dims) .def("from_dlpack", &PbTensor::FromDLPack) .def("__dlpack__", &PbTensor::DLPack, py::arg("stream") = py::none()) .def("__dlpack_device__", &PbTensor::DLPackDevice); py::class_>( module, "InferenceResponse") .def( py::init< const std::vector>&, std::shared_ptr>(), py::arg("output_tensors") = py::list(), py::arg("error") = static_cast>(nullptr)) .def( "output_tensors", &InferResponse::OutputTensors, py::return_value_policy::reference) .def("has_error", &InferResponse::HasError) .def("error", &InferResponse::Error); py::class_>( module, "InferenceResponseSender") .def( "send", &ResponseSender::Send, py::arg("response") = nullptr, py::arg("flags") = 0) .def("is_cancelled", &ResponseSender::IsCancelled); py::class_>( module, "ResponseIterator") .def(py::init&>()) .def( "__iter__", [](ResponseIterator& it) -> ResponseIterator& { it.Iter(); return it; }) .def("__next__", &ResponseIterator::Next); py::class_ logger(module, "Logger"); py::enum_(logger, "LogLevel") .value("INFO", LogLevel::kInfo) .value("WARNING", LogLevel::kWarning) .value("ERROR", LogLevel::kError) .value("VERBOSE", LogLevel::kVerbose) .export_values(); logger.def_static( "log", py::overload_cast(&Logger::Log), py::arg("message"), py::arg("level") = LogLevel::kInfo); logger.def_static("log_info", &Logger::LogInfo, py::arg("message")); logger.def_static("log_warn", &Logger::LogWarn, py::arg("message")); logger.def_static("log_error", &Logger::LogError, py::arg("message")); logger.def_static("log_verbose", &Logger::LogVerbose, py::arg("message")); py::class_>(module, "Metric") .def("increment", &Metric::SendIncrementRequest) .def("set", &Metric::SendSetValueRequest) .def("observe", &Metric::SendObserveRequest) .def("value", &Metric::SendGetValueRequest); py::enum_(module, "MetricKind") .value("COUNTER", MetricKind::kCounter) .value("GAUGE", MetricKind::kGauge) .value("HISTOGRAM", MetricKind::kHistogram) .export_values(); py::class_>( module, "MetricFamily") .def( py::init(&MetricFamily::CreateMetricFamily), py::arg("name").none(false), py::arg("description").none(false), py::arg("kind").none(false)) .def( "Metric", &MetricFamily::CreateMetric, py::arg("labels").none(true) = py::none(), py::arg("buckets").none(true) = py::none()); module.attr("MetricFamily").attr("COUNTER") = MetricKind::kCounter; module.attr("MetricFamily").attr("GAUGE") = MetricKind::kGauge; module.attr("MetricFamily").attr("HISTOGRAM") = MetricKind::kHistogram; module.def( "load_model", &LoadModel, py::arg("model_name").none(false), py::arg("config").none(false) = "", py::arg("files").none(true) = py::none()); module.def( "unload_model", &UnloadModel, py::arg("model_name").none(false), py::arg("unload_dependents").none(false) = false); module.def( "is_model_ready", &IsModelReady, py::arg("model_name").none(false), py::arg("model_version").none(false) = ""); // This function is not part of the public API for Python backend. This is // only used for internal callbacks. module.def( "async_event_future_done_callback", &AsyncEventFutureDoneCallback, py::arg("py_future").none(false)); // This class is not part of the public API for Python backend. This is only // used for internal testing purposes. py::class_(module, "SharedMemory") .def("free_memory", &SharedMemoryManager::FreeMemory); py::register_exception( module, "TritonModelException"); } void ModelContext::Init( const std::string& model_path, const std::string& runtime_modeldir, const std::string& triton_install_path, const std::string& model_version) { const char os_slash = std::filesystem::path::preferred_separator; type_ = ModelType::kDefault; if (runtime_modeldir != "DEFAULT") { // For python based backends, existence of `model.py` in the corresponding // backend folder happens on the core side, so we can omit this check here. python_model_path_ = runtime_modeldir + os_slash + "model.py"; type_ = ModelType::kBackend; } else { python_model_path_ = model_path; // Check if model file exists in this path. struct stat buffer; if (stat(python_model_path_.c_str(), &buffer) != 0) { throw PythonBackendException( ("Python model file not found in \'" + model_path + "\'")); } } model_dir_ = model_path.substr(0, model_path.find_last_of(os_slash)); python_backend_folder_ = triton_install_path; model_version_ = model_version; runtime_modeldir_ = runtime_modeldir; } void ModelContext::StubSetup(py::module& sys) { const char os_slash = std::filesystem::path::preferred_separator; std::string model_name = python_model_path_.substr(python_model_path_.find_last_of(os_slash) + 1); // Model name without the .py extension auto dotpy_pos = model_name.find_last_of(".py"); if (dotpy_pos == std::string::npos || dotpy_pos != model_name.size() - 1) { throw PythonBackendException( "Model name must end with '.py'. Model name is \"" + model_name + "\"."); } // The position of last character of the string that is searched for is // returned by 'find_last_of'. Need to manually adjust the position. std::string model_name_trimmed = model_name.substr(0, dotpy_pos - 2); if (type_ == ModelType::kDefault) { std::string model_path_parent = python_model_path_.substr(0, python_model_path_.find_last_of(os_slash)); std::string model_path_parent_parent = model_path_parent.substr(0, model_path_parent.find_last_of(os_slash)); sys.attr("path").attr("append")(model_path_parent); sys.attr("path").attr("append")(model_path_parent_parent); sys.attr("path").attr("append")(python_backend_folder_); sys = py::module_::import( (std::string(model_version_) + "." + model_name_trimmed).c_str()); } else { std::string model_path_parent = python_model_path_.substr(0, python_model_path_.find_last_of(os_slash)); std::string backend_model_dir(model_path_parent); sys.attr("path").attr("append")(backend_model_dir); sys.attr("path").attr("append")(python_backend_folder_); sys = py::module_::import(model_name_trimmed.c_str()); } } #ifdef _WIN32 bool ParentProcessActive(DWORD parent_id) { HANDLE parent = OpenProcess(PROCESS_ALL_ACCESS, FALSE, parent_id); DWORD exit_code; GetExitCodeProcess(parent, &exit_code); return (exit_code == STILL_ACTIVE); } #else bool ParentProcessActive(pid_t parent_id) { return (kill(parent_id, 0) == 0); } #endif extern "C" { int main(int argc, char** argv) { std::unique_ptr& logger = Logger::GetOrCreateInstance(); if (argc < 9) { LOG_INFO << "Expected 9 arguments, found " << argc << " arguments."; logger.reset(); exit(1); } signal(SIGINT, SignalHandler); signal(SIGTERM, SignalHandler); // Path to model std::string model_path = argv[1]; std::string shm_region_name = argv[2]; int64_t shm_default_size = std::stol(argv[3]); std::vector model_path_tokens; // Find the package name from model path. size_t prev = 0, pos = 0; const char os_slash = std::filesystem::path::preferred_separator; do { pos = model_path.find(os_slash, prev); if (pos == std::string::npos) pos = model_path.length(); std::string token = model_path.substr(prev, pos - prev); if (!token.empty()) model_path_tokens.push_back(token); prev = pos + 1; } while (pos < model_path.length() && prev < model_path.length()); if (model_path_tokens.size() < 2) { LOG_INFO << "Model path does not look right: " << model_path; logger.reset(); exit(1); } std::string model_version = model_path_tokens[model_path_tokens.size() - 2]; int64_t shm_growth_size = std::stol(argv[4]); std::string triton_install_path = argv[6]; std::string name = argv[8]; std::string runtime_modeldir = argv[9]; std::unique_ptr& stub = Stub::GetOrCreateInstance(); try { stub->Instantiate( shm_growth_size, shm_default_size, shm_region_name, model_path, model_version, argv[6] /* triton install path */, std::stoi(argv[7]) /* IPCControl handle */, name, runtime_modeldir); } catch (const PythonBackendException& pb_exception) { LOG_INFO << "Failed to preinitialize Python stub: " << pb_exception.what(); logger.reset(); exit(1); } // Start the Python Interpreter py::scoped_interpreter guard{}; #ifdef _WIN32 DWORD parent_pid = (DWORD)std::stoul(argv[5]); #else pid_t parent_pid = std::stoi(argv[5]); #endif std::atomic background_thread_running = {true}; std::thread background_thread = std::thread([&parent_pid, &background_thread_running, &stub, &logger] { // Send a dummy message after the stub process is launched to notify the // parent process that the health thread has started. std::unique_ptr ipc_message = IPCMessage::Create( stub->SharedMemory(), false /* inline_response */); stub->SendIPCMessage(ipc_message); while (background_thread_running) { // Every 300ms set the health variable to true. This variable is in // shared memory and will be set to false by the parent process. // The parent process expects that the stub process sets this // variable to true within 1 second. std::this_thread::sleep_for(std::chrono::milliseconds(300)); stub->UpdateHealth(); if (!ParentProcessActive(parent_pid)) { // When unhealthy, we should stop attempting to send // messages to the backend ASAP. if (stub->StubToParentServiceActive()) { stub->TerminateStubToParentQueueMonitor(); } if (stub->ParentToStubServiceActive()) { stub->TerminateParentToStubQueueMonitor(); } // Destroy Stub LOG_INFO << "Non-graceful termination detected. "; background_thread_running = false; non_graceful_exit = true; // Destroy stub and exit. logger.reset(); stub.reset(); exit(1); } } }); // The stub process will always keep listening for new notifications from the // parent process. After the notification is received the stub process will // run the appropriate command and wait for new notifications. bool finalize = false; while (true) { if (finalize) { stub->Finalize(); // Need check or may receive not joinable error if (stub->StubToParentServiceActive()) { stub->TerminateStubToParentQueueMonitor(); } if (stub->ParentToStubServiceActive()) { stub->TerminateParentToStubQueueMonitor(); } background_thread_running = false; background_thread.join(); break; } finalize = stub->RunCommand(); } // Stub must be destroyed before the py::scoped_interpreter goes out of // scope. The reason is that stub object has some attributes that are Python // objects. If the scoped_interpreter is destroyed before the stub object, // this process will no longer hold the GIL lock and destruction of the stub // will result in segfault. logger.reset(); stub.reset(); return 0; } } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_stub.h000066400000000000000000000372251527052371200250630ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include "infer_request.h" #include "infer_response.h" #include "ipc_message.h" #include "message_queue.h" #include "metric.h" #include "metric_family.h" #include "pb_cancel.h" #include "pb_log.h" #include "pb_response_iterator.h" #include "pb_utils.h" namespace bi = boost::interprocess; namespace py = pybind11; using namespace pybind11::literals; #ifndef TRITON_ENABLE_GPU using cudaStream_t = void*; #endif namespace triton { namespace backend { namespace python { #define LOG_IF_EXCEPTION(X) \ do { \ try { \ (X); \ } \ catch (const PythonBackendException& pb_exception) { \ LOG_INFO << pb_exception.what(); \ } \ } while (false) #define LOG_EXCEPTION(E) \ do { \ LOG_INFO << E.what(); \ } while (false) /// Macros that use current filename and line number. #define LOG_INFO LOG_FL(__FILE__, __LINE__, LogLevel::kInfo) #define LOG_WARN LOG_FL(__FILE__, __LINE__, LogLevel::kWarning) #define LOG_ERROR LOG_FL(__FILE__, __LINE__, LogLevel::kError) #define LOG_VERBOSE LOG_FL(__FILE__, __LINE__, LogLevel::kVerbose) class Logger { public: Logger() { backend_logging_active_ = false; }; ~Logger() { log_instance_.reset(); }; /// Python client log function static void Log(const std::string& message, LogLevel level = LogLevel::kInfo); /// Python client log info function static void LogInfo(const std::string& message); /// Python client warning function static void LogWarn(const std::string& message); /// Python client log error function static void LogError(const std::string& message); /// Python client log verbose function static void LogVerbose(const std::string& message); /// Internal log function void Log( const std::string& filename, uint32_t lineno, LogLevel level, const std::string& message); /// Log format helper function const std::string LeadingLogChar(const LogLevel& level); /// Set PYBE Logging Status void SetBackendLoggingActive(bool status); /// Get PYBE Logging Status bool BackendLoggingActive(); /// Singleton Getter Function static std::unique_ptr& GetOrCreateInstance(); DISALLOW_COPY_AND_ASSIGN(Logger); /// Flush the log. void Flush() { std::cerr << std::flush; } private: static std::unique_ptr log_instance_; bool backend_logging_active_; }; class LogMessage { public: /// Create a log message, stripping the path down to the filename only LogMessage(const char* file, int line, LogLevel level) : level_(level) { std::string path(file); const char os_slash = std::filesystem::path::preferred_separator; size_t pos = path.rfind(os_slash); if (pos != std::string::npos) { path = path.substr(pos + 1, std::string::npos); } file_ = path; line_ = static_cast(line); } /// Log message to console or send to backend (see Logger::Log for details) ~LogMessage() { Logger::GetOrCreateInstance()->Log(file_, line_, level_, stream_.str()); } std::stringstream& stream() { return stream_; } private: std::stringstream stream_; std::string file_; uint32_t line_; LogLevel level_; }; #define LOG_FL(FN, LN, LVL) LogMessage((char*)(FN), LN, LVL).stream() class ModelContext { public: // Scans and establishes path for serving the python model. void Init( const std::string& model_path, const std::string& platform, const std::string& triton_install_path, const std::string& model_version); // Sets up the python stub with appropriate paths. void StubSetup(py::module& sys); std::string& PythonModelPath() { return python_model_path_; } std::string& ModelDir() { return model_dir_; } private: std::string python_model_path_; std::string model_dir_; std::string model_version_; std::string python_backend_folder_; std::string runtime_modeldir_; // Triton supports python-based backends, // i.e. backends that provide common `model.py`, that can be re-used // between different models. `ModelType` helps to differentiate // between models running with c++ python backend (ModelType::kDefault) // and models running with python-based backend (ModelType::kBackend) // at the time of ModelContext::StubSetup to properly set up paths. enum ModelType { kDefault, kBackend }; ModelType type_; }; // The payload for the stub_to_parent message queue. This struct serves as a // wrapper for different types of messages so that they can be sent through the // same buffer. struct UtilsMessagePayload { UtilsMessagePayload( const PYTHONSTUB_CommandType& command_type, void* utils_message_ptr) : command_type(command_type), utils_message_ptr(utils_message_ptr) { } PYTHONSTUB_CommandType command_type; void* utils_message_ptr; }; class Stub { public: Stub() : stub_to_parent_thread_(false), parent_to_stub_thread_(false){}; static std::unique_ptr& GetOrCreateInstance(); /// Instantiate a new Python backend Stub. void Instantiate( int64_t shm_growth_size, int64_t shm_default_size, const std::string& shm_region_name, const std::string& model_path, const std::string& model_version, const std::string& triton_install_path, bi::managed_external_buffer::handle_t ipc_control_handle, const std::string& model_instance_name, const std::string& runtime_modeldir); /// Get the health of the stub process. bool& Health(); /// Get the shared memory manager. std::unique_ptr& SharedMemory(); /// Run a single command from the shared memory. bool RunCommand(); /// Setup for the stub process py::module StubSetup(); /// Return the path to the model py::str GetModelDir() { return model_context_.ModelDir(); } /// Set the model configuration for auto-complete void AutoCompleteModelConfig( bi::managed_external_buffer::handle_t string_handle, std::string* auto_complete_config); /// Initialize the user's Python code. void Initialize(bi::managed_external_buffer::handle_t map_handle); /// Send a message to the parent process. void SendIPCMessage(std::unique_ptr& ipc_message); /// Send a utils message to the parent process. void SendIPCUtilsMessage(std::unique_ptr& ipc_message); /// Receive a message from the parent process. std::unique_ptr PopMessage(); /// Update the health variable in the stub process. void UpdateHealth(); /// Finalize and terminate the stub process void Finalize(); /// Load all the requests from shared memory py::list LoadRequestsFromSharedMemory(RequestBatch* request_batch_shm_ptr); /// Execute a batch of requests. void ProcessRequests(RequestBatch* request_batch_shm_ptr); void ProcessReturnedResponses( py::list py_requests, py::object py_responses_obj, std::optional>& response_batch); void ProcessResponse(InferResponse* response); py::object GetAsyncEventLoop(); py::object RunCoroutine(py::object coroutine, bool in_background); void BackgroundFutureDone(const py::object& py_future); /// Get the memory manager message queue std::unique_ptr>& MemoryManagerQueue(); /// Get the shared memory pool std::unique_ptr& ShmPool() { return shm_pool_; } void ProcessBLSResponseDecoupled(std::unique_ptr& ipc_message); void LoadGPUBuffers(std::unique_ptr& ipc_message); bool IsDecoupled(); ~Stub(); /// Start stub to parent message handler process void LaunchStubToParentQueueMonitor(); /// End stub to parent message handler process void TerminateStubToParentQueueMonitor(); /// Add client log to queue void EnqueueLogRequest(std::unique_ptr& log_ptr); /// Thread process void ServiceStubToParentRequests(); /// Send client log to the python backend void SendLogMessage(std::unique_ptr& utils_msg_payload); /// Check if stub to parent message handler is running bool StubToParentServiceActive(); /// Start parent to stub message handler process void LaunchParentToStubQueueMonitor(); /// End parent to stub message handler process void TerminateParentToStubQueueMonitor(); /// Check if parent to stub message handler is running bool ParentToStubServiceActive(); /// Thread process void ParentToStubMQMonitor(); /// Get the ResponseIterator object associated with the infer response std::shared_ptr GetResponseIterator( std::shared_ptr infer_response); /// Send the id to the python backend for object cleanup void SendCleanupId( std::unique_ptr& utils_msg_payload, const PYTHONSTUB_CommandType& command_type); /// Add cleanup id to queue. This is used for cleaning up the infer_payload /// and the response factory for BLS decoupled response. void EnqueueCleanupId(void* id, const PYTHONSTUB_CommandType& command_type); /// Add request cancellation query to queue void EnqueueIsCancelled(PbCancel* pb_cancel); /// Send request cancellation query to python backend void SendIsCancelled(std::unique_ptr& utils_msg_payload); /// Is the stub initialized bool IsInitialized(); /// Is the stub in the finalize stage bool IsFinalizing(); /// Helper function to enqueue a utils message to the stub to parent message /// buffer void EnqueueUtilsMessage( std::unique_ptr utils_msg_payload); /// Send the message to the python backend. MessageType should be either // 'MetricFamilyMessage', 'MetricMessage' or 'ModelLoaderMessage'. template void SendMessage( AllocatedSharedMemory& msg_shm, PYTHONSTUB_CommandType command_type, bi::managed_external_buffer::handle_t handle); /// Helper function to prepare the message. MessageType should be either // 'MetricFamilyMessage', 'MetricMessage' or 'ModelLoaderMessage'. template void PrepareMessage(AllocatedSharedMemory& msg_shm); /// Helper function to retrieve a proxy stream for dlpack synchronization /// for provided device cudaStream_t GetProxyStream(const int& device_id); /// Get the CUDA memory pool address from the parent process. void GetCUDAMemoryPoolAddress(std::unique_ptr& ipc_message); private: bi::interprocess_mutex* stub_mutex_; bi::interprocess_condition* stub_cond_; bi::interprocess_mutex* parent_mutex_; bi::interprocess_condition* parent_cond_; bi::interprocess_mutex* health_mutex_; ModelContext model_context_; std::string name_; IPCControlShm* ipc_control_; std::unique_ptr shm_pool_; py::object model_instance_; py::object deserialize_bytes_; py::object serialize_bytes_; py::object async_event_loop_; py::object background_futures_; std::unique_ptr> stub_message_queue_; std::unique_ptr> parent_message_queue_; std::unique_ptr> stub_to_parent_mq_; std::unique_ptr> parent_to_stub_mq_; std::unique_ptr> memory_manager_message_queue_; bool initialized_; bool finalizing_; static std::unique_ptr stub_instance_; std::vector> gpu_tensors_; std::queue> stub_to_parent_buffer_; std::thread stub_to_parent_queue_monitor_; bool stub_to_parent_thread_; std::mutex stub_to_parent_message_mu_; std::condition_variable stub_to_parent_message_cv_; std::thread parent_to_stub_queue_monitor_; bool parent_to_stub_thread_; std::mutex response_iterator_map_mu_; std::unordered_map> response_iterator_map_; std::mutex dlpack_proxy_stream_pool_mu_; std::unordered_map dlpack_proxy_stream_pool_; }; template void Stub::PrepareMessage(AllocatedSharedMemory& msg_shm) { msg_shm = shm_pool_->Construct(); MessageType* msg = msg_shm.data_.get(); new (&(msg->mu)) bi::interprocess_mutex; new (&(msg->cv)) bi::interprocess_condition; msg->waiting_on_stub = false; msg->is_error_set = false; msg->has_error = false; } template void Stub::SendMessage( AllocatedSharedMemory& msg_shm, PYTHONSTUB_CommandType command_type, bi::managed_external_buffer::handle_t handle) { PrepareMessage(msg_shm); MessageType* msg = msg_shm.data_.get(); msg->message = handle; std::unique_ptr ipc_message = IPCMessage::Create(shm_pool_, false /* inline_response */); ipc_message->Command() = command_type; ipc_message->Args() = msg_shm.handle_; std::unique_lock guard{stub_to_parent_message_mu_}; { ScopedDefer _([&ipc_message, msg] { { bi::scoped_lock guard{msg->mu}; msg->waiting_on_stub = false; msg->cv.notify_all(); } }); { bi::scoped_lock guard{msg->mu}; SendIPCUtilsMessage(ipc_message); while (!msg->waiting_on_stub) { msg->cv.wait(guard); } } } if (msg->has_error) { if (msg->is_error_set) { std::unique_ptr pb_string = PbString::LoadFromSharedMemory(shm_pool_, msg->error); std::string err_message = std::string( "Failed to process the request for model '" + name_ + "', message: ") + pb_string->String(); throw PythonBackendException(err_message); } else { std::string err_message = std::string( "Failed to process the request for model '" + name_ + "'."); throw PythonBackendException(err_message); } } } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_stub_utils.cc000066400000000000000000000242301527052371200264310ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_stub_utils.h" #include "pb_utils.h" namespace triton { namespace backend { namespace python { TRITONSERVER_DataType numpy_to_triton_type(py::object data_type) { py::module np = py::module::import("numpy"); if (data_type.equal(np.attr("bool_"))) return TRITONSERVER_TYPE_BOOL; else if (data_type.equal(np.attr("uint8"))) return TRITONSERVER_TYPE_UINT8; else if (data_type.equal(np.attr("uint16"))) return TRITONSERVER_TYPE_UINT16; else if (data_type.equal(np.attr("uint32"))) return TRITONSERVER_TYPE_UINT32; else if (data_type.equal(np.attr("uint64"))) return TRITONSERVER_TYPE_UINT64; else if (data_type.equal(np.attr("int8"))) return TRITONSERVER_TYPE_INT8; else if (data_type.equal(np.attr("int16"))) return TRITONSERVER_TYPE_INT16; else if (data_type.equal(np.attr("int32"))) return TRITONSERVER_TYPE_INT32; else if (data_type.equal(np.attr("int64"))) return TRITONSERVER_TYPE_INT64; else if (data_type.equal(np.attr("float16"))) return TRITONSERVER_TYPE_FP16; else if (data_type.equal(np.attr("float32"))) return TRITONSERVER_TYPE_FP32; else if (data_type.equal(np.attr("float64"))) return TRITONSERVER_TYPE_FP64; else if ( data_type.equal(np.attr("object_")) || data_type.equal(np.attr("bytes_")) || data_type.attr("type").equal(np.attr("bytes_"))) return TRITONSERVER_TYPE_BYTES; throw PythonBackendException("NumPy dtype is not supported."); } py::object triton_to_numpy_type(TRITONSERVER_DataType data_type) { py::module np = py::module::import("numpy"); py::object np_type; switch (data_type) { case TRITONSERVER_TYPE_BOOL: np_type = np.attr("bool_"); break; case TRITONSERVER_TYPE_UINT8: np_type = np.attr("uint8"); break; case TRITONSERVER_TYPE_UINT16: np_type = np.attr("uint16"); break; case TRITONSERVER_TYPE_UINT32: np_type = np.attr("uint32"); break; case TRITONSERVER_TYPE_UINT64: np_type = np.attr("uint64"); break; case TRITONSERVER_TYPE_INT8: np_type = np.attr("int8"); break; case TRITONSERVER_TYPE_INT16: np_type = np.attr("int16"); break; case TRITONSERVER_TYPE_INT32: np_type = np.attr("int32"); break; case TRITONSERVER_TYPE_INT64: np_type = np.attr("int64"); break; case TRITONSERVER_TYPE_FP16: np_type = np.attr("float16"); break; case TRITONSERVER_TYPE_FP32: np_type = np.attr("float32"); break; case TRITONSERVER_TYPE_FP64: np_type = np.attr("float64"); break; case TRITONSERVER_TYPE_BYTES: np_type = np.attr("object_"); break; default: throw PythonBackendException( "Unsupported triton dtype" + std::to_string(static_cast(data_type))); } return np_type; } py::dtype triton_to_pybind_dtype(TRITONSERVER_DataType data_type) { py::dtype dtype_numpy; switch (data_type) { case TRITONSERVER_TYPE_BOOL: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_UINT8: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_UINT16: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_UINT32: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_UINT64: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_INT8: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_INT16: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_INT32: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_INT64: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_FP16: // Will be reinterpreted in the python code. dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_FP32: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_FP64: dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_BYTES: // Will be reinterpreted in the python code. dtype_numpy = py::dtype(py::format_descriptor::format()); break; case TRITONSERVER_TYPE_BF16: // NOTE: Currently skipping this call via `if (BF16)` check, but may // want to better handle this or set some default/invalid dtype. throw PythonBackendException("TYPE_BF16 not currently supported."); case TRITONSERVER_TYPE_INVALID: throw PythonBackendException("Dtype is invalid."); default: throw PythonBackendException("Unsupported triton dtype."); } return dtype_numpy; } DLDataType triton_to_dlpack_type(TRITONSERVER_DataType triton_dtype) { DLDataType dl_dtype; DLDataTypeCode dl_code; // Number of bits required for the data type. size_t dt_size = 0; dl_dtype.lanes = 1; switch (triton_dtype) { case TRITONSERVER_TYPE_BOOL: dl_code = DLDataTypeCode::kDLBool; dt_size = 8; break; case TRITONSERVER_TYPE_UINT8: dl_code = DLDataTypeCode::kDLUInt; dt_size = 8; break; case TRITONSERVER_TYPE_UINT16: dl_code = DLDataTypeCode::kDLUInt; dt_size = 16; break; case TRITONSERVER_TYPE_UINT32: dl_code = DLDataTypeCode::kDLUInt; dt_size = 32; break; case TRITONSERVER_TYPE_UINT64: dl_code = DLDataTypeCode::kDLUInt; dt_size = 64; break; case TRITONSERVER_TYPE_INT8: dl_code = DLDataTypeCode::kDLInt; dt_size = 8; break; case TRITONSERVER_TYPE_INT16: dl_code = DLDataTypeCode::kDLInt; dt_size = 16; break; case TRITONSERVER_TYPE_INT32: dl_code = DLDataTypeCode::kDLInt; dt_size = 32; break; case TRITONSERVER_TYPE_INT64: dl_code = DLDataTypeCode::kDLInt; dt_size = 64; break; case TRITONSERVER_TYPE_FP16: dl_code = DLDataTypeCode::kDLFloat; dt_size = 16; break; case TRITONSERVER_TYPE_FP32: dl_code = DLDataTypeCode::kDLFloat; dt_size = 32; break; case TRITONSERVER_TYPE_FP64: dl_code = DLDataTypeCode::kDLFloat; dt_size = 64; break; case TRITONSERVER_TYPE_BYTES: throw PythonBackendException( "TYPE_BYTES tensors cannot be converted to DLPack."); case TRITONSERVER_TYPE_BF16: dl_code = DLDataTypeCode::kDLBfloat; dt_size = 16; break; default: throw PythonBackendException( std::string("DType code \"") + std::to_string(static_cast(triton_dtype)) + "\" is not supported."); } dl_dtype.code = dl_code; dl_dtype.bits = dt_size; return dl_dtype; } TRITONSERVER_DataType dlpack_to_triton_type(const DLDataType& data_type) { if (data_type.lanes != 1) { // lanes != 1 is not supported in Python backend. return TRITONSERVER_TYPE_INVALID; } if (data_type.code == DLDataTypeCode::kDLFloat) { if (data_type.bits == 16) { return TRITONSERVER_TYPE_FP16; } else if (data_type.bits == 32) { return TRITONSERVER_TYPE_FP32; } else if (data_type.bits == 64) { return TRITONSERVER_TYPE_FP64; } } if (data_type.code == DLDataTypeCode::kDLInt) { if (data_type.bits == 8) { return TRITONSERVER_TYPE_INT8; } else if (data_type.bits == 16) { return TRITONSERVER_TYPE_INT16; } else if (data_type.bits == 32) { return TRITONSERVER_TYPE_INT32; } else if (data_type.bits == 64) { return TRITONSERVER_TYPE_INT64; } } if (data_type.code == DLDataTypeCode::kDLUInt) { if (data_type.bits == 8) { return TRITONSERVER_TYPE_UINT8; } else if (data_type.bits == 16) { return TRITONSERVER_TYPE_UINT16; } else if (data_type.bits == 32) { return TRITONSERVER_TYPE_UINT32; } else if (data_type.bits == 64) { return TRITONSERVER_TYPE_UINT64; } } if (data_type.code == DLDataTypeCode::kDLBool) { if (data_type.bits == 8) { return TRITONSERVER_TYPE_BOOL; } } if (data_type.code == DLDataTypeCode::kDLBfloat) { if (data_type.bits != 16) { throw PythonBackendException( "Expected BF16 tensor to have 16 bits, but had: " + std::to_string(data_type.bits)); } return TRITONSERVER_TYPE_BF16; } return TRITONSERVER_TYPE_INVALID; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_stub_utils.h000066400000000000000000000054401527052371200262750ustar00rootroot00000000000000// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include "triton/core/tritonserver.h" namespace py = pybind11; namespace triton { namespace backend { namespace python { /// Convert numpy dtype to triton dtype /// \param data_type numpy data type to be converted. /// \return equivalent triton dtype TRITONSERVER_DataType numpy_to_triton_type(py::object data_type); /// Convert triton dtype to numpy dtype /// \param data_type triton dtype to be converted. /// \return equivalent numpy data type. py::object triton_to_numpy_type(TRITONSERVER_DataType data_type); /// Convert triton dtype to dlpack dtype /// \param data_type triton dtype to be converted /// \return equivalent DLPack data type. DLDataType triton_to_dlpack_type(TRITONSERVER_DataType data_type); /// Convert dlpack type to triton type /// \param data_type triton dtype to be converted /// \return equivalent Triton dtype TRITONSERVER_DataType dlpack_to_triton_type(const DLDataType& data_type); /// Convert triton data to pybind data type. /// \param data_type triton dtype to be converted. /// \return equivalent pybind numpy dtype. py::dtype triton_to_pybind_dtype(TRITONSERVER_DataType data_type); }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_tensor.cc000066400000000000000000000520051527052371200255470ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU #ifdef TRITON_PB_STUB #include "pb_stub.h" #include "pb_stub_utils.h" namespace py = pybind11; #endif #include "pb_tensor.h" // WAR for undefined ssize_t on Windows: https://stackoverflow.com/a/35368387 #if defined(_MSC_VER) #include typedef SSIZE_T ssize_t; #endif namespace triton { namespace backend { namespace python { #ifdef TRITON_PB_STUB PbTensor::PbTensor(const std::string& name, py::array& numpy_array) : name_(name) { if (name == "") { throw PythonBackendException("Tensor name cannot be an empty string."); } dtype_ = numpy_to_triton_type(numpy_array.attr("dtype")); memory_type_ = TRITONSERVER_MEMORY_CPU; memory_type_id_ = 0; dl_managed_tensor_ = nullptr; bool is_contiguous = numpy_array.attr("data").attr("c_contiguous").cast(); if (!is_contiguous) { py::module numpy = py::module::import("numpy"); numpy_array = numpy.attr("ascontiguousarray")(numpy_array); } numpy_array_ = numpy_array; if (dtype_ == TRITONSERVER_TYPE_BYTES) { py::module triton_pb_utils = py::module::import("triton_python_backend_utils"); numpy_array_serialized_ = triton_pb_utils.attr("serialize_byte_tensor")(numpy_array); memory_ptr_ = numpy_array_serialized_.request().ptr; byte_size_ = numpy_array_serialized_.nbytes(); } else { memory_ptr_ = numpy_array_.request().ptr; byte_size_ = numpy_array_.nbytes(); } // Initialize tensor dimension size_t dims_count = numpy_array_.ndim(); const ssize_t* numpy_shape = numpy_array_.shape(); for (size_t i = 0; i < dims_count; i++) { dims_.push_back(numpy_shape[i]); } } PbTensor::PbTensor( const std::string& name, py::array& numpy_array, TRITONSERVER_DataType dtype) : name_(name) { if (name == "") { throw PythonBackendException("Tensor name cannot be an empty string."); } if (numpy_to_triton_type(numpy_array.attr("dtype")) != dtype) { numpy_array = numpy_array.attr("view")(triton_to_numpy_type(dtype)); } bool is_contiguous = numpy_array.attr("data").attr("c_contiguous").cast(); if (!is_contiguous) { py::module numpy = py::module::import("numpy"); numpy_array = numpy.attr("ascontiguousarray")(numpy_array); } numpy_array_ = numpy_array; if (dtype == TRITONSERVER_TYPE_BYTES) { py::module triton_pb_utils = py::module::import("triton_python_backend_utils"); numpy_array_serialized_ = triton_pb_utils.attr("serialize_byte_tensor")(numpy_array); memory_ptr_ = numpy_array_serialized_.request().ptr; byte_size_ = numpy_array_serialized_.nbytes(); } else { memory_ptr_ = numpy_array_.request().ptr; byte_size_ = numpy_array_.nbytes(); } memory_type_ = TRITONSERVER_MEMORY_CPU; dtype_ = dtype; // Initialize tensor dimension size_t dims_count = numpy_array_.ndim(); const ssize_t* numpy_shape = numpy_array_.shape(); for (size_t i = 0; i < dims_count; i++) { dims_.push_back(numpy_shape[i]); } memory_type_id_ = 0; dl_managed_tensor_ = nullptr; } #endif // TRITON_PB_STUB PbTensor::PbTensor( const std::string& name, const std::vector& dims, TRITONSERVER_DataType dtype, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* memory_ptr, uint64_t byte_size, DLManagedTensor* dl_managed_tensor) { if (name == "") { throw PythonBackendException("Tensor name cannot be an empty string."); } name_ = name; memory_ptr_ = memory_ptr; memory_type_ = memory_type; memory_type_id_ = memory_type_id; dtype_ = dtype; dims_ = dims; #ifdef TRITON_PB_STUB if (memory_type_ == TRITONSERVER_MEMORY_CPU || memory_type_ == TRITONSERVER_MEMORY_CPU_PINNED) { if (dtype == TRITONSERVER_TYPE_BF16) { // No native numpy representation for BF16. DLPack should be used instead. numpy_array_ = py::none(); } else if (dtype != TRITONSERVER_TYPE_BYTES) { py::object numpy_array = py::array(triton_to_pybind_dtype(dtype_), dims_, (void*)memory_ptr_); numpy_array_ = numpy_array.attr("view")(triton_to_numpy_type(dtype_)); } else { py::object numpy_array = py::array( triton_to_pybind_dtype(TRITONSERVER_TYPE_UINT8), {byte_size}, (void*)memory_ptr_); py::module triton_pb_utils = py::module::import("triton_python_backend_utils"); numpy_array_ = triton_pb_utils.attr("deserialize_bytes_tensor")(numpy_array) .attr("reshape")(dims); } } else { numpy_array_ = py::none(); } #endif byte_size_ = byte_size; dl_managed_tensor_ = dl_managed_tensor; } bool PbTensor::IsCPU() const { if (memory_type_ == TRITONSERVER_MEMORY_CPU || memory_type_ == TRITONSERVER_MEMORY_CPU_PINNED) { return true; } else { return false; } } TRITONSERVER_MemoryType PbTensor::MemoryType() const { return memory_type_; } int64_t PbTensor::MemoryTypeId() const { return memory_type_id_; } uint64_t PbTensor::ByteSize() const { return byte_size_; } const std::vector& PbTensor::Dims() const { return dims_; } void PbTensor::SetMemory(std::unique_ptr&& memory) { pb_memory_ = std::move(memory); memory_type_ = pb_memory_->MemoryType(); memory_type_id_ = pb_memory_->MemoryTypeId(); byte_size_ = pb_memory_->ByteSize(); memory_ptr_ = pb_memory_->DataPtr(); } #ifdef TRITON_PB_STUB void delete_unused_dltensor(PyObject* dlp) { if (PyCapsule_IsValid(dlp, "dltensor")) { DLManagedTensor* dl_managed_tensor = static_cast(PyCapsule_GetPointer(dlp, "dltensor")); dl_managed_tensor->deleter(dl_managed_tensor); } } std::shared_ptr PbTensor::FromNumpy(const std::string& name, py::array& numpy_array) { return std::make_shared(name, numpy_array); } DLDeviceType PbTensor::DeviceType() { DLDeviceType device_type{}; switch (memory_type_) { case TRITONSERVER_MEMORY_GPU: device_type = DLDeviceType::kDLCUDA; break; case TRITONSERVER_MEMORY_CPU: device_type = DLDeviceType::kDLCPU; break; case TRITONSERVER_MEMORY_CPU_PINNED: device_type = DLDeviceType::kDLCUDAHost; break; } return device_type; } py::capsule PbTensor::DLPack(const py::object& stream) { // Here external tensor requests PbTensor's `__dlpack__` method to provide // a PyCapsule. By the design of PbTensor, in a GPU case no pending work // is scheduled to work with PbTensor's data and we can simply pass // the capsule without a synchronization. return this->ToDLPack(); } py::capsule PbTensor::ToDLPack() { if (dtype_ == TRITONSERVER_TYPE_BYTES) { throw PythonBackendException( "DLPack does not have support for string tensors."); } DLManagedTensor* dlpack_tensor = new DLManagedTensor; dlpack_tensor->dl_tensor.ndim = dims_.size(); dlpack_tensor->dl_tensor.byte_offset = 0; dlpack_tensor->dl_tensor.data = memory_ptr_; dlpack_tensor->dl_tensor.shape = &dims_[0]; dlpack_tensor->dl_tensor.strides = nullptr; dlpack_tensor->manager_ctx = this; dlpack_tensor->deleter = [](DLManagedTensor* m) { // We need to acquire GIL since the framework that deleted the dlpack tensor // may not have acquired GIL when calling this function. py::gil_scoped_acquire gil; if (m->manager_ctx == nullptr) { return; } PbTensor* tensor = reinterpret_cast(m->manager_ctx); py::handle tensor_handle = py::cast(tensor); tensor_handle.dec_ref(); free(m); }; PbTensor* tensor = reinterpret_cast(this); py::handle tensor_handle = py::cast(tensor); // Increase the reference count by one to make sure that the DLPack // representation doesn't become invalid when the tensor object goes out of // scope. tensor_handle.inc_ref(); dlpack_tensor->dl_tensor.device.device_id = memory_type_id_; dlpack_tensor->dl_tensor.device.device_type = this->DeviceType(); dlpack_tensor->dl_tensor.dtype = triton_to_dlpack_type(dtype_); return py::capsule( static_cast(dlpack_tensor), "dltensor", &delete_unused_dltensor); } std::pair PbTensor::DLPackDevice() { return std::pair(this->DeviceType(), memory_type_id_); } #endif // TRITON_PB_STUB void PbTensor::DeleteDLPack() { if (dl_managed_tensor_ != nullptr) { dl_managed_tensor_->deleter(dl_managed_tensor_); dl_managed_tensor_ = nullptr; } } std::unique_ptr& PbTensor::Memory() { return pb_memory_; } #ifdef TRITON_PB_STUB std::shared_ptr PbTensor::FromDLPack(const std::string& name, const py::object& tensor) { if (name == "") { throw PythonBackendException("Tensor name cannot be an empty string."); } if (py::isinstance(tensor)) { return FromDLPackCapsule(name, tensor); } if (!py::hasattr(tensor, "__dlpack__") || !py::hasattr(tensor, "__dlpack_device__")) { throw PythonBackendException( "Provided tensor is not supported. Tensor must be a DLPack capsule \ or have `__dlpack__` and `__dlpack_device__` attributes"); } auto capsule_device_info = tensor.attr("__dlpack_device__")().cast>(); if (capsule_device_info.first == DLDeviceType::kDLCUDA) { #ifdef TRITON_ENABLE_GPU int current_device; cudaError_t err = cudaGetDevice(¤t_device); std::unique_ptr& stub = Stub::GetOrCreateInstance(); if (err != cudaSuccess) { throw PythonBackendException("Failed to get current CUDA device id."); } ScopedSetDevice scoped_set_device(capsule_device_info.second); bool overridden = (current_device != capsule_device_info.second); cudaStream_t proxy_stream = stub->GetProxyStream(current_device); // Array API requirements for the stream argument: // stream = 1 the legacy default stream (in this case should // synchronize on CUDA stream 0) // For CPU, `stream=None` is the only accepted argument // according to array API. For GPU, when `stream=None` producer // must assume the legacy default stream. Reference: // https://data-apis.org/array-api/latest/API_specification/generated/array_api.array.__dlpack__.html auto ptr_to_tensor = FromDLPackCapsule( name, tensor.attr("__dlpack__")( py::arg("stream") = py::int_(reinterpret_cast(proxy_stream)))); // In case there is a pending job on the data, where this capsule // is pointing to, we need to wait for it to finish before returning // capsule. // We synchronize on the proxy stream explicitly since that what we // pass to external tensor's `__dlpack__` method. err = cudaStreamSynchronize(proxy_stream); if (err != cudaSuccess) { throw PythonBackendException( "Failed to synchronize CUDA device with id " + std::to_string( overridden ? capsule_device_info.second : current_device)); } return ptr_to_tensor; #else throw PythonBackendException( "DLPack capsule passed pointer to memory allocated on GPU device, \ when GPU is not available"); #endif } else if ( capsule_device_info.first != DLDeviceType::kDLCPU && capsule_device_info.first != DLDeviceType::kDLCUDAHost) { throw PythonBackendException( "DLDevice type " + std::to_string(capsule_device_info.first) + " is not support by Python backend."); } // If data is located on a CPU, `stream=None` is the only accepted argument // according to array API. // Reference: // https://data-apis.org/array-api/latest/API_specification/generated/array_api.array.__dlpack__.html return FromDLPackCapsule( name, tensor.attr("__dlpack__")(py::arg("stream") = py::none())); } std::shared_ptr PbTensor::FromDLPackCapsule( const std::string& name, const py::capsule& dlpack_tensor) { DLManagedTensor* dl_managed_tensor = static_cast(dlpack_tensor.get_pointer()); void* memory_ptr = dl_managed_tensor->dl_tensor.data; memory_ptr = reinterpret_cast(memory_ptr) + dl_managed_tensor->dl_tensor.byte_offset; int64_t* strides = dl_managed_tensor->dl_tensor.strides; int ndim = dl_managed_tensor->dl_tensor.ndim; std::vector dims( dl_managed_tensor->dl_tensor.shape, dl_managed_tensor->dl_tensor.shape + ndim); // Check if the input is contiguous and in C order if (strides != nullptr) { int64_t calculated_stride{1}; bool is_contiguous_c_order = true; for (size_t i = 1; i < dims.size(); i++) { if (dims[ndim - i] != 1) { if (strides[ndim - i] != calculated_stride) { is_contiguous_c_order = false; break; } calculated_stride *= dims[ndim - i]; } } if (!is_contiguous_c_order) { throw PythonBackendException( "DLPack tensor is not contiguous. Only contiguous DLPack " "tensors that are stored in C-Order are supported."); } } TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; switch (dl_managed_tensor->dl_tensor.device.device_type) { case DLDeviceType::kDLCUDA: memory_type = TRITONSERVER_MEMORY_GPU; memory_type_id = dl_managed_tensor->dl_tensor.device.device_id; break; case DLDeviceType::kDLCPU: memory_type = TRITONSERVER_MEMORY_CPU; memory_type_id = 0; break; case DLDeviceType::kDLCUDAHost: memory_type = TRITONSERVER_MEMORY_CPU; memory_type_id = 0; break; default: throw PythonBackendException( "DLDevice type " + std::to_string(dl_managed_tensor->dl_tensor.device.device_type) + " is not support by Python backend."); break; } TRITONSERVER_DataType dtype = dlpack_to_triton_type(dl_managed_tensor->dl_tensor.dtype); // Calculate tensor size. uint64_t byte_size = 1; for (auto& dim : dims) { byte_size *= dim; } byte_size *= (dl_managed_tensor->dl_tensor.dtype.bits + 7) / 8; PyCapsule_SetName(dlpack_tensor.ptr(), "used_dlpack"); return std::make_unique( name, dims, dtype, memory_type, memory_type_id, memory_ptr, byte_size, dl_managed_tensor); } #endif // TRITON_PB_STUB PbTensor::~PbTensor() noexcept(false) { pb_memory_.reset(); DeleteDLPack(); } const std::string& PbTensor::Name() const { return name_; } #ifdef TRITON_PB_STUB const py::array* PbTensor::AsNumpy() const { if (!IsCPU()) { throw PythonBackendException( "Tensor is stored in GPU and cannot be converted to NumPy."); } if (dtype_ == TRITONSERVER_TYPE_BF16) { throw PythonBackendException( "Tensor dtype is BF16 and cannot be converted to NumPy. Use " "to_dlpack() and from_dlpack() instead."); } return &numpy_array_; } #endif // TRITON_PB_STUB void PbTensor::SaveToSharedMemory( std::unique_ptr& shm_pool, bool copy_gpu) { if (!tensor_shm_.data_) { uint64_t byte_size; if (!pb_memory_) { byte_size = sizeof(TensorShm) + sizeof(int64_t) * dims_.size() + PbString::ShmStructSize(name_) + PbMemory::ShmStructSize(memory_type_, byte_size_); } else { byte_size = sizeof(TensorShm) + sizeof(int64_t) * dims_.size() + PbString::ShmStructSize(name_); } tensor_shm_ = shm_pool->Construct(byte_size); tensor_shm_ptr_ = reinterpret_cast(tensor_shm_.data_.get()); tensor_shm_ptr_->dtype = dtype_; tensor_shm_ptr_->dims_count = dims_.size(); shm_handle_ = tensor_shm_.handle_; dims_shm_ptr_ = reinterpret_cast( reinterpret_cast(tensor_shm_ptr_) + sizeof(TensorShm)); // Write the dimensions data to shared memory. for (size_t i = 0; i < dims_.size(); i++) { dims_shm_ptr_[i] = dims_[i]; } std::size_t name_offset = sizeof(TensorShm) + sizeof(int64_t) * dims_.size(); name_shm_ = PbString::Create( name_, reinterpret_cast(tensor_shm_ptr_) + name_offset, shm_handle_ + name_offset); std::size_t pb_memory_offset = name_offset + PbString::ShmStructSize(name_); if (!pb_memory_) { pb_memory_ = PbMemory::Create( shm_pool, memory_type_, memory_type_id_, byte_size_, reinterpret_cast(memory_ptr_), reinterpret_cast(tensor_shm_ptr_) + pb_memory_offset, shm_handle_ + pb_memory_offset, copy_gpu); tensor_shm_ptr_->memory = 0; } else { tensor_shm_ptr_->memory = pb_memory_->ShmHandle(); } memory_ptr_ = pb_memory_->DataPtr(); } } std::unique_ptr PbTensor::LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t tensor_handle, bool open_cuda_handle) { AllocatedSharedMemory tensor_shm = shm_pool->Load(tensor_handle); TensorShm* tensor_shm_ptr = reinterpret_cast(tensor_shm.data_.get()); size_t name_offset = sizeof(TensorShm) + sizeof(int64_t) * tensor_shm_ptr->dims_count; std::unique_ptr name_shm = PbString::LoadFromSharedMemory( tensor_handle + name_offset, tensor_shm.data_.get() + name_offset); std::unique_ptr pb_memory; if (tensor_shm_ptr->memory == 0) { std::size_t pb_memory_offset = name_offset + name_shm->Size(); pb_memory = PbMemory::LoadFromSharedMemory( shm_pool, pb_memory_offset, tensor_shm.data_.get() + pb_memory_offset, open_cuda_handle); } else { pb_memory = PbMemory::LoadFromSharedMemory( shm_pool, tensor_shm_ptr->memory, open_cuda_handle); } return std::unique_ptr( new PbTensor(tensor_shm, name_shm, pb_memory)); } TRITONSERVER_DataType PbTensor::TritonDtype() const { return dtype_; } void* PbTensor::DataPtr() { return memory_ptr_; } bi::managed_external_buffer::handle_t PbTensor::ShmHandle() { return shm_handle_; } PbTensor::PbTensor( AllocatedSharedMemory& tensor_shm, std::unique_ptr& name_shm, std::unique_ptr& pb_memory) : tensor_shm_(std::move(tensor_shm)), name_shm_(std::move(name_shm)), pb_memory_(std::move(pb_memory)) { tensor_shm_ptr_ = reinterpret_cast(tensor_shm_.data_.get()); dims_shm_ptr_ = reinterpret_cast( reinterpret_cast(tensor_shm_ptr_) + sizeof(TensorShm)); name_ = name_shm_->String(); dims_ = std::vector( dims_shm_ptr_, dims_shm_ptr_ + tensor_shm_ptr_->dims_count); dtype_ = tensor_shm_ptr_->dtype; dl_managed_tensor_ = nullptr; byte_size_ = pb_memory_->ByteSize(); memory_ptr_ = pb_memory_->DataPtr(); memory_type_ = pb_memory_->MemoryType(); memory_type_id_ = pb_memory_->MemoryTypeId(); shm_handle_ = tensor_shm_.handle_; #ifdef TRITON_PB_STUB if (memory_type_ == TRITONSERVER_MEMORY_CPU || memory_type_ == TRITONSERVER_MEMORY_CPU_PINNED) { if (dtype_ == TRITONSERVER_TYPE_BF16) { // No native numpy representation for BF16. DLPack should be used instead. numpy_array_ = py::none(); } else if (dtype_ != TRITONSERVER_TYPE_BYTES) { py::object numpy_array = py::array(triton_to_pybind_dtype(dtype_), dims_, (void*)memory_ptr_); numpy_array_ = numpy_array.attr("view")(triton_to_numpy_type(dtype_)); } else { py::object numpy_array = py::array( triton_to_pybind_dtype(TRITONSERVER_TYPE_UINT8), {byte_size_}, (void*)memory_ptr_); py::module triton_pb_utils = py::module::import("triton_python_backend_utils"); numpy_array_ = triton_pb_utils.attr("deserialize_bytes_tensor")(numpy_array) .attr("reshape")(dims_); } } else { numpy_array_ = py::none(); } #endif } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_tensor.h000066400000000000000000000224441527052371200254150ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU #include #ifdef TRITON_PB_STUB #include #include #include namespace py = pybind11; #endif #include #include #include "pb_memory.h" #include "pb_string.h" #include "pb_utils.h" #include "triton/backend/backend_common.h" #include "triton/backend/backend_memory.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { namespace python { // // Represents a Tensor object in shared memory. // struct TensorShm { // Handle for the pointer data in shared memory. bi::managed_external_buffer::handle_t memory; TRITONSERVER_DataType dtype; size_t dims_count; }; // PbTensor class is the representation of Triton tensors inside Python backend. class PbTensor { public: #ifdef TRITON_PB_STUB /// Create a PbTensor using a numpy array /// \param name The name of the tensor /// \param numpy_array Numpy array to use for the initialization of the tensor PbTensor(const std::string& name, py::array& numpy_array); /// Create a PbTensor using a numpy array. This constructor is used for types /// that are not natively available in C++ such as float16. This constructor /// will fix the type of the NumPy array to match the Triton dtype. /// \param name The name of the tensor /// \param numpy_array Numpy array to use for the initialization of the tensor /// \param dtype The triton dtype PbTensor( const std::string& name, py::array& numpy_array, TRITONSERVER_DataType dtype); #endif /// Create a PbTensor from raw pointer. This constructor is used for /// interfacing with DLPack tensors. /// \param name The name of the tensor /// \param dims Tensor dimensions /// \param dtype Triton dtype /// \param memory_type The memory type of the tensor /// \param memory_type_id The memory type_id of the tensor /// \param memory_ptr Pointer to the location of the data. Data must be /// contiguous and in C-order. /// \param byte_size Total number of bytes that the tensor uses. /// \param shm_handle The shared memory handle of pointer if it is stored in /// shared memory. PbTensor( const std::string& name, const std::vector& dims, TRITONSERVER_DataType dtype, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* memory_ptr, uint64_t byte_size, DLManagedTensor* dl_managed_tensor = nullptr); /// This constructor is used when loading the tensor from shared memory. /// \param tensor_shm The name of the tensor /// \param dims_shm Tensor dimensions /// \param pb_string Triton dtype PbTensor( AllocatedSharedMemory& tensor_shm, std::unique_ptr& name_shm, std::unique_ptr& pb_memory); // Copying tensor objects is not allowed. DISALLOW_COPY_AND_ASSIGN(PbTensor); #ifdef TRITON_PB_STUB /// Construct a Python backend tensor from an /// external tensor. /// \param dlpack source dlpack tensor /// \param name name of the tensor static std::shared_ptr FromDLPack( const std::string& name, const py::object& dlpack); /// Construct a Python backend tensor using a DLPack /// capsule. static std::shared_ptr FromDLPackCapsule( const std::string& name, const py::capsule& dlpack); /// Construct a Python backend tensor using a NumPy object. /// \param numpy_array Numpy array /// \param name name of the tensor static std::shared_ptr FromNumpy( const std::string& name, py::array& numpy_array); /// Get device type in DLPack format. DLDeviceType DeviceType(); /// Exports tensor for consumption by `from_dlpack()` as a DLPack capsule. /// \param stream a Python integer representing a pointer to a stream, /// on devices that support streams /// \return Capsule object containing pointer to a DLPack object. py::capsule DLPack(const py::object& stream); /// Get a PyCapsule object containing the DLPack representation of the tensor. /// \return Capsule object containing pointer to a DLPack object. py::capsule ToDLPack(); /// Returns device type and device ID. /// Meant for use within `from_dlpack()`. /// \return a pair (device_type, device_id). std::pair DLPackDevice(); #endif /// Get the name of the tensor /// \return name of the tensor. const std::string& Name() const; /// Set the name of the tensor /// \param name Name of the tensor. void SetName(const std::string& name); /// Get the shared memory handle corresponding to this tensor /// \return returns the shared memory handle. bi::managed_external_buffer::handle_t ShmHandle(); /// Load the tensor object from shared memory. /// \param shm_pool The shared memory manager object /// \param tensor_handle The handle of the object in shared memory. /// \param open_cuda_handle If the tensor is in GPU, setting this option to /// true will call cudaIpcOpenMemHandle on it. In the main process this option /// should be set to false because we never want to call cudaIpcOpenMemHandle /// in the main process. /// \return returns the tensor loaded from shared memory. static std::unique_ptr LoadFromSharedMemory( std::unique_ptr& shm_pool, bi::managed_external_buffer::handle_t tensor_handle, bool open_cuda_handle); #ifdef TRITON_PB_STUB /// Get NumPy representation of the tensor. /// \throw If the tensor is stored in GPU, an exception is thrown /// \return NumPy representation of the Tensor const py::array* AsNumpy() const; #endif /// Save tensor inside shared memory. void SaveToSharedMemory( std::unique_ptr& shm_pool, bool copy_gpu); /// Get the triton dtype /// \return Triton dtype TRITONSERVER_DataType TritonDtype() const; /// Get the data ptr /// \return Get the raw pointer. void* DataPtr(); /// This function will be automatically called by the stub when the tensor is /// no longer required. void DeleteDLPack(); /// Tells whether the Tensor is stored in CPU or not. /// \return A boolean value indicating whether the tensor is stored in CPU /// or not. bool IsCPU() const; /// Get the total byte size of the tensor. uint64_t ByteSize() const; /// Get the triton memory type of the Tensor. /// \return the memory type of the tensor. TRITONSERVER_MemoryType MemoryType() const; /// Get a mutable reference to the MemoryType. /// \return the pointer to the memory type of the tensor. TRITONSERVER_MemoryType* MutableMemoryType(); /// Get the triton memory type of the Tensor. /// \return the memory type of the tensor. int64_t MemoryTypeId() const; /// Get the dimensions of the tensor /// \return A vector containing the tensor dimensions. const std::vector& Dims() const; /// Get the underlying memory std::unique_ptr& Memory(); /// Set the underlying memory void SetMemory(std::unique_ptr&& memory); PbTensor(); /// Destructor ~PbTensor() noexcept(false); private: std::string name_; #ifdef TRITON_PB_STUB py::array numpy_array_; // Storing the serialized version of the numpy array py::array numpy_array_serialized_; #endif TRITONSERVER_DataType dtype_; void* memory_ptr_; int64_t memory_type_id_; std::vector dims_; TRITONSERVER_MemoryType memory_type_; uint64_t byte_size_; DLManagedTensor* dl_managed_tensor_; bi::managed_external_buffer::handle_t shm_handle_; AllocatedSharedMemory tensor_shm_; TensorShm* tensor_shm_ptr_; int64_t* dims_shm_ptr_; std::unique_ptr name_shm_; // The pointer is null when the object is not stored in shared memory. std::unique_ptr pb_memory_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_utils.cc000066400000000000000000000224131527052371200253750ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "pb_utils.h" #ifdef _WIN32 #include #include #else #include #endif #ifdef TRITON_ENABLE_GPU #include #include #endif namespace triton { namespace backend { namespace python { #ifdef TRITON_ENABLE_GPU CUDAHandler::CUDAHandler() { dl_open_handle_ = LoadSharedObject("libcuda.so"); // If libcuda.so is successfully opened, it must be able to find // "cuPointerGetAttribute", "cuGetErrorString", and // "cuDevicePrimaryCtxGetState" symbols. if (dl_open_handle_ != nullptr) { void* cu_pointer_get_attribute_fn = LocateSymbol("cuPointerGetAttribute"); if (cu_pointer_get_attribute_fn == nullptr) { throw PythonBackendException( std::string("Failed to locate 'cuPointerGetAttribute'. Error: ") + LocateSymbolError()); } *((void**)&cu_pointer_get_attribute_fn_) = cu_pointer_get_attribute_fn; void* cu_get_error_string_fn = LocateSymbol("cuGetErrorString"); if (cu_get_error_string_fn == nullptr) { throw PythonBackendException( std::string("Failed to locate 'cuGetErrorString'. Error: ") + LocateSymbolError()); } *((void**)&cu_get_error_string_fn_) = cu_get_error_string_fn; void* cu_init_fn = LocateSymbol("cuInit"); if (cu_init_fn == nullptr) { throw PythonBackendException( std::string("Failed to locate 'cuInit'. Error: ") + LocateSymbolError()); } *((void**)&cu_init_fn_) = cu_init_fn; void* cu_device_primary_ctx_get_state_fn = LocateSymbol("cuDevicePrimaryCtxGetState"); if (cu_device_primary_ctx_get_state_fn == nullptr) { throw PythonBackendException( std::string( "Failed to locate 'cuDevicePrimaryCtxGetState'. Error: ") + LocateSymbolError()); } *((void**)&cu_device_primary_ctx_get_state_fn_) = cu_device_primary_ctx_get_state_fn; // Initialize the driver API. CUresult cuda_err = (*cu_init_fn_)(0 /* flags */); if (cuda_err != CUDA_SUCCESS) { const char* error_string; (*cu_get_error_string_fn_)(cuda_err, &error_string); error_str_ = std::string("failed to call cuInit: ") + error_string; CloseLibrary(); dl_open_handle_ = nullptr; } } } void CUDAHandler::PointerGetAttribute( CUdeviceptr* start_address, CUpointer_attribute attribute, CUdeviceptr dev_ptr) { CUresult cuda_err = (*cu_pointer_get_attribute_fn_)(start_address, attribute, dev_ptr); if (cuda_err != CUDA_SUCCESS) { const char* error_string; (*cu_get_error_string_fn_)(cuda_err, &error_string); throw PythonBackendException( std::string( "failed to get cuda pointer device attribute: " + std::string(error_string)) .c_str()); } } bool CUDAHandler::IsAvailable() { return dl_open_handle_ != nullptr; } void CUDAHandler::OpenCudaHandle( int64_t memory_type_id, cudaIpcMemHandle_t* cuda_mem_handle, void** data_ptr) { std::lock_guard guard{mu_}; ScopedSetDevice scoped_set_device(memory_type_id); cudaError_t err = cudaIpcOpenMemHandle( data_ptr, *cuda_mem_handle, cudaIpcMemLazyEnablePeerAccess); if (err != cudaSuccess) { throw PythonBackendException( std::string("Failed to open the cudaIpcHandle. error: ") + cudaGetErrorString(err)); } } void CUDAHandler::CloseCudaHandle(int64_t memory_type_id, void* data_ptr) { std::lock_guard guard{mu_}; int current_device; // Save the previous device cudaError_t err = cudaGetDevice(¤t_device); if (err != cudaSuccess) { throw PythonBackendException( std::string("Failed to get the current CUDA device. error: ") + cudaGetErrorString(err)); } // Restore the previous device before returning from the function. ScopedSetDevice scoped_set_device(memory_type_id); err = cudaIpcCloseMemHandle(data_ptr); if (err != cudaSuccess) { throw PythonBackendException( std::string("Failed to close the cudaIpcHandle. error: ") + cudaGetErrorString(err)); } } bool CUDAHandler::HasPrimaryContext(int device) { unsigned int ctx_flags; int ctx_is_active = 0; CUresult cuda_err = (*cu_device_primary_ctx_get_state_fn_)( device, &ctx_flags, &ctx_is_active); if (cuda_err != CUDA_SUCCESS) { const char* error_string; (*cu_get_error_string_fn_)(cuda_err, &error_string); throw PythonBackendException( std::string( "failed to get primary context state: " + std::string(error_string)) .c_str()); } return ctx_is_active == 1; } void CUDAHandler::MaybeSetDevice(int device) { if (HasPrimaryContext(device)) { cudaError_t err = cudaSetDevice(device); if (err != cudaSuccess) { throw PythonBackendException( std::string("Failed to set the CUDA device to ") + std::to_string(device) + ". error: " + cudaGetErrorString(err)); } } } CUDAHandler::~CUDAHandler() noexcept(false) { if (dl_open_handle_ != nullptr) { CloseLibrary(); } } void* CUDAHandler::LoadSharedObject(const char* filename) { #ifdef _WIN32 // NOTE: 'nvcuda.dll' is a placeholder library. Apparently, this should be the // equivalent library for Windows, but need to verify. return LoadLibraryA("nvcuda.dll"); #else return dlopen("libcuda.so", RTLD_LAZY); #endif } void* CUDAHandler::LocateSymbol(const char* symbol) { #ifdef _WIN32 return GetProcAddress(static_cast(dl_open_handle_), symbol); #else return dlsym(dl_open_handle_, symbol); #endif } std::string CUDAHandler::LocateSymbolError() { #ifdef _WIN32 return std::to_string(GetLastError()); #else return dlerror(); #endif } void CUDAHandler::CloseLibrary() { bool successful = true; #ifdef _WIN32 successful = (FreeLibrary(static_cast(dl_open_handle_)) != 0); #else successful = (dlclose(dl_open_handle_) == 0); #endif if (!successful) { throw PythonBackendException("Failed to close the cuda library handle."); } } ScopedSetDevice::ScopedSetDevice(int device) { device_ = device; THROW_IF_CUDA_ERROR(cudaGetDevice(¤t_device_)); if (current_device_ != device_) { THROW_IF_CUDA_ERROR(cudaSetDevice(device_)); } } ScopedSetDevice::~ScopedSetDevice() { if (current_device_ != device_) { CUDAHandler& cuda_handler = CUDAHandler::getInstance(); cuda_handler.MaybeSetDevice(current_device_); } } bool IsUsingCUDAPool( std::unique_ptr& cuda_pool, int64_t memory_type_id, void* data) { CUDAHandler& cuda_api = CUDAHandler::getInstance(); CUdeviceptr cuda_pool_address = 0; cuda_api.PointerGetAttribute( &cuda_pool_address, CU_POINTER_ATTRIBUTE_RANGE_START_ADDR, reinterpret_cast(data)); return ( cuda_pool->CUDAPoolAddress(memory_type_id) == reinterpret_cast(cuda_pool_address)); } #endif // TRITON_ENABLE_GPU // FIXME: [DLIS-6078]: We should not need this function. However, some paths are // being retrieved from core that are not platform-agnostic. void SanitizePath(std::string& path) { std::replace(path.begin(), path.end(), '/', '\\'); } #ifndef TRITON_PB_STUB std::shared_ptr WrapTritonErrorInSharedPtr(TRITONSERVER_Error* error) { std::shared_ptr response_error( new TRITONSERVER_Error*, [](TRITONSERVER_Error** error) { if (error != nullptr && *error != nullptr) { TRITONSERVER_ErrorDelete(*error); } if (error != nullptr) { delete error; } }); *response_error = error; return response_error; } #endif // NOT TRITON_PB_STUB std::string GenerateUUID() { static boost::uuids::random_generator generator; boost::uuids::uuid uuid = generator(); return boost::uuids::to_string(uuid); } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/pb_utils.h000066400000000000000000000303171527052371200252410ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #ifdef TRITON_ENABLE_GPU #include #endif // TRITON_ENABLE_GPU #include #include #include #include #include #include #include #include #include #include #include #include "pb_exception.h" #include "shm_manager.h" #include "triton/backend/backend_common.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { namespace python { namespace bi = boost::interprocess; #define STUB_SET_RESPONSE_ERROR_IF_ERROR(SHM_POOL, RESPONSE, R, X) \ do { \ try { \ (X); \ } \ catch (cont PythonBackendException & pb_exception) { \ bi::managed_external_buffer::handle_t string_handle__; \ try { \ SaveStringToSharedMemory( \ SHM_POOL, string_handle__, pb_exception.what()); \ RESPONSE->has_error = true; \ RESPONSE->error = string_handle__; \ if (R) \ return; \ } \ catch (cont PythonBackendException & pb2_exception) { \ printf(TRITONSERVER_ErrorMessage(pb_exception.what())); \ printf( \ TRITONSERVER_LOG_ERROR, \ TRITONSERVER_ErrorMessage(pb2_exception.what())); \ } \ } \ while (false) #define THROW_IF_TRITON_ERROR(X) \ do { \ TRITONSERVER_Error* tie_err__ = (X); \ if (tie_err__ != nullptr) { \ auto error_message = std::string(TRITONSERVER_ErrorMessage(tie_err__)); \ TRITONSERVER_ErrorDelete(tie_err__); \ throw PythonBackendException(error_message); \ } \ } while (false) #define THROW_IF_CUDA_ERROR(X) \ do { \ cudaError_t cuda_err__ = (X); \ if (cuda_err__ != cudaSuccess) { \ throw PythonBackendException( \ std::string(cudaGetErrorString(cuda_err__))); \ } \ } while (false) #define THROW_IF_ERROR(MSG, X) \ do { \ int return__ = (X); \ if (return__ != 0) { \ throw PythonBackendException(MSG); \ } \ } while (false) #define DUMMY_MESSAGE 0 #define DISALLOW_COPY(TypeName) TypeName(const TypeName&) = delete; #define DISALLOW_ASSIGN(TypeName) void operator=(const TypeName&) = delete; #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ DISALLOW_COPY(TypeName) \ DISALLOW_ASSIGN(TypeName) struct InitializeResponseShm { // Indicates whether the response has an error or not. bool response_has_error; // Indicates whether the response error is set or not. bool response_is_error_set; // Contains the error message. bi::managed_external_buffer::handle_t response_error; }; struct AutoCompleteResponseShm { // Indicates whether the response has an error or not. bool response_has_error; // Indicates whether the response error is set or not. bool response_is_error_set; // Contains the error message. bi::managed_external_buffer::handle_t response_error; // Indicates whether the response has model config or not. bool response_has_model_config; // Contains the model config bi::managed_external_buffer::handle_t response_model_config; }; // Control data structure for the communication between the Python stub and the // main stub. struct IPCControlShm { bool stub_health; bool parent_health; bool uses_env; bool decoupled; bi::interprocess_mutex parent_health_mutex; bi::interprocess_mutex stub_health_mutex; bi::managed_external_buffer::handle_t stub_message_queue; bi::managed_external_buffer::handle_t parent_message_queue; bi::managed_external_buffer::handle_t stub_to_parent_mq; bi::managed_external_buffer::handle_t parent_to_stub_mq; bi::managed_external_buffer::handle_t memory_manager_message_queue; }; struct SendMessageBase { bi::interprocess_mutex mu; bi::interprocess_condition cv; bool waiting_on_stub; }; struct ResponseBatch : SendMessageBase { uint32_t batch_size; bi::managed_external_buffer::handle_t error; bool has_error; // Indicates whether an additional call to stub is required for the clean up // of the resources. bool cleanup; // Indicates whether this error has a message or not. bool is_error_set; uint32_t response_size; // Indicates whether the response factory has been deleted or not. bool is_response_factory_deleted = false; }; enum LogLevel { kInfo = 0, kWarning, kError, kVerbose }; enum MetricKind { kCounter = 0, kGauge, kHistogram }; struct LogSendMessage : SendMessageBase { bi::managed_external_buffer::handle_t filename; int32_t line; bi::managed_external_buffer::handle_t log_message; LogLevel level; }; struct CleanupMessage : SendMessageBase { void* id; }; struct IsCancelledMessage : SendMessageBase { intptr_t response_factory_address; intptr_t request_address; bool is_cancelled; }; struct CustomMetricsMessage : SendMessageBase { bi::managed_external_buffer::handle_t message; bool has_error; bool is_error_set; bi::managed_external_buffer::handle_t error; // This field is specifically utilized when making the // 'PYTHONSTUB_MetricRequestValue' request. It is used to hold the metric // value after the Python backend calls the Triton C API to retrieve the // metric value and pass it back to the stub process. double value; // This field is specifically utilized when making the // 'PYTHONSTUB_MetricFamilyRequestNew' or 'PYTHONSTUB_MetricRequestNew' // requests. It is used to hold the memory address of // TRITONSERVER_MetricFamily' or 'TRITONSERVER_Metric' objects created in the // Python backend and pass back to the stub process. void* address; }; struct ModelLoaderMessage : SendMessageBase { bi::managed_external_buffer::handle_t message; bool has_error; bool is_error_set; bi::managed_external_buffer::handle_t error; bool is_model_ready; }; struct ResponseSenderBase { bi::interprocess_mutex mu; bi::interprocess_condition cv; bool is_stub_turn; bool has_error; bool is_error_set; bi::managed_external_buffer::handle_t error; intptr_t request_address; intptr_t response_factory_address; }; struct ResponseSendMessage : ResponseSenderBase { bi::managed_external_buffer::handle_t response; // A shm handle to a GPUBuffersShm object. bi::managed_external_buffer::handle_t gpu_buffers_handle; uint32_t flags; }; struct RequestBatch { uint32_t batch_size; // A shm handle to a GPUBuffersShm object. bi::managed_external_buffer::handle_t gpu_buffers_handle; }; struct MemoryReleaseMessage { std::mutex mu; std::condition_variable cv; uint64_t id; bool waiting_on_stub; }; #ifdef TRITON_ENABLE_GPU struct CUDAMemPoolMessage : SendMessageBase { cudaIpcMemHandle_t cuda_handle; int32_t device_id; bi::managed_external_buffer::handle_t error; bool has_error; bool is_error_set; }; class CUDAHandler { public: static CUDAHandler& getInstance() { static CUDAHandler instance; return instance; } private: std::mutex mu_; void* dl_open_handle_ = nullptr; std::string error_str_; CUresult (*cu_pointer_get_attribute_fn_)( CUdeviceptr*, CUpointer_attribute, CUdeviceptr) = nullptr; CUresult (*cu_get_error_string_fn_)(CUresult, const char**) = nullptr; CUresult (*cu_init_fn_)(unsigned int) = nullptr; CUresult (*cu_device_primary_ctx_get_state_fn_)( CUdevice, unsigned int*, int*) = nullptr; CUDAHandler(); /// Check if a primary context has already been created for a device. bool HasPrimaryContext(int device); ~CUDAHandler() noexcept(false); public: CUDAHandler(CUDAHandler const&) = delete; void operator=(CUDAHandler const&) = delete; bool IsAvailable(); const std::string& GetErrorString() const { return error_str_; } void ClearErrorString() { return error_str_.clear(); } void PointerGetAttribute( CUdeviceptr* start_address, CUpointer_attribute attr, CUdeviceptr device_ptr); void OpenCudaHandle( int64_t memory_type_id, cudaIpcMemHandle_t* cuda_mem_handle, void** data_ptr); void CloseCudaHandle(int64_t memory_type_id, void* data_ptr); void* LoadSharedObject(const char* filename); void* LocateSymbol(const char* symbol); std::string LocateSymbolError(); void CloseLibrary(); /// Set the device only if the primary context has already been created for /// this device. Inspired from PyTorch's MaybeSetDevice. /// \param device The cuda device index. void MaybeSetDevice(int device); }; /// A helper class to change the current device and restore the old context. The /// old context will be restored only if the primary context for that device is /// already created, otherwise the CUDA context will remain as the primary /// context of 'device'. class ScopedSetDevice { public: ScopedSetDevice(int device); ~ScopedSetDevice(); private: int device_; int current_device_; }; // Check if the data is allocated from the pool by the base address. bool IsUsingCUDAPool( std::unique_ptr& cuda_pool, int64_t memory_type_id, void* data); #endif // TRITON_ENABLE_GPU // FIXME: [DLIS-6078]: We should not need this function. However, some paths are // being retrieved from core that are not platform-agnostic. void SanitizePath(std::string& path); #ifndef TRITON_PB_STUB std::shared_ptr WrapTritonErrorInSharedPtr( TRITONSERVER_Error* error); #endif std::string GenerateUUID(); }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/python_be.cc000066400000000000000000002566371527052371200255640ustar00rootroot00000000000000// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "python_be.h" #include #include "correlation_id.h" #include "gpu_buffers.h" #include "infer_payload.h" #include "model_loader.h" #include "pb_log.h" namespace triton { namespace backend { namespace python { namespace bi = boost::interprocess; ModelInstanceState::ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), stub_to_parent_thread_(false) { } TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; } TRITONSERVER_Error* ModelInstanceState::CheckIncomingRequests( TRITONBACKEND_Request** requests, const uint32_t request_count, size_t& total_batch_size) { ModelState* model_state = reinterpret_cast(Model()); int max_batch_size = model_state->MaxBatchSize(); // For each request collect the total batch size for this inference // execution. The batch-size, number of inputs, and size of each // input has already been checked so don't need to do that here. total_batch_size = 0; for (size_t i = 0; i < request_count; i++) { // If we get a nullptr request then something is badly wrong. Fail // and release all requests. if (requests[i] == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "null request given to Python backend for '" + Name() + "'") .c_str()); } } for (size_t i = 0; i < request_count; i++) { if (max_batch_size > 0) { // Retrieve the batch size from one of the inputs, if the model // supports batching, the first dimension size is batch size TRITONBACKEND_Input* input; TRITONSERVER_Error* err = TRITONBACKEND_RequestInputByIndex(requests[i], 0 /* index */, &input); if (err == nullptr) { const int64_t* shape; err = TRITONBACKEND_InputProperties( input, nullptr, nullptr, &shape, nullptr, nullptr, nullptr); total_batch_size += shape[0]; } if (err != nullptr) { return err; } } else { ++total_batch_size; } } // If there are no valid payloads then no need to run the inference. if (total_batch_size == 0) { return nullptr; } // Make sure the maximum batch size is not exceeded. The // total_batch_size must be 1 for models that don't support batching // (i.e. max_batch_size == 0). If max_batch_size is exceeded then // scheduler has done something badly wrong so fail and release all // requests. if ((total_batch_size != 1) && (total_batch_size > (size_t)max_batch_size)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "batch size " + std::to_string(total_batch_size) + " for '" + Name() + "', max allowed is " + std::to_string(max_batch_size)) .c_str()); } return nullptr; // success } bool ModelInstanceState::ExistsInClosedRequests(intptr_t closed_request) { std::lock_guard guard{closed_requests_mutex_}; return std::find( closed_requests_.begin(), closed_requests_.end(), closed_request) != closed_requests_.end(); } void ModelInstanceState::SetErrorForResponseSendMessage( ResponseSendMessage* response_send_message, std::shared_ptr error, std::unique_ptr& error_message) { if (error && *error != nullptr) { response_send_message->has_error = true; LOG_IF_EXCEPTION( error_message = PbString::Create( Stub()->ShmPool(), TRITONSERVER_ErrorMessage(*error))); response_send_message->error = error_message->ShmHandle(); response_send_message->is_error_set = true; } } bool ModelInstanceState::IsStubProcessAlive() { boost::posix_time::ptime timeout = boost::get_system_time() + boost::posix_time::seconds(1); bi::scoped_lock lock(*Stub()->HealthMutex(), timeout); // Check if lock has been acquired. if (lock) { return Stub()->IpcControl()->stub_health; } else { // If It failed to obtain the lock, it means that the stub has been // stuck or exited while holding the health mutex lock. return false; } } TRITONSERVER_Error* ModelInstanceState::SaveRequestsToSharedMemory( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector>& pb_infer_requests, AllocatedSharedMemory& request_batch, std::shared_ptr>& responses) { // Clear any existing items in the requests vector pb_infer_requests.clear(); ModelState* model_state = reinterpret_cast(Model()); RETURN_IF_EXCEPTION( request_batch = Stub()->ShmPool()->Construct( sizeof(RequestBatch) + request_count * sizeof(bi::managed_external_buffer::handle_t))); RequestBatch* request_batch_shm_ptr = reinterpret_cast(request_batch.data_.get()); request_batch_shm_ptr->batch_size = request_count; bi::managed_external_buffer::handle_t* requests_shm = reinterpret_cast( request_batch.data_.get() + sizeof(RequestBatch)); for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; uint32_t requested_input_count = 0; RETURN_IF_ERROR( TRITONBACKEND_RequestInputCount(request, &requested_input_count)); uint32_t requested_output_count = 0; RETURN_IF_ERROR( TRITONBACKEND_RequestOutputCount(request, &requested_output_count)); std::vector> pb_input_tensors; for (size_t iidx = 0; iidx < requested_input_count; ++iidx) { std::shared_ptr pb_input_tensor; RETURN_IF_ERROR( GetInputTensor(iidx, pb_input_tensor, request, responses)); pb_input_tensors.emplace_back(std::move(pb_input_tensor)); } std::set requested_output_names; // Append the list of requested outputs to the inference_request for (size_t iidx = 0; iidx < requested_output_count; ++iidx) { const char* requested_output_name; RETURN_IF_ERROR(TRITONBACKEND_RequestOutputName( request, iidx, &requested_output_name)); requested_output_names.emplace(requested_output_name); } triton::common::TritonJson::Value parameters_json( triton::common::TritonJson::ValueType::OBJECT); uint32_t parameter_count; RETURN_IF_ERROR( TRITONBACKEND_RequestParameterCount(request, ¶meter_count)); for (size_t i = 0; i < parameter_count; i++) { const char* name; TRITONSERVER_ParameterType type; const void* vvalue; RETURN_IF_ERROR( TRITONBACKEND_RequestParameter(request, i, &name, &type, &vvalue)); if (type == TRITONSERVER_PARAMETER_INT) { RETURN_IF_ERROR(parameters_json.AddInt( name, *(reinterpret_cast(vvalue)))); } else if (type == TRITONSERVER_PARAMETER_BOOL) { RETURN_IF_ERROR(parameters_json.AddBool( name, *(reinterpret_cast(vvalue)))); } else if (type == TRITONSERVER_PARAMETER_STRING) { std::string string = reinterpret_cast(vvalue); RETURN_IF_ERROR(parameters_json.AddString(name, string)); } else if (type == TRITONSERVER_PARAMETER_DOUBLE) { RETURN_IF_ERROR(parameters_json.AddDouble( name, *(reinterpret_cast(vvalue)))); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unsupported parameter type for parameter '") + name + "'.") .c_str()); } } triton::common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(parameters_json.Write(&buffer)); const auto& parameters_string = buffer.Contents(); // request id const char* id; RETURN_IF_ERROR(TRITONBACKEND_RequestId(request, &id)); uint64_t correlation_id_uint = 0; CorrelationId correlation_id; auto error = TRITONBACKEND_RequestCorrelationId(request, &correlation_id_uint); if (error != nullptr) { TRITONSERVER_ErrorDelete(error); const char* correlation_id_string = ""; RETURN_IF_ERROR(TRITONBACKEND_RequestCorrelationIdString( request, &correlation_id_string)); correlation_id = CorrelationId(std::string(correlation_id_string)); } else { correlation_id = CorrelationId(correlation_id_uint); } uint32_t flags; RETURN_IF_ERROR(TRITONBACKEND_RequestFlags(request, &flags)); // Do not return if error in this case, because Triton core // will return an error if tracing is disabled (see PYBE PR#295). // For the same reason, we do not log the error message, otherwise // when Triton is compiled without tracing, it'll constantly log // this error. TRITONSERVER_InferenceTrace* triton_trace; auto err = TRITONBACKEND_RequestTrace(request, &triton_trace); if (err != nullptr) { triton_trace = nullptr; TRITONSERVER_ErrorDelete(err); } const char* val = nullptr; if (triton_trace != nullptr) { LOG_IF_ERROR( TRITONSERVER_InferenceTraceContext(triton_trace, &val), "failed to retrieve trace context"); } std::string context = (val != nullptr) ? std::string(val) : ""; InferenceTrace trace = InferenceTrace(reinterpret_cast(triton_trace), context); uint64_t request_timeout; RETURN_IF_ERROR(TRITONBACKEND_InferenceRequestTimeoutMicroseconds( request, &request_timeout)); std::unique_ptr infer_request; TRITONBACKEND_ResponseFactory* factory_ptr = nullptr; RETURN_IF_ERROR(TRITONBACKEND_ResponseFactoryNew(&factory_ptr, request)); infer_request = std::make_unique( id, correlation_id, pb_input_tensors, requested_output_names, model_state->Name(), model_state->Version(), parameters_string, flags, request_timeout, reinterpret_cast(factory_ptr), reinterpret_cast(request), PreferredMemory(PreferredMemory::kDefault, 0), trace); RETURN_IF_EXCEPTION(infer_request->SaveToSharedMemory(Stub()->ShmPool())); requests_shm[r] = infer_request->ShmHandle(); pb_infer_requests.emplace_back(std::move(infer_request)); } return nullptr; // success } TRITONSERVER_Error* ModelInstanceState::LaunchStubProcess() { ModelState* model_state = reinterpret_cast(Model()); Stub() = std::make_unique( "MODEL_INSTANCE_STUB", Name(), DeviceId(), TRITONSERVER_InstanceGroupKindString(Kind())); RETURN_IF_ERROR(Stub()->Initialize(model_state)); RETURN_IF_ERROR(Stub()->Setup()); StartMonitor(); RETURN_IF_ERROR(Stub()->Launch()); thread_pool_ = std::make_unique( model_state->StateForBackend()->thread_pool_size); request_executor_ = std::make_unique( Stub()->ShmPool(), model_state->TritonServer()); return nullptr; } TRITONSERVER_Error* ModelInstanceState::GetInputTensor( const uint32_t input_idx, std::shared_ptr& input_tensor, TRITONBACKEND_Request* request, std::shared_ptr>& responses) { NVTX_RANGE(nvtx_, "GetInputTensor " + Name()); const char* input_name; // Load iidx'th input name RETURN_IF_ERROR( TRITONBACKEND_RequestInputName(request, input_idx, &input_name)); // Load iidx'th input TRITONBACKEND_Input* in; RETURN_IF_ERROR(TRITONBACKEND_RequestInput(request, input_name, &in)); // Load input properties TRITONSERVER_DataType input_dtype; const int64_t* input_shape; uint32_t input_dims_count; uint64_t input_byte_size; uint32_t input_buffer_count; RETURN_IF_ERROR(TRITONBACKEND_InputPropertiesForHostPolicy( in, HostPolicyName().c_str(), &input_name, &input_dtype, &input_shape, &input_dims_count, &input_byte_size, &input_buffer_count)); // Only use input collector when a response array is provided. std::unique_ptr collector; if (responses) { collector = std::make_unique( &request, 1, responses.get(), Model()->TritonMemoryManager(), false /* pinned_enable */, CudaStream(), nullptr, nullptr, 0, HostPolicyName().c_str()); } ModelState* model_state = reinterpret_cast(Model()); bool cpu_only_tensors = model_state->ForceCPUOnlyInputTensors(); if (input_dtype == TRITONSERVER_TYPE_BYTES) { cpu_only_tensors = true; } #ifdef TRITON_ENABLE_GPU CUDAHandler& cuda_handler = CUDAHandler::getInstance(); // If CUDA driver API is not available, the input tensors will be moved to // CPU. if (!cuda_handler.IsAvailable() && !cpu_only_tensors) { if (!cuda_handler.GetErrorString().empty()) { LOG_MESSAGE( TRITONSERVER_LOG_WARN, (std::string( "Forcing CPU only input tensors: " + cuda_handler.GetErrorString())) .c_str()); } cuda_handler.ClearErrorString(); cpu_only_tensors = true; } #endif TRITONSERVER_MemoryType src_memory_type; int64_t src_memory_type_id; size_t src_byte_size; const void* src_ptr; RETURN_IF_ERROR(TRITONBACKEND_InputBuffer( in, 0 /* input buffer index */, &src_ptr, &src_byte_size, &src_memory_type, &src_memory_type_id)); // If TRITON_ENABLE_GPU is false, we need to copy the tensors // to the CPU. #ifndef TRITON_ENABLE_GPU cpu_only_tensors = true; #endif // TRITON_ENABLE_GPU if (cpu_only_tensors || src_memory_type != TRITONSERVER_MEMORY_GPU) { input_tensor = std::make_shared( std::string(input_name), std::vector(input_shape, input_shape + input_dims_count), input_dtype, TRITONSERVER_MEMORY_CPU /* memory_type */, 0 /* memory_type_id */, nullptr /* buffer ptr*/, input_byte_size, nullptr /* DLManagedTensor */); RETURN_IF_EXCEPTION(input_tensor->SaveToSharedMemory( Stub()->ShmPool(), false /* copy_gpu */)); char* input_buffer = reinterpret_cast(input_tensor->DataPtr()); if (collector) { collector->ProcessTensor( input_name, input_buffer, input_byte_size, TRITONSERVER_MEMORY_CPU /* memory_type */, 0 /* memory_type_id */); } else { size_t byte_size = input_byte_size; RETURN_IF_ERROR(backend::ReadInputTensor( request, input_name, input_buffer, &byte_size)); } if (input_dtype == TRITONSERVER_TYPE_BYTES) { const char* content = reinterpret_cast(input_tensor->DataPtr()); size_t content_byte_size = input_tensor->ByteSize(); const size_t request_element_cnt = GetElementCount(input_tensor->Dims()); RETURN_IF_ERROR(ValidateStringBuffer( content, content_byte_size, request_element_cnt, input_name, nullptr /* str_list */)); } } else { #ifdef TRITON_ENABLE_GPU // Attempt to use the cuda shared memory pool for GPU tensor. ShareCUDAMemoryPool(src_memory_type_id); // Retrieving GPU input tensors const void* buffer = nullptr; std::vector> alloc_perference; alloc_perference = {{TRITONSERVER_MEMORY_GPU, src_memory_type_id}}; // collector is used in the non-decoupled mode. if (collector) { // The ProcessTensor function will try to allocate the buffer in the CUDA // pool first. RETURN_IF_ERROR(collector->ProcessTensor( input_name, nullptr, 0, alloc_perference, reinterpret_cast(&buffer), &input_byte_size, &src_memory_type, &src_memory_type_id)); // If the tensor is using the cuda shared memory, we need to extract the // handle that was used to create the device pointer. This is because of a // limitation in the legacy CUDA IPC API that doesn't allow getting the // handle of an exported pointer. If the cuda handle exists, it indicates // that the cuda shared memory was used and the input is in a single // buffer. // [FIXME] For the case where the input is in cuda shared memory and uses // multiple input buffers this needs to be changed. TRITONSERVER_BufferAttributes* buffer_attributes; // This value is not used. const void* buffer_p; RETURN_IF_ERROR(TRITONBACKEND_InputBufferAttributes( in, 0, &buffer_p, &buffer_attributes)); input_tensor = std::make_shared( std::string(input_name), std::vector(input_shape, input_shape + input_dims_count), input_dtype, src_memory_type, src_memory_type_id, const_cast(buffer), input_byte_size, nullptr /* DLManagedTensor */); cudaIpcMemHandle_t* cuda_ipc_handle; RETURN_IF_ERROR(TRITONSERVER_BufferAttributesCudaIpcHandle( buffer_attributes, reinterpret_cast(&cuda_ipc_handle))); if (cuda_ipc_handle != nullptr) { RETURN_IF_EXCEPTION(input_tensor->SaveToSharedMemory( Stub()->ShmPool(), false /* copy_gpu */)); RETURN_IF_EXCEPTION( input_tensor->Memory()->SetCudaIpcHandle(cuda_ipc_handle)); } else { RETURN_IF_EXCEPTION(input_tensor->SaveToSharedMemory( Stub()->ShmPool(), true /* copy_gpu */)); } } else { // Try to use the cuda shared memory pool first. void* dev_ptr; BackendMemory* backend_memory; std::unique_ptr lbackend_memory; RETURN_IF_ERROR(BackendMemory::Create( reinterpret_cast( Stub() ->ShmPool() ->GetCUDAMemoryPoolManager() ->TritonMemoryManager()), {BackendMemory::AllocationType::GPU_POOL, BackendMemory::AllocationType::GPU}, src_memory_type_id, input_byte_size, &backend_memory)); dev_ptr = backend_memory->MemoryPtr(); lbackend_memory.reset(backend_memory); size_t byte_size = input_byte_size; bool cuda_used = false; RETURN_IF_ERROR(backend::ReadInputTensor( request, input_name, reinterpret_cast(dev_ptr), &byte_size, TRITONSERVER_MEMORY_GPU, src_memory_type_id, CudaStream(), &cuda_used)); if (cuda_used) { #ifdef TRITON_ENABLE_GPU cudaStreamSynchronize(stream_); #endif } input_tensor = std::make_shared( std::string(input_name), std::vector(input_shape, input_shape + input_dims_count), input_dtype, src_memory_type, src_memory_type_id, const_cast(dev_ptr), input_byte_size, nullptr /* DLManagedTensor */); input_tensor->SetMemory(std::move( PbMemory::Create(Stub()->ShmPool(), std::move(lbackend_memory)))); RETURN_IF_EXCEPTION(input_tensor->SaveToSharedMemory( Stub()->ShmPool(), true /* copy_gpu */)); } #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Python backend does not support GPU tensors."); #endif // TRITON_ENABLE_GPU } return nullptr; } void ModelInstanceState::ExecuteBLSRequest( std::shared_ptr ipc_message, const bool is_decoupled) { bool is_response_batch_set = false; std::unique_ptr infer_response; ResponseBatch* response_batch = nullptr; std::unique_ptr pb_error_message; std::unique_ptr bls_response; AllocatedSharedMemory response_batch_shm; bi::managed_external_buffer::handle_t* response_handle = nullptr; try { bls_response = IPCMessage::Create(Stub()->ShmPool(), false /* inline_response */); AllocatedSharedMemory request_batch = Stub()->ShmPool()->Load(ipc_message->Args()); RequestBatch* request_batch_shm_ptr = reinterpret_cast(request_batch.data_.get()); bls_response->Command() = PYTHONSTUB_InferExecResponse; ipc_message->ResponseHandle() = bls_response->ShmHandle(); // The response batch of the handle will contain a ResponseBatch PrepareResponseBatch( &response_batch, response_batch_shm, &bls_response, &response_handle); is_response_batch_set = true; bool has_gpu_tensor = false; GPUBuffersHelper gpu_buffer_helper; PythonBackendException pb_exception(std::string{}); if (request_batch_shm_ptr->batch_size == 1) { std::shared_ptr infer_request; bi::managed_external_buffer::handle_t* request_handle = reinterpret_cast( request_batch.data_.get() + sizeof(RequestBatch)); infer_request = InferRequest::LoadFromSharedMemory( Stub()->ShmPool(), *request_handle, false /* open_cuda_handle */, nullptr /* is_model_decoupled */); // If the BLS inputs are in GPU an additional round trip between the // stub process and the main process is required. The reason is that we // need to first allocate the GPU memory from the memory pool and then // ask the stub process to fill in those allocated buffers. try { for (auto& input_tensor : infer_request->Inputs()) { if (!input_tensor->IsCPU()) { #ifdef TRITON_ENABLE_GPU // Attempt to use the cuda shared memory pool for GPU tensor. ShareCUDAMemoryPool(input_tensor->MemoryTypeId()); BackendMemory* backend_memory; std::unique_ptr lbackend_memory; has_gpu_tensor = true; TRITONSERVER_Error* error = BackendMemory::Create( Model()->TritonMemoryManager(), {BackendMemory::AllocationType::GPU_POOL, BackendMemory::AllocationType::GPU}, input_tensor->MemoryTypeId(), input_tensor->ByteSize(), &backend_memory); if (error != nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, TRITONSERVER_ErrorMessage(error)); break; } lbackend_memory.reset(backend_memory); input_tensor->SetMemory(std::move(PbMemory::Create( Stub()->ShmPool(), std::move(lbackend_memory)))); gpu_buffer_helper.AddBuffer(input_tensor->Memory()->ShmHandle()); #endif // TRITON_ENABLE_GPU } } } catch (const PythonBackendException& exception) { gpu_buffer_helper.SetError(Stub()->ShmPool(), exception.what()); pb_exception = exception; } // Wait for the extra round trip to complete. The stub process will fill // in the data for the GPU tensors. If there is an error, the extra round // trip must be still completed, otherwise the stub process will always be // waiting for a message from the parent process. if (has_gpu_tensor) { gpu_buffer_helper.Complete(Stub()->ShmPool()); request_batch_shm_ptr->gpu_buffers_handle = gpu_buffer_helper.ShmHandle(); bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; ipc_message->ResponseCondition()->notify_all(); ipc_message->ResponseCondition()->wait(lock); } if (pb_exception.what() == std::string{""}) { auto callback = std::bind( &ModelInstanceState::SendBLSDecoupledResponse, this, std::placeholders::_1); std::shared_ptr infer_payload = std::make_shared(is_decoupled, callback); auto response_future = request_executor_->Infer(infer_request, infer_payload); infer_response = response_future.get(); if (is_decoupled && (infer_response->Id() != nullptr)) { // Need to manage the lifetime of InferPayload object for bls // decoupled responses. std::lock_guard lock(infer_payload_mu_); infer_payload_[reinterpret_cast(infer_payload.get())] = infer_payload; } PrepareResponseHandle(&infer_response, response_handle); } else { throw pb_exception; } } } catch (const PythonBackendException& pb_exception) { if (is_response_batch_set) { response_batch->has_error = true; LOG_IF_EXCEPTION( pb_error_message = PbString::Create(Stub()->ShmPool(), pb_exception.what())); if (pb_error_message != nullptr) { response_batch->is_error_set = true; response_batch->error = pb_error_message->ShmHandle(); } } else { LOG_MESSAGE(TRITONSERVER_LOG_ERROR, pb_exception.what()); } } // At this point, the stub has notified the parent process that it has // finished loading the inference response from shared memory. { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; ipc_message->ResponseCondition()->notify_all(); ipc_message->ResponseCondition()->wait(lock); } } void ModelInstanceState::StubToParentMQMonitor() { while (stub_to_parent_thread_) { bi::managed_external_buffer::handle_t handle = Stub()->StubToParentMessageQueue()->Pop(); if (handle == DUMMY_MESSAGE) { break; } std::unique_ptr message = IPCMessage::LoadFromSharedMemory(Stub()->ShmPool(), handle); switch (message->Command()) { case PYTHONSTUB_LogRequest: { ProcessLogRequest(message); break; } case PYTHONSTUB_BLSDecoupledInferPayloadCleanup: case PYTHONSTUB_DecoupledResponseFactoryCleanup: { ProcessCleanupRequest(message); break; } case PYTHONSTUB_IsRequestCancelled: { ProcessIsRequestCancelled(message); break; } case PYTHONSTUB_MetricFamilyRequestNew: case PYTHONSTUB_MetricFamilyRequestDelete: { ProcessMetricFamilyRequest(message); break; } case PYTHONSTUB_MetricRequestNew: case PYTHONSTUB_MetricRequestDelete: case PYTHONSTUB_MetricRequestValue: case PYTHONSTUB_MetricRequestIncrement: case PYTHONSTUB_MetricRequestSet: case PYTHONSTUB_MetricRequestObserve: { ProcessMetricRequest(message); break; } case PYTHONSTUB_ModelReadinessRequest: case PYTHONSTUB_LoadModelRequest: case PYTHONSTUB_UnloadModelRequest: { ProcessModelControlRequest(message); break; } case PYTHONSTUB_ResponseSend: { std::shared_ptr response_send_message = std::move(message); std::packaged_task task([this, response_send_message] { ResponseSendDecoupled(response_send_message); }); boost::asio::post(*thread_pool_, std::move(task)); break; } case PYTHONSTUB_InferExecRequest: case PYTHONSTUB_InferStreamExecRequest: { std::shared_ptr bls_execute = std::move(message); std::packaged_task task([this, bls_execute] { ExecuteBLSRequest( bls_execute, (bls_execute->Command() == PYTHONSTUB_InferStreamExecRequest)); }); boost::asio::post(*thread_pool_, std::move(task)); break; } default: { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, "Unexpected message type received."); break; } } } } void ModelInstanceState::ProcessLogRequest( const std::unique_ptr& message) { AllocatedSharedMemory log_message_response = Stub()->ShmPool()->Load(message->Args()); std::unique_ptr pb_log_message = PbLogShm::LoadFromSharedMemory(Stub()->ShmPool(), message->Args()); const std::string& filename = pb_log_message->Filename(); uint32_t line = pb_log_message->Line(); const std::string& log_message = pb_log_message->Message(); LogLevel level = pb_log_message->Level(); switch (level) { case LogLevel::kInfo: { TRITONSERVER_LogMessage( TRITONSERVER_LOG_INFO, (filename.c_str()), line, (log_message.c_str())); break; } case LogLevel::kWarning: { TRITONSERVER_LogMessage( TRITONSERVER_LOG_WARN, (filename.c_str()), line, (log_message.c_str())); break; } case LogLevel::kError: { TRITONSERVER_LogMessage( TRITONSERVER_LOG_ERROR, (filename.c_str()), line, (log_message.c_str())); break; } case LogLevel::kVerbose: { TRITONSERVER_LogMessage( TRITONSERVER_LOG_VERBOSE, (filename.c_str()), line, (log_message.c_str())); break; } } // Send confirmation back to pb_stub.cc that the message // was received. LogSendMessage* send_message_payload = reinterpret_cast(log_message_response.data_.get()); { bi::scoped_lock guard{send_message_payload->mu}; send_message_payload->waiting_on_stub = true; send_message_payload->cv.notify_all(); while (send_message_payload->waiting_on_stub) { send_message_payload->cv.wait(guard); } } } void ModelInstanceState::ProcessCleanupRequest( const std::unique_ptr& message) { AllocatedSharedMemory cleanup_request_message = Stub()->ShmPool()->Load(message->Args()); CleanupMessage* cleanup_message_ptr = reinterpret_cast(cleanup_request_message.data_.get()); intptr_t id = reinterpret_cast(cleanup_message_ptr->id); if (message->Command() == PYTHONSTUB_BLSDecoupledInferPayloadCleanup) { // Remove the InferPayload object from the map. std::lock_guard lock(infer_payload_mu_); infer_payload_.erase(id); } else if (message->Command() == PYTHONSTUB_DecoupledResponseFactoryCleanup) { // Delete response factory std::unique_ptr< TRITONBACKEND_ResponseFactory, backend::ResponseFactoryDeleter> response_factory(reinterpret_cast(id)); } { bi::scoped_lock lock{*(message->ResponseMutex())}; cleanup_message_ptr->waiting_on_stub = true; message->ResponseCondition()->notify_all(); } } void ModelInstanceState::ProcessIsRequestCancelled( const std::unique_ptr& message) { AllocatedSharedMemory message_shm = Stub()->ShmPool()->Load(message->Args()); IsCancelledMessage* message_payload = reinterpret_cast(message_shm.data_.get()); { bi::scoped_lock lk{message_payload->mu}; if (message_payload->response_factory_address != 0) { TRITONBACKEND_ResponseFactory* response_factory = reinterpret_cast( message_payload->response_factory_address); TRITONBACKEND_ResponseFactoryIsCancelled( response_factory, &message_payload->is_cancelled); } else if (message_payload->request_address != 0) { TRITONBACKEND_Request* request = reinterpret_cast( message_payload->request_address); TRITONBACKEND_RequestIsCancelled(request, &message_payload->is_cancelled); } else { throw PythonBackendException("Cannot determine request cancellation"); } message_payload->waiting_on_stub = true; message_payload->cv.notify_all(); while (message_payload->waiting_on_stub) { message_payload->cv.wait(lk); } } } template void ModelInstanceState::ProcessMessage( const std::unique_ptr& ipc_message, std::function&, MessageType*)> request_handler) { AllocatedSharedMemory message = Stub()->ShmPool()->Load(ipc_message->Args()); MessageType* message_ptr = reinterpret_cast(message.data_.get()); std::unique_ptr pb_error_message; PythonBackendException pb_exception(std::string{}); std::unique_ptr object = T::LoadFromSharedMemory(Stub()->ShmPool(), message_ptr->message); ScopedDefer _([message_ptr] { { bi::scoped_lock guard{message_ptr->mu}; message_ptr->waiting_on_stub = true; message_ptr->cv.notify_all(); while (message_ptr->waiting_on_stub) { message_ptr->cv.wait(guard); } } }); try { request_handler(object, message_ptr); } catch (const PythonBackendException& exception) { pb_exception = exception; } if (pb_exception.what() != std::string{}) { message_ptr->has_error = true; LOG_IF_EXCEPTION( pb_error_message = PbString::Create(Stub()->ShmPool(), pb_exception.what())); message_ptr->error = pb_error_message->ShmHandle(); message_ptr->is_error_set = true; } } void ModelInstanceState::ProcessMetricFamilyRequest( const std::unique_ptr& message) { auto command = message->Command(); ProcessMessage( message, [this, command]( std::unique_ptr& metric_family, CustomMetricsMessage* metrics_message_ptr) { switch (command) { case PYTHONSTUB_MetricFamilyRequestNew: { metrics_message_ptr->address = metric_family->InitializeTritonMetricFamily(); break; } case PYTHONSTUB_MetricFamilyRequestDelete: { metric_family->ClearTritonMetricFamily(); break; } default: { throw PythonBackendException("Unknown metric family request kind"); } } }); } void ModelInstanceState::ProcessMetricRequest( const std::unique_ptr& message) { auto command = message->Command(); ProcessMessage( message, [this, command]( std::unique_ptr& metric, CustomMetricsMessage* metrics_message_ptr) { try { switch (command) { case PYTHONSTUB_MetricRequestNew: { metrics_message_ptr->address = metric->InitializeTritonMetric(); break; } case PYTHONSTUB_MetricRequestIncrement: case PYTHONSTUB_MetricRequestSet: case PYTHONSTUB_MetricRequestObserve: case PYTHONSTUB_MetricRequestValue: { metric->HandleMetricOperation(metrics_message_ptr, command); break; } case PYTHONSTUB_MetricRequestDelete: { metric->ClearTritonMetric(); break; } default: { throw PythonBackendException("Unknown metric request kind"); } } } catch (const PythonBackendException& exception) { throw exception; } }); } void ModelInstanceState::ProcessModelControlRequest( const std::unique_ptr& message) { auto command = message->Command(); ModelState* model_state = reinterpret_cast(Model()); ProcessMessage( message, [this, command, model_state]( std::unique_ptr& model_loader, ModelLoaderMessage* model_loader_msg_ptr) { switch (command) { case PYTHONSTUB_LoadModelRequest: { model_loader->LoadModel(model_state->TritonServer()); break; } case PYTHONSTUB_UnloadModelRequest: { model_loader->UnloadModel(model_state->TritonServer()); break; } case PYTHONSTUB_ModelReadinessRequest: { model_loader_msg_ptr->is_model_ready = model_loader->IsModelReady(model_state->TritonServer()); break; } default: { throw PythonBackendException("Unknown model loader request kind"); } } }); } TRITONSERVER_Error* ModelInstanceState::SendMessageToStub( bi::managed_external_buffer::handle_t message) { bool success = false; while (!success) { uint64_t timeout_miliseconds = 1000; { boost::posix_time::ptime timeout = boost::get_system_time() + boost::posix_time::milliseconds(timeout_miliseconds); bi::scoped_lock lock( *(Stub()->HealthMutex()), timeout); // Check if lock has been acquired. if (lock) { Stub()->IpcControl()->stub_health = false; } else { // If it failed to obtain the lock, it means that the stub has been // stuck or exited while holding the health mutex lock. return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to obtain the health mutex."); } } Stub()->StubMessageQueue()->Push( message, timeout_miliseconds /* duration ms */, success); if (!success && !IsStubProcessAlive()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Stub process is not healthy."); } } return nullptr; // success } void ModelInstanceState::SendMessageAndReceiveResponse( bi::managed_external_buffer::handle_t message, bi::managed_external_buffer::handle_t& response, std::shared_ptr>& responses, TRITONBACKEND_Request** requests, const uint32_t request_count) { auto error = SendMessageToStub(message); if (error != nullptr) { RespondErrorToAllRequests( TRITONSERVER_ErrorMessage(error), responses, requests, request_count); return; } bi::managed_external_buffer::handle_t response_message; error = Stub()->ReceiveMessageFromStub(response_message); if (error != nullptr) { RespondErrorToAllRequests( TRITONSERVER_ErrorMessage(error), responses, requests, request_count); return; } response = response_message; } void ModelInstanceState::RespondErrorToAllRequests( const char* message, std::shared_ptr>& responses, TRITONBACKEND_Request** requests, const uint32_t request_count) { for (uint32_t r = 0; r < request_count; ++r) { if ((*responses)[r] == nullptr) continue; std::string err_message = std::string( "Failed to process the request(s) for model instance '" + Name() + "', message: ") + message; TRITONSERVER_Error* err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, err_message.c_str()); LOG_IF_ERROR( TRITONBACKEND_ResponseSend( (*responses)[r], TRITONSERVER_RESPONSE_COMPLETE_FINAL, err), "failed sending response"); (*responses)[r] = nullptr; TRITONSERVER_ErrorDelete(err); } } void ModelInstanceState::StartMonitor() { stub_to_parent_thread_ = true; stub_to_parent_queue_monitor_ = std::thread(&ModelInstanceState::StubToParentMQMonitor, this); } void ModelInstanceState::TerminateMonitor() { if (stub_to_parent_thread_) { stub_to_parent_thread_ = false; // Push a dummy message to signal the thread to terminate. Stub()->StubToParentMessageQueue()->Push(DUMMY_MESSAGE); stub_to_parent_queue_monitor_.join(); } } void ModelInstanceState::ResponseSendDecoupled( std::shared_ptr response_send_message) { AllocatedSharedMemory send_message = Stub()->ShmPool()->Load( response_send_message->Args()); ResponseSendMessage* send_message_payload = reinterpret_cast(send_message.data_.get()); std::unique_ptr error_message; ScopedDefer response_factory_deleter([send_message_payload] { if (send_message_payload->flags == TRITONSERVER_RESPONSE_COMPLETE_FINAL) { TRITONBACKEND_ResponseFactory* response_factory = reinterpret_cast( send_message_payload->response_factory_address); std::unique_ptr< TRITONBACKEND_ResponseFactory, backend::ResponseFactoryDeleter> lresponse_factory(reinterpret_cast( response_factory)); } }); ScopedDefer _([send_message_payload] { { bi::scoped_lock guard{send_message_payload->mu}; send_message_payload->is_stub_turn = true; send_message_payload->cv.notify_all(); while (send_message_payload->is_stub_turn) { send_message_payload->cv.wait(guard); } } }); TRITONBACKEND_ResponseFactory* response_factory = reinterpret_cast( send_message_payload->response_factory_address); if (send_message_payload->flags == TRITONSERVER_RESPONSE_COMPLETE_FINAL) { { std::lock_guard guard{closed_requests_mutex_}; closed_requests_.push_back(send_message_payload->request_address); } } if (send_message_payload->response != 0) { std::unique_ptr infer_response = InferResponse::LoadFromSharedMemory( Stub()->ShmPool(), send_message_payload->response, false /* open cuda ipc handle */); bool requires_deferred_callback = false; TRITONBACKEND_Response* response; SetErrorForResponseSendMessage( send_message_payload, WrapTritonErrorInSharedPtr( TRITONBACKEND_ResponseNewFromFactory(&response, response_factory)), error_message); std::vector, void*>> gpu_output_buffers; GPUBuffersHelper gpu_buffer_helper; #ifdef TRITON_ENABLE_GPU for (auto& output_tensor : infer_response->OutputTensors()) { if (!output_tensor->IsCPU()) { // Attempt to use the cuda shared memory pool for GPU tensor. ShareCUDAMemoryPool(output_tensor->MemoryTypeId()); } } #endif // TRITON_ENABLE_GPU infer_response->Send( response, CudaStream(), requires_deferred_callback, send_message_payload->flags, Stub()->ShmPool(), gpu_buffer_helper, gpu_output_buffers); if (requires_deferred_callback) { gpu_buffer_helper.Complete(Stub()->ShmPool()); send_message_payload->gpu_buffers_handle = gpu_buffer_helper.ShmHandle(); // Additional round trip so that the stub can fill the GPU output buffers. { bi::scoped_lock guard{send_message_payload->mu}; send_message_payload->is_stub_turn = true; send_message_payload->cv.notify_all(); while (send_message_payload->is_stub_turn) { send_message_payload->cv.wait(guard); } } bool cuda_copy = false; for (auto& output_buffer_pair : gpu_output_buffers) { auto& pb_memory = output_buffer_pair.first; void* pointer = output_buffer_pair.second; bool cuda_used; try { if (pb_memory->MemoryType() == TRITONSERVER_MEMORY_CPU) { THROW_IF_TRITON_ERROR(CopyBuffer( "Failed to copy the CPU output tensor to buffer.", TRITONSERVER_MEMORY_CPU, 0, TRITONSERVER_MEMORY_CPU, 0, pb_memory->ByteSize(), pb_memory->DataPtr(), pointer, CudaStream(), &cuda_used)); cuda_copy |= cuda_used; } else if ( (pb_memory->MemoryType() == TRITONSERVER_MEMORY_GPU) && pb_memory->UseCUDASharedPool() && (pb_memory->DataPtr() != pointer)) { // If the data pointer from pb_memory is not the same as the // pointer, it means that the Triton-provided buffer is not used // during tensor transfer. Instead, an intermediate buffer that uses // CUDA shared memory pool is used. In this case, we need to copy // the data from the intermediate buffer back to the Triton-provided // buffer. THROW_IF_TRITON_ERROR(CopyBuffer( "Failed to copy the GPU output tensor to buffer.", TRITONSERVER_MEMORY_GPU, pb_memory->MemoryTypeId(), TRITONSERVER_MEMORY_GPU, pb_memory->MemoryTypeId(), pb_memory->ByteSize(), pb_memory->DataPtr(), pointer, CudaStream(), &cuda_used)); cuda_copy |= cuda_used; } #ifdef TRITON_ENABLE_GPU if (cuda_copy) { cudaStreamSynchronize(stream_); } #endif // TRITON_ENABLE_GPU } catch (const PythonBackendException& pb_exception) { TRITONSERVER_Error* error = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string( "Failed to copy output tensor to Triton-provided buffer: ") + pb_exception.what()) .c_str()); SetErrorForResponseSendMessage( send_message_payload, WrapTritonErrorInSharedPtr(error), error_message); } } } } else { TRITONSERVER_Error* error = TRITONBACKEND_ResponseFactorySendFlags( response_factory, send_message_payload->flags); SetErrorForResponseSendMessage( send_message_payload, WrapTritonErrorInSharedPtr(error), error_message); } } TRITONSERVER_Error* ModelInstanceState::ProcessRequests( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector>& pb_infer_requests, PbMetricReporter& reporter) { NVTX_RANGE(nvtx_, "ProcessRequests " + Name()); closed_requests_ = {}; ModelState* model_state = reinterpret_cast(Model()); size_t total_batch_size = 0; RETURN_IF_ERROR( CheckIncomingRequests(requests, request_count, total_batch_size)); // No request to process if (total_batch_size == 0) { return nullptr; // success } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("model ") + model_state->Name() + ", instance " + Name() + ", executing " + std::to_string(request_count) + " requests") .c_str()); AllocatedSharedMemory request_batch; std::shared_ptr> responses; RETURN_IF_ERROR(SaveRequestsToSharedMemory( requests, request_count, pb_infer_requests, request_batch, responses)); uint64_t compute_start_ns = 0; SET_TIMESTAMP(compute_start_ns); reporter.SetComputeStartNs(compute_start_ns); std::unique_ptr ipc_message; RETURN_IF_EXCEPTION( ipc_message = IPCMessage::Create(Stub()->ShmPool(), false /*inline_response*/)); ipc_message->Command() = PYTHONSTUB_CommandType::PYTHONSTUB_ExecuteRequest; ipc_message->Args() = request_batch.handle_; ScopedDefer execute_finalize([this] { // Push a dummy message to signal the thread to terminate. Stub()->StubMessageQueue()->Push(DUMMY_MESSAGE); }); std::unique_ptr response; { Stub()->StubMessageQueue()->Push(ipc_message->ShmHandle()); bi::managed_external_buffer::handle_t response_message; RETURN_IF_ERROR(Stub()->ReceiveMessageFromStub(response_message)); response = IPCMessage::LoadFromSharedMemory(Stub()->ShmPool(), response_message); } char* ipc_message_shm = reinterpret_cast(response->GetAllocatedSharedMemory().data_.get()); ResponseBatch* response_batch_shm_ptr = reinterpret_cast(ipc_message_shm + sizeof(IPCMessageShm)); uint64_t compute_end_ns = 0; SET_TIMESTAMP(compute_end_ns); reporter.SetComputeEndNs(compute_end_ns); reporter.SetBatchStatistics(total_batch_size); if (response_batch_shm_ptr->has_error) { // Clean up the response factory if an error occurred. The // `is_response_factory_deleted` flag indicates whether the response factory // has been deleted for some corner cases. if (!response_batch_shm_ptr->is_response_factory_deleted) { for (uint32_t r = 0; r < request_count; r++) { TRITONBACKEND_ResponseFactory* response_factory = reinterpret_cast( pb_infer_requests[r]->GetResponseFactoryAddress()); std::unique_ptr< TRITONBACKEND_ResponseFactory, backend::ResponseFactoryDeleter> lresponse_factory(reinterpret_cast( response_factory)); } } if (response_batch_shm_ptr->is_error_set) { auto error = PbString::LoadFromSharedMemory( Stub()->ShmPool(), response_batch_shm_ptr->error); return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, error->String().c_str()); } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to process the requests."); } if (response_batch_shm_ptr->batch_size > 0) { bi::managed_external_buffer::handle_t* response_shm_handle = reinterpret_cast( ipc_message_shm + sizeof(ResponseBatch) + sizeof(IPCMessageShm)); std::shared_ptr> responses( new std::vector()); responses->reserve(request_count); for (size_t i = 0; i < request_count; i++) { // It is possible to have multiple responses batched together in a single // response batch shm, where some of the responses are None due to the // usage of response sender, so only create a TRITONBACKEND_Response // object for the valid responses. if (response_shm_handle[i] == 0) { responses->emplace_back(nullptr); } else { TRITONBACKEND_Response* response; auto err = TRITONBACKEND_ResponseNew(&response, requests[i]); if (err == nullptr) { responses->emplace_back(response); } else { responses->emplace_back(nullptr); LOG_MESSAGE(TRITONSERVER_LOG_ERROR, "Fail to create response"); TRITONSERVER_ErrorDelete(err); } } } std::vector requires_deferred_callback; bool has_gpu_output = false; std::vector> shm_responses; std::vector, void*>>> gpu_output_buffers(request_count); GPUBuffersHelper gpu_buffer_helper; for (uint32_t r = 0; r < request_count; ++r) { NVTX_RANGE(nvtx_, "LoadingResponse " + Name()); requires_deferred_callback.push_back(false); if (response_shm_handle[r] == 0) { continue; } TRITONBACKEND_Response* response = (*responses)[r]; TRITONBACKEND_Request* request = requests[r]; uint32_t requested_output_count = 0; shm_responses.emplace_back(nullptr); std::unique_ptr& infer_response = shm_responses.back(); try { if (pb_infer_requests[r]->ReleaseFlags() == TRITONSERVER_REQUEST_RELEASE_RESCHEDULE) { // For rescheduled requests, we do not need to send a response. LOG_IF_ERROR( TRITONBACKEND_ResponseDelete((*responses)[r]), "failed to delete response"); (*responses)[r] = nullptr; continue; } { TRITONBACKEND_ResponseFactory* response_factory = reinterpret_cast( pb_infer_requests[r]->GetResponseFactoryAddress()); std::unique_ptr< TRITONBACKEND_ResponseFactory, backend::ResponseFactoryDeleter> lresponse_factory( reinterpret_cast( response_factory)); } infer_response = InferResponse::LoadFromSharedMemory( Stub()->ShmPool(), response_shm_handle[r], false /* open_cuda_handle */); if (infer_response->HasError()) { TRITONSERVER_Error* err = TRITONSERVER_ErrorNew( infer_response->Error()->Code(), infer_response->Error()->Message().c_str()); LOG_IF_ERROR( TRITONBACKEND_ResponseSend( (*responses)[r], TRITONSERVER_RESPONSE_COMPLETE_FINAL, err), "failed sending response"); TRITONSERVER_ErrorDelete(err); (*responses)[r] = nullptr; // Reset the release flags for the request. pb_infer_requests[r]->SetReleaseFlags( TRITONSERVER_REQUEST_RELEASE_ALL); // If has_error is true, we do not look at the response tensors. continue; } } catch (const PythonBackendException& pb_exception) { TRITONSERVER_Error* err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, pb_exception.what()); LOG_IF_ERROR( TRITONBACKEND_ResponseSend( (*responses)[r], TRITONSERVER_RESPONSE_COMPLETE_FINAL, err), "failed sending response"); TRITONSERVER_ErrorDelete(err); (*responses)[r] = nullptr; // Reset the release flags for the request. pb_infer_requests[r]->SetReleaseFlags(TRITONSERVER_REQUEST_RELEASE_ALL); continue; } GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestOutputCount(request, &requested_output_count)); std::set requested_output_names; for (size_t j = 0; j < requested_output_count; ++j) { const char* output_name; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestOutputName(request, j, &output_name)); requested_output_names.insert(output_name); } bool require_deferred_callback = false; #ifdef TRITON_ENABLE_GPU for (auto& output_tensor : infer_response->OutputTensors()) { if (output_tensor->MemoryType() == TRITONSERVER_MEMORY_GPU) { // Attempt to use the cuda shared memory pool for GPU tensor. ShareCUDAMemoryPool(output_tensor->MemoryTypeId()); } } #endif // TRITON_ENABLE_GPU gpu_output_buffers[r] = std::vector, void*>>{}; infer_response->Send( response, CudaStream(), require_deferred_callback, TRITONSERVER_RESPONSE_COMPLETE_FINAL, Stub()->ShmPool(), gpu_buffer_helper, gpu_output_buffers[r], requested_output_names); requires_deferred_callback[r] = require_deferred_callback; if (requires_deferred_callback[r]) { has_gpu_output = true; } } execute_finalize.Complete(); // If the output tensor is in GPU, there will be a second round trip // required for filling the GPU buffers provided by the main process. if (has_gpu_output) { ipc_message->Command() = PYTHONSTUB_CommandType::PYTHONSTUB_LoadGPUBuffers; gpu_buffer_helper.Complete(Stub()->ShmPool()); ipc_message->Args() = gpu_buffer_helper.ShmHandle(); bi::managed_external_buffer::handle_t response_message; SendMessageAndReceiveResponse( ipc_message->ShmHandle(), response_message, responses, requests, 0); bool cuda_copy = false; uint32_t response_index = 0; for (auto& gpu_output_buffer : gpu_output_buffers) { for (auto& buffer_memory_pair : gpu_output_buffer) { auto& pb_memory = buffer_memory_pair.first; void* pointer = buffer_memory_pair.second; bool cuda_used = false; if (pb_memory->MemoryType() == TRITONSERVER_MEMORY_CPU) { GUARDED_RESPOND_IF_ERROR( responses, response_index, CopyBuffer( "Failed to copy the output tensor to buffer.", TRITONSERVER_MEMORY_CPU, 0, TRITONSERVER_MEMORY_CPU, 0, pb_memory->ByteSize(), pb_memory->DataPtr(), pointer, CudaStream(), &cuda_used)); cuda_copy |= cuda_used; } else if ( (pb_memory->MemoryType() == TRITONSERVER_MEMORY_GPU) && pb_memory->UseCUDASharedPool() && (pb_memory->DataPtr() != pointer)) { // If the data pointer from pb_memory is not the same as the // pointer, it means that the Triton-provided buffer is not used // during tensor transfer. Instead, an intermediate buffer that uses // CUDA shared memory pool is used. In this case, we need to copy // the data from the intermediate buffer back to the Triton-provided // buffer. GUARDED_RESPOND_IF_ERROR( responses, response_index, CopyBuffer( "Failed to copy the output tensor to buffer.", TRITONSERVER_MEMORY_GPU, pb_memory->MemoryTypeId(), TRITONSERVER_MEMORY_GPU, pb_memory->MemoryTypeId(), pb_memory->ByteSize(), pb_memory->DataPtr(), pointer, CudaStream(), &cuda_used)); cuda_copy |= cuda_used; } } response_index++; #ifdef TRITON_ENABLE_GPU if (cuda_copy) { cudaStreamSynchronize(stream_); } #endif // TRITON_ENABLE_GPU } } for (uint32_t r = 0; r < request_count; ++r) { if (requires_deferred_callback[r]) { shm_responses[r]->DeferredSendCallback(); } } } return nullptr; // success } void ModelInstanceState::PrepareResponseBatch( ResponseBatch** response_batch, AllocatedSharedMemory& response_batch_shm, std::unique_ptr* ipc_message, bi::managed_external_buffer::handle_t** response_handle) { response_batch_shm = Stub()->ShmPool()->Construct( sizeof(ResponseBatch) + sizeof(bi::managed_external_buffer::handle_t)); *response_batch = reinterpret_cast(response_batch_shm.data_.get()); (*ipc_message)->Args() = response_batch_shm.handle_; *response_handle = reinterpret_cast( response_batch_shm.data_.get() + sizeof(ResponseBatch)); (*response_batch)->batch_size = 1; (*response_batch)->has_error = false; (*response_batch)->is_error_set = false; (*response_batch)->cleanup = false; (*response_batch)->response_size = 1; } void ModelInstanceState::PrepareResponseHandle( std::unique_ptr* infer_response, bi::managed_external_buffer::handle_t* response_handle) { #ifdef TRITON_ENABLE_GPU for (auto& output_tensor : (*infer_response)->OutputTensors()) { if (!output_tensor->IsCPU()) { // Attempt to use the cuda shared memory pool for GPU tensor. ShareCUDAMemoryPool(output_tensor->MemoryTypeId()); // It's possible that the CUDA memory pool offset isn't set correctly, // even if the BLS output is using CUDA memory. This can occur when the // CUDA memory pool hasn't been shared with the stub process at the time // the BLS output is allocated during the ResponseAlloc callback. In such // cases, we need to adjust the CUDA pool offset accordingly. if (!output_tensor->Memory()->UseCUDASharedPool()) { output_tensor->Memory()->UpdateCUDAOffset( Stub()->ShmPool()->GetCUDAMemoryPoolManager()); } } } #endif // TRITON_ENABLE_GPU (*infer_response)->SaveToSharedMemory(Stub()->ShmPool()); for (auto& output_tensor : (*infer_response)->OutputTensors()) { if (!output_tensor->IsCPU()) { #ifdef TRITON_ENABLE_GPU std::unique_ptr memory_record; // Need to transfer the ownership of the BackendMemory to the // MemoryManager so that the lifetime of the BackendMemory is managed. memory_record = std::make_unique( output_tensor->Memory()->GetBackendMemory()); uint64_t memory_release_id = Stub()->GetMemoryManager()->AddRecord(std::move(memory_record)); output_tensor->Memory()->SetMemoryReleaseId(memory_release_id); #endif } } *response_handle = (*infer_response)->ShmHandle(); } void ModelInstanceState::SendBLSDecoupledResponse( std::unique_ptr infer_response) { bool is_response_batch_set = false; ResponseBatch* response_batch = nullptr; std::unique_ptr pb_error_message; std::unique_ptr ipc_message; AllocatedSharedMemory response_batch_shm; bi::managed_external_buffer::handle_t* response_handle = nullptr; try { ipc_message = IPCMessage::Create(Stub()->ShmPool(), true /* inline_response */); ipc_message->Args() = response_batch_shm.handle_; ipc_message->Command() = PYTHONSTUB_InferStreamExecResponse; PrepareResponseBatch( &response_batch, response_batch_shm, &ipc_message, &response_handle); is_response_batch_set = true; response_batch->waiting_on_stub = false; PrepareResponseHandle(&infer_response, response_handle); } catch (const PythonBackendException& pb_exception) { if (is_response_batch_set) { response_batch->has_error = true; LOG_IF_EXCEPTION( pb_error_message = PbString::Create(Stub()->ShmPool(), pb_exception.what())); if (pb_error_message != nullptr) { response_batch->is_error_set = true; response_batch->error = pb_error_message->ShmHandle(); } } else { LOG_MESSAGE(TRITONSERVER_LOG_ERROR, pb_exception.what()); } } { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; Stub()->ParentToStubMessageQueue()->Push(ipc_message->ShmHandle()); while (!response_batch->waiting_on_stub) { ipc_message->ResponseCondition()->wait(lock); } } } void ModelInstanceState::ShareCUDAMemoryPool(const int32_t device_id) { #ifdef TRITON_ENABLE_GPU try { Stub()->ShareCUDAMemoryPool(Model()->TritonMemoryManager(), device_id); } catch (const PythonBackendException& ex) { LOG_MESSAGE( TRITONSERVER_LOG_WARN, (std::string("Failed to share CUDA memory pool with stub process: ") + ex.what() + ". Will use CUDA IPC.") .c_str()); } #endif // TRITON_ENABLE_GPU } ModelInstanceState::~ModelInstanceState() { Stub()->UpdateHealth(); if (Stub()->IsHealthy()) { // Wait for all the pending tasks to finish. thread_pool_->wait(); } // Terminate stub first to allow any last messages to be received by the back // end before deallocating the queue memory Stub()->TerminateStub(); TerminateMonitor(); Stub()->ClearQueues(); Stub().reset(); } TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } // Auto-complete the configuration if requested... bool auto_complete_config = false; RETURN_IF_ERROR(TRITONBACKEND_ModelAutoCompleteConfig( triton_model, &auto_complete_config)); if (auto_complete_config) { RETURN_IF_ERROR((*state)->LaunchAutoCompleteStubProcess()); (*state)->ModelConfig() = std::move((*state)->Stub()->AutoCompleteConfig()); RETURN_IF_ERROR((*state)->SetModelConfig()); (*state)->Stub()->UpdateHealth(); (*state)->Stub()->TerminateStub(); (*state)->Stub()->ClearQueues(); (*state)->Stub().reset(); } RETURN_IF_ERROR((*state)->ValidateModelConfig()); return nullptr; // success } ModelState::ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model, true /* allow_optional */) { TRITONBACKEND_Backend* backend; THROW_IF_BACKEND_MODEL_ERROR( TRITONBACKEND_ModelBackend(triton_model, &backend)); const char* path = nullptr; TRITONBACKEND_ArtifactType artifact_type; THROW_IF_BACKEND_MODEL_ERROR( TRITONBACKEND_ModelRepository(triton_model, &artifact_type, &path)); python_execution_env_ = ""; force_cpu_only_input_tensors_ = true; decoupled_ = false; void* bstate; THROW_IF_BACKEND_MODEL_ERROR(TRITONBACKEND_BackendState(backend, &bstate)); backend_state_ = reinterpret_cast(bstate); runtime_modeldir_ = backend_state_->runtime_modeldir; triton::common::TritonJson::Value params; common::TritonJson::Value model_config; if (model_config_.Find("parameters", ¶ms)) { // Skip the EXECUTION_ENV_PATH variable if it doesn't exist. TRITONSERVER_Error* error = GetParameterValue(params, "EXECUTION_ENV_PATH", &python_execution_env_); if (error == nullptr) { std::string relative_path_keyword = "$$TRITON_MODEL_DIRECTORY"; size_t relative_path_loc = python_execution_env_.find(relative_path_keyword); if (relative_path_loc != std::string::npos) { python_execution_env_.replace( relative_path_loc, relative_path_loc + relative_path_keyword.size(), path); } LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Using Python execution env ") + python_execution_env_) .c_str()); } else { // Delete the error TRITONSERVER_ErrorDelete(error); } triton::common::TritonJson::Value model_transaction_policy; if (model_config_.Find( "model_transaction_policy", &model_transaction_policy)) { triton::common::TritonJson::Value decoupled; if (model_transaction_policy.Find("decoupled", &decoupled)) { auto error = decoupled.AsBool(&decoupled_); if (error != nullptr) { throw BackendModelException(error); } } } // Skip the FORCE_CPU_ONLY_INPUT_TENSORS variable if it doesn't exits. std::string force_cpu_only_input_tensor; error = nullptr; error = GetParameterValue( params, "FORCE_CPU_ONLY_INPUT_TENSORS", &force_cpu_only_input_tensor); if (error == nullptr) { if (force_cpu_only_input_tensor == "yes") { force_cpu_only_input_tensors_ = true; LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Forcing CPU only input tensors.")).c_str()); } else if (force_cpu_only_input_tensor == "no") { force_cpu_only_input_tensors_ = false; LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Input tensors can be both in CPU and GPU. " "FORCE_CPU_ONLY_INPUT_TENSORS is off.")) .c_str()); } else { throw BackendModelException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, (std::string("Incorrect value for FORCE_CPU_ONLY_INPUT_TENSORS: ") + force_cpu_only_input_tensor + "'") .c_str())); } } else { // Delete the error TRITONSERVER_ErrorDelete(error); } } if (artifact_type != TRITONBACKEND_ARTIFACT_FILESYSTEM) { throw BackendModelException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, (std::string("unsupported artifact type for model '") + Name() + "'") .c_str())); } } TRITONSERVER_Error* ModelState::LaunchAutoCompleteStubProcess() { Stub() = std::make_unique("AUTOCOMPLETE_STUB"); RETURN_IF_ERROR(Stub()->Initialize(this)); try { RETURN_IF_ERROR(Stub()->Setup()); RETURN_IF_ERROR(Stub()->Launch()); } catch (const BackendModelException& ex) { Stub()->UpdateHealth(); Stub()->TerminateStub(); Stub()->ClearQueues(); Stub().reset(); RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; } TRITONSERVER_Error* ModelState::ValidateModelConfig() { // We have the json DOM for the model configuration... triton::common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(ModelConfig().PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("model configuration:\n") + buffer.Contents()).c_str()); return nullptr; } TRITONSERVER_Error* ModelState::SetModelConfig() { BackendModel::SetModelConfig(); // `Update model_transaction_policy` if setting was set // with `set_model_transaction_policy` triton::common::TritonJson::Value model_transaction_policy; bool is_decoupled = false; if (ModelConfig().Find( "model_transaction_policy", &model_transaction_policy)) { triton::common::TritonJson::Value decoupled; if (model_transaction_policy.Find("decoupled", &decoupled)) { auto error = decoupled.AsBool(&is_decoupled); if (error != nullptr) { throw BackendModelException(error); } SetDecoupled(is_decoupled); } } return nullptr; } extern "C" { TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); // Check backend version to ensure compatibility uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("'") + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Triton backend API version does not support this backend"); } TRITONSERVER_Message* backend_config_message; RETURN_IF_ERROR( TRITONBACKEND_BackendConfig(backend, &backend_config_message)); const char* buffer; size_t byte_size; RETURN_IF_ERROR(TRITONSERVER_MessageSerializeToJson( backend_config_message, &buffer, &byte_size)); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("backend configuration:\n") + buffer).c_str()); triton::common::TritonJson::Value backend_config; if (byte_size != 0) { RETURN_IF_ERROR(backend_config.Parse(buffer, byte_size)); } std::unique_ptr backend_state(new BackendState()); triton::common::TritonJson::Value cmdline; backend_state->shm_default_byte_size = 1 * 1024 * 1024; // 1 MB backend_state->shm_growth_byte_size = 1 * 1024 * 1024; // 1 MB backend_state->stub_timeout_seconds = 30; backend_state->shm_message_queue_size = 1000; backend_state->thread_pool_size = 32; // Initialize shared memory region prefix to include backend's name // to avoid collision between python backend and python-based backends. backend_state->shared_memory_region_prefix = "triton_" + name + "_backend_shm_region_"; std::string default_backend_dir_string; if (backend_config.Find("cmdline", &cmdline)) { triton::common::TritonJson::Value shm_growth_size; std::string shm_growth_byte_size; if (cmdline.Find("shm-growth-byte-size", &shm_growth_size)) { RETURN_IF_ERROR(shm_growth_size.AsString(&shm_growth_byte_size)); try { backend_state->shm_growth_byte_size = std::stol(shm_growth_byte_size); if (backend_state->shm_growth_byte_size <= 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("shm-growth-byte-size") + " can't be smaller than or equal to zero.") .c_str()); } } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, ia.what()); } } triton::common::TritonJson::Value shm_default_size; std::string shm_default_byte_size; if (cmdline.Find("shm-default-byte-size", &shm_default_size)) { RETURN_IF_ERROR(shm_default_size.AsString(&shm_default_byte_size)); try { backend_state->shm_default_byte_size = std::stol(shm_default_byte_size); // Shared memory default byte size can't be less than 1 MB. if (backend_state->shm_default_byte_size < 1 * 1024 * 1024) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("shm-default-byte-size") + " can't be smaller than 4 MiBs") .c_str()); } } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, ia.what()); } } triton::common::TritonJson::Value thread_pool_size; std::string thread_pool_count; if (cmdline.Find("thread-pool-size", &thread_pool_size)) { RETURN_IF_ERROR(thread_pool_size.AsString(&thread_pool_count)); try { backend_state->thread_pool_size = std::stol(thread_pool_count); // Shared memory default byte size can't be less than 4 MBs. if (backend_state->thread_pool_size < 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("thread-pool-size") + " can't be less than 1.") .c_str()); } } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, ia.what()); } } triton::common::TritonJson::Value shm_region_prefix; std::string shm_region_prefix_str; if (cmdline.Find("shm-region-prefix-name", &shm_region_prefix)) { RETURN_IF_ERROR(shm_region_prefix.AsString(&shm_region_prefix_str)); // Shared memory default byte size can't be less than 4 MBs. if (shm_region_prefix_str.size() == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("shm-region-prefix-name") + " must at least contain one character.") .c_str()); } backend_state->shared_memory_region_prefix = shm_region_prefix_str; } triton::common::TritonJson::Value shm_message_queue_size; std::string shm_message_queue_size_str; if (cmdline.Find("shm_message_queue_size", &shm_message_queue_size)) { RETURN_IF_ERROR( shm_message_queue_size.AsString(&shm_message_queue_size_str)); try { backend_state->shm_message_queue_size = std::stol(shm_message_queue_size_str); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, ia.what()); } } triton::common::TritonJson::Value stub_timeout_seconds; std::string stub_timeout_string_seconds; if (cmdline.Find("stub-timeout-seconds", &stub_timeout_seconds)) { RETURN_IF_ERROR( stub_timeout_seconds.AsString(&stub_timeout_string_seconds)); try { backend_state->stub_timeout_seconds = std::stol(stub_timeout_string_seconds); if (backend_state->stub_timeout_seconds <= 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("stub-timeout-seconds") + " can't be smaller than or equal to zero.") .c_str()); } } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, ia.what()); } } triton::common::TritonJson::Value default_backend_dir; if (cmdline.Find("backend-directory", &default_backend_dir)) { RETURN_IF_ERROR( default_backend_dir.AsString(&default_backend_dir_string)); } } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Shared memory configuration is shm-default-byte-size=") + std::to_string(backend_state->shm_default_byte_size) + ",shm-growth-byte-size=" + std::to_string(backend_state->shm_growth_byte_size) + ",stub-timeout-seconds=" + std::to_string(backend_state->stub_timeout_seconds)) .c_str()); // Use BackendArtifacts to determine the location of Python files const char* clocation; TRITONBACKEND_ArtifactType artifact_type; RETURN_IF_ERROR( TRITONBACKEND_BackendArtifacts(backend, &artifact_type, &clocation)); const char os_slash = std::filesystem::path::preferred_separator; std::string location(clocation); #ifdef _WIN32 const std::string stub_executable_name = "triton_python_backend_stub.exe"; SanitizePath(location); SanitizePath(default_backend_dir_string); #else const std::string stub_executable_name = "triton_python_backend_stub"; #endif // Check if `triton_python_backend_stub` and `triton_python_backend_utils.py` // are located under `location`. std::string default_python_backend_dir = default_backend_dir_string + os_slash + "python"; std::string backend_stub_path = location + os_slash + stub_executable_name; std::string backend_utils = location + os_slash + "triton_python_backend_utils.py"; // Both, stub and utils should be in the same location if (FileExists(backend_stub_path) && FileExists(backend_utils)) { backend_state->python_lib = location; // If `location` is default location of a python backend, // then we are using default python backend. if (default_python_backend_dir == location) { backend_state->runtime_modeldir = ""; } else { // If `location` is not default location of a python backend, // then we are using a python backend based backend and model.py stored // in the received location. backend_state->runtime_modeldir = location; } } else { // If stub and utils are not found in received `location`, // then we are using a python backend based backend and stub and utils are // stored in the default python backend location. if (!default_backend_dir_string.empty()) { std::string backend_stub_path = default_backend_dir_string + os_slash + "python" + os_slash + stub_executable_name; if (!FileExists(backend_stub_path)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, (stub_executable_name + " is not found. Searched paths: " + default_backend_dir_string + os_slash + "python and " + location) .c_str()); } } backend_state->runtime_modeldir = location; backend_state->python_lib = default_backend_dir_string + os_slash + "python"; } // FIXME [DLIS-5969]: Enable for Windows when custom execution environments // are supported. #ifndef _WIN32 backend_state->env_manager = std::make_unique(); #endif RETURN_IF_ERROR(TRITONBACKEND_BackendSetState( backend, reinterpret_cast(backend_state.get()))); backend_state.release(); return nullptr; } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Finalize(TRITONBACKEND_Backend* backend) { LOG_MESSAGE(TRITONSERVER_LOG_VERBOSE, "TRITONBACKEND_Finalize: Start"); void* vstate; RETURN_IF_ERROR(TRITONBACKEND_BackendState(backend, &vstate)); auto backend_state = reinterpret_cast(vstate); delete backend_state; LOG_MESSAGE(TRITONSERVER_LOG_VERBOSE, "TRITONBACKEND_Finalize: End"); return nullptr; // success } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); TRITONSERVER_LogMessage( TRITONSERVER_LOG_VERBOSE, __FILE__, __LINE__, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); TRITONBACKEND_Backend* backend; RETURN_IF_ERROR(TRITONBACKEND_ModelBackend(model, &backend)); ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast(model_state))); return nullptr; } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return nullptr; } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast(vmodelstate); ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast(instance_state))); RETURN_IF_ERROR(instance_state->LaunchStubProcess()); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("TRITONBACKEND_ModelInstanceInitialize: instance " "initialization successful ") + name + " (device " + std::to_string(device_id) + ")") .c_str()); return nullptr; } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast(&instance_state))); TRITONSERVER_Error* error = nullptr; // If restart is equal to true, it indicates that the stub process is // unhealthy and needs a restart. // TODO: Implement restart on decoupled std::vector> infer_requests; { uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); PbMetricReporter reporter( instance_state->TritonModelInstance(), requests, request_count, nullptr); reporter.SetExecStartNs(exec_start_ns); error = instance_state->ProcessRequests( requests, request_count, infer_requests, reporter); uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); reporter.SetExecEndNs(exec_end_ns); if (error != nullptr) { reporter.SetSuccessStatus(false); for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; if (!instance_state->ExistsInClosedRequests( reinterpret_cast(request))) { TRITONBACKEND_Response* response = nullptr; LOG_IF_ERROR( TRITONBACKEND_ResponseNew(&response, request), "Failed to create a new response."); if (response != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, error), "Failed to send the error response."); } } } for (auto& infer_request : infer_requests) { // Reset the release flags for all the requests. infer_request->SetReleaseFlags(TRITONSERVER_REQUEST_RELEASE_ALL); } } } // The InferRequest object might not be created if an error occurs. Explicitly // update the release flags here based on the number of InferRequest objects. std::vector request_release_flags( request_count, TRITONSERVER_REQUEST_RELEASE_ALL); for (size_t i = 0; i < infer_requests.size(); ++i) { request_release_flags[i] = infer_requests[i]->ReleaseFlags(); } for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; try { THROW_IF_TRITON_ERROR( TRITONBACKEND_RequestRelease(request, request_release_flags[r])); } catch (const PythonBackendException& pb_exception) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("Failed to release request: ") + pb_exception.what()) .c_str()); if (request_release_flags[r] == TRITONSERVER_REQUEST_RELEASE_RESCHEDULE) { // If error occurs during request rescheduling, release the request with // `TRITONSERVER_REQUEST_RELEASE_ALL` flag. LOG_IF_ERROR( TRITONBACKEND_RequestRelease( request, TRITONSERVER_REQUEST_RELEASE_ALL), "Failed to release request."); } } } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("TRITONBACKEND_ModelInstanceExecute: model instance name ") + instance_state->Name() + " released " + std::to_string(request_count) + " requests") .c_str()); return nullptr; } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast(vstate); LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; } TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_GetBackendAttribute( TRITONBACKEND_Backend* backend, TRITONBACKEND_BackendAttribute* backend_attributes) { LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, "TRITONBACKEND_GetBackendAttribute: setting attributes"); // Specify different preferred instance kind based on backend compatibility, // so Triton core won't blindly auto-complete kind that may not be supported. // Other instance groups setting are set to "no value" so that Triton core // will auto-complete them with default policy. #ifdef TRITON_ENABLE_GPU RETURN_IF_ERROR(TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( backend_attributes, TRITONSERVER_INSTANCEGROUPKIND_GPU, 0, nullptr, 0)); #else RETURN_IF_ERROR(TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup( backend_attributes, TRITONSERVER_INSTANCEGROUPKIND_CPU, 0, nullptr, 0)); #endif // This backend can safely handle parallel calls to // TRITONBACKEND_ModelInstanceInitialize (thread-safe). RETURN_IF_ERROR(TRITONBACKEND_BackendAttributeSetParallelModelInstanceLoading( backend_attributes, true)); return nullptr; } } // extern "C" }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/python_be.h000066400000000000000000000451421527052371200254110ustar00rootroot00000000000000// Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "infer_request.h" #include "infer_response.h" #include "ipc_message.h" #include "memory_manager.h" #include "message_queue.h" #include "metric.h" #include "metric_family.h" #include "pb_env.h" #include "pb_map.h" #include "pb_metric_reporter.h" #include "pb_utils.h" #include "request_executor.h" #include "scoped_defer.h" #include "shm_manager.h" #include "stub_launcher.h" #include "triton/backend/backend_common.h" #include "triton/backend/backend_input_collector.h" #include "triton/backend/backend_memory.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" #include "triton/common/nvtx.h" #include "triton/common/triton_json.h" #include "triton/core/tritonbackend.h" #include "triton/core/tritonserver.h" #ifdef _WIN32 #define NOMINMAX #include #else #include #include #endif #define LOG_IF_EXCEPTION(X) \ do { \ try { \ (X); \ } \ catch (const PythonBackendException& pb_exception) { \ LOG_MESSAGE(TRITONSERVER_LOG_ERROR, pb_exception.what()); \ } \ } while (false) #define RESPOND_ALL_AND_RETURN_IF_ERROR(RESPONSES, RESPONSES_COUNT, X) \ do { \ TRITONSERVER_Error* raasnie_err__ = (X); \ if (raasnie_err__ != nullptr) { \ for (size_t ridx = 0; ridx < RESPONSES_COUNT; ++ridx) { \ if ((*RESPONSES)[ridx] != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (*RESPONSES)[ridx], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ raasnie_err__), \ "failed to send error response"); \ (*RESPONSES)[ridx] = nullptr; \ } \ } \ TRITONSERVER_ErrorDelete(raasnie_err__); \ return; \ } \ } while (false) #define RESPOND_ALL_AND_RETURN_IF_EXCEPTION(RESPONSES, RESPONSES_COUNT, X) \ do { \ try { \ (X); \ } \ catch (const PythonBackendException& exception) { \ TRITONSERVER_Error* raarie_err__ = TRITONSERVER_ErrorNew( \ TRITONSERVER_ERROR_INTERNAL, exception.what()); \ for (size_t ridx = 0; ridx < RESPONSES_COUNT; ++ridx) { \ if ((*RESPONSES)[ridx] != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (*RESPONSES)[ridx], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ raarie_err__), \ "failed to send error response"); \ (*RESPONSES)[ridx] = nullptr; \ } \ } \ TRITONSERVER_ErrorDelete(raarie_err__); \ return; \ } \ } while (false) #define RESPOND_AND_RETURN_IF_ERROR(REQUEST, X) \ do { \ TRITONSERVER_Error* rarie_err__ = (X); \ if (rarie_err__ != nullptr) { \ TRITONBACKEND_Response* rarie_response__ = nullptr; \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseNew(&rarie_response__, REQUEST), \ "failed to create response"); \ if (rarie_response__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ rarie_response__, TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ rarie_err__), \ "failed to send error response"); \ } \ return rarie_err__; \ } \ } while (false) #define GUARDED_RESPOND_IF_ERROR(RESPONSES, IDX, X) \ do { \ if ((*RESPONSES)[IDX] != nullptr) { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (*RESPONSES)[IDX], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ err__), \ "failed to send error response"); \ (*RESPONSES)[IDX] = nullptr; \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) #define GUARDED_RESPOND_IF_EXCEPTION(RESPONSES, IDX, X) \ do { \ if ((*RESPONSES)[IDX] != nullptr) { \ try { \ (X); \ } \ catch (const PythonBackendException& pb_exception) { \ TRITONSERVER_Error* err__ = TRITONSERVER_ErrorNew( \ TRITONSERVER_ERROR_INTERNAL, pb_exception.what()); \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (*RESPONSES)[IDX], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ err__), \ "failed to send error response"); \ (*RESPONSES)[IDX] = nullptr; \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) #define RETURN_IF_EXCEPTION(X) \ do { \ try { \ (X); \ } \ catch (const PythonBackendException& pb_exception) { \ TRITONSERVER_Error* rarie_err__ = TRITONSERVER_ErrorNew( \ TRITONSERVER_ERROR_INTERNAL, pb_exception.what()); \ return rarie_err__; \ } \ } while (false) namespace triton { namespace backend { namespace python { namespace bi = boost::interprocess; struct BackendState { std::string python_lib; int64_t shm_default_byte_size; int64_t shm_growth_byte_size; int64_t stub_timeout_seconds; int64_t shm_message_queue_size; std::atomic number_of_instance_inits; std::string shared_memory_region_prefix; int64_t thread_pool_size; // FIXME [DLIS-5969]: Enable for Windows when custom execution environments // are supported. #ifndef _WIN32 std::unique_ptr env_manager; #endif std::string runtime_modeldir; }; class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); // Get backend state BackendState* StateForBackend() { return backend_state_; } // Get the Python execution environment std::string PythonExecutionEnv() { return python_execution_env_; } // Force CPU only tensors bool ForceCPUOnlyInputTensors() { return force_cpu_only_input_tensors_; } // Is decoupled API being used. bool IsDecoupled() { return decoupled_; } // Set decoupled mode void SetDecoupled(bool decoupled) { decoupled_ = decoupled; } // Returns the value in the `runtime_modeldir_` field std::string RuntimeModelDir() { return runtime_modeldir_; } // Launch auto-complete stub process. TRITONSERVER_Error* LaunchAutoCompleteStubProcess(); // Validate Model Configuration TRITONSERVER_Error* ValidateModelConfig(); // Overrides `BackendModel::SetModelConfig` to also // set `ModelState::decoupled_` TRITONSERVER_Error* SetModelConfig(); // Auto-complete stub std::unique_ptr& Stub() { return auto_complete_stub_; } private: ModelState(TRITONBACKEND_Model* triton_model); BackendState* backend_state_; std::string python_execution_env_; bool force_cpu_only_input_tensors_; bool decoupled_; std::string runtime_modeldir_; std::unique_ptr auto_complete_stub_; }; class ModelInstanceState : public BackendModelInstance { ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* model_instance); TRITONBACKEND_Model* triton_model_; std::unique_ptr model_instance_stub_; std::vector closed_requests_; std::mutex closed_requests_mutex_; std::thread stub_to_parent_queue_monitor_; bool stub_to_parent_thread_; std::mutex mu_; std::condition_variable cv_; std::unique_ptr received_message_; std::vector> futures_; std::unique_ptr thread_pool_; std::unordered_map> infer_payload_; std::mutex infer_payload_mu_; std::unique_ptr request_executor_; public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* model_instance, ModelInstanceState** model_instance_state); ~ModelInstanceState(); // Launch stub process. TRITONSERVER_Error* LaunchStubProcess(); void ResponseSendDecoupled(std::shared_ptr response_send_message); // The parent message queue is monitored only by this function during the // execute phase. No other thread should pop any message from the message // queue. void MessageQueueMonitor(); // This function is executed on a separate thread and monitors the queue for // message sent from stub to parent process. void StubToParentMQMonitor(); // Process the log request. void ProcessLogRequest(const std::unique_ptr& message); // Convert TRITONBACKEND_Input to Python backend tensors. TRITONSERVER_Error* GetInputTensor( const uint32_t input_idx, std::shared_ptr& input_tensor, TRITONBACKEND_Request* request, std::shared_ptr>& responses); // Process all the requests in the decoupled mode. TRITONSERVER_Error* ProcessRequests( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector>& pb_infer_requests, PbMetricReporter& pb_metric_reporter); bool ExistsInClosedRequests(intptr_t closed_request); // Execute a BLS Request void ExecuteBLSRequest( std::shared_ptr ipc_message, const bool is_stream); // Cleanup BLS responses void CleanupBLSResponses(); // Check the incoming requests for errors TRITONSERVER_Error* CheckIncomingRequests( TRITONBACKEND_Request** requests, const uint32_t request_count, size_t& total_batch_size); // Set error for response send message void SetErrorForResponseSendMessage( ResponseSendMessage* response_send_message, std::shared_ptr error, std::unique_ptr& error_message); TRITONSERVER_Error* SaveRequestsToSharedMemory( TRITONBACKEND_Request** requests, const uint32_t request_count, std::vector>& pb_inference_requests, AllocatedSharedMemory& request_batch, std::shared_ptr>& responses); void SendMessageAndReceiveResponse( bi::managed_external_buffer::handle_t message, bi::managed_external_buffer::handle_t& response, std::shared_ptr>& responses, TRITONBACKEND_Request** requests, const uint32_t request_count); void RespondErrorToAllRequests( const char* message, std::shared_ptr>& responses, TRITONBACKEND_Request** requests, const uint32_t request_count); // void SendMessageToStub(bi::managed_external_buffer::handle_t message); TRITONSERVER_Error* SendMessageToStub( bi::managed_external_buffer::handle_t message); // Checks whether the stub process is live bool IsStubProcessAlive(); // Model instance stub std::unique_ptr& Stub() { return model_instance_stub_; } // Stop the stub_to_parent_queue_monitor thread void TerminateMonitor(); // Start the stub_to_parent_queue_monitor thread void StartMonitor(); // Send bls decoupled response to the stub process void SendBLSDecoupledResponse(std::unique_ptr infer_response); // Prepare the response batch object void PrepareResponseBatch( ResponseBatch** response_batch, AllocatedSharedMemory& response_batch_shm, std::unique_ptr* ipc_message, bi::managed_external_buffer::handle_t** response_handle); // Prepare the response handle void PrepareResponseHandle( std::unique_ptr* infer_response, bi::managed_external_buffer::handle_t* response_handle); // Process the decoupled cleanup request for InferPayload and ResponseFactory void ProcessCleanupRequest(const std::unique_ptr& message); // Process request cancellation query void ProcessIsRequestCancelled(const std::unique_ptr& message); // Process a message. The function 'request_handler' is invoked // to handle the request. T should be either 'MetricFamily', 'Metric' or // 'ModelLoader', and MessageType should be either 'MetricFamilyMessage', // 'MetricMessage' or 'ModelLoaderMessage'. template void ProcessMessage( const std::unique_ptr& message, std::function&, MessageType*)> request_handler); // Process a metric family request void ProcessMetricFamilyRequest(const std::unique_ptr& message); // Process a metric request void ProcessMetricRequest(const std::unique_ptr& message); // Process a model control request void ProcessModelControlRequest(const std::unique_ptr& message); // Attempt to share CUDA memory pool with the stub process void ShareCUDAMemoryPool(const int32_t device_id); }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/request_executor.cc000066400000000000000000000431031527052371200271610ustar00rootroot00000000000000// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "request_executor.h" #include #include "correlation_id.h" #include "pb_utils.h" #include "scoped_defer.h" #include "triton/backend/backend_common.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { namespace python { TRITONSERVER_Error* CreateTritonErrorFromException(const PythonBackendException& pb_exception) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, pb_exception.what()); } TRITONSERVER_Error* MemoryTypeToTritonMemoryType( TRITONSERVER_MemoryType* triton_memory_type, const PreferredMemory::MemoryType& memory_type) { switch (memory_type) { case PreferredMemory::MemoryType::kCPU: *triton_memory_type = TRITONSERVER_MEMORY_CPU; break; case PreferredMemory::MemoryType::kGPU: *triton_memory_type = TRITONSERVER_MEMORY_GPU; break; default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Unknown memory type"); } return nullptr; } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { if (request != nullptr) { LOG_IF_ERROR( TRITONSERVER_InferenceRequestDelete(request), "Failed to delete inference request."); } } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { auto linfer_payload = reinterpret_cast(userp); std::shared_ptr infer_payload = linfer_payload->GetPtr(); std::unique_ptr infer_response; std::vector> output_tensors; std::shared_ptr pb_error; if (response != nullptr) { try { THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceResponseError(response)); uint32_t output_count; THROW_IF_TRITON_ERROR( TRITONSERVER_InferenceResponseOutputCount(response, &output_count)); for (uint32_t idx = 0; idx < output_count; ++idx) { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceResponseOutput( response, idx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); std::string sname = cname; std::vector dims_vector{shape, shape + dim_count}; if (memory_type != TRITONSERVER_MEMORY_GPU) { if (byte_size != 0) { std::shared_ptr pb_tensor = std::make_shared( sname, dims_vector, datatype, memory_type, memory_type_id, const_cast(base), byte_size, nullptr /* DLManagedTensor */); // Load the data so that it is deallocated automatically. std::unique_ptr pb_memory( reinterpret_cast(userp)); pb_tensor->SetMemory(std::move(pb_memory)); output_tensors.push_back(pb_tensor); } else { output_tensors.push_back(std::make_shared( sname, dims_vector, datatype, memory_type, memory_type_id, const_cast(base), byte_size, nullptr /* DLManagedTensor */)); } } else { std::shared_ptr pb_tensor = std::make_shared( sname, dims_vector, datatype, memory_type, memory_type_id, const_cast(base), byte_size, nullptr /* DLManagedTensor */); std::unique_ptr pb_memory( reinterpret_cast(userp)); pb_tensor->SetMemory(std::move(pb_memory)); output_tensors.push_back(pb_tensor); } } } catch (const PythonBackendException& pb_exception) { if (response != nullptr) { LOG_IF_ERROR( TRITONSERVER_InferenceResponseDelete(response), "Failed to delete inference response."); response = nullptr; } pb_error = std::make_shared(pb_exception.what()); output_tensors.clear(); } if (!infer_payload->IsDecoupled()) { infer_response = std::make_unique( output_tensors, pb_error, true /* is_last_response */); } else { if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { // Not the last response. infer_response = std::make_unique( output_tensors, pb_error, false /* is_last_response */, userp /* id */); } else { // The last response. infer_response = std::make_unique( output_tensors, pb_error, true /* is_last_response */, userp /* id */); } } LOG_IF_ERROR( TRITONSERVER_InferenceResponseDelete(response), "Failed to release BLS inference response."); } else if ( (infer_payload)->IsDecoupled() && (flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) != 0) { // An empty response may be the last response for decoupled models. infer_response = std::make_unique( output_tensors, pb_error, true /* is_last_response */, userp /* id */); } else { pb_error = std::make_shared("Unexpected empty response."); infer_response = std::make_unique( output_tensors, pb_error, true /* is_last_response */, userp /* id */); } infer_payload->SetValue(std::move(infer_response)); } TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { auto p = reinterpret_cast(userp); std::unique_ptr shm_pool( reinterpret_cast(p->shm_pool)); ScopedDefer _([&shm_pool] { shm_pool.release(); }); if (p->preferred_memory.PreferredMemoryType() == PreferredMemory::MemoryType::kDefault) { *actual_memory_type = preferred_memory_type; *actual_memory_type_id = preferred_memory_type_id; } else { TRITONSERVER_MemoryType user_preferred_memory_type; RETURN_IF_ERROR(MemoryTypeToTritonMemoryType( &user_preferred_memory_type, p->preferred_memory.PreferredMemoryType())); *actual_memory_type = user_preferred_memory_type; *actual_memory_type_id = p->preferred_memory.PreferredDeviceId(); } // If 'byte_size' is zero just return 'buffer' == nullptr, we don't // need to do any other book-keeping. if (byte_size == 0) { *buffer = nullptr; *buffer_userp = nullptr; } else { switch (*actual_memory_type) { case TRITONSERVER_MEMORY_CPU: #ifndef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_GPU: #endif case TRITONSERVER_MEMORY_CPU_PINNED: { *actual_memory_type = TRITONSERVER_MEMORY_CPU; *actual_memory_type_id = 0; try { std::unique_ptr pb_memory = PbMemory::Create( shm_pool, *actual_memory_type, *actual_memory_type_id, byte_size, nullptr /* data */, false /* copy_gpu */); *buffer = pb_memory->DataPtr(); *buffer_userp = reinterpret_cast(pb_memory.get()); pb_memory.release(); } catch (const PythonBackendException& pb_exception) { TRITONSERVER_Error* err = CreateTritonErrorFromException(pb_exception); return err; } } break; #ifdef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_GPU: { BackendMemory* backend_memory; std::unique_ptr lbackend_memory; try { THROW_IF_TRITON_ERROR(BackendMemory::Create( reinterpret_cast( shm_pool->GetCUDAMemoryPoolManager()->TritonMemoryManager()), {BackendMemory::AllocationType::GPU_POOL, BackendMemory::AllocationType::GPU}, *actual_memory_type_id, byte_size, &backend_memory)); lbackend_memory.reset(backend_memory); std::unique_ptr pb_memory = PbMemory::Create( shm_pool, std::move(lbackend_memory), true /* copy_gpu */); *buffer = pb_memory->DataPtr(); *buffer_userp = reinterpret_cast(pb_memory.get()); pb_memory.release(); } catch (const PythonBackendException& pb_exception) { TRITONSERVER_Error* err = CreateTritonErrorFromException(pb_exception); return err; } break; } #endif } } return nullptr; // Success } TRITONSERVER_Error* OutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { // Always attempt to return the memory in the requested memory_type and // memory_type_id. return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { return nullptr; // Success } RequestExecutor::RequestExecutor( std::unique_ptr& shm_pool, TRITONSERVER_Server* server) : server_(server), shm_pool_(shm_pool) { TRITONSERVER_ResponseAllocator* allocator; THROW_IF_TRITON_ERROR(TRITONSERVER_ResponseAllocatorNew( &allocator, ResponseAlloc, ResponseRelease, nullptr /* start_fn */)); THROW_IF_TRITON_ERROR(TRITONSERVER_ResponseAllocatorSetQueryFunction( allocator, OutputBufferQuery)); response_allocator_ = allocator; } std::future> RequestExecutor::Infer( std::shared_ptr& infer_request, std::shared_ptr& infer_payload) { std::future> response_future; std::unique_ptr infer_response; bool is_ready = false; const char* model_name = infer_request->ModelName().c_str(); TRITONSERVER_InferenceRequest* irequest = nullptr; try { int64_t model_version = infer_request->ModelVersion(); THROW_IF_TRITON_ERROR(TRITONSERVER_ServerModelIsReady( server_, model_name, model_version, &is_ready)); if (!is_ready) { throw PythonBackendException( (std::string("Failed for execute the inference request. Model '") + model_name + "' is not ready.") .c_str()); } uint32_t txn_flags; THROW_IF_TRITON_ERROR(TRITONSERVER_ServerModelTransactionProperties( server_, model_name, model_version, &txn_flags, nullptr /* voidp */)); infer_request->SetIsDecoupled( (txn_flags & TRITONSERVER_TXN_DECOUPLED) != 0); if (!infer_payload->IsDecoupled() && infer_request->IsDecoupled()) { // Decoupled API is only supported by using stream API throw PythonBackendException( std::string("Model ") + model_name + " is using the decoupled. The current BLS request call doesn't " "support models using the decoupled transaction policy. Please use " "'decoupled=True' argument to the 'exec' or 'async_exec' calls for " "decoupled models.'"); } // Inference THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestNew( &irequest, server_, model_name, model_version)); THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetId( irequest, infer_request->RequestId().c_str())); if (infer_request->GetCorrelationId().Type() == CorrelationIdDataType::UINT64) { THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetCorrelationId( irequest, infer_request->GetCorrelationId().UnsignedIntValue())); } else { THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetCorrelationIdString( irequest, infer_request->GetCorrelationId().StringValue().c_str())); } THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetFlags( irequest, infer_request->Flags())); THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetTimeoutMicroseconds( irequest, infer_request->Timeout())); THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestComplete, nullptr /* request_release_userp */)); TRITONSERVER_InferenceTrace* trace = nullptr; if (infer_request->GetTrace().TritonTrace() != nullptr) { THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceTraceSpawnChildTrace( reinterpret_cast( infer_request->GetTrace().TritonTrace()), &trace)); } const std::string& param_str = infer_request->Parameters(); triton::common::TritonJson::Value param; THROW_IF_TRITON_ERROR(param.Parse(param_str.c_str(), param_str.length())); std::vector param_keys; THROW_IF_TRITON_ERROR(param.Members(¶m_keys)); for (const auto& key : param_keys) { triton::common::TritonJson::Value value; if (!param.Find(key.c_str(), &value)) { throw PythonBackendException("Unexpected missing key on parameters"); } if (value.IsString()) { std::string string_value; THROW_IF_TRITON_ERROR(value.AsString(&string_value)); THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetStringParameter( irequest, key.c_str(), string_value.c_str())); } else if (value.IsInt()) { int64_t int_value = 0; THROW_IF_TRITON_ERROR(value.AsInt(&int_value)); THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetIntParameter( irequest, key.c_str(), int_value)); } else if (value.IsBool()) { bool bool_value = false; THROW_IF_TRITON_ERROR(value.AsBool(&bool_value)); THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetBoolParameter( irequest, key.c_str(), bool_value)); } else { throw PythonBackendException("Unsupported value type on parameters"); } } for (auto& infer_input : infer_request->Inputs()) { THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestAddInput( irequest, infer_input->Name().c_str(), static_cast(infer_input->TritonDtype()), infer_input->Dims().data(), infer_input->Dims().size())); THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestAppendInputData( irequest, infer_input->Name().c_str(), infer_input->DataPtr(), infer_input->ByteSize(), infer_input->MemoryType(), infer_input->MemoryTypeId())); } for (auto& requested_output_name : infer_request->RequestedOutputNames()) { THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestAddRequestedOutput( irequest, requested_output_name.c_str())); } { infer_payload->SetFuture(response_future); ResponseAllocatorUserp response_allocator_userp( shm_pool_.get(), infer_request->GetPreferredMemory()); infer_payload->SetResponseAllocUserp(response_allocator_userp); THROW_IF_TRITON_ERROR(TRITONSERVER_InferenceRequestSetResponseCallback( irequest, response_allocator_, reinterpret_cast(infer_payload->ResponseAllocUserp().get()), InferResponseComplete, reinterpret_cast(infer_payload.get()))); THROW_IF_TRITON_ERROR( TRITONSERVER_ServerInferAsync(server_, irequest, trace)); } } catch (const PythonBackendException& pb_exception) { LOG_IF_ERROR( TRITONSERVER_InferenceRequestDelete(irequest), "Failed to delete inference request."); throw PythonBackendException( std::string("Model ") + model_name + " - Error when running inference: " + pb_exception.what()); } return response_future; } RequestExecutor::~RequestExecutor() { if (response_allocator_ != nullptr) { LOG_IF_ERROR( TRITONSERVER_ResponseAllocatorDelete(response_allocator_), "Failed to delete allocator."); } } }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/request_executor.h000066400000000000000000000045001527052371200270210ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include "infer_payload.h" #include "infer_request.h" #include "infer_response.h" namespace triton { namespace backend { namespace python { TRITONSERVER_Error* CreateTritonErrorFromException( const PythonBackendException& pb_exception); class RequestExecutor { TRITONSERVER_ResponseAllocator* response_allocator_ = nullptr; TRITONSERVER_Server* server_; std::unique_ptr& shm_pool_; public: std::future> Infer( std::shared_ptr& infer_request, std::shared_ptr& infer_payload); RequestExecutor( std::unique_ptr& shm_pool, TRITONSERVER_Server* server); ~RequestExecutor(); }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/resources/000077500000000000000000000000001527052371200252555ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/src/resources/triton_python_backend_utils.py000066400000000000000000000525651527052371200334530ustar00rootroot00000000000000# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import struct import numpy as np TRITON_STRING_TO_NUMPY = { "TYPE_BOOL": bool, "TYPE_UINT8": np.uint8, "TYPE_UINT16": np.uint16, "TYPE_UINT32": np.uint32, "TYPE_UINT64": np.uint64, "TYPE_INT8": np.int8, "TYPE_INT16": np.int16, "TYPE_INT32": np.int32, "TYPE_INT64": np.int64, "TYPE_FP16": np.float16, "TYPE_FP32": np.float32, "TYPE_FP64": np.float64, "TYPE_STRING": np.object_, } def serialize_byte_tensor(input_tensor): """ Serializes a bytes tensor into a flat numpy array of length prepended bytes. The numpy array should use dtype of np.object_. For np.bytes_, numpy will remove trailing zeros at the end of byte sequence and because of this it should be avoided. Parameters ---------- input_tensor : np.array The bytes tensor to serialize. Returns ------- serialized_bytes_tensor : np.array The 1-D numpy array of type uint8 containing the serialized bytes in 'C' order. Raises ------ InferenceServerException If unable to serialize the given tensor. """ if input_tensor.size == 0: return () # If the input is a tensor of string/bytes objects, then must flatten those # into a 1-dimensional array containing the 4-byte byte size followed by the # actual element bytes. All elements are concatenated together in "C" order. if (input_tensor.dtype == np.object_) or (input_tensor.dtype.type == np.bytes_): flattened_ls = [] for obj in np.nditer(input_tensor, flags=["refs_ok"], order="C"): # If directly passing bytes to BYTES type, # don't convert it to str as Python will encode the # bytes which may distort the meaning if input_tensor.dtype == np.object_: if type(obj.item()) == bytes: s = obj.item() else: s = str(obj.item()).encode("utf-8") else: s = obj.item() flattened_ls.append(struct.pack(" max_batch_size: raise ValueError( "configuration specified max_batch_size " + str(self._model_config["max_batch_size"]) + ", but in auto-complete-config function for model '" + self._model_config["name"] + "' specified max_batch_size " + str(max_batch_size) ) else: self._model_config["max_batch_size"] = max_batch_size def set_dynamic_batching(self): """Set dynamic_batching as the scheduler for the model if no scheduler is set. If dynamic_batching is set in the model configuration, then no action is taken and return success. Raises ------ ValueError If the 'sequence_batching' or 'ensemble_scheduling' scheduler is set for this model configuration. """ found_scheduler = None if "sequence_batching" in self._model_config: found_scheduler = "sequence_batching" elif "ensemble_scheduling" in self._model_config: found_scheduler = "ensemble_scheduling" if found_scheduler != None: raise ValueError( "Configuration specified scheduling_choice as '" + found_scheduler + "', but auto-complete-config " "function for model '" + self._model_config["name"] + "' tries to set scheduling_choice as 'dynamic_batching'" ) if "dynamic_batching" not in self._model_config: self._model_config["dynamic_batching"] = {} def add_input(self, input): """Add the input for the model. Parameters ---------- input : dict The input to be added. Raises ------ ValueError If input contains property other than 'name', 'data_type', 'dims', 'optional' or any of the non-optional properties are not set, or if an input with the same name already exists in the configuration but has different data_type or dims property """ valid_properties = ["name", "data_type", "dims", "optional"] for current_property in input: if current_property not in valid_properties: raise ValueError( "input '" + input["name"] + "' in auto-complete-config function for model '" + self._model_config["name"] + "' contains property other than 'name', 'data_type', 'dims' and 'optional'." ) if "name" not in input: raise ValueError( "input in auto-complete-config function for model '" + self._model_config["name"] + "' is missing 'name' property." ) elif "data_type" not in input: raise ValueError( "input '" + input["name"] + "' in auto-complete-config function for model '" + self._model_config["name"] + "' is missing 'data_type' property." ) elif "dims" not in input: raise ValueError( "input '" + input["name"] + "' in auto-complete-config function for model '" + self._model_config["name"] + "' is missing 'dims' property." ) for current_input in self._model_config["input"]: if input["name"] == current_input["name"]: if ( current_input["data_type"] != "TYPE_INVALID" and current_input["data_type"] != input["data_type"] ): raise ValueError( "unable to load model '" + self._model_config["name"] + "', configuration expects datatype " + current_input["data_type"] + " for input '" + input["name"] + "', model provides " + input["data_type"] ) elif current_input["dims"] and current_input["dims"] != input["dims"]: raise ValueError( "model '" + self._model_config["name"] + "', tensor '" + input["name"] + "': the model expects dims " + str(input["dims"]) + " but the model configuration specifies dims " + str(current_input["dims"]) ) elif ( "optional" in current_input and "optional" in input and current_input["optional"] != input["optional"] ): raise ValueError( "model '" + self._model_config["name"] + "', tensor '" + input["name"] + "': the model expects optional " + str(input["optional"]) + " but the model configuration specifies optional " + str(current_input["optional"]) ) else: current_input["data_type"] = input["data_type"] current_input["dims"] = input["dims"] if "optional" in input: current_input["optional"] = input["optional"] return self._model_config["input"].append(input) def add_output(self, output): """Add the output for the model. Parameters ---------- output : dict The output to be added. Raises ------ ValueError If output contains property other than 'name', 'data_type' and 'dims' or any of the properties are not set, or if an output with the same name already exists in the configuration but has different data_type or dims property """ valid_properties = ["name", "data_type", "dims"] for current_property in output: if current_property not in valid_properties: raise ValueError( "output '" + output["name"] + "' in auto-complete-config function for model '" + self._model_config["name"] + "' contains property other than 'name', 'data_type' and 'dims'." ) if "name" not in output: raise ValueError( "output in auto-complete-config function for model '" + self._model_config["name"] + "' is missing 'name' property." ) elif "data_type" not in output: raise ValueError( "output '" + output["name"] + "' in auto-complete-config function for model '" + self._model_config["name"] + "' is missing 'data_type' property." ) elif "dims" not in output: raise ValueError( "output '" + output["name"] + "' in auto-complete-config function for model '" + self._model_config["name"] + "' is missing 'dims' property." ) for current_output in self._model_config["output"]: if output["name"] == current_output["name"]: if ( current_output["data_type"] != "TYPE_INVALID" and current_output["data_type"] != output["data_type"] ): raise ValueError( "unable to load model '" + self._model_config["name"] + "', configuration expects datatype " + current_output["data_type"] + " for output '" + output["name"] + "', model provides " + output["data_type"] ) elif ( current_output["dims"] and current_output["dims"] != output["dims"] ): raise ValueError( "model '" + self._model_config["name"] + "', tensor '" + output["name"] + "': the model expects dims " + str(output["dims"]) + " but the model configuration specifies dims " + str(current_output["dims"]) ) else: current_output["data_type"] = output["data_type"] current_output["dims"] = output["dims"] return self._model_config["output"].append(output) def set_model_transaction_policy(self, transaction_policy_dict): """ Set model transaction policy for the model. Parameters ---------- transaction_policy_dict : dict The dict, containing all properties to be set as a part of `model_transaction_policy` field. Raises ------ ValueError If transaction_policy_dict contains property other than 'decoupled', or if `model_transaction_policy` already exists in the configuration, but has different `decoupled` property. """ valid_properties = ["decoupled"] for current_property in transaction_policy_dict.keys(): if current_property not in valid_properties: raise ValueError( "model transaction property in auto-complete-config " + "function for model '" + self._model_config["name"] + "' contains property other than 'decoupled'." ) if "model_transaction_policy" not in self._model_config: self._model_config["model_transaction_policy"] = {} if "decoupled" in transaction_policy_dict.keys(): if ( "decoupled" in self._model_config["model_transaction_policy"] and self._model_config["model_transaction_policy"]["decoupled"] != transaction_policy_dict["decoupled"] ): raise ValueError( "trying to change decoupled property in auto-complete-config " + "for model '" + self._model_config["name"] + "', which is already set to '" + str(self._model_config["model_transaction_policy"]["decoupled"]) + "'." ) self._model_config["model_transaction_policy"][ "decoupled" ] = transaction_policy_dict["decoupled"] TRITONSERVER_REQUEST_FLAG_SEQUENCE_START = 1 TRITONSERVER_REQUEST_FLAG_SEQUENCE_END = 2 TRITONSERVER_RESPONSE_COMPLETE_FINAL = 1 TRITONSERVER_REQUEST_RELEASE_ALL = 1 TRITONSERVER_REQUEST_RELEASE_RESCHEDULE = 2 kytensor-2.49.0.7/framework_backends/python_backend/src/response_sender.cc000066400000000000000000000240061527052371200267520ustar00rootroot00000000000000// Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "response_sender.h" #include #include #include "pb_stub.h" #include "pb_stub_utils.h" #include "scoped_defer.h" namespace triton { namespace backend { namespace python { void CheckResponseSenderArguments( const std::shared_ptr& response, const uint32_t flags) { // Check the correctness of the provided flags. if (flags != TRITONSERVER_RESPONSE_COMPLETE_FINAL && flags != 0) { throw PythonBackendException( "Unable to send response. Unsupported flag provided."); } if (flags == 0 && response == nullptr) { throw PythonBackendException( "Inference Response object must be provided when the response flags is " "set to zero."); } } ResponseSender::ResponseSender( intptr_t request_address, intptr_t response_factory_address, bool const* is_decoupled, const std::set& requested_output_names, std::unique_ptr& shm_pool, const std::shared_ptr& pb_cancel) : request_address_(request_address), response_factory_address_(response_factory_address), is_decoupled_(is_decoupled), requested_output_names_(requested_output_names), shm_pool_(shm_pool), pb_cancel_(pb_cancel), closed_(false), number_of_response_sent_(0), response_factory_deleted_(false) { } ResponseSender::~ResponseSender() { DeleteResponseFactory(); } void ResponseSender::UpdateStateAndCounters( InferResponse* response, const uint32_t flags) { if (is_decoupled_ == nullptr) { // TODO: Can a model access the response sender on a BLS infer request? throw PythonBackendException( "Unable to send response. Response sender has no reference to the " "decoupled state of the model."); } bool is_decoupled = *is_decoupled_; std::lock_guard lk(mu_); if (!is_decoupled) { if (response != nullptr && number_of_response_sent_ > 0) { throw PythonBackendException( "Unable to send response. Non-decoupled model cannot send more than " "one response."); } if (response == nullptr && flags == TRITONSERVER_RESPONSE_COMPLETE_FINAL && number_of_response_sent_ == 0) { throw PythonBackendException( "Unable to send response. Non-decoupled model cannot send complete " "final before sending a response."); } } if (closed_) { throw PythonBackendException( "Unable to send response. Response sender has been closed."); } if (flags == TRITONSERVER_RESPONSE_COMPLETE_FINAL) { response_factory_deleted_.exchange(true); closed_ = true; } number_of_response_sent_++; } void ResponseSender::Send( std::shared_ptr infer_response, const uint32_t flags) { // Release the GIL. This avoids a potential deadlock situation in the parent // process, where every thread in the thread pool is indirectly waiting for a // function in the stub process that acquires the GIL. Meanwhile, the current // thread, which holds the GIL, is also waiting for the parent side to have // the next available thread to pick up the job during resource contention. py::gil_scoped_release release; CheckResponseSenderArguments(infer_response, flags); UpdateStateAndCounters(infer_response.get(), flags); if (infer_response) { infer_response->PruneOutputTensors(requested_output_names_); } std::unique_ptr& stub = Stub::GetOrCreateInstance(); AllocatedSharedMemory response_send_message = shm_pool_->Construct( 1 /* count */, true /* aligned */); if (infer_response) { infer_response->SaveToSharedMemory(shm_pool_, false /* copy_gpu */); } ResponseSendMessage* send_message_payload = response_send_message.data_.get(); new (&(send_message_payload->mu)) bi::interprocess_mutex; new (&(send_message_payload->cv)) bi::interprocess_condition; send_message_payload->is_stub_turn = false; send_message_payload->request_address = request_address_; send_message_payload->response_factory_address = response_factory_address_; if (infer_response) { send_message_payload->response = infer_response->ShmHandle(); } else { send_message_payload->response = 0; } send_message_payload->has_error = false; send_message_payload->is_error_set = false; send_message_payload->flags = flags; std::unique_ptr ipc_message = IPCMessage::Create(shm_pool_, false /* inline_response */); ipc_message->Command() = PYTHONSTUB_ResponseSend; ipc_message->Args() = response_send_message.handle_; ScopedDefer _([send_message_payload] { { bi::scoped_lock guard{send_message_payload->mu}; send_message_payload->is_stub_turn = false; send_message_payload->cv.notify_all(); } }); { bi::scoped_lock guard{send_message_payload->mu}; // The server will destruct the response factory if the final flag is set. if (flags == TRITONSERVER_RESPONSE_COMPLETE_FINAL) { response_factory_deleted_.exchange(true); } stub->SendIPCUtilsMessage(ipc_message); while (!send_message_payload->is_stub_turn) { send_message_payload->cv.wait(guard); } } bool has_gpu_output = false; std::vector> gpu_tensors; if (infer_response) { for (auto& tensor : infer_response->OutputTensors()) { if (!tensor->IsCPU()) { has_gpu_output = true; gpu_tensors.push_back(tensor); } } } if (has_gpu_output) { ScopedDefer _([send_message_payload] { bi::scoped_lock guard{send_message_payload->mu}; send_message_payload->is_stub_turn = false; send_message_payload->cv.notify_one(); while (!send_message_payload->is_stub_turn) { // Wait for the stub process to send the response and populate error // message if any. send_message_payload->cv.wait(guard); } }); AllocatedSharedMemory gpu_buffers_handle = shm_pool_->Load( send_message_payload->gpu_buffers_handle); if (!gpu_buffers_handle.data_->success) { std::unique_ptr error = PbString::LoadFromSharedMemory( shm_pool_, gpu_buffers_handle.data_->error); throw PythonBackendException( "Failed to load GPU buffers: " + error->String()); } AllocatedSharedMemory gpu_buffers_handle_shm = shm_pool_->Load( gpu_buffers_handle.data_->buffers); uint64_t gpu_buffer_count = gpu_buffers_handle.data_->buffer_count; if (gpu_tensors.size() != gpu_buffer_count) { throw PythonBackendException( std::string( "GPU buffers size does not match the provided buffers: ") + std::to_string(gpu_tensors.size()) + " != " + std::to_string(gpu_buffer_count)); } std::vector> dst_buffers; for (size_t i = 0; i < gpu_tensors.size(); i++) { std::unique_ptr dst_buffer = PbMemory::LoadFromSharedMemory( shm_pool_, gpu_buffers_handle_shm.data_.get()[i], true /* open_cuda_handle */); dst_buffers.emplace_back(std::move(dst_buffer)); std::shared_ptr& src_buffer = gpu_tensors[i]; PbMemory::CopyBuffer(dst_buffers[i], src_buffer->Memory()); } } if (send_message_payload->has_error) { if (send_message_payload->is_error_set) { std::unique_ptr error = PbString::LoadFromSharedMemory( shm_pool_, send_message_payload->error); throw PythonBackendException(error->String()); } else { throw PythonBackendException( "An error occurred while sending a response."); } } } bool ResponseSender::IsCancelled() { return pb_cancel_->IsCancelled(); } bool ResponseSender::IsClosed() { std::lock_guard lk(mu_); return closed_; } void ResponseSender::Close() { std::lock_guard lk(mu_); closed_ = true; response_factory_deleted_.exchange(true); } void ResponseSender::DeleteResponseFactory() { bool already_deleted = response_factory_deleted_.exchange(true); if (!already_deleted) { std::unique_ptr& stub = Stub::GetOrCreateInstance(); stub->EnqueueCleanupId( reinterpret_cast(response_factory_address_), PYTHONSTUB_DecoupledResponseFactoryCleanup); } } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/response_sender.h000066400000000000000000000055111527052371200266140ustar00rootroot00000000000000// Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include "infer_response.h" #include "pb_cancel.h" #include "shm_manager.h" namespace triton { namespace backend { namespace python { class ResponseSender { public: ResponseSender( intptr_t request_address, intptr_t response_factory_address, bool const* is_decoupled, const std::set& requested_output_names, std::unique_ptr& shm_pool, const std::shared_ptr& pb_cancel); intptr_t ResponseFactory() { return response_factory_address_; } ~ResponseSender(); void Send(std::shared_ptr response, const uint32_t flags); bool IsCancelled(); void UpdateStateAndCounters(InferResponse* response, const uint32_t flags); // Can be useful at stopping the model from sending any more responses. void Close(); bool IsClosed(); private: void DeleteResponseFactory(); intptr_t request_address_; intptr_t response_factory_address_; bool const* is_decoupled_; std::set requested_output_names_; std::unique_ptr& shm_pool_; std::shared_ptr pb_cancel_; std::mutex mu_; bool closed_; size_t number_of_response_sent_; std::atomic response_factory_deleted_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/scoped_defer.cc000066400000000000000000000036261527052371200262030ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "scoped_defer.h" namespace triton { namespace backend { namespace python { ScopedDefer::ScopedDefer(std::function task) { task_ = task; done_ = false; } void ScopedDefer::Complete() { if (!done_) { task_(); done_ = true; } } ScopedDefer::~ScopedDefer() { if (!done_) { task_(); } } }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/scoped_defer.h000066400000000000000000000035251527052371200260430ustar00rootroot00000000000000// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include namespace triton { namespace backend { namespace python { class ScopedDefer { public: ScopedDefer(std::function task); ~ScopedDefer(); void Complete(); private: std::function task_; bool done_; }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/shm_manager.cc000066400000000000000000000207121527052371200260350ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "shm_manager.h" #include #include #include #include namespace triton { namespace backend { namespace python { void CUDAMemoryPoolManager::SetCUDAPoolAddress( const int32_t device_id, void* cuda_pool_address) { std::lock_guard lock(mu_); cuda_pool_address_map_[device_id] = cuda_pool_address; } void* CUDAMemoryPoolManager::CUDAPoolAddress(const int32_t device_id) { if (cuda_pool_address_map_.find(device_id) != cuda_pool_address_map_.end()) { return cuda_pool_address_map_[device_id]; } else { throw PythonBackendException( "CUDA pool address for device " + std::to_string(device_id) + " is not set."); } } void CUDAMemoryPoolManager::SetTritonMemoryManager(void* triton_memory_manager) { triton_memory_manager_ = triton_memory_manager; } void* CUDAMemoryPoolManager::TritonMemoryManager() { return triton_memory_manager_; } bool CUDAMemoryPoolManager::UseCudaSharedPool(const int32_t device_id) { return (cuda_pool_address_map_.find(device_id) != cuda_pool_address_map_.end()) && (cuda_pool_address_map_[device_id] != nullptr) && (triton_memory_manager_ != nullptr); } std::unordered_map& CUDAMemoryPoolManager::CUDAPoolAddressMap() { return cuda_pool_address_map_; } SharedMemoryManager::SharedMemoryManager( const std::string& shm_region_name, size_t shm_size, size_t shm_growth_bytes, bool create) { shm_region_name_ = shm_region_name; create_ = create; shm_growth_bytes_ = shm_growth_bytes; cuda_memory_pool_manager_ = std::make_unique(); try { if (create) { // Remove (if any) and create the region. bi::shared_memory_object::remove(shm_region_name.c_str()); shm_obj_ = std::make_unique( bi::create_only, shm_region_name.c_str(), bi::read_write); shm_obj_->truncate(shm_size); } else { // Open the existing region. shm_obj_ = std::make_unique( bi::open_only, shm_region_name.c_str(), bi::read_write); } current_capacity_ = shm_size; shm_map_ = std::make_shared(*shm_obj_, bi::read_write); old_shm_maps_.push_back(shm_map_); // Only create the managed external buffer for the stub process. if (create) { managed_buffer_ = std::make_unique( bi::create_only, shm_map_->get_address(), shm_size); } else { int64_t shm_size = 0; shm_obj_->get_size(shm_size); managed_buffer_ = std::make_unique( bi::open_only, shm_map_->get_address(), shm_size); current_capacity_ = shm_size; } } catch (bi::interprocess_exception& ex) { std::string error_message = ("Unable to initialize shared memory key '" + shm_region_name + "' to requested size (" + std::to_string(shm_size) + " bytes). If you are running Triton inside docker, use '--shm-size' " "flag to control the shared memory region size. Each Python backend " "model instance requires at least 1 MB of shared memory. Error: " + ex.what()); // Remove the shared memory region if there was an error. bi::shared_memory_object::remove(shm_region_name.c_str()); throw PythonBackendException(std::move(error_message)); } // Construct a mutex in shared memory. shm_mutex_ = managed_buffer_->find_or_construct("shm_mutex")(); total_size_ = managed_buffer_->find_or_construct("total size")(); delete_region_ = true; if (create) { *total_size_ = current_capacity_; new (shm_mutex_) bi::interprocess_mutex; } } SharedMemoryManager::SharedMemoryManager(const std::string& shm_region_name) { shm_region_name_ = shm_region_name; create_ = false; shm_growth_bytes_ = 1024; cuda_memory_pool_manager_ = std::make_unique(); shm_obj_ = std::make_unique( bi::open_only, shm_region_name.c_str(), bi::read_write); shm_map_ = std::make_shared(*shm_obj_, bi::read_write); old_shm_maps_.push_back(shm_map_); int64_t shm_size = 0; shm_obj_->get_size(shm_size); managed_buffer_ = std::make_unique( bi::open_only, shm_map_->get_address(), shm_size); current_capacity_ = shm_size; // Construct a mutex in shared memory. shm_mutex_ = managed_buffer_->find_or_construct("shm_mutex")(); total_size_ = managed_buffer_->find_or_construct("total size")(); delete_region_ = false; } void SharedMemoryManager::GrowIfNeeded(uint64_t byte_size) { if (*total_size_ != current_capacity_) { shm_map_ = std::make_shared(*shm_obj_, bi::read_write); managed_buffer_ = std::make_unique( bi::open_only, shm_map_->get_address(), *total_size_); old_shm_maps_.push_back(shm_map_); current_capacity_ = *total_size_; } if (byte_size != 0) { uint64_t bytes_to_be_added = shm_growth_bytes_ * (byte_size / shm_growth_bytes_ + 1); uint64_t new_size = *total_size_ + bytes_to_be_added; try { shm_obj_->truncate(new_size); } catch (bi::interprocess_exception& ex) { std::string error_message = ("Failed to increase the shared memory pool size for key '" + shm_region_name_ + "' to " + std::to_string(*total_size_) + " bytes. If you are running Triton inside docker, use '--shm-size' " "flag to control the shared memory region size. Error: " + ex.what()); throw PythonBackendException(error_message); } try { shm_obj_->truncate(new_size); shm_map_ = std::make_shared(*shm_obj_, bi::read_write); old_shm_maps_.push_back(shm_map_); managed_buffer_ = std::make_unique( bi::open_only, shm_map_->get_address(), new_size); managed_buffer_->grow(new_size - current_capacity_); current_capacity_ = managed_buffer_->get_size(); *total_size_ = new_size; } catch (bi::interprocess_exception& ex) { shm_obj_->truncate(*total_size_); std::string error_message = ("Failed to create new mapped region for the grown shared memory " "region '" + shm_region_name_ + "'. " + ex.what()); throw PythonBackendException(error_message); } } } size_t SharedMemoryManager::FreeMemory() { GrowIfNeeded(0); return managed_buffer_->get_free_memory(); } SharedMemoryManager::~SharedMemoryManager() noexcept(false) { if (delete_region_) { bi::shared_memory_object::remove(shm_region_name_.c_str()); } } void SharedMemoryManager::SetDeleteRegion(bool delete_region) { delete_region_ = delete_region; } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/shm_manager.h000066400000000000000000000207141527052371200257010ustar00rootroot00000000000000// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include #include #include #include "pb_exception.h" namespace triton { namespace backend { namespace python { namespace bi = boost::interprocess; class CUDAMemoryPoolManager { public: CUDAMemoryPoolManager() : triton_memory_manager_(nullptr) {} void SetCUDAPoolAddress(const int32_t device_id, void* cuda_pool_address); void* CUDAPoolAddress(const int32_t device_id); void SetTritonMemoryManager(void* triton_memory_manager); void* TritonMemoryManager(); bool UseCudaSharedPool(const int32_t device_id); // Return cuda pool address map std::unordered_map& CUDAPoolAddressMap(); private: // The base address of the Triton CUDA memory pool std::unordered_map cuda_pool_address_map_; // The mutex to protect the cuda_pool_address_map_ std::mutex mu_; // TRITONBACKEND_MemoryManager void* triton_memory_manager_; }; template struct AllocatedSharedMemory { AllocatedSharedMemory() = default; AllocatedSharedMemory( std::unique_ptr>& data, bi::managed_external_buffer::handle_t handle) : data_(std::move(data)), handle_(handle) { } std::unique_ptr> data_; bi::managed_external_buffer::handle_t handle_; }; // The alignment here is used to extend the size of the shared memory allocation // struct to 16 bytes. The reason for this change is that when an aligned shared // memory location is requested using the `Construct` method, the memory // alignment of the object will be incorrect since the shared memory ownership // info is placed in the beginning and the actual object is placed after that // (i.e. 4 plus the aligned address is not 16-bytes aligned). The aligned memory // is required by semaphore otherwise it may lead to SIGBUS error on ARM. struct alignas(16) AllocatedShmOwnership { uint32_t ref_count_; }; class SharedMemoryManager { public: SharedMemoryManager( const std::string& shm_region_name, size_t shm_size, size_t shm_growth_bytes, bool create); SharedMemoryManager(const std::string& shm_region_name); template AllocatedSharedMemory Construct(uint64_t count = 1, bool aligned = false) { T* obj = nullptr; AllocatedShmOwnership* shm_ownership_data = nullptr; bi::managed_external_buffer::handle_t handle = 0; { bi::scoped_lock guard{*shm_mutex_}; std::size_t requested_bytes = sizeof(T) * count + sizeof(AllocatedShmOwnership); GrowIfNeeded(0); void* allocated_data; try { allocated_data = Allocate(requested_bytes, aligned); } catch (bi::bad_alloc& ex) { // Try to grow the shared memory region if the allocate failed. GrowIfNeeded(requested_bytes); allocated_data = Allocate(requested_bytes, aligned); } shm_ownership_data = reinterpret_cast(allocated_data); obj = reinterpret_cast( (reinterpret_cast(shm_ownership_data)) + sizeof(AllocatedShmOwnership)); shm_ownership_data->ref_count_ = 1; handle = managed_buffer_->get_handle_from_address( reinterpret_cast(shm_ownership_data)); } return WrapObjectInUniquePtr(obj, shm_ownership_data, handle); } template AllocatedSharedMemory Load( bi::managed_external_buffer::handle_t handle, bool unsafe = false) { T* object_ptr; AllocatedShmOwnership* shm_ownership_data; { bi::scoped_lock guard{*shm_mutex_}; GrowIfNeeded(0); shm_ownership_data = reinterpret_cast( managed_buffer_->get_address_from_handle(handle)); object_ptr = reinterpret_cast( reinterpret_cast(shm_ownership_data) + sizeof(AllocatedShmOwnership)); if (!unsafe) { shm_ownership_data->ref_count_ += 1; } } return WrapObjectInUniquePtr(object_ptr, shm_ownership_data, handle); } size_t FreeMemory(); void Deallocate(bi::managed_external_buffer::handle_t handle) { bi::scoped_lock guard{*shm_mutex_}; GrowIfNeeded(0); void* ptr = managed_buffer_->get_address_from_handle(handle); managed_buffer_->deallocate(ptr); } void DeallocateUnsafe(bi::managed_external_buffer::handle_t handle) { void* ptr = managed_buffer_->get_address_from_handle(handle); managed_buffer_->deallocate(ptr); } void GrowIfNeeded(uint64_t bytes); bi::interprocess_mutex* Mutex() { return shm_mutex_; } void SetDeleteRegion(bool delete_region); std::unique_ptr& GetCUDAMemoryPoolManager() { return cuda_memory_pool_manager_; } ~SharedMemoryManager() noexcept(false); private: std::string shm_region_name_; std::unique_ptr managed_buffer_; std::unique_ptr shm_obj_; std::shared_ptr shm_map_; std::vector> old_shm_maps_; uint64_t current_capacity_; bi::interprocess_mutex* shm_mutex_; size_t shm_growth_bytes_; uint64_t* total_size_; bool create_; bool delete_region_; std::unique_ptr cuda_memory_pool_manager_; template AllocatedSharedMemory WrapObjectInUniquePtr( T* object, AllocatedShmOwnership* shm_ownership_data, const bi::managed_external_buffer::handle_t& handle) { // Custom deleter to conditionally deallocate the object std::function deleter = [this, handle, shm_ownership_data](T* memory) { bool destroy = false; bi::scoped_lock guard{*shm_mutex_}; // Before using any shared memory function you need to make sure that you // are using the correct mapping. For example, shared memory growth may // happen between the time an object was created and the time the object // gets destructed. GrowIfNeeded(0); shm_ownership_data->ref_count_ -= 1; if (shm_ownership_data->ref_count_ == 0) { destroy = true; } if (destroy) { DeallocateUnsafe(handle); } }; auto data = std::unique_ptr(object, deleter); return AllocatedSharedMemory(data, handle); } void* Allocate(uint64_t requested_bytes, bool aligned) { void* ptr; if (aligned) { const std::size_t alignment = 32; ptr = managed_buffer_->allocate_aligned(requested_bytes, alignment); } else { ptr = managed_buffer_->allocate(requested_bytes); } return ptr; } }; }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/shm_monitor/000077500000000000000000000000001527052371200256015ustar00rootroot00000000000000kytensor-2.49.0.7/framework_backends/python_backend/src/shm_monitor/CMakeLists.txt000066400000000000000000000037021527052371200303430ustar00rootroot00000000000000# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) pybind11_add_module( triton-shm-monitor EXCLUDE_FROM_ALL ./shm_monitor.cc ../shm_manager.h ../shm_manager.cc ) target_link_libraries( triton-shm-monitor PRIVATE -lrt # shared memory ) set_property(TARGET triton-shm-monitor PROPERTY OUTPUT_NAME triton_shm_monitor) install( TARGETS triton-shm-monitor LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/python OPTIONAL ) kytensor-2.49.0.7/framework_backends/python_backend/src/shm_monitor/shm_monitor.cc000066400000000000000000000036531527052371200304550ustar00rootroot00000000000000// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include "../shm_manager.h" namespace triton { namespace backend { namespace python { namespace py = pybind11; PYBIND11_MODULE(triton_shm_monitor, m) { py::class_(m, "SharedMemoryManager") .def(py::init()) .def("free_memory", &SharedMemoryManager::FreeMemory); } }}} // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/stub_launcher.cc000066400000000000000000000751121527052371200264160ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "stub_launcher.h" #include #include "python_be.h" #ifdef _WIN32 #include // getpid() #endif namespace triton { namespace backend { namespace python { StubLauncher::StubLauncher(const std::string stub_process_kind) : parent_pid_(0), is_initialized_(false), stub_process_kind_(stub_process_kind), model_instance_name_(""), device_id_(0), kind_("") { } StubLauncher::StubLauncher( const std::string stub_process_kind, const std::string model_instance_name, const int32_t device_id, const std::string kind) : is_initialized_(false), stub_process_kind_(stub_process_kind), model_instance_name_(model_instance_name), device_id_(device_id), kind_(kind) { } TRITONSERVER_Error* StubLauncher::Initialize(ModelState* model_state) { model_name_ = model_state->Name(); shm_default_byte_size_ = model_state->StateForBackend()->shm_default_byte_size; shm_growth_byte_size_ = model_state->StateForBackend()->shm_growth_byte_size; shm_message_queue_size_ = model_state->StateForBackend()->shm_message_queue_size; python_execution_env_ = model_state->PythonExecutionEnv(); python_lib_ = model_state->StateForBackend()->python_lib; model_state->ModelConfig().Write(&model_config_buffer_); is_decoupled_ = model_state->IsDecoupled(); model_repository_path_ = model_state->RepositoryPath(); runtime_modeldir_ = model_state->RuntimeModelDir(); if (runtime_modeldir_.empty()) { runtime_modeldir_ = "DEFAULT"; } #ifdef _WIN32 ZeroMemory(&startup_info_, sizeof(startup_info_)); startup_info_.cb = sizeof(startup_info_); ZeroMemory(&stub_pid_, sizeof(stub_pid_)); #else stub_pid_ = 0; #endif shm_region_name_ = model_state->StateForBackend()->shared_memory_region_prefix + GenerateUUID(); model_version_ = model_state->Version(); std::stringstream ss; const char os_slash = std::filesystem::path::preferred_separator; ss << model_repository_path_ << os_slash << model_version_ << os_slash; std::string artifact_name; RETURN_IF_ERROR(model_state->ModelConfig().MemberAsString( "default_model_filename", &artifact_name)); if (artifact_name.size() > 0) { ss << artifact_name; } else { // Default artifact name. ss << "model.py"; } model_path_ = ss.str(); // FIXME [DLIS-5969]: Enable for Windows when custom execution environments // are supported. if (python_execution_env_ != "") { #ifndef _WIN32 RETURN_IF_ERROR(GetPythonEnvironment(model_state)); #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Custom execution environments are not currently supported on " "Windows."); #endif } parent_pid_ = getpid(); return nullptr; } TRITONSERVER_Error* StubLauncher::Setup() { // Destruct any in-use shared memory object before starting the stub process. ipc_control_ = nullptr; stub_message_queue_ = nullptr; parent_message_queue_ = nullptr; stub_to_parent_mq_ = nullptr; parent_to_stub_mq_ = nullptr; memory_manager_ = nullptr; try { // It is necessary for restart to make sure that the previous shared memory // pool is destructed before the new pool is created. shm_pool_ = nullptr; shm_pool_ = std::make_unique( shm_region_name_, shm_default_byte_size_, shm_growth_byte_size_, true /* create */); } catch (const PythonBackendException& pb_exception) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, pb_exception.what()); } AllocatedSharedMemory current_ipc_control = shm_pool_->Construct(); ipc_control_ = std::move(current_ipc_control.data_); ipc_control_handle_ = current_ipc_control.handle_; RETURN_IF_EXCEPTION( stub_message_queue_ = MessageQueue::Create( shm_pool_, shm_message_queue_size_)); RETURN_IF_EXCEPTION( parent_message_queue_ = MessageQueue::Create( shm_pool_, shm_message_queue_size_)); RETURN_IF_EXCEPTION( stub_to_parent_mq_ = MessageQueue::Create( shm_pool_, shm_message_queue_size_)); RETURN_IF_EXCEPTION( parent_to_stub_mq_ = MessageQueue::Create( shm_pool_, shm_message_queue_size_)); std::unique_ptr> memory_manager_message_queue; RETURN_IF_EXCEPTION( memory_manager_message_queue = MessageQueue::Create(shm_pool_, shm_message_queue_size_)); memory_manager_message_queue->ResetSemaphores(); ipc_control_->memory_manager_message_queue = memory_manager_message_queue->ShmHandle(); ipc_control_->decoupled = is_decoupled_; memory_manager_ = std::make_unique(std::move(memory_manager_message_queue)); ipc_control_->parent_message_queue = parent_message_queue_->ShmHandle(); ipc_control_->stub_to_parent_mq = stub_to_parent_mq_->ShmHandle(); ipc_control_->stub_message_queue = stub_message_queue_->ShmHandle(); ipc_control_->parent_to_stub_mq = parent_to_stub_mq_->ShmHandle(); new (&(ipc_control_->stub_health_mutex)) bi::interprocess_mutex; health_mutex_ = &(ipc_control_->stub_health_mutex); stub_message_queue_->ResetSemaphores(); parent_message_queue_->ResetSemaphores(); stub_to_parent_mq_->ResetSemaphores(); parent_to_stub_mq_->ResetSemaphores(); is_initialized_ = false; return nullptr; } // FIXME: This should be merged with the Unix launch function once Windows // CI and functionality are demonstrably stable. The goal of keeping the // functions separate is to help debug Windows-specific issues without worrying // about the impact to our Unix builds. #ifdef _WIN32 TRITONSERVER_Error* StubLauncher::Launch() { std::string stub_name; if (stub_process_kind_ == "AUTOCOMPLETE_STUB") { stub_name = model_name_; } else { stub_name = model_instance_name_; } const char os_slash = std::filesystem::path::preferred_separator; const std::string stub_executable_name = "triton_python_backend_stub.exe"; SanitizePath(model_path_); SanitizePath(model_repository_path_); // Default Python backend stub std::string python_backend_stub = python_lib_ + os_slash + stub_executable_name; LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Stub path ") + python_backend_stub).c_str()); // Path to alternative Python backend stub std::string model_python_backend_stub = std::string(model_repository_path_) + os_slash + stub_executable_name; LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Alt path ") + python_backend_stub).c_str()); // Check if file exists // TODO: Integrate win32 and pb_env if (FileExists(model_python_backend_stub)) { python_backend_stub = model_python_backend_stub; } std::string launch_command; std::stringstream ss; ss << python_backend_stub << " " << model_path_ << " " << shm_region_name_ << " " << shm_default_byte_size_ << " " << shm_growth_byte_size_ << " " << parent_pid_ << " " << python_lib_ << " " << ipc_control_handle_ << " " << stub_name << " " << runtime_modeldir_; launch_command = ss.str(); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Starting Python backend stub: ") + launch_command).c_str()); LPSTR launch_command_lpstr = const_cast(launch_command.c_str()); // Start the child process. Unlike fork(), the remainder of this // function exists in the context of the parent, only. if (!CreateProcess( NULL, // No module name (use command line) launch_command_lpstr, // Command line NULL, // Process handle not inheritable NULL, // Thread handle not inheritable FALSE, // Set handle inheritance to FALSE 0, // No creation flags NULL, // Use parent's environment block NULL, // Use parent's starting directory &startup_info_, // Pointer to STARTUPINFO structure &stub_pid_) // Pointer to PROCESS_INFORMATION structure ) { std::stringstream ss; ss << "Failed to run python backend stub. Errno = " << errno << '\n' << "Python backend stub path: " << python_backend_stub << '\n' << "Shared Memory Region Name: " << shm_region_name_ << '\n' << "Shared Memory Default Byte Size: " << shm_default_byte_size_ << '\n' << "Shared Memory Growth Byte Size: " << shm_growth_byte_size_ << '\n'; // Print the error message directly because the underlying mutexes in // LOG_MESSAGE() could be forked when it is locked by other thread(s). std::cerr << '\n' << ss.str() << '\n'; _Exit(1); } ScopedDefer _([&] { // Push a dummy message to the message queue so that the stub // process is notified that it can release the object stored in // shared memory. stub_message_queue_->Push(DUMMY_MESSAGE); // If the model is not initialized, wait for the stub process to exit. if (!is_initialized_) { stub_message_queue_.reset(); parent_message_queue_.reset(); memory_manager_.reset(); WaitForStubProcess(); } }); // The stub process would send two messages to the parent process during the // initialization. // 1. When the stub process's health monitoring thread has started. // 2. When the initialization is fully completed and the Python model is // loaded. // // The reason it is broken into two steps is that creation of the health // monitoring thread may take longer which can make the server process think // that the stub process is unhealthy and return early. Waiting until the // health thread is spawn would make sure would prevent this issue. parent_message_queue_->Pop(); if (stub_process_kind_ == "AUTOCOMPLETE_STUB") { try { AutocompleteStubProcess(); } catch (const PythonBackendException& ex) { // Need to kill the stub process first KillStubProcess(); throw BackendModelException( TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, ex.what())); } } else if (stub_process_kind_ == "MODEL_INSTANCE_STUB") { RETURN_IF_ERROR(ModelInstanceStubProcess()); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("Unknown stub_process_kind: ") + stub_process_kind_) .c_str()); } is_initialized_ = true; return nullptr; } #else TRITONSERVER_Error* StubLauncher::Launch() { std::string stub_name; if (stub_process_kind_ == "AUTOCOMPLETE_STUB") { stub_name = model_name_; } else { stub_name = model_instance_name_; } const char* stub_args[4]; stub_args[0] = "bash"; stub_args[1] = "-c"; stub_args[3] = nullptr; // Last argument must be nullptr // Default Python backend stub std::string python_backend_stub = python_lib_ + "/triton_python_backend_stub"; // Path to alternative Python backend stub std::string model_python_backend_stub = std::string(model_repository_path_) + "/triton_python_backend_stub"; if (FileExists(model_python_backend_stub)) { python_backend_stub = model_python_backend_stub; } std::string bash_argument; // This shared memory variable indicates whether the stub process should // revert the LD_LIBRARY_PATH changes to avoid shared library issues in // executables and libraries. ipc_control_->uses_env = false; if (python_execution_env_ != "") { std::stringstream ss; // Need to properly set the LD_LIBRARY_PATH so that Python environments // using different python versions load properly. ss << "source " << path_to_activate_ << " && exec env LD_LIBRARY_PATH=" << path_to_libpython_ << ":$LD_LIBRARY_PATH " << python_backend_stub << " " << model_path_ << " " << shm_region_name_ << " " << shm_default_byte_size_ << " " << shm_growth_byte_size_ << " " << parent_pid_ << " " << python_lib_ << " " << ipc_control_handle_ << " " << stub_name << " " << runtime_modeldir_; ipc_control_->uses_env = true; bash_argument = ss.str(); } else { std::stringstream ss; ss << " exec " << python_backend_stub << " " << model_path_ << " " << shm_region_name_ << " " << shm_default_byte_size_ << " " << shm_growth_byte_size_ << " " << parent_pid_ << " " << python_lib_ << " " << ipc_control_handle_ << " " << stub_name << " " << runtime_modeldir_; bash_argument = ss.str(); } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("Starting Python backend stub: ") + bash_argument).c_str()); stub_args[2] = bash_argument.c_str(); int stub_status_code = system((python_backend_stub + "> /dev/null 2>&1").c_str()); // If running stub process without any arguments returns any status code, // other than 1, it can indicate a permission issue as a result of // downloading the stub process from a cloud object storage service. if (WEXITSTATUS(stub_status_code) != 1) { // Give the execute permission for the triton_python_backend_stub to the // owner. int error = chmod(python_backend_stub.c_str(), S_IXUSR); if (error != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("Failed to give execute permission to " "triton_python_backend_stub in ") + python_backend_stub + " " + stub_name + " Error No.: " + std::to_string(error)) .c_str()); } } pid_t pid = fork(); if (pid < 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to fork the stub process for auto-complete."); } if (pid == 0) { // Replace this child process with the new stub process. execvp("bash", (char**)stub_args); // execvp() never return if succeeded. Otherwise, an error has occurred. std::stringstream ss; ss << "Failed to run python backend stub. Errno = " << errno << '\n' << "Python backend stub path: " << python_backend_stub << '\n' << "Shared Memory Region Name: " << shm_region_name_ << '\n' << "Shared Memory Default Byte Size: " << shm_default_byte_size_ << '\n' << "Shared Memory Growth Byte Size: " << shm_growth_byte_size_ << '\n'; // Print the error message directly because the underlying mutexes in // LOG_MESSAGE() could be forked when it is locked by other thread(s). std::cerr << '\n' << ss.str() << '\n'; // Terminate the child execution immediately to avoid any issues. _Exit(1); } else { ScopedDefer _([&] { // Push a dummy message to the message queue so that the stub // process is notified that it can release the object stored in // shared memory. stub_message_queue_->Push(DUMMY_MESSAGE); // If the model is not initialized, wait for the stub process to exit. if (!is_initialized_) { stub_message_queue_.reset(); parent_message_queue_.reset(); memory_manager_.reset(); WaitForStubProcess(); } }); stub_pid_ = pid; // The stub process would send two messages to the parent process during the // initialization. // 1. When the stub process's health monitoring thread has started. // 2. When the initialization is fully completed and the Python model is // loaded. // // The reason it is broken into two steps is that creation of the health // monitoring thread may take longer which can make the server process think // that the stub process is unhealthy and return early. Waiting until the // health thread is spawn would prevent this issue. parent_message_queue_->Pop(); if (stub_process_kind_ == "AUTOCOMPLETE_STUB") { try { AutocompleteStubProcess(); } catch (const PythonBackendException& ex) { // Need to kill the stub process first KillStubProcess(); throw BackendModelException( TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, ex.what())); } } else if (stub_process_kind_ == "MODEL_INSTANCE_STUB") { RETURN_IF_ERROR(ModelInstanceStubProcess()); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("Unknown stub_process_kind: ") + stub_process_kind_) .c_str()); } is_initialized_ = true; } return nullptr; } TRITONSERVER_Error* StubLauncher::GetPythonEnvironment(ModelState* model_state) { std::string python_execution_env = ""; try { python_execution_env = model_state->StateForBackend()->env_manager->ExtractIfNotExtracted( python_execution_env_); } catch (PythonBackendException& pb_exception) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, pb_exception.what()); } path_to_activate_ = python_execution_env + "/bin/activate"; path_to_libpython_ = python_execution_env + "/lib"; if (python_execution_env.length() > 0 && !FileExists(path_to_activate_)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, ("Path " + path_to_activate_ + " does not exist. The Python environment should contain an " "'activate' script.") .c_str()); } return nullptr; } #endif void StubLauncher::AutocompleteStubProcess() { std::string model_config = model_config_buffer_.MutableContents(); std::unique_ptr auto_complete_message = IPCMessage::Create(shm_pool_, false /* inline_response */); auto_complete_message->Command() = PYTHONSTUB_AutoCompleteRequest; std::unique_ptr pb_string = PbString::Create(shm_pool_, model_config); bi::managed_external_buffer::handle_t string_handle = pb_string->ShmHandle(); auto_complete_message->Args() = string_handle; stub_message_queue_->Push(auto_complete_message->ShmHandle()); std::unique_ptr auto_complete_response_message = IPCMessage::LoadFromSharedMemory(shm_pool_, parent_message_queue_->Pop()); if (auto_complete_response_message->Command() != PYTHONSTUB_AutoCompleteResponse) { throw PythonBackendException( "Received unexpected response from Python backend stub: " + model_name_); } auto auto_complete_response = std::move((shm_pool_->Load( auto_complete_response_message->Args()))) .data_; if (auto_complete_response->response_has_error) { if (auto_complete_response->response_is_error_set) { std::unique_ptr error_message = PbString::LoadFromSharedMemory( shm_pool_, auto_complete_response->response_error); throw PythonBackendException(error_message->String()); } else { throw PythonBackendException("Auto-complete failed for " + model_name_); } } if (auto_complete_response->response_has_model_config) { std::unique_ptr auto_complete_config = PbString::LoadFromSharedMemory( shm_pool_, auto_complete_response->response_model_config); std::string auto_complete_config_string = auto_complete_config->String(); if (!auto_complete_config_string.empty()) { TRITONSERVER_Error* err = auto_complete_config_.Parse(auto_complete_config_string); if (err != nullptr) { throw PythonBackendException("Failed to parse auto-complete JSON."); } } } } TRITONSERVER_Error* StubLauncher::ModelInstanceStubProcess() { std::unordered_map initialize_map = { {"model_config", model_config_buffer_.MutableContents()}, {"model_instance_kind", kind_}, {"model_instance_name", model_instance_name_}, {"model_instance_device_id", std::to_string(device_id_)}, {"model_repository", model_repository_path_}, {"model_version", std::to_string(model_version_)}, {"model_name", model_name_}}; std::unique_ptr initialize_message = IPCMessage::Create(shm_pool_, false /* inline_response */); initialize_message->Command() = PYTHONSTUB_InitializeRequest; std::unique_ptr pb_map = PbMap::Create(shm_pool_, initialize_map); bi::managed_external_buffer::handle_t initialize_map_handle = pb_map->ShmHandle(); initialize_message->Args() = initialize_map_handle; stub_message_queue_->Push(initialize_message->ShmHandle()); bi::managed_external_buffer::handle_t message; RETURN_IF_ERROR(ReceiveMessageFromStub(message)); std::unique_ptr initialize_response_message = IPCMessage::LoadFromSharedMemory(shm_pool_, message); if (initialize_response_message->Command() != PYTHONSTUB_InitializeResponse) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string( "Received unexpected response from Python backend stub: ") + model_instance_name_) .c_str()); } auto initialize_response = std::move((shm_pool_->Load( initialize_response_message->Args()))) .data_; if (initialize_response->response_has_error) { if (initialize_response->response_is_error_set) { std::unique_ptr error_message = PbString::LoadFromSharedMemory( shm_pool_, initialize_response->response_error); return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, error_message->String().c_str()); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("Launch stub process failed for ") + model_name_) .c_str()); } } return nullptr; } bool StubLauncher::StubActive() { #ifdef _WIN32 DWORD ec; GetExitCodeProcess(stub_pid_.hProcess, &ec); return (ec == STILL_ACTIVE); #else return (stub_pid_ != 0); #endif } void StubLauncher::UpdateHealth() { is_healthy_ = false; if (is_initialized_) { { bi::scoped_lock lock(*health_mutex_); ipc_control_->stub_health = false; } // Sleep 1 second so that the child process has a chance to change the // health variable #ifdef _WIN32 Sleep(1); #else sleep(1); #endif { bi::scoped_lock lock(*health_mutex_); is_healthy_ = ipc_control_->stub_health; } } } void StubLauncher::TerminateStub() { if (is_initialized_) { bool force_kill = false; if (is_healthy_) { // Finalize command does not have any arguments. std::unique_ptr ipc_message = IPCMessage::Create(shm_pool_, false /* inline_response */); ipc_message->Command() = PYTHONSTUB_FinalizeRequest; stub_message_queue_->Push(ipc_message->ShmHandle()); parent_message_queue_->Pop(); stub_message_queue_.reset(); parent_message_queue_.reset(); memory_manager_.reset(); } else { force_kill = true; } if (force_kill) { KillStubProcess(); } else { WaitForStubProcess(); } } // First destroy the IPCControl. This makes sure that IPCControl is // destroyed before the shared memory manager goes out of scope. ipc_control_.reset(); stub_message_queue_.reset(); parent_message_queue_.reset(); memory_manager_.reset(); } void StubLauncher::ClearQueues() { stub_to_parent_mq_.reset(); parent_to_stub_mq_.reset(); } void StubLauncher::KillStubProcess() { #ifdef _WIN32 unsigned int exit_code; TerminateProcess(stub_pid_.hProcess, exit_code); CloseHandle(stub_pid_.hProcess); CloseHandle(stub_pid_.hThread); #else kill(stub_pid_, SIGKILL); WaitForStubProcess(); stub_pid_ = 0; #endif } TRITONSERVER_Error* StubLauncher::ReceiveMessageFromStub( bi::managed_external_buffer::handle_t& message) { bool success = false; while (!success) { uint64_t timeout_miliseconds = 1000; { boost::posix_time::ptime timeout = boost::get_system_time() + boost::posix_time::milliseconds(timeout_miliseconds); bi::scoped_lock lock(*health_mutex_, timeout); // Check if lock has been acquired. if (lock) { ipc_control_->stub_health = false; } else { // If it failed to obtain the lock, it means that the stub has been // stuck or exited while holding the health mutex lock. return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to obtain the health mutex."); } } message = parent_message_queue_->Pop( timeout_miliseconds /* duration ms */, success); bool is_stub_alive = false; { boost::posix_time::ptime timeout = boost::get_system_time() + boost::posix_time::seconds(1); bi::scoped_lock lock(*health_mutex_, timeout); if (lock) { is_stub_alive = ipc_control_->stub_health; } else { // If It failed to obtain the lock, it means that the stub has been // stuck or exited while holding the health mutex lock. is_stub_alive = false; } } if (!success && !is_stub_alive) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("Stub process '") + model_instance_name_ + "' is not healthy.") .c_str()); } } return nullptr; // success } void StubLauncher::WaitForStubProcess() { #ifdef _WIN32 WaitForSingleObject(stub_pid_.hProcess, INFINITE); CloseHandle(stub_pid_.hProcess); CloseHandle(stub_pid_.hThread); #else int status; if (stub_pid_ != 0) { // Added this check to ensure server doesn't hang waiting after stub // process has already be killed and cannot be waited on waitpid(stub_pid_, &status, 0); } #endif } #ifdef TRITON_ENABLE_GPU void StubLauncher::ShareCUDAMemoryPool( TRITONBACKEND_MemoryManager* triton_mem_manager, const int32_t device_id) { std::lock_guard lock(cuda_shm_pool_mutex_); if ((tried_sharing_cuda_pool_map_.find(device_id) != tried_sharing_cuda_pool_map_.end()) && tried_sharing_cuda_pool_map_[device_id]) { return; } std::unique_ptr ipc_message = IPCMessage::Create(shm_pool_, true /* inline_response */); CUDAMemPoolMessage* cuda_pool_message_ptr = nullptr; PythonBackendException pb_exception(std::string{}); try { // Create a dummy BackendMemory object to get the start address of the CUDA // memory pool. BackendMemory* backend_memory; std::unique_ptr lbackend_memory; THROW_IF_TRITON_ERROR(BackendMemory::Create( triton_mem_manager, BackendMemory::AllocationType::GPU_POOL, device_id, 1 /* byte size*/, &backend_memory)); lbackend_memory.reset(backend_memory); CUDAHandler& cuda_api = CUDAHandler::getInstance(); CUdeviceptr cuda_pool_address = 0; cuda_api.PointerGetAttribute( &cuda_pool_address, CU_POINTER_ATTRIBUTE_RANGE_START_ADDR, reinterpret_cast(lbackend_memory->MemoryPtr())); shm_pool_->GetCUDAMemoryPoolManager()->SetCUDAPoolAddress( device_id, reinterpret_cast(cuda_pool_address)); shm_pool_->GetCUDAMemoryPoolManager()->SetTritonMemoryManager( reinterpret_cast(triton_mem_manager)); // Get the memory handle from the CUDA memory pool. AllocatedSharedMemory cuda_pool_message = shm_pool_->Construct(); cuda_pool_message_ptr = cuda_pool_message.data_.get(); { ScopedSetDevice scoped_set_device(device_id); THROW_IF_CUDA_ERROR(cudaIpcGetMemHandle( reinterpret_cast( &cuda_pool_message_ptr->cuda_handle), reinterpret_cast(shm_pool_->GetCUDAMemoryPoolManager() ->CUDAPoolAddress(device_id)))); } ipc_message->Command() = PYTHONSTUB_CUDAPoolInitializeRequest; ipc_message->Args() = cuda_pool_message.handle_; cuda_pool_message_ptr->device_id = device_id; cuda_pool_message_ptr->has_error = false; cuda_pool_message_ptr->is_error_set = false; cuda_pool_message_ptr->waiting_on_stub = false; { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; parent_to_stub_mq_->Push(ipc_message->ShmHandle()); while (!cuda_pool_message_ptr->waiting_on_stub) { ipc_message->ResponseCondition()->wait(lock); } } if (cuda_pool_message_ptr->has_error) { if (cuda_pool_message_ptr->is_error_set) { std::unique_ptr error_message = PbString::LoadFromSharedMemory( shm_pool_, cuda_pool_message_ptr->error); throw PythonBackendException(error_message->String()); } else { throw PythonBackendException( "Failed to share CUDA memory pool with stub process: " + model_name_); } } } catch (const PythonBackendException& exception) { shm_pool_->GetCUDAMemoryPoolManager()->SetCUDAPoolAddress( device_id, nullptr); pb_exception = exception; } { bi::scoped_lock lock{ *(ipc_message->ResponseMutex())}; cuda_pool_message_ptr->waiting_on_stub = false; ipc_message->ResponseCondition()->notify_all(); } tried_sharing_cuda_pool_map_[device_id] = true; if (pb_exception.what() != std::string{""}) { throw pb_exception; } } #endif // TRITON_ENABLE_GPU }}}; // namespace triton::backend::python kytensor-2.49.0.7/framework_backends/python_backend/src/stub_launcher.h000066400000000000000000000155351527052371200262630ustar00rootroot00000000000000// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ipc_message.h" #include "memory_manager.h" #include "message_queue.h" #include "pb_utils.h" #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" #include "triton/core/tritonbackend.h" #include "triton/core/tritonserver.h" namespace triton { namespace backend { namespace python { class ModelState; class StubLauncher { public: StubLauncher(const std::string stub_process_kind); StubLauncher( const std::string stub_process_kind, const std::string model_instance_name, const int32_t device_id, const std::string kind); // Initialize stub process TRITONSERVER_Error* Initialize(ModelState* model_state); // Stub process setup TRITONSERVER_Error* Setup(); // Launch stub process TRITONSERVER_Error* Launch(); // Auto-complete stub process void AutocompleteStubProcess(); // Model instance stub process TRITONSERVER_Error* ModelInstanceStubProcess(); // Check if Stub PID is active bool StubActive(); // Health mutex bi::interprocess_mutex* HealthMutex() { return health_mutex_; } // Stub message queue std::unique_ptr>& StubMessageQueue() { return stub_message_queue_; } // Parent message queue std::unique_ptr>& ParentMessageQueue() { return parent_message_queue_; } // Stub to parent message queue std::unique_ptr>& StubToParentMessageQueue() { return stub_to_parent_mq_; } // Parent to stub message queue std::unique_ptr>& ParentToStubMessageQueue() { return parent_to_stub_mq_; } // Memory Manager std::unique_ptr& GetMemoryManager() { return memory_manager_; } // IPC control std::unique_ptr>& IpcControl() { return ipc_control_; } // Shared memory pool std::unique_ptr& ShmPool() { return shm_pool_; } // Get auto-complete model configuration common::TritonJson::Value& AutoCompleteConfig() { return auto_complete_config_; } // Update health variable void UpdateHealth(); // Is Healthy bool IsHealthy() { return is_healthy_; } // Destruct Stub process void TerminateStub(); // Reset log queue and bls decoupled queue pointers void ClearQueues(); // Kill stub process void KillStubProcess(); // Get a message from the stub process TRITONSERVER_Error* ReceiveMessageFromStub( bi::managed_external_buffer::handle_t& message); // Wait for stub process void WaitForStubProcess(); #ifndef _WIN32 // FIXME [DLIS-5969]: Enable for Windows when custom execution environments // are supported. TRITONSERVER_Error* GetPythonEnvironment(ModelState* model_state); #endif #ifdef TRITON_ENABLE_GPU // Share CUDA memory pool with stub process void ShareCUDAMemoryPool( TRITONBACKEND_MemoryManager* triton_mem_manager, const int32_t device_id); #endif // TRITON_ENABLE_GPU private: #ifdef _WIN32 STARTUPINFO startup_info_; DWORD parent_pid_; PROCESS_INFORMATION stub_pid_; #else pid_t parent_pid_; pid_t stub_pid_; #endif bool is_initialized_; bool is_decoupled_; bool is_healthy_; std::string shm_region_name_; std::string model_repository_path_; std::string model_path_; std::string runtime_modeldir_; const std::string stub_process_kind_; std::string model_name_; const std::string model_instance_name_; const int32_t device_id_; const std::string kind_; uint64_t model_version_; std::string python_lib_; int64_t shm_default_byte_size_; int64_t shm_growth_byte_size_; int64_t shm_message_queue_size_; // Path to python execution environment std::string path_to_libpython_; std::string path_to_activate_; std::string python_execution_env_; common::TritonJson::WriteBuffer model_config_buffer_; common::TritonJson::Value auto_complete_config_; bi::interprocess_mutex* health_mutex_; std::unique_ptr> stub_message_queue_; std::unique_ptr> parent_message_queue_; std::unique_ptr> stub_to_parent_mq_; std::unique_ptr> parent_to_stub_mq_; std::unique_ptr memory_manager_; std::unique_ptr> ipc_control_; bi::managed_external_buffer::handle_t ipc_control_handle_; std::unique_ptr shm_pool_; #ifdef TRITON_ENABLE_GPU std::mutex cuda_shm_pool_mutex_; std::unordered_map tried_sharing_cuda_pool_map_; #endif // TRITON_ENABLE_GPU }; }}} // namespace triton::backend::python kytensor-2.49.0.7/server/000077500000000000000000000000001527052371200151435ustar00rootroot00000000000000kytensor-2.49.0.7/server/.clang-format000066400000000000000000000014271527052371200175220ustar00rootroot00000000000000--- BasedOnStyle: Google IndentWidth: 2 ColumnLimit: 80 ContinuationIndentWidth: 4 UseTab: Never MaxEmptyLinesToKeep: 2 SortIncludes: true CompactNamespaces: true ReflowComments: true DerivePointerAlignment: false PointerAlignment: Left AllowShortIfStatementsOnASingleLine: false AllowShortBlocksOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AlwaysBreakAfterReturnType: TopLevelDefinitions AlignAfterOpenBracket: AlwaysBreak BreakBeforeBraces: Custom BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: true AfterNamespace: false AfterStruct: false AfterUnion: false BeforeCatch: true BinPackArguments: true BinPackParameters: true ConstructorInitializerAllOnOneLineOrOnePerLine: false IndentCaseLabels: true kytensor-2.49.0.7/server/.dockerignore000066400000000000000000000000061527052371200176130ustar00rootroot00000000000000.git* kytensor-2.49.0.7/server/.github/000077500000000000000000000000001527052371200165035ustar00rootroot00000000000000kytensor-2.49.0.7/server/.github/ISSUE_TEMPLATE/000077500000000000000000000000001527052371200206665ustar00rootroot00000000000000kytensor-2.49.0.7/server/.github/ISSUE_TEMPLATE/bug_report.md000066400000000000000000000011441527052371200233600ustar00rootroot00000000000000--- name: Bug report about: Create a report to help us improve title: '' labels: '' assignees: '' --- **Description** A clear and concise description of what the bug is. **Triton Information** What version of Triton are you using? Are you using the Triton container or did you build it yourself? **To Reproduce** Steps to reproduce the behavior. Describe the models (framework, inputs, outputs), ideally include the model configuration file (if using an ensemble include the model configuration file for that as well). **Expected behavior** A clear and concise description of what you expected to happen. kytensor-2.49.0.7/server/.github/ISSUE_TEMPLATE/feature_request.md000066400000000000000000000011231527052371200244100ustar00rootroot00000000000000--- name: Feature request about: Suggest an idea for this project title: '' labels: '' assignees: '' --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. kytensor-2.49.0.7/server/.github/PULL_REQUEST_TEMPLATE/000077500000000000000000000000001527052371200217625ustar00rootroot00000000000000kytensor-2.49.0.7/server/.github/PULL_REQUEST_TEMPLATE/pull_request_template_external_contrib.md000066400000000000000000000035621527052371200323530ustar00rootroot00000000000000#### What does the PR do? #### Checklist - [ ] I have read the [Contribution guidelines](#../../CONTRIBUTING.md). - [ ] PR title reflects the change and is of format `: ` - [ ] Changes are described in the pull request. - [ ] Related issues are referenced. - [ ] Populated [github labels](https://docs.github.com/en/issues/using-labels-and-milestones-to-track-work/managing-labels) field - [ ] Added [test plan](#test-plan) and verified test passes. - [ ] Verified that the PR passes existing CI. - [ ] I ran pre-commit locally (`pre-commit install, pre-commit run --all`) - [ ] Verified copyright is correct on all changed files. - [ ] Added _succinct_ git squash message before merging [ref](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). - [ ] All template sections are filled out. - [ ] Optional: Additional screenshots for behavior/output changes with before/after. #### Commit Type: Check the [conventional commit type](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#type) box here and add the label to the github PR. - [ ] build - [ ] ci - [ ] docs - [ ] feat - [ ] fix - [ ] perf - [ ] refactor - [ ] revert - [ ] style - [ ] test #### Related PRs: <!-- Related PRs from other Repositories --> #### Where should the reviewer start? <!-- call out specific files that should be looked at closely --> #### Test plan: <!-- list steps to verify feature works --> <!-- were e2e tests added?--> #### Caveats: <!-- any limitations or possible things missing from this PR --> #### Background <!-- e.g. what led to this change being made. this is optional extra information to help the reviewer --> #### Related Issues: (use one of the action keywords Closes / Fixes / Resolves / Relates to) - closes GitHub issue: #xxx ����������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/.github/PULL_REQUEST_TEMPLATE/pull_request_template_internal_contrib.md����0000664�0000000�0000000�00000003424�15270523712�0032342�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#### What does the PR do? <!-- Describe your pull request here. Please read the text below the line, and make sure you follow the checklist.--> #### Checklist - [ ] PR title reflects the change and is of format `<commit_type>: <Title>` - [ ] Changes are described in the pull request. - [ ] Related issues are referenced. - [ ] Populated [github labels](https://docs.github.com/en/issues/using-labels-and-milestones-to-track-work/managing-labels) field - [ ] Added [test plan](#test-plan) and verified test passes. - [ ] Verified that the PR passes existing CI. - [ ] Verified copyright is correct on all changed files. - [ ] Added _succinct_ git squash message before merging [ref](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). - [ ] All template sections are filled out. - [ ] Optional: Additional screenshots for behavior/output changes with before/after. #### Commit Type: Check the [conventional commit type](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#type) box here and add the label to the github PR. - [ ] build - [ ] ci - [ ] docs - [ ] feat - [ ] fix - [ ] perf - [ ] refactor - [ ] revert - [ ] style - [ ] test #### Related PRs: <!-- Related PRs from other Repositories --> #### Where should the reviewer start? <!-- call out specific files that should be looked at closely --> #### Test plan: <!-- list steps to verify --> <!-- were e2e tests added?--> - CI Pipeline ID: <!-- Only Pipeline ID and no direct link here --> #### Caveats: <!-- any limitations or possible things missing from this PR --> #### Background <!-- e.g. what led to this change being made. this is optional extra information to help the reviewer --> #### Related Issues: (use one of the action keywords Closes / Fixes / Resolves / Relates to) - closes GitHub issue: #xxx ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/.github/pull_request_template.md�������������������������������������������0000664�0000000�0000000�00000001456�15270523712�0023452�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Thanks for submitting a PR to Triton! Please go the the `Preview` tab above this description box and select the appropriate sub-template: * [PR description template for Triton Engineers](?expand=1&template=pull_request_template_internal_contrib.md) * [PR description template for External Contributors](?expand=1&template=pull_request_template_external_contrib.md) If you already created the PR, please replace this message with one of * [External contribution template](https://raw.githubusercontent.com/triton-inference-server/server/main/.github/PULL_REQUEST_TEMPLATE/pull_request_template_external_contrib.md) * [Internal contribution template](https://raw.githubusercontent.com/triton-inference-server/server/main/.github/PULL_REQUEST_TEMPLATE/pull_request_template_internal_contrib.md) and fill it out. ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/.github/workflows/���������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020540�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/.github/workflows/codeql.yml�����������������������������������������������0000664�0000000�0000000�00000007145�15270523712�0022541�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "CodeQL" on: pull_request: jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to: # https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs queries: +security-and-quality # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 # Command-line programs to run using the OS shell. # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | # echo "Run, Build Application using script" # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: category: "/language:${{matrix.language}}" ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/.github/workflows/pre-commit.yaml������������������������������������������0000664�0000000�0000000�00000003321�15270523712�0023477�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: pre-commit on: pull_request: jobs: pre-commit: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - uses: pre-commit/action@v3.0.0 ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/.pre-commit-config.yaml����������������������������������������������������0000664�0000000�0000000�00000005671�15270523712�0021435�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black rev: 23.1.0 hooks: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] types_or: [python, cython] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.5 hooks: - id: clang-format types_or: [c, c++, cuda, proto, textproto, java] args: ["-fallback-style=none", "-style=file", "-i"] - repo: https://github.com/codespell-project/codespell rev: v2.2.4 hooks: - id: codespell additional_dependencies: [tomli] args: ["--toml", "pyproject.toml"] exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$) # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-json - id: check-toml - id: check-yaml exclude: ^deploy(\/[^\/]+)*\/templates\/.*$ - id: check-shebang-scripts-are-executable - id: end-of-file-fixer types_or: [c, c++, cuda, proto, textproto, java, python] - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace �����������������������������������������������������������������������kytensor-2.49.0.7/server/CITATION.cff���������������������������������������������������������������0000664�0000000�0000000�00000000500�15270523712�0017030�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������cff-version: 1.2.0 message: "If you use this software, please cite it as below." title: "Triton Inference Server: An Optimized Cloud and Edge Inferencing Solution." url: https://github.com/triton-inference-server repository-code: https://github.com/triton-inference-server/server authors: - name: "NVIDIA Corporation" ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/CMakeLists.txt�������������������������������������������������������������0000664�0000000�0000000�00000031047�15270523712�0017710�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonserver LANGUAGES C CXX) include(CMakeDependentOption) # Use C++17 standard as Triton's minimum required. set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.") set(TRITON_VERSION "0.0.0" CACHE STRING "The version of the Triton shared library" ) option(TRITON_ENABLE_LOGGING "Include logging support in server" ON) option(TRITON_ENABLE_STATS "Include statistics collections in server" ON) option(TRITON_ENABLE_TRACING "Include tracing support in server" OFF) option(TRITON_ENABLE_NVTX "Include NVTX support in server" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in server" ON) option(TRITON_ENABLE_MALI_GPU "Enable Arm Mali GPU support in server" OFF) option(TRITON_IGPU_BUILD "Enable options for iGPU compilation in sever" OFF) set(TRITON_MIN_COMPUTE_CAPABILITY "6.0" CACHE STRING "The minimum CUDA compute capability supported by Triton" ) set(TRITON_EXTRA_LIB_PATHS "" CACHE PATH "Extra library paths for Triton Server build") # Ensemble option(TRITON_ENABLE_ENSEMBLE "Include ensemble support in server" OFF) # Endpoints option(TRITON_ENABLE_HTTP "Include HTTP API in server" ON) option(TRITON_ENABLE_GRPC "Include GRPC API in server" ON) option(TRITON_ENABLE_SAGEMAKER "Include AWS SageMaker API in server" OFF) option(TRITON_ENABLE_VERTEX_AI "Include Vertex AI API in server" OFF) # Metrics option(TRITON_ENABLE_METRICS "Include metrics support in server" ON) option(TRITON_ENABLE_METRICS_GPU "Include GPU metrics support in server" ON) option(TRITON_ENABLE_METRICS_CPU "Include CPU metrics support in server" ON) # Cloud storage option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF) option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF) option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF) # Need to know if TensorRT is available when building unit tests option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF) # ASAN option(TRITON_ENABLE_ASAN "Build with address sanitizer" OFF) # Repo tags set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from") set(TRITON_THIRD_PARTY_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/third_party repo") set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") # Third-party location set(TRITON_THIRD_PARTY_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/third-party" CACHE STRING "Location of third-party build") set(TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/third-party-src" CACHE STRING "Location of third-party source") if(TRITON_ENABLE_METRICS AND NOT TRITON_ENABLE_STATS) message(FATAL_ERROR "TRITON_ENABLE_METRICS=ON requires TRITON_ENABLE_STATS=ON") endif() if(TRITON_ENABLE_TRACING AND NOT TRITON_ENABLE_STATS) message(FATAL_ERROR "TRITON_ENABLE_TRACING=ON requires TRITON_ENABLE_STATS=ON") endif() if (TRITON_ENABLE_METRICS_CPU AND NOT TRITON_ENABLE_METRICS) message(FATAL_ERROR "TRITON_ENABLE_METRICS_CPU=ON requires TRITON_ENABLE_METRICS=ON") endif() if (TRITON_ENABLE_METRICS_GPU AND NOT TRITON_ENABLE_METRICS) message(FATAL_ERROR "TRITON_ENABLE_METRICS_GPU=ON requires TRITON_ENABLE_METRICS=ON") endif() if (TRITON_ENABLE_METRICS_GPU AND NOT TRITON_ENABLE_GPU) message(FATAL_ERROR "TRITON_ENABLE_METRICS_GPU=ON requires TRITON_ENABLE_GPU=ON") endif() if(TRITON_ENABLE_ASAN AND TRITON_ENABLE_GPU) message(FATAL_ERROR "TRITON_ENABLE_ASAN=ON requires TRITON_ENABLE_GPU=OFF") endif() # # Dependencies # include(FetchContent) FetchContent_Declare( repo-core GIT_REPOSITORY ${TRITON_REPO_ORGANIZATION}/core.git GIT_TAG ${TRITON_CORE_REPO_TAG} ) FetchContent_Declare( repo-third-party GIT_REPOSITORY ${TRITON_REPO_ORGANIZATION}/third_party.git GIT_TAG ${TRITON_THIRD_PARTY_REPO_TAG} ) # Some libs are installed to ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib64 instead # of ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib on Centos set(LIB_DIR "lib") if(LINUX) file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE") if(${DISTRO_ID_LIKE} MATCHES "rhel|centos") set (LIB_DIR "lib64") endif(${DISTRO_ID_LIKE} MATCHES "rhel|centos") endif(LINUX) set(TRITON_CORE_HEADERS_ONLY OFF) FetchContent_MakeAvailable(repo-third-party repo-core) # # Triton server executable and examples # # Need to use ExternalProject for our builds so that we can get the # correct dependencies between Triton executable and the # ExternalProject dependencies (found in the third_party repo) include(ExternalProject) # If CMAKE_TOOLCHAIN_FILE is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "") if (CMAKE_TOOLCHAIN_FILE) set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "-DCMAKE_TOOLCHAIN_FILE:PATH=${CMAKE_TOOLCHAIN_FILE}") endif() # If VCPKG_TARGET_TRIPLET is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "") if (VCPKG_TARGET_TRIPLET) set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "-DVCPKG_TARGET_TRIPLET:STRING=${VCPKG_TARGET_TRIPLET}") endif() # If OPENSSL_ROOT_DIR is set, propagate that hint path to the external # projects with OpenSSL dependency. set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "") if (OPENSSL_ROOT_DIR) set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") endif() # Location where protobuf-config.cmake will be installed varies by # platform if (WIN32) set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/cmake") else() set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf") endif() # Triton with Opentelemetry is not supported on Windows # FIXME: add location for Windows, when support is added # JIRA DLIS-4786 if (WIN32) set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "") else() set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/opentelemetry-cpp/${LIB_DIR}/cmake/opentelemetry-cpp") endif() if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(TRITON_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/install) else() set(TRITON_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) endif() set(TRITON_DEPENDS triton-core protobuf googletest re2) if(${TRITON_ENABLE_GCS}) set(TRITON_DEPENDS ${TRITON_DEPENDS} google-cloud-cpp) endif() # TRITON_ENABLE_GCS if(${TRITON_ENABLE_S3}) set(TRITON_DEPENDS ${TRITON_DEPENDS} aws-sdk-cpp) endif() # TRITON_ENABLE_S3 if(${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) set(TRITON_DEPENDS ${TRITON_DEPENDS} libevent libevhtp) endif() # TRITON_ENABLE_HTTP || TRITON_ENABLE_METRICS || TRITON_ENABLE_SAGEMAKER || TRITON_ENABLE_VERTEX_AI if(${TRITON_ENABLE_GRPC}) set(TRITON_DEPENDS ${TRITON_DEPENDS} grpc) endif() # TRITON_ENABLE_GRPC if(NOT WIN32 AND ${TRITON_ENABLE_TRACING}) set(TRITON_DEPENDS ${TRITON_DEPENDS} opentelemetry-cpp) endif() # TRITON_ENABLE_TRACING ExternalProject_Add(triton-server PREFIX triton-server SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/triton-server" CMAKE_CACHE_ARGS -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DGTEST_ROOT:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/googletest -DgRPC_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc/lib/cmake/grpc -Dc-ares_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares/${LIB_DIR}/cmake/c-ares -Dre2_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/re2/${LIB_DIR}/cmake/re2 -Dabsl_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl/${LIB_DIR}/cmake/absl -DCURL_DIR:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/curl/${LIB_DIR}/cmake/CURL -Dnlohmann_json_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/nlohmann_json/${LIB_DIR}/cmake/nlohmann_json -DLibevent_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevent/lib/cmake/libevent -Dlibevhtp_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevhtp/lib/cmake/libevhtp -Dstorage_client_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/google-cloud-cpp/${LIB_DIR}/cmake/storage_client -Dgoogle_cloud_cpp_common_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/google-cloud-cpp/${LIB_DIR}/cmake/google_cloud_cpp_common -DCrc32c_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/crc32c/${LIB_DIR}/cmake/Crc32c -DAWSSDK_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/AWSSDK -Daws-cpp-sdk-core_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/aws-cpp-sdk-core -Daws-cpp-sdk-s3_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/cmake/aws-cpp-sdk-s3 -Daws-c-event-stream_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-c-event-stream/cmake -Daws-c-common_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-c-common/cmake -Daws-checksums_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp/${LIB_DIR}/aws-checksums/cmake -Dopentelemetry-cpp_DIR:PATH=${_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR} -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_IGPU_BUILD:BOOL=${TRITON_IGPU_BUILD} -DTRITON_THIRD_PARTY_REPO_TAG:STRING=${TRITON_THIRD_PARTY_REPO_TAG} -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} -DTRITON_EXTRA_LIB_PATHS:PATH=${TRITON_EXTRA_LIB_PATHS} -DTRITON_ENABLE_ASAN:BOOL=${TRITON_ENABLE_ASAN} -DTRITON_ENABLE_NVTX:BOOL=${TRITON_ENABLE_NVTX} -DTRITON_ENABLE_TRACING:BOOL=${TRITON_ENABLE_TRACING} -DTRITON_ENABLE_LOGGING:BOOL=${TRITON_ENABLE_LOGGING} -DTRITON_ENABLE_STATS:BOOL=${TRITON_ENABLE_STATS} -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU} -DTRITON_ENABLE_MALI_GPU:BOOL=${TRITON_ENABLE_MALI_GPU} -DTRITON_ENABLE_HTTP:BOOL=${TRITON_ENABLE_HTTP} -DTRITON_ENABLE_SAGEMAKER:BOOL=${TRITON_ENABLE_SAGEMAKER} -DTRITON_ENABLE_VERTEX_AI:BOOL=${TRITON_ENABLE_VERTEX_AI} -DTRITON_ENABLE_GRPC:BOOL=${TRITON_ENABLE_GRPC} -DTRITON_MIN_COMPUTE_CAPABILITY:STRING=${TRITON_MIN_COMPUTE_CAPABILITY} -DTRITON_ENABLE_METRICS:BOOL=${TRITON_ENABLE_METRICS} -DTRITON_ENABLE_METRICS_GPU:BOOL=${TRITON_ENABLE_METRICS_GPU} -DTRITON_ENABLE_METRICS_CPU:BOOL=${TRITON_ENABLE_METRICS_CPU} -DTRITON_ENABLE_GCS:BOOL=${TRITON_ENABLE_GCS} -DTRITON_ENABLE_AZURE_STORAGE:BOOL=${TRITON_ENABLE_AZURE_STORAGE} -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3} -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT} -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} -DTRITON_VERSION:STRING=${TRITON_VERSION} DEPENDS ${TRITON_DEPENDS} ) �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/CONTRIBUTING.md������������������������������������������������������������0000664�0000000�0000000�00000012564�15270523712�0017404�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������<!-- # Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> # Contribution Guidelines Contributions that fix documentation errors or that make small changes to existing code can be contributed directly by following the rules below and submitting an appropriate PR. Contributions intended to add significant new functionality must follow a more collaborative path described in the following points. Before submitting a large PR that adds a major enhancement or extension, be sure to submit a GitHub issue that describes the proposed change so that the Triton team can provide feedback. - As part of the GitHub issue discussion, a design for your change will be agreed upon. An up-front design discussion is required to ensure that your enhancement is done in a manner that is consistent with Triton's overall architecture. - The Triton project is spread across multiple repos. The Triton team will provide guidance about how and where your enhancement should be implemented. - [Testing](docs/customization_guide/test.md) is a critical part of any Triton enhancement. You should plan on spending significant time on creating tests for your change. The Triton team will help you to design your testing so that it is compatible with existing testing infrastructure. - If your enhancement provides a user visible feature then you need to provide documentation. # Contribution Rules - The code style convention is enforced by clang-format. See below on how to ensure your contributions conform. In general please follow the existing conventions in the relevant file, submodule, module, and project when you add new code or when you extend/fix existing functionality. - Avoid introducing unnecessary complexity into existing code so that maintainability and readability are preserved. - Try to keep pull requests (PRs) as concise as possible: - Avoid committing commented-out code. - Wherever possible, each PR should address a single concern. If there are several otherwise-unrelated things that should be fixed to reach a desired endpoint, it is perfectly fine to open several PRs and state in the description which PR depends on another PR. The more complex the changes are in a single PR, the more time it will take to review those changes. - Make sure that the build log is clean, meaning no warnings or errors should be present. - Make sure all `L0_*` tests pass: - In the `qa/` directory, there are basic sanity tests scripted in directories named `L0_...`. See the [Test](docs/customization_guide/test.md) documentation for instructions on running these tests. - Triton Inference Server's default build assumes recent versions of dependencies (CUDA, TensorFlow, PyTorch, TensorRT, etc.). Contributions that add compatibility with older versions of those dependencies will be considered, but NVIDIA cannot guarantee that all possible build configurations work, are not broken by future contributions, and retain highest performance. - Make sure that you can contribute your work to open source (no license and/or patent conflict is introduced by your code). - Thanks in advance for your patience as we review your contributions; we do appreciate them! # Coding Convention All pull requests are checked against the [pre-commit hooks](https://github.com/pre-commit/pre-commit-hooks) located [in the repository's top-level .pre-commit-config.yaml](https://github.com/NVIDIA/triton-inference-server/blob/master/pre-commit-config.yaml). The hooks do some sanity checking like linting and formatting. These checks must pass to merge a change. To run these locally, you can [install pre-commit,](https://pre-commit.com/#install) then run `pre-commit install` inside the cloned repo. When you commit a change, the pre-commit hooks will run automatically. If a fix is implemented by a pre-commit hook, adding the file again and running `git commit` a second time will pass and successfully commit. ��������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/Dockerfile.QA��������������������������������������������������������������0000664�0000000�0000000�00000047226�15270523712�0017450�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ARG BASE_IMAGE=tritonserver ARG CIBASE_IMAGE=tritonserver_cibase ARG SDK_IMAGE=tritonserver_sdk ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server ARG TRITON_COMMON_REPO_TAG=main ARG TRITON_CORE_REPO_TAG=main ARG TRITON_THIRD_PARTY_REPO_TAG=main ARG TRITON_BACKEND_REPO_TAG=main ARG TRITONTMP_DIR=/tmp ARG IGPU_BUILD=0 ############################################################################ ## Test artifacts built as part of the tritonserver build are ## available in CIBASE_IMAGE. Copy these artifacts into the QA area. ############################################################################ FROM ${CIBASE_IMAGE} AS cibase ARG TRITONTMP_DIR ARG TRITON_REPO_ORGANIZATION ARG TRITON_COMMON_REPO_TAG ARG TRITON_CORE_REPO_TAG ARG TRITON_THIRD_PARTY_REPO_TAG ARG TRITON_BACKEND_REPO_TAG ARG IGPU_BUILD # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ libarchive-dev \ libboost-dev \ python3-dev \ python3-pip \ rapidjson-dev \ software-properties-common && \ rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade pip && \ pip3 install --upgrade wheel setuptools RUN apt update -q=2 \ && apt install -y gpg wget \ && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \ && . /etc/os-release \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ && apt-get update -q=2 \ && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7* # Add inception_graphdef model to example repo WORKDIR /workspace/docs/examples/model_repository RUN mkdir -p inception_graphdef/1 && \ wget -O ${TRITONTMP_DIR}/inception_v3_2016_08_28_frozen.pb.tar.gz \ https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz && \ (cd ${TRITONTMP_DIR} && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) && \ mv ${TRITONTMP_DIR}/inception_v3_2016_08_28_frozen.pb inception_graphdef/1/model.graphdef # Update the qa/ directory with test executables, models, etc. WORKDIR /workspace RUN mkdir -p qa/common && \ cp -r /workspace/src/test/models/repeat_int32 qa/L0_decoupled/models/ && \ cp -r /workspace/src/test/models/square_int32 qa/L0_decoupled/models/ && \ mkdir qa/L0_simple_example/models && \ cp -r docs/examples/model_repository/simple qa/L0_simple_example/models/. && \ mkdir qa/L0_simple_go_client/models && \ cp -r docs/examples/model_repository/simple qa/L0_simple_go_client/models/. && \ mkdir qa/L0_backend_release/simple_models && \ cp -r docs/examples/model_repository/simple qa/L0_backend_release/simple_models/. && \ mkdir qa/L0_simple_nodejs_client/models && \ cp -r docs/examples/model_repository/simple qa/L0_simple_nodejs_client/models/. && \ mkdir qa/L0_backend_release/simple_seq_models && \ cp -r /workspace/docs/examples/model_repository/simple_sequence qa/L0_backend_release/simple_seq_models/. && \ mkdir qa/L0_shared_memory/models && \ cp -r docs/examples/model_repository/simple qa/L0_shared_memory/models/. && \ mkdir qa/L0_cuda_shared_memory/models && \ cp -r docs/examples/model_repository/simple qa/L0_cuda_shared_memory/models/. && \ mkdir qa/L0_client_java/models && \ cp -r docs/examples/model_repository/simple qa/L0_client_java/models && \ mkdir qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_dyna_sequence qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_int8 qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_identity qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_sequence qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_string qa/L0_grpc/models && \ cp -r docs/examples/model_repository/inception_graphdef qa/L0_grpc/models && \ mkdir qa/L0_grpc_state_cleanup/models && \ cp -r /workspace/src/test/models/repeat_int32 qa/L0_grpc_state_cleanup/models/ && \ mkdir qa/L0_http/models && \ cp -r docs/examples/model_repository/simple qa/L0_http/models && \ cp -r docs/examples/model_repository/simple_dyna_sequence qa/L0_http/models && \ cp -r docs/examples/model_repository/simple_identity qa/L0_http/models && \ cp -r docs/examples/model_repository/simple_sequence qa/L0_http/models && \ cp -r docs/examples/model_repository/simple_string qa/L0_http/models && \ cp -r docs/examples/model_repository/inception_graphdef qa/L0_http/models && \ mkdir qa/L0_https/models && \ cp -r docs/examples/model_repository/simple qa/L0_https/models/. && \ mkdir qa/L0_secure_grpc/models && \ cp -r docs/examples/model_repository/simple qa/L0_secure_grpc/models/. && \ cp bin/simple qa/L0_simple_lib/. && \ cp bin/memory_alloc qa/L0_io/. && \ cp bin/multi_server qa/L0_multi_server/. && \ cp bin/memory_test qa/L0_memory/. && \ cp bin/pinned_memory_manager_test qa/L0_memory/. && \ cp bin/repo_agent_test qa/L0_triton_repo_agent/. && \ cp lib/libtritonrepoagent_relocation.so qa/L0_triton_repo_agent/. && \ mkdir qa/L0_query/models/query/1 && \ cp tritonbuild/tritonserver/backends/query/libtriton_query.so qa/L0_query/models/query/1/. && \ cp bin/query_test qa/L0_query/. && \ mkdir qa/L0_iterative_sequence/models/iterative_sequence/1 && \ cp tritonbuild/tritonserver/backends/iterative_sequence/libtriton_iterative_sequence.so qa/L0_iterative_sequence/models/iterative_sequence/1/. && \ cp bin/register_api_test qa/L0_register/. && \ cp bin/async_work_queue_test qa/L0_async_work_queue/. && \ cp tritonbuild/tritonserver/backends/implicit_state/libtriton_implicit_state.so \ qa/L0_implicit_state/. && \ mkdir qa/L0_data_compression/models && \ cp -r docs/examples/model_repository/simple qa/L0_data_compression/models && \ cp bin/data_compressor_test qa/L0_data_compression/. && \ cp bin/metrics_api_test qa/L0_metrics/. && \ cp bin/response_cache_test qa/L0_response_cache/. && \ cp bin/request_cancellation_test qa/L0_request_cancellation/. && \ cp bin/triton_json_test qa/L0_json/. && \ cp bin/backend_output_detail_test qa/L0_backend_output_detail/. && \ cp -r deploy/mlflow-triton-plugin qa/L0_mlflow/. && \ cp bin/input_byte_size_test qa/L0_input_validation/. && \ cp -r docs/examples/model_repository/simple_identity qa/L0_input_validation/models RUN mkdir -p qa/pkgs && \ cp python/triton*.whl qa/pkgs/. && \ cp -rf python/test/. qa/L0_python_api/. RUN mkdir -p qa/L0_simple_ensemble/models/simple/1 && \ cp docs/examples/model_repository/simple/1/model.graphdef \ qa/L0_simple_ensemble/models/simple/1/. && \ mkdir -p qa/L0_simple_ensemble/models/simple/2 && \ cp docs/examples/model_repository/simple/1/model.graphdef \ qa/L0_simple_ensemble/models/simple/2/. && \ mkdir -p qa/L0_socket/models/simple/1 && \ cp docs/examples/model_repository/simple/1/model.graphdef \ qa/L0_socket/models/simple/1/. RUN mkdir -p qa/L0_backend_identity/models && \ cp -r src/test/models/identity_fp32 qa/L0_backend_identity/models/. && \ mkdir -p qa/L0_backend_identity/models/identity_fp32/1 RUN mkdir -p qa/custom_models/custom_sequence_int32/1 && \ cp tritonbuild/tritonserver/backends/sequence/libtriton_sequence.so \ qa/custom_models/custom_sequence_int32/1/. && \ mkdir -p qa/custom_models/custom_dyna_sequence_int32/1 && \ cp tritonbuild/tritonserver/backends/dyna_sequence/libtriton_dyna_sequence.so \ qa/custom_models/custom_dyna_sequence_int32/1/. # L0_lifecycle needs No-GPU build of identity backend. RUN cd tritonbuild/identity && \ rm -rf install build && mkdir build && cd build && \ cmake -DTRITON_ENABLE_GPU=OFF \ -DCMAKE_INSTALL_PREFIX:PATH=/workspace/tritonbuild/identity/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \ -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \ -DTRITON_THIRD_PARTY_REPO_TAG:STRING=${TRITON_THIRD_PARTY_REPO_TAG} \ -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \ make -j16 install # L0_backend_python test require triton_shm_monitor RUN cd tritonbuild/python && \ rm -rf install build && mkdir build && cd build && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=/workspace/tritonbuild/python/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \ -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \ -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \ make -j16 triton-shm-monitor install RUN cp tritonbuild/identity/install/backends/identity/libtriton_identity.so \ qa/L0_lifecycle/. && \ cp tritonbuild/python/install/backends/python/triton_shm_monitor*.so \ qa/common/. && \ mkdir -p qa/L0_perf_nomodel/custom_models/custom_zero_1_float32/1 && \ mkdir -p qa/L0_perf_pyclients/custom_models/custom_zero_1_int32/1 && \ mkdir -p qa/L0_infer_shm && \ cp -r qa/L0_infer/. qa/L0_infer_shm && \ mkdir -p qa/L0_infer_cudashm && \ cp -r qa/L0_infer/. qa/L0_infer_cudashm && \ mkdir -p qa/L0_infer_valgrind && \ cp -r qa/L0_infer/. qa/L0_infer_valgrind && \ mkdir -p qa/L0_trt_shape_tensors_shm && \ cp -r qa/L0_trt_shape_tensors/. qa/L0_trt_shape_tensors_shm && \ mkdir -p qa/L0_trt_shape_tensors_cudashm && \ cp -r qa/L0_trt_shape_tensors/. qa/L0_trt_shape_tensors_cudashm && \ mkdir -p qa/L0_batcher_shm && \ cp -r qa/L0_batcher/. qa/L0_batcher_shm && \ mkdir -p qa/L0_batcher_cudashm && \ cp -r qa/L0_batcher/. qa/L0_batcher_cudashm && \ mkdir -p qa/L0_batcher_valgrind && \ cp -r qa/L0_batcher/. qa/L0_batcher_valgrind && \ mkdir -p qa/L0_sequence_batcher_shm && \ cp -r qa/L0_sequence_batcher/. qa/L0_sequence_batcher_shm && \ mkdir -p qa/L0_sequence_batcher_cudashm && \ cp -r qa/L0_sequence_batcher/. qa/L0_sequence_batcher_cudashm && \ mkdir -p qa/L0_sequence_batcher_valgrind && \ cp -r qa/L0_sequence_batcher/. qa/L0_sequence_batcher_valgrind && \ mkdir -p qa/L0_perf_nomodel_shm && \ cp -r qa/L0_perf_nomodel/. qa/L0_perf_nomodel_shm && \ mkdir -p qa/L0_perf_nomodel_cudashm && \ cp -r qa/L0_perf_nomodel/. qa/L0_perf_nomodel_cudashm # L0_model_control_stress will not be present if gitlab tests are not available RUN if [ -d qa/L0_model_control_stress ]; then \ mkdir -p qa/L0_model_control_stress_valgrind && \ cp -r qa/L0_model_control_stress/. qa/L0_model_control_stress_valgrind && \ mkdir -p qa/L0_model_control_stress_valgrind_massif && \ cp -r qa/L0_model_control_stress/. qa/L0_model_control_stress_valgrind_massif; \ fi RUN mkdir -p qa/L0_decoupled/models/repeat_int32/1 && \ mkdir -p qa/L0_decoupled/models/square_int32/1 && \ mkdir -p qa/L0_decoupled/models/identity_int32/1 && \ mkdir -p qa/L0_decoupled/models/simple_repeat/1 && \ mkdir -p qa/L0_decoupled/models/fan_repeat/1 && \ mkdir -p qa/L0_decoupled/models/sequence_repeat/1 && \ mkdir -p qa/L0_decoupled/models/repeat_square/1 && \ mkdir -p qa/L0_decoupled/models/nested_square/1 && \ mkdir -p qa/L0_grpc_state_cleanup/models/repeat_int32/1 RUN if [ "$IGPU_BUILD" == "0" ]; then \ cp backends/repeat/libtriton_repeat.so qa/L0_model_config && \ cp backends/repeat/libtriton_repeat.so qa/L0_decoupled/models/repeat_int32/1 && \ cp backends/repeat/libtriton_repeat.so qa/L0_grpc_state_cleanup/models/repeat_int32/1/. && \ cp backends/square/libtriton_square.so qa/L0_decoupled/models/square_int32/1; \ fi RUN cp -r qa/L0_decoupled/models qa/L0_decoupled/python_models/ && \ cp /workspace/tritonbuild/python/examples/decoupled/repeat_model.py \ qa/L0_decoupled/python_models/repeat_int32/1/. && \ cp /workspace/tritonbuild/python/examples/decoupled/repeat_config.pbtxt \ qa/L0_decoupled/python_models/repeat_int32/. && \ cp /workspace/tritonbuild/python/examples/decoupled/square_model.py \ qa/L0_decoupled/python_models/square_int32/1/. && \ cp /workspace/tritonbuild/python/examples/decoupled/square_config.pbtxt \ qa/L0_decoupled/python_models/square_int32/. RUN mkdir -p qa/L0_decoupled_grpc_error && \ cp -r qa/L0_decoupled/. qa/L0_decoupled_grpc_error RUN mkdir -p qa/L0_grpc_error_state_cleanup && \ cp -r qa/L0_grpc_state_cleanup/. qa/L0_grpc_error_state_cleanup RUN mkdir -p qa/L0_repoagent_checksum/models/identity_int32/1 && \ cp tritonbuild/identity/install/backends/identity/libtriton_identity.so \ qa/L0_repoagent_checksum/models/identity_int32/1/. RUN mkdir -p qa/L0_passive_instance/models/distributed_int32_int32_int32/1 && \ cp tritonbuild/tritonserver/backends/distributed_addsub/libtriton_distributed_addsub.so \ qa/L0_passive_instance/models/distributed_int32_int32_int32/1/. ############################################################################ ## Copy artifacts from sdk container ############################################################################ FROM ${SDK_IMAGE} AS sdk ARG TARGETPLATFORM WORKDIR /workspace COPY --from=cibase /workspace/qa/ qa/ RUN mkdir -p qa/clients && mkdir -p qa/pkgs && \ cp -a install/bin/* qa/clients/. && \ cp install/lib/libgrpcclient.so qa/clients/. && \ cp install/lib/libhttpclient.so qa/clients/. && \ cp install/python/*.py qa/clients/. && \ cp install/python/triton*.whl qa/pkgs/. && \ cp install/java/examples/*.jar qa/clients/. RUN cp client/src/grpc_generated/go/*.go qa/L0_simple_go_client/. && \ cp client/src/grpc_generated/javascript/*.js qa/L0_simple_nodejs_client/. && \ cp client/src/grpc_generated/javascript/*.json qa/L0_simple_nodejs_client/. && \ cp -r client/src/grpc_generated/java qa/L0_client_java/. ############################################################################ ## Create CI enabled image ############################################################################ FROM $BASE_IMAGE ARG TARGETPLATFORM # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive # install platform specific packages RUN if [ $(cat /etc/os-release | grep 'VERSION_ID="20.04"' | wc -l) -ne 0 ]; then \ apt-get update && \ apt-get install -y --no-install-recommends \ libpng-dev; \ elif [ $(cat /etc/os-release | grep 'VERSION_ID="22.04"' | wc -l) -ne 0 ]; then \ apt-get update && \ apt-get install -y --no-install-recommends \ libpng-dev; \ elif [ $(cat /etc/os-release | grep 'VERSION_ID="18.04"' | wc -l) -ne 0 ]; then \ apt-get update && \ apt-get install -y --no-install-recommends \ libpng-dev; \ else \ echo "Ubuntu version must be either 18.04, 20.04 or 22.04" && \ exit 1; \ fi # CI/QA for memcheck requires valgrind # libarchive-dev is required by Python backend RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ gdb \ libopencv-dev \ libarchive-dev \ libopencv-core-dev \ libzmq3-dev \ maven \ openjdk-11-jdk \ nginx \ npm \ protobuf-compiler \ python3-dev \ python3-pip \ python3-protobuf \ python3-setuptools \ swig \ valgrind && \ rm -rf /var/lib/apt/lists/* # CI/QA expects "python" executable (not python3). RUN rm -f /usr/bin/python && \ ln -s /usr/bin/python3 /usr/bin/python RUN pip3 install --upgrade wheel setuptools && \ pip3 install --upgrade "numpy<2" pillow attrdict future grpcio requests gsutil \ awscli six grpcio-channelz prettytable virtualenv \ check-jsonschema # go needed for example go client test. RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ wget https://golang.org/dl/go1.22.3.linux-arm64.tar.gz && \ rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.3.linux-arm64.tar.gz && \ rm -f go1.22.3.linux-arm64.tar.gz; \ else \ wget https://golang.org/dl/go1.22.3.linux-amd64.tar.gz && \ rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz && \ rm -f go1.22.3.linux-amd64.tar.gz; \ fi ENV GOPATH /root/go ENV PATH $PATH:/usr/local/go/bin:$GOPATH/bin RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && \ go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest # CI expects tests in /opt/tritonserver/qa. The triton-server (1000) # user should own all artifacts in case CI is run using triton-server # user. WORKDIR /opt/tritonserver COPY --chown=1000:1000 --from=sdk /workspace/qa/ qa/ # Remove CI tests that are meant to run only on build image and # install the tritonserver/triton python client APIs. RUN rm -fr qa/L0_copyrights qa/L0_build_variants && \ find qa/pkgs/ -maxdepth 1 -type f -name \ "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \ xargs pip3 install --upgrade # Install Triton Python API RUN find qa/pkgs/ -maxdepth 1 -type f -name \ "tritonserver-*.whl" | xargs -I {} pip3 install --upgrade {}[all] ENV LD_LIBRARY_PATH /opt/tritonserver/qa/clients:${LD_LIBRARY_PATH} # DLIS-3631: Needed to run Perf Analyzer CI tests correctly ENV LD_LIBRARY_PATH /opt/hpcx/ompi/lib:${LD_LIBRARY_PATH} # Required for PyTorch to pickup the correct HPCX libraries ENV LD_LIBRARY_PATH /opt/hpcx/ucc/lib/:/opt/hpcx/ucx/lib/:${LD_LIBRARY_PATH} ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/Dockerfile.sdk�������������������������������������������������������������0000664�0000000�0000000�00000026136�15270523712�0017725�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Multistage build. # # Base image on the minimum Triton container ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:24.08-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo ARG TRITON_PA_REPO_SUBDIR=perfanalyzerrepo ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server ARG TRITON_COMMON_REPO_TAG=main ARG TRITON_CORE_REPO_TAG=main ARG TRITON_CLIENT_REPO_TAG=main ARG TRITON_THIRD_PARTY_REPO_TAG=main ARG TRITON_MODEL_ANALYZER_REPO_TAG=main ARG TRITON_ENABLE_GPU=ON ARG JAVA_BINDINGS_MAVEN_VERSION=3.8.4 ARG JAVA_BINDINGS_JAVACPP_PRESETS_TAG=1.5.8 # DCGM version to install for Model Analyzer ARG DCGM_VERSION=3.2.6 ARG NVIDIA_TRITON_SERVER_SDK_VERSION=unknown ARG NVIDIA_BUILD_ID=unknown ############################################################################ ## Build image ############################################################################ FROM ${BASE_IMAGE} AS sdk_build # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y --no-install-recommends \ ca-certificates \ software-properties-common \ autoconf \ automake \ build-essential \ curl \ git \ gperf \ libb64-dev \ libgoogle-perftools-dev \ libopencv-dev \ libopencv-core-dev \ libssl-dev \ libtool \ pkg-config \ python3 \ python3-pip \ python3-dev \ rapidjson-dev \ vim \ wget \ python3-pdfkit \ openjdk-11-jdk \ maven && \ pip3 install --upgrade wheel setuptools && \ pip3 install --upgrade grpcio-tools && \ pip3 install --upgrade pip # Client build requires recent version of CMake (FetchContent required) # Using CMAKE installation instruction from:: https://apt.kitware.com/ RUN apt update -q=2 \ && apt install -y gpg wget \ && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \ && . /etc/os-release \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ && apt-get update -q=2 \ && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7* \ && cmake --version # Build expects "python" executable (not python3). RUN rm -f /usr/bin/python && \ ln -s /usr/bin/python3 /usr/bin/python # Build the client library and examples ARG TRITON_REPO_ORGANIZATION ARG TRITON_CLIENT_REPO_SUBDIR ARG TRITON_PA_REPO_SUBDIR ARG TRITON_COMMON_REPO_TAG ARG TRITON_CORE_REPO_TAG ARG TRITON_CLIENT_REPO_TAG ARG TRITON_THIRD_PARTY_REPO_TAG ARG TRITON_ENABLE_GPU ARG JAVA_BINDINGS_MAVEN_VERSION ARG JAVA_BINDINGS_JAVACPP_PRESETS_TAG ARG TARGETPLATFORM WORKDIR /workspace COPY TRITON_VERSION . COPY ${TRITON_CLIENT_REPO_SUBDIR} client COPY ${TRITON_PA_REPO_SUBDIR} perf_analyzer WORKDIR /workspace/client_build RUN cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ -DTRITON_VERSION=`cat /workspace/TRITON_VERSION` \ -DTRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION} \ -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ -DTRITON_THIRD_PARTY_REPO_TAG=${TRITON_THIRD_PARTY_REPO_TAG} \ -DTRITON_ENABLE_PERF_ANALYZER=OFF \ -DTRITON_ENABLE_CC_HTTP=ON -DTRITON_ENABLE_CC_GRPC=ON \ -DTRITON_ENABLE_PYTHON_HTTP=OFF -DTRITON_ENABLE_PYTHON_GRPC=OFF \ -DTRITON_ENABLE_JAVA_HTTP=ON \ -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_ENABLE_TESTS=ON \ -DTRITON_ENABLE_GPU=${TRITON_ENABLE_GPU} /workspace/client RUN make -j16 cc-clients java-clients && \ rm -fr ~/.m2 # TODO: PA will rebuild the CC clients since it depends on it. # This should be optimized so that we do not have to build # the CC clients twice. Similarly, because the SDK expectation is # that PA is packaged with the python client, we hold off on building # the python client until now. Post-migration we should focus # effort on de-tangling these flows. WORKDIR /workspace/pa_build RUN cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ -DTRITON_VERSION=`cat /workspace/TRITON_VERSION` \ -DTRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION} \ -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ -DTRITON_ENABLE_PERF_ANALYZER_C_API=ON \ -DTRITON_ENABLE_PERF_ANALYZER_TFS=ON \ -DTRITON_ENABLE_PERF_ANALYZER_TS=ON \ -DTRITON_ENABLE_PERF_ANALYZER_OPENAI=ON \ -DTRITON_ENABLE_CC_HTTP=ON \ -DTRITON_ENABLE_CC_GRPC=ON \ -DTRITON_ENABLE_PYTHON_HTTP=ON \ -DTRITON_ENABLE_PYTHON_GRPC=ON \ -DTRITON_PACKAGE_PERF_ANALYZER=ON \ -DTRITON_ENABLE_GPU=${TRITON_ENABLE_GPU} \ /workspace/perf_analyzer RUN make -j16 perf-analyzer python-clients RUN pip3 install build \ && cd /workspace/perf_analyzer/genai-perf \ && python3 -m build --wheel --outdir /workspace/install/python # Install Java API Bindings RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \ source /workspace/client/src/java-api-bindings/scripts/install_dependencies_and_build.sh \ --maven-version ${JAVA_BINDINGS_MAVEN_VERSION} \ --core-tag ${TRITON_CORE_REPO_TAG} \ --javacpp-tag ${JAVA_BINDINGS_JAVACPP_PRESETS_TAG} \ --jar-install-path /workspace/install/java-api-bindings; \ fi ############################################################################ ## Create sdk container ############################################################################ FROM ${BASE_IMAGE} # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive ARG DCGM_VERSION ARG TRITON_REPO_ORGANIZATION ARG TRITON_CORE_REPO_TAG ARG TARGETPLATFORM ARG TRITON_ENABLE_GPU RUN apt-get update && \ apt-get install -y --no-install-recommends \ software-properties-common \ curl \ git \ gperf \ libb64-dev \ libgoogle-perftools-dev \ libopencv-dev \ libopencv-core-dev \ libssl-dev \ libtool \ python3 \ python3-pip \ python3-dev \ vim \ wget \ python3-pdfkit \ maven \ default-jdk && \ pip3 install --upgrade wheel setuptools && \ pip3 install --upgrade grpcio-tools && \ pip3 install --upgrade pip WORKDIR /workspace COPY TRITON_VERSION . COPY --from=sdk_build /workspace/client/ client/ COPY --from=sdk_build /workspace/perf_analyzer/ perf_analyzer/ COPY --from=sdk_build /workspace/install/ install/ RUN cd install && \ export VERSION=`cat /workspace/TRITON_VERSION` && \ tar zcf /workspace/v$VERSION.clients.tar.gz * # For CI testing need to copy over L0_sdk test and L0_client_build_variants test. RUN mkdir qa COPY qa/L0_sdk qa/L0_sdk COPY qa/L0_client_build_variants qa/L0_client_build_variants # Create a directory for all the python client tests to enable unit testing RUN mkdir -p qa/python_client_unit_tests/ COPY --from=sdk_build /workspace/client/src/python/library/tests/* qa/python_client_unit_tests/ # Install an image needed by the quickstart and other documentation. COPY qa/images/mug.jpg images/mug.jpg # Install the dependencies needed to run the client examples. These # are not needed for building but including them allows this image to # be used to run the client examples. RUN pip3 install --upgrade "numpy<2" pillow attrdict && \ find install/python/ -maxdepth 1 -type f -name \ "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \ xargs pip3 install --upgrade RUN pip3 install install/python/genai_perf-*.whl # Install DCGM RUN if [ "$TRITON_ENABLE_GPU" = "ON" ]; then \ [ "$(uname -m)" != "x86_64" ] && arch="sbsa" || arch="x86_64" && \ curl -o /tmp/cuda-keyring.deb \ https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$arch/cuda-keyring_1.0-1_all.deb \ && apt install /tmp/cuda-keyring.deb && rm /tmp/cuda-keyring.deb && \ apt-get update && apt-get install -y datacenter-gpu-manager=1:${DCGM_VERSION}; \ fi # Build expects "python" executable (not python3). RUN rm -f /usr/bin/python && \ ln -s /usr/bin/python3 /usr/bin/python # Install Model Analyzer ARG TRITON_MODEL_ANALYZER_REPO_TAG ARG TRITON_MODEL_ANALYZER_REPO="${TRITON_REPO_ORGANIZATION}/model_analyzer@${TRITON_MODEL_ANALYZER_REPO_TAG}" RUN pip3 install "git+${TRITON_MODEL_ANALYZER_REPO}" # Entrypoint Banner ENV NVIDIA_PRODUCT_NAME="Triton Server SDK" COPY docker/entrypoint.d/ /opt/nvidia/entrypoint.d/ RUN sed 's/Server/Server SDK/' /opt/nvidia/entrypoint.d/10-banner.txt | \ sed 's/^===/=======/' > /opt/nvidia/entrypoint.d/10-banner.new && \ mv /opt/nvidia/entrypoint.d/10-banner.new /opt/nvidia/entrypoint.d/10-banner.txt ARG NVIDIA_TRITON_SERVER_SDK_VERSION ARG NVIDIA_BUILD_ID ENV NVIDIA_TRITON_SERVER_SDK_VERSION=${NVIDIA_TRITON_SERVER_SDK_VERSION} ENV NVIDIA_BUILD_ID=${NVIDIA_BUILD_ID} ENV PATH /workspace/install/bin:${PATH} ENV LD_LIBRARY_PATH /workspace/install/lib:${LD_LIBRARY_PATH} # DLIS-3631: Needed to run Perf Analyzer CI tests correctly ENV LD_LIBRARY_PATH /opt/hpcx/ompi/lib:${LD_LIBRARY_PATH} # Set TCMALLOC_RELEASE_RATE for users setting LD_PRELOAD with tcmalloc ENV TCMALLOC_RELEASE_RATE 200 ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/Dockerfile.win10.min�������������������������������������������������������0000664�0000000�0000000�00000020773�15270523712�0020665�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Windows min container for Triton build ARG BASE_IMAGE=mcr.microsoft.com/windows:10.0.19042.1889 FROM ${BASE_IMAGE} as dependency_base RUN powershell.exe Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope LocalMachine RUN powershell.exe [Net.ServicePointManager]::Expect100Continue=$true;[Net.ServicePointManager]::SecurityProtocol=[Net.SecurityProtocolType]::Tls,[Net.SecurityProtocolType]::Tls11,[Net.SecurityProtocolType]::Tls12,[Net.SecurityProtocolType]::Ssl3;Invoke-Expression( New-Object System.Net.WebClient ).DownloadString('https://chocolatey.org/install.ps1') RUN choco install unzip -y # # Installing TensorRT # ARG TENSORRT_VERSION=10.3.0.26 ARG TENSORRT_ZIP="TensorRT-${TENSORRT_VERSION}.Windows10.x86_64.cuda-12.5.zip" ARG TENSORRT_SOURCE=https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.2.0/zip/TensorRT-10.2.0.19.Windows10.x86_64.cuda-12.5.zip # COPY ${TENSORRT_ZIP} /tmp/${TENSORRT_ZIP} ADD ${TENSORRT_SOURCE} /tmp/${TENSORRT_ZIP} RUN unzip /tmp/%TENSORRT_ZIP% RUN move TensorRT-* TensorRT LABEL TENSORRT_VERSION="${TENSORRT_VERSION}" # # Installing cuDNN # ARG CUDNN_VERSION=9.3.0.75 ARG CUDNN_ZIP=cudnn-windows-x86_64-${CUDNN_VERSION}_cuda12-archive.zip ARG CUDNN_SOURCE=https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/windows-x86_64/cudnn-windows-x86_64-9.2.1.18_cuda12-archive.zip ADD ${CUDNN_SOURCE} /tmp/${CUDNN_ZIP} RUN unzip /tmp/%CUDNN_ZIP% RUN move cudnn-* cudnn LABEL CUDNN_VERSION="${CUDNN_VERSION}" FROM ${BASE_IMAGE} as build_base SHELL ["cmd", "/S", "/C"] RUN mkdir c:\tmp WORKDIR /tmp RUN powershell.exe Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope LocalMachine RUN powershell.exe [Net.ServicePointManager]::Expect100Continue=$true;[Net.ServicePointManager]::SecurityProtocol=[Net.SecurityProtocolType]::Tls,[Net.SecurityProtocolType]::Tls11,[Net.SecurityProtocolType]::Tls12,[Net.SecurityProtocolType]::Ssl3;Invoke-Expression( New-Object System.Net.WebClient ).DownloadString('https://chocolatey.org/install.ps1') RUN choco install git docker unzip -y # # Installing python # ARG PYTHON_VERSION=3.10.11 ARG PYTHON_SOURCE=https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-amd64.exe ADD ${PYTHON_SOURCE} python-${PYTHON_VERSION}-amd64.exe RUN python-%PYTHON_VERSION%-amd64.exe /quiet InstallAllUsers=1 PrependPath=1 Include_doc=0 TargetDir="C:\python%PYTHON_VERSION%" RUN mklink "C:\python%PYTHON_VERSION%\python3.exe" "C:\python%PYTHON_VERSION%\python.exe" RUN pip install --upgrade wheel setuptools docker RUN pip install grpcio-tools psutil LABEL PYTHON_VERSION=${PYTHON_VERSION} # # Installing CMake # ARG CMAKE_VERSION=3.30.0 RUN pip install cmake==%CMAKE_VERSION% ENV CMAKE_TOOLCHAIN_FILE /vcpkg/scripts/buildsystems/vcpkg.cmake ENV VCPKG_TARGET_TRIPLET x64-windows LABEL CMAKE_VERSION=${CMAKE_VERSION} # Be aware that pip can interact badly with VS cmd shell so need to pip install before # vsdevcmd.bat (see https://bugs.python.org/issue38989) # # Installing Visual Studio BuildTools: VS17 2022 # ARG BUILDTOOLS_VERSION=17.10.35201.131 # Download collect.exe in case of an install failure. ADD https://aka.ms/vscollect.exe "C:\tmp\collect.exe" # Use the latest release channel. For more control, specify the location of an internal layout. # Download the Build Tools bootstrapper. # ARG BUILD_TOOLS_SOURCE=https://aka.ms/vs/17/release/vs_buildtools.exe ARG BUILD_TOOLS_SOURCE=https://download.visualstudio.microsoft.com/download/pr/28626b4b-f88f-4b55-a0cf-f3eaa2c643fb/e6c43d4dfb36338d954cdb3ad9010ab2a479e712088f4f6b016eadcc721bab28/vs_BuildTools.exe ADD ${BUILD_TOOLS_SOURCE} vs_buildtools.exe # Install Build Tools with the Microsoft.VisualStudio.Workload.VCTools workload, including recommended. ARG VS_INSTALL_PATH_WP="C:\BuildTools" RUN vs_buildtools.exe --quiet --wait --norestart --nocache install \ --installPath %VS_INSTALL_PATH_WP% \ --add Microsoft.VisualStudio.Workload.VCTools \ --includeRecommended \ --locale "En-us" LABEL BUILDTOOLS_VERSION=${BUILDTOOLS_VERSION} WORKDIR / # # Installing Vcpkg # ARG VCPGK_VERSION=2024.03.19 RUN git clone --single-branch --depth=1 -b %VCPGK_VERSION% https://github.com/microsoft/vcpkg.git WORKDIR /vcpkg RUN bootstrap-vcpkg.bat RUN vcpkg.exe update RUN vcpkg.exe install \ boost-interprocess:x64-windows \ boost-stacktrace:x64-windows \ b64:x64-windows \ openssl-windows:x64-windows \ openssl:x64-windows \ pthread:x64-windows \ rapidjson:x64-windows \ zlib:x64-windows RUN vcpkg.exe integrate install LABEL VCPGK_VERSION=${VCPGK_VERSION} WORKDIR / # # Installing CUDA # ARG CUDA_MAJOR=12 ARG CUDA_MINOR=5 ARG CUDA_PATCH=1 ARG CUDA_VERSION=${CUDA_MAJOR}.${CUDA_MINOR}.${CUDA_PATCH} ARG CUDA_PACKAGES="nvcc_${CUDA_MAJOR}.${CUDA_MINOR} \ cudart_${CUDA_MAJOR}.${CUDA_MINOR} \ nvml_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ cublas_${CUDA_MAJOR}.${CUDA_MINOR} cublas_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ cufft_${CUDA_MAJOR}.${CUDA_MINOR} cufft_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ curand_${CUDA_MAJOR}.${CUDA_MINOR} curand_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ cusolver_${CUDA_MAJOR}.${CUDA_MINOR} cusolver_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ cusparse_${CUDA_MAJOR}.${CUDA_MINOR} cusparse_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ cupti_${CUDA_MAJOR}.${CUDA_MINOR} \ thrust_${CUDA_MAJOR}.${CUDA_MINOR} \ visual_studio_integration_${CUDA_MAJOR}.${CUDA_MINOR}" ARG CUDA_INSTALL_ROOT_WP="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${CUDA_MAJOR}.${CUDA_MINOR}" ARG CUDA_SOURCE=https://developer.download.nvidia.com/compute/cuda/${CUDA_VERSION}/network_installers/cuda_${CUDA_VERSION}_windows_network.exe ADD ${CUDA_SOURCE} cuda_${CUDA_VERSION}_windows_network.exe RUN cuda_%CUDA_VERSION%_windows_network.exe -s %CUDA_PACKAGES% # Copy the CUDA visualstudio integration from where it was installed # into the appropriate place in BuildTools RUN copy "%CUDA_INSTALL_ROOT_WP%\extras\visual_studio_integration\MSBuildExtensions\*" "%VS_INSTALL_PATH_WP%\MSBuild\Microsoft\VC\v170\BuildCustomizations" RUN setx PATH "%CUDA_INSTALL_ROOT_WP%\bin;%PATH%" ARG CUDNN_VERSION=9.3.0.75 ENV CUDNN_VERSION ${CUDNN_VERSION} COPY --from=dependency_base /cudnn /cudnn RUN copy cudnn\bin\cudnn*.dll "%CUDA_INSTALL_ROOT_WP%\bin\." RUN copy cudnn\lib\x64\cudnn*.lib "%CUDA_INSTALL_ROOT_WP%\lib\x64\." RUN copy cudnn\include\cudnn*.h "%CUDA_INSTALL_ROOT_WP%\include\." LABEL CUDNN_VERSION="${CUDNN_VERSION}" ARG TENSORRT_VERSION=10.3.0.26 ENV TRT_VERSION ${TENSORRT_VERSION} COPY --from=dependency_base /TensorRT /TensorRT RUN setx PATH "c:\TensorRT\lib;%PATH%" LABEL TENSORRT_VERSION="${TENSORRT_VERSION}" LABEL CUDA_VERSION="${CUDA_VERSION}" # It is important that the entrypoint initialize VisualStudio # environment otherwise the build will fail. Also set # CMAKE_TOOLCHAIN_FILE and VCPKG_TARGET_TRIPLET so # that cmake can find the packages installed by vcpkg. ENTRYPOINT C:\BuildTools\VC\Auxiliary\Build\vcvars64.bat && �����kytensor-2.49.0.7/server/LICENSE��������������������������������������������������������������������0000664�0000000�0000000�00000002715�15270523712�0016155�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ���������������������������������������������������kytensor-2.49.0.7/server/README.md������������������������������������������������������������������0000664�0000000�0000000�00000036325�15270523712�0016433�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������<!-- # Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> # Triton Inference Server 📣 **vLLM x Triton Meetup at Fort Mason on Sept 9th 4:00 - 9:00 pm** We are excited to announce that we will be hosting our Triton user meetup with the vLLM team at [Fort Mason](https://maps.app.goo.gl/9Lr3fxRssrpQCGK58) on Sept 9th 4:00 - 9:00 pm. Join us for this exclusive event where you will learn about the newest vLLM and Triton features, get a glimpse into the roadmaps, and connect with fellow users, the NVIDIA Triton and vLLM teams. Seating is limited and registration confirmation is required to attend - please register [here](https://lu.ma/87q3nvnh) to join the meetup. ___ [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) [!WARNING] ##### LATEST RELEASE You are currently on the `main` branch which tracks under-development progress towards the next release. The current release is version [2.49.0](https://github.com/triton-inference-server/server/releases/latest) and corresponds to the 24.08 container release on NVIDIA GPU Cloud (NGC). Triton Inference Server is an open source inference serving software that streamlines AI inferencing. Triton enables teams to deploy any AI model from multiple deep learning and machine learning frameworks, including TensorRT, TensorFlow, PyTorch, ONNX, OpenVINO, Python, RAPIDS FIL, and more. Triton Inference Server supports inference across cloud, data center, edge and embedded devices on NVIDIA GPUs, x86 and ARM CPU, or AWS Inferentia. Triton Inference Server delivers optimized performance for many query types, including real time, batched, ensembles and audio/video streaming. Triton inference Server is part of [NVIDIA AI Enterprise](https://www.nvidia.com/en-us/data-center/products/ai-enterprise/), a software platform that accelerates the data science pipeline and streamlines the development and deployment of production AI. Major features include: - [Supports multiple deep learning frameworks](https://github.com/triton-inference-server/backend#where-can-i-find-all-the-backends-that-are-available-for-triton) - [Supports multiple machine learning frameworks](https://github.com/triton-inference-server/fil_backend) - [Concurrent model execution](docs/user_guide/architecture.md#concurrent-model-execution) - [Dynamic batching](docs/user_guide/model_configuration.md#dynamic-batcher) - [Sequence batching](docs/user_guide/model_configuration.md#sequence-batcher) and [implicit state management](docs/user_guide/architecture.md#implicit-state-management) for stateful models - Provides [Backend API](https://github.com/triton-inference-server/backend) that allows adding custom backends and pre/post processing operations - Supports writing custom backends in python, a.k.a. [Python-based backends.](https://github.com/triton-inference-server/backend/blob/main/docs/python_based_backends.md#python-based-backends) - Model pipelines using [Ensembling](docs/user_guide/architecture.md#ensemble-models) or [Business Logic Scripting (BLS)](https://github.com/triton-inference-server/python_backend#business-logic-scripting) - [HTTP/REST and GRPC inference protocols](docs/customization_guide/inference_protocols.md) based on the community developed [KServe protocol](https://github.com/kserve/kserve/tree/master/docs/predict-api/v2) - A [C API](docs/customization_guide/inference_protocols.md#in-process-triton-server-api) and [Java API](docs/customization_guide/inference_protocols.md#java-bindings-for-in-process-triton-server-api) allow Triton to link directly into your application for edge and other in-process use cases - [Metrics](docs/user_guide/metrics.md) indicating GPU utilization, server throughput, server latency, and more **New to Triton Inference Server?** Make use of [these tutorials](https://github.com/triton-inference-server/tutorials) to begin your Triton journey! Join the [Triton and TensorRT community](https://www.nvidia.com/en-us/deep-learning-ai/triton-tensorrt-newsletter/) and stay current on the latest product updates, bug fixes, content, best practices, and more. Need enterprise support? NVIDIA global support is available for Triton Inference Server with the [NVIDIA AI Enterprise software suite](https://www.nvidia.com/en-us/data-center/products/ai-enterprise/). ## Serve a Model in 3 Easy Steps ```bash # Step 1: Create the example model repository git clone -b r24.08 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:24.08-py3 tritonserver --model-repository=/models # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:24.08-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': 15.346230 (504) = COFFEE MUG 13.224326 (968) = CUP 10.422965 (505) = COFFEEPOT ``` Please read the [QuickStart](docs/getting_started/quickstart.md) guide for additional information regarding this example. The quickstart guide also contains an example of how to launch Triton on [CPU-only systems](docs/getting_started/quickstart.md#run-on-cpu-only-system). New to Triton and wondering where to get started? Watch the [Getting Started video](https://youtu.be/NQDtfSi5QF4). ## Examples and Tutorials Check out [NVIDIA LaunchPad](https://www.nvidia.com/en-us/data-center/products/ai-enterprise-suite/trial/) for free access to a set of hands-on labs with Triton Inference Server hosted on NVIDIA infrastructure. Specific end-to-end examples for popular models, such as ResNet, BERT, and DLRM are located in the [NVIDIA Deep Learning Examples](https://github.com/NVIDIA/DeepLearningExamples) page on GitHub. The [NVIDIA Developer Zone](https://developer.nvidia.com/nvidia-triton-inference-server) contains additional documentation, presentations, and examples. ## Documentation ### Build and Deploy The recommended way to build and use Triton Inference Server is with Docker images. - [Install Triton Inference Server with Docker containers](docs/customization_guide/build.md#building-with-docker) (*Recommended*) - [Install Triton Inference Server without Docker containers](docs/customization_guide/build.md#building-without-docker) - [Build a custom Triton Inference Server Docker container](docs/customization_guide/compose.md) - [Build Triton Inference Server from source](docs/customization_guide/build.md#building-on-unsupported-platforms) - [Build Triton Inference Server for Windows 10](docs/customization_guide/build.md#building-for-windows-10) - Examples for deploying Triton Inference Server with Kubernetes and Helm on [GCP](deploy/gcp/README.md), [AWS](deploy/aws/README.md), and [NVIDIA FleetCommand](deploy/fleetcommand/README.md) - [Secure Deployment Considerations](docs/customization_guide/deploy.md) ### Using Triton #### Preparing Models for Triton Inference Server The first step in using Triton to serve your models is to place one or more models into a [model repository](docs/user_guide/model_repository.md). Depending on the type of the model and on what Triton capabilities you want to enable for the model, you may need to create a [model configuration](docs/user_guide/model_configuration.md) for the model. - [Add custom operations to Triton if needed by your model](docs/user_guide/custom_operations.md) - Enable model pipelining with [Model Ensemble](docs/user_guide/architecture.md#ensemble-models) and [Business Logic Scripting (BLS)](https://github.com/triton-inference-server/python_backend#business-logic-scripting) - Optimize your models setting [scheduling and batching](docs/user_guide/architecture.md#models-and-schedulers) parameters and [model instances](docs/user_guide/model_configuration.md#instance-groups). - Use the [Model Analyzer tool](https://github.com/triton-inference-server/model_analyzer) to help optimize your model configuration with profiling - Learn how to [explicitly manage what models are available by loading and unloading models](docs/user_guide/model_management.md) #### Configure and Use Triton Inference Server - Read the [Quick Start Guide](docs/getting_started/quickstart.md) to run Triton Inference Server on both GPU and CPU - Triton supports multiple execution engines, called [backends](https://github.com/triton-inference-server/backend#where-can-i-find-all-the-backends-that-are-available-for-triton), including [TensorRT](https://github.com/triton-inference-server/tensorrt_backend), [TensorFlow](https://github.com/triton-inference-server/tensorflow_backend), [PyTorch](https://github.com/triton-inference-server/pytorch_backend), [ONNX](https://github.com/triton-inference-server/onnxruntime_backend), [OpenVINO](https://github.com/triton-inference-server/openvino_backend), [Python](https://github.com/triton-inference-server/python_backend), and more - Not all the above backends are supported on every platform supported by Triton. Look at the [Backend-Platform Support Matrix](https://github.com/triton-inference-server/backend/blob/main/docs/backend_platform_support_matrix.md) to learn which backends are supported on your target platform. - Learn how to [optimize performance](docs/user_guide/optimization.md) using the [Performance Analyzer](https://github.com/triton-inference-server/perf_analyzer/blob/main/README.md) and [Model Analyzer](https://github.com/triton-inference-server/model_analyzer) - Learn how to [manage loading and unloading models](docs/user_guide/model_management.md) in Triton - Send requests directly to Triton with the [HTTP/REST JSON-based or gRPC protocols](docs/customization_guide/inference_protocols.md#httprest-and-grpc-protocols) #### Client Support and Examples A Triton *client* application sends inference and other requests to Triton. The [Python and C++ client libraries](https://github.com/triton-inference-server/client) provide APIs to simplify this communication. - Review client examples for [C++](https://github.com/triton-inference-server/client/blob/main/src/c%2B%2B/examples), [Python](https://github.com/triton-inference-server/client/blob/main/src/python/examples), and [Java](https://github.com/triton-inference-server/client/blob/main/src/java/src/main/java/triton/client/examples) - Configure [HTTP](https://github.com/triton-inference-server/client#http-options) and [gRPC](https://github.com/triton-inference-server/client#grpc-options) client options - Send input data (e.g. a jpeg image) directly to Triton in the [body of an HTTP request without any additional metadata](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_binary_data.md#raw-binary-request) ### Extend Triton [Triton Inference Server's architecture](docs/user_guide/architecture.md) is specifically designed for modularity and flexibility - [Customize Triton Inference Server container](docs/customization_guide/compose.md) for your use case - [Create custom backends](https://github.com/triton-inference-server/backend) in either [C/C++](https://github.com/triton-inference-server/backend/blob/main/README.md#triton-backend-api) or [Python](https://github.com/triton-inference-server/python_backend) - Create [decoupled backends and models](docs/user_guide/decoupled_models.md) that can send multiple responses for a request or not send any responses for a request - Use a [Triton repository agent](docs/customization_guide/repository_agents.md) to add functionality that operates when a model is loaded and unloaded, such as authentication, decryption, or conversion - Deploy Triton on [Jetson and JetPack](docs/user_guide/jetson.md) - [Use Triton on AWS Inferentia](https://github.com/triton-inference-server/python_backend/tree/main/inferentia) ### Additional Documentation - [FAQ](docs/user_guide/faq.md) - [User Guide](docs/README.md#user-guide) - [Customization Guide](docs/README.md#customization-guide) - [Release Notes](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/index.html) - [GPU, Driver, and CUDA Support Matrix](https://docs.nvidia.com/deeplearning/dgx/support-matrix/index.html) ## Contributing Contributions to Triton Inference Server are more than welcome. To contribute please review the [contribution guidelines](CONTRIBUTING.md). If you have a backend, client, example or similar contribution that is not modifying the core of Triton, then you should file a PR in the [contrib repo](https://github.com/triton-inference-server/contrib). ## Reporting problems, asking questions We appreciate any feedback, questions or bug reporting regarding this project. When posting [issues in GitHub](https://github.com/triton-inference-server/server/issues), follow the process outlined in the [Stack Overflow document](https://stackoverflow.com/help/mcve). Ensure posted examples are: - minimal – use as little code as possible that still produces the same problem - complete – provide all parts needed to reproduce the problem. Check if you can strip external dependencies and still show the problem. The less time we spend on reproducing problems the more time we have to fix it - verifiable – test the code you're about to provide to make sure it reproduces the problem. Remove all other problems that are not related to your request/question. For issues, please use the provided bug report and feature request templates. For questions, we recommend posting in our community [GitHub Discussions.](https://github.com/triton-inference-server/server/discussions) ## For more information Please refer to the [NVIDIA Developer Triton page](https://developer.nvidia.com/nvidia-triton-inference-server) for more information. �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/SECURITY.md����������������������������������������������������������������0000664�0000000�0000000�00000005014�15270523712�0016734�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������<!-- # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> # Report a Security Vulnerability To report a potential security vulnerability in any NVIDIA product, please use either: * This web form: [Security Vulnerability Submission Form](https://www.nvidia.com/object/submit-security-vulnerability.html), or * Send email to: [NVIDIA PSIRT](mailto:psirt@nvidia.com) **OEM Partners should contact their NVIDIA Customer Program Manager** If reporting a potential vulnerability via email, please encrypt it using NVIDIA’s public PGP key ([see PGP Key page](https://www.nvidia.com/en-us/security/pgp-key/)) and include the following information: 1. Product/Driver name and version/branch that contains the vulnerability 2. Type of vulnerability (code execution, denial of service, buffer overflow, etc.) 3. Instructions to reproduce the vulnerability 4. Proof-of-concept or exploit code 5. Potential impact of the vulnerability, including how an attacker could exploit the vulnerability See https://www.nvidia.com/en-us/security/ for past NVIDIA Security Bulletins and Notices. ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/TRITON_VERSION�������������������������������������������������������������0000664�0000000�0000000�00000000011�15270523712�0017302�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������2.50.0dev�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/build.py�������������������������������������������������������������������0000775�0000000�0000000�00000315454�15270523712�0016633�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/usr/bin/env python3 # Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import importlib.util import multiprocessing import os import os.path import pathlib import platform import stat import subprocess import sys from inspect import getsourcefile import distro import requests # # Build Triton Inference Server. # # By default build.py builds the Triton Docker image, but can also be # used to build without Docker. See docs/build.md and --help for more # information. # # The TRITON_VERSION file indicates the Triton version and # TRITON_VERSION_MAP is used to determine the corresponding container # version and upstream container version (upstream containers are # dependencies required by Triton). These versions may be overridden. # Map from Triton version to corresponding container and component versions. # # triton version -> # (triton container version, # upstream container version, # ORT version, # ORT OpenVINO version (use None to disable OpenVINO in ORT), # Standalone OpenVINO version, # DCGM version # ) # # Currently the OpenVINO versions used in ORT and standalone must # match because of the way dlopen works with loading the backends. If # different versions are used then one backend or the other will # incorrectly load the other version of the openvino libraries. # TRITON_VERSION_MAP = { "2.50.0dev": ( "24.09dev", # triton container "24.08", # upstream container "1.18.1", # ORT "2024.0.0", # ORT OpenVINO "2024.0.0", # Standalone OpenVINO "3.2.6", # DCGM version "0.5.3.post1", # vLLM version ) } CORE_BACKENDS = ["ensemble"] FLAGS = None EXTRA_CORE_CMAKE_FLAGS = {} OVERRIDE_CORE_CMAKE_FLAGS = {} EXTRA_BACKEND_CMAKE_FLAGS = {} OVERRIDE_BACKEND_CMAKE_FLAGS = {} THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(getsourcefile(lambda: 0))) def log(msg, force=False): if force or not FLAGS.quiet: try: print(msg, file=sys.stderr) except Exception: print("<failed to log>", file=sys.stderr) def log_verbose(msg): if FLAGS.verbose: log(msg, force=True) def fail(msg): fail_if(True, msg) def fail_if(p, msg): if p: print("error: {}".format(msg), file=sys.stderr) sys.exit(1) def target_platform(): # When called by compose.py, FLAGS will be None if FLAGS and FLAGS.target_platform is not None: return FLAGS.target_platform platform_string = platform.system().lower() if platform_string == "linux": # Need to inspect the /etc/os-release file to get # the distribution of linux id_like_list = distro.like().split() if "debian" in id_like_list: return "linux" else: return "rhel" else: return platform_string def target_machine(): # When called by compose.py, FLAGS will be None if FLAGS and FLAGS.target_machine is not None: return FLAGS.target_machine return platform.machine().lower() def container_versions(version, container_version, upstream_container_version): if container_version is None: if version not in TRITON_VERSION_MAP: fail("container version not known for {}".format(version)) container_version = TRITON_VERSION_MAP[version][0] if upstream_container_version is None: if version not in TRITON_VERSION_MAP: fail("upstream container version not known for {}".format(version)) upstream_container_version = TRITON_VERSION_MAP[version][1] return container_version, upstream_container_version class BuildScript: """Utility class for writing build scripts""" def __init__(self, filepath, desc=None, verbose=False): self._filepath = filepath self._file = open(self._filepath, "w") self._verbose = verbose self.header(desc) def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __del__(self): self.close() def close(self): if self._file is not None: if target_platform() == "windows": self.blankln() self._file.write("}\n") self._file.write("catch {\n") self._file.write(" $_;\n") self._file.write(" ExitWithCode 1;\n") self._file.write("}\n") """Close the file""" self._file.close() self._file = None st = os.stat(self._filepath) os.chmod(self._filepath, st.st_mode | stat.S_IEXEC) def blankln(self): self._file.write("\n") def commentln(self, cnt): self._file.write("#" * cnt + "\n") def comment(self, msg=""): if not isinstance(msg, str): try: for m in msg: self._file.write(f"# {msg}\n") return except TypeError: pass self._file.write(f"# {msg}\n") def comment_verbose(self, msg=""): if self._verbose: self.comment(msg) def header(self, desc=None): if target_platform() != "windows": self._file.write("#!/usr/bin/env bash\n\n") if desc is not None: self.comment() self.comment(desc) self.comment() self.blankln() self.comment("Exit script immediately if any command fails") if target_platform() == "windows": self._file.write("$UseStructuredOutput = $false\n") self.blankln() self._file.write("function ExitWithCode($exitcode) {\n") self._file.write(" $host.SetShouldExit($exitcode)\n") self._file.write(" exit $exitcode\n") self._file.write("}\n") self.blankln() if self._verbose: self._file.write("Set-PSDebug -Trace 1\n") self.blankln() self._file.write("try {\n") else: self._file.write("set -e\n") if self._verbose: self._file.write("set -x\n") self.blankln() def envvar_ref(self, v): if target_platform() == "windows": return f"${{env:{v}}}" return f"${{{v}}}" def cmd(self, clist, check_exitcode=False): if isinstance(clist, str): self._file.write(f"{clist}\n") else: for c in clist: self._file.write(f"{c} ") self.blankln() if check_exitcode: if target_platform() == "windows": self._file.write("if ($LASTEXITCODE -ne 0) {\n") self._file.write( ' Write-Output "exited with status code $LASTEXITCODE";\n' ) self._file.write(" ExitWithCode 1;\n") self._file.write("}\n") def cwd(self, path): if target_platform() == "windows": self.cmd(f"Set-Location -EV Err -EA Stop {path}") else: self.cmd(f"cd {path}") def cp(self, src, dest): if target_platform() == "windows": self.cmd(f"Copy-Item -EV Err -EA Stop {src} -Destination {dest}") else: self.cmd(f"cp {src} {dest}") def mkdir(self, path): if target_platform() == "windows": self.cmd( f"New-Item -EV Err -EA Stop -ItemType Directory -Force -Path {path}" ) else: self.cmd(f"mkdir -p {pathlib.Path(path)}") def rmdir(self, path): if target_platform() == "windows": self.cmd(f"if (Test-Path -Path {path}) {{") self.cmd(f" Remove-Item -EV Err -EA Stop -Recurse -Force {path}") self.cmd("}") else: self.cmd(f"rm -fr {pathlib.Path(path)}") def cpdir(self, src, dest): if target_platform() == "windows": self.cmd(f"Copy-Item -EV Err -EA Stop -Recurse {src} -Destination {dest}") else: self.cmd(f"cp -r {src} {dest}") def tar(self, subdir, tar_filename): if target_platform() == "windows": fail("unsupported operation: tar") else: self.cmd(f"tar zcf {tar_filename} {subdir}") def cmake(self, args): # Pass some additional envvars into cmake... env_args = [] for k in ("TRT_VERSION", "CMAKE_TOOLCHAIN_FILE", "VCPKG_TARGET_TRIPLET"): env_args += [f'"-D{k}={self.envvar_ref(k)}"'] self.cmd(f'cmake {" ".join(env_args)} {" ".join(args)}', check_exitcode=True) def makeinstall(self, target="install"): verbose_flag = "-v" if self._verbose else "" self.cmd( f"cmake --build . --config {FLAGS.build_type} -j{FLAGS.build_parallel} {verbose_flag} -t {target}" ) def gitclone(self, repo, tag, subdir, org): clone_dir = subdir if not FLAGS.no_force_clone: self.rmdir(clone_dir) if target_platform() == "windows": self.cmd(f"if (-Not (Test-Path -Path {clone_dir})) {{") else: self.cmd(f"if [[ ! -e {clone_dir} ]]; then") # FIXME [DLIS-4045 - Currently the tag starting with "pull/" is not # working with "--repo-tag" as the option is not forwarded to the # individual repo build correctly.] # If 'tag' starts with "pull/" then it must be of form # "pull/<pr>/head". We just clone at "main" and then fetch the # reference onto a new branch we name "tritonbuildref". if tag.startswith("pull/"): self.cmd( f" git clone --recursive --depth=1 {org}/{repo}.git {subdir};", check_exitcode=True, ) self.cmd("}" if target_platform() == "windows" else "fi") self.cwd(subdir) self.cmd(f"git fetch origin {tag}:tritonbuildref", check_exitcode=True) self.cmd(f"git checkout tritonbuildref", check_exitcode=True) else: self.cmd( f" git clone --recursive --single-branch --depth=1 -b {tag} {org}/{repo}.git {subdir};", check_exitcode=True, ) self.cmd("}" if target_platform() == "windows" else "fi") def cmake_core_arg(name, type, value): # Return cmake -D setting to set name=value for core build. Use # command-line specified value if one is given. if name in OVERRIDE_CORE_CMAKE_FLAGS: value = OVERRIDE_CORE_CMAKE_FLAGS[name] if type is None: type = "" else: type = ":{}".format(type) return '"-D{}{}={}"'.format(name, type, value) def cmake_core_enable(name, flag): # Return cmake -D setting to set name=flag?ON:OFF for core # build. Use command-line specified value for 'flag' if one is # given. if name in OVERRIDE_CORE_CMAKE_FLAGS: value = OVERRIDE_CORE_CMAKE_FLAGS[name] else: value = "ON" if flag else "OFF" return '"-D{}:BOOL={}"'.format(name, value) def cmake_core_extra_args(): args = [] for k, v in EXTRA_CORE_CMAKE_FLAGS.items(): args.append('"-D{}={}"'.format(k, v)) return args def cmake_backend_arg(backend, name, type, value): # Return cmake -D setting to set name=value for backend build. Use # command-line specified value if one is given. if backend in OVERRIDE_BACKEND_CMAKE_FLAGS: if name in OVERRIDE_BACKEND_CMAKE_FLAGS[backend]: value = OVERRIDE_BACKEND_CMAKE_FLAGS[backend][name] if type is None: type = "" else: type = ":{}".format(type) return '"-D{}{}={}"'.format(name, type, value) def cmake_backend_enable(backend, name, flag): # Return cmake -D setting to set name=flag?ON:OFF for backend # build. Use command-line specified value for 'flag' if one is # given. value = None if backend in OVERRIDE_BACKEND_CMAKE_FLAGS: if name in OVERRIDE_BACKEND_CMAKE_FLAGS[backend]: value = OVERRIDE_BACKEND_CMAKE_FLAGS[backend][name] if value is None: value = "ON" if flag else "OFF" return '"-D{}:BOOL={}"'.format(name, value) def cmake_backend_extra_args(backend): args = [] if backend in EXTRA_BACKEND_CMAKE_FLAGS: for k, v in EXTRA_BACKEND_CMAKE_FLAGS[backend].items(): args.append('"-D{}={}"'.format(k, v)) return args def cmake_repoagent_arg(name, type, value): # For now there is no override for repo-agents if type is None: type = "" else: type = ":{}".format(type) return '"-D{}{}={}"'.format(name, type, value) def cmake_repoagent_enable(name, flag): # For now there is no override for repo-agents value = "ON" if flag else "OFF" return '"-D{}:BOOL={}"'.format(name, value) def cmake_repoagent_extra_args(): # For now there is no extra args for repo-agents args = [] return args def cmake_cache_arg(name, type, value): # For now there is no override for caches if type is None: type = "" else: type = ":{}".format(type) return '"-D{}{}={}"'.format(name, type, value) def cmake_cache_enable(name, flag): # For now there is no override for caches value = "ON" if flag else "OFF" return '"-D{}:BOOL={}"'.format(name, value) def cmake_cache_extra_args(): # For now there is no extra args for caches args = [] return args def core_cmake_args(components, backends, cmake_dir, install_dir): cargs = [ cmake_core_arg("CMAKE_BUILD_TYPE", None, FLAGS.build_type), cmake_core_arg("CMAKE_INSTALL_PREFIX", "PATH", install_dir), cmake_core_arg("TRITON_VERSION", "STRING", FLAGS.version), cmake_core_arg("TRITON_REPO_ORGANIZATION", "STRING", FLAGS.github_organization), cmake_core_arg("TRITON_COMMON_REPO_TAG", "STRING", components["common"]), cmake_core_arg("TRITON_CORE_REPO_TAG", "STRING", components["core"]), cmake_core_arg("TRITON_BACKEND_REPO_TAG", "STRING", components["backend"]), cmake_core_arg( "TRITON_THIRD_PARTY_REPO_TAG", "STRING", components["thirdparty"] ), ] cargs.append(cmake_core_enable("TRITON_ENABLE_LOGGING", FLAGS.enable_logging)) cargs.append(cmake_core_enable("TRITON_ENABLE_STATS", FLAGS.enable_stats)) cargs.append(cmake_core_enable("TRITON_ENABLE_METRICS", FLAGS.enable_metrics)) cargs.append( cmake_core_enable("TRITON_ENABLE_METRICS_GPU", FLAGS.enable_gpu_metrics) ) cargs.append( cmake_core_enable("TRITON_ENABLE_METRICS_CPU", FLAGS.enable_cpu_metrics) ) cargs.append(cmake_core_enable("TRITON_ENABLE_TRACING", FLAGS.enable_tracing)) cargs.append(cmake_core_enable("TRITON_ENABLE_NVTX", FLAGS.enable_nvtx)) cargs.append(cmake_core_enable("TRITON_ENABLE_GPU", FLAGS.enable_gpu)) cargs.append( cmake_core_arg( "TRITON_MIN_COMPUTE_CAPABILITY", None, FLAGS.min_compute_capability ) ) cargs.append(cmake_core_enable("TRITON_ENABLE_MALI_GPU", FLAGS.enable_mali_gpu)) cargs.append(cmake_core_enable("TRITON_ENABLE_GRPC", "grpc" in FLAGS.endpoint)) cargs.append(cmake_core_enable("TRITON_ENABLE_HTTP", "http" in FLAGS.endpoint)) cargs.append( cmake_core_enable("TRITON_ENABLE_SAGEMAKER", "sagemaker" in FLAGS.endpoint) ) cargs.append( cmake_core_enable("TRITON_ENABLE_VERTEX_AI", "vertex-ai" in FLAGS.endpoint) ) cargs.append(cmake_core_enable("TRITON_ENABLE_GCS", "gcs" in FLAGS.filesystem)) cargs.append(cmake_core_enable("TRITON_ENABLE_S3", "s3" in FLAGS.filesystem)) cargs.append( cmake_core_enable( "TRITON_ENABLE_AZURE_STORAGE", "azure_storage" in FLAGS.filesystem ) ) cargs.append(cmake_core_enable("TRITON_ENABLE_ENSEMBLE", "ensemble" in backends)) cargs.append(cmake_core_enable("TRITON_ENABLE_TENSORRT", "tensorrt" in backends)) cargs += cmake_core_extra_args() cargs.append(cmake_dir) return cargs def repoagent_repo(ra): return "{}_repository_agent".format(ra) def repoagent_cmake_args(images, components, ra, install_dir): args = [] cargs = args + [ cmake_repoagent_arg("CMAKE_BUILD_TYPE", None, FLAGS.build_type), cmake_repoagent_arg("CMAKE_INSTALL_PREFIX", "PATH", install_dir), cmake_repoagent_arg( "TRITON_REPO_ORGANIZATION", "STRING", FLAGS.github_organization ), cmake_repoagent_arg("TRITON_COMMON_REPO_TAG", "STRING", components["common"]), cmake_repoagent_arg("TRITON_CORE_REPO_TAG", "STRING", components["core"]), ] cargs.append(cmake_repoagent_enable("TRITON_ENABLE_GPU", FLAGS.enable_gpu)) cargs += cmake_repoagent_extra_args() cargs.append("..") return cargs def cache_repo(cache): # example: "local", or "redis" return "{}_cache".format(cache) def cache_cmake_args(images, components, cache, install_dir): args = [] cargs = args + [ cmake_cache_arg("CMAKE_BUILD_TYPE", None, FLAGS.build_type), cmake_cache_arg("CMAKE_INSTALL_PREFIX", "PATH", install_dir), cmake_cache_arg( "TRITON_REPO_ORGANIZATION", "STRING", FLAGS.github_organization ), cmake_cache_arg("TRITON_COMMON_REPO_TAG", "STRING", components["common"]), cmake_cache_arg("TRITON_CORE_REPO_TAG", "STRING", components["core"]), ] cargs.append(cmake_cache_enable("TRITON_ENABLE_GPU", FLAGS.enable_gpu)) cargs += cmake_cache_extra_args() cargs.append("..") return cargs def backend_repo(be): return "{}_backend".format(be) def backend_cmake_args(images, components, be, install_dir, library_paths): cmake_build_type = FLAGS.build_type if be == "onnxruntime": args = onnxruntime_cmake_args(images, library_paths) elif be == "openvino": args = openvino_cmake_args() elif be == "tensorflow": args = tensorflow_cmake_args(images, library_paths) elif be == "python": args = [] elif be == "dali": args = dali_cmake_args() elif be == "pytorch": args = pytorch_cmake_args(images) elif be == "armnn_tflite": args = armnn_tflite_cmake_args() elif be == "fil": args = fil_cmake_args(images) # DLIS-4618: FIL backend fails debug build, so override it for now. cmake_build_type = "Release" elif be == "fastertransformer": args = fastertransformer_cmake_args() elif be == "tensorrt": args = tensorrt_cmake_args() elif be == "tensorrtllm": args = tensorrtllm_cmake_args(images) else: args = [] cargs = args + [ cmake_backend_arg(be, "CMAKE_BUILD_TYPE", None, cmake_build_type), cmake_backend_arg(be, "CMAKE_INSTALL_PREFIX", "PATH", install_dir), cmake_backend_arg( be, "TRITON_REPO_ORGANIZATION", "STRING", FLAGS.github_organization ), cmake_backend_arg(be, "TRITON_COMMON_REPO_TAG", "STRING", components["common"]), cmake_backend_arg(be, "TRITON_CORE_REPO_TAG", "STRING", components["core"]), cmake_backend_arg( be, "TRITON_BACKEND_REPO_TAG", "STRING", components["backend"] ), ] cargs.append(cmake_backend_enable(be, "TRITON_ENABLE_GPU", FLAGS.enable_gpu)) cargs.append( cmake_backend_enable(be, "TRITON_ENABLE_MALI_GPU", FLAGS.enable_mali_gpu) ) cargs.append(cmake_backend_enable(be, "TRITON_ENABLE_STATS", FLAGS.enable_stats)) cargs.append( cmake_backend_enable(be, "TRITON_ENABLE_METRICS", FLAGS.enable_metrics) ) # [DLIS-4950] always enable below once Windows image is updated with CUPTI # cargs.append(cmake_backend_enable(be, 'TRITON_ENABLE_MEMORY_TRACKER', True)) if (target_platform() == "windows") and (not FLAGS.no_container_build): print( "Warning: Detected docker build is used for Windows, backend utility 'device memory tracker' will be disabled due to missing library in CUDA Windows docker image." ) cargs.append(cmake_backend_enable(be, "TRITON_ENABLE_MEMORY_TRACKER", False)) elif target_platform() == "igpu": print( "Warning: Detected iGPU build, backend utility 'device memory tracker' will be disabled as iGPU doesn't contain required version of the library." ) cargs.append(cmake_backend_enable(be, "TRITON_ENABLE_MEMORY_TRACKER", False)) elif FLAGS.enable_gpu: cargs.append(cmake_backend_enable(be, "TRITON_ENABLE_MEMORY_TRACKER", True)) cargs += cmake_backend_extra_args(be) if be == "tensorrtllm": cargs.append("-S ../inflight_batcher_llm -B .") else: cargs.append("..") return cargs def pytorch_cmake_args(images): if "pytorch" in images: image = images["pytorch"] else: image = "nvcr.io/nvidia/pytorch:{}-py3".format(FLAGS.upstream_container_version) cargs = [ cmake_backend_arg("pytorch", "TRITON_PYTORCH_DOCKER_IMAGE", None, image), ] # TODO: TPRD-372 TorchTRT extension is not currently supported by our manylinux build # TODO: TPRD-373 NVTX extension is not currently supported by our manylinux build if target_platform() != "rhel": if FLAGS.enable_gpu: cargs.append( cmake_backend_enable("pytorch", "TRITON_PYTORCH_ENABLE_TORCHTRT", True) ) cargs.append( cmake_backend_enable("pytorch", "TRITON_ENABLE_NVTX", FLAGS.enable_nvtx) ) return cargs def onnxruntime_cmake_args(images, library_paths): cargs = [ cmake_backend_arg( "onnxruntime", "TRITON_BUILD_ONNXRUNTIME_VERSION", None, os.getenv("TRITON_BUILD_ONNXRUNTIME_VERSION") if os.getenv("TRITON_BUILD_ONNXRUNTIME_VERSION") else TRITON_VERSION_MAP[FLAGS.version][2], ) ] # TRITON_ENABLE_GPU is already set for all backends in backend_cmake_args() # TODO: TPRD-334 TensorRT extension is not currently supported by our manylinux build if FLAGS.enable_gpu and target_platform() != "rhel": cargs.append( cmake_backend_enable( "onnxruntime", "TRITON_ENABLE_ONNXRUNTIME_TENSORRT", True ) ) if target_platform() == "windows": if "base" in images: cargs.append( cmake_backend_arg( "onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"] ) ) else: if "base" in images: cargs.append( cmake_backend_arg( "onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"] ) ) else: cargs.append( cmake_backend_arg( "onnxruntime", "TRITON_BUILD_CONTAINER_VERSION", None, TRITON_VERSION_MAP[FLAGS.version][1], ) ) # TODO: TPRD-333 OpenVino extension is not currently supported by our manylinux build if ( (target_machine() != "aarch64") and (target_platform() != "rhel") and (TRITON_VERSION_MAP[FLAGS.version][3] is not None) ): cargs.append( cmake_backend_enable( "onnxruntime", "TRITON_ENABLE_ONNXRUNTIME_OPENVINO", True ) ) cargs.append( cmake_backend_arg( "onnxruntime", "TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION", None, TRITON_VERSION_MAP[FLAGS.version][3], ) ) if (target_platform() == "igpu") or (target_platform() == "rhel"): cargs.append( cmake_backend_arg( "onnxruntime", "TRITON_BUILD_TARGET_PLATFORM", None, target_platform(), ) ) return cargs def openvino_cmake_args(): cargs = [ cmake_backend_arg( "openvino", "TRITON_BUILD_OPENVINO_VERSION", None, TRITON_VERSION_MAP[FLAGS.version][4], ) ] if target_platform() == "windows": if "base" in images: cargs.append( cmake_backend_arg( "openvino", "TRITON_BUILD_CONTAINER", None, images["base"] ) ) else: if "base" in images: cargs.append( cmake_backend_arg( "openvino", "TRITON_BUILD_CONTAINER", None, images["base"] ) ) else: cargs.append( cmake_backend_arg( "openvino", "TRITON_BUILD_CONTAINER_VERSION", None, TRITON_VERSION_MAP[FLAGS.version][1], ) ) return cargs def tensorrt_cmake_args(): cargs = [ cmake_backend_enable("tensorrt", "TRITON_ENABLE_NVTX", FLAGS.enable_nvtx), ] if target_platform() == "windows": cargs.append( cmake_backend_arg( "tensorrt", "TRITON_TENSORRT_INCLUDE_PATHS", None, "c:/TensorRT/include" ) ) return cargs def tensorflow_cmake_args(images, library_paths): backend_name = "tensorflow" extra_args = [] # If a specific TF image is specified use it, otherwise pull from NGC. if backend_name in images: image = images[backend_name] else: image = "nvcr.io/nvidia/tensorflow:{}-tf2-py3".format( FLAGS.upstream_container_version ) extra_args = [ cmake_backend_arg(backend_name, "TRITON_TENSORFLOW_DOCKER_IMAGE", None, image) ] return extra_args def dali_cmake_args(): return [ cmake_backend_enable("dali", "TRITON_DALI_SKIP_DOWNLOAD", False), ] def fil_cmake_args(images): cargs = [cmake_backend_enable("fil", "TRITON_FIL_DOCKER_BUILD", True)] if "base" in images: cargs.append( cmake_backend_arg("fil", "TRITON_BUILD_CONTAINER", None, images["base"]) ) else: cargs.append( cmake_backend_arg( "fil", "TRITON_BUILD_CONTAINER_VERSION", None, TRITON_VERSION_MAP[FLAGS.version][1], ) ) return cargs def armnn_tflite_cmake_args(): return [ cmake_backend_arg("armnn_tflite", "JOBS", None, multiprocessing.cpu_count()), ] def fastertransformer_cmake_args(): print("Warning: FasterTransformer backend is not officially supported.") cargs = [ cmake_backend_arg( "fastertransformer", "CMAKE_EXPORT_COMPILE_COMMANDS", None, 1 ), cmake_backend_arg("fastertransformer", "ENABLE_FP8", None, "OFF"), ] return cargs def tensorrtllm_cmake_args(images): cargs = [] cargs.append(cmake_backend_enable("tensorrtllm", "USE_CXX11_ABI", True)) return cargs def install_dcgm_libraries(dcgm_version, target_machine): if dcgm_version == "": fail( "unable to determine default repo-tag, DCGM version not known for {}".format( FLAGS.version ) ) return "" else: # RHEL has the same install instructions for both aarch64 and x86 if target_platform() == "rhel": if target_machine == "aarch64": return """ ENV DCGM_VERSION {} # Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/sbsa/cuda-rhel8.repo \\ && dnf clean expire-cache \\ && dnf install -y datacenter-gpu-manager-{} """.format( dcgm_version, dcgm_version ) else: return """ ENV DCGM_VERSION {} # Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo \\ && dnf clean expire-cache \\ && dnf install -y datacenter-gpu-manager-{} """.format( dcgm_version, dcgm_version ) else: if target_machine == "aarch64": return """ ENV DCGM_VERSION {} # Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads RUN curl -o /tmp/cuda-keyring.deb \\ https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/cuda-keyring_1.0-1_all.deb \\ && apt install /tmp/cuda-keyring.deb \\ && rm /tmp/cuda-keyring.deb \\ && apt-get update \\ && apt-get install -y datacenter-gpu-manager=1:{} """.format( dcgm_version, dcgm_version ) else: return """ ENV DCGM_VERSION {} # Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads RUN curl -o /tmp/cuda-keyring.deb \\ https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb \\ && apt install /tmp/cuda-keyring.deb \\ && rm /tmp/cuda-keyring.deb \\ && apt-get update \\ && apt-get install -y datacenter-gpu-manager=1:{} """.format( dcgm_version, dcgm_version ) def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} ARG BASE_IMAGE={} """.format( argmap["TRITON_VERSION"], argmap["TRITON_CONTAINER_VERSION"], argmap["BASE_IMAGE"], ) df += """ FROM ${BASE_IMAGE} ARG TRITON_VERSION ARG TRITON_CONTAINER_VERSION """ df += """ # Install docker docker buildx RUN yum install -y ca-certificates curl gnupg yum-utils \\ && yum-config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo \\ && yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin # && yum install -y docker.io docker-buildx-plugin # libcurl4-openSSL-dev is needed for GCS # python3-dev is needed by Torchvision # python3-pip and libarchive-dev is needed by python backend # libxml2-dev is needed for Azure Storage # scons is needed for armnn_tflite backend build dep RUN yum install -y \\ ca-certificates \\ autoconf \\ automake \\ git \\ gperf \\ re2-devel \\ openssl-devel \\ libtool \\ libcurl-devel \\ libb64-devel \\ gperftools-devel \\ patchelf \\ python3.11-devel \\ python3-pip \\ python3-setuptools \\ rapidjson-devel \\ python3-scons \\ pkg-config \\ unzip \\ wget \\ zlib-devel \\ libarchive-devel \\ libxml2-devel \\ numactl-devel \\ wget RUN pip3 install --upgrade pip \\ && pip3 install --upgrade \\ wheel \\ setuptools \\ docker \\ virtualenv # Install boost version >= 1.78 for boost::span # Current libboost-dev apt packages are < 1.78, so install from tar.gz RUN wget -O /tmp/boost.tar.gz \\ https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz \\ && (cd /tmp && tar xzf boost.tar.gz) \\ && mv /tmp/boost_1_80_0/boost /usr/include/boost # Server build requires recent version of CMake (FetchContent required) # Might not need this if the installed version of cmake is high enough for our build. # RUN apt update -q=2 \\ # && apt install -y gpg wget \\ # && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \\ # && . /etc/os-release \\ # && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \\ # && apt-get update -q=2 \\ # && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7* """ if FLAGS.enable_gpu: df += install_dcgm_libraries(argmap["DCGM_VERSION"], target_machine()) df += """ ENV TRITON_SERVER_VERSION ${TRITON_VERSION} ENV NVIDIA_TRITON_SERVER_VERSION ${TRITON_CONTAINER_VERSION} """ df += """ WORKDIR /workspace RUN rm -fr * COPY . . ENTRYPOINT [] """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} ARG BASE_IMAGE={} """.format( argmap["TRITON_VERSION"], argmap["TRITON_CONTAINER_VERSION"], argmap["BASE_IMAGE"], ) df += """ FROM ${BASE_IMAGE} ARG TRITON_VERSION ARG TRITON_CONTAINER_VERSION """ # Install the windows- or linux-specific buildbase dependencies if target_platform() == "windows": df += """ SHELL ["cmd", "/S", "/C"] """ else: df += """ # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive # Install docker docker buildx RUN apt-get update \\ && apt-get install -y ca-certificates curl gnupg \\ && install -m 0755 -d /etc/apt/keyrings \\ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \\ && chmod a+r /etc/apt/keyrings/docker.gpg \\ && echo \\ "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \\ "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \\ tee /etc/apt/sources.list.d/docker.list > /dev/null \\ && apt-get update \\ && apt-get install -y docker.io docker-buildx-plugin # libcurl4-openSSL-dev is needed for GCS # python3-dev is needed by Torchvision # python3-pip and libarchive-dev is needed by python backend # libxml2-dev is needed for Azure Storage # scons is needed for armnn_tflite backend build dep RUN apt-get update \\ && apt-get install -y --no-install-recommends \\ ca-certificates \\ autoconf \\ automake \\ build-essential \\ git \\ gperf \\ libre2-dev \\ libssl-dev \\ libtool \\ libcurl4-openssl-dev \\ libb64-dev \\ libgoogle-perftools-dev \\ patchelf \\ python3-dev \\ python3-pip \\ python3-setuptools \\ rapidjson-dev \\ scons \\ software-properties-common \\ pkg-config \\ unzip \\ wget \\ zlib1g-dev \\ libarchive-dev \\ libxml2-dev \\ libnuma-dev \\ wget \\ && rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade pip \\ && pip3 install --upgrade \\ wheel \\ setuptools \\ docker \\ virtualenv # Install boost version >= 1.78 for boost::span # Current libboost-dev apt packages are < 1.78, so install from tar.gz RUN wget -O /tmp/boost.tar.gz \\ https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz \\ && (cd /tmp && tar xzf boost.tar.gz) \\ && mv /tmp/boost_1_80_0/boost /usr/include/boost # Server build requires recent version of CMake (FetchContent required) RUN apt update -q=2 \\ && apt install -y gpg wget \\ && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \\ && . /etc/os-release \\ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \\ && apt-get update -q=2 \\ && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7* """ if FLAGS.enable_gpu: df += install_dcgm_libraries(argmap["DCGM_VERSION"], target_machine()) df += """ ENV TRITON_SERVER_VERSION ${TRITON_VERSION} ENV NVIDIA_TRITON_SERVER_VERSION ${TRITON_CONTAINER_VERSION} """ # Copy in the triton source. We remove existing contents first in # case the FROM container has something there already. if target_platform() == "windows": df += """ WORKDIR /workspace RUN rmdir /S/Q * || exit 0 COPY . . """ else: df += """ WORKDIR /workspace RUN rm -fr * COPY . . ENTRYPOINT [] """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) def create_dockerfile_cibase(ddir, dockerfile_name, argmap): df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} ARG BASE_IMAGE={} """.format( argmap["TRITON_VERSION"], argmap["TRITON_CONTAINER_VERSION"], argmap["BASE_IMAGE"], ) df += """ FROM ${BASE_IMAGE} ARG TRITON_VERSION ARG TRITON_CONTAINER_VERSION COPY build/ci /workspace WORKDIR /workspace ENV TRITON_SERVER_VERSION ${TRITON_VERSION} ENV NVIDIA_TRITON_SERVER_VERSION ${TRITON_CONTAINER_VERSION} """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) def create_dockerfile_linux( ddir, dockerfile_name, argmap, backends, repoagents, caches, endpoints ): df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} ARG BASE_IMAGE={} """.format( argmap["TRITON_VERSION"], argmap["TRITON_CONTAINER_VERSION"], argmap["BASE_IMAGE"], ) # PyTorch and TensorFlow backends need extra CUDA and other # dependencies during runtime that are missing in the CPU-only base container. # These dependencies must be copied from the Triton Min image. if not FLAGS.enable_gpu and (("pytorch" in backends) or ("tensorflow" in backends)): df += """ ############################################################################ ## Triton Min image ############################################################################ FROM {} AS min_container """.format( argmap["GPU_BASE_IMAGE"] ) df += """ ############################################################################ ## Production stage: Create container with just inference server executable ############################################################################ FROM ${BASE_IMAGE} """ df += dockerfile_prepare_container_linux( argmap, backends, FLAGS.enable_gpu, target_machine() ) df += """ WORKDIR /opt COPY --chown=1000:1000 build/install tritonserver WORKDIR /opt/tritonserver """ if not FLAGS.no_core_build: # Add feature labels for SageMaker endpoint if "sagemaker" in endpoints: df += """ LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true LABEL com.amazonaws.sagemaker.capabilities.multi-models=true COPY --chown=1000:1000 docker/sagemaker/serve /usr/bin/. """ # This is required since libcublasLt.so is not present during the build # stage of the PyTorch backend if not FLAGS.enable_gpu and ("pytorch" in backends): df += """ RUN patchelf --add-needed /usr/local/cuda/lib64/stubs/libcublasLt.so.12 backends/pytorch/libtorch_cuda.so """ if "tensorrtllm" in backends: df += """ # Install required packages for TRT-LLM models # Remove contents that are not needed in runtime # Setuptools has breaking changes in version 70.0.0, so fix it to 69.5.1 # The generated code in grpc_service_pb2_grpc.py depends on grpcio>=1.64.0, so fix it to 1.64.0 RUN ldconfig && \\ ARCH="$(uname -i)" && \\ rm -fr ${TRT_ROOT}/bin ${TRT_ROOT}/targets/${ARCH}-linux-gnu/bin ${TRT_ROOT}/data && \\ rm -fr ${TRT_ROOT}/doc ${TRT_ROOT}/onnx_graphsurgeon ${TRT_ROOT}/python && \\ rm -fr ${TRT_ROOT}/samples ${TRT_ROOT}/targets/${ARCH}-linux-gnu/samples && \\ python3 -m pip install --upgrade pip && \\ pip3 install --no-cache-dir transformers && \\ find /usr -name libtensorrt_llm.so -exec dirname {} \; > /etc/ld.so.conf.d/tensorrt-llm.conf && \\ find /opt/tritonserver -name libtritonserver.so -exec dirname {} \; > /etc/ld.so.conf.d/triton-tensorrtllm-worker.conf && \\ pip3 install --no-cache-dir grpcio-tools==1.64.0 && \\ pip3 uninstall -y setuptools ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib/:/opt/tritonserver/backends/tensorrtllm:$LD_LIBRARY_PATH # There are some ucc issues when spawning mpi processes with ompi v4.1.7a1. # Downgrade to ompi v4.1.5rc2 to avoid the issue. RUN rm -fr /opt/hpcx/ompi COPY --from=nvcr.io/nvidia/tritonserver:24.02-py3-min /opt/hpcx/ompi /opt/hpcx/ompi """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_machine): gpu_enabled = 1 if enable_gpu else 0 # Common steps to produce docker images shared by build.py and compose.py. # Sets environment variables, installs dependencies and adds entrypoint df = """ ARG TRITON_VERSION ARG TRITON_CONTAINER_VERSION ENV TRITON_SERVER_VERSION ${TRITON_VERSION} ENV NVIDIA_TRITON_SERVER_VERSION ${TRITON_CONTAINER_VERSION} LABEL com.nvidia.tritonserver.version="${TRITON_SERVER_VERSION}" ENV PATH /opt/tritonserver/bin:${PATH} # Remove once https://github.com/openucx/ucx/pull/9148 is available # in the min container. ENV UCX_MEM_EVENTS no """ # Necessary for libtorch.so to find correct HPCX libraries if "pytorch" in backends: df += """ ENV LD_LIBRARY_PATH /opt/hpcx/ucc/lib/:/opt/hpcx/ucx/lib/:${LD_LIBRARY_PATH} """ backend_dependencies = "" # libgomp1 is needed by both onnxruntime and pytorch backends if ("onnxruntime" in backends) or ("pytorch" in backends): backend_dependencies = "libgomp1" # libgfortran5 is needed by pytorch backend on ARM if ("pytorch" in backends) and (target_machine == "aarch64"): backend_dependencies += " libgfortran5" # openssh-server is needed for fastertransformer if "fastertransformer" in backends: backend_dependencies += " openssh-server" df += """ ENV TF_ADJUST_HUE_FUSED 1 ENV TF_ADJUST_SATURATION_FUSED 1 ENV TF_ENABLE_WINOGRAD_NONFUSED 1 ENV TF_AUTOTUNE_THRESHOLD 2 ENV TRITON_SERVER_GPU_ENABLED {gpu_enabled} # Create a user that can be used to run triton as # non-root. Make sure that this user to given ID 1000. All server # artifacts copied below are assign to this user. ENV TRITON_SERVER_USER=triton-server RUN userdel tensorrt-server > /dev/null 2>&1 || true \\ && if ! id -u $TRITON_SERVER_USER > /dev/null 2>&1 ; then \\ useradd $TRITON_SERVER_USER; \\ fi \\ && [ `id -u $TRITON_SERVER_USER` -eq 1000 ] \\ && [ `id -g $TRITON_SERVER_USER` -eq 1000 ] """.format( gpu_enabled=gpu_enabled ) if target_platform() == "rhel": df += """ # Common dpeendencies. RUN yum install -y \\ git \\ gperf \\ re2-devel \\ openssl-devel \\ libtool \\ libcurl-devel \\ libb64-devel \\ gperftools-devel \\ patchelf \\ wget \\ numactl-devel """ else: df += """ # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive # Common dependencies. FIXME (can any of these be conditional? For # example libcurl only needed for GCS?) RUN apt-get update \\ && apt-get install -y --no-install-recommends \\ clang \\ curl \\ dirmngr \\ git \\ gperf \\ libb64-0d \\ libcurl4-openssl-dev \\ libgoogle-perftools-dev \\ libjemalloc-dev \\ libnuma-dev \\ libre2-9 \\ software-properties-common \\ wget \\ {backend_dependencies} \\ && rm -rf /var/lib/apt/lists/* """.format( backend_dependencies=backend_dependencies ) df += """ # Set TCMALLOC_RELEASE_RATE for users setting LD_PRELOAD with tcmalloc ENV TCMALLOC_RELEASE_RATE 200 """ if "fastertransformer" in backends: be = "fastertransformer" url = "https://raw.githubusercontent.com/triton-inference-server/fastertransformer_backend/{}/docker/create_dockerfile_and_build.py".format( backends[be] ) response = requests.get(url) spec = importlib.util.spec_from_loader( "fastertransformer_buildscript", loader=None, origin=url ) fastertransformer_buildscript = importlib.util.module_from_spec(spec) exec(response.content, fastertransformer_buildscript.__dict__) df += fastertransformer_buildscript.create_postbuild(is_multistage_build=False) if enable_gpu: df += install_dcgm_libraries(argmap["DCGM_VERSION"], target_machine) df += """ # Extra defensive wiring for CUDA Compat lib RUN ln -sf ${_CUDA_COMPAT_PATH}/lib.real ${_CUDA_COMPAT_PATH}/lib \\ && echo ${_CUDA_COMPAT_PATH}/lib > /etc/ld.so.conf.d/00-cuda-compat.conf \\ && ldconfig \\ && rm -f ${_CUDA_COMPAT_PATH}/lib """ else: df += add_cpu_libs_to_linux_dockerfile(backends, target_machine) # Add dependencies needed for python backend if "python" in backends: df += """ # python3, python3-pip and some pip installs required for the python backend RUN apt-get update \\ && apt-get install -y --no-install-recommends \\ python3 \\ libarchive-dev \\ python3-pip \\ libpython3-dev \\ && pip3 install --upgrade pip \\ && pip3 install --upgrade \\ wheel \\ setuptools \\ \"numpy<2\" \\ virtualenv \\ && rm -rf /var/lib/apt/lists/* """ if "tensorrtllm" in backends: df += """ # Updating the openssh-client to fix for the CVE-2024-6387. This can be removed when trtllm uses a later CUDA container(12.5 or later) RUN apt-get update \\ && apt-get install -y --no-install-recommends \\ openssh-client \\ && rm -rf /var/lib/apt/lists/* """ if "vllm" in backends: df += """ # vLLM needed for vLLM backend RUN pip3 install vllm=={} """.format( TRITON_VERSION_MAP[FLAGS.version][6] ) if "dali" in backends: df += """ # Update Python path to include DALI ENV PYTHONPATH=/opt/tritonserver/backends/dali/wheel/dali:$PYTHONPATH """ df += """ WORKDIR /opt/tritonserver RUN rm -fr /opt/tritonserver/* ENV NVIDIA_PRODUCT_NAME="Triton Server" COPY docker/entrypoint.d/ /opt/nvidia/entrypoint.d/ """ # The CPU-only build uses ubuntu as the base image, and so the # entrypoint files are not available in /opt/nvidia in the base # image, so we must provide them ourselves. if not enable_gpu: df += """ COPY docker/cpu_only/ /opt/nvidia/ ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] """ df += """ ENV NVIDIA_BUILD_ID {} LABEL com.nvidia.build.id={} LABEL com.nvidia.build.ref={} """.format( argmap["NVIDIA_BUILD_ID"], argmap["NVIDIA_BUILD_ID"], argmap["NVIDIA_BUILD_REF"] ) return df def add_cpu_libs_to_linux_dockerfile(backends, target_machine): df = "" libs_arch = "aarch64" if target_machine == "aarch64" else "x86_64" if "pytorch" in backends: # Add extra dependencies for pytorch backend. # Note: Even though the build is CPU-only, the version of pytorch # we are using depend upon libraries like cuda and cudnn. Since # these dependencies are not present in the ubuntu base image, # we must copy these from the Triton min container ourselves. cuda_arch = "sbsa" if target_machine == "aarch64" else "x86_64" df += """ RUN mkdir -p /usr/local/cuda/lib64/stubs COPY --from=min_container /usr/local/cuda/lib64/stubs/libcusparse.so /usr/local/cuda/lib64/stubs/libcusparse.so.12 COPY --from=min_container /usr/local/cuda/lib64/stubs/libcusolver.so /usr/local/cuda/lib64/stubs/libcusolver.so.11 COPY --from=min_container /usr/local/cuda/lib64/stubs/libcurand.so /usr/local/cuda/lib64/stubs/libcurand.so.10 COPY --from=min_container /usr/local/cuda/lib64/stubs/libcufft.so /usr/local/cuda/lib64/stubs/libcufft.so.11 COPY --from=min_container /usr/local/cuda/lib64/stubs/libcublas.so /usr/local/cuda/lib64/stubs/libcublas.so.12 COPY --from=min_container /usr/local/cuda/lib64/stubs/libcublasLt.so /usr/local/cuda/lib64/stubs/libcublasLt.so.12 COPY --from=min_container /usr/local/cuda/lib64/stubs/libcublasLt.so /usr/local/cuda/lib64/stubs/libcublasLt.so.11 RUN mkdir -p /usr/local/cuda/targets/{cuda_arch}-linux/lib COPY --from=min_container /usr/local/cuda/lib64/libcudart.so.12 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. COPY --from=min_container /usr/local/cuda/lib64/libcupti.so.12 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. COPY --from=min_container /usr/local/cuda/lib64/libnvToolsExt.so.1 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. COPY --from=min_container /usr/local/cuda/lib64/libnvJitLink.so.12 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. RUN mkdir -p /opt/hpcx/ucc/lib/ /opt/hpcx/ucx/lib/ COPY --from=min_container /opt/hpcx/ucc/lib/libucc.so.1 /opt/hpcx/ucc/lib/libucc.so.1 COPY --from=min_container /opt/hpcx/ucx/lib/libucm.so.0 /opt/hpcx/ucx/lib/libucm.so.0 COPY --from=min_container /opt/hpcx/ucx/lib/libucp.so.0 /opt/hpcx/ucx/lib/libucp.so.0 COPY --from=min_container /opt/hpcx/ucx/lib/libucs.so.0 /opt/hpcx/ucx/lib/libucs.so.0 COPY --from=min_container /opt/hpcx/ucx/lib/libuct.so.0 /opt/hpcx/ucx/lib/libuct.so.0 COPY --from=min_container /usr/lib/{libs_arch}-linux-gnu/libcudnn.so.9 /usr/lib/{libs_arch}-linux-gnu/libcudnn.so.9 # patchelf is needed to add deps of libcublasLt.so.12 to libtorch_cuda.so RUN apt-get update \\ && apt-get install -y --no-install-recommends openmpi-bin patchelf ENV LD_LIBRARY_PATH /usr/local/cuda/targets/{cuda_arch}-linux/lib:/usr/local/cuda/lib64/stubs:${{LD_LIBRARY_PATH}} """.format( cuda_arch=cuda_arch, libs_arch=libs_arch ) if ("pytorch" in backends) or ("tensorflow" in backends): # Add NCCL dependency for tensorflow/pytorch backend. # Note: Even though the build is CPU-only, the version of # tensorflow/pytorch we are using depends upon the NCCL library. # Since this dependency is not present in the ubuntu base image, # we must copy it from the Triton min container ourselves. df += """ COPY --from=min_container /usr/lib/{libs_arch}-linux-gnu/libnccl.so.2 /usr/lib/{libs_arch}-linux-gnu/libnccl.so.2 """.format( libs_arch=libs_arch ) return df def create_dockerfile_windows( ddir, dockerfile_name, argmap, backends, repoagents, caches ): df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} ARG BASE_IMAGE={} ############################################################################ ## Production stage: Create container with just inference server executable ############################################################################ FROM ${{BASE_IMAGE}} ARG TRITON_VERSION ARG TRITON_CONTAINER_VERSION ENV TRITON_SERVER_VERSION ${{TRITON_VERSION}} ENV NVIDIA_TRITON_SERVER_VERSION ${{TRITON_CONTAINER_VERSION}} LABEL com.nvidia.tritonserver.version="${{TRITON_SERVER_VERSION}}" RUN setx path "%path%;C:\opt\tritonserver\bin" """.format( argmap["TRITON_VERSION"], argmap["TRITON_CONTAINER_VERSION"], argmap["BASE_IMAGE"], ) df += """ WORKDIR /opt RUN rmdir /S/Q tritonserver || exit 0 COPY --chown=1000:1000 build/install tritonserver WORKDIR /opt/tritonserver """ df += """ ENTRYPOINT [] ENV NVIDIA_BUILD_ID {} LABEL com.nvidia.build.id={} LABEL com.nvidia.build.ref={} """.format( argmap["NVIDIA_BUILD_ID"], argmap["NVIDIA_BUILD_ID"], argmap["NVIDIA_BUILD_REF"] ) with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) def create_build_dockerfiles( container_build_dir, images, backends, repoagents, caches, endpoints ): if "base" in images: base_image = images["base"] elif target_platform() == "windows": base_image = "mcr.microsoft.com/dotnet/framework/sdk:4.8" elif FLAGS.enable_gpu: base_image = "nvcr.io/nvidia/tritonserver:{}-py3-min".format( FLAGS.upstream_container_version ) else: base_image = "ubuntu:22.04" dockerfileargmap = { "NVIDIA_BUILD_REF": "" if FLAGS.build_sha is None else FLAGS.build_sha, "NVIDIA_BUILD_ID": "<unknown>" if FLAGS.build_id is None else FLAGS.build_id, "TRITON_VERSION": FLAGS.version, "TRITON_CONTAINER_VERSION": FLAGS.container_version, "BASE_IMAGE": base_image, "DCGM_VERSION": "" if FLAGS.version is None or FLAGS.version not in TRITON_VERSION_MAP else TRITON_VERSION_MAP[FLAGS.version][5], } # For CPU-only image we need to copy some cuda libraries and dependencies # since we are using PyTorch and TensorFlow containers that # are not CPU-only. if ( not FLAGS.enable_gpu and (("pytorch" in backends) or ("tensorflow" in backends)) and (target_platform() != "windows") ): if "gpu-base" in images: gpu_base_image = images["gpu-base"] else: gpu_base_image = "nvcr.io/nvidia/tritonserver:{}-py3-min".format( FLAGS.upstream_container_version ) dockerfileargmap["GPU_BASE_IMAGE"] = gpu_base_image if target_platform() == "rhel": create_dockerfile_buildbase_rhel( FLAGS.build_dir, "Dockerfile.buildbase", dockerfileargmap ) else: create_dockerfile_buildbase( FLAGS.build_dir, "Dockerfile.buildbase", dockerfileargmap ) if target_platform() == "windows": create_dockerfile_windows( FLAGS.build_dir, "Dockerfile", dockerfileargmap, backends, repoagents, caches, ) else: create_dockerfile_linux( FLAGS.build_dir, "Dockerfile", dockerfileargmap, backends, repoagents, caches, endpoints, ) # Dockerfile used for the creating the CI base image. create_dockerfile_cibase(FLAGS.build_dir, "Dockerfile.cibase", dockerfileargmap) def create_docker_build_script(script_name, container_install_dir, container_ci_dir): with BuildScript( os.path.join(FLAGS.build_dir, script_name), verbose=FLAGS.verbose, desc=("Docker-based build script for Triton Inference Server"), ) as docker_script: # # Build base image... tritonserver_buildbase # docker_script.commentln(8) docker_script.comment("Create Triton base build image") docker_script.comment( "This image contains all dependencies necessary to build Triton" ) docker_script.comment() cachefrommap = [ "tritonserver_buildbase", "tritonserver_buildbase_cache0", "tritonserver_buildbase_cache1", ] baseargs = [ "docker", "build", "-t", "tritonserver_buildbase", "-f", os.path.join(FLAGS.build_dir, "Dockerfile.buildbase"), ] if not FLAGS.no_container_pull: baseargs += [ "--pull", ] # Windows docker runs in a VM and memory needs to be specified # explicitly (at least for some configurations of docker). if target_platform() == "windows": if FLAGS.container_memory: baseargs += ["--memory", FLAGS.container_memory] baseargs += ["--cache-from={}".format(k) for k in cachefrommap] baseargs += ["."] docker_script.cwd(THIS_SCRIPT_DIR) docker_script.cmd(baseargs, check_exitcode=True) # # Build... # docker_script.blankln() docker_script.commentln(8) docker_script.comment("Run build in tritonserver_buildbase container") docker_script.comment("Mount a directory into the container where the install") docker_script.comment("artifacts will be placed.") docker_script.comment() # Don't use '-v' to communicate the built artifacts out of the # build, because we want this code to work even if run within # Docker (i.e. docker-in-docker) and not just if run directly # from host. runargs = [ "docker", "run", "-w", "/workspace/build", "--name", "tritonserver_builder", ] if not FLAGS.no_container_interactive: runargs += ["-it"] if target_platform() == "windows": if FLAGS.container_memory: runargs += ["--memory", FLAGS.container_memory] runargs += ["-v", "\\\\.\pipe\docker_engine:\\\\.\pipe\docker_engine"] else: runargs += ["-v", "/var/run/docker.sock:/var/run/docker.sock"] runargs += ["tritonserver_buildbase"] if target_platform() == "windows": runargs += ["powershell.exe", "-noexit", "-File", "./cmake_build.ps1"] else: runargs += ["./cmake_build"] # Remove existing tritonserver_builder container... if target_platform() == "windows": docker_script.cmd(["docker", "rm", "tritonserver_builder"]) else: docker_script._file.write( 'if [ "$(docker ps -a | grep tritonserver_builder)" ]; then docker rm -f tritonserver_builder; fi\n' ) docker_script.cmd(runargs, check_exitcode=True) docker_script.cmd( [ "docker", "cp", "tritonserver_builder:/tmp/tritonbuild/install", FLAGS.build_dir, ], check_exitcode=True, ) docker_script.cmd( [ "docker", "cp", "tritonserver_builder:/tmp/tritonbuild/ci", FLAGS.build_dir, ], check_exitcode=True, ) # # Final image... tritonserver # docker_script.blankln() docker_script.commentln(8) docker_script.comment("Create final tritonserver image") docker_script.comment() finalargs = [ "docker", "build", "-t", "tritonserver", "-f", os.path.join(FLAGS.build_dir, "Dockerfile"), ".", ] docker_script.cwd(THIS_SCRIPT_DIR) docker_script.cmd(finalargs, check_exitcode=True) # # CI base image... tritonserver_cibase # docker_script.blankln() docker_script.commentln(8) docker_script.comment("Create CI base image") docker_script.comment() cibaseargs = [ "docker", "build", "-t", "tritonserver_cibase", "-f", os.path.join(FLAGS.build_dir, "Dockerfile.cibase"), ".", ] docker_script.cwd(THIS_SCRIPT_DIR) docker_script.cmd(cibaseargs, check_exitcode=True) def core_build( cmake_script, repo_dir, cmake_dir, build_dir, install_dir, components, backends ): repo_build_dir = os.path.join(build_dir, "tritonserver", "build") repo_install_dir = os.path.join(build_dir, "tritonserver", "install") cmake_script.commentln(8) cmake_script.comment("Triton core library and tritonserver executable") cmake_script.comment() cmake_script.mkdir(repo_build_dir) cmake_script.cwd(repo_build_dir) cmake_script.cmake( core_cmake_args(components, backends, cmake_dir, repo_install_dir) ) cmake_script.makeinstall() if target_platform() == "windows": cmake_script.mkdir(os.path.join(install_dir, "bin")) cmake_script.cp( os.path.join(repo_install_dir, "bin", "tritonserver.exe"), os.path.join(install_dir, "bin"), ) cmake_script.cp( os.path.join(repo_install_dir, "bin", "tritonserver.dll"), os.path.join(install_dir, "bin"), ) cmake_script.cp( os.path.join(repo_install_dir, "lib", "tritonserver.lib"), os.path.join(install_dir, "bin"), ) elif target_platform() == "rhel": cmake_script.mkdir(os.path.join(install_dir, "bin")) cmake_script.cp( os.path.join(repo_install_dir, "bin", "tritonserver"), os.path.join(install_dir, "bin"), ) cmake_script.mkdir(os.path.join(install_dir, "lib64")) cmake_script.cp( os.path.join(repo_install_dir, "lib64", "libtritonserver.so"), os.path.join(install_dir, "lib64"), ) else: cmake_script.mkdir(os.path.join(install_dir, "bin")) cmake_script.cp( os.path.join(repo_install_dir, "bin", "tritonserver"), os.path.join(install_dir, "bin"), ) cmake_script.mkdir(os.path.join(install_dir, "lib")) cmake_script.cp( os.path.join(repo_install_dir, "lib", "libtritonserver.so"), os.path.join(install_dir, "lib"), ) # [FIXME] Placing the Triton server wheel file in 'python' for now, should # have been upload to pip registry and be able to install directly cmake_script.mkdir(os.path.join(install_dir, "python")) cmake_script.cp( os.path.join(repo_install_dir, "python", "tritonserver*.whl"), os.path.join(install_dir, "python"), ) cmake_script.mkdir(os.path.join(install_dir, "include", "triton")) cmake_script.cpdir( os.path.join(repo_install_dir, "include", "triton", "core"), os.path.join(install_dir, "include", "triton", "core"), ) cmake_script.cp(os.path.join(repo_dir, "LICENSE"), install_dir) cmake_script.cp(os.path.join(repo_dir, "TRITON_VERSION"), install_dir) # If requested, package the source code for all OSS used to build # For windows, Triton is not delivered as a container so skip for # windows platform. if target_platform() != "windows": if ( (not FLAGS.no_container_build) and (not FLAGS.no_core_build) and (not FLAGS.no_container_source) ): cmake_script.mkdir(os.path.join(install_dir, "third-party-src")) cmake_script.cwd(repo_build_dir) cmake_script.tar( "third-party-src", os.path.join(install_dir, "third-party-src", "src.tar.gz"), ) cmake_script.cp( os.path.join(repo_dir, "docker", "README.third-party-src"), os.path.join(install_dir, "third-party-src", "README"), ) cmake_script.comment() cmake_script.comment("end Triton core library and tritonserver executable") cmake_script.commentln(8) cmake_script.blankln() def tensorrtllm_prebuild(cmake_script): # Export the TRT_ROOT environment variable cmake_script.cmd("export TRT_ROOT=/usr/local/tensorrt") cmake_script.cmd("export ARCH=$(uname -m)") cmake_script.cmd( 'export LD_LIBRARY_PATH="/usr/local/cuda/compat/lib.real:${LD_LIBRARY_PATH}"' ) def tensorrtllm_postbuild(cmake_script, repo_install_dir, tensorrtllm_be_dir): # TODO: Update the CMakeLists.txt of TRT-LLM backend to install the artifacts to the correct location cmake_destination_dir = os.path.join(repo_install_dir, "backends/tensorrtllm") cmake_script.mkdir(cmake_destination_dir) # Copy over the TRT-LLM backend libraries cmake_script.cp( os.path.join(tensorrtllm_be_dir, "build", "libtriton_tensorrtllm*.so"), cmake_destination_dir, ) cmake_script.cp( os.path.join(tensorrtllm_be_dir, "build", "trtllmExecutorWorker"), cmake_destination_dir, ) def backend_build( be, cmake_script, tag, build_dir, install_dir, github_organization, images, components, library_paths, ): repo_build_dir = os.path.join(build_dir, be, "build") repo_install_dir = os.path.join(build_dir, be, "install") cmake_script.commentln(8) cmake_script.comment(f"'{be}' backend") cmake_script.comment("Delete this section to remove backend from build") cmake_script.comment() cmake_script.mkdir(build_dir) cmake_script.cwd(build_dir) cmake_script.gitclone(backend_repo(be), tag, be, github_organization) if be == "tensorrtllm": tensorrtllm_prebuild(cmake_script) cmake_script.mkdir(repo_build_dir) cmake_script.cwd(repo_build_dir) cmake_script.cmake( backend_cmake_args(images, components, be, repo_install_dir, library_paths) ) cmake_script.makeinstall() if be == "tensorrtllm": tensorrtllm_be_dir = os.path.join(build_dir, be) tensorrtllm_postbuild(cmake_script, repo_install_dir, tensorrtllm_be_dir) cmake_script.mkdir(os.path.join(install_dir, "backends")) cmake_script.rmdir(os.path.join(install_dir, "backends", be)) cmake_script.cpdir( os.path.join(repo_install_dir, "backends", be), os.path.join(install_dir, "backends"), ) cmake_script.comment() cmake_script.comment(f"end '{be}' backend") cmake_script.commentln(8) cmake_script.blankln() def backend_clone( be, clone_script, tag, build_dir, install_dir, github_organization, ): clone_script.commentln(8) clone_script.comment(f"'{be}' backend") clone_script.comment("Delete this section to remove backend from build") clone_script.comment() clone_script.mkdir(build_dir) clone_script.cwd(build_dir) clone_script.gitclone(backend_repo(be), tag, be, github_organization) repo_target_dir = os.path.join(install_dir, "backends") clone_script.mkdir(repo_target_dir) backend_dir = os.path.join(repo_target_dir, be) clone_script.rmdir(backend_dir) clone_script.mkdir(backend_dir) clone_script.cp( os.path.join(build_dir, be, "src", "model.py"), backend_dir, ) clone_script.cpdir( os.path.join(build_dir, be, "src", "utils"), backend_dir, ) clone_script.comment() clone_script.comment(f"end '{be}' backend") clone_script.commentln(8) clone_script.blankln() def repo_agent_build( ra, cmake_script, build_dir, install_dir, repoagent_repo, repoagents ): repo_build_dir = os.path.join(build_dir, ra, "build") repo_install_dir = os.path.join(build_dir, ra, "install") cmake_script.commentln(8) cmake_script.comment(f"'{ra}' repository agent") cmake_script.comment("Delete this section to remove repository agent from build") cmake_script.comment() cmake_script.mkdir(build_dir) cmake_script.cwd(build_dir) cmake_script.gitclone( repoagent_repo(ra), repoagents[ra], ra, FLAGS.github_organization ) cmake_script.mkdir(repo_build_dir) cmake_script.cwd(repo_build_dir) cmake_script.cmake(repoagent_cmake_args(images, components, ra, repo_install_dir)) cmake_script.makeinstall() cmake_script.mkdir(os.path.join(install_dir, "repoagents")) cmake_script.rmdir(os.path.join(install_dir, "repoagents", ra)) cmake_script.cpdir( os.path.join(repo_install_dir, "repoagents", ra), os.path.join(install_dir, "repoagents"), ) cmake_script.comment() cmake_script.comment(f"end '{ra}' repository agent") cmake_script.commentln(8) cmake_script.blankln() def cache_build(cache, cmake_script, build_dir, install_dir, cache_repo, caches): repo_build_dir = os.path.join(build_dir, cache, "build") repo_install_dir = os.path.join(build_dir, cache, "install") cmake_script.commentln(8) cmake_script.comment(f"'{cache}' cache") cmake_script.comment("Delete this section to remove cache from build") cmake_script.comment() cmake_script.mkdir(build_dir) cmake_script.cwd(build_dir) cmake_script.gitclone( cache_repo(cache), caches[cache], cache, FLAGS.github_organization ) cmake_script.mkdir(repo_build_dir) cmake_script.cwd(repo_build_dir) cmake_script.cmake(cache_cmake_args(images, components, cache, repo_install_dir)) cmake_script.makeinstall() cmake_script.mkdir(os.path.join(install_dir, "caches")) cmake_script.rmdir(os.path.join(install_dir, "caches", cache)) cmake_script.cpdir( os.path.join(repo_install_dir, "caches", cache), os.path.join(install_dir, "caches"), ) cmake_script.comment() cmake_script.comment(f"end '{cache}' cache") cmake_script.commentln(8) cmake_script.blankln() def cibase_build( cmake_script, repo_dir, cmake_dir, build_dir, install_dir, ci_dir, backends ): repo_install_dir = os.path.join(build_dir, "tritonserver", "install") cmake_script.commentln(8) cmake_script.comment("Collect Triton CI artifacts") cmake_script.comment() cmake_script.mkdir(ci_dir) # On windows we are not yet using a CI/QA docker image for # testing, so don't do anything... if target_platform() == "windows": return # The core build produces some artifacts that are needed for CI # testing, so include those in the install. cmake_script.cpdir(os.path.join(repo_dir, "qa"), ci_dir) cmake_script.cpdir(os.path.join(repo_dir, "deploy"), ci_dir) cmake_script.mkdir(os.path.join(ci_dir, "docs")) cmake_script.cpdir( os.path.join(repo_dir, "docs", "examples"), os.path.join(ci_dir, "docs") ) cmake_script.mkdir(os.path.join(ci_dir, "src", "test")) cmake_script.cpdir( os.path.join(repo_dir, "src", "test", "models"), os.path.join(ci_dir, "src", "test"), ) # Skip copying the artifacts in the bin, lib, and python as those directories will # be missing when the core build is not enabled. if not FLAGS.no_core_build: cmake_script.cpdir(os.path.join(repo_install_dir, "bin"), ci_dir) cmake_script.mkdir(os.path.join(ci_dir, "lib")) cmake_script.cp( os.path.join(repo_install_dir, "lib", "libtritonrepoagent_relocation.so"), os.path.join(ci_dir, "lib"), ) cmake_script.cpdir(os.path.join(repo_install_dir, "python"), ci_dir) # Some of the backends are needed for CI testing cmake_script.mkdir(os.path.join(ci_dir, "backends")) for be in ("identity", "repeat", "square"): be_install_dir = os.path.join(build_dir, be, "install", "backends", be) if target_platform() == "windows": cmake_script.cmd(f"if (Test-Path -Path {be_install_dir}) {{") else: cmake_script.cmd(f"if [[ -e {be_install_dir} ]]; then") cmake_script.cpdir(be_install_dir, os.path.join(ci_dir, "backends")) cmake_script.cmd("}" if target_platform() == "windows" else "fi") # Some of the unit-test built backends are needed for CI testing cmake_script.mkdir(os.path.join(ci_dir, "tritonbuild", "tritonserver", "backends")) for be in ( "query", "implicit_state", "sequence", "dyna_sequence", "distributed_addsub", "iterative_sequence", ): be_install_dir = os.path.join(repo_install_dir, "backends", be) if target_platform() == "windows": cmake_script.cmd(f"if (Test-Path -Path {be_install_dir}) {{") else: cmake_script.cmd(f"if [[ -e {be_install_dir} ]]; then") cmake_script.cpdir( be_install_dir, os.path.join(ci_dir, "tritonbuild", "tritonserver", "backends"), ) cmake_script.cmd("}" if target_platform() == "windows" else "fi") # The onnxruntime_backend build produces some artifacts that # are needed for CI testing. if "onnxruntime" in backends: ort_install_dir = os.path.join(build_dir, "onnxruntime", "install") cmake_script.mkdir(os.path.join(ci_dir, "qa", "L0_custom_ops")) if target_platform() != "igpu": cmake_script.cp( os.path.join(ort_install_dir, "test", "libcustom_op_library.so"), os.path.join(ci_dir, "qa", "L0_custom_ops"), ) cmake_script.cp( os.path.join(ort_install_dir, "test", "custom_op_test.onnx"), os.path.join(ci_dir, "qa", "L0_custom_ops"), ) # [WIP] other way than wildcard? backend_tests = os.path.join(build_dir, "onnxruntime", "test", "*") cmake_script.cpdir(backend_tests, os.path.join(ci_dir, "qa")) # Need the build area for some backends so that they can be # rebuilt with specific options. cmake_script.mkdir(os.path.join(ci_dir, "tritonbuild")) for be in ("identity", "python"): if be in backends: cmake_script.rmdir(os.path.join(build_dir, be, "build")) cmake_script.rmdir(os.path.join(build_dir, be, "install")) cmake_script.cpdir( os.path.join(build_dir, be), os.path.join(ci_dir, "tritonbuild") ) cmake_script.comment() cmake_script.comment("end Triton CI artifacts") cmake_script.commentln(8) cmake_script.blankln() def finalize_build(cmake_script, install_dir, ci_dir): cmake_script.cmd(f"chmod -R a+rw {install_dir}") cmake_script.cmd(f"chmod -R a+rw {ci_dir}") def enable_all(): if target_platform() != "windows": all_backends = [ "ensemble", "identity", "square", "repeat", "tensorflow", "onnxruntime", "python", "dali", "pytorch", "openvino", "fil", "tensorrt", ] all_repoagents = ["checksum"] all_caches = ["local", "redis"] all_filesystems = ["gcs", "s3", "azure_storage"] all_endpoints = ["http", "grpc", "sagemaker", "vertex-ai"] FLAGS.enable_logging = True FLAGS.enable_stats = True FLAGS.enable_metrics = True FLAGS.enable_gpu_metrics = True FLAGS.enable_cpu_metrics = True FLAGS.enable_tracing = True FLAGS.enable_nvtx = True FLAGS.enable_gpu = True else: all_backends = [ "ensemble", "identity", "square", "repeat", "onnxruntime", "openvino", "tensorrt", ] all_repoagents = ["checksum"] all_caches = ["local", "redis"] all_filesystems = [] all_endpoints = ["http", "grpc"] FLAGS.enable_logging = True FLAGS.enable_stats = True FLAGS.enable_tracing = True FLAGS.enable_gpu = True requested_backends = [] for be in FLAGS.backend: parts = be.split(":") requested_backends += [parts[0]] for be in all_backends: if be not in requested_backends: FLAGS.backend += [be] requested_repoagents = [] for ra in FLAGS.repoagent: parts = ra.split(":") requested_repoagents += [parts[0]] for ra in all_repoagents: if ra not in requested_repoagents: FLAGS.repoagent += [ra] requested_caches = [] for cache in FLAGS.cache: parts = cache.split(":") requested_caches += [parts[0]] for cache in all_caches: if cache not in requested_caches: FLAGS.cache += [cache] for fs in all_filesystems: if fs not in FLAGS.filesystem: FLAGS.filesystem += [fs] for ep in all_endpoints: if ep not in FLAGS.endpoint: FLAGS.endpoint += [ep] if __name__ == "__main__": parser = argparse.ArgumentParser() group_qv = parser.add_mutually_exclusive_group() group_qv.add_argument( "-q", "--quiet", action="store_true", required=False, help="Disable console output.", ) group_qv.add_argument( "-v", "--verbose", action="store_true", required=False, help="Enable verbose output.", ) parser.add_argument( "--dryrun", action="store_true", required=False, help="Output the build scripts, but do not perform build.", ) parser.add_argument( "--no-container-build", action="store_true", required=False, help="Do not use Docker container for build.", ) parser.add_argument( "--no-container-interactive", action="store_true", required=False, help='Do not use -it argument to "docker run" when performing container build.', ) parser.add_argument( "--no-container-pull", action="store_true", required=False, help="Do not use Docker --pull argument when building container.", ) parser.add_argument( "--container-memory", default=None, required=False, help="Value for Docker --memory argument. Used only for windows builds.", ) parser.add_argument( "--target-platform", required=False, default=None, help='Target platform for build, can be "linux", "rhel", "windows" or "igpu". If not specified, build targets the current platform.', ) parser.add_argument( "--target-machine", required=False, default=None, help="Target machine/architecture for build. If not specified, build targets the current machine/architecture.", ) parser.add_argument( "--build-id", type=str, required=False, help="Build ID associated with the build.", ) parser.add_argument( "--build-sha", type=str, required=False, help="SHA associated with the build." ) parser.add_argument( "--build-dir", type=str, required=False, help="Build directory. All repo clones and builds will be performed in this directory.", ) parser.add_argument( "--install-dir", type=str, required=False, default=None, help="Install directory, default is <builddir>/opt/tritonserver.", ) parser.add_argument( "--cmake-dir", type=str, required=False, help="Directory containing the CMakeLists.txt file for Triton server.", ) parser.add_argument( "--tmp-dir", type=str, required=False, default="/tmp", help="Temporary directory used for building inside docker. Default is /tmp.", ) parser.add_argument( "--library-paths", action="append", required=False, default=None, help="Specify library paths for respective backends in build as <backend-name>[:<library_path>].", ) parser.add_argument( "--build-type", required=False, default="Release", help='Build type, one of "Release", "Debug", "RelWithDebInfo" or "MinSizeRel". Default is "Release".', ) parser.add_argument( "-j", "--build-parallel", type=int, required=False, default=None, help="Build parallelism. Defaults to 2 * number-of-cores.", ) parser.add_argument( "--github-organization", type=str, required=False, default="https://github.com/triton-inference-server", help='The GitHub organization containing the repos used for the build. Defaults to "https://github.com/triton-inference-server".', ) parser.add_argument( "--version", type=str, required=False, help="The Triton version. If not specified defaults to the value in the TRITON_VERSION file.", ) parser.add_argument( "--container-version", type=str, required=False, help="The Triton container version to build. If not specified the container version will be chosen automatically based on --version value.", ) parser.add_argument( "--upstream-container-version", type=str, required=False, help="The upstream container version to use for the build. If not specified the upstream container version will be chosen automatically based on --version value.", ) parser.add_argument( "--container-prebuild-command", type=str, required=False, help="When performing a container build, this command will be executed within the container just before the build it performed.", ) parser.add_argument( "--no-container-source", action="store_true", required=False, help="Do not include OSS source code in Docker container.", ) parser.add_argument( "--image", action="append", required=False, help='Use specified Docker image in build as <image-name>,<full-image-name>. <image-name> can be "base", "gpu-base", "tensorflow", or "pytorch".', ) parser.add_argument( "--enable-all", action="store_true", required=False, help="Enable all standard released Triton features, backends, repository agents, caches, endpoints and file systems.", ) parser.add_argument( "--enable-logging", action="store_true", required=False, help="Enable logging." ) parser.add_argument( "--enable-stats", action="store_true", required=False, help="Enable statistics collection.", ) parser.add_argument( "--enable-metrics", action="store_true", required=False, help="Enable metrics reporting.", ) parser.add_argument( "--enable-gpu-metrics", action="store_true", required=False, help="Include GPU metrics in reported metrics.", ) parser.add_argument( "--enable-cpu-metrics", action="store_true", required=False, help="Include CPU metrics in reported metrics.", ) parser.add_argument( "--enable-tracing", action="store_true", required=False, help="Enable tracing." ) parser.add_argument( "--enable-nvtx", action="store_true", required=False, help="Enable NVTX." ) parser.add_argument( "--enable-gpu", action="store_true", required=False, help="Enable GPU support." ) parser.add_argument( "--enable-mali-gpu", action="store_true", required=False, help="Enable ARM MALI GPU support.", ) parser.add_argument( "--min-compute-capability", type=str, required=False, default="6.0", help="Minimum CUDA compute capability supported by server.", ) parser.add_argument( "--endpoint", action="append", required=False, help='Include specified endpoint in build. Allowed values are "grpc", "http", "vertex-ai" and "sagemaker".', ) parser.add_argument( "--filesystem", action="append", required=False, help='Include specified filesystem in build. Allowed values are "gcs", "azure_storage" and "s3".', ) parser.add_argument( "--no-core-build", action="store_true", required=False, help="Do not build Triton core shared library or executable.", ) parser.add_argument( "--backend", action="append", required=False, help='Include specified backend in build as <backend-name>[:<repo-tag>]. If <repo-tag> starts with "pull/" then it refers to a pull-request reference, otherwise <repo-tag> indicates the git tag/branch to use for the build. If the version is non-development then the default <repo-tag> is the release branch matching the container version (e.g. version YY.MM -> branch rYY.MM); otherwise the default <repo-tag> is "main" (e.g. version YY.MMdev -> branch main).', ) parser.add_argument( "--repo-tag", action="append", required=False, help='The version of a component to use in the build as <component-name>:<repo-tag>. <component-name> can be "common", "core", "backend" or "thirdparty". <repo-tag> indicates the git tag/branch to use for the build. Currently <repo-tag> does not support pull-request reference. If the version is non-development then the default <repo-tag> is the release branch matching the container version (e.g. version YY.MM -> branch rYY.MM); otherwise the default <repo-tag> is "main" (e.g. version YY.MMdev -> branch main).', ) parser.add_argument( "--repoagent", action="append", required=False, help='Include specified repo agent in build as <repoagent-name>[:<repo-tag>]. If <repo-tag> starts with "pull/" then it refers to a pull-request reference, otherwise <repo-tag> indicates the git tag/branch to use for the build. If the version is non-development then the default <repo-tag> is the release branch matching the container version (e.g. version YY.MM -> branch rYY.MM); otherwise the default <repo-tag> is "main" (e.g. version YY.MMdev -> branch main).', ) parser.add_argument( "--cache", action="append", required=False, help='Include specified cache in build as <cache-name>[:<repo-tag>]. If <repo-tag> starts with "pull/" then it refers to a pull-request reference, otherwise <repo-tag> indicates the git tag/branch to use for the build. If the version is non-development then the default <repo-tag> is the release branch matching the container version (e.g. version YY.MM -> branch rYY.MM); otherwise the default <repo-tag> is "main" (e.g. version YY.MMdev -> branch main).', ) parser.add_argument( "--no-force-clone", action="store_true", default=False, help="Do not create fresh clones of repos that have already been cloned.", ) parser.add_argument( "--extra-core-cmake-arg", action="append", required=False, help="Extra CMake argument as <name>=<value>. The argument is passed to CMake as -D<name>=<value> and is included after all CMake arguments added by build.py for the core builds.", ) parser.add_argument( "--override-core-cmake-arg", action="append", required=False, help="Override specified CMake argument in the build as <name>=<value>. The argument is passed to CMake as -D<name>=<value>. This flag only impacts CMake arguments that are used by build.py. To unconditionally add a CMake argument to the core build use --extra-core-cmake-arg.", ) parser.add_argument( "--extra-backend-cmake-arg", action="append", required=False, help="Extra CMake argument for a backend build as <backend>:<name>=<value>. The argument is passed to CMake as -D<name>=<value> and is included after all CMake arguments added by build.py for the backend.", ) parser.add_argument( "--override-backend-cmake-arg", action="append", required=False, help="Override specified backend CMake argument in the build as <backend>:<name>=<value>. The argument is passed to CMake as -D<name>=<value>. This flag only impacts CMake arguments that are used by build.py. To unconditionally add a CMake argument to the backend build use --extra-backend-cmake-arg.", ) FLAGS = parser.parse_args() if FLAGS.image is None: FLAGS.image = [] if FLAGS.repo_tag is None: FLAGS.repo_tag = [] if FLAGS.backend is None: FLAGS.backend = [] if FLAGS.endpoint is None: FLAGS.endpoint = [] if FLAGS.filesystem is None: FLAGS.filesystem = [] if FLAGS.repoagent is None: FLAGS.repoagent = [] if FLAGS.cache is None: FLAGS.cache = [] if FLAGS.library_paths is None: FLAGS.library_paths = [] if FLAGS.extra_core_cmake_arg is None: FLAGS.extra_core_cmake_arg = [] if FLAGS.override_core_cmake_arg is None: FLAGS.override_core_cmake_arg = [] if FLAGS.override_backend_cmake_arg is None: FLAGS.override_backend_cmake_arg = [] if FLAGS.extra_backend_cmake_arg is None: FLAGS.extra_backend_cmake_arg = [] # if --enable-all is specified, then update FLAGS to enable all # settings, backends, repo-agents, caches, file systems, endpoints, etc. if FLAGS.enable_all: enable_all() # When doing a docker build, --build-dir, --install-dir and # --cmake-dir must not be set. We will use the build/ subdir # within the server/ repo that contains this build.py script for # --build-dir. If not doing a docker build, --build-dir must be # set. if FLAGS.no_container_build: if FLAGS.build_dir is None: fail("--no-container-build requires --build-dir") if FLAGS.install_dir is None: FLAGS.install_dir = os.path.join(FLAGS.build_dir, "opt", "tritonserver") if FLAGS.cmake_dir is None: FLAGS.cmake_dir = THIS_SCRIPT_DIR else: if FLAGS.build_dir is not None: fail("--build-dir must not be set for container-based build") if FLAGS.install_dir is not None: fail("--install-dir must not be set for container-based build") if FLAGS.cmake_dir is not None: fail("--cmake-dir must not be set for container-based build") FLAGS.build_dir = os.path.join(THIS_SCRIPT_DIR, "build") # Determine the versions. Start with Triton version, if --version # is not explicitly specified read from TRITON_VERSION file. if FLAGS.version is None: with open(os.path.join(THIS_SCRIPT_DIR, "TRITON_VERSION"), "r") as vfile: FLAGS.version = vfile.readline().strip() if FLAGS.build_parallel is None: FLAGS.build_parallel = multiprocessing.cpu_count() * 2 log("Building Triton Inference Server") log("platform {}".format(target_platform())) log("machine {}".format(target_machine())) log("version {}".format(FLAGS.version)) log("build dir {}".format(FLAGS.build_dir)) log("install dir {}".format(FLAGS.install_dir)) log("cmake dir {}".format(FLAGS.cmake_dir)) # Determine the default repo-tag that should be used for images, # backends, repo-agents, and caches if a repo-tag is not given # explicitly. For release branches we use the release branch as # the default, otherwise we use 'main'. default_repo_tag = "main" cver = FLAGS.container_version if cver is None: if FLAGS.version not in TRITON_VERSION_MAP: fail( "unable to determine default repo-tag, container version not known for {}".format( FLAGS.version ) ) cver = TRITON_VERSION_MAP[FLAGS.version][0] if not cver.endswith("dev"): default_repo_tag = "r" + cver log("default repo-tag: {}".format(default_repo_tag)) # For other versions use the TRITON_VERSION_MAP unless explicitly # given. FLAGS.container_version, FLAGS.upstream_container_version = container_versions( FLAGS.version, FLAGS.container_version, FLAGS.upstream_container_version ) log("container version {}".format(FLAGS.container_version)) log("upstream container version {}".format(FLAGS.upstream_container_version)) for ep in FLAGS.endpoint: log(f'endpoint "{ep}"') for fs in FLAGS.filesystem: log(f'filesystem "{fs}"') # Initialize map of backends to build and repo-tag for each. backends = {} for be in FLAGS.backend: parts = be.split(":") if len(parts) == 1: parts.append(default_repo_tag) if parts[0] == "tensorflow1": fail( "Starting from Triton version 23.04, support for TensorFlow 1 has been discontinued. Please switch to Tensorflow 2." ) if parts[0] == "tensorflow2": parts[0] = "tensorflow" log('backend "{}" at tag/branch "{}"'.format(parts[0], parts[1])) backends[parts[0]] = parts[1] if "vllm" in backends: if "python" not in backends: log( "vLLM backend requires Python backend, adding Python backend with tag {}".format( backends["vllm"] ) ) backends["python"] = backends["vllm"] # Initialize map of repo agents to build and repo-tag for each. repoagents = {} for be in FLAGS.repoagent: parts = be.split(":") if len(parts) == 1: parts.append(default_repo_tag) log('repoagent "{}" at tag/branch "{}"'.format(parts[0], parts[1])) repoagents[parts[0]] = parts[1] # Initialize map of caches to build and repo-tag for each. caches = {} for be in FLAGS.cache: parts = be.split(":") if len(parts) == 1: parts.append(default_repo_tag) log('cache "{}" at tag/branch "{}"'.format(parts[0], parts[1])) caches[parts[0]] = parts[1] # Initialize map of docker images. images = {} for img in FLAGS.image: parts = img.split(",") fail_if( len(parts) != 2, "--image must specify <image-name>,<full-image-registry>" ) fail_if( parts[0] not in ["base", "gpu-base", "pytorch", "tensorflow", "tensorflow2"], "unsupported value for --image", ) log('image "{}": "{}"'.format(parts[0], parts[1])) if parts[0] == "tensorflow2": parts[0] = "tensorflow" images[parts[0]] = parts[1] # Initialize map of library paths for each backend. library_paths = {} for lpath in FLAGS.library_paths: parts = lpath.split(":") if len(parts) == 2: log('backend "{}" library path "{}"'.format(parts[0], parts[1])) if parts[0] == "tensorflow2": parts[0] = "tensorflow" library_paths[parts[0]] = parts[1] # Parse any explicitly specified cmake arguments for cf in FLAGS.extra_core_cmake_arg: parts = cf.split("=") fail_if(len(parts) != 2, "--extra-core-cmake-arg must specify <name>=<value>") log('CMake core extra "-D{}={}"'.format(parts[0], parts[1])) EXTRA_CORE_CMAKE_FLAGS[parts[0]] = parts[1] for cf in FLAGS.override_core_cmake_arg: parts = cf.split("=") fail_if( len(parts) != 2, "--override-core-cmake-arg must specify <name>=<value>" ) log('CMake core override "-D{}={}"'.format(parts[0], parts[1])) OVERRIDE_CORE_CMAKE_FLAGS[parts[0]] = parts[1] for cf in FLAGS.extra_backend_cmake_arg: parts = cf.split(":", 1) fail_if( len(parts) != 2, "--extra-backend-cmake-arg must specify <backend>:<name>=<value>", ) be = parts[0] parts = parts[1].split("=", 1) fail_if( len(parts) != 2, "--extra-backend-cmake-arg must specify <backend>:<name>=<value>", ) fail_if( be not in backends, '--extra-backend-cmake-arg specifies backend "{}" which is not included in build'.format( be ), ) log('backend "{}" CMake extra "-D{}={}"'.format(be, parts[0], parts[1])) if be not in EXTRA_BACKEND_CMAKE_FLAGS: EXTRA_BACKEND_CMAKE_FLAGS[be] = {} EXTRA_BACKEND_CMAKE_FLAGS[be][parts[0]] = parts[1] for cf in FLAGS.override_backend_cmake_arg: parts = cf.split(":", 1) fail_if( len(parts) != 2, "--override-backend-cmake-arg must specify <backend>:<name>=<value>", ) be = parts[0] parts = parts[1].split("=", 1) fail_if( len(parts) != 2, "--override-backend-cmake-arg must specify <backend>:<name>=<value>", ) fail_if( be not in backends, '--override-backend-cmake-arg specifies backend "{}" which is not included in build'.format( be ), ) log('backend "{}" CMake override "-D{}={}"'.format(be, parts[0], parts[1])) if be not in OVERRIDE_BACKEND_CMAKE_FLAGS: OVERRIDE_BACKEND_CMAKE_FLAGS[be] = {} OVERRIDE_BACKEND_CMAKE_FLAGS[be][parts[0]] = parts[1] # Initialize map of common components and repo-tag for each. components = { "common": default_repo_tag, "core": default_repo_tag, "backend": default_repo_tag, "thirdparty": default_repo_tag, } for be in FLAGS.repo_tag: parts = be.split(":") fail_if(len(parts) != 2, "--repo-tag must specify <component-name>:<repo-tag>") fail_if( parts[0] not in components, '--repo-tag <component-name> must be "common", "core", "backend", or "thirdparty"', ) components[parts[0]] = parts[1] for c in components: log('component "{}" at tag/branch "{}"'.format(c, components[c])) # Set the build, install, and cmake directories to use for the # generated build scripts and Dockerfiles. If building without # Docker, these are the directories specified on the cmdline. If # building with Docker, we change these to be directories within # FLAGS.tmp_dir inside the Docker container. script_repo_dir = THIS_SCRIPT_DIR script_build_dir = FLAGS.build_dir script_install_dir = script_ci_dir = FLAGS.install_dir script_cmake_dir = FLAGS.cmake_dir if not FLAGS.no_container_build: # FLAGS.tmp_dir may be specified with "\" on Windows, adjust # to "/" for docker usage. script_build_dir = os.path.normpath( os.path.join(FLAGS.tmp_dir, "tritonbuild").replace("\\", "/") ) script_install_dir = os.path.normpath(os.path.join(script_build_dir, "install")) script_ci_dir = os.path.normpath(os.path.join(script_build_dir, "ci")) if target_platform() == "windows": script_repo_dir = script_cmake_dir = os.path.normpath("c:/workspace") else: script_repo_dir = script_cmake_dir = "/workspace" script_name = "cmake_build" if target_platform() == "windows": script_name += ".ps1" # Write the build script that invokes cmake for the core, backends, repo-agents, and caches. pathlib.Path(FLAGS.build_dir).mkdir(parents=True, exist_ok=True) with BuildScript( os.path.join(FLAGS.build_dir, script_name), verbose=FLAGS.verbose, desc=("Build script for Triton Inference Server"), ) as cmake_script: # Run the container pre-build command if the cmake build is # being done within the build container. if not FLAGS.no_container_build and FLAGS.container_prebuild_command: cmake_script.cmd(FLAGS.container_prebuild_command, check_exitcode=True) cmake_script.blankln() # Commands to build the core shared library and the server executable. if not FLAGS.no_core_build: core_build( cmake_script, script_repo_dir, script_cmake_dir, script_build_dir, script_install_dir, components, backends, ) # Commands to build each backend... for be in backends: # Core backends are not built separately from core so skip... if be in CORE_BACKENDS: continue # If armnn_tflite backend, source from external repo for git clone if be == "armnn_tflite": github_organization = "https://gitlab.com/arm-research/smarter/" else: github_organization = FLAGS.github_organization if be == "vllm": backend_clone( be, cmake_script, backends[be], script_build_dir, script_install_dir, github_organization, ) else: backend_build( be, cmake_script, backends[be], script_build_dir, script_install_dir, github_organization, images, components, library_paths, ) # Commands to build each repo agent... for ra in repoagents: repo_agent_build( ra, cmake_script, script_build_dir, script_install_dir, repoagent_repo, repoagents, ) # Commands to build each cache... for cache in caches: cache_build( cache, cmake_script, script_build_dir, script_install_dir, cache_repo, caches, ) # Commands needed only when building with Docker... if not FLAGS.no_container_build: # Commands to collect all the build artifacts needed for CI # testing. cibase_build( cmake_script, script_repo_dir, script_cmake_dir, script_build_dir, script_install_dir, script_ci_dir, backends, ) # When building with Docker the install and ci artifacts # written to the build-dir while running the docker container # may have root ownership, so give them permissions to be # managed by all users on the host system. if target_platform() != "windows": finalize_build(cmake_script, script_install_dir, script_ci_dir) # If --no-container-build is not specified then we perform the # actual build within a docker container and from that create the # final tritonserver docker image. For the build we need to # generate a few Dockerfiles and a top-level script that drives # the build process. if not FLAGS.no_container_build: script_name = "docker_build" if target_platform() == "windows": script_name += ".ps1" create_build_dockerfiles( script_build_dir, images, backends, repoagents, caches, FLAGS.endpoint ) create_docker_build_script(script_name, script_install_dir, script_ci_dir) # In not dry-run, execute the script to perform the build... If a # container-based build is requested use 'docker_build' script, # otherwise build directly on this system using cmake script. if not FLAGS.dryrun: if target_platform() == "windows": p = subprocess.Popen( ["powershell.exe", "-noexit", "-File", f"./{script_name}"], cwd=FLAGS.build_dir, ) else: p = subprocess.Popen([f"./{script_name}"], cwd=FLAGS.build_dir) p.wait() fail_if(p.returncode != 0, "build failed") ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/compose.py�����������������������������������������������������������������0000775�0000000�0000000�00000042177�15270523712�0017200�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/usr/bin/env python3 # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os import platform import subprocess import sys FLAGS = None #### helper functions def log(msg, force=False): if force or not FLAGS.quiet: try: print(msg, file=sys.stderr) except Exception: print("<failed to log>", file=sys.stderr) def log_verbose(msg): if FLAGS.verbose: log(msg, force=True) def fail(msg): print("error: {}".format(msg), file=sys.stderr) sys.exit(1) def fail_if(p, msg): if p: fail(msg) def start_dockerfile(ddir, images, argmap, dockerfile_name, backends): # Set environment variables, set default user and install dependencies df = """ # # Multistage build. # ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} FROM {} AS full """.format( argmap["TRITON_VERSION"], argmap["TRITON_CONTAINER_VERSION"], images["full"] ) # PyTorch, TensorFlow backends need extra CUDA and other # dependencies during runtime that are missing in the CPU-only base container. # These dependencies must be copied from the Triton Min image. if not FLAGS.enable_gpu and ( ("pytorch" in backends) or ("tensorflow" in backends) or ("tensorflow2" in backends) ): df += """ FROM {} AS min_container """.format( images["gpu-min"] ) df += """ FROM {} """.format( images["min"] ) import build df += build.dockerfile_prepare_container_linux( argmap, backends, FLAGS.enable_gpu, platform.machine().lower() ) # Copy over files df += """ WORKDIR /opt/tritonserver COPY --chown=1000:1000 --from=full /opt/tritonserver/LICENSE . COPY --chown=1000:1000 --from=full /opt/tritonserver/TRITON_VERSION . COPY --chown=1000:1000 --from=full /opt/tritonserver/bin bin/ COPY --chown=1000:1000 --from=full /opt/tritonserver/lib lib/ COPY --chown=1000:1000 --from=full /opt/tritonserver/include include/ """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) def add_requested_backends(ddir, dockerfile_name, backends): df = "# Copying over backends \n" for backend in backends: df += """COPY --chown=1000:1000 --from=full /opt/tritonserver/backends/{} /opt/tritonserver/backends/{} """.format( backend, backend ) if len(backends) > 0: df += """ # Top-level /opt/tritonserver/backends not copied so need to explicitly set permissions here RUN chown triton-server:triton-server /opt/tritonserver/backends """ with open(os.path.join(ddir, dockerfile_name), "a") as dfile: dfile.write(df) def add_requested_repoagents(ddir, dockerfile_name, repoagents): df = "# Copying over repoagents \n" for ra in repoagents: df += """COPY --chown=1000:1000 --from=full /opt/tritonserver/repoagents/{} /opt/tritonserver/repoagents/{} """.format( ra, ra ) if len(repoagents) > 0: df += """ # Top-level /opt/tritonserver/repoagents not copied so need to explicitly set permissions here RUN chown triton-server:triton-server /opt/tritonserver/repoagents """ with open(os.path.join(ddir, dockerfile_name), "a") as dfile: dfile.write(df) def add_requested_caches(ddir, dockerfile_name, caches): df = "# Copying over caches \n" for cache in caches: df += """COPY --chown=1000:1000 --from=full /opt/tritonserver/caches/{} /opt/tritonserver/caches/{} """.format( cache, cache ) if len(caches) > 0: df += """ # Top-level /opt/tritonserver/caches not copied so need to explicitly set permissions here RUN chown triton-server:triton-server /opt/tritonserver/caches """ with open(os.path.join(ddir, dockerfile_name), "a") as dfile: dfile.write(df) def end_dockerfile(ddir, dockerfile_name, argmap): # Install additional dependencies df = "" if argmap["SAGEMAKER_ENDPOINT"]: df += """ LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true COPY --chown=1000:1000 --from=full /usr/bin/serve /usr/bin/. """ with open(os.path.join(ddir, dockerfile_name), "a") as dfile: dfile.write(df) def build_docker_image(ddir, dockerfile_name, container_name): # Create container with docker build p = subprocess.Popen( [ "docker", "build", "-t", container_name, "-f", os.path.join(ddir, dockerfile_name), ".", ] ) p.wait() fail_if(p.returncode != 0, "docker build {} failed".format(container_name)) def get_container_version_if_not_specified(): if FLAGS.container_version is None: # Read from TRITON_VERSION file in server repo to determine version with open("TRITON_VERSION", "r") as vfile: version = vfile.readline().strip() import build _, FLAGS.container_version = build.container_versions( version, None, FLAGS.container_version ) log("version {}".format(version)) log("using container version {}".format(FLAGS.container_version)) def create_argmap(images, skip_pull): # Extract information from upstream build and create map other functions can # use full_docker_image = images["full"] min_docker_image = images["min"] enable_gpu = FLAGS.enable_gpu # Docker inspect environment variables base_run_args = ["docker", "inspect", "-f"] import re # parse all PATH environment variables # first pull docker images if not skip_pull: log("pulling container:{}".format(full_docker_image)) p = subprocess.run(["docker", "pull", full_docker_image]) fail_if( p.returncode != 0, "docker pull container {} failed, {}".format(full_docker_image, p.stderr), ) if enable_gpu: if not skip_pull: pm = subprocess.run(["docker", "pull", min_docker_image]) fail_if( pm.returncode != 0 and not skip_pull, "docker pull container {} failed, {}".format( min_docker_image, pm.stderr ), ) pm_path = subprocess.run( base_run_args + [ "{{range $index, $value := .Config.Env}}{{$value}} {{end}}", min_docker_image, ], capture_output=True, text=True, ) fail_if( pm_path.returncode != 0, "docker inspect to find triton environment variables for min container failed, {}".format( pm_path.stderr ), ) # min container needs to be GPU-support-enabled if the build is GPU build vars = pm_path.stdout e = re.search("CUDA_VERSION", vars) gpu_enabled = False if e is None else True fail_if( not gpu_enabled, "Composing container with gpu support enabled but min container provided does not have CUDA installed", ) # Check full container environment variables p_path = subprocess.run( base_run_args + [ "{{range $index, $value := .Config.Env}}{{$value}} {{end}}", full_docker_image, ], capture_output=True, text=True, ) fail_if( p_path.returncode != 0, "docker inspect to find environment variables for full container failed, {}".format( p_path.stderr ), ) vars = p_path.stdout log_verbose("inspect args: {}".format(vars)) e0 = re.search("TRITON_SERVER_GPU_ENABLED=([\S]{1,}) ", vars) e1 = re.search("CUDA_VERSION", vars) gpu_enabled = False if e0 != None: gpu_enabled = e0.group(1) == "1" elif e1 != None: gpu_enabled = True fail_if( gpu_enabled != enable_gpu, "Error: full container provided was build with " "'TRITON_SERVER_GPU_ENABLED' as {} and you are composing container" "with 'TRITON_SERVER_GPU_ENABLED' as {}".format(gpu_enabled, enable_gpu), ) e = re.search("TRITON_SERVER_VERSION=([\S]{6,}) ", vars) version = "" if e is None else e.group(1) fail_if( len(version) == 0, "docker inspect to find triton server version failed, {}".format(p_path.stderr), ) e = re.search("NVIDIA_TRITON_SERVER_VERSION=([\S]{5,}) ", vars) container_version = "" if e is None else e.group(1) fail_if( len(container_version) == 0, "docker inspect to find triton container version failed, {}".format(vars), ) dcgm_ver = re.search("DCGM_VERSION=([\S]{4,}) ", vars) dcgm_version = "" if dcgm_ver is None: dcgm_version = "2.2.3" log( "WARNING: DCGM version not found from image, installing the earlierst version {}".format( dcgm_version ) ) else: dcgm_version = dcgm_ver.group(1) fail_if( len(dcgm_version) == 0, "docker inspect to find DCGM version failed, {}".format(vars), ) p_sha = subprocess.run( base_run_args + ['{{ index .Config.Labels "com.nvidia.build.ref"}}', full_docker_image], capture_output=True, text=True, ) fail_if( p_sha.returncode != 0, "docker inspect of upstream docker image build sha failed, {}".format( p_sha.stderr ), ) p_build = subprocess.run( base_run_args + ['{{ index .Config.Labels "com.nvidia.build.id"}}', full_docker_image], capture_output=True, text=True, ) fail_if( p_build.returncode != 0, "docker inspect of upstream docker image build sha failed, {}".format( p_build.stderr ), ) p_find = subprocess.run( ["docker", "run", full_docker_image, "bash", "-c", "ls /usr/bin/"], capture_output=True, text=True, ) f = re.search("serve", p_find.stdout) fail_if( p_find.returncode != 0, "Cannot search for 'serve' in /usr/bin, {}".format(p_find.stderr), ) argmap = { "NVIDIA_BUILD_REF": p_sha.stdout.rstrip(), "NVIDIA_BUILD_ID": p_build.stdout.rstrip(), "TRITON_VERSION": version, "TRITON_CONTAINER_VERSION": container_version, "DCGM_VERSION": dcgm_version, "SAGEMAKER_ENDPOINT": f is not None, } return argmap if __name__ == "__main__": parser = argparse.ArgumentParser() group_qv = parser.add_mutually_exclusive_group() group_qv.add_argument( "-q", "--quiet", action="store_true", required=False, help="Disable console output.", ) group_qv.add_argument( "-v", "--verbose", action="store_true", required=False, help="Enable verbose output.", ) parser.add_argument( "--output-name", type=str, required=False, help='Name for the generated Docker image. Default is "tritonserver".', ) parser.add_argument( "--work-dir", type=str, required=False, help="Generated dockerfiles are placed here. Default to current directory.", ) parser.add_argument( "--container-version", type=str, required=False, help="The version to use for the generated Docker image. If not specified " "the container version will be chosen automatically based on the " "repository branch.", ) parser.add_argument( "--image", action="append", required=False, help="Use specified Docker image to generate Docker image. Specified as " '<image-name>,<full-image-name>. <image-name> can be "min", "gpu-min" ' 'or "full". Both "min" and "full" need to be specified at the same time.' 'This will override "--container-version". "gpu-min" is needed for ' "CPU-only container to copy TensorFlow and PyTorch deps.", ) parser.add_argument( "--enable-gpu", nargs="?", type=lambda x: (str(x).lower() == "true"), const=True, default=True, required=False, help=argparse.SUPPRESS, ) parser.add_argument( "--backend", action="append", required=False, help="Include <backend-name> in the generated Docker image. The flag may be " "specified multiple times.", ) parser.add_argument( "--repoagent", action="append", required=False, help="Include <repoagent-name> in the generated Docker image. The flag may " "be specified multiple times.", ) parser.add_argument( "--cache", action="append", required=False, help="Include <cache-name> in the generated Docker image. The flag may " "be specified multiple times.", ) parser.add_argument( "--skip-pull", action="store_true", required=False, help="Do not pull the required docker images. The user is responsible " "for pulling the upstream images needed to compose the image.", ) parser.add_argument( "--dry-run", action="store_true", required=False, help="Only creates Dockerfile.compose, does not build the Docker image.", ) FLAGS = parser.parse_args() if FLAGS.work_dir is None: FLAGS.work_dir = "." if FLAGS.output_name is None: FLAGS.output_name = "tritonserver" dockerfile_name = "Dockerfile.compose" if FLAGS.backend is None: FLAGS.backend = [] if FLAGS.repoagent is None: FLAGS.repoagent = [] if FLAGS.cache is None: FLAGS.cache = [] # Initialize map of docker images. images = {} if FLAGS.image: for img in FLAGS.image: parts = img.split(",") fail_if( len(parts) != 2, "--image must specific <image-name>,<full-image-registry>", ) fail_if( parts[0] not in ["min", "full", "gpu-min"], "unsupported image-name '{}' for --image".format(parts[0]), ) log('image "{}": "{}"'.format(parts[0], parts[1])) images[parts[0]] = parts[1] else: get_container_version_if_not_specified() if FLAGS.enable_gpu: images = { "full": "nvcr.io/nvidia/tritonserver:{}-py3".format( FLAGS.container_version ), "min": "nvcr.io/nvidia/tritonserver:{}-py3-min".format( FLAGS.container_version ), } else: images = { "full": "nvcr.io/nvidia/tritonserver:{}-cpu-only-py3".format( FLAGS.container_version ), "min": "ubuntu:22.04", } fail_if(len(images) < 2, "Need to specify both 'full' and 'min' images if at all") # For CPU-only image we need to copy some cuda libraries and dependencies # since we are using PyTorch, TensorFlow 1, TensorFlow 2 containers that # are not CPU-only. if ( ("pytorch" in FLAGS.backend) or ("tensorflow" in FLAGS.backend) or ("tensorflow2" in FLAGS.backend) ) and ("gpu-min" not in images): images["gpu-min"] = "nvcr.io/nvidia/tritonserver:{}-py3-min".format( FLAGS.container_version ) argmap = create_argmap(images, FLAGS.skip_pull) start_dockerfile(FLAGS.work_dir, images, argmap, dockerfile_name, FLAGS.backend) add_requested_backends(FLAGS.work_dir, dockerfile_name, FLAGS.backend) add_requested_repoagents(FLAGS.work_dir, dockerfile_name, FLAGS.repoagent) add_requested_caches(FLAGS.work_dir, dockerfile_name, FLAGS.cache) end_dockerfile(FLAGS.work_dir, dockerfile_name, argmap) if not FLAGS.dry_run: build_docker_image(FLAGS.work_dir, dockerfile_name, FLAGS.output_name) �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/��������������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0016412�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/README.third-party-src����������������������������������������������0000664�0000000�0000000�00000000243�15270523712�0022324�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������This directory contains the licenses and source code for software included in the Triton Inference Server build. To extract the files use: $ tar zxf src.tar.gz �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/cpu_only/�����������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020242�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/cpu_only/entrypoint.d/����������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022677�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/cpu_only/entrypoint.d/12-banner.sh����������������������������������0000775�0000000�0000000�00000001014�15270523712�0024717�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/bash # Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. prodname_uc=$(echo "${NVIDIA_PRODUCT_NAME}" | tr [:lower:] [:upper:] | sed 's/ /_/g' | sed 's/^NVIDIA_//') # Product name _prodver="NVIDIA_${prodname_uc}_VERSION" # Container product version variable name _compver="${prodname_uc}_VERSION" # Upstream component version variable name echo echo "NVIDIA Release ${!_prodver} (build ${NVIDIA_BUILD_ID})" [ -n "${!_compver}" ] && echo "${NVIDIA_PRODUCT_NAME} Version ${!_compver}" ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/cpu_only/entrypoint.d/50-gpu-driver-check2.sh�����������������������0000775�0000000�0000000�00000000171�15270523712�0026700�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/bash # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. export TRITON_SERVER_CPU_ONLY=1 �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/cpu_only/nvidia_entrypoint.sh���������������������������������������0000775�0000000�0000000�00000004102�15270523712�0024343�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/bash # Copyright 2016-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Gather parts in alpha order shopt -s nullglob extglob SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" declare -a PARTS=( "${SCRIPT_DIR}/entrypoint.d"/*@(.txt|.sh) ) shopt -u nullglob extglob # Execute the entrypoint parts for file in "${PARTS[@]}"; do case "${file}" in *.txt) cat "${file}";; *.sh) source "${file}";; esac done echo # This script can either be a wrapper around arbitrary command lines, # or it will simply exec bash if no arguments were given if [[ $# -eq 0 ]]; then exec "/bin/bash" else exec "$@" fi ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/entrypoint.d/�������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021047�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/entrypoint.d/10-banner.txt������������������������������������������0000664�0000000�0000000�00000000133�15270523712�0023270�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������ ============================= == Triton Inference Server == ============================= �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/entrypoint.d/15-container-copyright.txt�����������������������������0000664�0000000�0000000�00000000121�15270523712�0026015�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������ Copyright (c) 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/entrypoint.d/50-gpu-driver-check2.sh��������������������������������0000775�0000000�0000000�00000000254�15270523712�0025052�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/bash # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. if [[ "${NVIDIA_CPU_ONLY:-0}" == "1" ]]; then export TRITON_SERVER_CPU_ONLY=1 fi ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/entrypoint.d/56-network-driver-version-check.sh���������������������0000664�0000000�0000000�00000000001�15270523712�0027342�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������ �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/entrypoint.d/70-shm-check.sh����������������������������������������0000664�0000000�0000000�00000000001�15270523712�0023460�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������ �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/entrypoint.d/99-check-run-aip-mode.sh�������������������������������0000775�0000000�0000000�00000000603�15270523712�0025214�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/bash # Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # If detect Vertex AI environment, launch tritonserver with supplied arguments # This has the effect of "unshifting" the tritonserver command onto the front # of $@ if AIP_MODE is nonempty; it will then be exec'd by entrypoint.sh set -- ${AIP_MODE:+"/opt/tritonserver/bin/tritonserver"} "$@" �����������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/sagemaker/����������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020351�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/docker/sagemaker/serve�����������������������������������������������������0000775�0000000�0000000�00000020256�15270523712�0021430�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/bash # Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAGEMAKER_SINGLE_MODEL_REPO=/opt/ml/model/ # Use 'ready' for ping check in single-model endpoint mode, and use 'live' for ping check in multi-model endpoint model # https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/rest_predict_v2.yaml#L10-L26 if [ -n "$SAGEMAKER_TRITON_OVERRIDE_PING_MODE" ]; then SAGEMAKER_TRITON_PING_MODE=${SAGEMAKER_TRITON_OVERRIDE_PING_MODE} else SAGEMAKER_TRITON_PING_MODE="ready" fi # Note: in Triton on SageMaker, each model url is registered as a separate repository # e.g., /opt/ml/models/<hash>/model. Specifying MME model repo path as /opt/ml/models causes Triton # to treat it as an additional empty repository and changes # the state of all models to be UNAVAILABLE in the model repository # https://github.com/triton-inference-server/core/blob/main/src/model_repository_manager.cc#L914,L922 # On Triton, this path will be a dummy path as it's mandatory to specify a model repo when starting triton SAGEMAKER_MULTI_MODEL_REPO=/tmp/sagemaker SAGEMAKER_MODEL_REPO=${SAGEMAKER_SINGLE_MODEL_REPO} is_mme_mode=false if [ -n "$SAGEMAKER_MULTI_MODEL" ]; then if [ "$SAGEMAKER_MULTI_MODEL" == "true" ]; then mkdir -p ${SAGEMAKER_MULTI_MODEL_REPO} SAGEMAKER_MODEL_REPO=${SAGEMAKER_MULTI_MODEL_REPO} if [ -n "$SAGEMAKER_TRITON_OVERRIDE_PING_MODE" ]; then SAGEMAKER_TRITON_PING_MODE=${SAGEMAKER_TRITON_OVERRIDE_PING_MODE} else SAGEMAKER_TRITON_PING_MODE="live" fi is_mme_mode=true echo -e "Triton is running in SageMaker MME mode. Using Triton ping mode: \"${SAGEMAKER_TRITON_PING_MODE}\"" fi fi SAGEMAKER_ARGS="--model-repository=${SAGEMAKER_MODEL_REPO}" #Set model namespacing to true, but allow disabling if required if [ -n "$SAGEMAKER_TRITON_DISABLE_MODEL_NAMESPACING" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --model-namespacing=${SAGEMAKER_TRITON_DISABLE_MODEL_NAMESPACING}" else SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --model-namespacing=true" fi if [ -n "$SAGEMAKER_BIND_TO_PORT" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --sagemaker-port=${SAGEMAKER_BIND_TO_PORT}" fi if [ -n "$SAGEMAKER_SAFE_PORT_RANGE" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --sagemaker-safe-port-range=${SAGEMAKER_SAFE_PORT_RANGE}" fi if [ -n "$SAGEMAKER_TRITON_ALLOW_GRPC" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --allow-grpc=${SAGEMAKER_TRITON_ALLOW_GRPC}" else SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --allow-grpc=false" fi if [ -n "$SAGEMAKER_TRITON_ALLOW_METRICS" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --allow-metrics=${SAGEMAKER_TRITON_ALLOW_METRICS}" else SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --allow-metrics=false" fi if [ -n "$SAGEMAKER_TRITON_METRICS_PORT" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --metrics-port=${SAGEMAKER_TRITON_METRICS_PORT}" fi if [ -n "$SAGEMAKER_TRITON_GRPC_PORT" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --grpc-port=${SAGEMAKER_TRITON_GRPC_PORT}" fi if [ -n "$SAGEMAKER_TRITON_BUFFER_MANAGER_THREAD_COUNT" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --buffer-manager-thread-count=${SAGEMAKER_TRITON_BUFFER_MANAGER_THREAD_COUNT}" fi if [ -n "$SAGEMAKER_TRITON_THREAD_COUNT" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --sagemaker-thread-count=${SAGEMAKER_TRITON_THREAD_COUNT}" fi # Enable verbose logging by default. If env variable is specified, use value from env variable if [ -n "$SAGEMAKER_TRITON_LOG_VERBOSE" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --log-verbose=${SAGEMAKER_TRITON_LOG_VERBOSE}" else SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --log-verbose=true" fi if [ -n "$SAGEMAKER_TRITON_LOG_INFO" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --log-info=${SAGEMAKER_TRITON_LOG_INFO}" fi if [ -n "$SAGEMAKER_TRITON_LOG_WARNING" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --log-warning=${SAGEMAKER_TRITON_LOG_WARNING}" fi if [ -n "$SAGEMAKER_TRITON_LOG_ERROR" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --log-error=${SAGEMAKER_TRITON_LOG_ERROR}" fi if [ -n "$SAGEMAKER_TRITON_SHM_DEFAULT_BYTE_SIZE" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --backend-config=python,shm-default-byte-size=${SAGEMAKER_TRITON_SHM_DEFAULT_BYTE_SIZE}" else SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --backend-config=python,shm-default-byte-size=16777216" #16MB fi if [ -n "$SAGEMAKER_TRITON_SHM_GROWTH_BYTE_SIZE" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --backend-config=python,shm-growth-byte-size=${SAGEMAKER_TRITON_SHM_GROWTH_BYTE_SIZE}" else SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --backend-config=python,shm-growth-byte-size=1048576" #1MB fi if [ -n "$SAGEMAKER_TRITON_TENSORFLOW_VERSION" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --backend-config=tensorflow,version=${SAGEMAKER_TRITON_TENSORFLOW_VERSION}" fi if [ -n "$SAGEMAKER_TRITON_MODEL_LOAD_GPU_LIMIT" ]; then num_gpus=$(nvidia-smi -L | wc -l) for ((i=0; i<${num_gpus}; i++)); do SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --model-load-gpu-limit ${i}:${SAGEMAKER_TRITON_MODEL_LOAD_GPU_LIMIT}" done fi if [ -n "$SAGEMAKER_TRITON_ADDITIONAL_ARGS" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} ${SAGEMAKER_TRITON_ADDITIONAL_ARGS}" fi if [ "${is_mme_mode}" = false ] && [ -f "${SAGEMAKER_MODEL_REPO}/config.pbtxt" ]; then echo "ERROR: Incorrect directory structure." echo " Model directory needs to contain the top level folder" exit 1 fi if [ "${is_mme_mode}" = false ] && [ -n "$SAGEMAKER_TRITON_DEFAULT_MODEL_NAME" ]; then if [ -d "${SAGEMAKER_MODEL_REPO}/$SAGEMAKER_TRITON_DEFAULT_MODEL_NAME" ]; then SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --load-model=${SAGEMAKER_TRITON_DEFAULT_MODEL_NAME}" else echo "ERROR: Directory with provided SAGEMAKER_TRITON_DEFAULT_MODEL_NAME ${SAGEMAKER_TRITON_DEFAULT_MODEL_NAME} does not exist" exit 1 fi elif [ "${is_mme_mode}" = false ]; then MODEL_DIRS=(`find "${SAGEMAKER_MODEL_REPO}" -mindepth 1 -maxdepth 1 -type d -printf "%f\n"`) case ${#MODEL_DIRS[@]} in 0) echo "ERROR: No model found in model repository"; exit 1 ;; 1) echo "WARNING: No SAGEMAKER_TRITON_DEFAULT_MODEL_NAME provided." echo " Starting with the only existing model directory ${MODEL_DIRS[0]}"; export SAGEMAKER_TRITON_DEFAULT_MODEL_NAME=${MODEL_DIRS[0]} ;; *) echo "ERROR: More than 1 model directory found in model repository." echo " Either provide a single directory or set SAGEMAKER_TRITON_DEFAULT_MODEL_NAME to run the ensemble backend." echo " Directories found in model repository: ${MODEL_DIRS[@]}"; exit 1 ;; esac SAGEMAKER_ARGS="${SAGEMAKER_ARGS} --load-model=${SAGEMAKER_TRITON_DEFAULT_MODEL_NAME}" fi tritonserver --allow-sagemaker=true --allow-http=false --model-control-mode=explicit $SAGEMAKER_ARGS ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/kytensor.service�����������������������������������������������������������0000664�0000000�0000000�00000000531�15270523712�0020402�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������[Unit] Description=Kylin triton inference server [Service] ExecStart=/usr/bin/kytensor --model-repository=/usr/share/kylin-ai/model-repository --model-repository=/opt/appdata/kylin-ai/model-repository --model-control-mode=explicit --http-address=127.0.0.1 --grpc-address=127.0.0.1 Restart=always RestartSec=5 [Install] WantedBy=default.target �����������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/pyproject.toml�������������������������������������������������������������0000664�0000000�0000000�00000004367�15270523712�0020071�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [tool.codespell] # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - # this is only to allow you to run codespell interactively skip = "./.git,./.github" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" # ignore allowed words ignore-words-list = "passin" # use the 'clear' dictionary for unambiguous spelling mistakes builtin = "clear" # disable warnings about binary files and wrong encoding quiet-level = 3 [tool.isort] profile = "black" use_parentheses = true multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 ensure_newline_before_comments = true line_length = 88 balanced_wrapping = true indent = " " skip = ["build"] �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/server/tritonserver.desktop�������������������������������������������������������0000664�0000000�0000000�00000000711�15270523712�0021303�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������[Desktop Entry] Type=Application Name=TritonServer Exec=/bin/bash -c "source /usr/share/kylin-ai-python-env/python-env/bin/activate && tritonserver --model-repository=/opt/appdata/kylin-ai/model-repository --model-control-mode=explicit --http-address=127.0.0.1 --grpc-address=127.0.0.1 --log-file=$HOME/.log/tritonserver.log" Comment=Triton Server NoDisplay=true StartupNotify=false Terminal=false X-UKUI-AutoRestart=true X-UKUI-Autostart-Phase=Application �������������������������������������������������������kytensor-2.49.0.7/src/������������������������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0014424�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/CMakeLists.txt����������������������������������������������������������������0000664�0000000�0000000�00000047021�15270523712�0017170�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) project(tritonserverexe LANGUAGES C CXX) include(GNUInstallDirs) # # Dependencies # # We must include the transitive closure of all repos so that we can # override the tag. The backend repo is needed for the tests. # include(FetchContent) FetchContent_Declare( repo-common SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/common ) FetchContent_Declare( repo-core SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/core ) FetchContent_Declare( repo-backend SOURCE_DIR ${TRITON_MODULE_SOURCE_DIR}/backend ) if(TRITON_ENABLE_GRPC) set(TRITON_COMMON_ENABLE_PROTOBUF ON) set(TRITON_COMMON_ENABLE_GRPC ON) endif() # TRITON_ENABLE_GRPC FetchContent_MakeAvailable(repo-common repo-core repo-backend) find_package(PkgConfig REQUIRED) # CUDA # if(${TRITON_ENABLE_GPU}) find_package(CUDAToolkit REQUIRED) message(STATUS "Using CUDA ${CUDA_VERSION}") endif() # TRITON_ENABLE_GPU # libevent # if(${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) find_package(Libevent CONFIG REQUIRED) message(STATUS "Using libevent ${Libevent_VERSION}") endif() # OpenTelemetry # if (NOT WIN32 AND ${TRITON_ENABLE_TRACING}) find_package(absl CONFIG REQUIRED) find_package(CURL CONFIG REQUIRED) find_package(nlohmann_json CONFIG REQUIRED) find_package(opentelemetry-cpp CONFIG REQUIRED) message(STATUS "Using opentelemetry-cpp ${opentelemetry-cpp_VERSION}") endif() # re2 # pkg_check_modules(Re2 REQUIRED re2) message(STATUS "Using re2 ${Re2_VERSION}") # boost #set(BOOST_ROOT ${Boost_DIR}) #set(Boost_NO_SYSTEM_PATHS ON) #find_package(Boost REQUIRED) #include_directories(${Boost_INCLUDE_DIRS}) include_directories(${Protobuf_INCLUDE_DIRS}) # # tritonserver executable # add_executable( main classification.cc command_line_parser.cc common.cc main.cc shared_memory_manager.cc triton_signal.cc classification.h common.h shared_memory_manager.h triton_signal.h ) # On windows a *.lib file can be generated for a exe. When creating # tritonserver.exe if we try to create tritonserver.lib it will fail # because there is already a trtionserver.lib for tritonserver.dll, # this causes the build to fail. To avoid we keep the build name as # main.exe and then for windows after installing we rename it to # tritonserver.exe (below in the install steps). if (NOT WIN32) set_property(TARGET main PROPERTY OUTPUT_NAME kytensor) endif() target_compile_features(main PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message("Using MSVC as compiler, default target on Windows 10. " "If the target system is not Windows 10, please update _WIN32_WINNT " "to corresponding value.") target_compile_options( main PRIVATE /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor ) target_compile_definitions(main PRIVATE NOMINMAX) else() target_compile_options( main PRIVATE -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wno-maybe-uninitialized -Werror ) endif() set(LIB_DIR "lib") if(LINUX) file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE") if(${DISTRO_ID_LIKE} MATCHES "rhel|centos") set (LIB_DIR "lib64") endif(${DISTRO_ID_LIKE} MATCHES "rhel|centos") endif(LINUX) set(TRITON_CORE_HEADERS_ONLY OFF) set_target_properties( main PROPERTIES POSITION_INDEPENDENT_CODE ON SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH FALSE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "$\{ORIGIN\}/../${LIB_DIR}" ) target_link_libraries( main PRIVATE triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-common-logging # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) if(${TRITON_ENABLE_ASAN}) set(CMAKE_BUILD_TYPE Debug) target_compile_definitions( main PRIVATE TRITON_ENABLE_ASAN=1 ) set(_ASAN_FLAGS "-static-libstdc++ -static-libasan -fno-omit-frame-pointer -fsanitize=address") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${_ASAN_FLAGS}") set(CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG} ${_ASAN_FLAGS}") endif() # TRITON_ENABLE_ASAN if(${TRITON_ENABLE_GPU}) target_compile_definitions( main PRIVATE TRITON_ENABLE_GPU=1 PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) target_link_libraries( main PRIVATE CUDA::cudart ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) target_include_directories( main PRIVATE ${LIBEVENT_INCLUDE_DIRS} ) endif() if(${TRITON_ENABLE_HTTP}) target_compile_definitions( main PRIVATE TRITON_ENABLE_HTTP=1 ) endif() # TRITON_ENABLE_HTTP if(${TRITON_ENABLE_SAGEMAKER}) target_compile_definitions( main PRIVATE TRITON_ENABLE_SAGEMAKER=1 ) endif() # TRITON_ENABLE_SAGEMAKER if(${TRITON_ENABLE_VERTEX_AI}) target_compile_definitions( main PRIVATE TRITON_ENABLE_VERTEX_AI=1 ) endif() # TRITON_ENABLE_VERTEX_AI if(${TRITON_ENABLE_LOGGING}) target_compile_definitions( main PRIVATE TRITON_ENABLE_LOGGING=1 ) endif() # TRITON_ENABLE_LOGGING if(${TRITON_ENABLE_METRICS}) target_compile_definitions( main PRIVATE TRITON_ENABLE_METRICS=1 ) endif() # TRITON_ENABLE_METRICS if(${TRITON_ENABLE_STATS}) target_compile_definitions( main PRIVATE TRITON_ENABLE_STATS=1 ) endif() # TRITON_ENABLE_STATS if(${TRITON_ENABLE_TRACING}) target_compile_definitions( main PRIVATE TRITON_ENABLE_TRACING=1 ) # FIXME: remove, when Windows support is added for Opentelemetry if (NOT WIN32) target_include_directories( main PRIVATE ${OPENTELEMETRY_CPP_INCLUDE_DIRS} ) endif() endif() # TRITON_ENABLE_TRACING if(${TRITON_ENABLE_NVTX}) target_compile_definitions( main PRIVATE TRITON_ENABLE_NVTX=1 ) endif() # TRITON_ENABLE_NVTX if (NOT WIN32) target_link_libraries( main PRIVATE rt dl ) endif() # NOT WIN32 if (NOT WIN32) install( TARGETS main RUNTIME DESTINATION bin ) else() # See explanation above as to why we need to rename main.exe to # tritonserver.exe as part of the install process on windows. install( PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/main.exe DESTINATION bin RENAME tritonserver.exe ) endif() if(${TRITON_ENABLE_GRPC}) # # GRPC # find_package(gRPC CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") add_subdirectory(grpc) target_link_libraries( main PRIVATE grpc-endpoint-library ) target_include_directories( main PRIVATE $<TARGET_PROPERTY:gRPC::grpc,INTERFACE_INCLUDE_DIRECTORIES> ) target_compile_definitions( main PRIVATE TRITON_ENABLE_GRPC=1 ) endif() # http endpoint # if(${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) find_package(libevhtp CONFIG REQUIRED) message(STATUS "Using libevhtp ${libevhtp_VERSION}") list(APPEND HTTP_ENDPOINT_SRCS http_server.cc ) list(APPEND HTTP_ENDPOINT_HDRS http_server.h ) # Add header / src files based on HTTP related endpoint requested if(${TRITON_ENABLE_SAGEMAKER}) list(APPEND HTTP_ENDPOINT_SRCS sagemaker_server.cc ) list(APPEND HTTP_ENDPOINT_HDRS sagemaker_server.h ) endif() # TRITON_ENABLE_SAGEMAKER if(${TRITON_ENABLE_VERTEX_AI}) list(APPEND HTTP_ENDPOINT_SRCS vertex_ai_server.cc ) list(APPEND HTTP_ENDPOINT_HDRS vertex_ai_server.h ) endif() # TRITON_ENABLE_VERTEX_AI add_library( http-endpoint-library EXCLUDE_FROM_ALL ${HTTP_ENDPOINT_SRCS} ${HTTP_ENDPOINT_HDRS} ) target_compile_features(http-endpoint-library PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") target_compile_options( http-endpoint-library PRIVATE /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor ) else() target_compile_options( http-endpoint-library PRIVATE -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wno-maybe-uninitialized -Werror ) endif() set_target_properties( http-endpoint-library PROPERTIES POSITION_INDEPENDENT_CODE ON ) target_link_libraries( http-endpoint-library PUBLIC triton-common-json # from repo-common triton-common-logging # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ${LIBEVENT_LIBRARIES} libevhtp::evhtp libre2.a libbsd.a ) target_include_directories( http-endpoint-library PRIVATE $<TARGET_PROPERTY:libevhtp::evhtp,INTERFACE_INCLUDE_DIRECTORIES> ) # FIXME when Triton support of Opentelemetry is available on Windows # add ${OPENTELEMETRY_CPP_INCLUDE_DIRS} to above target_include_directories # JIRA DLIS-4786 if (NOT WIN32 AND ${TRITON_ENABLE_TRACING}) target_include_directories( http-endpoint-library PRIVATE ${OPENTELEMETRY_CPP_INCLUDE_DIRS} ) endif() if(${TRITON_ENABLE_GPU}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_GPU=1 PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) target_link_libraries( http-endpoint-library PUBLIC CUDA::cudart ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_HTTP}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_HTTP=1 ) endif() # TRITON_ENABLE_HTTP if(${TRITON_ENABLE_SAGEMAKER}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_SAGEMAKER=1 ) endif() # TRITON_ENABLE_SAGEMAKER if(${TRITON_ENABLE_VERTEX_AI}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_VERTEX_AI=1 ) endif() # TRITON_ENABLE_VERTEX_AI if(${TRITON_ENABLE_METRICS}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_METRICS=1 ) endif() # TRITON_ENABLE_METRICS if(${TRITON_ENABLE_LOGGING}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_LOGGING=1 ) endif() # TRITON_ENABLE_LOGGING if(${TRITON_ENABLE_STATS}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_STATS=1 ) endif() # TRITON_ENABLE_STATS if(${TRITON_ENABLE_TRACING}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_TRACING=1 ) endif() # TRITON_ENABLE_TRACING if(${TRITON_ENABLE_NVTX}) target_compile_definitions( http-endpoint-library PRIVATE TRITON_ENABLE_NVTX=1 ) endif() # TRITON_ENABLE_NVTX if (WIN32) find_library(B64_LIBRARY NAMES b64) find_library(ZLIB_LIBRARY NAMES zlib) target_link_libraries( http-endpoint-library PUBLIC ${B64_LIBRARY} ${ZLIB_LIBRARY} ) else() target_link_libraries( http-endpoint-library PUBLIC b64 z ) endif() target_link_libraries( main PRIVATE http-endpoint-library ) endif() # TRITON_ENABLE_HTTP || TRITON_ENABLE_METRICS || # TRITON_ENABLE_SAGEMAKER || TRITON_ENABLE_VERTEX_AI # tracing # if(${TRITON_ENABLE_TRACING}) message(STATUS "Using tracing ${TRITON_TRACE_INSTALL_PATH}") add_library( tracing-library EXCLUDE_FROM_ALL tracer.cc tracer.h ) target_compile_features(tracing-library PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if (NOT WIN32) target_include_directories( tracing-library PRIVATE ${OPENTELEMETRY_CPP_INCLUDE_DIRS} ) target_link_libraries( tracing-library PRIVATE ${OPENTELEMETRY_CPP_LIBRARIES}) endif() target_link_libraries( tracing-library PUBLIC triton-common-logging # from repo-common triton-common-json # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) target_compile_definitions( tracing-library PRIVATE TRITON_ENABLE_TRACING=1 ) if(${TRITON_ENABLE_GPU}) target_compile_definitions( tracing-library PRIVATE TRITON_ENABLE_GPU=1 PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) target_link_libraries( tracing-library PUBLIC CUDA::cudart ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_METRICS}) target_compile_definitions( tracing-library PRIVATE TRITON_ENABLE_METRICS=1 ) endif() # TRITON_ENABLE_METRICS if(${TRITON_ENABLE_LOGGING}) target_compile_definitions( tracing-library PRIVATE TRITON_ENABLE_LOGGING=1 ) endif() # TRITON_ENABLE_LOGGING if(${TRITON_ENABLE_STATS}) target_compile_definitions( tracing-library PRIVATE TRITON_ENABLE_STATS=1 ) endif() # TRITON_ENABLE_STATS if(${TRITON_ENABLE_NVTX}) target_compile_definitions( tracing-library PRIVATE TRITON_ENABLE_NVTX=1 ) endif() # TRITON_ENABLE_NVTX target_link_libraries( main PRIVATE tracing-library ) endif() # TRITON_ENABLE_TRACING if (NOT WIN32) # # simple # add_executable( simple simple.cc ) target_compile_features(simple PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message("Using MSVC as compiler, default target on Windows 10. " "If the target system is not Windows 10, please update _WIN32_WINNT " "to corresponding value.") target_compile_options( simple PRIVATE /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor ) else() target_compile_options( simple PRIVATE -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror ) endif() set_target_properties( simple PROPERTIES POSITION_INDEPENDENT_CODE ON SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_link_libraries( simple PRIVATE triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) if(${TRITON_ENABLE_GPU}) target_compile_definitions( simple PRIVATE TRITON_ENABLE_GPU=1 PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) target_link_libraries( simple PRIVATE CUDA::cudart ) endif() # TRITON_ENABLE_GPU install( TARGETS simple RUNTIME DESTINATION bin ) # # multi_server example # add_executable( multi_server multi_server.cc ) target_compile_features(multi_server PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message("Using MSVC as compiler, default target on Windows 10. " "If the target system is not Windows 10, please update _WIN32_WINNT " "to corresponding value.") target_compile_options( multi_server PRIVATE /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor ) else() target_compile_options( multi_server PRIVATE -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror ) endif() set_target_properties( multi_server PROPERTIES POSITION_INDEPENDENT_CODE ON SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_link_libraries( multi_server PRIVATE triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) if(${TRITON_ENABLE_GPU}) target_compile_definitions( multi_server PRIVATE TRITON_ENABLE_GPU=1 PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) target_link_libraries( multi_server PRIVATE CUDA::cudart ) endif() # TRITON_ENABLE_GPU install( TARGETS multi_server RUNTIME DESTINATION bin ) if(${TRITON_ENABLE_GPU}) # # memory_alloc example # add_executable( memory_alloc memory_alloc.cc ) target_compile_features(memory_alloc PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") message("Using MSVC as compiler, default target on Windows 10. " "If the target system is not Windows 10, please update _WIN32_WINNT " "to corresponding value.") target_compile_options( memory_alloc PRIVATE /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor ) else() target_compile_options( memory_alloc PRIVATE -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror ) endif() set_target_properties( memory_alloc PROPERTIES POSITION_INDEPENDENT_CODE ON SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_compile_definitions( memory_alloc PRIVATE TRITON_ENABLE_GPU=1 PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) target_link_libraries( memory_alloc PRIVATE triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core CUDA::cudart ) install( TARGETS memory_alloc RUNTIME DESTINATION bin ) endif() # TRITON_ENABLE_GPU endif() # NOT WIN32 # Currently unit tests do not build for windows... if ( NOT WIN32) # add_subdirectory(test test) endif() # NOT WIN32 ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/classification.cc�������������������������������������������������������������0000664�0000000�0000000�00000011672�15270523712�0017735�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "classification.h" #include <algorithm> #include <numeric> #include "common.h" namespace triton { namespace server { namespace { template <typename T> TRITONSERVER_Error* AddClassResults( TRITONSERVER_InferenceResponse* response, const uint32_t output_idx, const char* base, const size_t element_cnt, const uint32_t req_class_cnt, std::vector<std::string>* class_strs) { const T* probs = reinterpret_cast<const T*>(base); std::vector<size_t> idx(element_cnt); iota(idx.begin(), idx.end(), 0); sort(idx.begin(), idx.end(), [&probs](size_t i1, size_t i2) { return probs[i1] > probs[i2]; }); const size_t class_cnt = std::min(element_cnt, (size_t)req_class_cnt); for (size_t k = 0; k < class_cnt; ++k) { class_strs->push_back( std::to_string(probs[idx[k]]) + ":" + std::to_string(idx[k])); const char* label; RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutputClassificationLabel( response, output_idx, idx[k], &label)); if (label != nullptr) { class_strs->back() += ":"; class_strs->back().append(label); } } return nullptr; // success } } // namespace TRITONSERVER_Error* TopkClassifications( TRITONSERVER_InferenceResponse* response, const uint32_t output_idx, const char* base, const size_t byte_size, const TRITONSERVER_DataType datatype, const uint32_t req_class_count, std::vector<std::string>* class_strs) { const size_t element_cnt = byte_size / TRITONSERVER_DataTypeByteSize(datatype); switch (datatype) { case TRITONSERVER_TYPE_UINT8: return AddClassResults<uint8_t>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_UINT16: return AddClassResults<uint16_t>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_UINT32: return AddClassResults<uint32_t>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_UINT64: return AddClassResults<uint64_t>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_INT8: return AddClassResults<int8_t>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_INT16: return AddClassResults<int16_t>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_INT32: return AddClassResults<int32_t>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_INT64: return AddClassResults<int64_t>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_FP32: return AddClassResults<float>( response, output_idx, base, element_cnt, req_class_count, class_strs); case TRITONSERVER_TYPE_FP64: return AddClassResults<double>( response, output_idx, base, element_cnt, req_class_count, class_strs); default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( std::string("class result not available for output due to " "unsupported type '") + std::string(TRITONSERVER_DataTypeString(datatype)) + "'") .c_str()); } return nullptr; // success } }} // namespace triton::server ����������������������������������������������������������������������kytensor-2.49.0.7/src/classification.h��������������������������������������������������������������0000664�0000000�0000000�00000003715�15270523712�0017576�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <string> #include <vector> #include "triton/core/tritonserver.h" namespace triton { namespace server { TRITONSERVER_Error* TopkClassifications( TRITONSERVER_InferenceResponse* response, const uint32_t output_idx, const char* base, const size_t byte_size, const TRITONSERVER_DataType datatype, const uint32_t req_class_count, std::vector<std::string>* class_strs); }} // namespace triton::server ���������������������������������������������������kytensor-2.49.0.7/src/command_line_parser.cc��������������������������������������������������������0000664�0000000�0000000�00000271056�15270523712�0020747�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #include "command_line_parser.h" constexpr const char* GLOBAL_OPTION_GROUP = ""; #ifdef _WIN32 int optind = 1; const char* optarg = nullptr; /// Implementation of `getopt_long` for Windows. /// Linux uses available implementation: /// https://github.com/gcc-mirror/gcc/blob/fab08d12b40ad637c5a4ce8e026fb43cd3f0fad1/include/getopt.h /// and /// https://github.com/gcc-mirror/gcc/blob/fab08d12b40ad637c5a4ce8e026fb43cd3f0fad1/libiberty/getopt.c#L521 /// Parameters' description is available here: /// https://github.com/gcc-mirror/gcc/blob/fab08d12b40ad637c5a4ce8e026fb43cd3f0fad1/libiberty/getopt.c#L464-L518 /// `optind' is an index to iterate over `argv`, (whose length is `argc`), /// and starts from 1, since argv[0] is the program name. /// Text in the current `argv`-element is returned in `optarg'. /// Note: if option was provided in the form of --<key>=<value>, then /// optarg is (argv[optind] + found + 1), i.e. everything after `=`. /// Alternatively, option can be provided as --<key> <value>. /// In this case, <value> is storred as a separate parameter in `argv`. /// `longind` returns the index in `longopts` of the long-named option found. int getopt_long( int argc, char* const argv[], const char* optstring, const struct option* longopts, int* longind) { if (optind >= argc) { return -1; } const struct option* curr_longopt = longopts; std::string argv_str = argv[optind]; size_t found = argv_str.find_first_of("="); std::string key = argv_str.substr( 2, (found == std::string::npos) ? std::string::npos : (found - 2)); int option_index = 0; for (curr_longopt, option_index; curr_longopt->name; curr_longopt++, option_index++) { if (key == curr_longopt->name) { if (longind != NULL) (*longind) = option_index; if (curr_longopt->has_arg == required_argument) { if (found == std::string::npos) { optind++; if (optind >= argc) { std::cerr << argv[0] << ": option '" << argv_str << "' requires an argument" << std::endl; return '?'; } optarg = argv[optind]; } else { optarg = (argv[optind] + found + 1); } } optind++; return curr_longopt->val; } } return -1; } #endif #include <algorithm> #include <iomanip> #include <iostream> #include <string> #include "common.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr #include "triton/common/triton_json.h" namespace triton { namespace server { // [FIXME] expose following parse helpers for other type of parser namespace { // A wrapper around std::stoi, std::stoull, std::stoll, std::stod // to catch `invalid argument` and `out of range` exceptions template <typename T> T StringTo(const std::string& arg); template <> int StringTo(const std::string& arg) { return std::stoi(arg); } #ifdef TRITON_ENABLE_TRACING template <> uint32_t StringTo(const std::string& arg) { return std::stoul(arg); } #endif // TRITON_ENABLE_TRACING template <> uint64_t StringTo(const std::string& arg) { return std::stoull(arg); } template <> int64_t StringTo(const std::string& arg) { return std::stoll(arg); } template <> double StringTo(const std::string& arg) { return std::stod(arg); } // There must be specialization for the types to be parsed into so that // the argument is properly validated and parsed. Attempted to use input // operator (>>) but it will consume improper argument without error // (i.e. parse "1.4" to 'int' will return 1 but we want to report error). template <typename T> T ParseOption(const std::string& arg) { try { return StringTo<T>(arg); } catch (const std::invalid_argument& ia) { std::stringstream ss; ss << "Invalid option value. Got " << arg << std::endl; throw ParseException(ss.str()); } catch (const std::out_of_range& oor) { std::stringstream ss; ss << "Provided option value is out of bound. Got " << arg << std::endl; throw ParseException(ss.str()); } } template <> bool ParseOption(const std::string& arg) { // 'arg' need to comply with template declaration std::string larg = arg; std::transform(larg.begin(), larg.end(), larg.begin(), [](unsigned char c) { return std::tolower(c); }); if ((larg == "true") || (larg == "on") || (larg == "1")) { return true; } if ((larg == "false") || (larg == "off") || (larg == "0")) { return false; } throw ParseException("invalid value for bool option: " + arg); } // Condition here merely to avoid compilation error, this function will // be defined but not used otherwise. #ifdef TRITON_ENABLE_LOGGING int ParseIntBoolOption(std::string arg) { std::transform(arg.begin(), arg.end(), arg.begin(), [](unsigned char c) { return std::tolower(c); }); if (arg == "true") { return 1; } if (arg == "false") { return 0; } return ParseOption<int>(arg); } #endif // TRITON_ENABLE_LOGGING std::string PairsToJsonStr(std::vector<std::pair<std::string, std::string>> settings) { triton::common::TritonJson::Value json( triton::common::TritonJson::ValueType::OBJECT); for (const auto& setting : settings) { const auto& key = setting.first; const auto& value = setting.second; json.SetStringObject(key.c_str(), value); } triton::common::TritonJson::WriteBuffer buffer; auto err = json.Write(&buffer); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to convert config to JSON"); } return buffer.Contents(); } template <typename T1, typename T2> std::pair<T1, T2> ParsePairOption(const std::string& arg, const std::string& delim_str) { int delim = arg.find(delim_str); if ((delim < 0)) { std::stringstream ss; ss << "Cannot parse pair option due to incorrect number of inputs." "--<pair option> argument requires format <first>" << delim_str << "<second>. " << "Found: " << arg << std::endl; throw ParseException(ss.str()); } std::string first_string = arg.substr(0, delim); std::string second_string = arg.substr(delim + delim_str.length()); // Specific conversion from key-value string to actual key-value type, // should be extracted out of this function if we need to parse // more pair option of different types. return {ParseOption<T1>(first_string), ParseOption<T2>(second_string)}; } // Split 'options' by 'delim_str' and place split strings into a vector std::vector<std::string> SplitOptions(std::string options, const std::string& delim_str) { std::vector<std::string> res; int delim = options.find(delim_str); while ((delim >= 0)) { res.emplace_back(options.substr(0, delim)); options = options.substr(delim + delim_str.length()); delim = options.find(delim_str); } // include last element res.emplace_back(options); return res; } } // namespace enum TritonOptionId { OPTION_HELP = 1000, #ifdef TRITON_ENABLE_LOGGING OPTION_LOG_VERBOSE, OPTION_LOG_INFO, OPTION_LOG_WARNING, OPTION_LOG_ERROR, OPTION_LOG_FORMAT, OPTION_LOG_FILE, #endif // TRITON_ENABLE_LOGGING OPTION_ID, OPTION_MODEL_REPOSITORY, OPTION_EXIT_ON_ERROR, OPTION_DISABLE_AUTO_COMPLETE_CONFIG, OPTION_STRICT_MODEL_CONFIG, OPTION_STRICT_READINESS, #if defined(TRITON_ENABLE_HTTP) OPTION_ALLOW_HTTP, OPTION_HTTP_HEADER_FORWARD_PATTERN, OPTION_HTTP_PORT, OPTION_REUSE_HTTP_PORT, OPTION_HTTP_ADDRESS, OPTION_HTTP_THREAD_COUNT, OPTION_HTTP_RESTRICTED_API, #endif // TRITON_ENABLE_HTTP #if defined(TRITON_ENABLE_GRPC) OPTION_ALLOW_GRPC, OPTION_GRPC_PORT, OPTION_REUSE_GRPC_PORT, OPTION_GRPC_ADDRESS, OPTION_GRPC_HEADER_FORWARD_PATTERN, OPTION_GRPC_INFER_ALLOCATION_POOL_SIZE, OPTION_GRPC_USE_SSL, OPTION_GRPC_USE_SSL_MUTUAL, OPTION_GRPC_SERVER_CERT, OPTION_GRPC_SERVER_KEY, OPTION_GRPC_ROOT_CERT, OPTION_GRPC_RESPONSE_COMPRESSION_LEVEL, OPTION_GRPC_ARG_KEEPALIVE_TIME_MS, OPTION_GRPC_ARG_KEEPALIVE_TIMEOUT_MS, OPTION_GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, OPTION_GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, OPTION_GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, OPTION_GRPC_ARG_HTTP2_MAX_PING_STRIKES, OPTION_GRPC_RESTRICTED_PROTOCOL, OPTION_GRPC_ARG_MAX_CONNECTION_AGE_MS, OPTION_GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS, #endif // TRITON_ENABLE_GRPC #if defined(TRITON_ENABLE_SAGEMAKER) OPTION_ALLOW_SAGEMAKER, OPTION_SAGEMAKER_PORT, OPTION_SAGEMAKER_SAFE_PORT_RANGE, OPTION_SAGEMAKER_THREAD_COUNT, #endif // TRITON_ENABLE_SAGEMAKER #if defined(TRITON_ENABLE_VERTEX_AI) OPTION_ALLOW_VERTEX_AI, OPTION_VERTEX_AI_PORT, OPTION_VERTEX_AI_THREAD_COUNT, OPTION_VERTEX_AI_DEFAULT_MODEL, #endif // TRITON_ENABLE_VERTEX_AI #ifdef TRITON_ENABLE_METRICS OPTION_ALLOW_METRICS, OPTION_ALLOW_GPU_METRICS, OPTION_ALLOW_CPU_METRICS, OPTION_METRICS_ADDRESS, OPTION_METRICS_PORT, OPTION_METRICS_INTERVAL_MS, OPTION_METRICS_CONFIG, #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_TRACING OPTION_TRACE_FILEPATH, OPTION_TRACE_LEVEL, OPTION_TRACE_RATE, OPTION_TRACE_COUNT, OPTION_TRACE_LOG_FREQUENCY, OPTION_TRACE_CONFIG, #endif // TRITON_ENABLE_TRACING OPTION_MODEL_CONTROL_MODE, OPTION_POLL_REPO_SECS, OPTION_STARTUP_MODEL, OPTION_CUSTOM_MODEL_CONFIG_NAME, OPTION_RATE_LIMIT, OPTION_RATE_LIMIT_RESOURCE, OPTION_PINNED_MEMORY_POOL_BYTE_SIZE, OPTION_CUDA_MEMORY_POOL_BYTE_SIZE, OPTION_CUDA_VIRTUAL_ADDRESS_SIZE, OPTION_RESPONSE_CACHE_BYTE_SIZE, OPTION_CACHE_CONFIG, OPTION_CACHE_DIR, OPTION_MIN_SUPPORTED_COMPUTE_CAPABILITY, OPTION_EXIT_TIMEOUT_SECS, OPTION_BACKEND_DIR, OPTION_REPOAGENT_DIR, OPTION_BUFFER_MANAGER_THREAD_COUNT, OPTION_MODEL_LOAD_THREAD_COUNT, OPTION_MODEL_LOAD_RETRY_COUNT, OPTION_BACKEND_CONFIG, OPTION_HOST_POLICY, OPTION_MODEL_LOAD_GPU_LIMIT, OPTION_MODEL_NAMESPACING, OPTION_ENABLE_PEER_ACCESS }; void TritonParser::SetupOptions() { global_options_.push_back( {OPTION_HELP, "help", Option::ArgNone, "Print usage"}); server_options_.push_back( {OPTION_ID, "id", Option::ArgStr, "Identifier for this server."}); server_options_.push_back( {OPTION_EXIT_TIMEOUT_SECS, "exit-timeout-secs", Option::ArgInt, "Timeout (in seconds) when exiting to wait for in-flight inferences to " "finish. After the timeout expires the server exits even if inferences " "are still in flight."}); model_repo_options_.push_back( {OPTION_MODEL_REPOSITORY, "model-store", Option::ArgStr, "Equivalent to --model-repository."}); model_repo_options_.push_back( {OPTION_MODEL_REPOSITORY, "model-repository", Option::ArgStr, "Path to model repository directory. It may be specified multiple times " "to add multiple model repositories. Note that if a model is not unique " "across all model repositories at any time, the model will not be " "available."}); model_repo_options_.push_back( {OPTION_EXIT_ON_ERROR, "exit-on-error", Option::ArgBool, "Exit the inference server if an error occurs during initialization."}); model_repo_options_.push_back( {OPTION_DISABLE_AUTO_COMPLETE_CONFIG, "disable-auto-complete-config", Option::ArgNone, "If set, disables the triton and backends from auto completing model " "configuration files. Model configuration files must be provided and " "all required " "configuration settings must be specified."}); model_repo_options_.push_back( {OPTION_STRICT_READINESS, "strict-readiness", Option::ArgBool, "If true /v2/health/ready endpoint indicates ready if the server " "is responsive and all models are available. If false " "/v2/health/ready endpoint indicates ready if server is responsive " "even if some/all models are unavailable."}); model_repo_options_.push_back( {OPTION_MODEL_CONTROL_MODE, "model-control-mode", Option::ArgStr, "Specify the mode for model management. Options are \"none\", \"poll\" " "and \"explicit\". The default is \"none\". " "For \"none\", the server will load all models in the model " "repository(s) at startup and will not make any changes to the load " "models after that. For \"poll\", the server will poll the model " "repository(s) to detect changes and will load/unload models based on " "those changes. The poll rate is controlled by 'repository-poll-secs'. " "For \"explicit\", model load and unload is initiated by using the " "model control APIs, and only models specified with --load-model will " "be loaded at startup."}); model_repo_options_.push_back( {OPTION_POLL_REPO_SECS, "repository-poll-secs", Option::ArgInt, "Interval in seconds between each poll of the model repository to check " "for changes. Valid only when --model-control-mode=poll is " "specified."}); model_repo_options_.push_back( {OPTION_STARTUP_MODEL, "load-model", Option::ArgStr, "Name of the model to be loaded on server startup. It may be specified " "multiple times to add multiple models. To load ALL models at startup, " "specify '*' as the model name with --load-model=* as the ONLY " "--load-model argument, this does not imply any pattern matching. " "Specifying --load-model=* in conjunction with another --load-model " "argument will result in error. Note that this option will only take " "effect if --model-control-mode=explicit is true."}); model_repo_options_.push_back( {OPTION_CUSTOM_MODEL_CONFIG_NAME, "model-config-name", Option::ArgStr, "The custom configuration name for models to load." "The name should not contain any space character." "For example: --model-config-name=h100. " "If --model-config-name is not set, Triton will use the default " "config.pbtxt."}); model_repo_options_.push_back( {OPTION_MODEL_LOAD_THREAD_COUNT, "model-load-thread-count", Option::ArgInt, "The number of threads used to concurrently load models in " "model repositories. Default is 4."}); model_repo_options_.push_back( {OPTION_MODEL_LOAD_RETRY_COUNT, "model-load-retry-count", Option::ArgInt, "The number of retry to load a model in " "model repositories. Default is 0."}); model_repo_options_.push_back( {OPTION_MODEL_NAMESPACING, "model-namespacing", Option::ArgBool, "Whether model namespacing is enable or not. If true, models with the " "same name can be served if they are in different namespace."}); model_repo_options_.push_back( {OPTION_ENABLE_PEER_ACCESS, "enable-peer-access", Option::ArgBool, "Whether the server tries to enable peer access or not. Even when this " "options is set to true, " "peer access could still be not enabled because the underlying system " "doesn't support it." " The server will log a warning in this case. Default is true."}); #if defined(TRITON_ENABLE_HTTP) http_options_.push_back( {OPTION_ALLOW_HTTP, "allow-http", Option::ArgBool, "Allow the server to listen for HTTP requests."}); http_options_.push_back( {OPTION_HTTP_ADDRESS, "http-address", Option::ArgStr, "The address for the http server to bind to. Default is 0.0.0.0"}); http_options_.push_back( {OPTION_HTTP_PORT, "http-port", Option::ArgInt, "The port for the server to listen on for HTTP " "requests. Default is 8000."}); http_options_.push_back( {OPTION_REUSE_HTTP_PORT, "reuse-http-port", Option::ArgBool, "Allow multiple servers to listen on the same HTTP port when every " "server has this option set. If you plan to use this option as a way to " "load balance between different Triton servers, the same model " "repository or set of models must be used for every server."}); http_options_.push_back( {OPTION_HTTP_HEADER_FORWARD_PATTERN, "http-header-forward-pattern", Option::ArgStr, "The regular expression pattern that will be used for forwarding HTTP " "headers as inference request parameters."}); http_options_.push_back( {OPTION_HTTP_THREAD_COUNT, "http-thread-count", Option::ArgInt, "Number of threads handling HTTP requests."}); http_options_.push_back( {OPTION_HTTP_RESTRICTED_API, "http-restricted-api", "<string>:<string>=<string>", "Specify restricted HTTP api setting. The format of this " "flag is --http-restricted-api=<apis>,<key>=<value>. Where " "<api> is a comma-separated list of apis to be restricted. " "<key> will be additional header key to be checked when a HTTP request " "is received, and <value> is the value expected to be matched." " Allowed APIs: " + Join(RESTRICTED_CATEGORY_NAMES, ", ")}); #endif // TRITON_ENABLE_HTTP #if defined(TRITON_ENABLE_GRPC) grpc_options_.push_back( {OPTION_ALLOW_GRPC, "allow-grpc", Option::ArgBool, "Allow the server to listen for GRPC requests."}); grpc_options_.push_back( {OPTION_GRPC_ADDRESS, "grpc-address", Option::ArgStr, "The address for the grpc server to binds to. Default is 0.0.0.0"}); grpc_options_.push_back( {OPTION_GRPC_PORT, "grpc-port", Option::ArgInt, "The port for the server to listen on for GRPC " "requests. Default is 8001."}); grpc_options_.push_back( {OPTION_REUSE_GRPC_PORT, "reuse-grpc-port", Option::ArgBool, "Allow multiple servers to listen on the same GRPC port when every " "server has this option set. If you plan to use this option as a way to " "load balance between different Triton servers, the same model " "repository or set of models must be used for every server."}); grpc_options_.push_back( {OPTION_GRPC_HEADER_FORWARD_PATTERN, "grpc-header-forward-pattern", Option::ArgStr, "The regular expression pattern that will be used for forwarding GRPC " "headers as inference request parameters."}); grpc_options_.push_back( {OPTION_GRPC_INFER_ALLOCATION_POOL_SIZE, "grpc-infer-allocation-pool-size", Option::ArgInt, "The maximum number of inference request/response objects that remain " "allocated for reuse. As long as the number of in-flight requests " "doesn't exceed this value there will be no allocation/deallocation of " "request/response objects."}); grpc_options_.push_back( {OPTION_GRPC_USE_SSL, "grpc-use-ssl", Option::ArgBool, "Use SSL authentication for GRPC requests. Default is false."}); grpc_options_.push_back( {OPTION_GRPC_USE_SSL_MUTUAL, "grpc-use-ssl-mutual", Option::ArgBool, "Use mututal SSL authentication for GRPC requests. This option will " "preempt '--grpc-use-ssl' if it is also specified. Default is false."}); grpc_options_.push_back( {OPTION_GRPC_SERVER_CERT, "grpc-server-cert", Option::ArgStr, "File holding PEM-encoded server certificate. Ignored unless " "--grpc-use-ssl is true."}); grpc_options_.push_back( {OPTION_GRPC_SERVER_KEY, "grpc-server-key", Option::ArgStr, "File holding PEM-encoded server key. Ignored unless " "--grpc-use-ssl is true."}); grpc_options_.push_back( {OPTION_GRPC_ROOT_CERT, "grpc-root-cert", Option::ArgStr, "File holding PEM-encoded root certificate. Ignore unless " "--grpc-use-ssl is false."}); grpc_options_.push_back( {OPTION_GRPC_RESPONSE_COMPRESSION_LEVEL, "grpc-infer-response-compression-level", Option::ArgStr, "The compression level to be used while returning the infer response to " "the peer. Allowed values are none, low, medium and high. By default, " "compression level is selected as none."}); grpc_options_.push_back( {OPTION_GRPC_ARG_KEEPALIVE_TIME_MS, "grpc-keepalive-time", Option::ArgInt, "The period (in milliseconds) after which a keepalive ping is sent on " "the transport. Default is 7200000 (2 hours)."}); grpc_options_.push_back( {OPTION_GRPC_ARG_KEEPALIVE_TIMEOUT_MS, "grpc-keepalive-timeout", Option::ArgInt, "The period (in milliseconds) the sender of the keepalive ping waits " "for an acknowledgement. If it does not receive an acknowledgment " "within this time, it will close the connection. " "Default is 20000 (20 seconds)."}); grpc_options_.push_back( {OPTION_GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, "grpc-keepalive-permit-without-calls", Option::ArgBool, "Allows keepalive pings to be sent even if there are no calls in flight " "(0 : false; 1 : true). Default is 0 (false)."}); grpc_options_.push_back( {OPTION_GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, "grpc-http2-max-pings-without-data", Option::ArgInt, "The maximum number of pings that can be sent when there is no " "data/header frame to be sent. gRPC Core will not continue sending " "pings if we run over the limit. Setting it to 0 allows sending pings " "without such a restriction. Default is 2."}); grpc_options_.push_back( {OPTION_GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, "grpc-http2-min-recv-ping-interval-without-data", Option::ArgInt, "If there are no data/header frames being sent on the transport, this " "channel argument on the server side controls the minimum time " "(in milliseconds) that gRPC Core would expect between receiving " "successive pings. If the time between successive pings is less than " "this time, then the ping will be considered a bad ping from the peer. " "Such a ping counts as a ‘ping strike’. Default is 300000 (5 " "minutes)."}); grpc_options_.push_back( {OPTION_GRPC_ARG_HTTP2_MAX_PING_STRIKES, "grpc-http2-max-ping-strikes", Option::ArgInt, "Maximum number of bad pings that the server will tolerate before " "sending an HTTP2 GOAWAY frame and closing the transport. Setting it to " "0 allows the server to accept any number of bad pings. Default is 2."}); grpc_options_.push_back( {OPTION_GRPC_ARG_MAX_CONNECTION_AGE_MS, "grpc-max-connection-age", Option::ArgInt, "Maximum time that a channel may exist in milliseconds." "Default is undefined."}); grpc_options_.push_back( {OPTION_GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS, "grpc-max-connection-age-grace", Option::ArgInt, "Grace period after the channel reaches its max age. " "Default is undefined."}); grpc_options_.push_back( {OPTION_GRPC_RESTRICTED_PROTOCOL, "grpc-restricted-protocol", "<string>:<string>=<string>", "Specify restricted GRPC protocol setting. The format of this " "flag is --grpc-restricted-protocol=<protocols>,<key>=<value>. Where " "<protocol> is a comma-separated list of protocols to be restricted. " "<key> will be additional header key to be checked when a GRPC request " "is received, and <value> is the value expected to be matched." " Allowed protocols: " + Join(RESTRICTED_CATEGORY_NAMES, ", ")}); #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_LOGGING logging_options_.push_back( {OPTION_LOG_VERBOSE, "log-verbose", Option::ArgInt, "Set verbose logging level. Zero (0) disables verbose logging and " "values >= 1 enable verbose logging."}); logging_options_.push_back( {OPTION_LOG_INFO, "log-info", Option::ArgBool, "Enable/disable info-level logging."}); logging_options_.push_back( {OPTION_LOG_WARNING, "log-warning", Option::ArgBool, "Enable/disable warning-level logging."}); logging_options_.push_back( {OPTION_LOG_ERROR, "log-error", Option::ArgBool, "Enable/disable error-level logging."}); logging_options_.push_back( {OPTION_LOG_FORMAT, "log-format", Option::ArgStr, "Set the logging format. Options are \"default\" and \"ISO8601\". " "The default is \"default\". For \"default\", the log severity (L) and " "timestamp will be logged as \"LMMDD hh:mm:ss.ssssss\". " "For \"ISO8601\", the log format will be \"YYYY-MM-DDThh:mm:ssZ L\"."}); logging_options_.push_back( {OPTION_LOG_FILE, "log-file", Option::ArgStr, "Set the name of the log output file. If specified, log outputs will be " "saved to this file. If not specified, log outputs will stream to the " "console."}); #endif // TRITON_ENABLE_LOGGING #if defined(TRITON_ENABLE_SAGEMAKER) sagemaker_options_.push_back( {OPTION_ALLOW_SAGEMAKER, "allow-sagemaker", Option::ArgBool, "Allow the server to listen for Sagemaker requests. Default is false."}); sagemaker_options_.push_back( {OPTION_SAGEMAKER_PORT, "sagemaker-port", Option::ArgInt, "The port for the server to listen on for Sagemaker requests. Default " "is 8080."}); sagemaker_options_.push_back( {OPTION_SAGEMAKER_SAFE_PORT_RANGE, "sagemaker-safe-port-range", "<integer>-<integer>", "Set the allowed port range for endpoints other than the SageMaker " "endpoints."}); sagemaker_options_.push_back( {OPTION_SAGEMAKER_THREAD_COUNT, "sagemaker-thread-count", Option::ArgInt, "Number of threads handling Sagemaker requests. Default is 8."}); #endif // TRITON_ENABLE_SAGEMAKER #if defined(TRITON_ENABLE_VERTEX_AI) vertex_options_.push_back( {OPTION_ALLOW_VERTEX_AI, "allow-vertex-ai", Option::ArgBool, "Allow the server to listen for Vertex AI requests. Default is true if " "AIP_MODE=PREDICTION, false otherwise."}); vertex_options_.push_back( {OPTION_VERTEX_AI_PORT, "vertex-ai-port", Option::ArgInt, "The port for the server to listen on for Vertex AI requests. Default " "is AIP_HTTP_PORT if set, 8080 otherwise."}); vertex_options_.push_back( {OPTION_VERTEX_AI_THREAD_COUNT, "vertex-ai-thread-count", Option::ArgInt, "Number of threads handling Vertex AI requests. Default is 8."}); vertex_options_.push_back( {OPTION_VERTEX_AI_DEFAULT_MODEL, "vertex-ai-default-model", Option::ArgStr, "The name of the model to use for single-model inference requests."}); #endif // TRITON_ENABLE_VERTEX_AI #if defined(TRITON_ENABLE_METRICS) metric_options_.push_back( {OPTION_ALLOW_METRICS, "allow-metrics", Option::ArgBool, "Allow the server to provide prometheus metrics."}); metric_options_.push_back( {OPTION_ALLOW_GPU_METRICS, "allow-gpu-metrics", Option::ArgBool, "Allow the server to provide GPU metrics. Ignored unless " "--allow-metrics is true."}); metric_options_.push_back( {OPTION_ALLOW_CPU_METRICS, "allow-cpu-metrics", Option::ArgBool, "Allow the server to provide CPU metrics. Ignored unless " "--allow-metrics is true."}); metric_options_.push_back( {OPTION_METRICS_ADDRESS, "metrics-address", Option::ArgStr, "The address for the metrics server to bind to. Default is the same as " "--http-address if built with HTTP support. Otherwise, default is " "0.0.0.0"}); metric_options_.push_back( {OPTION_METRICS_PORT, "metrics-port", Option::ArgInt, "The port reporting prometheus metrics. Default is 8002."}); metric_options_.push_back( {OPTION_METRICS_INTERVAL_MS, "metrics-interval-ms", Option::ArgFloat, "Metrics will be collected once every <metrics-interval-ms> " "milliseconds. Default is 2000 milliseconds."}); metric_options_.push_back( {OPTION_METRICS_CONFIG, "metrics-config", "<string>=<string>", "Specify a metrics-specific configuration setting. The format of this " "flag is --metrics-config=<setting>=<value>. It can be specified " "multiple times."}); #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_TRACING tracing_options_.push_back( {OPTION_TRACE_CONFIG, "trace-config", "<string>,<string>=<string>", "Specify global or trace mode specific configuration setting. " "The format of this flag is --trace-config " "<mode>,<setting>=<value>. " "Where <mode> is either \"triton\" or \"opentelemetry\". " "The default is \"triton\". To specify global trace settings " "(level, rate, count, or mode), the format would be " "--trace-config <setting>=<value>. For \"triton\" mode, the server will " "use " "Triton's Trace APIs. For \"opentelemetry\" mode, the server will use " "OpenTelemetry's APIs to generate, collect and export traces for " "individual inference requests."}); #endif // TRITON_ENABLE_TRACING cache_options_.push_back( {OPTION_CACHE_CONFIG, "cache-config", "<string>,<string>=<string>", "Specify a cache-specific configuration setting. The format of this " "flag is --cache-config=<cache_name>,<setting>=<value>. Where " "<cache_name> is the name of the cache, such as 'local' or 'redis'. " "Example: --cache-config=local,size=1048576 will configure a 'local' " "cache implementation with a fixed buffer pool of size 1048576 bytes."}); cache_options_.push_back( {OPTION_CACHE_DIR, "cache-directory", Option::ArgStr, "The global directory searched for cache shared libraries. Default is " "'/opt/tritonserver/caches'. This directory is expected to contain a " "cache implementation as a shared library with the name " "'libtritoncache.so'."}); rate_limiter_options_.push_back( // FIXME: fix the default to execution_count once RL logic is complete. {OPTION_RATE_LIMIT, "rate-limit", Option::ArgStr, "Specify the mode for rate limiting. Options are \"execution_count\" " "and \"off\". The default is \"off\". For " "\"execution_count\", the server will determine the instance using " "configured priority and the number of time the instance has been " "used to run inference. The inference will finally be executed once " "the required resources are available. For \"off\", the server will " "ignore any rate limiter config and run inference as soon as an " "instance is ready."}); rate_limiter_options_.push_back( {OPTION_RATE_LIMIT_RESOURCE, "rate-limit-resource", "<string>:<integer>:<integer>", "The number of resources available to the server. The format of this " "flag is --rate-limit-resource=<resource_name>:<count>:<device>. The " "<device> is optional and if not listed will be applied to every " "device. If the resource is specified as \"GLOBAL\" in the model " "configuration the resource is considered shared among all the devices " "in the system. The <device> property is ignored for such resources. " "This flag can be specified multiple times to specify each resources " "and their availability. By default, the max across all instances that " "list the resource is selected as its availability. The values for this " "flag is case-insensitive."}); memory_device_options_.push_back( {OPTION_PINNED_MEMORY_POOL_BYTE_SIZE, "pinned-memory-pool-byte-size", Option::ArgInt, "The total byte size that can be allocated as pinned system memory. " "If GPU support is enabled, the server will allocate pinned system " "memory to accelerate data transfer between host and devices until it " "exceeds the specified byte size. If 'numa-node' is configured via " "--host-policy, the pinned system memory of the pool size will be " "allocated on each numa node. This option will not affect the " "allocation conducted by the backend frameworks. Default is 256 MB."}); memory_device_options_.push_back( {OPTION_CUDA_MEMORY_POOL_BYTE_SIZE, "cuda-memory-pool-byte-size", "<integer>:<integer>", "The total byte size that can be allocated as CUDA memory for the GPU " "device. If GPU support is enabled, the server will allocate CUDA " "memory to minimize data transfer between host and devices until it " "exceeds the specified byte size. This option will not affect the " "allocation conducted by the backend frameworks. The argument should be " "2 integers separated by colons in the format " "<GPU device ID>:<pool byte size>. This option can be used multiple " "times, but only once per GPU device. Subsequent uses will overwrite " "previous uses for the same GPU device. Default is 64 MB."}); memory_device_options_.push_back( {OPTION_CUDA_VIRTUAL_ADDRESS_SIZE, "cuda-virtual-address-size", "<integer>:<integer>", "The total CUDA virtual address size that will be used for each " "implicit state when growable memory is used. This value determines " "the maximum size of each implicit state. The state size cannot go " "beyond this value. The argument should be " "2 integers separated by colons in the format " "<GPU device ID>:<CUDA virtual address size>. This option can be used " "multiple " "times, but only once per GPU device. Subsequent uses will overwrite " "previous uses for the same GPU device. Default is 1 GB."}); memory_device_options_.push_back( {OPTION_MIN_SUPPORTED_COMPUTE_CAPABILITY, "min-supported-compute-capability", Option::ArgFloat, "The minimum supported CUDA compute capability. GPUs that don't support " "this compute capability will not be used by the server."}); memory_device_options_.push_back( {OPTION_BUFFER_MANAGER_THREAD_COUNT, "buffer-manager-thread-count", Option::ArgInt, "The number of threads used to accelerate copies and other operations " "required to manage input and output tensor contents. Default is 0."}); memory_device_options_.push_back( {OPTION_HOST_POLICY, "host-policy", "<string>,<string>=<string>", "Specify a host policy setting associated with a policy name. The " "format of this flag is --host-policy=<policy_name>,<setting>=<value>. " "Currently supported settings are 'numa-node', 'cpu-cores'. Note that " "'numa-node' setting will affect pinned memory pool behavior, see " "--pinned-memory-pool for more detail."}); memory_device_options_.push_back( {OPTION_MODEL_LOAD_GPU_LIMIT, "model-load-gpu-limit", "<device_id>:<fraction>", "Specify the limit on GPU memory usage as a fraction. If model loading " "on the device is requested and the current memory usage exceeds the " "limit, the load will be rejected. If not specified, the limit will " "not be set."}); backend_options_.push_back( {OPTION_BACKEND_DIR, "backend-directory", Option::ArgStr, "The global directory searched for backend shared libraries. Default is " "'/opt/tritonserver/backends'."}); backend_options_.push_back( {OPTION_BACKEND_CONFIG, "backend-config", "<string>,<string>=<string>", "Specify a backend-specific configuration setting. The format of this " "flag is --backend-config=<backend_name>,<setting>=<value>. Where " "<backend_name> is the name of the backend, such as 'tensorrt'."}); repo_agent_options_.push_back( {OPTION_REPOAGENT_DIR, "repoagent-directory", Option::ArgStr, "The global directory searched for repository agent shared libraries. " "Default is '/opt/tritonserver/repoagents'."}); // Deprecations deprecated_options_.push_back( {OPTION_STRICT_MODEL_CONFIG, "strict-model-config", Option::ArgBool, "DEPRECATED: If true model configuration files must be provided and all " "required " "configuration settings must be specified. If false the model " "configuration may be absent or only partially specified and the " "server will attempt to derive the missing required configuration."}); deprecated_options_.push_back( {OPTION_RESPONSE_CACHE_BYTE_SIZE, "response-cache-byte-size", Option::ArgInt, "DEPRECATED: Please use --cache-config instead."}); #ifdef TRITON_ENABLE_TRACING deprecated_options_.push_back( {OPTION_TRACE_FILEPATH, "trace-file", Option::ArgStr, "DEPRECATED: Please use --trace-config triton,file=<path/to/your/file>" " Set the file where trace output will be saved. If " "--trace-log-frequency" " is also specified, this argument value will be the prefix of the files" " to save the trace output. See --trace-log-frequency for detail."}); deprecated_options_.push_back( {OPTION_TRACE_LEVEL, "trace-level", Option::ArgStr, "DEPRECATED: Please use --trace-config level=<OFF|TIMESTAMPS|TENSORS>" "Specify a trace level. OFF to disable tracing, TIMESTAMPS to " "trace timestamps, TENSORS to trace tensors. It may be specified " "multiple times to trace multiple information. Default is OFF."}); deprecated_options_.push_back( {OPTION_TRACE_RATE, "trace-rate", Option::ArgInt, "DEPRECATED: Please use --trace-config rate=<rate value>" "Set the trace sampling rate. Default is 1000."}); deprecated_options_.push_back( {OPTION_TRACE_COUNT, "trace-count", Option::ArgInt, "DEPRECATED: Please use --trace-config count=<count value>" "Set the number of traces to be sampled. If the value is -1, the number " "of traces to be sampled will not be limited. Default is -1."}); deprecated_options_.push_back( {OPTION_TRACE_LOG_FREQUENCY, "trace-log-frequency", Option::ArgInt, "DEPRECATED: Please use --trace-config triton,log-frequency=<value>" "Set the trace log frequency. If the value is 0, Triton will only log " "the trace output to <trace-file> when shutting down. Otherwise, Triton " "will log the trace output to <trace-file>.<idx> when it collects the " "specified number of traces. For example, if the log frequency is 100, " "when Triton collects the 100-th trace, it logs the traces to file " "<trace-file>.0, and when it collects the 200-th trace, it logs the " "101-th to the 200-th traces to file <trace-file>.1. Default is 0."}); #endif // TRITON_ENABLE_TRACING } void TritonParser::SetupOptionGroups() { SetupOptions(); option_groups_.emplace_back(GLOBAL_OPTION_GROUP, global_options_); option_groups_.emplace_back("Server", server_options_); option_groups_.emplace_back("Logging", logging_options_); option_groups_.emplace_back("Model Repository", model_repo_options_); option_groups_.emplace_back("HTTP", http_options_); option_groups_.emplace_back("GRPC", grpc_options_); option_groups_.emplace_back("Sagemaker", sagemaker_options_); option_groups_.emplace_back("Vertex", vertex_options_); option_groups_.emplace_back("Metrics", metric_options_); option_groups_.emplace_back("Tracing", tracing_options_); option_groups_.emplace_back("Backend", backend_options_); option_groups_.emplace_back("Repository Agent", repo_agent_options_); option_groups_.emplace_back("Response Cache", cache_options_); option_groups_.emplace_back("Rate Limiter", rate_limiter_options_); option_groups_.emplace_back( "Memory/Device Management", memory_device_options_); option_groups_.emplace_back("DEPRECATED", deprecated_options_); } TritonParser::TritonParser() { SetupOptionGroups(); } void TritonServerParameters::CheckPortCollision() { // [FIXME] try to make this function endpoint type agnostic // List of enabled services and their constraints std::vector< std::tuple<std::string, std::string, int32_t, bool, int32_t, int32_t>> ports; #ifdef TRITON_ENABLE_HTTP if (allow_http_) { ports.emplace_back("HTTP", http_address_, http_port_, false, -1, -1); } #endif // TRITON_ENABLE_HTTP #ifdef TRITON_ENABLE_GRPC if (allow_grpc_) { ports.emplace_back( "GRPC", grpc_options_.socket_.address_, grpc_options_.socket_.port_, false, -1, -1); } #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_METRICS if (allow_metrics_) { ports.emplace_back( "metrics", metrics_address_, metrics_port_, false, -1, -1); } #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_SAGEMAKER if (allow_sagemaker_) { ports.emplace_back( "SageMaker", sagemaker_address_, sagemaker_port_, sagemaker_safe_range_set_, sagemaker_safe_range_.first, sagemaker_safe_range_.second); } #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI if (allow_vertex_ai_) { ports.emplace_back( "Vertex AI", vertex_ai_address_, vertex_ai_port_, false, -1, -1); } #endif // TRITON_ENABLE_VERTEX_AI for (auto curr_it = ports.begin(); curr_it != ports.end(); ++curr_it) { // If the current service doesn't specify the allow port range for other // services, then we don't need to revisit the checked services auto comparing_it = (std::get<3>(*curr_it)) ? ports.begin() : (curr_it + 1); for (; comparing_it != ports.end(); ++comparing_it) { if (comparing_it == curr_it) { continue; } if (std::get<1>(*curr_it) != std::get<1>(*comparing_it)) { continue; } // Set range and comparing service port is out of range if (std::get<3>(*curr_it) && ((std::get<2>(*comparing_it) < std::get<4>(*curr_it)) || (std::get<2>(*comparing_it) > std::get<5>(*curr_it)))) { std::stringstream ss; ss << "The server cannot listen to " << std::get<0>(*comparing_it) << " requests at port " << std::get<2>(*comparing_it) << ", allowed port range is [" << std::get<4>(*curr_it) << ", " << std::get<5>(*curr_it) << "]" << std::endl; throw ParseException(ss.str()); } if (std::get<2>(*curr_it) == std::get<2>(*comparing_it)) { std::stringstream ss; ss << "The server cannot listen to " << std::get<0>(*curr_it) << " requests " << "and " << std::get<0>(*comparing_it) << " requests at the same address and port " << std::get<1>(*curr_it) << ":" << std::get<2>(*curr_it) << std::endl; throw ParseException(ss.str()); } } } } TritonServerParameters::ManagedTritonServerOptionPtr TritonServerParameters::BuildTritonServerOptions() { TRITONSERVER_ServerOptions* loptions = nullptr; THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsNew(&loptions), "creating server options"); ManagedTritonServerOptionPtr managed_ptr( loptions, TRITONSERVER_ServerOptionsDelete); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetServerId(loptions, server_id_.c_str()), "setting server ID"); for (const auto& model_repository_path : model_repository_paths_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetModelRepositoryPath( loptions, model_repository_path.c_str()), "setting model repository path"); } THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetModelControlMode(loptions, control_mode_), "setting model control mode"); for (const auto& model : startup_models_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetStartupModel(loptions, model.c_str()), "setting startup model"); } THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetModelConfigName( loptions, model_config_name_.c_str()), "setting custom model configuration name for models"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetRateLimiterMode(loptions, rate_limit_mode_), "setting rate limiter configuration"); for (const auto& resource : rate_limit_resources_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsAddRateLimiterResource( loptions, std::get<0>(resource).c_str(), std::get<1>(resource), std::get<2>(resource)), "setting rate limiter resource"); } THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize( loptions, pinned_memory_pool_byte_size_), "setting total pinned memory byte size"); for (const auto& cuda_pool : cuda_pools_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize( loptions, cuda_pool.first, cuda_pool.second), "setting total CUDA memory byte size"); } for (const auto& cuda_virtual_address_size : cuda_virtual_address_size_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetCudaVirtualAddressSize( loptions, cuda_virtual_address_size.first, cuda_virtual_address_size.second), "setting total CUDA virtual address size"); } THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( loptions, min_supported_compute_capability_), "setting minimum supported CUDA compute capability"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetExitOnError(loptions, exit_on_error_), "setting exit on error"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetStrictModelConfig( loptions, strict_model_config_), "setting strict model configuration"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetStrictReadiness(loptions, strict_readiness_), "setting strict readiness"); // [FIXME] std::max seems to be part of Parse() THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetExitTimeout( loptions, std::max(0, exit_timeout_secs_)), "setting exit timeout"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetBufferManagerThreadCount( loptions, std::max(0, buffer_manager_thread_count_)), "setting buffer manager thread count"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetModelLoadThreadCount( loptions, std::max(1u, model_load_thread_count_)), "setting model load thread count"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetModelLoadRetryCount( loptions, std::max(0u, model_load_retry_count_)), "setting model load retry count"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetModelNamespacing( loptions, enable_model_namespacing_), "setting model namespacing"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetEnablePeerAccess( loptions, enable_peer_access_), "setting peer access"); #ifdef TRITON_ENABLE_LOGGING TRITONSERVER_ServerOptionsSetLogFile(loptions, log_file_.c_str()); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetLogInfo(loptions, log_info_), "setting log info enable"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetLogWarn(loptions, log_warn_), "setting log warn enable"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetLogError(loptions, log_error_), "setting log error enable"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetLogVerbose(loptions, log_verbose_), "setting log verbose level"); switch (log_format_) { case triton::common::Logger::Format::kDEFAULT: THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetLogFormat( loptions, TRITONSERVER_LOG_DEFAULT), "setting log format"); break; case triton::common::Logger::Format::kISO8601: THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetLogFormat( loptions, TRITONSERVER_LOG_ISO8601), "setting log format"); break; } #endif // TRITON_ENABLE_LOGGING #ifdef TRITON_ENABLE_METRICS THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetMetrics(loptions, allow_metrics_), "setting metrics enable"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetGpuMetrics(loptions, allow_gpu_metrics_), "setting GPU metrics enable"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetCpuMetrics(loptions, allow_cpu_metrics_), "setting CPU metrics enable"); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetMetricsInterval( loptions, metrics_interval_ms_), "setting metrics interval"); for (const auto& mcs : metrics_config_settings_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetMetricsConfig( loptions, std::get<0>(mcs).c_str(), std::get<1>(mcs).c_str(), std::get<2>(mcs).c_str()), "setting metrics configuration"); } #endif // TRITON_ENABLE_METRICS THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetBackendDirectory( loptions, backend_dir_.c_str()), "setting backend directory"); // Enable cache and configure it if a cache CLI arg is passed, // this will allow for an empty configuration. if (enable_cache_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetCacheDirectory( loptions, cache_dir_.c_str()), "setting cache directory"); for (const auto& cache_pair : cache_config_settings_) { const auto& cache_name = cache_pair.first; const auto& settings = cache_pair.second; const auto& json_config_str = PairsToJsonStr(settings); THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetCacheConfig( loptions, cache_name.c_str(), json_config_str.c_str()), "setting cache configuration"); } } THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetRepoAgentDirectory( loptions, repoagent_dir_.c_str()), "setting repository agent directory"); for (const auto& bcs : backend_config_settings_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetBackendConfig( loptions, std::get<0>(bcs).c_str(), std::get<1>(bcs).c_str(), std::get<2>(bcs).c_str()), "setting backend configuration"); } for (const auto& limit : load_gpu_limit_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit( loptions, TRITONSERVER_INSTANCEGROUPKIND_GPU, limit.first, limit.second), "setting model load GPU limit"); } for (const auto& hp : host_policies_) { THROW_IF_ERR( ParseException, TRITONSERVER_ServerOptionsSetHostPolicy( loptions, std::get<0>(hp).c_str(), std::get<1>(hp).c_str(), std::get<2>(hp).c_str()), "setting host policy"); } return managed_ptr; } std::pair<TritonServerParameters, std::vector<char*>> TritonParser::Parse(int argc, char** argv) { // // Step 1. Before parsing setup // TritonServerParameters lparams; bool strict_model_config_present{false}; bool disable_auto_complete_config{false}; bool cache_size_present{false}; bool cache_config_present{false}; #ifdef TRITON_ENABLE_TRACING bool explicit_disable_trace{false}; bool trace_filepath_present{false}; bool trace_level_present{false}; bool trace_rate_present{false}; bool trace_count_present{false}; bool trace_log_frequency_present{false}; #endif // TRITON_ENABLE_TRACING int option_index = 0; #ifdef TRITON_ENABLE_GRPC triton::server::grpc::Options& lgrpc_options = lparams.grpc_options_; #endif // TRITON_ENABLE_GRPC #if defined TRITON_ENABLE_HTTP || defined TRITON_ENABLE_GRPC // According to HTTP specification header names are case-insensitive. const std::string case_insensitive_prefix{"(?i)"}; #endif // TRITON_ENABLE_HTTP || TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_VERTEX_AI // Set different default value if specific flag is set { auto aip_mode = triton::server::GetEnvironmentVariableOrDefault("AIP_MODE", ""); // Enable Vertex AI service and disable HTTP / GRPC service by default // if detecting Vertex AI environment if (aip_mode == "PREDICTION") { lparams.allow_vertex_ai_ = true; #ifdef TRITON_ENABLE_HTTP lparams.allow_http_ = false; #endif // TRITON_ENABLE_HTTP #ifdef TRITON_ENABLE_GRPC lparams.allow_grpc_ = false; #endif // TRITON_ENABLE_GRPC } auto port = triton::server::GetEnvironmentVariableOrDefault( "AIP_HTTP_PORT", "8080"); lparams.vertex_ai_port_ = ParseOption<int>(port); } #endif // TRITON_ENABLE_VERTEX_AI // // Step 2. parse options // std::vector<struct option> long_options; for (const auto& group : option_groups_) { for (const auto& o : group.second) { long_options.push_back(o.GetLongOption()); } } long_options.push_back({nullptr, 0, nullptr, 0}); int flag; while ((flag = getopt_long( argc, argv, "", &long_options[0], &option_index)) != -1) { try { switch (flag) { case OPTION_HELP: // [FIXME] how help is printed? case '?': // [FIXME] fall through when seeing this, currently consumes all // options [FIXME] disable stderr output of `getopt_long` throw ParseException(); #ifdef TRITON_ENABLE_LOGGING case OPTION_LOG_VERBOSE: lparams.log_verbose_ = ParseIntBoolOption(optarg); break; case OPTION_LOG_INFO: lparams.log_info_ = ParseOption<bool>(optarg); break; case OPTION_LOG_WARNING: lparams.log_warn_ = ParseOption<bool>(optarg); break; case OPTION_LOG_ERROR: lparams.log_error_ = ParseOption<bool>(optarg); break; case OPTION_LOG_FORMAT: { std::string format_str(optarg); if (format_str == "default") { lparams.log_format_ = triton::common::Logger::Format::kDEFAULT; } else if (format_str == "ISO8601") { lparams.log_format_ = triton::common::Logger::Format::kISO8601; } else { throw ParseException("invalid argument for --log-format"); } break; } case OPTION_LOG_FILE: lparams.log_file_ = optarg; break; #endif // TRITON_ENABLE_LOGGING case OPTION_ID: lparams.server_id_ = optarg; break; case OPTION_MODEL_REPOSITORY: lparams.model_repository_paths_.insert(optarg); break; case OPTION_EXIT_ON_ERROR: lparams.exit_on_error_ = ParseOption<bool>(optarg); break; case OPTION_DISABLE_AUTO_COMPLETE_CONFIG: disable_auto_complete_config = true; break; case OPTION_STRICT_MODEL_CONFIG: std::cerr << "Warning: '--strict-model-config' has been deprecated! " "Please use '--disable-auto-complete-config' instead." << std::endl; strict_model_config_present = true; lparams.strict_model_config_ = ParseOption<bool>(optarg); break; case OPTION_STRICT_READINESS: lparams.strict_readiness_ = ParseOption<bool>(optarg); break; #ifdef TRITON_ENABLE_HTTP case OPTION_ALLOW_HTTP: lparams.allow_http_ = ParseOption<bool>(optarg); break; case OPTION_HTTP_PORT: lparams.http_port_ = ParseOption<int>(optarg); break; case OPTION_REUSE_HTTP_PORT: lparams.reuse_http_port_ = ParseOption<bool>(optarg); break; case OPTION_HTTP_ADDRESS: lparams.http_address_ = optarg; break; case OPTION_HTTP_HEADER_FORWARD_PATTERN: lparams.http_forward_header_pattern_ = std::move(case_insensitive_prefix + optarg); break; case OPTION_HTTP_THREAD_COUNT: lparams.http_thread_cnt_ = ParseOption<int>(optarg); break; case OPTION_HTTP_RESTRICTED_API: ParseRestrictedFeatureOption( optarg, long_options[option_index].name, "", "api", lparams.http_restricted_apis_); break; #endif // TRITON_ENABLE_HTTP #ifdef TRITON_ENABLE_SAGEMAKER case OPTION_ALLOW_SAGEMAKER: lparams.allow_sagemaker_ = ParseOption<bool>(optarg); break; case OPTION_SAGEMAKER_PORT: lparams.sagemaker_port_ = ParseOption<int>(optarg); break; case OPTION_SAGEMAKER_SAFE_PORT_RANGE: lparams.sagemaker_safe_range_set_ = true; lparams.sagemaker_safe_range_ = ParsePairOption<int, int>(optarg, "-"); break; case OPTION_SAGEMAKER_THREAD_COUNT: lparams.sagemaker_thread_cnt_ = ParseOption<int>(optarg); break; #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI case OPTION_ALLOW_VERTEX_AI: lparams.allow_vertex_ai_ = ParseOption<bool>(optarg); break; case OPTION_VERTEX_AI_PORT: lparams.vertex_ai_port_ = ParseOption<int>(optarg); break; case OPTION_VERTEX_AI_THREAD_COUNT: lparams.vertex_ai_thread_cnt_ = ParseOption<int>(optarg); break; case OPTION_VERTEX_AI_DEFAULT_MODEL: lparams.vertex_ai_default_model_ = optarg; break; #endif // TRITON_ENABLE_VERTEX_AI #ifdef TRITON_ENABLE_GRPC case OPTION_ALLOW_GRPC: lparams.allow_grpc_ = ParseOption<bool>(optarg); break; case OPTION_GRPC_PORT: lgrpc_options.socket_.port_ = ParseOption<int>(optarg); break; case OPTION_REUSE_GRPC_PORT: lgrpc_options.socket_.reuse_port_ = ParseOption<bool>(optarg); break; case OPTION_GRPC_ADDRESS: lgrpc_options.socket_.address_ = optarg; break; case OPTION_GRPC_INFER_ALLOCATION_POOL_SIZE: lgrpc_options.infer_allocation_pool_size_ = ParseOption<int>(optarg); break; case OPTION_GRPC_USE_SSL: lgrpc_options.ssl_.use_ssl_ = ParseOption<bool>(optarg); break; case OPTION_GRPC_USE_SSL_MUTUAL: lgrpc_options.ssl_.use_mutual_auth_ = ParseOption<bool>(optarg); lgrpc_options.ssl_.use_ssl_ = true; break; case OPTION_GRPC_SERVER_CERT: lgrpc_options.ssl_.server_cert_ = optarg; break; case OPTION_GRPC_SERVER_KEY: lgrpc_options.ssl_.server_key_ = optarg; break; case OPTION_GRPC_ROOT_CERT: lgrpc_options.ssl_.root_cert_ = optarg; break; case OPTION_GRPC_RESPONSE_COMPRESSION_LEVEL: { std::string mode_str(optarg); std::transform( mode_str.begin(), mode_str.end(), mode_str.begin(), ::tolower); if (mode_str == "none") { lgrpc_options.infer_compression_level_ = GRPC_COMPRESS_LEVEL_NONE; } else if (mode_str == "low") { lgrpc_options.infer_compression_level_ = GRPC_COMPRESS_LEVEL_LOW; } else if (mode_str == "medium") { lgrpc_options.infer_compression_level_ = GRPC_COMPRESS_LEVEL_MED; } else if (mode_str == "high") { lgrpc_options.infer_compression_level_ = GRPC_COMPRESS_LEVEL_HIGH; } else { throw ParseException( "invalid argument for " "--grpc_infer_response_compression_level"); } break; } case OPTION_GRPC_ARG_KEEPALIVE_TIME_MS: lgrpc_options.keep_alive_.keepalive_time_ms_ = ParseOption<int>(optarg); break; case OPTION_GRPC_ARG_KEEPALIVE_TIMEOUT_MS: lgrpc_options.keep_alive_.keepalive_timeout_ms_ = ParseOption<int>(optarg); break; case OPTION_GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS: lgrpc_options.keep_alive_.keepalive_permit_without_calls_ = ParseOption<bool>(optarg); break; case OPTION_GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA: lgrpc_options.keep_alive_.http2_max_pings_without_data_ = ParseOption<int>(optarg); break; case OPTION_GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS: lgrpc_options.keep_alive_ .http2_min_recv_ping_interval_without_data_ms_ = ParseOption<int>(optarg); break; case OPTION_GRPC_ARG_HTTP2_MAX_PING_STRIKES: lgrpc_options.keep_alive_.http2_max_ping_strikes_ = ParseOption<int>(optarg); break; case OPTION_GRPC_ARG_MAX_CONNECTION_AGE_MS: lgrpc_options.keep_alive_.max_connection_age_ms_ = ParseOption<int>(optarg); break; case OPTION_GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS: lgrpc_options.keep_alive_.max_connection_age_grace_ms_ = ParseOption<int>(optarg); break; case OPTION_GRPC_RESTRICTED_PROTOCOL: { ParseRestrictedFeatureOption( optarg, long_options[option_index].name, std::string( triton::server::grpc::kRestrictedProtocolHeaderTemplate), "protocol", lgrpc_options.restricted_protocols_); break; } case OPTION_GRPC_HEADER_FORWARD_PATTERN: lgrpc_options.forward_header_pattern_ = std::move(case_insensitive_prefix + optarg); break; #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_METRICS case OPTION_ALLOW_METRICS: lparams.allow_metrics_ = ParseOption<bool>(optarg); break; case OPTION_ALLOW_GPU_METRICS: lparams.allow_gpu_metrics_ = ParseOption<bool>(optarg); break; case OPTION_ALLOW_CPU_METRICS: lparams.allow_cpu_metrics_ = ParseOption<bool>(optarg); break; case OPTION_METRICS_ADDRESS: lparams.metrics_address_ = optarg; break; case OPTION_METRICS_PORT: lparams.metrics_port_ = ParseOption<int>(optarg); break; case OPTION_METRICS_INTERVAL_MS: lparams.metrics_interval_ms_ = ParseOption<int>(optarg); break; case OPTION_METRICS_CONFIG: lparams.metrics_config_settings_.push_back( ParseMetricsConfigOption(optarg)); break; #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_TRACING case OPTION_TRACE_FILEPATH: { std::cerr << "Warning: '--trace-file' has been deprecated and will be" " removed in future releases. Please use " "'--trace-config triton,file=<filepath> instead." << std::endl; trace_filepath_present = true; lparams.trace_filepath_ = optarg; break; } case OPTION_TRACE_LEVEL: { std::cerr << "Warning: '--trace-level' has been deprecated and will be" " removed in future releases. Please use " "'--trace-config level=<OFF|TIMESTAMPS|TENSORS> instead." << std::endl; trace_level_present = true; auto parsed_level = ParseTraceLevelOption(optarg); explicit_disable_trace |= (parsed_level == TRITONSERVER_TRACE_LEVEL_DISABLED); lparams.trace_level_ = static_cast<TRITONSERVER_InferenceTraceLevel>( lparams.trace_level_ | parsed_level); break; } case OPTION_TRACE_RATE: std::cerr << "Warning: '--trace-rate' has been deprecated and will be" " removed in future releases. Please use " "'--trace-config rate=<rate value> instead." << std::endl; trace_rate_present = true; lparams.trace_rate_ = ParseOption<int>(optarg); break; case OPTION_TRACE_COUNT: std::cerr << "Warning: '--trace-count' has been deprecated and will be" " removed in future releases. Please use " "'--trace-config count=<count value> instead." << std::endl; trace_count_present = true; lparams.trace_count_ = ParseOption<int>(optarg); break; case OPTION_TRACE_LOG_FREQUENCY: std::cerr << "Warning: '--trace-log-frequency' has been deprecated and " "will be" " removed in future releases. Please use " "'--trace-config triton,log-frequency=<log frequency " "value> instead." << std::endl; trace_log_frequency_present = true; lparams.trace_log_frequency_ = ParseOption<int>(optarg); break; case OPTION_TRACE_CONFIG: { auto trace_config_setting = ParseTraceConfigOption(optarg); triton::server::TraceConfig& tc = lparams .trace_config_map_[std::get<0>(trace_config_setting).c_str()]; tc.push_back(std::make_pair( std::get<1>(trace_config_setting).c_str(), std::get<2>(trace_config_setting).c_str())); break; } #endif // TRITON_ENABLE_TRACING case OPTION_POLL_REPO_SECS: lparams.repository_poll_secs_ = ParseOption<int>(optarg); break; case OPTION_STARTUP_MODEL: lparams.startup_models_.insert(optarg); break; case OPTION_CUSTOM_MODEL_CONFIG_NAME: if (std::strlen(optarg) == 0) { throw ParseException( "Error: empty argument for --model-config-name"); } lparams.model_config_name_ = optarg; break; case OPTION_MODEL_CONTROL_MODE: { std::string mode_str(optarg); std::transform( mode_str.begin(), mode_str.end(), mode_str.begin(), ::tolower); if (mode_str == "none") { lparams.control_mode_ = TRITONSERVER_MODEL_CONTROL_NONE; } else if (mode_str == "poll") { lparams.control_mode_ = TRITONSERVER_MODEL_CONTROL_POLL; } else if (mode_str == "explicit") { lparams.control_mode_ = TRITONSERVER_MODEL_CONTROL_EXPLICIT; } else { throw ParseException("invalid argument for --model-control-mode"); } break; } case OPTION_RATE_LIMIT: { std::string rate_limit_str(optarg); std::transform( rate_limit_str.begin(), rate_limit_str.end(), rate_limit_str.begin(), ::tolower); if (rate_limit_str == "execution_count") { lparams.rate_limit_mode_ = TRITONSERVER_RATE_LIMIT_EXEC_COUNT; } else if (rate_limit_str == "off") { lparams.rate_limit_mode_ = TRITONSERVER_RATE_LIMIT_OFF; } else { throw ParseException("invalid argument for --rate-limit"); } break; } case OPTION_RATE_LIMIT_RESOURCE: { std::string rate_limit_resource_str(optarg); std::transform( rate_limit_resource_str.begin(), rate_limit_resource_str.end(), rate_limit_resource_str.begin(), ::tolower); lparams.rate_limit_resources_.push_back( ParseRateLimiterResourceOption(optarg)); break; } case OPTION_PINNED_MEMORY_POOL_BYTE_SIZE: lparams.pinned_memory_pool_byte_size_ = ParseOption<int64_t>(optarg); break; case OPTION_CUDA_MEMORY_POOL_BYTE_SIZE: lparams.cuda_pools_.push_back( ParsePairOption<int, uint64_t>(optarg, ":")); break; case OPTION_CUDA_VIRTUAL_ADDRESS_SIZE: lparams.cuda_virtual_address_size_.push_back( ParsePairOption<int, size_t>(optarg, ":")); break; case OPTION_RESPONSE_CACHE_BYTE_SIZE: { cache_size_present = true; const auto byte_size = std::to_string(ParseOption<int64_t>(optarg)); lparams.cache_config_settings_["local"] = {{"size", byte_size}}; std::cerr << "Warning: '--response-cache-byte-size' has been deprecated! " "This will default to the 'local' cache implementation with " "the provided byte size for its config. Please use " "'--cache-config' instead. The equivalent " "--cache-config CLI args would be: " "'--cache-config=local,size=" + byte_size + "'" << std::endl; break; } case OPTION_CACHE_CONFIG: { cache_config_present = true; const auto cache_setting = ParseCacheConfigOption(optarg); const auto& cache_name = std::get<0>(cache_setting); const auto& key = std::get<1>(cache_setting); const auto& value = std::get<2>(cache_setting); lparams.cache_config_settings_[cache_name].push_back({key, value}); break; } case OPTION_CACHE_DIR: lparams.cache_dir_ = optarg; break; case OPTION_MIN_SUPPORTED_COMPUTE_CAPABILITY: lparams.min_supported_compute_capability_ = ParseOption<double>(optarg); break; case OPTION_EXIT_TIMEOUT_SECS: lparams.exit_timeout_secs_ = ParseOption<int>(optarg); break; case OPTION_BACKEND_DIR: lparams.backend_dir_ = optarg; break; case OPTION_REPOAGENT_DIR: lparams.repoagent_dir_ = optarg; break; case OPTION_BUFFER_MANAGER_THREAD_COUNT: lparams.buffer_manager_thread_count_ = ParseOption<int>(optarg); break; case OPTION_MODEL_LOAD_THREAD_COUNT: lparams.model_load_thread_count_ = ParseOption<int>(optarg); break; case OPTION_MODEL_LOAD_RETRY_COUNT: lparams.model_load_retry_count_ = ParseOption<int>(optarg); break; case OPTION_BACKEND_CONFIG: lparams.backend_config_settings_.push_back( ParseBackendConfigOption(optarg)); break; case OPTION_HOST_POLICY: lparams.host_policies_.push_back(ParseHostPolicyOption(optarg)); break; case OPTION_MODEL_LOAD_GPU_LIMIT: lparams.load_gpu_limit_.emplace( ParsePairOption<int, double>(optarg, ":")); break; case OPTION_MODEL_NAMESPACING: lparams.enable_model_namespacing_ = ParseOption<bool>(optarg); break; case OPTION_ENABLE_PEER_ACCESS: lparams.enable_peer_access_ = ParseOption<bool>(optarg); break; } } catch (const ParseException& pe) { if ((pe.what() != NULL) && (strlen(pe.what()) != 0)) { std::stringstream ss; ss << "Bad option: \"--" << long_options[option_index].name << "\".\n" << pe.what() << std::endl; throw ParseException(ss.str()); } else { // In case of `Unrecognized option` or `Help` option, just throw a // ParseException throw ParseException(); } } } if (optind < argc) { throw ParseException(std::string("Unexpected argument: ") + argv[optind]); } // // Step 3. Post parsing validation, usually for options that depend on the // others which are not determined until after parsing. // if (lparams.control_mode_ != TRITONSERVER_MODEL_CONTROL_POLL) { lparams.repository_poll_secs_ = 0; } if (lparams.startup_models_.size() > 0 && lparams.control_mode_ != TRITONSERVER_MODEL_CONTROL_EXPLICIT) { throw ParseException( "Error: Use of '--load-model' requires setting " "'--model-control-mode=explicit' as well."); } #ifdef TRITON_ENABLE_VERTEX_AI // Set default model repository if specific flag is set, postpone the // check to after parsing so we only monitor the default repository if // Vertex service is allowed if (lparams.model_repository_paths_.empty()) { auto aip_storage_uri = triton::server::GetEnvironmentVariableOrDefault("AIP_STORAGE_URI", ""); if (!aip_storage_uri.empty()) { lparams.model_repository_paths_.insert(aip_storage_uri); } } #endif // TRITON_ENABLE_VERTEX_AI #ifdef TRITON_ENABLE_METRICS lparams.allow_gpu_metrics_ &= lparams.allow_metrics_; lparams.allow_cpu_metrics_ &= lparams.allow_metrics_; // Set metrics_address to default if never specified if (lparams.metrics_address_.empty()) { #ifdef TRITON_ENABLE_HTTP // If built with HTTP support, default to HTTP address lparams.metrics_address_ = lparams.http_address_; #else // Otherwise have default for builds without HTTP support lparams.metrics_address_ = "0.0.0.0"; #endif // TRITON_ENABLE_HTTP } #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_TRACING PostProcessTraceArgs( lparams, trace_level_present, trace_rate_present, trace_count_present, trace_filepath_present, trace_log_frequency_present, explicit_disable_trace); #endif // TRITON_ENABLE_TRACING // Check if there is a conflict between --disable-auto-complete-config // and --strict-model-config if (disable_auto_complete_config) { if (strict_model_config_present && !lparams.strict_model_config_) { std::cerr << "Warning: Overriding deprecated '--strict-model-config' from " "False to True in favor of '--disable-auto-complete-config'!" << std::endl; } lparams.strict_model_config_ = true; } // Check if there is a conflict between --response-cache-byte-size // and --cache-config if (cache_size_present && cache_config_present) { throw ParseException( "Error: Incompatible flags --response-cache-byte-size and " "--cache-config both provided. Please provide one or the other."); } lparams.enable_cache_ = (cache_size_present || cache_config_present); return {lparams, {}}; } std::string TritonParser::FormatUsageMessage(std::string str, int offset) { int width = 60; int current_pos = offset; while (current_pos + width < int(str.length())) { int n = str.rfind(' ', current_pos + width); if (n != int(std::string::npos)) { str.replace(n, 1, "\n\t"); current_pos += (width + 9); } } return str; } std::string TritonParser::Usage() { std::stringstream ss; for (const auto& group : option_groups_) { if (!group.first.empty() && !group.second.empty()) { ss << std::endl << group.first << ":" << std::endl; } for (const auto& o : group.second) { if (!o.arg_desc_.empty()) { ss << " --" << o.flag_ << " <" << o.arg_desc_ << ">" << std::endl << "\t" << FormatUsageMessage(o.desc_, 0) << std::endl; } else { ss << " --" << o.flag_ << std::endl << "\t" << FormatUsageMessage(o.desc_, 0) << std::endl; } } } return ss.str(); } std::tuple<std::string, std::string, std::string> TritonParser::ParseMetricsConfigOption(const std::string& arg) { // Format is "<setting>=<value>" for generic configs/settings int delim_setting = arg.find("="); if (delim_setting < 0) { std::stringstream ss; ss << "--metrics-config option format is " << "<setting>=<value>. Got " << arg << std::endl; throw ParseException(ss.str()); } // Break section before "=" into substr to avoid matching commas // in setting values. auto name_substr = arg.substr(0, delim_setting); int delim_name = name_substr.find(","); // No name-specific configs currently supported, though it may be in // the future. Map global configs to empty string like other configs for // now. std::string name_string = std::string(); if (delim_name >= 0) { std::stringstream ss; ss << "--metrics-config option format is " << "<setting>=<value>. Got " << arg << std::endl; throw ParseException(ss.str()); } // else global metrics config std::string setting_string = arg.substr(delim_name + 1, delim_setting - delim_name - 1); std::string value_string = arg.substr(delim_setting + 1); if (setting_string.empty() || value_string.empty()) { std::stringstream ss; ss << "--metrics-config option format is " << "<setting>=<value>. Got " << arg << std::endl; throw ParseException(ss.str()); } return {name_string, setting_string, value_string}; } std::tuple<std::string, std::string, std::string> TritonParser::ParseCacheConfigOption(const std::string& arg) { // Format is "<cache_name>,<setting>=<value>" for specific // config/settings and "<setting>=<value>" for cache agnostic // configs/settings int delim_name = arg.find(","); int delim_setting = arg.find("=", delim_name + 1); std::string name_string = std::string(); if (delim_name > 0) { name_string = arg.substr(0, delim_name); } // No cache-agnostic global settings are currently supported else { std::stringstream ss; ss << "No cache specified. --cache-config option format is " << "<cache name>,<setting>=<value>. Got " << arg << std::endl; throw ParseException(ss.str()); } if (delim_setting < 0) { std::stringstream ss; ss << "--cache-config option format is '<cache " "name>,<setting>=<value>'. Got " << arg << std::endl; throw ParseException(ss.str()); } std::string setting_string = arg.substr(delim_name + 1, delim_setting - delim_name - 1); std::string value_string = arg.substr(delim_setting + 1); if (setting_string.empty() || value_string.empty()) { std::stringstream ss; ss << "--cache-config option format is '<cache " "name>,<setting>=<value>'. Got " << arg << std::endl; throw ParseException(ss.str()); } return {name_string, setting_string, value_string}; } std::tuple<std::string, int, int> TritonParser::ParseRateLimiterResourceOption(const std::string& arg) { std::string error_string( "--rate-limit-resource option format is " "'<resource_name>:<count>:<device>' or '<resource_name>:<count>'. " "Got " + arg); std::string name_string(""); int count = -1; int device_id = -1; size_t delim_first = arg.find(":"); size_t delim_second = arg.find(":", delim_first + 1); if (delim_second != std::string::npos) { // Handle format `<resource_name>:<count>:<device>' size_t delim_third = arg.find(":", delim_second + 1); if (delim_third != std::string::npos) { throw ParseException(error_string); } name_string = arg.substr(0, delim_first); count = ParseOption<int>( arg.substr(delim_first + 1, delim_second - delim_first - 1)); device_id = ParseOption<int>(arg.substr(delim_second + 1)); } else if (delim_first != std::string::npos) { // Handle format `<resource_name>:<count>' name_string = arg.substr(0, delim_first); count = ParseOption<int>(arg.substr(delim_first + 1)); } else { // If no colons found throw ParseException(error_string); } return {name_string, count, device_id}; } std::tuple<std::string, std::string, std::string> TritonParser::ParseBackendConfigOption(const std::string& arg) { // Format is "<backend_name>,<setting>=<value>" for specific // config/settings and "<setting>=<value>" for backend agnostic // configs/settings int delim_name = arg.find(","); int delim_setting = arg.find("=", delim_name + 1); std::string name_string = std::string(); if (delim_name > 0) { name_string = arg.substr(0, delim_name); } else if (delim_name == 0) { std::stringstream ss; ss << "No backend specified. --backend-config option format is " << "<backend name>,<setting>=<value> or " << "<setting>=<value>. Got " << arg << std::endl; throw ParseException(ss.str()); } // else global backend config if (delim_setting < 0) { std::stringstream ss; ss << "--backend-config option format is '<backend " "name>,<setting>=<value>'. Got " << arg << std::endl; throw ParseException(ss.str()); } std::string setting_string = arg.substr(delim_name + 1, delim_setting - delim_name - 1); std::string value_string = arg.substr(delim_setting + 1); if (setting_string.empty() || value_string.empty()) { std::stringstream ss; ss << "--backend-config option format is '<backend " "name>,<setting>=<value>'. Got " << arg << std::endl; throw ParseException(ss.str()); } return {name_string, setting_string, value_string}; } void TritonParser::ParseRestrictedFeatureOption( const std::string& arg, const std::string& option_name, const std::string& key_prefix, const std::string& feature_type, RestrictedFeatures& restricted_features) { const auto& parsed_tuple = ParseGenericConfigOption(arg, ":", "=", option_name, "config name"); const auto& features = SplitOptions(std::get<0>(parsed_tuple), ","); const auto& key = std::get<1>(parsed_tuple); const auto& value = std::get<2>(parsed_tuple); for (const auto& feature : features) { const auto& category = RestrictedFeatures::ToCategory(feature); if (category == RestrictedCategory::INVALID) { std::stringstream ss; ss << "unknown restricted " << feature_type << " '" << feature << "' " << std::endl; throw ParseException(ss.str()); } if (restricted_features.IsRestricted(category)) { // restricted feature can only be in one group std::stringstream ss; ss << "restricted " << feature_type << " '" << feature << "' can not be specified in multiple config groups" << std::endl; throw ParseException(ss.str()); } restricted_features.Insert( category, std::make_pair(key_prefix + key, value)); } } std::tuple<std::string, std::string, std::string> TritonParser::ParseHostPolicyOption(const std::string& arg) { return ParseGenericConfigOption(arg, ",", "=", "host-policy", "policy name"); } std::tuple<std::string, std::string, std::string> TritonParser::ParseGenericConfigOption( const std::string& arg, const std::string& first_delim, const std::string& second_delim, const std::string& option_name, const std::string& config_name) { // Format is "<string>,<string>=<string>" int delim_name = arg.find(first_delim); int delim_setting = arg.find(second_delim, delim_name + 1); std::string error_string = "--" + option_name + " option format is '<" + config_name + ">" + first_delim + "<setting>" + second_delim + "<value>'. Got " + arg + "\n"; // Check for 2 semicolons if ((delim_name < 0) || (delim_setting < 0)) { throw ParseException(error_string); } std::string name_string = arg.substr(0, delim_name); std::string setting_string = arg.substr(delim_name + 1, delim_setting - delim_name - 1); std::string value_string = arg.substr(delim_setting + 1); if (name_string.empty() || setting_string.empty() || value_string.empty()) { throw ParseException(error_string); } return {name_string, setting_string, value_string}; } #ifdef TRITON_ENABLE_TRACING TRITONSERVER_InferenceTraceLevel TritonParser::ParseTraceLevelOption(std::string arg) { std::transform(arg.begin(), arg.end(), arg.begin(), [](unsigned char c) { return std::tolower(c); }); if ((arg == "false") || (arg == "off")) { return TRITONSERVER_TRACE_LEVEL_DISABLED; } if ((arg == "true") || (arg == "on") || (arg == "min") || (arg == "max") || (arg == "timestamps")) { return TRITONSERVER_TRACE_LEVEL_TIMESTAMPS; } if (arg == "tensors") { return TRITONSERVER_TRACE_LEVEL_TENSORS; } throw ParseException("invalid value for trace level option: " + arg); } InferenceTraceMode TritonParser::ParseTraceModeOption(std::string arg) { std::transform(arg.begin(), arg.end(), arg.begin(), [](unsigned char c) { return std::tolower(c); }); if (arg == "triton") { return TRACE_MODE_TRITON; } if (arg == "opentelemetry") { return TRACE_MODE_OPENTELEMETRY; } throw ParseException( "invalid value for trace mode option: " + arg + ". Available options are \"triton\" and \"opentelemetry\""); } std::tuple<std::string, std::string, std::string> TritonParser::ParseTraceConfigOption(const std::string& arg) { int delim_name = arg.find(","); int delim_setting = arg.find("=", delim_name + 1); std::string name_string = std::string(); if (delim_name > 0) { name_string = std::to_string(ParseTraceModeOption(arg.substr(0, delim_name))); } else if (delim_name == 0) { std::stringstream ss; ss << "No trace mode specified. --trace-config option format is " << "<trace mode>,<setting>=<value> or " << "<setting>=<value>. Got " << arg << std::endl; throw ParseException(ss.str()); } // else global trace config if (delim_setting < 0) { std::stringstream ss; ss << "--trace-config option format is '<trace mode>,<setting>=<value>'. " "Got " << arg << std::endl; throw ParseException(ss.str()); } std::string setting_string = arg.substr(delim_name + 1, delim_setting - delim_name - 1); std::string value_string = arg.substr(delim_setting + 1); if (setting_string.empty() || value_string.empty()) { std::stringstream ss; ss << "--trace-config option format is '<trace mode>,<setting>=<value>'. " "Got " << arg << std::endl; throw ParseException(ss.str()); } return {name_string, setting_string, value_string}; } void TritonParser::SetGlobalTraceArgs( TritonServerParameters& lparams, bool trace_level_present, bool trace_rate_present, bool trace_count_present, bool explicit_disable_trace) { for (const auto& [setting, value_variant] : lparams.trace_config_map_[""]) { auto value = std::get<std::string>(value_variant); try { if (setting == "rate") { if (trace_rate_present) { std::cerr << "Warning: Overriding deprecated '--trace-rate' " "in favor of provided rate value in --trace-config!" << std::endl; } lparams.trace_rate_ = ParseOption<int>(value); } if (setting == "level") { if (trace_level_present) { std::cerr << "Warning: Overriding deprecated '--trace-level' " "in favor of provided level in --trace-config!" << std::endl; } auto parsed_level_config = ParseTraceLevelOption(value); explicit_disable_trace |= (parsed_level_config == TRITONSERVER_TRACE_LEVEL_DISABLED); lparams.trace_level_ = static_cast<TRITONSERVER_InferenceTraceLevel>( lparams.trace_level_ | parsed_level_config); } if (setting == "mode") { lparams.trace_mode_ = ParseTraceModeOption(value); } if (setting == "count") { if (trace_count_present) { std::cerr << "Warning: Overriding deprecated '--trace-count' " "in favor of provided count in --trace-config!" << std::endl; } lparams.trace_count_ = ParseOption<int>(value); } } catch (const ParseException& pe) { std::stringstream ss; ss << "Bad option: \"--trace-config " << setting << "\".\n" << pe.what() << std::endl; throw ParseException(ss.str()); } } } void TritonParser::SetTritonTraceArgs( TritonServerParameters& lparams, bool trace_filepath_present, bool trace_log_frequency_present) { for (const auto& [setting, value_variant] : lparams.trace_config_map_[std::to_string(TRACE_MODE_TRITON)]) { auto value = std::get<std::string>(value_variant); try { if (setting == "file") { if (trace_filepath_present) { std::cerr << "Warning: Overriding deprecated '--trace-file' " "in favor of provided file in --trace-config!" << std::endl; } lparams.trace_filepath_ = value; } else if (setting == "log-frequency") { if (trace_log_frequency_present) { std::cerr << "Warning: Overriding deprecated '--trace-log-frequency' " "in favor of provided log-frequency in --trace-config!" << std::endl; } lparams.trace_log_frequency_ = ParseOption<int>(value); } } catch (const ParseException& pe) { std::stringstream ss; ss << "Bad option: \"--trace-config triton," << setting << "\".\n" << pe.what() << std::endl; throw ParseException(ss.str()); } } } void TritonParser::SetOpenTelemetryTraceArgs( TritonServerParameters& lparams, bool trace_filepath_present, bool trace_log_frequency_present) { if (trace_filepath_present) { std::cerr << "Warning: '--trace-file' is deprecated and will " "be ignored with opentelemetry tracing mode. " << std::endl; } if (trace_log_frequency_present) { std::cerr << "Warning: '--trace-log-frequency' is deprecated " "and will be ignored with opentelemetry tracing mode." << std::endl; } triton::server::TraceConfig& otel_trace_settings = lparams.trace_config_map_[std::to_string(TRACE_MODE_OPENTELEMETRY)]; ProcessOpenTelemetryBatchSpanProcessorArgs(otel_trace_settings); } void TritonParser::ProcessOpenTelemetryBatchSpanProcessorArgs( TraceConfig& otel_trace_settings) { std::unordered_map<std::string, std::string> otel_bsp_default_settings = {}; // Set up default BatchSpanProcessor parameters, or use // parameters, specified by environment variables auto env_bsp_max_queue_size = triton::server::GetEnvironmentVariableOrDefault( "OTEL_BSP_MAX_QUEUE_SIZE", "2048"); otel_bsp_default_settings.insert(std::make_pair( std::string("bsp_max_queue_size"), env_bsp_max_queue_size)); auto env_bsp_schedule_delay = triton::server::GetEnvironmentVariableOrDefault( "OTEL_BSP_SCHEDULE_DELAY", "5000"); otel_bsp_default_settings.insert(std::make_pair( std::string("bsp_schedule_delay"), env_bsp_schedule_delay)); auto env_bsp_max_export_batch_size = triton::server::GetEnvironmentVariableOrDefault( "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "512"); otel_bsp_default_settings.insert(std::make_pair( std::string("bsp_max_export_batch_size"), env_bsp_max_export_batch_size)); // Process cmd args and convert string arguments to integers. // Throw a ParseException for invalid arguments for (auto& [setting, value_variant] : otel_trace_settings) { try { auto value = std::get<std::string>(value_variant); if (setting == "bsp_max_queue_size") { value_variant = ParseOption<uint32_t>(value); otel_bsp_default_settings.erase("bsp_max_queue_size"); } else if (setting == "bsp_schedule_delay") { value_variant = ParseOption<uint32_t>(value); otel_bsp_default_settings.erase("bsp_schedule_delay"); } else if (setting == "bsp_max_export_batch_size") { value_variant = ParseOption<uint32_t>(value); otel_bsp_default_settings.erase("bsp_max_export_batch_size"); } } catch (const ParseException& pe) { std::stringstream ss; ss << "Bad option: \"--trace-config opentelemetry," << setting << "\".\n" << pe.what() << std::endl; throw ParseException(ss.str()); } } // If not all BSP settings were provided through cmd, // populate OpenTelemetry's trace settings with the default value. if (!otel_bsp_default_settings.empty()) { for (const auto& [setting, value] : otel_bsp_default_settings) { try { otel_trace_settings.push_back( std::make_pair(setting, ParseOption<uint32_t>(value))); } catch (const ParseException& pe) { std::stringstream ss; ss << "Bad option: \"OTEL_"; for (auto& ch : setting) { ss << static_cast<char>(std::toupper(ch)); } ss << "\".\n" << pe.what() << std::endl; throw ParseException(ss.str()); } } } } void TritonParser::PostProcessTraceArgs( TritonServerParameters& lparams, bool trace_level_present, bool trace_rate_present, bool trace_count_present, bool trace_filepath_present, bool trace_log_frequency_present, bool explicit_disable_trace) { SetGlobalTraceArgs( lparams, trace_level_present, trace_rate_present, trace_count_present, explicit_disable_trace); if (lparams.trace_mode_ == TRACE_MODE_OPENTELEMETRY) { SetOpenTelemetryTraceArgs( lparams, trace_filepath_present, trace_log_frequency_present); } else if (lparams.trace_mode_ == TRACE_MODE_TRITON) { SetTritonTraceArgs( lparams, trace_filepath_present, trace_log_frequency_present); } if (explicit_disable_trace) { lparams.trace_level_ = TRITONSERVER_TRACE_LEVEL_DISABLED; } } #endif // TRITON_ENABLE_TRACING }} // namespace triton::server ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/command_line_parser.h���������������������������������������������������������0000664�0000000�0000000�00000032476�15270523712�0020612�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #pragma once #include <list> #include <map> #include <memory> #include <set> #include <string> #include <thread> #include <unordered_map> #include <vector> #include "restricted_features.h" #include "triton/common/logging.h" #include "triton/core/tritonserver.h" #ifdef TRITON_ENABLE_GRPC // To avoid ambiguous reference during build // grpc headers should be imported first // https://github.com/open-telemetry/opentelemetry-cpp/blob/main/examples/otlp/README.md#additional-notes-regarding-abseil-library #include "grpc/grpc_server.h" #endif // TRITON_ENABLE_GRPC #if defined(TRITON_ENABLE_HTTP) || defined(TRITON_ENABLE_METRICS) #include "http_server.h" #endif // TRITON_ENABLE_HTTP || TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_SAGEMAKER #include "sagemaker_server.h" #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI #include "vertex_ai_server.h" #endif // TRITON_ENABLE_VERTEX_AI #ifndef _WIN32 #include <getopt.h> #include <unistd.h> #else // Minimum implementation of <getopt.h> for Windows #define required_argument 1 #define no_argument 2 struct option { option(const char* name, int has_arg, int* flag, int val) : name(name), has_arg(has_arg), flag(flag), val(val) { } const char* name; int has_arg; int* flag; int val; }; #endif #ifdef TRITON_ENABLE_TRACING #include "tracer.h" #endif namespace triton { namespace server { // Command-line options struct Option { static constexpr const char* ArgNone = ""; static constexpr const char* ArgBool = "boolean"; static constexpr const char* ArgFloat = "float"; static constexpr const char* ArgInt = "integer"; static constexpr const char* ArgStr = "string"; Option(int id, std::string flag, std::string arg_desc, std::string desc) : id_(id), flag_(flag), arg_desc_(arg_desc), desc_(desc) { } struct option GetLongOption() const { struct option lo { flag_.c_str(), (!arg_desc_.empty()) ? required_argument : no_argument, nullptr, id_ }; return lo; } const int id_; const std::string flag_; const std::string arg_desc_; const std::string desc_; }; struct TritonServerParameters { std::string server_id_{"triton"}; bool exit_on_error_{true}; bool strict_model_config_{false}; bool strict_readiness_{true}; int32_t exit_timeout_secs_{30}; #ifdef TRITON_ENABLE_GPU double min_supported_compute_capability_{TRITON_MIN_COMPUTE_CAPABILITY}; #else double min_supported_compute_capability_{0.0}; #endif // TRITON_ENABLE_GPU std::string repoagent_dir_{"/opt/tritonserver/repoagents"}; std::string backend_dir_{"/opt/tritonserver/backends"}; std::vector<std::tuple<std::string, std::string, std::string>> backend_config_settings_; // Model repository manager configuration bool enable_model_namespacing_{false}; bool enable_peer_access_{true}; std::set<std::string> model_repository_paths_{}; TRITONSERVER_ModelControlMode control_mode_{TRITONSERVER_MODEL_CONTROL_NONE}; std::set<std::string> startup_models_{}; // Interval, in seconds, when the model repository is polled for changes. int32_t repository_poll_secs_{15}; // Number of threads to use for concurrently loading models uint32_t model_load_thread_count_{4}; uint32_t model_load_retry_count_{0}; std::map<int, double> load_gpu_limit_; // Custom model configuration file. Fall back to default config.pbtxt if not // set. std::string model_config_name_; // Rate limiter configuration // FIXME: Once the rate limiter implementation is complete make // EXEC_COUNT the default. // TRITONSERVER_RateLimitMode // rate_limit_mode_{TRITONSERVER_RATE_LIMIT_EXEC_COUNT}; TRITONSERVER_RateLimitMode rate_limit_mode_{TRITONSERVER_RATE_LIMIT_OFF}; std::vector<std::tuple<std::string, int, int>> rate_limit_resources_; // memory pool configuration int64_t pinned_memory_pool_byte_size_{1 << 28}; std::list<std::pair<int, uint64_t>> cuda_pools_; std::list<std::pair<int, size_t>> cuda_virtual_address_size_; // [FIXME] this option is broken after backend separation: this should have // controlled backend copy behavior but not properly propagate to backend // after separation, need to go through backend config. int32_t buffer_manager_thread_count_{0}; std::vector<std::tuple<std::string, std::string, std::string>> host_policies_; // Cache configuration bool enable_cache_{false}; std::string cache_dir_{"/opt/tritonserver/caches"}; std::unordered_map< std::string, std::vector<std::pair<std::string, std::string>>> cache_config_settings_; #ifdef TRITON_ENABLE_LOGGING bool log_info_{true}; bool log_warn_{true}; bool log_error_{true}; int32_t log_verbose_{0}; triton::common::Logger::Format log_format_{ triton::common::Logger::Format::kDEFAULT}; std::string log_file_{}; #endif // TRITON_ENABLE_LOGGING #ifdef TRITON_ENABLE_TRACING std::string trace_filepath_{}; TRITONSERVER_InferenceTraceLevel trace_level_{ TRITONSERVER_TRACE_LEVEL_DISABLED}; int32_t trace_rate_{1000}; int32_t trace_count_{-1}; int32_t trace_log_frequency_{0}; InferenceTraceMode trace_mode_{TRACE_MODE_TRITON}; TraceConfigMap trace_config_map_; #endif // TRITON_ENABLE_TRACING // The configurations for various endpoints (i.e. HTTP, GRPC and metrics) #ifdef TRITON_ENABLE_HTTP bool allow_http_{true}; std::string http_address_{"0.0.0.0"}; int32_t http_port_{8000}; bool reuse_http_port_{false}; std::string http_forward_header_pattern_; // The number of threads to initialize for the HTTP front-end. int http_thread_cnt_{8}; RestrictedFeatures http_restricted_apis_{}; #endif // TRITON_ENABLE_HTTP #ifdef TRITON_ENABLE_GRPC bool allow_grpc_{true}; triton::server::grpc::Options grpc_options_; #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_METRICS bool allow_metrics_{true}; // Defaults to http_address_ if TRITON_ENABLE_HTTP is enabled for backwards, // otherwise defaults to "0.0.0.0" for TRITON_ENABLE_HTTP is disabled. std::string metrics_address_{""}; int32_t metrics_port_{8002}; // Metric settings for Triton core float metrics_interval_ms_{2000}; bool allow_gpu_metrics_{true}; bool allow_cpu_metrics_{true}; std::vector<std::tuple<std::string, std::string, std::string>> metrics_config_settings_; #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_SAGEMAKER bool allow_sagemaker_{false}; std::string sagemaker_address_{"0.0.0.0"}; int32_t sagemaker_port_{8080}; bool sagemaker_safe_range_set_{false}; std::pair<int32_t, int32_t> sagemaker_safe_range_{-1, -1}; // The number of threads to initialize for the SageMaker HTTP front-end. int sagemaker_thread_cnt_{8}; #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI bool allow_vertex_ai_{false}; std::string vertex_ai_address_{"0.0.0.0"}; int32_t vertex_ai_port_{8080}; // The number of threads to initialize for the Vertex AI HTTP front-end. int vertex_ai_thread_cnt_{8}; std::string vertex_ai_default_model_{}; #endif // TRITON_ENABLE_VERTEX_AI // [FIXME] who should call this function? void CheckPortCollision(); using ManagedTritonServerOptionPtr = std::unique_ptr< TRITONSERVER_ServerOptions, decltype(&TRITONSERVER_ServerOptionsDelete)>; ManagedTritonServerOptionPtr BuildTritonServerOptions(); }; // Exception type to be thrown if the error is parsing related class ParseException : public std::exception { public: ParseException() = default; ParseException(const std::string& message) : message_(message) {} virtual const char* what() const throw() { return message_.c_str(); } private: const std::string message_{""}; }; // [WIP] Fall-through parser, Parse() will convert the recognized options into // parameter object and return the unrecognized options to be another argument // list for other parser to consume. // This allows the composition of parser chain. // [FIXME] abstract interface, concrete class below should only parse Triton // core and endpoint control options (endpoint specific options in their own // parser) class TritonParser { public: TritonParser(); // Parse command line arguments into a parameters struct and transform // the argument list to contain only unrecognized options. The content of // unrecognized argument list shares the same lifecycle as 'argv'. // Raise ParseException if fail to parse recognized options. std::pair<TritonServerParameters, std::vector<char*>> Parse( int argc, char** argv); // Return usage of all recognized options std::string Usage(); private: std::string FormatUsageMessage(std::string str, int offset); // Helper functions for parsing options that require multi-value parsing. std::tuple<std::string, std::string, std::string> ParseCacheConfigOption( const std::string& arg); std::tuple<std::string, int, int> ParseRateLimiterResourceOption( const std::string& arg); std::tuple<std::string, std::string, std::string> ParseBackendConfigOption( const std::string& arg); std::tuple<std::string, std::string, std::string> ParseHostPolicyOption( const std::string& arg); std::tuple<std::string, std::string, std::string> ParseMetricsConfigOption( const std::string& arg); void ParseRestrictedFeatureOption( const std::string& arg, const std::string& option_name, const std::string& header_prefix, const std::string& feature_type, RestrictedFeatures& restricted_features); #ifdef TRITON_ENABLE_TRACING TRITONSERVER_InferenceTraceLevel ParseTraceLevelOption(std::string arg); InferenceTraceMode ParseTraceModeOption(std::string arg); std::tuple<std::string, std::string, std::string> ParseTraceConfigOption( const std::string& arg); // Helper functions for post processing for collected trace arguments. void SetGlobalTraceArgs( TritonServerParameters& lparams, bool trace_level_present, bool trace_rate_present, bool trace_count_present, bool explicit_disable_trace); void SetTritonTraceArgs( TritonServerParameters& lparams, bool trace_filepath_present, bool trace_log_frequency_present); void SetOpenTelemetryTraceArgs( TritonServerParameters& lparams, bool trace_filepath_present, bool trace_log_frequency_present); void PostProcessTraceArgs( TritonServerParameters& lparams, bool trace_level_present, bool trace_rate_present, bool trace_count_present, bool trace_filepath_present, bool trace_log_frequency_present, bool explicit_disable_trace); void ProcessOpenTelemetryBatchSpanProcessorArgs( TraceConfig& otel_trace_settings); #endif // TRITON_ENABLE_TRACING // Helper function to parse option in // "<string>[1st_delim]<string>[2nd_delim]<string>" format std::tuple<std::string, std::string, std::string> ParseGenericConfigOption( const std::string& arg, const std::string& first_delim, const std::string& second_delim, const std::string& option_name, const std::string& config_name); // Initialize individual option groups void SetupOptions(); // Initialize option group mappings void SetupOptionGroups(); // Sum of option groups: vector to maintain insertion order for Usage() std::vector<std::pair<std::string, std::vector<Option>&>> option_groups_; // Individual option groups std::vector<Option> global_options_; std::vector<Option> server_options_; std::vector<Option> model_repo_options_; std::vector<Option> logging_options_; std::vector<Option> http_options_; std::vector<Option> grpc_options_; std::vector<Option> sagemaker_options_; std::vector<Option> vertex_options_; std::vector<Option> metric_options_; std::vector<Option> tracing_options_; std::vector<Option> backend_options_; std::vector<Option> repo_agent_options_; std::vector<Option> cache_options_; std::vector<Option> rate_limiter_options_; std::vector<Option> memory_device_options_; // Group deprecated options to keep preferred options more succinct std::vector<Option> deprecated_options_; }; }} // namespace triton::server ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/common.cc���������������������������������������������������������������������0000664�0000000�0000000�00000006372�15270523712�0016233�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common.h" #include <algorithm> #include <iterator> #include "restricted_features.h" #include "triton/core/tritonserver.h" namespace triton { namespace server { TRITONSERVER_Error* GetModelVersionFromString(const std::string& version_string, int64_t* version) { if (version_string.empty()) { *version = -1; return nullptr; // success } try { *version = std::stol(version_string); } catch (std::exception& e) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "failed to get model version from specified version string '" + version_string + "' (details: " + e.what() + "), version should be an integral value > 0") .c_str()); } if (*version < 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "invalid model version specified '" + version_string + "' , version should be an integral value > 0") .c_str()); } return nullptr; // success } std::string GetEnvironmentVariableOrDefault( const std::string& variable_name, const std::string& default_value) { const char* value = getenv(variable_name.c_str()); return value ? value : default_value; } int64_t GetElementCount(const std::vector<int64_t>& dims) { bool first = true; int64_t cnt = 0; for (auto dim : dims) { if (dim == WILDCARD_DIM) { return -1; } if (first) { cnt = dim; first = false; } else { cnt *= dim; } } return cnt; } bool Contains(const std::vector<std::string>& vec, const std::string& str) { return std::find(vec.begin(), vec.end(), str) != vec.end(); } }} // namespace triton::server ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/common.h����������������������������������������������������������������������0000664�0000000�0000000�00000020144�15270523712�0016066�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <iostream> #include <sstream> #include <string> #include <vector> #include "triton/core/tritonserver.h" namespace triton { namespace server { constexpr char kInferHeaderContentLengthHTTPHeader[] = "Inference-Header-Content-Length"; constexpr char kAcceptEncodingHTTPHeader[] = "Accept-Encoding"; constexpr char kContentEncodingHTTPHeader[] = "Content-Encoding"; constexpr char kContentTypeHeader[] = "Content-Type"; constexpr char kContentLengthHeader[] = "Content-Length"; constexpr int MAX_GRPC_MESSAGE_SIZE = INT32_MAX; /// The value for a dimension in a shape that indicates that that /// dimension can take on any size. constexpr int WILDCARD_DIM = -1; /// Request parameter keys that start with a "triton_" prefix for internal use const std::vector<std::string> TRITON_RESERVED_REQUEST_PARAMS{ "triton_enable_empty_final_response"}; #define RETURN_IF_ERR(X) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ return err__; \ } \ } while (false) #define RETURN_MSG_IF_ERR(X, MSG) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ auto new_err = TRITONSERVER_ErrorNew( \ TRITONSERVER_ErrorCode(err__), \ (std::string(MSG) + ": " + TRITONSERVER_ErrorMessage(err__)) \ .c_str()); \ TRITONSERVER_ErrorDelete(err__); \ return new_err; \ } \ } while (false) #define GOTO_IF_ERR(X, T) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ goto T; \ } \ } while (false) #define FAIL(MSG) \ do { \ std::cerr << "error: " << (MSG) << std::endl; \ exit(1); \ } while (false) #define FAIL_IF_ERR(X, MSG) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ std::cerr << "error: " << (MSG) << ": " \ << TRITONSERVER_ErrorCodeString(err__) << " - " \ << TRITONSERVER_ErrorMessage(err__) << std::endl; \ TRITONSERVER_ErrorDelete(err__); \ exit(1); \ } \ } while (false) #define THROW_IF_ERR(EX_TYPE, X, MSG) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ auto ex__ = (EX_TYPE)(std::string("error: ") + (MSG) + ": " + \ TRITONSERVER_ErrorCodeString(err__) + " - " + \ TRITONSERVER_ErrorMessage(err__)); \ TRITONSERVER_ErrorDelete(err__); \ throw ex__; \ } \ } while (false) #define IGNORE_ERR(X) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ TRITONSERVER_ErrorDelete(err__); \ } \ } while (false) #ifdef TRITON_ENABLE_GPU #define FAIL_IF_CUDA_ERR(X, MSG) \ do { \ cudaError_t err__ = (X); \ if (err__ != cudaSuccess) { \ std::cerr << "error: " << (MSG) << ": " << cudaGetErrorString(err__) \ << std::endl; \ exit(1); \ } \ } while (false) #endif // TRITON_ENABLE_GPU /// Get the integral version from a string, or fail if string does not /// represent a valid version. /// /// \param version_string The string version. /// \param version Returns the integral version. /// \return The error status. Failure if 'version_string' doesn't /// convert to valid version. TRITONSERVER_Error* GetModelVersionFromString( const std::string& version_string, int64_t* version); /// Get the value of the environment variable, or default value if not set /// /// \param variable_name The name of the environment variable. /// \param default_value The default value. /// \return The environment variable or the default value if not set. std::string GetEnvironmentVariableOrDefault( const std::string& variable_name, const std::string& default_value); /// Get the number of elements in a shape. /// /// \param dims The shape. /// \return The number of elements, or -1 if the number of elements /// cannot be determined because the shape contains one or more /// wildcard dimensions. int64_t GetElementCount(const std::vector<int64_t>& dims); /// Returns if 'vec' contains 'str'. /// /// \param vec The vector of strings to search. /// \param str The string to lookup. /// \return True if the str is found, false otherwise. bool Contains(const std::vector<std::string>& vec, const std::string& str); /// Joins container of strings into a single string delimited by /// 'delim'. /// /// \param container The container of strings to join. /// \param delim The delimiter to join with. /// \return The joint string. template <class T> std::string Join(const T& container, const std::string& delim) { if (container.empty()) { return ""; } std::stringstream ss; ss << container[0]; for (size_t i = 1; i < container.size(); ++i) { ss << delim << container[i]; } return ss.str(); } }} // namespace triton::server ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/data_compressor.h�������������������������������������������������������������0000664�0000000�0000000�00000030305�15270523712�0017763�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <event2/buffer.h> #include <zlib.h> #include <cassert> #include <cstring> #include <iostream> #include <memory> #include <string> #include <vector> #include "common.h" #include "triton/core/tritonserver.h" namespace triton { namespace server { // // DataCompressor // class DataCompressor { public: enum class Type { UNKNOWN, IDENTITY, GZIP, DEFLATE }; // Specialization where the source and destination buffer are stored as // evbuffer static TRITONSERVER_Error* CompressData( const Type type, evbuffer* source, evbuffer* compressed_data) { size_t expected_compressed_size = evbuffer_get_length(source); // nothing to be compressed if (expected_compressed_size == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "nothing to be compressed"); } z_stream stream; stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; switch (type) { case Type::UNKNOWN: case Type::IDENTITY: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "nothing to be compressed"); } case Type::GZIP: if (deflateInit2( &stream, Z_DEFAULT_COMPRESSION /* level */, Z_DEFLATED /* method */, 15 | 16 /* windowBits */, 8 /* memLevel */, Z_DEFAULT_STRATEGY /* strategy */) != Z_OK) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "failed to initialize state for gzip data compression"); } break; case Type::DEFLATE: { if (deflateInit(&stream, Z_DEFAULT_COMPRESSION /* level */) != Z_OK) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "failed to initialize state for deflate data compression"); } break; } } // ensure the internal state are cleaned up on function return std::unique_ptr<z_stream, decltype(&deflateEnd)> managed_stream( &stream, deflateEnd); // Get the addr and size of each chunk of memory in 'source' struct evbuffer_iovec* buffer_array = nullptr; int buffer_count = evbuffer_peek(source, -1, NULL, NULL, 0); if (buffer_count > 0) { buffer_array = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * buffer_count)); if (evbuffer_peek(source, -1, NULL, buffer_array, buffer_count) != buffer_count) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting buffers to be compressed"); } } // Reserve the same size as source for compressed data, it is less likely // that a negative compression happens. struct evbuffer_iovec current_reserved_space; RETURN_MSG_IF_ERR( AllocEVBuffer( expected_compressed_size, compressed_data, ¤t_reserved_space), "unexpected error allocating output buffer for compression: "); stream.next_out = reinterpret_cast<unsigned char*>(current_reserved_space.iov_base); stream.avail_out = expected_compressed_size; // Compress until end of 'source' for (int idx = 0; idx < buffer_count; ++idx) { stream.next_in = reinterpret_cast<unsigned char*>(buffer_array[idx].iov_base); stream.avail_in = buffer_array[idx].iov_len; // run deflate() on input until source has been read in do { // Need additional buffer if (stream.avail_out == 0) { RETURN_MSG_IF_ERR( CommitEVBuffer( compressed_data, ¤t_reserved_space, expected_compressed_size), "unexpected error committing output buffer for compression: "); RETURN_MSG_IF_ERR( AllocEVBuffer( expected_compressed_size, compressed_data, ¤t_reserved_space), "unexpected error allocating output buffer for compression: "); stream.next_out = reinterpret_cast<unsigned char*>(current_reserved_space.iov_base); stream.avail_out = expected_compressed_size; } auto flush = ((idx + 1) != buffer_count) ? Z_NO_FLUSH : Z_FINISH; auto ret = deflate(&stream, flush); if (ret == Z_STREAM_ERROR) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "encountered inconsistent stream state during compression"); } } while (stream.avail_out == 0); } // Make sure the last buffer is committed if (current_reserved_space.iov_base != nullptr) { RETURN_MSG_IF_ERR( CommitEVBuffer( compressed_data, ¤t_reserved_space, expected_compressed_size - stream.avail_out), "unexpected error committing output buffer for compression: "); } return nullptr; // success } static TRITONSERVER_Error* DecompressData( const Type type, evbuffer* source, evbuffer* decompressed_data) { size_t source_byte_size = evbuffer_get_length(source); // nothing to be decompressed if (evbuffer_get_length(source) == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "nothing to be decompressed"); } // Set reasonable size for each output buffer to be allocated size_t output_buffer_size = (source_byte_size > (1 << 20 /* 1MB */)) ? source_byte_size : (1 << 20 /* 1MB */); switch (type) { case Type::UNKNOWN: case Type::IDENTITY: { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "nothing to be decompressed"); } case Type::GZIP: case Type::DEFLATE: // zlib can automatically detect compression type { z_stream stream; stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; stream.avail_in = 0; stream.next_in = Z_NULL; if (inflateInit2(&stream, 15 | 32) != Z_OK) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "failed to initialize state for data decompression"); } // ensure the internal state are cleaned up on function return std::unique_ptr<z_stream, decltype(&inflateEnd)> managed_stream( &stream, inflateEnd); // Get the addr and size of each chunk of memory in 'source' struct evbuffer_iovec* buffer_array = nullptr; int buffer_count = evbuffer_peek(source, -1, NULL, NULL, 0); if (buffer_count > 0) { buffer_array = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * buffer_count)); if (evbuffer_peek(source, -1, NULL, buffer_array, buffer_count) != buffer_count) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting buffers to be decompressed"); } } // Reserve the same size as source for compressed data, it is less // likely that a negative compression happens. struct evbuffer_iovec current_reserved_space; RETURN_MSG_IF_ERR( AllocEVBuffer( output_buffer_size, decompressed_data, ¤t_reserved_space), "unexpected error allocating output buffer for decompression: "); stream.next_out = reinterpret_cast<unsigned char*>(current_reserved_space.iov_base); stream.avail_out = output_buffer_size; // Compress until end of 'source' for (int idx = 0; idx < buffer_count; ++idx) { stream.next_in = reinterpret_cast<unsigned char*>(buffer_array[idx].iov_base); stream.avail_in = buffer_array[idx].iov_len; // run inflate() on input until source has been read in do { // Need additional buffer if (stream.avail_out == 0) { RETURN_MSG_IF_ERR( CommitEVBuffer( decompressed_data, ¤t_reserved_space, output_buffer_size), "unexpected error committing output buffer for " "decompression: "); RETURN_MSG_IF_ERR( AllocEVBuffer( output_buffer_size, decompressed_data, ¤t_reserved_space), "unexpected error allocating output buffer for " "decompression: "); stream.next_out = reinterpret_cast<unsigned char*>( current_reserved_space.iov_base); stream.avail_out = output_buffer_size; } auto ret = inflate(&stream, Z_NO_FLUSH); if (ret == Z_STREAM_ERROR) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "encountered inconsistent stream state during " "decompression"); } } while (stream.avail_out == 0); } // Make sure the last buffer is committed if (current_reserved_space.iov_base != nullptr) { RETURN_MSG_IF_ERR( CommitEVBuffer( decompressed_data, ¤t_reserved_space, output_buffer_size - stream.avail_out), "unexpected error committing output buffer for compression: "); } break; } } return nullptr; // success } private: static TRITONSERVER_Error* AllocEVBuffer( const size_t byte_size, evbuffer* evb, struct evbuffer_iovec* current_reserved_space) { // Reserve requested space in evbuffer... if ((evbuffer_reserve_space(evb, byte_size, current_reserved_space, 1) != 1) || (current_reserved_space->iov_len < byte_size)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "failed to reserve " + std::to_string(byte_size) + " bytes in evbuffer") .c_str()); } return nullptr; // success } static TRITONSERVER_Error* CommitEVBuffer( evbuffer* evb, struct evbuffer_iovec* current_reserved_space, const size_t filled_byte_size) { current_reserved_space->iov_len = filled_byte_size; if (evbuffer_commit_space(evb, current_reserved_space, 1) != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "failed to commit allocated evbuffer"); } current_reserved_space->iov_base = nullptr; return nullptr; // success } }; }} // namespace triton::server ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/�������������������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0015357�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/CMakeLists.txt�����������������������������������������������������������0000664�0000000�0000000�00000010552�15270523712�0020122�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add_library( grpc-endpoint-library EXCLUDE_FROM_ALL grpc_server.cc grpc_server.h grpc_handler.h grpc_utils.cc grpc_utils.h infer_handler.cc infer_handler.h stream_infer_handler.h stream_infer_handler.cc ) target_compile_features(grpc-endpoint-library PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") target_compile_options( grpc-endpoint-library PRIVATE /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor ) else() target_compile_options( grpc-endpoint-library PRIVATE -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wno-maybe-uninitialized -Werror ) endif() set_target_properties( grpc-endpoint-library PROPERTIES POSITION_INDEPENDENT_CODE ON ) target_link_libraries( grpc-endpoint-library PUBLIC proto-library # from repo-common triton-common-logging # from repo-common triton-common-table-printer # from repo-common triton-common-json # from repo-common grpc-health-library # from repo-common grpc-service-library # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core gRPC::grpc++ gRPC::grpc protobuf::libprotobuf ) target_include_directories( grpc-endpoint-library PRIVATE $<TARGET_PROPERTY:gRPC::grpc,INTERFACE_INCLUDE_DIRECTORIES> ) # FIXME when Triton support of OpenTelemetry is available on Windows # add ${OPENTELEMETRY_CPP_INCLUDE_DIRS} to above target_include_directories # JIRA DLIS-4786 if (NOT WIN32 AND ${TRITON_ENABLE_TRACING}) target_include_directories( grpc-endpoint-library PRIVATE ${OPENTELEMETRY_CPP_INCLUDE_DIRS} ) endif() target_compile_definitions( grpc-endpoint-library PRIVATE TRITON_ENABLE_GRPC=1 ) if(${TRITON_ENABLE_GPU}) target_compile_definitions( grpc-endpoint-library PRIVATE TRITON_ENABLE_GPU=1 PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} ) target_link_libraries( grpc-endpoint-library PUBLIC CUDA::cudart ) endif() # TRITON_ENABLE_GPU if(${TRITON_ENABLE_METRICS}) target_compile_definitions( grpc-endpoint-library PRIVATE TRITON_ENABLE_METRICS=1 ) endif() # TRITON_ENABLE_METRICS if(${TRITON_ENABLE_LOGGING}) target_compile_definitions( grpc-endpoint-library PRIVATE TRITON_ENABLE_LOGGING=1 ) endif() # TRITON_ENABLE_LOGGING if(${TRITON_ENABLE_STATS}) target_compile_definitions( grpc-endpoint-library PRIVATE TRITON_ENABLE_STATS=1 ) endif() # TRITON_ENABLE_STATS if(${TRITON_ENABLE_TRACING}) target_compile_definitions( grpc-endpoint-library PRIVATE TRITON_ENABLE_TRACING=1 ) endif() # TRITON_ENABLE_TRACING if(${TRITON_ENABLE_NVTX}) target_compile_definitions( grpc-endpoint-library PRIVATE TRITON_ENABLE_NVTX=1 ) endif() # TRITON_ENABLE_NVTX ������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/grpc_handler.h�����������������������������������������������������������0000664�0000000�0000000�00000003677�15270523712�0020175�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <string> namespace triton { namespace server { namespace grpc { class HandlerBase { public: virtual ~HandlerBase() = default; virtual void Start() = 0; virtual void Stop() = 0; }; class ICallData { public: virtual ~ICallData() = default; virtual bool Process(bool ok) = 0; virtual std::string Name() = 0; virtual uint64_t Id() = 0; }; }}} // namespace triton::server::grpc �����������������������������������������������������������������kytensor-2.49.0.7/src/grpc/grpc_server.cc�����������������������������������������������������������0000664�0000000�0000000�00000266042�15270523712�0020221�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "grpc_server.h" #include <google/protobuf/arena.h> #include <grpc++/alarm.h> #include <chrono> #include <condition_variable> #include <cstdint> #include <fstream> #include <list> #include <map> #include <mutex> #include <queue> #include <sstream> #include <thread> #include "../classification.h" #include "../common.h" #include "grpc++/grpc++.h" #include "grpc++/security/server_credentials.h" #include "grpc++/server.h" #include "grpc++/server_builder.h" #include "grpc++/server_context.h" #include "grpc++/support/status.h" #include "triton/common/logging.h" #include "triton/common/table_printer.h" #include "triton/core/tritonserver.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr #include "triton/common/triton_json.h" #ifdef TRITON_ENABLE_TRACING #include "../tracer.h" #endif // TRITON_ENABLE_TRACING #define REGISTER_GRPC_INFER_THREAD_COUNT 2 namespace triton { namespace server { namespace grpc { namespace { // // The server has separate handling mechanisms for inference RPCs // and non-inference RPCs. // //========================================================================= // The following section contains the handling mechanism for non-inference // RPCs. A single thread is created to handle all these requests as they // are deemed to be not performance critical. //========================================================================= template <typename ResponderType, typename RequestType, typename ResponseType> class CommonCallData : public ICallData { public: using StandardRegisterFunc = std::function<void( ::grpc::ServerContext*, RequestType*, ResponderType*, void*)>; using StandardCallbackFunc = std::function<void(RequestType&, ResponseType*, ::grpc::Status*)>; CommonCallData( const std::string& name, const uint64_t id, const StandardRegisterFunc OnRegister, const StandardCallbackFunc OnExecute, const bool async, ::grpc::ServerCompletionQueue* cq, const std::pair<std::string, std::string>& restricted_kv, const uint64_t& response_delay = 0) : name_(name), id_(id), OnRegister_(OnRegister), OnExecute_(OnExecute), async_(async), cq_(cq), responder_(&ctx_), step_(Steps::START), restricted_kv_(restricted_kv), response_delay_(response_delay) { OnRegister_(&ctx_, &request_, &responder_, this); LOG_VERBOSE(1) << "Ready for RPC '" << name_ << "', " << id_; } ~CommonCallData() { if (async_thread_.joinable()) { async_thread_.join(); } } bool Process(bool ok) override; std::string Name() override { return name_; } uint64_t Id() override { return id_; } private: void Execute(); void AddToCompletionQueue(); void WriteResponse(); bool ExecutePrecondition(); const std::string name_; const uint64_t id_; const StandardRegisterFunc OnRegister_; const StandardCallbackFunc OnExecute_; const bool async_; ::grpc::ServerCompletionQueue* cq_; ::grpc::ServerContext ctx_; ::grpc::Alarm alarm_; ResponderType responder_; RequestType request_; ResponseType response_; ::grpc::Status status_; std::thread async_thread_; Steps step_; std::pair<std::string, std::string> restricted_kv_{"", ""}; const uint64_t response_delay_; }; template <typename ResponderType, typename RequestType, typename ResponseType> bool CommonCallData<ResponderType, RequestType, ResponseType>::Process(bool rpc_ok) { LOG_VERBOSE(1) << "Process for " << name_ << ", rpc_ok=" << rpc_ok << ", " << id_ << " step " << step_; // If RPC failed on a new request then the server is shutting down // and so we should do nothing (including not registering for a new // request). If RPC failed on a non-START step then there is nothing // we can do since we one execute one step. const bool shutdown = (!rpc_ok && (step_ == Steps::START)); if (shutdown) { if (async_thread_.joinable()) { async_thread_.join(); } step_ = Steps::FINISH; } if (step_ == Steps::START) { // Start a new request to replace this one... if (!shutdown) { new CommonCallData<ResponderType, RequestType, ResponseType>( name_, id_ + 1, OnRegister_, OnExecute_, async_, cq_, restricted_kv_, response_delay_); } if (!async_) { // For synchronous calls, execute and write response // here. Execute(); WriteResponse(); } else { // For asynchronous calls, delegate the execution to another // thread. step_ = Steps::ISSUED; async_thread_ = std::thread(&CommonCallData::Execute, this); } } else if (step_ == Steps::WRITEREADY) { // Will only come here for asynchronous mode. WriteResponse(); } else if (step_ == Steps::COMPLETE) { step_ = Steps::FINISH; } return step_ != Steps::FINISH; } template <typename ResponderType, typename RequestType, typename ResponseType> void CommonCallData<ResponderType, RequestType, ResponseType>::Execute() { if (ExecutePrecondition()) { OnExecute_(request_, &response_, &status_); } else { status_ = ::grpc::Status( ::grpc::StatusCode::UNAVAILABLE, std::string("This protocol is restricted, expecting header '") + restricted_kv_.first + "'"); } step_ = Steps::WRITEREADY; if (async_) { // For asynchronous operation, need to add itself onto the completion // queue so that the response can be written once the object is // taken up next for execution. AddToCompletionQueue(); } } template <typename ResponderType, typename RequestType, typename ResponseType> bool CommonCallData<ResponderType, RequestType, ResponseType>::ExecutePrecondition() { if (!restricted_kv_.first.empty()) { const auto& metadata = ctx_.client_metadata(); const auto it = metadata.find(restricted_kv_.first); return (it != metadata.end()) && (it->second == restricted_kv_.second); } return true; } template <typename ResponderType, typename RequestType, typename ResponseType> void CommonCallData<ResponderType, RequestType, ResponseType>::AddToCompletionQueue() { alarm_.Set(cq_, gpr_now(gpr_clock_type::GPR_CLOCK_REALTIME), this); } template <typename ResponderType, typename RequestType, typename ResponseType> void CommonCallData<ResponderType, RequestType, ResponseType>::WriteResponse() { if (response_delay_ != 0) { // Will delay the write of the response by the specified time. // This can be used to test the flow where there are other // responses available to be written. LOG_VERBOSE(1) << "Delaying the write of the response by " << response_delay_ << " seconds"; std::this_thread::sleep_for(std::chrono::seconds(response_delay_)); } step_ = Steps::COMPLETE; responder_.Finish(response_, status_, this); } // // CommonHandler // // A common handler for all non-inference requests. // class CommonHandler : public HandlerBase { public: CommonHandler( const std::string& name, const std::shared_ptr<TRITONSERVER_Server>& tritonserver, const std::shared_ptr<SharedMemoryManager>& shm_manager, TraceManager* trace_manager, inference::GRPCInferenceService::AsyncService* service, ::grpc::health::v1::Health::AsyncService* health_service, ::grpc::ServerCompletionQueue* cq, const RestrictedFeatures& restricted_keys, const uint64_t response_delay); // Descriptive name of of the handler. const std::string& Name() const { return name_; } // Start handling requests. void Start() override; // Stop handling requests. void Stop() override; private: void SetUpAllRequests(); // [FIXME] turn into generated code void RegisterServerLive(); void RegisterServerReady(); void RegisterHealthCheck(); void RegisterModelReady(); void RegisterServerMetadata(); void RegisterModelMetadata(); void RegisterModelConfig(); void RegisterModelStatistics(); void RegisterTrace(); void RegisterLogging(); void RegisterSystemSharedMemoryStatus(); void RegisterSystemSharedMemoryRegister(); void RegisterSystemSharedMemoryUnregister(); void RegisterCudaSharedMemoryStatus(); void RegisterCudaSharedMemoryRegister(); void RegisterCudaSharedMemoryUnregister(); void RegisterRepositoryIndex(); void RegisterRepositoryModelLoad(); void RegisterRepositoryModelUnload(); // Set count and cumulative duration for 'RegisterModelStatistics()' template <typename PBTYPE> TRITONSERVER_Error* SetStatisticsDuration( triton::common::TritonJson::Value& statistics_json, const std::string& statistics_name, PBTYPE* mutable_statistics_duration_protobuf) const; const std::string name_; std::shared_ptr<TRITONSERVER_Server> tritonserver_; std::shared_ptr<SharedMemoryManager> shm_manager_; TraceManager* trace_manager_; inference::GRPCInferenceService::AsyncService* service_; ::grpc::health::v1::Health::AsyncService* health_service_; ::grpc::ServerCompletionQueue* cq_; std::unique_ptr<std::thread> thread_; RestrictedFeatures restricted_keys_{}; const uint64_t response_delay_ = 0; }; CommonHandler::CommonHandler( const std::string& name, const std::shared_ptr<TRITONSERVER_Server>& tritonserver, const std::shared_ptr<SharedMemoryManager>& shm_manager, TraceManager* trace_manager, inference::GRPCInferenceService::AsyncService* service, ::grpc::health::v1::Health::AsyncService* health_service, ::grpc::ServerCompletionQueue* cq, const RestrictedFeatures& restricted_keys, const uint64_t response_delay = 0) : name_(name), tritonserver_(tritonserver), shm_manager_(shm_manager), trace_manager_(trace_manager), service_(service), health_service_(health_service), cq_(cq), restricted_keys_(restricted_keys), response_delay_(response_delay) { } void CommonHandler::Start() { // Use a barrier to make sure we don't return until thread has // started. auto barrier = std::make_shared<Barrier>(2); thread_.reset(new std::thread([this, barrier] { SetUpAllRequests(); barrier->Wait(); void* tag; bool ok; while (cq_->Next(&tag, &ok)) { ICallData* call_data = static_cast<ICallData*>(tag); if (!call_data->Process(ok)) { LOG_VERBOSE(1) << "Done for " << call_data->Name() << ", " << call_data->Id(); delete call_data; } } })); barrier->Wait(); LOG_VERBOSE(1) << "Thread started for " << Name(); } void CommonHandler::Stop() { if (thread_->joinable()) { thread_->join(); } LOG_VERBOSE(1) << "Thread exited for " << Name(); } void CommonHandler::SetUpAllRequests() { // Define all the RPCs to be handled by this handler below // // Within each of the Register function, the format of RPC specification is: // 1. A OnRegister function: This will be called when the // server is ready to receive the requests for this RPC. // 2. A OnExecute function: This will be called when the // to process the request. // 3. Create a CommonCallData object with the above callback // functions // health (GRPC standard) RegisterHealthCheck(); // health (Triton) RegisterServerLive(); RegisterServerReady(); RegisterModelReady(); // Metadata RegisterServerMetadata(); RegisterModelMetadata(); // model config RegisterModelConfig(); // shared memory // system.. RegisterSystemSharedMemoryStatus(); RegisterSystemSharedMemoryRegister(); RegisterSystemSharedMemoryUnregister(); // cuda.. RegisterCudaSharedMemoryStatus(); RegisterCudaSharedMemoryRegister(); RegisterCudaSharedMemoryUnregister(); // model repository RegisterRepositoryIndex(); RegisterRepositoryModelLoad(); RegisterRepositoryModelUnload(); // statistics RegisterModelStatistics(); // trace RegisterTrace(); // logging RegisterLogging(); } void CommonHandler::RegisterServerLive() { auto OnRegisterServerLive = [this]( ::grpc::ServerContext* ctx, inference::ServerLiveRequest* request, ::grpc::ServerAsyncResponseWriter<inference::ServerLiveResponse>* responder, void* tag) { this->service_->RequestServerLive( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteServerLive = [this]( inference::ServerLiveRequest& request, inference::ServerLiveResponse* response, ::grpc::Status* status) { bool live = false; TRITONSERVER_Error* err = TRITONSERVER_ServerIsLive(tritonserver_.get(), &live); response->set_live((err == nullptr) && live); GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::HEALTH); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::ServerLiveResponse>, inference::ServerLiveRequest, inference::ServerLiveResponse>( "ServerLive", 0, OnRegisterServerLive, OnExecuteServerLive, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterServerReady() { auto OnRegisterServerReady = [this]( ::grpc::ServerContext* ctx, inference::ServerReadyRequest* request, ::grpc::ServerAsyncResponseWriter<inference::ServerReadyResponse>* responder, void* tag) { this->service_->RequestServerReady( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteServerReady = [this]( inference::ServerReadyRequest& request, inference::ServerReadyResponse* response, ::grpc::Status* status) { bool ready = false; TRITONSERVER_Error* err = TRITONSERVER_ServerIsReady(tritonserver_.get(), &ready); response->set_ready((err == nullptr) && ready); GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::HEALTH); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::ServerReadyResponse>, inference::ServerReadyRequest, inference::ServerReadyResponse>( "ServerReady", 0, OnRegisterServerReady, OnExecuteServerReady, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterHealthCheck() { auto OnRegisterHealthCheck = [this]( ::grpc::ServerContext* ctx, ::grpc::health::v1::HealthCheckRequest* request, ::grpc::ServerAsyncResponseWriter< ::grpc::health::v1::HealthCheckResponse>* responder, void* tag) { this->health_service_->RequestCheck( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteHealthCheck = [this]( ::grpc::health::v1::HealthCheckRequest& request, ::grpc::health::v1::HealthCheckResponse* response, ::grpc::Status* status) { bool live = false; TRITONSERVER_Error* err = TRITONSERVER_ServerIsReady(tritonserver_.get(), &live); auto serving_status = ::grpc::health::v1::HealthCheckResponse_ServingStatus_UNKNOWN; if (err == nullptr) { serving_status = live ? ::grpc::health::v1::HealthCheckResponse_ServingStatus_SERVING : ::grpc::health::v1:: HealthCheckResponse_ServingStatus_NOT_SERVING; } response->set_status(serving_status); GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::HEALTH); new CommonCallData< ::grpc::ServerAsyncResponseWriter< ::grpc::health::v1::HealthCheckResponse>, ::grpc::health::v1::HealthCheckRequest, ::grpc::health::v1::HealthCheckResponse>( "Check", 0, OnRegisterHealthCheck, OnExecuteHealthCheck, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterModelReady() { auto OnRegisterModelReady = [this]( ::grpc::ServerContext* ctx, inference::ModelReadyRequest* request, ::grpc::ServerAsyncResponseWriter<inference::ModelReadyResponse>* responder, void* tag) { this->service_->RequestModelReady( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteModelReady = [this]( inference::ModelReadyRequest& request, inference::ModelReadyResponse* response, ::grpc::Status* status) { bool is_ready = false; int64_t requested_model_version; auto err = GetModelVersionFromString(request.version(), &requested_model_version); if (err == nullptr) { err = TRITONSERVER_ServerModelIsReady( tritonserver_.get(), request.name().c_str(), requested_model_version, &is_ready); } response->set_ready(is_ready); GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::HEALTH); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::ModelReadyResponse>, inference::ModelReadyRequest, inference::ModelReadyResponse>( "ModelReady", 0, OnRegisterModelReady, OnExecuteModelReady, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterServerMetadata() { auto OnRegisterServerMetadata = [this]( ::grpc::ServerContext* ctx, inference::ServerMetadataRequest* request, ::grpc::ServerAsyncResponseWriter<inference::ServerMetadataResponse>* responder, void* tag) { this->service_->RequestServerMetadata( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteServerMetadata = [this]( inference::ServerMetadataRequest& request, inference::ServerMetadataResponse* response, ::grpc::Status* status) { TRITONSERVER_Message* server_metadata_message = nullptr; TRITONSERVER_Error* err = TRITONSERVER_ServerMetadata( tritonserver_.get(), &server_metadata_message); GOTO_IF_ERR(err, earlyexit); const char* buffer; size_t byte_size; err = TRITONSERVER_MessageSerializeToJson( server_metadata_message, &buffer, &byte_size); GOTO_IF_ERR(err, earlyexit); { triton::common::TritonJson::Value server_metadata_json; err = server_metadata_json.Parse(buffer, byte_size); GOTO_IF_ERR(err, earlyexit); const char* name; size_t namelen; err = server_metadata_json.MemberAsString("name", &name, &namelen); GOTO_IF_ERR(err, earlyexit); const char* version; size_t versionlen; err = server_metadata_json.MemberAsString( "version", &version, &versionlen); GOTO_IF_ERR(err, earlyexit); response->set_name(std::string(name, namelen)); response->set_version(std::string(version, versionlen)); if (server_metadata_json.Find("extensions")) { triton::common::TritonJson::Value extensions_json; err = server_metadata_json.MemberAsArray( "extensions", &extensions_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < extensions_json.ArraySize(); ++idx) { const char* ext; size_t extlen; err = extensions_json.IndexAsString(idx, &ext, &extlen); GOTO_IF_ERR(err, earlyexit); response->add_extensions(std::string(ext, extlen)); } } TRITONSERVER_MessageDelete(server_metadata_message); } earlyexit: GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::METADATA); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::ServerMetadataResponse>, inference::ServerMetadataRequest, inference::ServerMetadataResponse>( "ServerMetadata", 0, OnRegisterServerMetadata, OnExecuteServerMetadata, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterModelMetadata() { auto OnRegisterModelMetadata = [this]( ::grpc::ServerContext* ctx, inference::ModelMetadataRequest* request, ::grpc::ServerAsyncResponseWriter<inference::ModelMetadataResponse>* responder, void* tag) { this->service_->RequestModelMetadata( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteModelMetadata = [this]( inference::ModelMetadataRequest& request, inference::ModelMetadataResponse* response, ::grpc::Status* status) { int64_t requested_model_version; auto err = GetModelVersionFromString(request.version(), &requested_model_version); GOTO_IF_ERR(err, earlyexit); { TRITONSERVER_Message* model_metadata_message = nullptr; err = TRITONSERVER_ServerModelMetadata( tritonserver_.get(), request.name().c_str(), requested_model_version, &model_metadata_message); GOTO_IF_ERR(err, earlyexit); const char* buffer; size_t byte_size; err = TRITONSERVER_MessageSerializeToJson( model_metadata_message, &buffer, &byte_size); GOTO_IF_ERR(err, earlyexit); triton::common::TritonJson::Value model_metadata_json; err = model_metadata_json.Parse(buffer, byte_size); GOTO_IF_ERR(err, earlyexit); const char* name; size_t namelen; err = model_metadata_json.MemberAsString("name", &name, &namelen); GOTO_IF_ERR(err, earlyexit); response->set_name(std::string(name, namelen)); if (model_metadata_json.Find("versions")) { triton::common::TritonJson::Value versions_json; err = model_metadata_json.MemberAsArray("versions", &versions_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < versions_json.ArraySize(); ++idx) { const char* version; size_t versionlen; err = versions_json.IndexAsString(idx, &version, &versionlen); GOTO_IF_ERR(err, earlyexit); response->add_versions(std::string(version, versionlen)); } } const char* platform; size_t platformlen; err = model_metadata_json.MemberAsString( "platform", &platform, &platformlen); GOTO_IF_ERR(err, earlyexit); response->set_platform(std::string(platform, platformlen)); if (model_metadata_json.Find("inputs")) { triton::common::TritonJson::Value inputs_json; err = model_metadata_json.MemberAsArray("inputs", &inputs_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < inputs_json.ArraySize(); ++idx) { triton::common::TritonJson::Value io_json; err = inputs_json.IndexAsObject(idx, &io_json); GOTO_IF_ERR(err, earlyexit); inference::ModelMetadataResponse::TensorMetadata* io = response->add_inputs(); const char* name; size_t namelen; err = io_json.MemberAsString("name", &name, &namelen); GOTO_IF_ERR(err, earlyexit); const char* datatype; size_t datatypelen; err = io_json.MemberAsString("datatype", &datatype, &datatypelen); GOTO_IF_ERR(err, earlyexit); io->set_name(std::string(name, namelen)); io->set_datatype(std::string(datatype, datatypelen)); if (io_json.Find("shape")) { triton::common::TritonJson::Value shape_json; err = io_json.MemberAsArray("shape", &shape_json); GOTO_IF_ERR(err, earlyexit); for (size_t sidx = 0; sidx < shape_json.ArraySize(); ++sidx) { int64_t d; err = shape_json.IndexAsInt(sidx, &d); GOTO_IF_ERR(err, earlyexit); io->add_shape(d); } } } } if (model_metadata_json.Find("outputs")) { triton::common::TritonJson::Value outputs_json; err = model_metadata_json.MemberAsArray("outputs", &outputs_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < outputs_json.ArraySize(); ++idx) { triton::common::TritonJson::Value io_json; err = outputs_json.IndexAsObject(idx, &io_json); GOTO_IF_ERR(err, earlyexit); inference::ModelMetadataResponse::TensorMetadata* io = response->add_outputs(); const char* name; size_t namelen; err = io_json.MemberAsString("name", &name, &namelen); GOTO_IF_ERR(err, earlyexit); const char* datatype; size_t datatypelen; err = io_json.MemberAsString("datatype", &datatype, &datatypelen); GOTO_IF_ERR(err, earlyexit); io->set_name(std::string(name, namelen)); io->set_datatype(std::string(datatype, datatypelen)); if (io_json.Find("shape")) { triton::common::TritonJson::Value shape_json; err = io_json.MemberAsArray("shape", &shape_json); GOTO_IF_ERR(err, earlyexit); for (size_t sidx = 0; sidx < shape_json.ArraySize(); ++sidx) { int64_t d; err = shape_json.IndexAsInt(sidx, &d); GOTO_IF_ERR(err, earlyexit); io->add_shape(d); } } } } TRITONSERVER_MessageDelete(model_metadata_message); } earlyexit: GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::METADATA); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::ModelMetadataResponse>, inference::ModelMetadataRequest, inference::ModelMetadataResponse>( "ModelMetadata", 0, OnRegisterModelMetadata, OnExecuteModelMetadata, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterModelConfig() { auto OnRegisterModelConfig = [this]( ::grpc::ServerContext* ctx, inference::ModelConfigRequest* request, ::grpc::ServerAsyncResponseWriter<inference::ModelConfigResponse>* responder, void* tag) { this->service_->RequestModelConfig( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteModelConfig = [this]( inference::ModelConfigRequest& request, inference::ModelConfigResponse* response, ::grpc::Status* status) { int64_t requested_model_version; auto err = GetModelVersionFromString(request.version(), &requested_model_version); if (err == nullptr) { TRITONSERVER_Message* model_config_message = nullptr; err = TRITONSERVER_ServerModelConfig( tritonserver_.get(), request.name().c_str(), requested_model_version, 1 /* config_version */, &model_config_message); if (err == nullptr) { const char* buffer; size_t byte_size; err = TRITONSERVER_MessageSerializeToJson( model_config_message, &buffer, &byte_size); if (err == nullptr) { ::google::protobuf::util::JsonStringToMessage( ::google::protobuf::stringpiece_internal::StringPiece( buffer, (int)byte_size), response->mutable_config()); } TRITONSERVER_MessageDelete(model_config_message); } } GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::MODEL_CONFIG); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::ModelConfigResponse>, inference::ModelConfigRequest, inference::ModelConfigResponse>( "ModelConfig", 0, OnRegisterModelConfig, OnExecuteModelConfig, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterModelStatistics() { auto OnRegisterModelStatistics = [this]( ::grpc::ServerContext* ctx, inference::ModelStatisticsRequest* request, ::grpc::ServerAsyncResponseWriter<inference::ModelStatisticsResponse>* responder, void* tag) { this->service_->RequestModelStatistics( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteModelStatistics = [this]( inference::ModelStatisticsRequest& request, inference::ModelStatisticsResponse* response, ::grpc::Status* status) { #ifdef TRITON_ENABLE_STATS triton::common::TritonJson::Value model_stats_json; int64_t requested_model_version; auto err = GetModelVersionFromString(request.version(), &requested_model_version); GOTO_IF_ERR(err, earlyexit); { TRITONSERVER_Message* model_stats_message = nullptr; err = TRITONSERVER_ServerModelStatistics( tritonserver_.get(), request.name().c_str(), requested_model_version, &model_stats_message); GOTO_IF_ERR(err, earlyexit); const char* buffer; size_t byte_size; err = TRITONSERVER_MessageSerializeToJson( model_stats_message, &buffer, &byte_size); GOTO_IF_ERR(err, earlyexit); err = model_stats_json.Parse(buffer, byte_size); GOTO_IF_ERR(err, earlyexit); TRITONSERVER_MessageDelete(model_stats_message); } if (model_stats_json.Find("model_stats")) { triton::common::TritonJson::Value stats_json; err = model_stats_json.MemberAsArray("model_stats", &stats_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < stats_json.ArraySize(); ++idx) { triton::common::TritonJson::Value model_stat; err = stats_json.IndexAsObject(idx, &model_stat); GOTO_IF_ERR(err, earlyexit); auto statistics = response->add_model_stats(); const char* name; size_t namelen; err = model_stat.MemberAsString("name", &name, &namelen); GOTO_IF_ERR(err, earlyexit); const char* version; size_t versionlen; err = model_stat.MemberAsString("version", &version, &versionlen); GOTO_IF_ERR(err, earlyexit); statistics->set_name(std::string(name, namelen)); statistics->set_version(std::string(version, versionlen)); uint64_t ucnt; err = model_stat.MemberAsUInt("last_inference", &ucnt); GOTO_IF_ERR(err, earlyexit); statistics->set_last_inference(ucnt); err = model_stat.MemberAsUInt("inference_count", &ucnt); GOTO_IF_ERR(err, earlyexit); statistics->set_inference_count(ucnt); err = model_stat.MemberAsUInt("execution_count", &ucnt); GOTO_IF_ERR(err, earlyexit); statistics->set_execution_count(ucnt); { triton::common::TritonJson::Value infer_stats_json; err = model_stat.MemberAsObject("inference_stats", &infer_stats_json); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( infer_stats_json, "success", statistics->mutable_inference_stats()->mutable_success()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( infer_stats_json, "fail", statistics->mutable_inference_stats()->mutable_fail()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( infer_stats_json, "queue", statistics->mutable_inference_stats()->mutable_queue()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( infer_stats_json, "compute_input", statistics->mutable_inference_stats()->mutable_compute_input()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( infer_stats_json, "compute_infer", statistics->mutable_inference_stats()->mutable_compute_infer()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( infer_stats_json, "compute_output", statistics->mutable_inference_stats()->mutable_compute_output()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( infer_stats_json, "cache_hit", statistics->mutable_inference_stats()->mutable_cache_hit()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( infer_stats_json, "cache_miss", statistics->mutable_inference_stats()->mutable_cache_miss()); GOTO_IF_ERR(err, earlyexit); } { triton::common::TritonJson::Value responses_json; err = model_stat.MemberAsObject("response_stats", &responses_json); GOTO_IF_ERR(err, earlyexit); std::vector<std::string> keys; err = responses_json.Members(&keys); GOTO_IF_ERR(err, earlyexit); for (const auto& key : keys) { triton::common::TritonJson::Value res_json; err = responses_json.MemberAsObject(key.c_str(), &res_json); GOTO_IF_ERR(err, earlyexit); inference::InferResponseStatistics res; err = SetStatisticsDuration( res_json, "compute_infer", res.mutable_compute_infer()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( res_json, "compute_output", res.mutable_compute_output()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( res_json, "success", res.mutable_success()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration(res_json, "fail", res.mutable_fail()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( res_json, "empty_response", res.mutable_empty_response()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration(res_json, "cancel", res.mutable_cancel()); GOTO_IF_ERR(err, earlyexit); (*statistics->mutable_response_stats())[key] = std::move(res); } } { triton::common::TritonJson::Value batches_json; err = model_stat.MemberAsArray("batch_stats", &batches_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < batches_json.ArraySize(); ++idx) { triton::common::TritonJson::Value batch_stat; err = batches_json.IndexAsObject(idx, &batch_stat); GOTO_IF_ERR(err, earlyexit); auto batch_statistics = statistics->add_batch_stats(); uint64_t ucnt; err = batch_stat.MemberAsUInt("batch_size", &ucnt); GOTO_IF_ERR(err, earlyexit); batch_statistics->set_batch_size(ucnt); err = SetStatisticsDuration( batch_stat, "compute_input", batch_statistics->mutable_compute_input()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( batch_stat, "compute_infer", batch_statistics->mutable_compute_infer()); GOTO_IF_ERR(err, earlyexit); err = SetStatisticsDuration( batch_stat, "compute_output", batch_statistics->mutable_compute_output()); GOTO_IF_ERR(err, earlyexit); } } { triton::common::TritonJson::Value memory_usage_json; err = model_stat.MemberAsArray("memory_usage", &memory_usage_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < memory_usage_json.ArraySize(); ++idx) { triton::common::TritonJson::Value usage; err = memory_usage_json.IndexAsObject(idx, &usage); GOTO_IF_ERR(err, earlyexit); auto memory_usage = statistics->add_memory_usage(); { const char* type; size_t type_len; err = usage.MemberAsString("type", &type, &type_len); GOTO_IF_ERR(err, earlyexit); memory_usage->set_type(std::string(type, type_len)); } { int64_t id; err = usage.MemberAsInt("id", &id); GOTO_IF_ERR(err, earlyexit); memory_usage->set_id(id); } { uint64_t byte_size; err = usage.MemberAsUInt("byte_size", &byte_size); GOTO_IF_ERR(err, earlyexit); memory_usage->set_byte_size(byte_size); } } } } } earlyexit: GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); #else auto err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "the server does not support model statistics"); GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); #endif }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::STATISTICS); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::ModelStatisticsResponse>, inference::ModelStatisticsRequest, inference::ModelStatisticsResponse>( "ModelStatistics", 0, OnRegisterModelStatistics, OnExecuteModelStatistics, false /* async */, cq_, restricted_kv, response_delay_); } template <typename PBTYPE> TRITONSERVER_Error* CommonHandler::SetStatisticsDuration( triton::common::TritonJson::Value& statistics_json, const std::string& statistics_name, PBTYPE* mutable_statistics_duration_protobuf) const { triton::common::TritonJson::Value statistics_duration_json; RETURN_IF_ERR(statistics_json.MemberAsObject( statistics_name.c_str(), &statistics_duration_json)); uint64_t value; RETURN_IF_ERR(statistics_duration_json.MemberAsUInt("count", &value)); mutable_statistics_duration_protobuf->set_count(value); RETURN_IF_ERR(statistics_duration_json.MemberAsUInt("ns", &value)); mutable_statistics_duration_protobuf->set_ns(value); return nullptr; } void CommonHandler::RegisterTrace() { auto OnRegisterTrace = [this]( ::grpc::ServerContext* ctx, inference::TraceSettingRequest* request, ::grpc::ServerAsyncResponseWriter<inference::TraceSettingResponse>* responder, void* tag) { this->service_->RequestTraceSetting( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteTrace = [this]( inference::TraceSettingRequest& request, inference::TraceSettingResponse* response, ::grpc::Status* status) { #ifdef TRITON_ENABLE_TRACING TRITONSERVER_Error* err = nullptr; TRITONSERVER_InferenceTraceLevel level = TRITONSERVER_TRACE_LEVEL_DISABLED; uint32_t rate; int32_t count; uint32_t log_frequency; std::string filepath; InferenceTraceMode trace_mode; TraceConfigMap config_map; if (!request.model_name().empty()) { bool ready = false; GOTO_IF_ERR( TRITONSERVER_ServerModelIsReady( tritonserver_.get(), request.model_name().c_str(), -1 /* model version */, &ready), earlyexit); if (!ready) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Request for unknown model : ") + request.model_name()) .c_str()); GOTO_IF_ERR(err, earlyexit); } } // Update trace setting if (!request.settings().empty()) { TraceManager::NewSetting new_setting; { static std::string setting_name = "trace_file"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "trace file location can not be updated through network " "protocol"); GOTO_IF_ERR(err, earlyexit); } } { static std::string setting_name = "trace_level"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { if (it->second.value().size() == 0) { new_setting.clear_level_ = true; } else { for (const auto& level_str : it->second.value()) { if (level_str == "OFF") { if (it->second.value().size() == 1) { level = TRITONSERVER_TRACE_LEVEL_DISABLED; new_setting.level_ = &level; } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Expect only one trace level 'OFF' is specified"); GOTO_IF_ERR(err, earlyexit); } } else if (level_str == "TIMESTAMPS") { level = static_cast<TRITONSERVER_InferenceTraceLevel>( level | TRITONSERVER_TRACE_LEVEL_TIMESTAMPS); new_setting.level_ = &level; } else if (level_str == "TENSORS") { level = static_cast<TRITONSERVER_InferenceTraceLevel>( level | TRITONSERVER_TRACE_LEVEL_TENSORS); new_setting.level_ = &level; } } } } } { static std::string setting_name = "trace_rate"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { if (it->second.value().size() == 0) { new_setting.clear_rate_ = true; } else if (it->second.value().size() == 1) { try { rate = std::stoi(it->second.value()[0]); new_setting.rate_ = &rate; } catch (const std::invalid_argument& ia) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse '") + setting_name + "', got: " + it->second.value()[0]) .c_str()); GOTO_IF_ERR(err, earlyexit); } catch (const std::out_of_range& oor) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse '") + setting_name + "', value is out of range [ " + std::to_string(std::numeric_limits<std::uint32_t>::min()) + ", " + std::to_string(std::numeric_limits<std::uint32_t>::max()) + " ], got: " + it->second.value()[0]) .c_str()); GOTO_IF_ERR(err, earlyexit); } } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect only 1 value for '") + setting_name + "'") .c_str()); GOTO_IF_ERR(err, earlyexit); } } } { static std::string setting_name = "trace_count"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { if (it->second.value().size() == 0) { new_setting.clear_count_ = true; } else if (it->second.value().size() == 1) { try { count = std::stoi(it->second.value()[0]); if (count < TraceManager::MIN_TRACE_COUNT_VALUE) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse '") + setting_name + "'. Expecting value >= " + std::to_string(TraceManager::MIN_TRACE_COUNT_VALUE) + ", got: " + it->second.value()[0]) .c_str()); GOTO_IF_ERR(err, earlyexit); } new_setting.count_ = &count; } catch (const std::invalid_argument& ia) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse '") + setting_name + "', got: " + it->second.value()[0]) .c_str()); GOTO_IF_ERR(err, earlyexit); } catch (const std::out_of_range& oor) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse '") + setting_name + "', value is out of range [ " + std::to_string(TraceManager::MIN_TRACE_COUNT_VALUE) + ", " + std::to_string(std::numeric_limits<std::int32_t>::max()) + " ], got: " + it->second.value()[0]) .c_str()); GOTO_IF_ERR(err, earlyexit); } } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect only 1 value for '") + setting_name + "'") .c_str()); GOTO_IF_ERR(err, earlyexit); } } } { static std::string setting_name = "log_frequency"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { if (it->second.value().size() == 0) { new_setting.clear_log_frequency_ = true; } else if (it->second.value().size() == 1) { try { log_frequency = std::stoi(it->second.value()[0]); new_setting.log_frequency_ = &log_frequency; } catch (const std::invalid_argument& ia) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse '") + setting_name + "', got: " + it->second.value()[0]) .c_str()); GOTO_IF_ERR(err, earlyexit); } catch (const std::out_of_range& oor) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse '") + setting_name + "', value is out of range [ " + std::to_string(std::numeric_limits<std::uint32_t>::min()) + ", " + std::to_string(std::numeric_limits<std::uint32_t>::max()) + " ], got: " + it->second.value()[0]) .c_str()); GOTO_IF_ERR(err, earlyexit); } } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect only 1 value for '") + setting_name + "'") .c_str()); GOTO_IF_ERR(err, earlyexit); } } } err = trace_manager_->UpdateTraceSetting(request.model_name(), new_setting); GOTO_IF_ERR(err, earlyexit); } // Get current trace setting, this is needed even if the setting // has been updated above as some values may not be provided in the request. trace_manager_->GetTraceSetting( request.model_name(), &level, &rate, &count, &log_frequency, &filepath, &trace_mode, &config_map); // level { inference::TraceSettingResponse::SettingValue level_setting; if (level == TRITONSERVER_TRACE_LEVEL_DISABLED) { level_setting.add_value("OFF"); } else { if (level & TRITONSERVER_TRACE_LEVEL_TIMESTAMPS) { level_setting.add_value("TIMESTAMPS"); } if (level & TRITONSERVER_TRACE_LEVEL_TENSORS) { level_setting.add_value("TENSORS"); } } (*response->mutable_settings())["trace_level"] = level_setting; } (*response->mutable_settings())["trace_rate"].add_value( std::to_string(rate)); (*response->mutable_settings())["trace_count"].add_value( std::to_string(count)); if (trace_mode == TRACE_MODE_TRITON) { (*response->mutable_settings())["log_frequency"].add_value( std::to_string(log_frequency)); (*response->mutable_settings())["trace_file"].add_value(filepath); } (*response->mutable_settings())["trace_mode"].add_value( trace_manager_->InferenceTraceModeString(trace_mode)); { auto mode_key = std::to_string(trace_mode); auto trace_options_it = config_map.find(mode_key); if (trace_options_it != config_map.end()) { for (const auto& [key, value] : trace_options_it->second) { if ((key == "file") || (key == "log-frequency")) { continue; } std::string valueAsString; if (std::holds_alternative<std::string>(value)) { valueAsString = std::get<std::string>(value); } else if (std::holds_alternative<int>(value)) { valueAsString = std::to_string(std::get<int>(value)); } else if (std::holds_alternative<uint32_t>(value)) { valueAsString = std::to_string(std::get<uint32_t>(value)); } (*response->mutable_settings())[key].add_value(valueAsString); } } } earlyexit: GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); #else auto err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "the server does not support trace"); GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); #endif }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::TRACE); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::TraceSettingResponse>, inference::TraceSettingRequest, inference::TraceSettingResponse>( "Trace", 0, OnRegisterTrace, OnExecuteTrace, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterLogging() { auto OnRegisterLogging = [this]( ::grpc::ServerContext* ctx, inference::LogSettingsRequest* request, ::grpc::ServerAsyncResponseWriter<inference::LogSettingsResponse>* responder, void* tag) { this->service_->RequestLogSettings( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteLogging = [this]( inference::LogSettingsRequest& request, inference::LogSettingsResponse* response, ::grpc::Status* status) { #ifdef TRITON_ENABLE_LOGGING TRITONSERVER_Error* err = nullptr; // Update log settings // Server and Core repos do not have the same Logger object // Each update must be applied to both server and core repo versions if (!request.settings().empty()) { { static std::string setting_name = "log_file"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "log file location can not be updated through network protocol"); GOTO_IF_ERR(err, earlyexit); } } { static std::string setting_name = "log_info"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { const auto& log_param = it->second; if (log_param.parameter_choice_case() != inference::LogSettingsRequest_SettingValue::ParameterChoiceCase:: kBoolParam) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect boolean for '") + setting_name + "'") .c_str()); GOTO_IF_ERR(err, earlyexit); } else { bool log_info_status = it->second.bool_param(); LOG_ENABLE_INFO(log_info_status); TRITONSERVER_ServerOptionsSetLogInfo(nullptr, log_info_status); } } } { static std::string setting_name = "log_warning"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { const auto& log_param = it->second; if (log_param.parameter_choice_case() != inference::LogSettingsRequest_SettingValue::ParameterChoiceCase:: kBoolParam) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect boolean for '") + setting_name + "'") .c_str()); GOTO_IF_ERR(err, earlyexit); } else { bool log_warn_status = it->second.bool_param(); LOG_ENABLE_WARNING(log_warn_status); TRITONSERVER_ServerOptionsSetLogWarn(nullptr, log_warn_status); } } } { static std::string setting_name = "log_error"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { const auto& log_param = it->second; if (log_param.parameter_choice_case() != inference::LogSettingsRequest_SettingValue::ParameterChoiceCase:: kBoolParam) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect boolean for '") + setting_name + "'") .c_str()); GOTO_IF_ERR(err, earlyexit); } else { bool log_error_status = it->second.bool_param(); LOG_ENABLE_ERROR(log_error_status); TRITONSERVER_ServerOptionsSetLogError(nullptr, log_error_status); } } } { static std::string setting_name = "log_verbose_level"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { const auto& log_param = it->second; if (log_param.parameter_choice_case() != inference::LogSettingsRequest_SettingValue::ParameterChoiceCase:: kUint32Param) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect int32 for '") + setting_name + "'") .c_str()); GOTO_IF_ERR(err, earlyexit); } else { uint32_t verbose_level = it->second.uint32_param(); LOG_SET_VERBOSE(static_cast<int32_t>(verbose_level)); TRITONSERVER_ServerOptionsSetLogVerbose(nullptr, verbose_level); } } } { static std::string setting_name = "log_format"; auto it = request.settings().find(setting_name); if (it != request.settings().end()) { const auto& log_param = it->second; if (log_param.parameter_choice_case() != inference::LogSettingsRequest_SettingValue::ParameterChoiceCase:: kStringParam) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect string for '") + setting_name + "'") .c_str()); GOTO_IF_ERR(err, earlyexit); } else { const std::string& log_format_parse = it->second.string_param(); triton::common::Logger::Format log_format_final = triton::common::Logger::Format::kDEFAULT; if (log_format_parse == "ISO8601") { log_format_final = triton::common::Logger::Format::kISO8601; } else if (log_format_parse != "default") { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, ("invalid argument for log_format, got: " + log_format_parse) .c_str()); GOTO_IF_ERR(err, earlyexit); } LOG_SET_FORMAT(log_format_final); switch (log_format_final) { case triton::common::Logger::Format::kDEFAULT: TRITONSERVER_ServerOptionsSetLogFormat( nullptr, TRITONSERVER_LOG_DEFAULT); break; case triton::common::Logger::Format::kISO8601: TRITONSERVER_ServerOptionsSetLogFormat( nullptr, TRITONSERVER_LOG_ISO8601); break; } } } } GOTO_IF_ERR(err, earlyexit); } (*response->mutable_settings())["log_file"].set_string_param(LOG_FILE); (*response->mutable_settings())["log_info"].set_bool_param(LOG_INFO_IS_ON); (*response->mutable_settings())["log_warning"].set_bool_param( LOG_WARNING_IS_ON); (*response->mutable_settings())["log_error"].set_bool_param( LOG_ERROR_IS_ON); (*response->mutable_settings())["log_verbose_level"].set_uint32_param( LOG_VERBOSE_LEVEL); (*response->mutable_settings())["log_format"].set_string_param( LOG_FORMAT_STRING); earlyexit: GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); #else auto err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "the server does not support dynamic logging"); GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); #endif }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::LOGGING); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::LogSettingsResponse>, inference::LogSettingsRequest, inference::LogSettingsResponse>( "Logging", 0, OnRegisterLogging, OnExecuteLogging, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterSystemSharedMemoryStatus() { auto OnRegisterSystemSharedMemoryStatus = [this]( ::grpc::ServerContext* ctx, inference::SystemSharedMemoryStatusRequest* request, ::grpc::ServerAsyncResponseWriter< inference::SystemSharedMemoryStatusResponse>* responder, void* tag) { this->service_->RequestSystemSharedMemoryStatus( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteSystemSharedMemoryStatus = [this]( inference::SystemSharedMemoryStatusRequest& request, inference::SystemSharedMemoryStatusResponse* response, ::grpc::Status* status) { triton::common::TritonJson::Value shm_status_json( triton::common::TritonJson::ValueType::ARRAY); TRITONSERVER_Error* err = shm_manager_->GetStatus( request.name(), TRITONSERVER_MEMORY_CPU, &shm_status_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < shm_status_json.ArraySize(); ++idx) { triton::common::TritonJson::Value shm_region_json; err = shm_status_json.IndexAsObject(idx, &shm_region_json); GOTO_IF_ERR(err, earlyexit); const char* name; size_t namelen; err = shm_region_json.MemberAsString("name", &name, &namelen); GOTO_IF_ERR(err, earlyexit); const char* key; size_t keylen; err = shm_region_json.MemberAsString("key", &key, &keylen); GOTO_IF_ERR(err, earlyexit); uint64_t offset; err = shm_region_json.MemberAsUInt("offset", &offset); GOTO_IF_ERR(err, earlyexit); uint64_t byte_size; err = shm_region_json.MemberAsUInt("byte_size", &byte_size); GOTO_IF_ERR(err, earlyexit); inference::SystemSharedMemoryStatusResponse::RegionStatus region_status; region_status.set_name(std::string(name, namelen)); region_status.set_key(std::string(key, keylen)); region_status.set_offset(offset); region_status.set_byte_size(byte_size); (*response->mutable_regions())[name] = region_status; } earlyexit: GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::SHARED_MEMORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter< inference::SystemSharedMemoryStatusResponse>, inference::SystemSharedMemoryStatusRequest, inference::SystemSharedMemoryStatusResponse>( "SystemSharedMemoryStatus", 0, OnRegisterSystemSharedMemoryStatus, OnExecuteSystemSharedMemoryStatus, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterSystemSharedMemoryRegister() { auto OnRegisterSystemSharedMemoryRegister = [this]( ::grpc::ServerContext* ctx, inference::SystemSharedMemoryRegisterRequest* request, ::grpc::ServerAsyncResponseWriter< inference::SystemSharedMemoryRegisterResponse>* responder, void* tag) { this->service_->RequestSystemSharedMemoryRegister( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteSystemSharedMemoryRegister = [this]( inference::SystemSharedMemoryRegisterRequest& request, inference::SystemSharedMemoryRegisterResponse* response, ::grpc::Status* status) { TRITONSERVER_Error* err = shm_manager_->RegisterSystemSharedMemory( request.name(), request.key(), request.offset(), request.byte_size()); GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::SHARED_MEMORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter< inference::SystemSharedMemoryRegisterResponse>, inference::SystemSharedMemoryRegisterRequest, inference::SystemSharedMemoryRegisterResponse>( "SystemSharedMemoryRegister", 0, OnRegisterSystemSharedMemoryRegister, OnExecuteSystemSharedMemoryRegister, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterSystemSharedMemoryUnregister() { auto OnRegisterSystemSharedMemoryUnregister = [this]( ::grpc::ServerContext* ctx, inference::SystemSharedMemoryUnregisterRequest* request, ::grpc::ServerAsyncResponseWriter< inference::SystemSharedMemoryUnregisterResponse>* responder, void* tag) { this->service_->RequestSystemSharedMemoryUnregister( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteSystemSharedMemoryUnregister = [this]( inference::SystemSharedMemoryUnregisterRequest& request, inference::SystemSharedMemoryUnregisterResponse* response, ::grpc::Status* status) { TRITONSERVER_Error* err = nullptr; if (request.name().empty()) { err = shm_manager_->UnregisterAll(TRITONSERVER_MEMORY_CPU); } else { err = shm_manager_->Unregister(request.name(), TRITONSERVER_MEMORY_CPU); } GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::SHARED_MEMORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter< inference::SystemSharedMemoryUnregisterResponse>, inference::SystemSharedMemoryUnregisterRequest, inference::SystemSharedMemoryUnregisterResponse>( "SystemSharedMemoryUnregister", 0, OnRegisterSystemSharedMemoryUnregister, OnExecuteSystemSharedMemoryUnregister, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterCudaSharedMemoryStatus() { auto OnRegisterCudaSharedMemoryStatus = [this]( ::grpc::ServerContext* ctx, inference::CudaSharedMemoryStatusRequest* request, ::grpc::ServerAsyncResponseWriter< inference::CudaSharedMemoryStatusResponse>* responder, void* tag) { this->service_->RequestCudaSharedMemoryStatus( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteCudaSharedMemoryStatus = [this]( inference::CudaSharedMemoryStatusRequest& request, inference::CudaSharedMemoryStatusResponse* response, ::grpc::Status* status) { triton::common::TritonJson::Value shm_status_json( triton::common::TritonJson::ValueType::ARRAY); TRITONSERVER_Error* err = shm_manager_->GetStatus( request.name(), TRITONSERVER_MEMORY_GPU, &shm_status_json); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < shm_status_json.ArraySize(); ++idx) { triton::common::TritonJson::Value shm_region_json; err = shm_status_json.IndexAsObject(idx, &shm_region_json); GOTO_IF_ERR(err, earlyexit); const char* name; size_t namelen; err = shm_region_json.MemberAsString("name", &name, &namelen); GOTO_IF_ERR(err, earlyexit); uint64_t device_id; err = shm_region_json.MemberAsUInt("device_id", &device_id); GOTO_IF_ERR(err, earlyexit); uint64_t byte_size; err = shm_region_json.MemberAsUInt("byte_size", &byte_size); GOTO_IF_ERR(err, earlyexit); inference::CudaSharedMemoryStatusResponse::RegionStatus region_status; region_status.set_name(std::string(name, namelen)); region_status.set_device_id(device_id); region_status.set_byte_size(byte_size); (*response->mutable_regions())[name] = region_status; } earlyexit: GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::SHARED_MEMORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter< inference::CudaSharedMemoryStatusResponse>, inference::CudaSharedMemoryStatusRequest, inference::CudaSharedMemoryStatusResponse>( "CudaSharedMemoryStatus", 0, OnRegisterCudaSharedMemoryStatus, OnExecuteCudaSharedMemoryStatus, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterCudaSharedMemoryRegister() { auto OnRegisterCudaSharedMemoryRegister = [this]( ::grpc::ServerContext* ctx, inference::CudaSharedMemoryRegisterRequest* request, ::grpc::ServerAsyncResponseWriter< inference::CudaSharedMemoryRegisterResponse>* responder, void* tag) { this->service_->RequestCudaSharedMemoryRegister( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteCudaSharedMemoryRegister = [this]( inference::CudaSharedMemoryRegisterRequest& request, inference::CudaSharedMemoryRegisterResponse* response, ::grpc::Status* status) { TRITONSERVER_Error* err = nullptr; #ifdef TRITON_ENABLE_GPU err = shm_manager_->RegisterCUDASharedMemory( request.name(), reinterpret_cast<const cudaIpcMemHandle_t*>( request.raw_handle().c_str()), request.byte_size(), request.device_id()); #else err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "failed to register CUDA shared memory region: '" + request.name() + "', GPUs not supported") .c_str()); #endif // TRITON_ENABLE_GPU GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::SHARED_MEMORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter< inference::CudaSharedMemoryRegisterResponse>, inference::CudaSharedMemoryRegisterRequest, inference::CudaSharedMemoryRegisterResponse>( "CudaSharedMemoryRegister", 0, OnRegisterCudaSharedMemoryRegister, OnExecuteCudaSharedMemoryRegister, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterCudaSharedMemoryUnregister() { auto OnRegisterCudaSharedMemoryUnregister = [this]( ::grpc::ServerContext* ctx, inference::CudaSharedMemoryUnregisterRequest* request, ::grpc::ServerAsyncResponseWriter< inference::CudaSharedMemoryUnregisterResponse>* responder, void* tag) { this->service_->RequestCudaSharedMemoryUnregister( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteCudaSharedMemoryUnregister = [this]( inference::CudaSharedMemoryUnregisterRequest& request, inference::CudaSharedMemoryUnregisterResponse* response, ::grpc::Status* status) { TRITONSERVER_Error* err = nullptr; if (request.name().empty()) { err = shm_manager_->UnregisterAll(TRITONSERVER_MEMORY_GPU); } else { err = shm_manager_->Unregister(request.name(), TRITONSERVER_MEMORY_GPU); } GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::SHARED_MEMORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter< inference::CudaSharedMemoryUnregisterResponse>, inference::CudaSharedMemoryUnregisterRequest, inference::CudaSharedMemoryUnregisterResponse>( "CudaSharedMemoryUnregister", 0, OnRegisterCudaSharedMemoryUnregister, OnExecuteCudaSharedMemoryUnregister, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterRepositoryIndex() { auto OnRegisterRepositoryIndex = [this]( ::grpc::ServerContext* ctx, inference::RepositoryIndexRequest* request, ::grpc::ServerAsyncResponseWriter<inference::RepositoryIndexResponse>* responder, void* tag) { this->service_->RequestRepositoryIndex( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteRepositoryIndex = [this]( inference::RepositoryIndexRequest& request, inference::RepositoryIndexResponse* response, ::grpc::Status* status) { TRITONSERVER_Error* err = nullptr; if (request.repository_name().empty()) { uint32_t flags = 0; if (request.ready()) { flags |= TRITONSERVER_INDEX_FLAG_READY; } TRITONSERVER_Message* model_index_message = nullptr; err = TRITONSERVER_ServerModelIndex( tritonserver_.get(), flags, &model_index_message); GOTO_IF_ERR(err, earlyexit); const char* buffer; size_t byte_size; err = TRITONSERVER_MessageSerializeToJson( model_index_message, &buffer, &byte_size); GOTO_IF_ERR(err, earlyexit); triton::common::TritonJson::Value model_index_json; err = model_index_json.Parse(buffer, byte_size); GOTO_IF_ERR(err, earlyexit); err = model_index_json.AssertType( triton::common::TritonJson::ValueType::ARRAY); GOTO_IF_ERR(err, earlyexit); for (size_t idx = 0; idx < model_index_json.ArraySize(); ++idx) { triton::common::TritonJson::Value index_json; err = model_index_json.IndexAsObject(idx, &index_json); GOTO_IF_ERR(err, earlyexit); auto model_index = response->add_models(); const char* name; size_t namelen; err = index_json.MemberAsString("name", &name, &namelen); GOTO_IF_ERR(err, earlyexit); model_index->set_name(std::string(name, namelen)); if (index_json.Find("version")) { const char* version; size_t versionlen; err = index_json.MemberAsString("version", &version, &versionlen); GOTO_IF_ERR(err, earlyexit); model_index->set_version(std::string(version, versionlen)); } if (index_json.Find("state")) { const char* state; size_t statelen; err = index_json.MemberAsString("state", &state, &statelen); GOTO_IF_ERR(err, earlyexit); model_index->set_state(std::string(state, statelen)); } if (index_json.Find("reason")) { const char* reason; size_t reasonlen; err = index_json.MemberAsString("reason", &reason, &reasonlen); GOTO_IF_ERR(err, earlyexit); model_index->set_reason(std::string(reason, reasonlen)); } } TRITONSERVER_MessageDelete(model_index_message); } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "'repository_name' specification is not supported"); } earlyexit: GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::MODEL_REPOSITORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::RepositoryIndexResponse>, inference::RepositoryIndexRequest, inference::RepositoryIndexResponse>( "RepositoryIndex", 0, OnRegisterRepositoryIndex, OnExecuteRepositoryIndex, false /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterRepositoryModelLoad() { auto OnRegisterRepositoryModelLoad = [this]( ::grpc::ServerContext* ctx, inference::RepositoryModelLoadRequest* request, ::grpc::ServerAsyncResponseWriter< inference::RepositoryModelLoadResponse>* responder, void* tag) { this->service_->RequestRepositoryModelLoad( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteRepositoryModelLoad = [this]( inference::RepositoryModelLoadRequest& request, inference::RepositoryModelLoadResponse* response, ::grpc::Status* status) { TRITONSERVER_Error* err = nullptr; if (request.repository_name().empty()) { std::vector<TRITONSERVER_Parameter*> params; // WAR for the const-ness check std::vector<const TRITONSERVER_Parameter*> const_params; for (const auto& param_proto : request.parameters()) { if (param_proto.first == "config") { if (param_proto.second.parameter_choice_case() != inference::ModelRepositoryParameter::ParameterChoiceCase:: kStringParam) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("invalid value type for load parameter '") + param_proto.first + "', expected string_param.") .c_str()); break; } else { auto param = TRITONSERVER_ParameterNew( param_proto.first.c_str(), TRITONSERVER_PARAMETER_STRING, param_proto.second.string_param().c_str()); if (param != nullptr) { params.emplace_back(param); const_params.emplace_back(param); } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error on creating Triton parameter"); break; } } } else if (param_proto.first.rfind("file:", 0) == 0) { if (param_proto.second.parameter_choice_case() != inference::ModelRepositoryParameter::ParameterChoiceCase:: kBytesParam) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("invalid value type for load parameter '") + param_proto.first + "', expected bytes_param.") .c_str()); break; } else { auto param = TRITONSERVER_ParameterBytesNew( param_proto.first.c_str(), param_proto.second.bytes_param().data(), param_proto.second.bytes_param().length()); if (param != nullptr) { params.emplace_back(param); const_params.emplace_back(param); } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error on creating Triton parameter"); break; } } } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unrecognized load parameter '") + param_proto.first + "'.") .c_str()); break; } } if (err == nullptr) { err = TRITONSERVER_ServerLoadModelWithParameters( tritonserver_.get(), request.model_name().c_str(), const_params.data(), const_params.size()); } // Assumes no further 'params' access after load API returns for (auto& param : params) { TRITONSERVER_ParameterDelete(param); } } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "'repository_name' specification is not supported"); } GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::MODEL_REPOSITORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter<inference::RepositoryModelLoadResponse>, inference::RepositoryModelLoadRequest, inference::RepositoryModelLoadResponse>( "RepositoryModelLoad", 0, OnRegisterRepositoryModelLoad, OnExecuteRepositoryModelLoad, true /* async */, cq_, restricted_kv, response_delay_); } void CommonHandler::RegisterRepositoryModelUnload() { auto OnRegisterRepositoryModelUnload = [this]( ::grpc::ServerContext* ctx, inference::RepositoryModelUnloadRequest* request, ::grpc::ServerAsyncResponseWriter< inference::RepositoryModelUnloadResponse>* responder, void* tag) { this->service_->RequestRepositoryModelUnload( ctx, request, responder, this->cq_, this->cq_, tag); }; auto OnExecuteRepositoryModelUnload = [this]( inference::RepositoryModelUnloadRequest& request, inference::RepositoryModelUnloadResponse* response, ::grpc::Status* status) { TRITONSERVER_Error* err = nullptr; if (request.repository_name().empty()) { // Check if the dependent models should be removed bool unload_dependents = false; for (auto param : request.parameters()) { if (param.first.compare("unload_dependents") == 0) { const auto& unload_param = param.second; if (unload_param.parameter_choice_case() != inference::ModelRepositoryParameter::ParameterChoiceCase:: kBoolParam) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value type for 'unload_dependents' parameter, " "expected " "bool_param."); } unload_dependents = unload_param.bool_param(); break; } } if (err == nullptr) { if (unload_dependents) { err = TRITONSERVER_ServerUnloadModelAndDependents( tritonserver_.get(), request.model_name().c_str()); } else { err = TRITONSERVER_ServerUnloadModel( tritonserver_.get(), request.model_name().c_str()); } } } else { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "'repository_name' specification is not supported"); } GrpcStatusUtil::Create(status, err); TRITONSERVER_ErrorDelete(err); }; const std::pair<std::string, std::string>& restricted_kv = restricted_keys_.Get(RestrictedCategory::MODEL_REPOSITORY); new CommonCallData< ::grpc::ServerAsyncResponseWriter< inference::RepositoryModelUnloadResponse>, inference::RepositoryModelUnloadRequest, inference::RepositoryModelUnloadResponse>( "RepositoryModelUnload", 0, OnRegisterRepositoryModelUnload, OnExecuteRepositoryModelUnload, true /* async */, cq_, restricted_kv, response_delay_); } } // namespace // // Server // Server::Server( const std::shared_ptr<TRITONSERVER_Server>& tritonserver, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const Options& options) : tritonserver_(tritonserver), trace_manager_(trace_manager), shm_manager_(shm_manager), server_addr_( options.socket_.address_ + ":" + std::to_string(options.socket_.port_)) { std::shared_ptr<::grpc::ServerCredentials> credentials; const auto& ssl_options = options.ssl_; if (ssl_options.use_ssl_) { std::string key; std::string cert; std::string root; ReadFile(ssl_options.server_cert_, cert); ReadFile(ssl_options.server_key_, key); ReadFile(ssl_options.root_cert_, root); ::grpc::SslServerCredentialsOptions::PemKeyCertPair keycert = {key, cert}; ::grpc::SslServerCredentialsOptions sslOpts; sslOpts.pem_root_certs = root; sslOpts.pem_key_cert_pairs.push_back(keycert); if (ssl_options.use_mutual_auth_) { sslOpts.client_certificate_request = GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY; } credentials = ::grpc::SslServerCredentials(sslOpts); } else { credentials = ::grpc::InsecureServerCredentials(); } builder_.AddListeningPort(server_addr_, credentials, &bound_port_); builder_.SetMaxMessageSize(MAX_GRPC_MESSAGE_SIZE); builder_.RegisterService(&service_); builder_.RegisterService(&health_service_); builder_.AddChannelArgument( GRPC_ARG_ALLOW_REUSEPORT, options.socket_.reuse_port_); { // GRPC KeepAlive Docs: // https://grpc.github.io/grpc/cpp/md_doc_keepalive.html NOTE: In order to // work properly, the client-side settings should be in agreement with // server-side settings. const auto& keepalive_options = options.keep_alive_; builder_.AddChannelArgument( GRPC_ARG_KEEPALIVE_TIME_MS, keepalive_options.keepalive_time_ms_); builder_.AddChannelArgument( GRPC_ARG_KEEPALIVE_TIMEOUT_MS, keepalive_options.keepalive_timeout_ms_); builder_.AddChannelArgument( GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, keepalive_options.keepalive_permit_without_calls_); builder_.AddChannelArgument( GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, keepalive_options.http2_max_pings_without_data_); builder_.AddChannelArgument( GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, keepalive_options.http2_min_recv_ping_interval_without_data_ms_); builder_.AddChannelArgument( GRPC_ARG_HTTP2_MAX_PING_STRIKES, keepalive_options.http2_max_ping_strikes_); if (keepalive_options.max_connection_age_ms_ != 0) { builder_.AddChannelArgument( GRPC_ARG_MAX_CONNECTION_AGE_MS, keepalive_options.max_connection_age_ms_); } if (keepalive_options.max_connection_age_grace_ms_ != 0) { builder_.AddChannelArgument( GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS, keepalive_options.max_connection_age_grace_ms_); } std::vector<std::string> headers{"GRPC KeepAlive Option", "Value"}; triton::common::TablePrinter table_printer(headers); std::vector<std::string> row{ "keepalive_time_ms", std::to_string(keepalive_options.keepalive_time_ms_)}; table_printer.InsertRow(row); row = { "keepalive_timeout_ms", std::to_string(keepalive_options.keepalive_timeout_ms_)}; table_printer.InsertRow(row); row = { "keepalive_permit_without_calls", std::to_string(keepalive_options.keepalive_permit_without_calls_)}; table_printer.InsertRow(row); row = { "http2_max_pings_without_data", std::to_string(keepalive_options.http2_max_pings_without_data_)}; table_printer.InsertRow(row); row = { "http2_min_recv_ping_interval_without_data_ms", std::to_string( keepalive_options.http2_min_recv_ping_interval_without_data_ms_)}; table_printer.InsertRow(row); row = { "http2_max_ping_strikes", std::to_string(keepalive_options.http2_max_ping_strikes_)}; table_printer.InsertRow(row); if (keepalive_options.max_connection_age_ms_ != 0) { row = { "max_connection_age_ms", std::to_string(keepalive_options.max_connection_age_ms_)}; table_printer.InsertRow(row); } if (keepalive_options.max_connection_age_grace_ms_ != 0) { row = { "max_connection_age_grace_ms", std::to_string(keepalive_options.max_connection_age_grace_ms_)}; table_printer.InsertRow(row); } LOG_TABLE_VERBOSE(1, table_printer); } common_cq_ = builder_.AddCompletionQueue(); model_infer_cq_ = builder_.AddCompletionQueue(); model_stream_infer_cq_ = builder_.AddCompletionQueue(); // For testing purposes only, add artificial delay in grpc responses. const char* dstr = getenv("TRITONSERVER_SERVER_DELAY_GRPC_RESPONSE_SEC"); uint64_t response_delay = 0; if (dstr != nullptr) { response_delay = atoi(dstr); } // A common Handler for other non-inference requests common_handler_.reset(new CommonHandler( "CommonHandler", tritonserver_, shm_manager_, trace_manager_, &service_, &health_service_, common_cq_.get(), options.restricted_protocols_, response_delay)); // [FIXME] "register" logic is different for infer // Handler for model inference requests. std::pair<std::string, std::string> restricted_kv = options.restricted_protocols_.Get(RestrictedCategory::INFERENCE); for (int i = 0; i < REGISTER_GRPC_INFER_THREAD_COUNT; ++i) { model_infer_handlers_.emplace_back(new ModelInferHandler( "ModelInferHandler", tritonserver_, trace_manager_, shm_manager_, &service_, model_infer_cq_.get(), options.infer_allocation_pool_size_ /* max_state_bucket_count */, options.infer_compression_level_, restricted_kv, options.forward_header_pattern_)); } // Handler for streaming inference requests. Keeps one handler for streaming // to avoid possible concurrent writes which is not allowed model_stream_infer_handlers_.emplace_back(new ModelStreamInferHandler( "ModelStreamInferHandler", tritonserver_, trace_manager_, shm_manager_, &service_, model_stream_infer_cq_.get(), options.infer_allocation_pool_size_ /* max_state_bucket_count */, options.infer_compression_level_, restricted_kv, options.forward_header_pattern_)); } Server::~Server() { IGNORE_ERR(Stop()); } TRITONSERVER_Error* Server::Create( const std::shared_ptr<TRITONSERVER_Server>& tritonserver, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const Options& server_options, std::unique_ptr<Server>* server) { const std::string addr = server_options.socket_.address_ + ":" + std::to_string(server_options.socket_.port_); try { server->reset( new Server(tritonserver, trace_manager, shm_manager, server_options)); } catch (const std::invalid_argument& pe) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, pe.what()); ; } return nullptr; // success } TRITONSERVER_Error* Server::Start() { if (running_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_ALREADY_EXISTS, "GRPC server is already running."); } server_ = builder_.BuildAndStart(); // Check if binding port failed if (bound_port_ == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, (std::string("Socket '") + server_addr_ + "' already in use ").c_str()); } common_handler_->Start(); for (auto& model_infer_handler : model_infer_handlers_) { model_infer_handler->Start(); } for (auto& model_stream_infer_handler : model_stream_infer_handlers_) { model_stream_infer_handler->Start(); } running_ = true; LOG_INFO << "Started GRPCInferenceService at " << server_addr_; return nullptr; // success } TRITONSERVER_Error* Server::Stop() { if (!running_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "GRPC server is not running."); } // Always shutdown the completion queue after the server. server_->Shutdown(); common_cq_->Shutdown(); model_infer_cq_->Shutdown(); model_stream_infer_cq_->Shutdown(); // Must stop all handlers explicitly to wait for all the handler // threads to join since they are referencing completion queue, etc. common_handler_->Stop(); for (auto& model_infer_handler : model_infer_handlers_) { model_infer_handler->Stop(); } for (auto& model_stream_infer_handler : model_stream_infer_handlers_) { model_stream_infer_handler->Stop(); } running_ = false; return nullptr; // success } }}} // namespace triton::server::grpc ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/grpc_server.h������������������������������������������������������������0000664�0000000�0000000�00000012152�15270523712�0020052�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <grpc++/grpc++.h> #include <vector> #include "../restricted_features.h" #include "../shared_memory_manager.h" #include "../tracer.h" #include "grpc_handler.h" #include "grpc_service.grpc.pb.h" #include "grpc_utils.h" #include "health.grpc.pb.h" #include "infer_handler.h" #include "stream_infer_handler.h" #include "triton/core/tritonserver.h" namespace triton { namespace server { namespace grpc { // GRPC uses HTTP2 which requires header to be in lowercase, so the Triton // specific header that may be set for GRPC is defined to be all lowercases constexpr char kRestrictedProtocolHeaderTemplate[] = "triton-grpc-protocol-"; struct SocketOptions { std::string address_{"0.0.0.0"}; int32_t port_{8001}; bool reuse_port_{false}; }; struct SslOptions { // Whether SSL is used for communication bool use_ssl_{false}; // File holding PEM-encoded server certificate std::string server_cert_{""}; // File holding PEM-encoded server key std::string server_key_{""}; // File holding PEM-encoded root certificate std::string root_cert_{""}; // Whether to use Mutual Authentication bool use_mutual_auth_{false}; }; // GRPC KeepAlive: https://grpc.github.io/grpc/cpp/md_doc_keepalive.html // https://grpc.io/docs/guides/keepalive/ struct KeepAliveOptions { int keepalive_time_ms_{7200000}; int keepalive_timeout_ms_{20000}; bool keepalive_permit_without_calls_{false}; int http2_max_pings_without_data_{2}; int http2_min_recv_ping_interval_without_data_ms_{300000}; int http2_max_ping_strikes_{2}; int max_connection_age_ms_{0}; int max_connection_age_grace_ms_{0}; }; struct Options { SocketOptions socket_; SslOptions ssl_; KeepAliveOptions keep_alive_; grpc_compression_level infer_compression_level_{GRPC_COMPRESS_LEVEL_NONE}; // The maximum number of inference request/response objects that // remain allocated for reuse. As long as the number of in-flight // requests doesn't exceed this value there will be no // allocation/deallocation of request/response objects. int infer_allocation_pool_size_{8}; RestrictedFeatures restricted_protocols_; std::string forward_header_pattern_; }; class Server { public: static TRITONSERVER_Error* Create( const std::shared_ptr<TRITONSERVER_Server>& tritonserver, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const Options& server_options, std::unique_ptr<Server>* server); ~Server(); TRITONSERVER_Error* Start(); TRITONSERVER_Error* Stop(); private: Server( const std::shared_ptr<TRITONSERVER_Server>& tritonserver, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const Options& server_options); std::shared_ptr<TRITONSERVER_Server> tritonserver_; TraceManager* trace_manager_; std::shared_ptr<SharedMemoryManager> shm_manager_; const std::string server_addr_; ::grpc::ServerBuilder builder_; inference::GRPCInferenceService::AsyncService service_; ::grpc::health::v1::Health::AsyncService health_service_; std::unique_ptr<::grpc::Server> server_; std::unique_ptr<::grpc::ServerCompletionQueue> common_cq_; std::unique_ptr<::grpc::ServerCompletionQueue> model_infer_cq_; std::unique_ptr<::grpc::ServerCompletionQueue> model_stream_infer_cq_; std::unique_ptr<HandlerBase> common_handler_; std::vector<std::unique_ptr<HandlerBase>> model_infer_handlers_; std::vector<std::unique_ptr<HandlerBase>> model_stream_infer_handlers_; int bound_port_{0}; bool running_{false}; }; }}} // namespace triton::server::grpc ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/grpc_utils.cc������������������������������������������������������������0000664�0000000�0000000�00000011605�15270523712�0020044�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "grpc_utils.h" namespace triton { namespace server { namespace grpc { std::ostream& operator<<(std::ostream& out, const Steps& step) { switch (step) { case START: out << "START"; break; case COMPLETE: out << "COMPLETE"; break; case FINISH: out << "FINISH"; break; case ISSUED: out << "ISSUED"; break; case READ: out << "READ"; break; case WRITEREADY: out << "WRITEREADY"; break; case WRITTEN: out << "WRITTEN"; break; case WAITING_NOTIFICATION: out << "WAITING_NOTIFICATION"; break; case CANCELLATION_ISSUED: out << "CANCELLATION_ISSUED"; break; case CANCELLED: out << "CANCELLED"; break; case PARTIAL_COMPLETION: out << "PARTIAL_COMPLETION"; break; } return out; } void GrpcStatusUtil::Create(::grpc::Status* status, TRITONSERVER_Error* err) { if (err == nullptr) { *status = ::grpc::Status::OK; } else { *status = ::grpc::Status( GrpcStatusUtil::CodeToStatus(TRITONSERVER_ErrorCode(err)), TRITONSERVER_ErrorMessage(err)); } } ::grpc::StatusCode GrpcStatusUtil::CodeToStatus(TRITONSERVER_Error_Code code) { // GRPC status codes: // https://github.com/grpc/grpc/blob/master/include/grpc/impl/codegen/status.h switch (code) { case TRITONSERVER_ERROR_UNKNOWN: return ::grpc::StatusCode::UNKNOWN; case TRITONSERVER_ERROR_INTERNAL: return ::grpc::StatusCode::INTERNAL; case TRITONSERVER_ERROR_NOT_FOUND: return ::grpc::StatusCode::NOT_FOUND; case TRITONSERVER_ERROR_INVALID_ARG: return ::grpc::StatusCode::INVALID_ARGUMENT; case TRITONSERVER_ERROR_UNAVAILABLE: return ::grpc::StatusCode::UNAVAILABLE; case TRITONSERVER_ERROR_UNSUPPORTED: return ::grpc::StatusCode::UNIMPLEMENTED; case TRITONSERVER_ERROR_ALREADY_EXISTS: return ::grpc::StatusCode::ALREADY_EXISTS; case TRITONSERVER_ERROR_CANCELLED: return ::grpc::StatusCode::CANCELLED; } return ::grpc::StatusCode::UNKNOWN; } TRITONSERVER_Error* ParseClassificationParams( const inference::ModelInferRequest::InferRequestedOutputTensor& output, bool* has_classification, uint32_t* classification_count) { *has_classification = false; const auto& class_it = output.parameters().find("classification"); if (class_it != output.parameters().end()) { *has_classification = true; const auto& param = class_it->second; if (param.parameter_choice_case() != inference::InferParameter::ParameterChoiceCase::kInt64Param) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value type for 'classification' parameter, expected " "int64_param"); } const int64_t cnt = param.int64_param(); if (cnt <= 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value for 'classification' parameter, expected >= 0"); } *classification_count = cnt; } return nullptr; // success } void ReadFile(const std::string& filename, std::string& data) { data.clear(); if (!filename.empty()) { std::ifstream file(filename.c_str(), std::ios::in); if (file.is_open()) { std::stringstream ss; ss << file.rdbuf(); file.close(); data = ss.str(); } } } }}} // namespace triton::server::grpc ���������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/grpc_utils.h�������������������������������������������������������������0000664�0000000�0000000�00000020520�15270523712�0017702�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <list> #include <memory> #include <unordered_map> #include "../classification.h" #include "../common.h" #include "../shared_memory_manager.h" #include "grpc_service.grpc.pb.h" #include "triton/common/logging.h" #include "triton/core/tritonserver.h" namespace triton { namespace server { namespace grpc { // The step of processing that the state is in. Every state must // recognize START, COMPLETE and FINISH and the others are optional. typedef enum { // This marks the starting stage of the RPC START, // This marks that RPC is complete. COMPLETE, // This marks the stage where all the notifications from the gRPC // completion queue is received and state can be safely released. FINISH, // This stage means that RPC has been issued to Triton for inference // and is waiting for the server callbacks or cancellation to be // invoked. ISSUED, // This stage means the request has been read from the network and // can be sent to Triton for execution. READ, // This stage means that the response is ready to be written back to // the network. WRITEREADY, // This stage means that response has been written completely to the // network. WRITTEN, // This marks the special stage for the state object to differentiate // the tag delivered from AsyncNotifyWhenDone() method. WAITING_NOTIFICATION, // This stage means that the cancellation for the RPC has been issued // to the server. CANCELLATION_ISSUED, // This stage marks that the state has been successfully cancelled. CANCELLED, // This is intermediary stage where the state has been been partially // completed by grpc responder Finish call or AsyncNotifyWhenDone() // notification. The other next call will move the stage to fully // complete. PARTIAL_COMPLETION } Steps; typedef enum { // No error from CORE seen yet NONE, // Error from CORE encountered, waiting to be picked up by completion queue to // initiate cancellation ERROR_ENCOUNTERED, // Error from CORE encountered, stream closed // This state is added to avoid double cancellation ERROR_HANDLING_COMPLETE } TritonGRPCErrorSteps; class gRPCErrorTracker { public: // True if set by user via header // Can be accessed without a lock, as set only once in startstream std::atomic<bool> triton_grpc_error_; // Indicates the state of triton_grpc_error, only relevant if special // triton_grpc_error feature set to true by client TritonGRPCErrorSteps grpc_stream_error_state_; // Constructor gRPCErrorTracker() : triton_grpc_error_(false), grpc_stream_error_state_(TritonGRPCErrorSteps::NONE) { } // Changes the state of grpc_stream_error_state_ to ERROR_HANDLING_COMPLETE, // indicating we have closed the stream and initiated the cancel flow void MarkGRPCErrorHandlingComplete(); // Returns true ONLY when GRPC_ERROR from CORE is waiting to be processed. bool CheckAndUpdateGRPCError(); // Marks error after it has been responded to void MarkGRPCErrorEncountered(); // Checks if error already responded to in triton_grpc_error mode bool GRPCErrorEncountered(); }; // Debugging helper std::ostream& operator<<(std::ostream& out, const Steps& step); // // GrpcStatusUtil // class GrpcStatusUtil { public: static void Create(::grpc::Status* status, TRITONSERVER_Error* err); static ::grpc::StatusCode CodeToStatus(TRITONSERVER_Error_Code code); }; template <typename TensorType> TRITONSERVER_Error* ParseSharedMemoryParams( const TensorType& tensor, bool* has_shared_memory, std::string* region_name, int64_t* offset, size_t* byte_size) { *has_shared_memory = false; *offset = 0 /* default value */; const auto& region_it = tensor.parameters().find("shared_memory_region"); if (region_it != tensor.parameters().end()) { *has_shared_memory = true; const auto& infer_param = region_it->second; if (infer_param.parameter_choice_case() != inference::InferParameter::ParameterChoiceCase::kStringParam) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "invalid value type for 'shared_memory_region' parameter for " "tensor '" + tensor.name() + "', expected string_param.") .c_str()); } *region_name = infer_param.string_param(); } const auto& offset_it = tensor.parameters().find("shared_memory_offset"); if (offset_it != tensor.parameters().end()) { if (!*has_shared_memory) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "'shared_memory_offset' can not be specified without " "'shared_memory_region' parameter for tensor '" + tensor.name() + "'") .c_str()); } const auto& infer_param = offset_it->second; if (infer_param.parameter_choice_case() != inference::InferParameter::ParameterChoiceCase::kInt64Param) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "invalid value type for 'shared_memory_offset' parameter for " "tensor '" + tensor.name() + "', expected int64_param.") .c_str()); } *offset = infer_param.int64_param(); } const auto& bs_it = tensor.parameters().find("shared_memory_byte_size"); if (bs_it != tensor.parameters().end()) { if (!*has_shared_memory) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "'shared_memory_byte_size' can not be specified without " "'shared_memory_region' parameter for tensor '" + tensor.name() + "'") .c_str()); } const auto& infer_param = bs_it->second; if (infer_param.parameter_choice_case() != inference::InferParameter::ParameterChoiceCase::kInt64Param) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "invalid value type for 'shared_memory_byte_size' parameter " "for " "tensor '" + tensor.name() + "', expected int64_param.") .c_str()); } *byte_size = infer_param.int64_param(); } else { if (*has_shared_memory) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "'shared_memory_byte_size' must be specified along with " "'shared_memory_region' parameter for tensor '" + tensor.name() + "'") .c_str()); } } return nullptr; } TRITONSERVER_Error* ParseClassificationParams( const inference::ModelInferRequest::InferRequestedOutputTensor& output, bool* has_classification, uint32_t* classification_count); void ReadFile(const std::string& filename, std::string& data); }}} // namespace triton::server::grpc ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/infer_handler.cc���������������������������������������������������������0000664�0000000�0000000�00000125546�15270523712�0020503�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "infer_handler.h" #ifndef NDEBUG uint64_t NextUniqueId() { static std::atomic<uint64_t> id(0); return ++id; } #endif // NDEBUG namespace triton { namespace server { namespace grpc { TRITONSERVER_Error* OutputBufferAttributesHelper( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, const TensorShmMap& shm_map, TRITONSERVER_BufferAttributes* buffer_attributes) { // We only need to set the cuda ipc handle here. The rest of the buffer // attributes have been properly populated by triton core. if (tensor_name != nullptr) { const auto& pr = shm_map.find(tensor_name); if (pr != shm_map.end()) { if (pr->second.memory_type_ == TRITONSERVER_MEMORY_GPU) { RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetCudaIpcHandle( buffer_attributes, pr->second.cuda_ipc_handle_)); } } } return nullptr; // Success } TRITONSERVER_Error* OutputBufferQueryHelper( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t* byte_size, const TensorShmMap& shm_map, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { // Check if shared memory is used if named tensor is provided if (tensor_name != nullptr) { const auto& pr = shm_map.find(tensor_name); if (pr != shm_map.end()) { // The output is in shared memory so check that shared memory // size is at least large enough for the output, if byte size is provided if ((byte_size != nullptr) && (*byte_size > pr->second.byte_size_)) { // Don't return error yet and just set to the default properties for // GRPC buffer, error will be raised when allocation happens *memory_type = TRITONSERVER_MEMORY_CPU; *memory_type_id = 0; } else { *memory_type = pr->second.memory_type_; *memory_type_id = pr->second.memory_type_id_; } return nullptr; // Success } } // Not using shared memory so a buffer created directly in // the response protobuf will be used, and the type will be CPU. *memory_type = TRITONSERVER_MEMORY_CPU; *memory_type_id = 0; return nullptr; // Success } // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* InferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { AllocPayload<inference::ModelInferResponse>* payload = reinterpret_cast<AllocPayload<inference::ModelInferResponse>*>(userp); // ModelInfer RPC expects exactly one response per request. Hence, // will be creating and using just one response object. inference::ModelInferResponse* response = payload->response_queue_->GetNonDecoupledResponse(); return ResponseAllocatorHelper( allocator, tensor_name, byte_size, preferred_memory_type, preferred_memory_type_id, response, payload->shm_map_, buffer, buffer_userp, actual_memory_type, actual_memory_type_id); } // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* OutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { AllocPayload<inference::ModelInferResponse>* payload = reinterpret_cast<AllocPayload<inference::ModelInferResponse>*>(userp); return OutputBufferQueryHelper( allocator, tensor_name, byte_size, payload->shm_map_, memory_type, memory_type_id); } // Make sure to keep InferResponseAlloc, OutputBufferQuery, and // OutputBufferAttributes logic in sync TRITONSERVER_Error* OutputBufferAttributes( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, TRITONSERVER_BufferAttributes* buffer_attributes, void* userp, void* buffer_userp) { AllocPayload<inference::ModelInferResponse>* payload = reinterpret_cast<AllocPayload<inference::ModelInferResponse>*>(userp); return OutputBufferAttributesHelper( allocator, tensor_name, payload->shm_map_, buffer_attributes); return nullptr; // Success } TRITONSERVER_Error* InferResponseFree( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { LOG_VERBOSE(1) << "GRPC free: " << "size " << byte_size << ", addr " << buffer; // Don't do anything when releasing a buffer since InferResponseAlloc // wrote directly into the response protobuf. return nullptr; // Success } TRITONSERVER_Error* InferGRPCToInputHelper( const std::string& input_name, const std::string& model_name, const TRITONSERVER_DataType tensor_dt, const TRITONSERVER_DataType input_dt, const size_t binary_data_byte_size); TRITONSERVER_Error* InferGRPCToInput( const std::shared_ptr<TRITONSERVER_Server>& tritonserver, const std::shared_ptr<SharedMemoryManager>& shm_manager, const inference::ModelInferRequest& request, std::list<std::string>* serialized_data, TRITONSERVER_InferenceRequest* inference_request); TRITONSERVER_Error* InferGRPCToInputHelper( const std::string& input_name, const std::string& model_name, const TRITONSERVER_DataType tensor_dt, const TRITONSERVER_DataType input_dt, const size_t binary_data_byte_size) { if (binary_data_byte_size != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected explicit tensor data for input tensor '" + input_name + "' for model '" + model_name + "', binary data was already supplied.") .c_str()); } if (tensor_dt != input_dt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected explicit tensor data for input tensor '" + input_name + "' for model '" + model_name + "' of type '" + TRITONSERVER_DataTypeString(tensor_dt) + "', expected datatype '" + TRITONSERVER_DataTypeString(input_dt) + "'") .c_str()); } return nullptr; // success } TRITONSERVER_Error* InferResponseStart(TRITONSERVER_ResponseAllocator* allocator, void* userp) { AllocPayload<inference::ModelInferResponse>* payload = reinterpret_cast<AllocPayload<inference::ModelInferResponse>*>(userp); // ModelInfer RPC expects exactly one response per request. Hence, always call // GetNonDecoupledResponse() to create one response object on response start. payload->response_queue_->GetNonDecoupledResponse(); return nullptr; // success } TRITONSERVER_Error* SetInferenceRequestMetadata( TRITONSERVER_InferenceRequest* inference_request, const inference::ModelInferRequest& request, StateParameters& state_params) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetId( inference_request, request.id().c_str())); uint32_t flags = 0; for (auto param : request.parameters()) { if (param.first.compare("sequence_id") == 0) { const auto& infer_param = param.second; if (infer_param.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kInt64Param) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetCorrelationId( inference_request, infer_param.int64_param())); } else if ( infer_param.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kStringParam) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetCorrelationIdString( inference_request, infer_param.string_param().c_str())); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value type for 'sequence_id' parameter, expected " "int64_param or string_param."); } } else if (param.first.compare("sequence_start") == 0) { const auto& infer_param = param.second; if (infer_param.parameter_choice_case() != inference::InferParameter::ParameterChoiceCase::kBoolParam) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value type for 'sequence_start' parameter, expected " "bool_param."); } if (infer_param.bool_param()) { flags |= TRITONSERVER_REQUEST_FLAG_SEQUENCE_START; } } else if (param.first.compare("sequence_end") == 0) { const auto& infer_param = param.second; if (infer_param.parameter_choice_case() != inference::InferParameter::ParameterChoiceCase::kBoolParam) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value type for 'sequence_end' parameter, expected " "bool_param."); } if (infer_param.bool_param()) { flags |= TRITONSERVER_REQUEST_FLAG_SEQUENCE_END; } } else if (param.first.compare("priority") == 0) { const auto& infer_param = param.second; if (infer_param.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kInt64Param) { if (infer_param.int64_param() < 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value for 'priority', expected value >= 0."); } RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetPriorityUInt64( inference_request, infer_param.int64_param())); } else if ( infer_param.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kUint64Param) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetPriorityUInt64( inference_request, infer_param.uint64_param())); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value type for 'priority' parameter, expected " "int64_param or uint64_param."); } } else if (param.first.compare("timeout") == 0) { const auto& infer_param = param.second; if (infer_param.parameter_choice_case() != inference::InferParameter::ParameterChoiceCase::kInt64Param) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "invalid value type for 'timeout' parameter, expected " "int64_param."); } RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetTimeoutMicroseconds( inference_request, infer_param.int64_param())); } else if (param.first.rfind("triton_", 0) == 0) { if (!Contains(TRITON_RESERVED_REQUEST_PARAMS, param.first)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "parameter keys starting with 'triton_' are reserved for " "Triton " "usage. Only the following keys starting with 'triton_' are " "allowed: ") + Join(TRITON_RESERVED_REQUEST_PARAMS, " ")) .c_str()); } RETURN_IF_ERR(SetStateParameterFromTritonParameter(state_params, param)); } else { const auto& infer_param = param.second; if (infer_param.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kInt64Param) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetIntParameter( inference_request, param.first.c_str(), infer_param.int64_param())); } else if ( infer_param.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kBoolParam) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetBoolParameter( inference_request, param.first.c_str(), infer_param.bool_param())); } else if ( infer_param.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kStringParam) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetStringParameter( inference_request, param.first.c_str(), infer_param.string_param().c_str())); } else if ( infer_param.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kDoubleParam) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetDoubleParameter( inference_request, param.first.c_str(), infer_param.double_param())); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "invalid value type for '" + param.first + "' parameter, expected " "int64_param, bool_param, or string_param.") .c_str()); } } } RETURN_IF_ERR( TRITONSERVER_InferenceRequestSetFlags(inference_request, flags)); for (const auto& input : request.inputs()) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput( inference_request, input.name().c_str(), TRITONSERVER_StringToDataType(input.datatype().c_str()), input.shape().data(), input.shape_size())); } for (const auto& output : request.outputs()) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddRequestedOutput( inference_request, output.name().c_str())); } return nullptr; // Success } TRITONSERVER_Error* SetStateParameterFromTritonParameter( StateParameters& state_params, const std::pair<std::string, inference::InferParameter>& param) { const auto& key = param.first; const auto& value = param.second; if (key == "triton_enable_empty_final_response") { if (value.parameter_choice_case() != inference::InferParameter::ParameterChoiceCase::kBoolParam) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("invalid value type for '") + key + std::string("' parameter, expected bool_param.")) .c_str()); } state_params.enable_empty_final_response_ = value.bool_param(); } return nullptr; // success } TRITONSERVER_Error* InferGRPCToInput( const std::shared_ptr<TRITONSERVER_Server>& tritonserver, const std::shared_ptr<SharedMemoryManager>& shm_manager, const inference::ModelInferRequest& request, std::list<std::string>* serialized_data, TRITONSERVER_InferenceRequest* inference_request) { // Verify that the batch-byte-size of each input matches the size of // the provided tensor data (provided raw or from shared memory) int index = 0; for (const auto& io : request.inputs()) { const void* base; size_t byte_size = 0; TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; int64_t memory_type_id = 0; std::string region_name; int64_t offset; bool has_shared_memory; RETURN_IF_ERR( ParseSharedMemoryParams<inference::ModelInferRequest::InferInputTensor>( io, &has_shared_memory, ®ion_name, &offset, &byte_size)); TRITONSERVER_BufferAttributes* buffer_attributes; RETURN_IF_ERR(TRITONSERVER_BufferAttributesNew(&buffer_attributes)); auto buffer_attributes_del = [](TRITONSERVER_BufferAttributes* buffer_attributes) { TRITONSERVER_BufferAttributesDelete(buffer_attributes); }; std::unique_ptr< TRITONSERVER_BufferAttributes, decltype(buffer_attributes_del)> buffer_attrsl(buffer_attributes, buffer_attributes_del); char* cuda_ipc_handle = nullptr; if (has_shared_memory) { if (io.has_contents()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected 'content' provided when using shared memory " "for " "input tensor '" + io.name() + "' for model '" + request.model_name() + "'") .c_str()); } void* tmp; RETURN_IF_ERR(shm_manager->GetMemoryInfo( region_name, offset, byte_size, &tmp, &memory_type, &memory_type_id)); base = tmp; if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU RETURN_IF_ERR(shm_manager->GetCUDAHandle( region_name, reinterpret_cast<cudaIpcMemHandle_t**>(&cuda_ipc_handle))); #endif } } else { if (io.has_contents() && (!request.raw_input_contents().empty())) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "contents field must not be specified when using " "raw_input_contents for '" + io.name() + "' for model '" + request.model_name() + "'") .c_str()); } else if (io.has_contents()) { // Check the presence of explicit tensors TRITONSERVER_DataType dtype = TRITONSERVER_StringToDataType(io.datatype().c_str()); const size_t elem_byte_size = TRITONSERVER_DataTypeByteSize(dtype); if (io.contents().bool_contents_size() != 0) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_BOOL, dtype, byte_size)); base = (const void*)io.contents().bool_contents().data(); byte_size = io.contents().bool_contents_size() * elem_byte_size; } if (io.contents().int_contents_size() != 0) { if (dtype == TRITONSERVER_TYPE_INT8) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_INT8, dtype, byte_size)); serialized_data->emplace_back(); auto& serialized = serialized_data->back(); serialized.reserve( io.contents().int_contents_size() * elem_byte_size); for (const auto& element : io.contents().int_contents()) { // Assuming the system is little-endian, picking the // least significant byte of 32-bit integer as a // int8 element serialized.append( reinterpret_cast<const char*>(&element), elem_byte_size); } base = serialized.c_str(); byte_size = serialized.size(); } else if (dtype == TRITONSERVER_TYPE_INT16) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_INT16, dtype, byte_size)); serialized_data->emplace_back(); auto& serialized = serialized_data->back(); serialized.reserve( io.contents().int_contents_size() * elem_byte_size); for (const auto& element : io.contents().int_contents()) { // Assuming the system is little-endian, picking the // least 2 significant bytes of 32-bit integer as a // int16 element serialized.append( reinterpret_cast<const char*>(&element), elem_byte_size); } base = serialized.c_str(); byte_size = serialized.size(); } else { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_INT32, dtype, byte_size)); base = (const void*)io.contents().int_contents().data(); byte_size = io.contents().int_contents_size() * elem_byte_size; } } if (io.contents().int64_contents_size() != 0) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_INT64, dtype, byte_size)); base = (const void*)io.contents().int64_contents().data(); byte_size = io.contents().int64_contents_size() * elem_byte_size; } if (io.contents().uint_contents_size() != 0) { if (dtype == TRITONSERVER_TYPE_UINT8) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_UINT8, dtype, byte_size)); serialized_data->emplace_back(); auto& serialized = serialized_data->back(); serialized.reserve( io.contents().uint_contents_size() * elem_byte_size); for (const auto& element : io.contents().uint_contents()) { // Assuming the system is little-endian, picking the // least significant byte of 32-bit unsigned integer as a // uint8 element serialized.append( reinterpret_cast<const char*>(&element), elem_byte_size); } base = serialized.c_str(); byte_size = serialized.size(); } else if (dtype == TRITONSERVER_TYPE_UINT16) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_UINT16, dtype, byte_size)); serialized_data->emplace_back(); auto& serialized = serialized_data->back(); serialized.reserve( io.contents().uint_contents_size() * elem_byte_size); for (const auto& element : io.contents().uint_contents()) { // Assuming the system is little-endian, picking the // least 2 significant bytes of 32-bit integer as a // uint16 element serialized.append( reinterpret_cast<const char*>(&element), elem_byte_size); } base = serialized.c_str(); byte_size = serialized.size(); } else { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_UINT32, dtype, byte_size)); base = (const void*)io.contents().uint_contents().data(); byte_size = io.contents().uint_contents_size() * elem_byte_size; } } if (io.contents().uint64_contents_size() != 0) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_UINT64, dtype, byte_size)); base = (const void*)io.contents().uint64_contents().data(); byte_size = io.contents().uint64_contents_size() * elem_byte_size; } if (io.contents().fp32_contents_size() != 0) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_FP32, dtype, byte_size)); base = (const void*)io.contents().fp32_contents().data(); byte_size = io.contents().fp32_contents_size() * elem_byte_size; } if (io.contents().fp64_contents_size() != 0) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_FP64, dtype, byte_size)); base = (const void*)io.contents().fp64_contents().data(); byte_size = io.contents().fp64_contents_size() * elem_byte_size; } if (io.contents().bytes_contents_size() != 0) { RETURN_IF_ERR(InferGRPCToInputHelper( io.name(), request.model_name(), TRITONSERVER_TYPE_BYTES, dtype, byte_size)); serialized_data->emplace_back(); auto& serialized = serialized_data->back(); // Serialize the output tensor strings. Each string is // serialized as a 4-byte length followed by the string itself // with no null-terminator. for (const auto& element : io.contents().bytes_contents()) { uint32_t len{(uint32_t)element.size()}; serialized.append( reinterpret_cast<const char*>(&len), sizeof(uint32_t)); if (element.size() > 0) { serialized.append(element.c_str(), len); } } base = serialized.c_str(); byte_size = serialized.size(); } } else if (request.raw_input_contents().size() > index) { // Try to read the raw contents if available const std::string& raw = request.raw_input_contents()[index++]; base = raw.c_str(); byte_size = raw.size(); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unable to find data for input tensor '" + io.name() + "' for model '" + request.model_name() + "' in request.") .c_str()); } } if (cuda_ipc_handle != nullptr) { RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetCudaIpcHandle( buffer_attributes, reinterpret_cast<void*>(cuda_ipc_handle))); } RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetMemoryType( buffer_attributes, memory_type)); RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetMemoryTypeId( buffer_attributes, memory_type_id)); RETURN_IF_ERR( TRITONSERVER_BufferAttributesSetByteSize(buffer_attributes, byte_size)); RETURN_IF_ERR( TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( inference_request, io.name().c_str(), base, buffer_attributes)); } return nullptr; // success } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { LOG_VERBOSE(1) << "ModelInferHandler::InferRequestComplete"; RequestReleasePayload* request_release_payload = static_cast<RequestReleasePayload*>(userp); if ((flags & TRITONSERVER_REQUEST_RELEASE_ALL) != 0) { delete request_release_payload; } } //=========================================================================== // The following section contains the handling mechanism for ModelInfer RPC. // This implementation is tuned towards performance and reducing latency. //=========================================================================== void ModelInferHandler::StartNewRequest() { auto context = std::make_shared<State::Context>(cq_); context->SetCompressionLevel(compression_level_); State* state = StateNew(tritonserver_.get(), context); #ifdef TRITON_ENABLE_TRACING // Can't create trace as we don't know the model to be requested, // track timestamps in 'state' state->trace_timestamps_.emplace_back( std::make_pair("GRPC_WAITREAD_START", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING service_->RequestModelInfer( state->context_->ctx_.get(), &state->request_, state->context_->responder_.get(), cq_, cq_, state); LOG_VERBOSE(1) << "New request handler for " << Name() << ", " << state->unique_id_; } bool ModelInferHandler::Process(InferHandler::State* state, bool rpc_ok) { // There are multiple handlers registered in the gRPC service. // Hence, there we can have a case where a handler thread is // making progress in the state machine for a request and the // other thread is issuing cancellation on the same request. // Need to protect the state transitions for these cases. std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); if (state->delay_process_ms_ != 0) { // Will delay the Process execution by the specified time. // This can be used to test the flow when cancellation request // issued for the request, which is still at START step. LOG_INFO << "Delaying the write of the response by " << state->delay_process_ms_ << " ms..."; std::this_thread::sleep_for( std::chrono::milliseconds(state->delay_process_ms_)); } // Handle notification for cancellation which can be raised // asynchronously if detected on the network. if (state->IsGrpcContextCancelled()) { if (rpc_ok && (state->step_ == Steps::START) && (state->context_->step_ != Steps::CANCELLED)) { #ifdef TRITON_ENABLE_TRACING // Can't create trace as we don't know the model to be requested, // track timestamps in 'state' state->trace_timestamps_.emplace_back(std::make_pair( "GRPC_WAITREAD_END", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING // Need to create a new request object here explicitly for step START, // because we will never leave this if body. Refer to PR 7325. // This is a special case for ModelInferHandler, since we have 2 threads, // and each of them can process cancellation. ModelStreamInfer has only 1 // thread, and cancellation at step START was not reproducible in a // single thread scenario. StartNewRequest(); } bool resume = state->context_->HandleCancellation(state, rpc_ok, Name()); return resume; } LOG_VERBOSE(1) << "Process for " << Name() << ", rpc_ok=" << rpc_ok << ", " << state->unique_id_ << " step " << state->step_; // We need an explicit finish indicator. Can't use 'state->step_' // because we launch an async thread that could update 'state's // step_ to be FINISH before this thread exits this function. bool finished = false; // If RPC failed on a new request then the server is shutting down // and so we should do nothing (including not registering for a new // request). If RPC failed on a non-START step then there is nothing // we can do since we one execute one step. const bool shutdown = (!rpc_ok && (state->step_ == Steps::START)); if (shutdown) { state->step_ = Steps::FINISH; finished = true; } if (state->step_ == Steps::START) { #ifdef TRITON_ENABLE_TRACING // Can't create trace as we don't know the model to be requested, // track timestamps in 'state' state->trace_timestamps_.emplace_back( std::make_pair("GRPC_WAITREAD_END", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING // Start a new request to replace this one... if (!shutdown) { StartNewRequest(); } if (ExecutePrecondition(state)) { Execute(state); } else { ::grpc::Status status = ::grpc::Status( ::grpc::StatusCode::UNAVAILABLE, std::string("This protocol is restricted, expecting header '") + restricted_kv_.first + "'"); #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_SEND_START", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING state->step_ = COMPLETE; state->context_->responder_->Finish( inference::ModelInferResponse(), status, state); } } else if (state->step_ == Steps::COMPLETE) { #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_SEND_END", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING state->step_ = Steps::FINISH; } else if (state->step_ == Steps::FINISH) { finished = true; } return !finished; } TRITONSERVER_Error* ResponseAllocatorHelper( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, inference::ModelInferResponse* response, const TensorShmMap& shm_map, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { *buffer = nullptr; *buffer_userp = nullptr; *actual_memory_type = preferred_memory_type; *actual_memory_type_id = preferred_memory_type_id; // We add an output contents even if the 'byte_size' == 0 because we // expect to have a contents for every output. inference::ModelInferResponse::InferOutputTensor* output_tensor = response->add_outputs(); output_tensor->set_name(tensor_name); std::string* raw_output = response->add_raw_output_contents(); if (byte_size > 0) { const auto& pr = shm_map.find(tensor_name); if (pr != shm_map.end()) { // The output is in shared memory so check that shared memory // size is at least large enough for the output. if (byte_size > pr->second.byte_size_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "shared memory size specified with the request for output '" + std::string(tensor_name) + "' (" + std::to_string(pr->second.byte_size_) + " bytes) should be at least " + std::to_string(byte_size) + " bytes to hold the results") .c_str()); } *buffer = const_cast<void*>(pr->second.base_); *actual_memory_type = pr->second.memory_type_; *actual_memory_type_id = pr->second.memory_type_id_; LOG_VERBOSE(1) << "GRPC: using shared-memory for '" << tensor_name << "', size: " << byte_size << ", addr: " << *buffer; return nullptr; // Success } // Not using shared memory so allocate a buffer. The buffer we // create is directly in the response protobuf so we can't // allocate any type other than CPU. // // FIXME we could use pinned CPU memory here. if (*actual_memory_type != TRITONSERVER_MEMORY_CPU) { LOG_VERBOSE(1) << "GRPC: unable to provide '" << tensor_name << "' in " << TRITONSERVER_MemoryTypeString(*actual_memory_type) << ", will use " << TRITONSERVER_MemoryTypeString(TRITONSERVER_MEMORY_CPU); *actual_memory_type = TRITONSERVER_MEMORY_CPU; *actual_memory_type_id = 0; } raw_output->resize(byte_size); *buffer = static_cast<void*>(&((*raw_output)[0])); LOG_VERBOSE(1) << "GRPC: using buffer for '" << tensor_name << "', size: " << byte_size << ", addr: " << *buffer; } return nullptr; // Success } void ModelInferHandler::Execute(InferHandler::State* state) { TRITONSERVER_Error* err = nullptr; const inference::ModelInferRequest& request = state->request_; auto response_queue = state->response_queue_; int64_t requested_model_version; if (err == nullptr) { err = GetModelVersionFromString( request.model_version(), &requested_model_version); } if (err == nullptr) { uint32_t txn_flags; err = TRITONSERVER_ServerModelTransactionProperties( tritonserver_.get(), request.model_name().c_str(), requested_model_version, &txn_flags, nullptr /* voidp */); if ((err == nullptr) && (txn_flags & TRITONSERVER_TXN_DECOUPLED) != 0) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "ModelInfer RPC doesn't support models with decoupled " "transaction policy"); } } // Create the inference request which contains all the // input information needed for an inference. TRITONSERVER_InferenceRequest* irequest = nullptr; if (err == nullptr) { err = TRITONSERVER_InferenceRequestNew( &irequest, tritonserver_.get(), request.model_name().c_str(), requested_model_version); } if (err == nullptr) { state->inference_request_ = { irequest, [](TRITONSERVER_InferenceRequest* request) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestDelete(request), "deleting gRPC inference request"); }}; err = SetInferenceRequestMetadata(irequest, request, state->parameters_); } if (err == nullptr) { err = ForwardHeadersAsParameters(irequest, state); } // Will be used to hold the serialized data in case explicit string // tensors are present in the request. std::list<std::string> serialized_data; if (err == nullptr) { err = InferGRPCToInput( tritonserver_, shm_manager_, request, &serialized_data, irequest); } if (err == nullptr) { err = InferAllocatorPayload<inference::ModelInferResponse>( tritonserver_, shm_manager_, request, std::move(serialized_data), response_queue, &state->alloc_payload_); } auto request_release_payload = std::make_unique<RequestReleasePayload>(state->inference_request_); if (err == nullptr) { err = TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestComplete, request_release_payload.get() /* request_release_userp */); } if (err == nullptr) { err = TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator_, &state->alloc_payload_ /* response_allocator_userp */, InferResponseComplete, reinterpret_cast<void*>(state)); } // Get request ID for logging in case of error. const char* request_id = ""; if (irequest != nullptr) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestId(irequest, &request_id), "unable to retrieve request ID string"); } if (!strncmp(request_id, "", 1)) { request_id = "<id_unknown>"; } if (err == nullptr) { TRITONSERVER_InferenceTrace* triton_trace = nullptr; #ifdef TRITON_ENABLE_TRACING GrpcServerCarrier carrier(state->context_->ctx_.get()); auto start_options = trace_manager_->GetTraceStartOptions(carrier, request.model_name()); state->trace_ = std::move(trace_manager_->SampleTrace(start_options)); if (state->trace_ != nullptr) { triton_trace = state->trace_->trace_; } #endif // TRITON_ENABLE_TRACING state->step_ = ISSUED; err = TRITONSERVER_ServerInferAsync( tritonserver_.get(), irequest, triton_trace); } // If not error then state->step_ == ISSUED and inference request // has initiated... completion callback will transition to // COMPLETE or CANCELLED. Recording the state and the irequest // to handle gRPC stream cancellation. if (err == nullptr) { state->context_->InsertInflightState(state); // The payload will be cleaned in request release callback. request_release_payload.release(); } else { // If error go immediately to COMPLETE. LOG_VERBOSE(1) << "[request id: " << request_id << "] " << "Infer failed: " << TRITONSERVER_ErrorMessage(err); ::grpc::Status status; GrpcStatusUtil::Create(&status, err); TRITONSERVER_ErrorDelete(err); inference::ModelInferResponse error_response; #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_SEND_START", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING state->step_ = COMPLETE; state->context_->responder_->Finish(error_response, status, state); } } void ModelInferHandler::InferResponseComplete( TRITONSERVER_InferenceResponse* iresponse, const uint32_t flags, void* userp) { State* state = reinterpret_cast<State*>(userp); // There are multiple handlers registered in the gRPC service // Hence, we would need to properly synchronize this thread // and the handler thread handling async cancellation // notification. std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); // Increment the callback index if received valid 'iresponse' if (iresponse != nullptr) { state->cb_count_++; } LOG_VERBOSE(1) << "ModelInferHandler::InferResponseComplete, " << state->unique_id_ << " step " << state->step_; // Allow sending 1 response and final flag separately, only mark // non-inflight when seeing final flag if (flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) { state->context_->EraseInflightState(state); } // If gRPC Stream is cancelled then no need of forming and returning // a response. if (state->IsGrpcContextCancelled()) { // Clean-up the received response object. LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(iresponse), "deleting GRPC inference response"); state->context_->EraseInflightState(state); state->step_ = Steps::CANCELLED; LOG_VERBOSE(1) << "ModelInferHandler::InferResponseComplete, " << state->unique_id_ << ", skipping response generation as grpc transaction was " "cancelled... "; // Send state back to the queue so that state can be released // in the next cycle. state->context_->PutTaskBackToQueue(state); return; } TRITONSERVER_Error* err = nullptr; // This callback is expected to be called exactly once for each request. // Will use the single response object in the response list to hold the // information. inference::ModelInferResponse* response = state->response_queue_->GetResponseAt(0); bool response_created = false; if (response == nullptr) { LOG_ERROR << "expected allocator to have created a response object"; err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "No response object found in the callback"); response_created = true; response = new inference::ModelInferResponse(); } if (state->cb_count_ != 1) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "expected a single response, got " + std::to_string(state->cb_count_)) .c_str()); } else if (iresponse != nullptr) { err = InferResponseCompleteCommon<inference::ModelInferResponse>( state->tritonserver_, iresponse, *response, state->alloc_payload_); #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back(std::make_pair( "INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING } if (err != nullptr) { response->Clear(); } GrpcStatusUtil::Create(&state->status_, err); TRITONSERVER_ErrorDelete(err); LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(iresponse), "deleting GRPC inference response"); // Defer sending the response until FINAL flag is seen or // there is error if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { return; } #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_SEND_START", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING state->step_ = COMPLETE; state->context_->responder_->Finish(*response, state->status_, state); if (response_created) { delete response; } } }}} // namespace triton::server::grpc ����������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/infer_handler.h����������������������������������������������������������0000664�0000000�0000000�00000145063�15270523712�0020341�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <grpc++/alarm.h> #include <grpc++/grpc++.h> #include <re2/re2.h> #include <condition_variable> #include <queue> #include <regex> #include <thread> #include "../tracer.h" #include "grpc_handler.h" #include "grpc_service.grpc.pb.h" #include "grpc_utils.h" #include "triton/common/logging.h" #include "triton/core/tritonserver.h" // Unique IDs are only needed when debugging. They only appear in // verbose logging. #ifndef NDEBUG uint64_t NextUniqueId(); #define NEXT_UNIQUE_ID NextUniqueId() #else #define NEXT_UNIQUE_ID (0) #endif // NDEBUG namespace triton { namespace server { namespace grpc { // Options used in InferHandler/StreamInferHandler states that are set from // request parameters struct StateParameters { // Whether to generate an empty response when a FINAL flag is received with // no corresponding response. Only applicable to StreamInferHandlerState. bool enable_empty_final_response_ = false; }; // // C++11 doesn't have a barrier so we implement our own. // class Barrier { public: explicit Barrier(size_t cnt) : threshold_(cnt), count_(cnt), generation_(0) {} void Wait() { std::unique_lock<std::mutex> lock(mu_); auto lgen = generation_; if (--count_ == 0) { generation_++; count_ = threshold_; cv_.notify_all(); } else { cv_.wait(lock, [this, lgen] { return lgen != generation_; }); } } private: std::mutex mu_; std::condition_variable cv_; const size_t threshold_; size_t count_; size_t generation_; }; // Simple structure that carries the userp payload needed for // request release callback. struct RequestReleasePayload final { explicit RequestReleasePayload( const std::shared_ptr<TRITONSERVER_InferenceRequest>& inference_request) : inference_request_(inference_request){}; private: std::shared_ptr<TRITONSERVER_InferenceRequest> inference_request_ = nullptr; }; // // ResponseQueue // // A simple queue holding the responses to be written. Uses a // vector of persistent message objects to prevent allocating // memory for each response to be written. // template <typename ResponseType> class ResponseQueue { public: explicit ResponseQueue() { Reset(); } ~ResponseQueue() { for (auto response : responses_) { delete response; } } // Resets the queue void Reset() { alloc_count_ = 0; ready_count_ = 0; current_index_ = 0; for (auto response : responses_) { response->Clear(); } } // Gets the response for the non-decoupled models. // Note that there will be a single response in // non-decoupled cases. ResponseType* GetNonDecoupledResponse() { std::lock_guard<std::mutex> lock(mtx_); alloc_count_ = 1; if (responses_.size() < 1) { responses_.push_back(new ResponseType()); } return responses_[0]; } // Allocates a response on the head of the queue void AllocateResponse() { std::lock_guard<std::mutex> lock(mtx_); alloc_count_++; if (responses_.size() < alloc_count_) { responses_.push_back(new ResponseType()); } } // Gets the last allocated response ResponseType* GetLastAllocatedResponse() { std::lock_guard<std::mutex> lock(mtx_); if (responses_.size() < alloc_count_) { LOG_ERROR << "[INTERNAL] Attempting to access the response not yet allocated"; return nullptr; } return responses_[alloc_count_ - 1]; } // Marks the next non-ready response complete bool MarkNextResponseComplete() { std::lock_guard<std::mutex> lock(mtx_); if (alloc_count_ <= ready_count_) { LOG_ERROR << "[INTERNAL] Attempting to mark an unallocated response complete"; return false; } ready_count_++; return true; } // Gets the current response from the tail of // the queue. ResponseType* GetCurrentResponse() { std::lock_guard<std::mutex> lock(mtx_); if (current_index_ >= ready_count_) { LOG_ERROR << "[INTERNAL] Attempting to access current response when it " "is not ready"; return nullptr; } return responses_[current_index_]; } // Gets the response at the specified index ResponseType* GetResponseAt(const uint32_t index) { std::lock_guard<std::mutex> lock(mtx_); if (index >= alloc_count_) { LOG_ERROR << "[INTERNAL] Attempting to access response which is not yet " "allocated"; return nullptr; } return responses_[index]; } // Pops the response from the tail of the queue void PopResponse() { std::lock_guard<std::mutex> lock(mtx_); current_index_++; } // Returns whether the queue is empty bool IsEmpty() { std::lock_guard<std::mutex> lock(mtx_); return ((alloc_count_ == ready_count_) && (alloc_count_ == current_index_)); } // Returns whether the queue has responses // ready to be written. bool HasReadyResponse() { std::lock_guard<std::mutex> lock(mtx_); return (ready_count_ > current_index_); } private: std::vector<ResponseType*> responses_; std::mutex mtx_; // There are three indices to track the responses in the queue // Tracks the allocated response uint32_t alloc_count_; // Tracks the response that is ready to be written uint32_t ready_count_; // Tracks the response next in the queue to be written uint32_t current_index_; }; // // ShmInfo // // Simple structure that carries the shared memory information // struct ShmInfo { ShmInfo( void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, char* cuda_ipc_handle) : base_(base), byte_size_(byte_size), memory_type_(memory_type), memory_type_id_(memory_type_id), cuda_ipc_handle_(cuda_ipc_handle) { } void* base_; size_t byte_size_; TRITONSERVER_MemoryType memory_type_; int64_t memory_type_id_; char* cuda_ipc_handle_; }; using TensorShmMap = std::unordered_map<std::string, ShmInfo>; // // AllocPayload // // Simple structure that carries the userp payload needed for // allocation. // template <typename ResponseType> struct AllocPayload { using ClassificationMap = std::unordered_map<std::string, uint32_t>; explicit AllocPayload() : response_queue_(nullptr) {} ~AllocPayload() { // Don't delete 'response_'.. it is owned by the InferHandlerState } std::shared_ptr<ResponseQueue<ResponseType>> response_queue_; uint32_t response_alloc_count_; TensorShmMap shm_map_; ClassificationMap classification_map_; // Used to extend the lifetime of the serialized data in case // non-raw contents were provided in the request. Serialized data's // actual lifetime is that of the request whereas AllocPayload's // lifetime is that of a response... but it is convenient to keep it // here. std::list<std::string> serialized_data_; }; template <typename ResponseType> TRITONSERVER_Error* InferAllocatorPayload( const std::shared_ptr<TRITONSERVER_Server>& tritonserver, const std::shared_ptr<SharedMemoryManager>& shm_manager, const inference::ModelInferRequest& request, std::list<std::string>&& serialized_data, std::shared_ptr<ResponseQueue<ResponseType>> response_queue, AllocPayload<ResponseType>* alloc_payload) { alloc_payload->response_queue_ = response_queue; alloc_payload->shm_map_.clear(); alloc_payload->classification_map_.clear(); alloc_payload->serialized_data_ = std::move(serialized_data); // If any of the outputs use shared memory, then we must calculate // the memory address for that output and store it in the allocator // payload so that it is available when the allocation callback is // invoked. for (const auto& io : request.outputs()) { std::string region_name; int64_t offset; size_t byte_size; bool has_shared_memory; RETURN_IF_ERR(ParseSharedMemoryParams< inference::ModelInferRequest::InferRequestedOutputTensor>( io, &has_shared_memory, ®ion_name, &offset, &byte_size)); bool has_classification; uint32_t classification_count; RETURN_IF_ERR(ParseClassificationParams( io, &has_classification, &classification_count)); if (has_shared_memory && has_classification) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "output can't set both 'shared_memory_region' and " "'classification'"); } if (has_shared_memory) { void* base; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; RETURN_IF_ERR(shm_manager->GetMemoryInfo( region_name, offset, byte_size, &base, &memory_type, &memory_type_id)); if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU char* cuda_handle; RETURN_IF_ERR(shm_manager->GetCUDAHandle( region_name, reinterpret_cast<cudaIpcMemHandle_t**>(&cuda_handle))); alloc_payload->shm_map_.emplace( io.name(), ShmInfo(base, byte_size, memory_type, memory_type_id, cuda_handle)); #endif } else { alloc_payload->shm_map_.emplace( io.name(), ShmInfo( base, byte_size, memory_type, memory_type_id, nullptr /* cuda_ipc_handle */)); } } else if (has_classification) { alloc_payload->classification_map_.emplace( io.name(), classification_count); } } return nullptr; // Success } TRITONSERVER_Error* InferGRPCToInputHelper( const std::string& input_name, const std::string& model_name, const TRITONSERVER_DataType tensor_dt, const TRITONSERVER_DataType input_dt, const size_t binary_data_byte_size); TRITONSERVER_Error* InferGRPCToInput( const std::shared_ptr<TRITONSERVER_Server>& tritonserver, const std::shared_ptr<SharedMemoryManager>& shm_manager, const inference::ModelInferRequest& request, std::list<std::string>* serialized_data, TRITONSERVER_InferenceRequest* inference_request); TRITONSERVER_Error* ResponseAllocatorHelper( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, inference::ModelInferResponse* response, const TensorShmMap& shm_map, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id); TRITONSERVER_Error* OutputBufferAttributesHelper( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, const TensorShmMap& shm_map, TRITONSERVER_BufferAttributes* buffer_attributes); TRITONSERVER_Error* OutputBufferQueryHelper( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t* byte_size, const TensorShmMap& shm_map, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* InferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id); TRITONSERVER_Error* SetInferenceRequestMetadata( TRITONSERVER_InferenceRequest* inference_request, const inference::ModelInferRequest& request, StateParameters& state_params); // Helper to set options for StreamInferHandler state when parsing // request parameters. TRITONSERVER_Error* SetStateParameterFromTritonParameter( StateParameters& state_params, const std::pair<std::string, inference::InferParameter>& param); void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp); // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* OutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); // Make sure to keep InferResponseAlloc, OutputBufferQuery, and // OutputBufferAttributes logic in sync TRITONSERVER_Error* OutputBufferAttributes( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, TRITONSERVER_BufferAttributes* buffer_attributes, void* userp, void* buffer_userp); TRITONSERVER_Error* InferResponseFree( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); TRITONSERVER_Error* InferResponseStart( TRITONSERVER_ResponseAllocator* allocator, void* userp); template <typename ResponseType> TRITONSERVER_Error* InferResponseCompleteCommon( TRITONSERVER_Server* server, TRITONSERVER_InferenceResponse* iresponse, inference::ModelInferResponse& response, const AllocPayload<ResponseType>& alloc_payload) { RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(iresponse)); const char *model_name, *id; int64_t model_version; RETURN_IF_ERR(TRITONSERVER_InferenceResponseModel( iresponse, &model_name, &model_version)); RETURN_IF_ERR(TRITONSERVER_InferenceResponseId(iresponse, &id)); response.set_id(id); response.set_model_name(model_name); response.set_model_version(std::to_string(model_version)); // Propagate response parameters. uint32_t parameter_count; RETURN_IF_ERR(TRITONSERVER_InferenceResponseParameterCount( iresponse, ¶meter_count)); for (uint32_t pidx = 0; pidx < parameter_count; ++pidx) { const char* name; TRITONSERVER_ParameterType type; const void* vvalue; RETURN_IF_ERR(TRITONSERVER_InferenceResponseParameter( iresponse, pidx, &name, &type, &vvalue)); inference::InferParameter& param = (*response.mutable_parameters())[name]; switch (type) { case TRITONSERVER_PARAMETER_BOOL: param.set_bool_param(*(reinterpret_cast<const bool*>(vvalue))); break; case TRITONSERVER_PARAMETER_INT: param.set_int64_param(*(reinterpret_cast<const int64_t*>(vvalue))); break; case TRITONSERVER_PARAMETER_STRING: param.set_string_param(reinterpret_cast<const char*>(vvalue)); break; case TRITONSERVER_PARAMETER_DOUBLE: param.set_double_param(*(reinterpret_cast<const double*>(vvalue))); break; case TRITONSERVER_PARAMETER_BYTES: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Response parameter of type 'TRITONSERVER_PARAMETER_BYTES' is not " "currently supported"); break; } } // Go through each response output and transfer information to the // corresponding GRPC response output. uint32_t output_count; RETURN_IF_ERR( TRITONSERVER_InferenceResponseOutputCount(iresponse, &output_count)); if (output_count != (uint32_t)response.outputs_size()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "response output count mismatch"); } for (uint32_t output_idx = 0; output_idx < output_count; ++output_idx) { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput( iresponse, output_idx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); const std::string name(cname); // There are usually very few outputs so fastest just to look for // the one we want... could create a map for cases where there are // a large number of outputs. Or rely on order to be same... inference::ModelInferResponse::InferOutputTensor* output = nullptr; for (auto& io : *(response.mutable_outputs())) { if (io.name() == name) { output = &io; break; } } if (output == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unable to find expected response output"); } // If this output was requested as classification then remove the // raw output from the response and instead return classification // results as a string tensor const auto itr = alloc_payload.classification_map_.find(name); if (itr == alloc_payload.classification_map_.end()) { // Not classification... output->set_datatype(TRITONSERVER_DataTypeString(datatype)); for (size_t idx = 0; idx < dim_count; idx++) { output->add_shape(shape[idx]); } } else { // Classification const uint32_t classification_count = itr->second; // For classification need to determine the batch size, if any, // because need to use that to break up the response for each // batch entry. uint32_t batch_size = 0; uint32_t batch_flags; RETURN_IF_ERR(TRITONSERVER_ServerModelBatchProperties( server, model_name, model_version, &batch_flags, nullptr /* voidp */)); if ((dim_count > 0) && ((batch_flags & TRITONSERVER_BATCH_FIRST_DIM) != 0)) { batch_size = shape[0]; } // Determine the batch1 byte size of the tensor... needed when // the response tensor batch-size > 1 so that we know how to // stride though the tensor data. size_t batch1_element_count = 1; for (size_t idx = ((batch_size == 0) ? 0 : 1); idx < dim_count; idx++) { batch1_element_count *= shape[idx]; } const size_t batch1_byte_size = batch1_element_count * TRITONSERVER_DataTypeByteSize(datatype); // Create the classification contents std::string serialized; size_t class_offset = 0; for (uint32_t bs = 0; bs < std::max((uint32_t)1, batch_size); ++bs) { std::vector<std::string> class_strs; RETURN_IF_ERR(TopkClassifications( iresponse, output_idx, reinterpret_cast<const char*>(base) + class_offset, ((class_offset + batch1_byte_size) > byte_size) ? 0 : batch1_byte_size, datatype, classification_count, &class_strs)); // Serialize for binary representation... for (const auto& str : class_strs) { uint32_t len = str.size(); serialized.append(reinterpret_cast<const char*>(&len), sizeof(len)); if (len > 0) { serialized.append(str); } } class_offset += batch1_byte_size; } // Update the output with new datatype, shape and contents. output->set_datatype( TRITONSERVER_DataTypeString(TRITONSERVER_TYPE_BYTES)); if (batch_size > 0) { output->add_shape(batch_size); } output->add_shape( std::min(classification_count, (uint32_t)batch1_element_count)); (*response.mutable_raw_output_contents())[output_idx] = std::move(serialized); } } // Make sure response doesn't exceed GRPC limits. if (response.ByteSizeLong() > MAX_GRPC_MESSAGE_SIZE) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "Response has byte size " + std::to_string(response.ByteSizeLong()) + " which exceeds gRPC's byte size limit " + std::to_string(INT_MAX) + ".") .c_str()); } return nullptr; // success } // // InferHandlerState // template < typename ServerResponderType, typename RequestType, typename ResponseType> class InferHandlerState { public: using InferHandlerStateType = InferHandlerState<ServerResponderType, RequestType, ResponseType>; // State that is shared across all state objects that make up a GRPC // transaction (e.g. a stream). struct Context { explicit Context( ::grpc::ServerCompletionQueue* cq, const uint64_t unique_id = 0) : cq_(cq), unique_id_(unique_id), ongoing_requests_(0), step_(Steps::START), finish_ok_(true), ongoing_write_(false), received_notification_(false) { ctx_.reset(new ::grpc::ServerContext()); responder_.reset(new ServerResponderType(ctx_.get())); gRPCErrorTracker_ = std::make_unique<gRPCErrorTracker>(); } void SetCompressionLevel(grpc_compression_level compression_level) { ctx_->set_compression_level(compression_level); } void GrpcContextAsyncNotifyWhenDone(InferHandlerStateType* state) { notify_state_ = std::unique_ptr<InferHandlerStateType>( new InferHandlerStateType(Steps::WAITING_NOTIFICATION, state)); ctx_->AsyncNotifyWhenDone(notify_state_.get()); } void SetReceivedNotification(bool value) { received_notification_ = true; } bool ReceivedNotification() { return received_notification_; } bool IsCancelled() { std::lock_guard<std::recursive_mutex> lock(mu_); return received_notification_ ? (ctx_->IsCancelled() || gRPCErrorTracker_->CheckAndUpdateGRPCError()) : false; } // Increments the ongoing request counter void IncrementRequestCounter() { ongoing_requests_++; } // Decrements the ongoing request counter void DecrementRequestCounter() { ongoing_requests_--; } // Adds the state object created on this context void InsertState(InferHandlerStateType* state) { all_states_.insert(state); } // Erases the state object created on this context void EraseState(InferHandlerStateType* state) { EraseInflightState(state); all_states_.erase(state); } bool HandleCompletion() { if (step_ != Steps::FINISH) { for (auto state : all_states_) { std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); // There is no order guarantee on when the AsyncNotifyWhenDone // event is placed on the completion queue vs when the actual // state RPC is processed. Need to transition through two steps // to preserve the lifetime of the state object. if (state->step_ == Steps::PARTIAL_COMPLETION) { state->step_ = Steps::COMPLETE; } else { state->step_ = Steps::FINISH; } PutTaskBackToQueue(state); } step_ = Steps::FINISH; return true; } return false; } // Extracts headers from GRPC request and updates state void ExtractStateFromHeaders(InferHandlerStateType* state) { const auto& metadata = state->context_->ctx_->client_metadata(); std::string triton_grpc_error_key = "triton_grpc_error"; auto it = metadata.find( {triton_grpc_error_key.data(), triton_grpc_error_key.size()}); if (it != metadata.end()) { if (it->second == "true") { LOG_VERBOSE(2) << "GRPC: triton_grpc_error mode detected in new grpc stream"; state->context_->gRPCErrorTracker_->triton_grpc_error_ = true; } } } void WriteGRPCErrorResponse(InferHandlerStateType* state) { std::lock_guard<std::recursive_mutex> lock(state->context_->mu_); // Check if Error not responded previously // Avoid closing connection twice on multiple errors from core if (!state->context_->gRPCErrorTracker_->GRPCErrorEncountered()) { state->step_ = Steps::COMPLETE; state->context_->responder_->Finish(state->status_, state); // Mark error for this stream state->context_->gRPCErrorTracker_->MarkGRPCErrorEncountered(); } } const std::string DebugString(InferHandlerStateType* state) { std::string debug_string(""); debug_string.append( "Running state_id " + std::to_string(state->unique_id_) + "\n"); debug_string.append( "\tContext step " + std::to_string(state->context_->step_) + " id " + std::to_string(state->context_->unique_id_) + "\n"); for (auto new_state : all_states_) { debug_string.append( "\t\t State id " + std::to_string(new_state->unique_id_) + ": State step " + std::to_string(new_state->step_) + "\n"); } return debug_string; } // Inserts the state to a set tracking active requests // within the server core. Should only be called when // the request was successfully enqueued on Triton. void InsertInflightState(InferHandlerStateType* state) { std::lock_guard<std::recursive_mutex> lock(mu_); inflight_states_.insert(state); } // Erases the state to a set tracking active requests // within the server core. void EraseInflightState(InferHandlerStateType* state) { std::lock_guard<std::recursive_mutex> lock(mu_); inflight_states_.erase(state); } // Issues the cancellation for all inflight requests // being tracked by this context. void IssueRequestCancellation() { { std::lock_guard<std::recursive_mutex> lock(mu_); // Issues the request cancellation to the core. for (auto state : inflight_states_) { std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); if (state->step_ != Steps::CANCELLED && state->step_ != Steps::COMPLETE) { LOG_VERBOSE(1) << "Issuing cancellation for " << state->unique_id_; if (state->inference_request_.get() == nullptr) { // The context might be holding some states that have // not been issued to Triton core. Need to skip calling // issuing cancellation for such requests. continue; } // Note that request may or may not be valid at this point. // Assuming if RequestComplete callback is run asynchronously // before this point. TRITONSERVER_Error* err = nullptr; err = TRITONSERVER_InferenceRequestCancel( state->inference_request_.get()); // TODO: Add request id to the message if (err != nullptr) { LOG_INFO << "Failed to cancel the request: " << TRITONSERVER_ErrorMessage(err); } state->step_ = Steps::CANCELLATION_ISSUED; } else if (state->step_ == Steps::COMPLETE) { // The RPC is complete and no callback will be invoked to retrieve // the object. Hence, need to explicitly place the state on the // completion queue. PutTaskBackToQueue(state); } } } } // Handles the gRPC context cancellation. This function can be called // multiple times and is supposed to be re-entrant. // Returns whether or not to continue cycling through the gRPC // completion queue or not. bool HandleCancellation( InferHandlerStateType* state, bool rpc_ok, const std::string& name) { // Check to avoid early exit in case of triton_grpc_error if (!IsCancelled()) { LOG_ERROR << "[INTERNAL] HandleCancellation called even when the context was " "not cancelled for " << name << ", rpc_ok=" << rpc_ok << ", context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_; return true; } if ((state->step_ != Steps::CANCELLATION_ISSUED) && (state->step_ != Steps::CANCELLED)) { LOG_VERBOSE(1) << "Cancellation notification received for " << name << ", rpc_ok=" << rpc_ok << ", context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_; // If the context has not been cancelled then // issue cancellation request to all the inflight // states belonging to the context. if (state->context_->step_ != Steps::CANCELLED) { IssueRequestCancellation(); // Mark the context as cancelled state->context_->step_ = Steps::CANCELLED; // The state returns true because the CancelExecution // call above would have raised alarm objects on all // pending inflight states objects. This state will // be taken up along with all the other states in the // next iteration from the completion queue which // would release the state. return true; } } if (state->step_ != Steps::CANCELLATION_ISSUED) { // The cancellation has not been issued hence the state can // be released. LOG_VERBOSE(1) << "Completing cancellation for " << name << ", rpc_ok=" << rpc_ok << ", context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_; return false; } else { // Should wait for the ResponseComplete callbacks to be invoked. LOG_VERBOSE(1) << "Waiting for the callback to retrieve cancellation for " << name << ", rpc_ok=" << rpc_ok << ", context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_; return true; } } // Enqueue 'state' so that its response is delivered in the // correct order. void EnqueueForResponse(InferHandlerStateType* state) { std::lock_guard<std::recursive_mutex> lock(mu_); states_.push(state); } // Write the response to the stream directly. void DecoupledWriteResponse(InferHandlerStateType* state) { #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_SEND_START", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING state->step_ = Steps::WRITTEN; ResponseType* response = state->response_queue_->GetCurrentResponse(); responder_->Write(*response, state); // Clear the response after writing response->mutable_infer_response()->Clear(); // Pop the response from queue state->response_queue_->PopResponse(); } // Adds the state object to the completion queue so // that it can be processed later void PutTaskBackToQueue(InferHandlerStateType* state) { std::lock_guard<std::recursive_mutex> lock(mu_); // FIXME: Is there a better way to put task on the // completion queue rather than using alarm object? // The alarm object will add a new task to the back of the // completion queue when it expires or when it’s cancelled. state->alarm_.Set( cq_, gpr_now(gpr_clock_type::GPR_CLOCK_REALTIME), state); } // Check the state at the front of the queue and write it if // ready. The state at the front of the queue is ready if it is in // the WRITEREADY state and it equals 'required_state' (or // 'required_state' is nullptr). Return nullptr if front of queue // was not ready (and so not written), or return the state if it // was ready and written. InferHandlerStateType* WriteResponseIfReady( InferHandlerStateType* required_state) { std::lock_guard<std::recursive_mutex> lock(mu_); if (states_.empty()) { return nullptr; } InferHandlerStateType* state = states_.front(); if (state->step_ != Steps::WRITEREADY) { return nullptr; } if ((required_state != nullptr) && (state != required_state)) { return nullptr; } #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_SEND_START", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING state->step_ = Steps::WRITTEN; state->context_->ongoing_write_ = true; // Non decoupled writes use only one response responder_->Write(*state->response_queue_->GetResponseAt(0), state); return state; } // If 'state' is at the front of the queue and written, pop it and // return true. Other return false. bool PopCompletedResponse(InferHandlerStateType* state) { std::lock_guard<std::recursive_mutex> lock(mu_); if (states_.empty()) { return false; } InferHandlerStateType* front = states_.front(); if ((front == state) && (state->step_ == Steps::WRITTEN)) { states_.pop(); return true; } return false; } // Return true if this context has completed all reads and writes. bool IsRequestsCompleted() { std::lock_guard<std::recursive_mutex> lock(mu_); return ( (step_ == Steps::WRITEREADY) && states_.empty() && (ongoing_requests_ == 0)); } // The grpc completion queue associated with the RPC. ::grpc::ServerCompletionQueue* cq_; // Unique ID for the context. Used only for debugging so will // always be 0 in non-debug builds. const uint64_t unique_id_; // Context for the rpc, allowing to tweak aspects of it such as // the use of compression, authentication, as well as to send // metadata back to the client. std::unique_ptr<::grpc::ServerContext> ctx_; std::unique_ptr<ServerResponderType> responder_; // The states associated with this context that are currently // active. Used by stream handlers to maintain request / response // orders. A state enters this queue when it has successfully read // a request and exits the queue when it is written. std::recursive_mutex mu_; std::queue<InferHandlerStateType*> states_; std::atomic<uint32_t> ongoing_requests_; // Tracks the inflight requests sent to Triton core via this // context. We will use this structure to issue cancellations // on these requests. std::set<InferHandlerStateType*> inflight_states_; // Tracks all the states that have been created on this context. std::set<InferHandlerStateType*> all_states_; // Ready to write queue for decoupled std::queue<InferHandlerStateType*> ready_to_write_states_; // The step of the entire context. Steps step_; // True if this context should finish with OK status, false if // should finish with CANCELLED status. bool finish_ok_; // True if there is an ongoing write to the grpc stream std::atomic<bool> ongoing_write_; // The state object that is sent to grpc async notification // for tracking the gRPC stream. std::unique_ptr<InferHandlerState> notify_state_; // Tracks whether the async notification has been delivered by // completion queue. bool received_notification_; std::unique_ptr<gRPCErrorTracker> gRPCErrorTracker_; }; // This constructor is used to build a wrapper state object // pointing to the actual state object. The wrapper state // object is used to distinguish a tag from AsyncNotifyWhenDone() // signal. explicit InferHandlerState(Steps start_step, InferHandlerState* state) : step_(start_step), state_ptr_(state), async_notify_state_(false) { state->MarkAsAsyncNotifyState(); } explicit InferHandlerState( TRITONSERVER_Server* tritonserver, const std::shared_ptr<Context>& context, Steps start_step = Steps::START) : tritonserver_(tritonserver), async_notify_state_(false) { // For debugging and testing const char* dstr = getenv("TRITONSERVER_DELAY_GRPC_RESPONSE"); delay_response_ms_ = 0; if (dstr != nullptr) { delay_response_ms_ = atoi(dstr); } const char* cstr = getenv("TRITONSERVER_DELAY_GRPC_COMPLETE"); delay_complete_ms_ = 0; if (cstr != nullptr) { delay_complete_ms_ = atoi(cstr); } const char* pstr = getenv("TRITONSERVER_DELAY_GRPC_PROCESS"); delay_process_ms_ = 0; if (pstr != nullptr) { delay_process_ms_ = atoi(pstr); } response_queue_.reset(new ResponseQueue<ResponseType>()); Reset(context, start_step); } ~InferHandlerState() { ClearTraceTimestamps(); } bool IsGrpcContextCancelled() { return context_->IsCancelled(); } void Reset( const std::shared_ptr<Context>& context, Steps start_step = Steps::START) { unique_id_ = NEXT_UNIQUE_ID; context_ = context; step_ = start_step; status_ = ::grpc::Status{}; cb_count_ = 0; is_decoupled_ = false; complete_ = false; parameters_ = {}; request_.Clear(); response_queue_->Reset(); // Clear trace_timestamps_ here so they do not grow indefinitely since // states are re-used for performance. ClearTraceTimestamps(); // The pointer should be nullptr for all state objects instead of // wrapper state object in WAITING_NOTIFICATION step. state_ptr_ = nullptr; async_notify_state_ = false; } void Release() { context_ = nullptr; inference_request_.reset(); ClearTraceTimestamps(); } void ClearTraceTimestamps() { #ifdef TRITON_ENABLE_TRACING if (trace_ != nullptr) { for (const auto& timestamp : trace_timestamps_) { trace_->CaptureTimestamp(timestamp.first, timestamp.second); } trace_.reset(); } trace_timestamps_.clear(); #endif // TRITON_ENABLE_TRACING } // Returns whether all the responses from the state // are delivered and successfully written on the // stream. bool IsComplete() { return (complete_ && response_queue_->IsEmpty()); } void MarkAsAsyncNotifyState() { async_notify_state_ = true; } bool IsAsyncNotifyState() { return async_notify_state_; } // Needed in the response handle for classification outputs. TRITONSERVER_Server* tritonserver_; // Unique ID for the state. Used only for debugging so will // always be 0 in non-debug builds. uint64_t unique_id_; std::shared_ptr<Context> context_; Steps step_; std::recursive_mutex step_mtx_; // Shared pointer to the inference request object. The lifetime of // inference request object is extended till all the responses from // the request are processed and the request is released. std::shared_ptr<TRITONSERVER_InferenceRequest> inference_request_; #ifdef TRITON_ENABLE_TRACING std::shared_ptr<TraceManager::Trace> trace_; // Additional timestamps that are captured before a trace stream is acquired std::deque<std::pair<std::string, uint64_t>> trace_timestamps_; #endif // TRITON_ENABLE_TRACING bool is_decoupled_ = false; StateParameters parameters_; ::grpc::Status status_; std::atomic<uint32_t> cb_count_; bool complete_; RequestType request_; std::shared_ptr<ResponseQueue<ResponseType>> response_queue_; ::grpc::Alarm alarm_; // For testing and debugging int delay_response_ms_; int delay_complete_ms_; int delay_process_ms_; // For inference requests the allocator payload, unused for other // requests. AllocPayload<ResponseType> alloc_payload_; // The below pointer is only set when using this state object as a // wrapper over actual state when being sent to completion queue // using AsyncNotifyWhenDone function. Otherwise it is nullptr. InferHandlerState* state_ptr_; // Tracks whether this state object has been wrapped and send to // AsyncNotifyWhenDone() function as a tag. bool async_notify_state_; }; // // InferHandler // template < typename ServiceType, typename ServerResponderType, typename RequestType, typename ResponseType> class InferHandler : public HandlerBase { public: InferHandler( const std::string& name, const std::shared_ptr<TRITONSERVER_Server>& tritonserver, ServiceType* service, ::grpc::ServerCompletionQueue* cq, size_t max_state_bucket_count, std::pair<std::string, std::string> restricted_kv, const std::string& header_forward_pattern); virtual ~InferHandler(); // Descriptive name of of the handler. const std::string& Name() const { return name_; } // Start handling requests. void Start() override; // Stop handling requests. void Stop() override; protected: using State = InferHandlerState<ServerResponderType, RequestType, ResponseType>; using StateContext = typename State::Context; State* StateNew( TRITONSERVER_Server* tritonserver, const std::shared_ptr<StateContext>& context, Steps start_step = Steps::START) { State* state = nullptr; if (max_state_bucket_count_ > 0) { std::lock_guard<std::mutex> lock(alloc_mu_); if (!state_bucket_.empty()) { state = state_bucket_.back(); state->Reset(context, start_step); state_bucket_.pop_back(); } } if (state == nullptr) { state = new State(tritonserver, context, start_step); } if (start_step == Steps::START) { // Need to be called to receive an asynchronous notification // when the transaction is cancelled. context->GrpcContextAsyncNotifyWhenDone(state); } context->InsertState(state); LOG_VERBOSE(2) << "StateNew, " << state->unique_id_ << " Step " << state->step_; return state; } void StateRelease(State* state) { LOG_VERBOSE(2) << "StateRelease, " << state->unique_id_ << " Step " << state->step_; if (max_state_bucket_count_ > 0) { std::lock_guard<std::mutex> lock(alloc_mu_); if (state_bucket_.size() < max_state_bucket_count_) { state->Release(); state_bucket_.push_back(state); return; } } delete state; } virtual void StartNewRequest() = 0; virtual bool Process(State* state, bool rpc_ok) = 0; bool ExecutePrecondition(InferHandler::State* state); TRITONSERVER_Error* ForwardHeadersAsParameters( TRITONSERVER_InferenceRequest* irequest, InferHandler::State* state); const std::string name_; std::shared_ptr<TRITONSERVER_Server> tritonserver_; ServiceType* service_; ::grpc::ServerCompletionQueue* cq_; std::unique_ptr<std::thread> thread_; // Mutex to serialize State allocation std::mutex alloc_mu_; // Keep some number of state objects for reuse to avoid the overhead // of creating a state for every new request. const size_t max_state_bucket_count_; std::vector<State*> state_bucket_; std::pair<std::string, std::string> restricted_kv_; std::string header_forward_pattern_; re2::RE2 header_forward_regex_; }; template < typename ServiceType, typename ServerResponderType, typename RequestType, typename ResponseType> InferHandler<ServiceType, ServerResponderType, RequestType, ResponseType>:: InferHandler( const std::string& name, const std::shared_ptr<TRITONSERVER_Server>& tritonserver, ServiceType* service, ::grpc::ServerCompletionQueue* cq, size_t max_state_bucket_count, std::pair<std::string, std::string> restricted_kv, const std::string& header_forward_pattern) : name_(name), tritonserver_(tritonserver), service_(service), cq_(cq), max_state_bucket_count_(max_state_bucket_count), restricted_kv_(restricted_kv), header_forward_pattern_(header_forward_pattern), header_forward_regex_(header_forward_pattern_) { } template < typename ServiceType, typename ServerResponderType, typename RequestType, typename ResponseType> InferHandler<ServiceType, ServerResponderType, RequestType, ResponseType>:: ~InferHandler() { for (State* state : state_bucket_) { delete state; } state_bucket_.clear(); LOG_VERBOSE(1) << "Destructed " << Name(); } template < typename ServiceType, typename ServerResponderType, typename RequestType, typename ResponseType> void InferHandler< ServiceType, ServerResponderType, RequestType, ResponseType>::Start() { // Use a barrier to make sure we don't return until thread has // started. auto barrier = std::make_shared<Barrier>(2); thread_.reset(new std::thread([this, barrier] { StartNewRequest(); barrier->Wait(); void* tag; bool ok; while (cq_->Next(&tag, &ok)) { State* state = static_cast<State*>(tag); if (state->step_ == Steps::WAITING_NOTIFICATION) { State* state_wrapper = state; state = state_wrapper->state_ptr_; state->context_->SetReceivedNotification(true); LOG_VERBOSE(1) << "Received notification for " << Name() << ", " << state->unique_id_; } LOG_VERBOSE(2) << "Grpc::CQ::Next() " << state->context_->DebugString(state); if (!Process(state, ok)) { LOG_VERBOSE(1) << "Done for " << Name() << ", " << state->unique_id_; state->context_->EraseState(state); StateRelease(state); } else { LOG_VERBOSE(2) << "Returning from " << Name() << ", " << state->unique_id_ << ", " << state->step_; } } })); barrier->Wait(); LOG_VERBOSE(1) << "Thread started for " << Name(); } template < typename ServiceType, typename ServerResponderType, typename RequestType, typename ResponseType> void InferHandler< ServiceType, ServerResponderType, RequestType, ResponseType>::Stop() { if (thread_->joinable()) { thread_->join(); } LOG_VERBOSE(1) << "Thread exited for " << Name(); } template < typename ServiceType, typename ServerResponderType, typename RequestType, typename ResponseType> bool InferHandler<ServiceType, ServerResponderType, RequestType, ResponseType>:: ExecutePrecondition(InferHandler::State* state) { if (!restricted_kv_.first.empty()) { const auto& metadata = state->context_->ctx_->client_metadata(); const auto it = metadata.find(restricted_kv_.first); return (it != metadata.end()) && (it->second == restricted_kv_.second); } return true; } template < typename ServiceType, typename ServerResponderType, typename RequestType, typename ResponseType> TRITONSERVER_Error* InferHandler<ServiceType, ServerResponderType, RequestType, ResponseType>:: ForwardHeadersAsParameters( TRITONSERVER_InferenceRequest* irequest, InferHandler::State* state) { TRITONSERVER_Error* err = nullptr; if (!header_forward_pattern_.empty()) { const auto& metadata = state->context_->ctx_->client_metadata(); for (const auto& pair : metadata) { auto& key = pair.first; auto& value = pair.second; std::string param_key = std::string(key.begin(), key.end()); if (RE2::PartialMatch(param_key, header_forward_regex_)) { std::string param_value = std::string(value.begin(), value.end()); err = TRITONSERVER_InferenceRequestSetStringParameter( irequest, param_key.c_str(), param_value.c_str()); if (err != nullptr) { break; } } } } return err; } // // ModelInferHandler // class ModelInferHandler : public InferHandler< inference::GRPCInferenceService::AsyncService, ::grpc::ServerAsyncResponseWriter<inference::ModelInferResponse>, inference::ModelInferRequest, inference::ModelInferResponse> { public: ModelInferHandler( const std::string& name, const std::shared_ptr<TRITONSERVER_Server>& tritonserver, TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, inference::GRPCInferenceService::AsyncService* service, ::grpc::ServerCompletionQueue* cq, size_t max_state_bucket_count, grpc_compression_level compression_level, std::pair<std::string, std::string> restricted_kv, const std::string& forward_header_pattern) : InferHandler( name, tritonserver, service, cq, max_state_bucket_count, restricted_kv, forward_header_pattern), trace_manager_(trace_manager), shm_manager_(shm_manager), compression_level_(compression_level) { // Create the allocator that will be used to allocate buffers for // the result tensors. FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator_, InferResponseAlloc, InferResponseFree, InferResponseStart), "creating inference response allocator"); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorSetQueryFunction( allocator_, OutputBufferQuery), "setting allocator's query function"); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( allocator_, OutputBufferAttributes), "setting allocator's output buffer attributes function"); } ~ModelInferHandler() { LOG_TRITONSERVER_ERROR( TRITONSERVER_ResponseAllocatorDelete(allocator_), "deleting response allocator"); } protected: void StartNewRequest() override; bool Process(State* state, bool rpc_ok) override; private: void Execute(State* state); static void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); TraceManager* trace_manager_; std::shared_ptr<SharedMemoryManager> shm_manager_; TRITONSERVER_ResponseAllocator* allocator_; grpc_compression_level compression_level_; }; #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING) class GrpcServerCarrier : public otel_cntxt::propagation::TextMapCarrier { public: GrpcServerCarrier(::grpc::ServerContext* context) : context_(context) {} GrpcServerCarrier() = default; virtual opentelemetry::nostd::string_view Get( opentelemetry::nostd::string_view key) const noexcept override { auto it = context_->client_metadata().find({key.data(), key.size()}); if (it != context_->client_metadata().end()) { return it->second.data(); } return ""; } // Not required on server side virtual void Set( opentelemetry::nostd::string_view key, opentelemetry::nostd::string_view value) noexcept override { return; } ::grpc::ServerContext* context_; }; #else using GrpcServerCarrier = void*; #endif // TRITON_ENABLE_TRACING }}} // namespace triton::server::grpc �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/stream_infer_handler.cc��������������������������������������������������0000664�0000000�0000000�00000103363�15270523712�0022047�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "stream_infer_handler.h" #include <regex> namespace triton { namespace server { namespace grpc { // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* StreamInferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { AllocPayload<inference::ModelStreamInferResponse>* payload = reinterpret_cast<AllocPayload<inference::ModelStreamInferResponse>*>( userp); auto response = payload->response_queue_->GetLastAllocatedResponse(); if (response == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Unable to access the last allocated response"); } return ResponseAllocatorHelper( allocator, tensor_name, byte_size, preferred_memory_type, preferred_memory_type_id, response->mutable_infer_response(), payload->shm_map_, buffer, buffer_userp, actual_memory_type, actual_memory_type_id); } TRITONSERVER_Error* StreamInferResponseStart(TRITONSERVER_ResponseAllocator* allocator, void* userp) { AllocPayload<inference::ModelStreamInferResponse>* payload = reinterpret_cast<AllocPayload<inference::ModelStreamInferResponse>*>( userp); // Move to the next response object payload->response_queue_->AllocateResponse(); return nullptr; // success } // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* StreamOutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { AllocPayload<inference::ModelStreamInferResponse>* payload = reinterpret_cast<AllocPayload<inference::ModelStreamInferResponse>*>( userp); return OutputBufferQueryHelper( allocator, tensor_name, byte_size, payload->shm_map_, memory_type, memory_type_id); } // Make sure to keep InferResponseAlloc, OutputBufferQuery, and // OutputBufferAttributes logic in sync TRITONSERVER_Error* StreamOutputBufferAttributes( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, TRITONSERVER_BufferAttributes* buffer_attributes, void* userp, void* buffer_userp) { AllocPayload<inference::ModelStreamInferResponse>* payload = reinterpret_cast<AllocPayload<inference::ModelStreamInferResponse>*>( userp); return OutputBufferAttributesHelper( allocator, tensor_name, payload->shm_map_, buffer_attributes); } //============================================================================= // The following section contains the handling mechanism for ModelStreamInfer // RPC. This implementation is tuned towards performance and reducing latency. //============================================================================= void ModelStreamInferHandler::StartNewRequest() { auto context = std::make_shared<State::Context>(cq_, NEXT_UNIQUE_ID); context->SetCompressionLevel(compression_level_); State* state = StateNew(tritonserver_.get(), context); #ifdef TRITON_ENABLE_TRACING // Can't create trace as we don't know the model to be requested, // track timestamps in 'state' state->trace_timestamps_.emplace_back( std::make_pair("GRPC_WAITREAD_START", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING service_->RequestModelStreamInfer( state->context_->ctx_.get(), state->context_->responder_.get(), cq_, cq_, state); LOG_VERBOSE(1) << "New request handler for " << Name() << ", " << state->unique_id_; } bool ModelStreamInferHandler::Process(InferHandler::State* state, bool rpc_ok) { // Because gRPC doesn't allow concurrent writes on the // the stream we only have a single handler thread that // reads from the completion queue. Hence, cancellation // notification will be received on the same handler // thread. // This means that we only need to take care of // synchronizing this thread and the ResponseComplete // threads. if (state->context_->ReceivedNotification()) { std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); if (state->IsGrpcContextCancelled()) { bool resume = state->context_->HandleCancellation(state, rpc_ok, Name()); return resume; } else { if (state->context_->HandleCompletion()) { return true; } } } LOG_VERBOSE(1) << "Process for " << Name() << ", rpc_ok=" << rpc_ok << ", context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_; // We need an explicit finish indicator. Can't use 'state->step_' // because we launch an async thread that could update 'state's // step_ to be FINISH before this thread exits this function. bool finished = false; if (state->step_ == Steps::START) { // A new stream connection... If RPC failed on a new request then // the server is shutting down and so we should do nothing. if (!rpc_ok) { state->step_ = Steps::FINISH; return false; } // Start a new request to replace this one... StartNewRequest(); if (ExecutePrecondition(state)) { // Since this is the start of a connection, 'state' hasn't been // used yet so use it to read a request off the connection. state->context_->step_ = Steps::READ; state->step_ = Steps::READ; state->context_->responder_->Read(&state->request_, state); } else { // Precondition is not satisfied, cancel the stream state->context_->step_ = Steps::COMPLETE; state->step_ = Steps::PARTIAL_COMPLETION; ::grpc::Status status = ::grpc::Status( ::grpc::StatusCode::UNAVAILABLE, std::string("This protocol is restricted, expecting header '") + restricted_kv_.first + "'"); state->context_->responder_->Finish(status, state); return !finished; } state->context_->ExtractStateFromHeaders(state); } else if (state->step_ == Steps::READ) { TRITONSERVER_Error* err = nullptr; const inference::ModelInferRequest& request = state->request_; #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_WAITREAD_END", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING // If done reading and no in-flight requests then can finish the // entire stream. Otherwise just finish this state. if (!rpc_ok) { state->context_->step_ = Steps::WRITEREADY; if (state->context_->IsRequestsCompleted()) { state->context_->step_ = Steps::COMPLETE; state->step_ = Steps::PARTIAL_COMPLETION; LOG_VERBOSE(2) << "Finishing responder from state " << state->unique_id_; state->context_->responder_->Finish( state->context_->finish_ok_ ? ::grpc::Status::OK : ::grpc::Status::CANCELLED, state); } else { state->step_ = Steps::FINISH; finished = true; } return !finished; } int64_t requested_model_version; err = GetModelVersionFromString( request.model_version(), &requested_model_version); // Record the transaction policy of the model into the current state // object. if (err == nullptr) { uint32_t txn_flags; err = TRITONSERVER_ServerModelTransactionProperties( tritonserver_.get(), request.model_name().c_str(), requested_model_version, &txn_flags, nullptr /* voidp */); if (err == nullptr) { state->is_decoupled_ = ((txn_flags & TRITONSERVER_TXN_DECOUPLED) != 0); } } // Request has been successfully read, increment the context request // counter. state->context_->IncrementRequestCounter(); // If the request is not for a model with decoupled transaction policy // then put it in the context queue so that its response is sent in // the same order as the request was received. if (!state->is_decoupled_) { state->context_->EnqueueForResponse(state); } // Need to get context here as it is needed below. 'state' can // complete inference, write response, and finish (which releases // context) before we make any forward progress.... so need to // hold onto context here while we know it is good. std::shared_ptr<StateContext> context = state->context_; // Issue the inference request into server... auto response_queue_ = state->response_queue_; // Create the inference request which contains all the // input information needed for an inference. TRITONSERVER_InferenceRequest* irequest = nullptr; if (err == nullptr) { err = TRITONSERVER_InferenceRequestNew( &irequest, tritonserver_.get(), request.model_name().c_str(), requested_model_version); } if (err == nullptr) { state->inference_request_ = { irequest, [](TRITONSERVER_InferenceRequest* request) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestDelete(request), "deleting gRPC inference request"); }}; err = SetInferenceRequestMetadata(irequest, request, state->parameters_); } if (err == nullptr) { err = ForwardHeadersAsParameters(irequest, state); } // Will be used to hold the serialized data in case explicit string // tensors are present in the request. std::list<std::string> serialized_data; if (err == nullptr) { err = InferGRPCToInput( tritonserver_, shm_manager_, request, &serialized_data, irequest); } if (err == nullptr) { err = InferAllocatorPayload<inference::ModelStreamInferResponse>( tritonserver_, shm_manager_, request, std::move(serialized_data), response_queue_, &state->alloc_payload_); } auto request_release_payload = std::make_unique<RequestReleasePayload>(state->inference_request_); if (err == nullptr) { err = TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestComplete, request_release_payload.get() /* request_release_userp */); } if (err == nullptr) { err = TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator_, &state->alloc_payload_ /* response_allocator_userp */, StreamInferResponseComplete, reinterpret_cast<void*>(state)); } if (err == nullptr) { TRITONSERVER_InferenceTrace* triton_trace = nullptr; #ifdef TRITON_ENABLE_TRACING GrpcServerCarrier carrier(state->context_->ctx_.get()); auto start_options = trace_manager_->GetTraceStartOptions(carrier, request.model_name()); state->trace_ = std::move(trace_manager_->SampleTrace(start_options)); if (state->trace_ != nullptr) { triton_trace = state->trace_->trace_; } #endif // TRITON_ENABLE_TRACING state->step_ = ISSUED; err = TRITONSERVER_ServerInferAsync( tritonserver_.get(), irequest, triton_trace); } // If there was not an error in issuing the 'state' request then // state->step_ == ISSUED and inference request has // initiated... the completion callback will transition to // WRITEREADY or WRITTEN or CANCELLED. Recording the state and the // irequest to handle gRPC stream cancellation. if (err == nullptr) { state->context_->InsertInflightState(state); // The payload will be cleaned in request release callback. request_release_payload.release(); } else { // If there was an error then enqueue the error response and show // it to be ready for writing. inference::ModelStreamInferResponse* response; if (state->is_decoupled_) { state->response_queue_->AllocateResponse(); response = state->response_queue_->GetLastAllocatedResponse(); } else { response = state->response_queue_->GetNonDecoupledResponse(); } // Get request ID for logging in case of error. std::string log_request_id = request.id(); if (log_request_id.empty()) { log_request_id = "<id_unknown>"; } LOG_VERBOSE(1) << "[request id: " << log_request_id << "] " << "Infer failed: " << TRITONSERVER_ErrorMessage(err); ::grpc::Status status; GrpcStatusUtil::Create(&status, err); TRITONSERVER_ErrorDelete(err); response->set_error_message(status.error_message()); response->mutable_infer_response()->Clear(); // repopulate the id so that client knows which request failed. response->mutable_infer_response()->set_id(request.id()); if (!state->is_decoupled_) { state->step_ = Steps::WRITEREADY; state->context_->WriteResponseIfReady(state); } else { InferHandler::State* writing_state = nullptr; std::lock_guard<std::recursive_mutex> lk1(state->context_->mu_); { std::lock_guard<std::recursive_mutex> lk2(state->step_mtx_); state->response_queue_->MarkNextResponseComplete(); state->context_->ready_to_write_states_.push(state); if (!state->context_->ongoing_write_) { // Only one write is allowed per gRPC stream / context at any time. // If the stream is not currently writing, start writing the next // ready to write response from the next ready to write state from // 'ready_to_write_states_'. If there are other responses on the // state ready to be written after starting the write, the state // will be placed at the back of the 'ready_to_write_states_'. If // there are no other response, the state will be marked as 'ISSUED' // if complete final flag is not received yet from the backend or // completed if complete final flag is received. // The 'ongoing_write_' will reset once the completion queue returns // a written state and no additional response on the stream is ready // to be written. state->context_->ongoing_write_ = true; writing_state = state->context_->ready_to_write_states_.front(); state->context_->ready_to_write_states_.pop(); } state->complete_ = true; } if (writing_state != nullptr) { StateWriteResponse(writing_state); } } } // Now that the inference request is in flight, create a copy of // 'state' and use it to attempt another read from the connection // (i.e the next request in the stream). State* next_read_state = StateNew(tritonserver_.get(), context, Steps::READ); #ifdef TRITON_ENABLE_TRACING // Capture a timestamp for the time when we start waiting for this // next request to read. // Can't create trace as we don't know the model to be requested, // track timestamps in 'state' next_read_state->trace_timestamps_.emplace_back(std::make_pair( "GRPC_WAITREAD_START", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING next_read_state->context_->responder_->Read( &next_read_state->request_, next_read_state); } else if (state->step_ == Steps::PARTIAL_COMPLETION) { state->step_ = Steps::COMPLETE; } else if (state->step_ == Steps::COMPLETE) { state->step_ = Steps::FINISH; } else if (state->step_ == Steps::FINISH) { // The RPC execution is finished hence the state // can be released. finished = true; } else if (!state->is_decoupled_) { // We handle the WRITTEN and WRITEREADY states little // differently depending whether the inference request // is for a decoupled model or not. This is because the // grpc contract requires us to call Write() only once // on a task. Hence, for decoupled writes, we call only // one write and then wait for another notification from // the completion queue to execute pending Write()'s, if // any. // // Non-Decoupled state transitions // if (state->step_ == Steps::WRITTEN) { state->context_->ongoing_write_ = false; #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_SEND_END", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING // If the write failed (for example, client closed the stream) // mark that the stream did not complete successfully but don't // cancel right away... need to wait for any pending reads, // inferences and writes to complete. if (!rpc_ok) { LOG_VERBOSE(1) << "Write for " << Name() << ", rpc_ok=" << rpc_ok << ", context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_ << ", failed"; state->context_->finish_ok_ = false; } // Log an error if 'state' is not the expected next response. Mark // that the stream did not complete successfully but don't cancel // right away... need to wait for any pending reads, inferences // and writes to complete. if (!state->context_->PopCompletedResponse(state)) { LOG_ERROR << "Unexpected response for " << Name() << ", rpc_ok=" << rpc_ok << ", context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_; state->context_->finish_ok_ = false; } // Write the next response if it is ready... state->context_->WriteResponseIfReady(nullptr); // The response for the request has been written completely. // The counter can be safely decremented. state->context_->DecrementRequestCounter(); finished = Finish(state); } } else { // // Decoupled state transitions // if (state->step_ == Steps::WRITTEN) { #ifdef TRITON_ENABLE_TRACING state->trace_timestamps_.emplace_back( std::make_pair("GRPC_SEND_END", TraceManager::CaptureTimestamp())); #endif // TRITON_ENABLE_TRACING // If the write failed (for example, client closed the stream) // mark that the stream did not complete successfully but don't // cancel right away... need to wait for any pending reads, // inferences and writes to complete. if (!rpc_ok) { LOG_VERBOSE(1) << "Write for " << Name() << ", rpc_ok=" << rpc_ok << ", context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_ << ", failed"; state->context_->finish_ok_ = false; } { InferHandler::State* writing_state = nullptr; std::lock_guard<std::recursive_mutex> lk1(state->context_->mu_); { std::lock_guard<std::recursive_mutex> lk2(state->step_mtx_); if (!state->context_->ready_to_write_states_.empty()) { writing_state = state->context_->ready_to_write_states_.front(); state->context_->ready_to_write_states_.pop(); } else { state->context_->ongoing_write_ = false; } // Finish the state if all the transactions associated with // the state have completed. if (state != writing_state) { if (state->IsComplete()) { state->context_->DecrementRequestCounter(); finished = Finish(state); } else { state->step_ = Steps::ISSUED; } } } if (writing_state != nullptr) { StateWriteResponse(writing_state); } } } else if (state->step_ == Steps::WRITEREADY) { // Finish the state if all the transactions associated with // the state have completed. if (state->IsComplete()) { state->context_->DecrementRequestCounter(); finished = Finish(state); } else { LOG_ERROR << "Should not print this! Decoupled should NOT write via " "WRITEREADY!"; // Remove the state from the completion queue std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); state->step_ = Steps::ISSUED; } } } return !finished; } // For decoupled only. Caller must ensure exclusive write. void ModelStreamInferHandler::StateWriteResponse(InferHandler::State* state) { if (state->delay_response_ms_ != 0) { // Will delay the write of the response by the specified time. // This can be used to test the flow where there are other // responses available to be written. LOG_INFO << "Delaying the write of the response by " << state->delay_response_ms_ << " ms..."; std::this_thread::sleep_for( std::chrono::milliseconds(state->delay_response_ms_)); } { std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); state->step_ = Steps::WRITTEN; // gRPC doesn't allow to issue another write till the notification from // previous write has been delivered. state->context_->DecoupledWriteResponse(state); if (state->response_queue_->HasReadyResponse()) { state->context_->ready_to_write_states_.push(state); } } } bool ModelStreamInferHandler::Finish(InferHandler::State* state) { // If done reading and no in-flight requests then can finish the // entire stream. if (state->context_->IsRequestsCompleted()) { state->context_->step_ = Steps::COMPLETE; state->step_ = Steps::PARTIAL_COMPLETION; LOG_VERBOSE(2) << "Finishing responder from state " << state->unique_id_; state->context_->responder_->Finish( state->context_->finish_ok_ ? ::grpc::Status::OK : ::grpc::Status::CANCELLED, state); } else if (state->IsAsyncNotifyState()) { // Should only mark the state complete as the state has been sent // to AsyncNotifyWhenDone() tag and the completion event should take // care of finally releasing the state object. state->step_ = Steps::COMPLETE; } else { // Can finish this state. state->step_ = Steps::FINISH; return true; } return false; } void ModelStreamInferHandler::StreamInferResponseComplete( TRITONSERVER_InferenceResponse* iresponse, const uint32_t flags, void* userp) { State* state = reinterpret_cast<State*>(userp); // Ignore Response from CORE in case GRPC Strict as we dont care about if (state->context_->gRPCErrorTracker_->triton_grpc_error_) { std::lock_guard<std::recursive_mutex> lock(state->context_->mu_); if (state->context_->gRPCErrorTracker_->GRPCErrorEncountered()) { return; } } // Increment the callback index uint32_t response_index = state->cb_count_++; LOG_VERBOSE(1) << "ModelStreamInferHandler::StreamInferComplete, context " << state->context_->unique_id_ << ", " << state->unique_id_ << " step " << state->step_ << ", callback index " << state->cb_count_ << ", flags " << flags << ", response is nullptr " << (iresponse == nullptr); #ifdef TRITON_ENABLE_TRACING if (state->cb_count_ == 1) { state->trace_timestamps_.emplace_back(std::make_pair( "INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp())); } #endif // TRITON_ENABLE_TRACING bool is_complete = state->complete_ || (flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) != 0; // If receiving the final callback then erase the state from the inflight // state data structure to prevent cancellation being called on the request. // Also make sure that if this state was sent to gRPC async notification // mechanism then the state is not removed as it would be needed for handling // the cancellation if detected. if (is_complete && (!state->IsAsyncNotifyState())) { state->context_->EraseInflightState(state); } if (state->IsGrpcContextCancelled()) { std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); // Clean-up the received response object. LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(iresponse), "deleting GRPC inference response"); LOG_VERBOSE(1) << "ModelStreamInferHandler::StreamInferResponseComplete, " << state->unique_id_ << ", skipping response generation as grpc transaction was " "cancelled... "; // If this was the final callback for the state // then cycle through the completion queue so // that state object can be released. if (is_complete) { state->step_ = Steps::CANCELLED; state->context_->PutTaskBackToQueue(state); } state->complete_ = is_complete; return; } auto& response_queue = state->response_queue_; std::string log_request_id = state->request_.id(); if (log_request_id.empty()) { log_request_id = "<id_unknown>"; } inference::ModelStreamInferResponse* response = nullptr; bool failed = false; if (iresponse) { // Backend returned a non-null response TRITONSERVER_Error* err = nullptr; response = response_queue->GetResponseAt(response_index); if (response) { inference::ModelInferResponse& infer_response = *(response->mutable_infer_response()); // Validate Triton iresponse and set grpc/protobuf response fields from it err = InferResponseCompleteCommon<inference::ModelStreamInferResponse>( state->tritonserver_, iresponse, infer_response, state->alloc_payload_); } else { LOG_ERROR << "expected the response allocator to have added the response"; } if (err != nullptr) { failed = true; ::grpc::Status status; // Converts CORE errors to GRPC error codes GrpcStatusUtil::Create(&status, err); response->mutable_infer_response()->Clear(); response->set_error_message(status.error_message()); LOG_VERBOSE(1) << "Failed for ID: " << log_request_id << std::endl; if (state->context_->gRPCErrorTracker_->triton_grpc_error_) { state->status_ = status; // Finish only once, if backend ignores cancellation LOG_VERBOSE(1) << "GRPC streaming error detected with status: " << status.error_code() << "Closing stream connection." << std::endl; state->context_->WriteGRPCErrorResponse(state); TRITONSERVER_ErrorDelete(err); LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(iresponse), "deleting GRPC inference response"); return; } } TRITONSERVER_ErrorDelete(err); LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(iresponse), "deleting GRPC inference response"); } // Decoupled backends can return a null response via // TRITONBACKEND_ResponseFactorySendFlags. By default, these null // "empty" responses are not sent back to the client. Clients can // opt-in to receiving these empty responses via request parameters. // NOTE: The complete flag is the only flag used for this case at this time. const bool empty_final = !iresponse && state->is_decoupled_ && is_complete; const bool enable_empty_final = state->parameters_.enable_empty_final_response_; const bool create_empty_response = (empty_final && enable_empty_final); if (create_empty_response) { // Assume decoupled here based on prior checks. state->response_queue_->AllocateResponse(); response = state->response_queue_->GetLastAllocatedResponse(); if (response) { LOG_VERBOSE(1) << "[request id: " << log_request_id << "] " << "Creating empty final response"; response->mutable_infer_response()->Clear(); } else { LOG_ERROR << "expected the response allocator to have added the response"; } } if (response) { auto& infer_response = *(response->mutable_infer_response()); // Set response metadata to associate it with request. These will be set // by InferResponseCompleteCommon for successful inference. if (create_empty_response || failed) { infer_response.set_id(state->request_.id()); infer_response.set_model_name(state->request_.model_name()); infer_response.set_model_version(state->request_.model_version()); } auto& params = *(infer_response.mutable_parameters()); params["triton_final_response"].set_bool_param(is_complete); } if (state->delay_complete_ms_ != 0) { // Delay updating the state. This is useful for testing race condition with // the thread that runs Process(). LOG_INFO << "Delaying the completion of reporting response / flag by " << state->delay_complete_ms_ << " ms..."; void* context_ptr_before_delay = (void*)state->context_.get(); std::this_thread::sleep_for( std::chrono::milliseconds(state->delay_complete_ms_)); void* context_ptr_after_delay = (void*)state->context_.get(); if (context_ptr_before_delay != context_ptr_after_delay) { LOG_ERROR << "Should not print this! The state context object has " "changed after delay, pointer before: " << context_ptr_before_delay << ", pointer after: " << context_ptr_after_delay; } } if (state->IsGrpcContextCancelled()) { // Need to hold lock because the handler thread processing context // cancellation might have cancelled or marked the state for cancellation. std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); LOG_VERBOSE(1) << "ModelStreamInferHandler::StreamInferResponseComplete, " << state->unique_id_ << ", skipping writing response because of transaction was cancelled"; // If this was the final callback for the state // then cycle through the completion queue so // that state object can be released. if (is_complete) { state->step_ = Steps::CANCELLED; state->context_->PutTaskBackToQueue(state); } state->complete_ = is_complete; return; } if (state->is_decoupled_) { InferHandler::State* writing_state = nullptr; std::lock_guard<std::recursive_mutex> lk1(state->context_->mu_); { std::lock_guard<std::recursive_mutex> lk2(state->step_mtx_); bool has_prev_ready_response = state->response_queue_->HasReadyResponse(); if (response) { state->response_queue_->MarkNextResponseComplete(); } if (!has_prev_ready_response && response) { state->context_->ready_to_write_states_.push(state); } if (!state->context_->ongoing_write_ && !state->context_->ready_to_write_states_.empty()) { state->context_->ongoing_write_ = true; writing_state = state->context_->ready_to_write_states_.front(); state->context_->ready_to_write_states_.pop(); } if (is_complete && state->response_queue_->IsEmpty() && state->step_ == Steps::ISSUED) { // The response queue is empty and complete final flag is received, so // mark the state as 'WRITEREADY' so it can be cleaned up later. state->step_ = Steps::WRITEREADY; state->context_->PutTaskBackToQueue(state); } state->complete_ = is_complete; } if (writing_state != nullptr) { StateWriteResponse(writing_state); } } else { // non-decoupled std::lock_guard<std::recursive_mutex> lock(state->step_mtx_); state->step_ = Steps::WRITEREADY; if (is_complete) { state->context_->WriteResponseIfReady(state); } state->complete_ = is_complete; } } // Changes the state of grpc_stream_error_state_ to ERROR_HANDLING_COMPLETE, // indicating we have closed the stream and initiated the cancel flow void gRPCErrorTracker::MarkGRPCErrorHandlingComplete() { grpc_stream_error_state_ = TritonGRPCErrorSteps::ERROR_HANDLING_COMPLETE; } // Returns true ONLY when GRPC_ERROR from CORE is waiting to be processed. bool gRPCErrorTracker::CheckAndUpdateGRPCError() { if (grpc_stream_error_state_ == TritonGRPCErrorSteps::ERROR_ENCOUNTERED) { // Change the state to ERROR_HANDLING_COMPLETE as we have called // HandleCancellation MarkGRPCErrorHandlingComplete(); return true; } return false; } // Marks error after it has been responded to void gRPCErrorTracker::MarkGRPCErrorEncountered() { grpc_stream_error_state_ = TritonGRPCErrorSteps::ERROR_ENCOUNTERED; } // Checks if error already responded to in triton_grpc_error mode bool gRPCErrorTracker::GRPCErrorEncountered() { if (grpc_stream_error_state_ == TritonGRPCErrorSteps::NONE) { return false; } return true; } }}} // namespace triton::server::grpc �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/grpc/stream_infer_handler.h���������������������������������������������������0000664�0000000�0000000�00000012421�15270523712�0021703�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "infer_handler.h" namespace triton { namespace server { namespace grpc { // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* StreamInferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id); // // Additional Stream Infer utilities // TRITONSERVER_Error* StreamInferResponseStart( TRITONSERVER_ResponseAllocator* allocator, void* userp); // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* StreamOutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); // Make sure to keep InferResponseAlloc, OutputBufferQuery, and // OutputBufferAttributes logic in sync TRITONSERVER_Error* StreamOutputBufferAttributes( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, TRITONSERVER_BufferAttributes* buffer_attributes, void* userp, void* buffer_userp); class ModelStreamInferHandler : public InferHandler< inference::GRPCInferenceService::AsyncService, ::grpc::ServerAsyncReaderWriter< inference::ModelStreamInferResponse, inference::ModelInferRequest>, inference::ModelInferRequest, inference::ModelStreamInferResponse> { public: ModelStreamInferHandler( const std::string& name, const std::shared_ptr<TRITONSERVER_Server>& tritonserver, TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, inference::GRPCInferenceService::AsyncService* service, ::grpc::ServerCompletionQueue* cq, size_t max_state_bucket_count, grpc_compression_level compression_level, std::pair<std::string, std::string> restricted_kv, const std::string& header_forward_pattern) : InferHandler( name, tritonserver, service, cq, max_state_bucket_count, restricted_kv, header_forward_pattern), trace_manager_(trace_manager), shm_manager_(shm_manager), compression_level_(compression_level) { // Create the allocator that will be used to allocate buffers for // the result tensors. FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator_, StreamInferResponseAlloc, InferResponseFree, StreamInferResponseStart), "creating response allocator"); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorSetQueryFunction( allocator_, StreamOutputBufferQuery), "setting allocator's query function"); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( allocator_, StreamOutputBufferAttributes), "setting allocator's output buffer attribute query function"); } ~ModelStreamInferHandler() { LOG_TRITONSERVER_ERROR( TRITONSERVER_ResponseAllocatorDelete(allocator_), "deleting response allocator"); } protected: void StartNewRequest() override; bool Process(State* state, bool rpc_ok) override; private: static void StreamInferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); static void StateWriteResponse(InferHandler::State* state); bool Finish(State* state); TraceManager* trace_manager_; std::shared_ptr<SharedMemoryManager> shm_manager_; TRITONSERVER_ResponseAllocator* allocator_; grpc_compression_level compression_level_; }; }}} // namespace triton::server::grpc �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/http_server.cc����������������������������������������������������������������0000664�0000000�0000000�00000510344�15270523712�0017307�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef _WIN32 #define NOMINMAX #endif #include "http_server.h" #include <event2/buffer.h> #include <re2/re2.h> #include <algorithm> #include <list> #include <regex> #include <thread> #include "classification.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr #include "triton/common/triton_json.h" extern "C" { #include <b64/cdecode.h> } namespace triton { namespace server { #define RETURN_AND_CALLBACK_IF_ERR(X, CALLBACK) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ CALLBACK(err__); \ TRITONSERVER_ErrorDelete(err__); \ return; \ } \ } while (false) #define RETURN_AND_RESPOND_IF_ERR(REQ, X) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ EVBufferAddErrorJson((REQ)->buffer_out, err__); \ evhtp_send_reply((REQ), HttpCodeFromError(err__)); \ TRITONSERVER_ErrorDelete(err__); \ return; \ } \ } while (false) #define RETURN_AND_RESPOND_WITH_ERR(REQ, CODE, MSG) \ do { \ EVBufferAddErrorJson((REQ)->buffer_out, MSG); \ evhtp_send_reply((REQ), CODE); \ return; \ } while (false) #define RETURN_AND_RESPOND_IF_RESTRICTED( \ REQ, RESTRICTED_CATEGORY, RESTRICTED_APIS) \ do { \ auto const& is_restricted_api = \ RESTRICTED_APIS.IsRestricted(RESTRICTED_CATEGORY); \ auto const& restriction = RESTRICTED_APIS.Get(RESTRICTED_CATEGORY); \ if (is_restricted_api && RespondIfRestricted(REQ, restriction)) { \ return; \ } \ } while (false) namespace { int HttpCodeFromError(TRITONSERVER_Error* error) { if (error == nullptr) { return EVHTP_RES_OK; } switch (TRITONSERVER_ErrorCode(error)) { case TRITONSERVER_ERROR_INTERNAL: return EVHTP_RES_SERVERR; case TRITONSERVER_ERROR_NOT_FOUND: return EVHTP_RES_NOTFOUND; case TRITONSERVER_ERROR_UNAVAILABLE: return EVHTP_RES_SERVUNAVAIL; case TRITONSERVER_ERROR_UNSUPPORTED: return EVHTP_RES_NOTIMPL; // cases that has no direct matching code case TRITONSERVER_ERROR_UNKNOWN: case TRITONSERVER_ERROR_INVALID_ARG: case TRITONSERVER_ERROR_ALREADY_EXISTS: case TRITONSERVER_ERROR_CANCELLED: return EVHTP_RES_BADREQ; } return EVHTP_RES_BADREQ; } void EVBufferAddErrorJson(evbuffer* buffer, const char* message) { triton::common::TritonJson::Value response( triton::common::TritonJson::ValueType::OBJECT); response.AddStringRef("error", message, strlen(message)); triton::common::TritonJson::WriteBuffer buffer_json; response.Write(&buffer_json); evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); } void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { const char* message = TRITONSERVER_ErrorMessage(err); EVBufferAddErrorJson(buffer, message); } void AddContentTypeHeader(evhtp_request_t* req, const char* type) { // Remove existing header if found auto content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); if (content_header) { evhtp_header_rm_and_free(req->headers_out, content_header); } evhtp_headers_add_header( req->headers_out, evhtp_header_new(kContentTypeHeader, type, 1, 1)); } TRITONSERVER_Error* SetTritonParameterFromJsonParameter( const std::string& parameter, triton::common::TritonJson::Value& params_json, TRITONSERVER_InferenceRequest* irequest) { triton::common::TritonJson::Value value; if (!params_json.Find(parameter.c_str(), &value)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, ("parameter key '" + parameter + "' was not found in the JSON") .c_str()); } if (value.IsString()) { std::string string_value; RETURN_IF_ERR(value.AsString(&string_value)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetStringParameter( irequest, parameter.c_str(), string_value.c_str())); } else if (value.IsInt()) { int64_t int_value; RETURN_IF_ERR(value.AsInt(&int_value)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetIntParameter( irequest, parameter.c_str(), int_value)); } else if (value.IsBool()) { bool bool_value; RETURN_IF_ERR(value.AsBool(&bool_value)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetBoolParameter( irequest, parameter.c_str(), bool_value)); } else if (value.IsNumber()) { double double_value; RETURN_IF_ERR(value.AsDouble(&double_value)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetDoubleParameter( irequest, parameter.c_str(), double_value)); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, ("parameter '" + parameter + "' has invalid type. It should be either " "'int', 'bool', or 'string'.") .c_str()); } return nullptr; // success } } // namespace TRITONSERVER_Error* HTTPServer::Start() { if (!worker_.joinable()) { evbase_ = event_base_new(); htp_ = evhtp_new(evbase_, NULL); evhtp_enable_flag(htp_, EVHTP_FLAG_ENABLE_NODELAY); if (reuse_port_) { evhtp_enable_flag(htp_, EVHTP_FLAG_ENABLE_REUSEPORT); } evhtp_set_gencb(htp_, HTTPServer::Dispatch, this); evhtp_set_pre_accept_cb(htp_, HTTPServer::NewConnection, this); evhtp_use_threads_wexit(htp_, NULL, NULL, thread_cnt_, NULL); if (evhtp_bind_socket(htp_, address_.c_str(), port_, 1024) != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, (std::string("Socket '") + address_ + ":" + std::to_string(port_) + "' already in use ") .c_str()); } // Set listening event for breaking event loop evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, fds_); break_ev_ = event_new(evbase_, fds_[0], EV_READ, StopCallback, evbase_); event_add(break_ev_, NULL); worker_ = std::thread(event_base_loop, evbase_, 0); return nullptr; } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_ALREADY_EXISTS, "HTTP server is already running."); } TRITONSERVER_Error* HTTPServer::Stop(uint32_t* exit_timeout_secs, const std::string& service_name) { { std::lock_guard<std::mutex> lock(conn_mu_); accepting_new_conn_ = false; } if (exit_timeout_secs != nullptr) { // Note: conn_cnt_ can only decrease while (*exit_timeout_secs > 0 && conn_cnt_ > 0) { LOG_INFO << "Timeout " << *exit_timeout_secs << ": Found " << conn_cnt_ << " " << service_name << " service connections"; std::this_thread::sleep_for(std::chrono::seconds(1)); (*exit_timeout_secs)--; } } if (worker_.joinable()) { // Notify event loop to break via fd write send(fds_[1], (const char*)&evbase_, sizeof(event_base*), 0); worker_.join(); event_free(break_ev_); evutil_closesocket(fds_[0]); evutil_closesocket(fds_[1]); evhtp_unbind_socket(htp_); evhtp_free(htp_); event_base_free(evbase_); return nullptr; } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "HTTP server is not running."); } void HTTPServer::StopCallback(evutil_socket_t sock, short events, void* arg) { struct event_base* base = (struct event_base*)arg; event_base_loopbreak(base); } void HTTPServer::Dispatch(evhtp_request_t* req, void* arg) { (static_cast<HTTPServer*>(arg))->Handle(req); } evhtp_res HTTPServer::NewConnection(evhtp_connection_t* conn, void* arg) { HTTPServer* server = static_cast<HTTPServer*>(arg); { std::lock_guard<std::mutex> lock(server->conn_mu_); if (!server->accepting_new_conn_) { return EVHTP_RES_SERVUNAVAIL; // reset connection } server->conn_cnt_++; } evhtp_connection_set_hook( conn, evhtp_hook_on_connection_fini, (evhtp_hook)(void*)HTTPServer::EndConnection, arg); return EVHTP_RES_OK; } evhtp_res HTTPServer::EndConnection(evhtp_connection_t* conn, void* arg) { HTTPServer* server = static_cast<HTTPServer*>(arg); { std::lock_guard<std::mutex> lock(server->conn_mu_); server->conn_cnt_--; } return EVHTP_RES_OK; } #ifdef TRITON_ENABLE_METRICS void HTTPMetricsServer::Handle(evhtp_request_t* req) { LOG_VERBOSE(1) << "HTTP request: " << req->method << " " << req->uri->path->full; if (req->method != htp_method_GET) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } evhtp_headers_add_header( req->headers_out, evhtp_header_new(kContentTypeHeader, "text/plain; charset=utf-8", 1, 1)); // Call to metric endpoint should not have any trailing string if (RE2::FullMatch(std::string(req->uri->path->full), api_regex_)) { TRITONSERVER_Metrics* metrics = nullptr; TRITONSERVER_Error* err = TRITONSERVER_ServerMetrics(server_.get(), &metrics); if (err == nullptr) { const char* base; size_t byte_size; err = TRITONSERVER_MetricsFormatted( metrics, TRITONSERVER_METRIC_PROMETHEUS, &base, &byte_size); if (err == nullptr) { evbuffer_add(req->buffer_out, base, byte_size); } } TRITONSERVER_MetricsDelete(metrics); RETURN_AND_RESPOND_IF_ERR(req, err); TRITONSERVER_ErrorDelete(err); } evhtp_send_reply(req, EVHTP_RES_OK); } TRITONSERVER_Error* HTTPMetricsServer::Create( const std::shared_ptr<TRITONSERVER_Server>& server, const int32_t port, std::string address, const int thread_cnt, std::unique_ptr<HTTPServer>* metrics_server) { metrics_server->reset( new HTTPMetricsServer(server, port, address, thread_cnt)); const std::string addr = address + ":" + std::to_string(port); LOG_INFO << "Started Metrics Service at " << addr; return nullptr; } #endif // TRITON_ENABLE_METRICS namespace { // Allocate an evbuffer of size 'byte_size'. Return the 'evb' and // the 'base' address of the buffer contents. TRITONSERVER_Error* AllocEVBuffer(const size_t byte_size, evbuffer** evb, void** base) { evbuffer* evhttp_buffer = evbuffer_new(); if (evhttp_buffer == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "failed to create evbuffer for output tensor"); } // Reserve requested space in evbuffer... struct evbuffer_iovec output_iovec; if (evbuffer_reserve_space(evhttp_buffer, byte_size, &output_iovec, 1) != 1) { evbuffer_free(evhttp_buffer); return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "failed to reserve " + std::to_string(byte_size) + " bytes in output tensor buffer") .c_str()); } if (output_iovec.iov_len < byte_size) { evbuffer_free(evhttp_buffer); return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "reserved " + std::to_string(output_iovec.iov_len) + " bytes in output tensor buffer, need " + std::to_string(byte_size)) .c_str()); } output_iovec.iov_len = byte_size; *base = output_iovec.iov_base; // Immediately commit the buffer space. We are relying on evbuffer // not to relocate this space. Because we request a contiguous // chunk every time (above by allowing only a single entry in // output_iovec), this seems to be a valid assumption. if (evbuffer_commit_space(evhttp_buffer, &output_iovec, 1) != 0) { *base = nullptr; evbuffer_free(evhttp_buffer); return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "failed to commit output tensors to output buffer"); } *evb = evhttp_buffer; return nullptr; // success } // Recursively adds to byte_size from multi dimensional data input TRITONSERVER_Error* JsonBytesArrayByteSize( triton::common::TritonJson::Value& tensor_data, size_t* byte_size) { *byte_size = 0; // Recurse if not last dimension... if (tensor_data.IsArray()) { for (size_t i = 0; i < tensor_data.ArraySize(); i++) { triton::common::TritonJson::Value el; RETURN_IF_ERR(tensor_data.At(i, &el)); size_t byte_size_; RETURN_IF_ERR(JsonBytesArrayByteSize(el, &byte_size_)); *byte_size += byte_size_; } } else { // Serialized data size is the length of the string itself plus // 4 bytes to record the string length. const char* str; size_t len = 0; RETURN_MSG_IF_ERR( tensor_data.AsString(&str, &len), "Unable to parse JSON bytes array"); *byte_size += len + sizeof(uint32_t); } return nullptr; // success } TRITONSERVER_Error* ReadDataFromJsonHelper( char* base, const TRITONSERVER_DataType dtype, triton::common::TritonJson::Value& tensor_data, int* counter, int64_t expected_cnt) { // FIXME should move 'switch' statement outside the recursive function and // pass in a read data callback once data type is confirmed. // Currently 'switch' is performed on each element even through all elements // have the same data type. // Recurse on array element if not last dimension... if (tensor_data.IsArray()) { for (size_t i = 0; i < tensor_data.ArraySize(); i++) { triton::common::TritonJson::Value el; RETURN_IF_ERR(tensor_data.At(i, &el)); RETURN_IF_ERR( ReadDataFromJsonHelper(base, dtype, el, counter, expected_cnt)); } } else { // Check if writing to 'serialized' is overrunning the expected byte_size if (*counter >= expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Shape does not match true shape of 'data' field"); } switch (dtype) { case TRITONSERVER_TYPE_BOOL: { bool b = false; RETURN_IF_ERR(tensor_data.AsBool(&b)); uint8_t* data_vec = reinterpret_cast<uint8_t*>(base); // FIXME for unsigned should bounds check and raise error // since otherwise the actually used value will be // unexpected. data_vec[*counter] = (uint8_t)(b ? 1 : 0); *counter += 1; break; } case TRITONSERVER_TYPE_UINT8: { uint64_t ui = 0; RETURN_IF_ERR(tensor_data.AsUInt(&ui)); uint8_t* data_vec = reinterpret_cast<uint8_t*>(base); data_vec[*counter] = (uint8_t)ui; *counter += 1; break; } case TRITONSERVER_TYPE_UINT16: { uint64_t ui = 0; RETURN_IF_ERR(tensor_data.AsUInt(&ui)); uint16_t* data_vec = reinterpret_cast<uint16_t*>(base); data_vec[*counter] = (uint16_t)ui; *counter += 1; break; } case TRITONSERVER_TYPE_UINT32: { uint64_t ui = 0; RETURN_IF_ERR(tensor_data.AsUInt(&ui)); uint32_t* data_vec = reinterpret_cast<uint32_t*>(base); data_vec[*counter] = (uint32_t)ui; *counter += 1; break; } case TRITONSERVER_TYPE_UINT64: { uint64_t ui = 0; RETURN_IF_ERR(tensor_data.AsUInt(&ui)); uint64_t* data_vec = reinterpret_cast<uint64_t*>(base); data_vec[*counter] = ui; *counter += 1; break; } case TRITONSERVER_TYPE_INT8: { // FIXME for signed type just assigning to smaller type is // "implementation defined" and so really need to bounds // check. int64_t si = 0; RETURN_IF_ERR(tensor_data.AsInt(&si)); int8_t* data_vec = reinterpret_cast<int8_t*>(base); data_vec[*counter] = (int8_t)si; *counter += 1; break; } case TRITONSERVER_TYPE_INT16: { int64_t si = 0; RETURN_IF_ERR(tensor_data.AsInt(&si)); int16_t* data_vec = reinterpret_cast<int16_t*>(base); data_vec[*counter] = (int16_t)si; *counter += 1; break; } case TRITONSERVER_TYPE_INT32: { int64_t si = 0; RETURN_IF_ERR(tensor_data.AsInt(&si)); int32_t* data_vec = reinterpret_cast<int32_t*>(base); data_vec[*counter] = (int32_t)si; *counter += 1; break; } case TRITONSERVER_TYPE_INT64: { int64_t si = 0; RETURN_IF_ERR(tensor_data.AsInt(&si)); int64_t* data_vec = reinterpret_cast<int64_t*>(base); data_vec[*counter] = si; *counter += 1; break; } case TRITONSERVER_TYPE_FP32: { double fp64 = 0; RETURN_IF_ERR(tensor_data.AsDouble(&fp64)); float* data_vec = reinterpret_cast<float*>(base); data_vec[*counter] = fp64; *counter += 1; break; } case TRITONSERVER_TYPE_FP64: { double fp64 = 0; RETURN_IF_ERR(tensor_data.AsDouble(&fp64)); double* data_vec = reinterpret_cast<double*>(base); data_vec[*counter] = fp64; *counter += 1; break; } case TRITONSERVER_TYPE_BYTES: { const char* cstr; size_t len = 0; RETURN_IF_ERR(tensor_data.AsString(&cstr, &len)); if (static_cast<int64_t>(*counter + len + sizeof(uint32_t)) > expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Shape does not match true shape of 'data' field"); } memcpy( base + *counter, reinterpret_cast<char*>(&len), sizeof(uint32_t)); std::copy(cstr, cstr + len, base + *counter + sizeof(uint32_t)); *counter += len + sizeof(uint32_t); break; } default: break; } } return nullptr; // success } TRITONSERVER_Error* ReadDataFromJson( const char* tensor_name, triton::common::TritonJson::Value& tensor_data, char* base, const TRITONSERVER_DataType dtype, int64_t expected_cnt) { int counter = 0; switch (dtype) { // FP16 not supported via JSON case TRITONSERVER_TYPE_FP16: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "receiving FP16 data via JSON is not supported. Please use the " "binary data format for input " + std::string(tensor_name)) .c_str()); // BF16 not supported via JSON case TRITONSERVER_TYPE_BF16: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "receiving BF16 data via JSON is not supported. Please use the " "binary data format for input " + std::string(tensor_name)) .c_str()); case TRITONSERVER_TYPE_INVALID: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("invalid datatype for input " + std::string(tensor_name)) .c_str()); default: RETURN_MSG_IF_ERR( ReadDataFromJsonHelper( base, dtype, tensor_data, &counter, expected_cnt), "Unable to parse 'data'"); break; } // Check if 'ReadDataFromJsonHelper' reads less than the expected byte size if (counter != expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Unable to parse 'data': Shape does not match true shape of 'data' " "field"); } return nullptr; } TRITONSERVER_Error* WriteDataToJsonCheck( const std::string& output_name, const size_t byte_size, const size_t expected_size) { if (byte_size != expected_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "output tensor shape does not match size of output for '" + output_name + "'") .c_str()); } return nullptr; // success } TRITONSERVER_Error* WriteDataToJson( triton::common::TritonJson::Value* data_json, const std::string& output_name, const TRITONSERVER_DataType datatype, const void* base, const size_t byte_size, const size_t element_count) { switch (datatype) { case TRITONSERVER_TYPE_BOOL: { const uint8_t* bool_base = reinterpret_cast<const uint8_t*>(base); if (byte_size != (element_count * sizeof(uint8_t))) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "output tensor shape does not match size of output for '" + output_name + "'") .c_str()); } for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR( data_json->AppendBool((bool_base[e] == 0) ? false : true)); } break; } case TRITONSERVER_TYPE_UINT8: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(uint8_t) * element_count)); const uint8_t* cbase = reinterpret_cast<const uint8_t*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendUInt(cbase[e])); } break; } case TRITONSERVER_TYPE_UINT16: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(uint16_t) * element_count)); const uint16_t* cbase = reinterpret_cast<const uint16_t*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendUInt(cbase[e])); } break; } case TRITONSERVER_TYPE_UINT32: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(uint32_t) * element_count)); const uint32_t* cbase = reinterpret_cast<const uint32_t*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendUInt(cbase[e])); } break; } case TRITONSERVER_TYPE_UINT64: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(uint64_t) * element_count)); const uint64_t* cbase = reinterpret_cast<const uint64_t*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendUInt(cbase[e])); } break; } case TRITONSERVER_TYPE_INT8: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(int8_t) * element_count)); const int8_t* cbase = reinterpret_cast<const int8_t*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendInt(cbase[e])); } break; } case TRITONSERVER_TYPE_INT16: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(int16_t) * element_count)); const int16_t* cbase = reinterpret_cast<const int16_t*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendInt(cbase[e])); } break; } case TRITONSERVER_TYPE_INT32: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(int32_t) * element_count)); const int32_t* cbase = reinterpret_cast<const int32_t*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendInt(cbase[e])); } break; } case TRITONSERVER_TYPE_INT64: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(int64_t) * element_count)); const int64_t* cbase = reinterpret_cast<const int64_t*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendInt(cbase[e])); } break; } // FP16 not supported via JSON case TRITONSERVER_TYPE_FP16: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "sending FP16 data via JSON is not supported. Please use the " "binary data format for output"); // BF16 not supported via JSON case TRITONSERVER_TYPE_BF16: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "sending BF16 data via JSON is not supported. Please use the " "binary data format for output"); case TRITONSERVER_TYPE_FP32: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(float) * element_count)); const float* cbase = reinterpret_cast<const float*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendDouble(cbase[e])); } break; } case TRITONSERVER_TYPE_FP64: { RETURN_IF_ERR(WriteDataToJsonCheck( output_name, byte_size, sizeof(double) * element_count)); const double* cbase = reinterpret_cast<const double*>(base); for (size_t e = 0; e < element_count; ++e) { RETURN_IF_ERR(data_json->AppendDouble(cbase[e])); } break; } case TRITONSERVER_TYPE_BYTES: { const char* cbase = reinterpret_cast<const char*>(base); size_t offset = 0; for (size_t e = 0; e < element_count; ++e) { if ((offset + sizeof(uint32_t)) > byte_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "output tensor shape does not match size of output for '" + output_name + "'") .c_str()); } const size_t len = *(reinterpret_cast<const uint32_t*>(cbase + offset)); offset += sizeof(uint32_t); if ((offset + len) > byte_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "output tensor shape does not match size of output for '" + output_name + "'") .c_str()); } // Can use stringref because 'base' buffer is not deleted // until response is deleted and that happens after this json // is serialized. RETURN_IF_ERR(data_json->AppendStringRef(cbase + offset, len)); offset += len; } break; } case TRITONSERVER_TYPE_INVALID: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Invalid data type for output tensor"); } return nullptr; // success } TRITONSERVER_Error* CheckBinaryInputData( triton::common::TritonJson::Value& request_input, bool* is_binary, size_t* byte_size) { *is_binary = false; triton::common::TritonJson::Value params_json; if (request_input.Find("parameters", ¶ms_json)) { triton::common::TritonJson::Value binary_data_size_json; if (params_json.Find("binary_data_size", &binary_data_size_json)) { RETURN_MSG_IF_ERR( binary_data_size_json.AsUInt(reinterpret_cast<uint64_t*>(byte_size)), "Unable to parse 'binary_data_size'"); *is_binary = true; } } return nullptr; // success } TRITONSERVER_Error* CheckBinaryOutputData( triton::common::TritonJson::Value& request_output, bool* is_binary) { *is_binary = false; triton::common::TritonJson::Value params_json; if (request_output.Find("parameters", ¶ms_json)) { triton::common::TritonJson::Value binary_data_json; if (params_json.Find("binary_data", &binary_data_json)) { RETURN_MSG_IF_ERR( binary_data_json.AsBool(is_binary), "Unable to parse 'binary_data'"); } } return nullptr; // success } TRITONSERVER_Error* CheckSharedMemoryData( triton::common::TritonJson::Value& request_input, bool* use_shm, const char** shm_region, uint64_t* offset, uint64_t* byte_size) { *use_shm = false; *offset = 0; *byte_size = 0; triton::common::TritonJson::Value params_json; if (request_input.Find("parameters", ¶ms_json)) { { triton::common::TritonJson::Value region_json; if (params_json.Find("shared_memory_region", ®ion_json)) { *use_shm = true; size_t len; RETURN_MSG_IF_ERR( region_json.AsString(shm_region, &len), "Unable to parse 'shared_memory_region'"); } } { triton::common::TritonJson::Value offset_json; if (params_json.Find("shared_memory_offset", &offset_json)) { RETURN_MSG_IF_ERR( offset_json.AsUInt(offset), "Unable to parse 'shared_memory_offset'"); } } { triton::common::TritonJson::Value size_json; if (params_json.Find("shared_memory_byte_size", &size_json)) { RETURN_MSG_IF_ERR( size_json.AsUInt(byte_size), "Unable to parse 'shared_memory_byte_size'"); } } } return nullptr; // success } TRITONSERVER_Error* CheckClassificationOutput( triton::common::TritonJson::Value& request_output, uint64_t* num_classes) { *num_classes = 0; triton::common::TritonJson::Value params_json; if (request_output.Find("parameters", ¶ms_json)) { triton::common::TritonJson::Value cls_json; if (params_json.Find("classification", &cls_json)) { RETURN_MSG_IF_ERR( cls_json.AsUInt(num_classes), "Unable to set 'classification'"); } } return nullptr; // success } TRITONSERVER_Error* ValidateInputContentType(triton::common::TritonJson::Value& io) { bool has_data = false; bool has_binary = false; bool has_shared_memory = false; has_data = io.Find("data"); triton::common::TritonJson::Value params_json; if (io.Find("parameters", ¶ms_json)) { has_binary = params_json.Find("binary_data_size"); has_shared_memory = params_json.Find("shared_memory_region"); } int set_count = has_data + has_binary + has_shared_memory; if (set_count != 1) { std::string err_str = "Input must set only one of the following fields: 'data', " "'binary_data_size' in 'parameters', 'shared_memory_region' in " "'parameters'. But"; if (set_count == 0) { err_str += " no field is set"; } else { err_str += " set"; if (has_data) { err_str += " 'data'"; } if (has_binary) { err_str += " 'binary_data_size'"; } if (has_shared_memory) { err_str += " 'shared_memory_region'"; } } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, err_str.c_str()); } return nullptr; // success } TRITONSERVER_Error* ValidateOutputParameter(triton::common::TritonJson::Value& io) { triton::common::TritonJson::Value params_json; if (io.Find("parameters", ¶ms_json)) { const bool has_shared_memory = params_json.Find("shared_memory_region"); if (has_shared_memory) { // Currently shared memory can't set with classification because // cls results are not stored in shared memory, internally it is computed // based on results in shared memory. if (params_json.Find("classification")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Output can't set both 'shared_memory_region' and " "'classification'"); } triton::common::TritonJson::Value binary_data_json; if (params_json.Find("binary_data", &binary_data_json)) { bool is_binary = false; RETURN_MSG_IF_ERR( binary_data_json.AsBool(&is_binary), "Unable to set 'binary_data'"); if (is_binary) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Output can't set both 'shared_memory_region' and 'binary_data'"); } } } } return nullptr; // success } TRITONSERVER_Error* EVBufferToJson( triton::common::TritonJson::Value* document, evbuffer_iovec* v, int* v_idx, const size_t length, int n) { size_t offset = 0, remaining_length = length; char* json_base; std::vector<char> json_buffer; // No need to memcpy when number of iovecs is 1 if ((n > 0) && (v[0].iov_len >= remaining_length)) { json_base = static_cast<char*>(v[0].iov_base); if (v[0].iov_len > remaining_length) { v[0].iov_base = static_cast<void*>(json_base + remaining_length); v[0].iov_len -= remaining_length; remaining_length = 0; } else if (v[0].iov_len == remaining_length) { remaining_length = 0; *v_idx += 1; } } else { json_buffer.resize(length); json_base = json_buffer.data(); while ((remaining_length > 0) && (*v_idx < n)) { char* base = static_cast<char*>(v[*v_idx].iov_base); size_t base_size; if (v[*v_idx].iov_len > remaining_length) { base_size = remaining_length; v[*v_idx].iov_base = static_cast<void*>(base + remaining_length); v[*v_idx].iov_len -= remaining_length; remaining_length = 0; } else { base_size = v[*v_idx].iov_len; remaining_length -= v[*v_idx].iov_len; *v_idx += 1; } memcpy(json_base + offset, base, base_size); offset += base_size; } } if (remaining_length != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected size for request JSON, expecting " + std::to_string(remaining_length) + " more bytes") .c_str()); } RETURN_IF_ERR(document->Parse(json_base, length)); return nullptr; // success } std::string CompressionTypeUsed(const std::string accept_encoding) { std::vector<std::string> encodings; size_t offset = 0; size_t delimeter_pos = accept_encoding.find(','); while (delimeter_pos != std::string::npos) { encodings.emplace_back( accept_encoding.substr(offset, delimeter_pos - offset)); offset = delimeter_pos + 1; delimeter_pos = accept_encoding.find(',', offset); } std::string res = "identity"; double weight = 0; encodings.emplace_back(accept_encoding.substr(offset)); for (const auto& encoding : encodings) { auto start_pos = encoding.find_first_not_of(' '); auto weight_pos = encoding.find(";q="); // Skip if the encoding is malformed if ((start_pos == std::string::npos) || ((weight_pos != std::string::npos) && (start_pos >= weight_pos))) { continue; } const std::string type = (weight_pos == std::string::npos) ? encoding.substr(start_pos) : encoding.substr(start_pos, weight_pos - start_pos); double type_weight = 1; if (weight_pos != std::string::npos) { try { type_weight = std::stod(encoding.substr(weight_pos + 3)); } catch (const std::invalid_argument& ia) { continue; } } if (((type == "identity") || (type == "deflate") || (type == "gzip")) && (type_weight > weight)) { res = type; weight = type_weight; } } return res; } } // namespace HTTPAPIServer::HTTPAPIServer( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const int32_t port, const bool reuse_port, const std::string& address, const std::string& header_forward_pattern, const int thread_cnt, const RestrictedFeatures& restricted_apis) : HTTPServer(port, reuse_port, address, header_forward_pattern, thread_cnt), server_(server), trace_manager_(trace_manager), shm_manager_(shm_manager), allocator_(nullptr), server_regex_(R"(/v2(?:/health/(live|ready))?)"), model_regex_( R"(/v2/models/([^/]+)(?:/versions/([0-9]+))?(?:/(infer|generate|generate_stream|ready|config|stats|trace/setting))?)"), modelcontrol_regex_( R"(/v2/repository(?:/([^/]+))?/(index|models/([^/]+)/(load|unload)))"), systemsharedmemory_regex_( R"(/v2/systemsharedmemory(?:/region/([^/]+))?/(status|register|unregister))"), cudasharedmemory_regex_( R"(/v2/cudasharedmemory(?:/region/([^/]+))?/(status|register|unregister))"), trace_regex_(R"(/v2/trace/setting)"), restricted_apis_(restricted_apis) { // FIXME, don't cache server metadata. The http endpoint should // not be deciding that server metadata will not change during // execution. TRITONSERVER_Message* message = nullptr; server_metadata_err_ = TRITONSERVER_ServerMetadata(server_.get(), &message); if (server_metadata_err_ == nullptr) { const char* buffer; size_t byte_size; server_metadata_err_ = TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size); server_metadata_ = std::string(buffer, byte_size); } if (message != nullptr) { TRITONSERVER_MessageDelete(message); } FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator_, InferResponseAlloc, InferResponseFree, nullptr /* start_fn */), "creating response allocator"); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorSetQueryFunction( allocator_, OutputBufferQuery), "setting allocator's query function"); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction( allocator_, OutputBufferAttributes), "setting allocator's buffer attributes function"); ConfigureGenerateMappingSchema(); } HTTPAPIServer::~HTTPAPIServer() { if (server_metadata_err_ != nullptr) { TRITONSERVER_ErrorDelete(server_metadata_err_); } LOG_TRITONSERVER_ERROR( TRITONSERVER_ResponseAllocatorDelete(allocator_), "deleting response allocator"); } // Make sure to keep InferResponseAlloc, OutputBufferQuery, and // OutputBufferAttributes logic in sync TRITONSERVER_Error* HTTPAPIServer::InferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { AllocPayload* payload = reinterpret_cast<AllocPayload*>(userp); std::unordered_map<std::string, AllocPayload::OutputInfo*>& output_map = payload->output_map_; const AllocPayload::OutputInfo::Kind default_output_kind = payload->default_output_kind_; *buffer = nullptr; *buffer_userp = nullptr; *actual_memory_type = preferred_memory_type; *actual_memory_type_id = preferred_memory_type_id; AllocPayload::OutputInfo* info = nullptr; // If we don't find an output then it means that the output wasn't // explicitly specified in the request. In that case we create an // OutputInfo for it that uses default setting of JSON. auto pr = output_map.find(tensor_name); if (pr == output_map.end()) { info = new AllocPayload::OutputInfo(default_output_kind, 0); } else { // Take ownership of the OutputInfo object. info = pr->second; output_map.erase(pr); } // If the output is in shared memory... if (info->kind_ == AllocPayload::OutputInfo::SHM) { // ...then make sure shared memory size is at least as big as // the size of the output. if (byte_size > info->byte_size_) { const auto info_byte_size = info->byte_size_; delete info; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "shared memory size specified with the request for output '" + std::string(tensor_name) + "' (" + std::to_string(info_byte_size) + " bytes) should be at least " + std::to_string(byte_size) + " bytes to hold the results") .c_str()); } *buffer = const_cast<void*>(info->base_); *actual_memory_type = info->memory_type_; *actual_memory_type_id = info->device_id_; *buffer_userp = reinterpret_cast<void*>(info); LOG_VERBOSE(1) << "HTTP: using shared-memory for '" << tensor_name << "', size: " << byte_size << ", addr: " << *buffer; return nullptr; // Success } // Don't need to do anything if no memory was requested. if (byte_size > 0) { // Can't allocate for any memory type other than CPU. If asked to // allocate on GPU memory then force allocation on CPU instead. if (*actual_memory_type != TRITONSERVER_MEMORY_CPU) { LOG_VERBOSE(1) << "HTTP: unable to provide '" << tensor_name << "' in " << TRITONSERVER_MemoryTypeString(*actual_memory_type) << ", will use " << TRITONSERVER_MemoryTypeString(TRITONSERVER_MEMORY_CPU); *actual_memory_type = TRITONSERVER_MEMORY_CPU; *actual_memory_type_id = 0; } evbuffer* evhttp_buffer; TRITONSERVER_Error* err = AllocEVBuffer(byte_size, &evhttp_buffer, buffer); if (err != nullptr) { delete info; return err; } // Associate info with the evbuffer with this allocation. // Ownership passes to 'buffer_userp' which has the same lifetime // as the buffer itself. info->evbuffer_ = evhttp_buffer; LOG_VERBOSE(1) << "HTTP using buffer for: '" << tensor_name << "', size: " << byte_size << ", addr: " << *buffer; } *buffer_userp = reinterpret_cast<void*>(info); return nullptr; // Success } // Make sure to keep InferResponseAlloc, OutputBufferQuery, and // OutputBufferAttributes logic in sync TRITONSERVER_Error* HTTPAPIServer::OutputBufferAttributes( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, TRITONSERVER_BufferAttributes* buffer_attributes, void* userp, void* buffer_userp) { AllocPayload::OutputInfo* info = reinterpret_cast<AllocPayload::OutputInfo*>(buffer_userp); // We only need to set the cuda ipc handle here. The rest of the buffer // attributes have been properly populated by triton core. if (tensor_name != nullptr) { if (info->kind_ == AllocPayload::OutputInfo::SHM && info->memory_type_ == TRITONSERVER_MEMORY_GPU) { RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetCudaIpcHandle( buffer_attributes, info->cuda_ipc_handle_)); } } return nullptr; // Success } // Make sure to keep InferResponseAlloc and OutputBufferQuery logic in sync TRITONSERVER_Error* HTTPAPIServer::OutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id) { AllocPayload* payload = reinterpret_cast<AllocPayload*>(userp); if (tensor_name != nullptr) { auto pr = payload->output_map_.find(tensor_name); if ((pr != payload->output_map_.end()) && (pr->second->kind_ == AllocPayload::OutputInfo::SHM)) { // The output is in shared memory so check that shared memory // size is at least large enough for the output, if byte size is provided if ((byte_size != nullptr) && (*byte_size > pr->second->byte_size_)) { // Don't return error yet and just set to the default properties for // GRPC buffer, error will be raised when allocation happens *memory_type = TRITONSERVER_MEMORY_CPU; *memory_type_id = 0; } else { *memory_type = pr->second->memory_type_; *memory_type_id = pr->second->device_id_; } return nullptr; // Success } } // Not using shared memory so a evhtp buffer will be used, // and the type will be CPU. *memory_type = TRITONSERVER_MEMORY_CPU; *memory_type_id = 0; return nullptr; // Success } TRITONSERVER_Error* HTTPAPIServer::InferResponseFree( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { LOG_VERBOSE(1) << "HTTP release: " << "size " << byte_size << ", addr " << buffer; // 'buffer' is backed by shared memory or evbuffer so we don't // delete directly. auto info = reinterpret_cast<AllocPayload::OutputInfo*>(buffer_userp); delete info; return nullptr; // Success } void HTTPAPIServer::HandleServerHealth(evhtp_request_t* req, const std::string& kind) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::HEALTH, restricted_apis_); if (req->method != htp_method_GET) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } TRITONSERVER_Error* err = nullptr; bool ready = false; if (kind == "live") { err = TRITONSERVER_ServerIsLive(server_.get(), &ready); } else { err = TRITONSERVER_ServerIsReady(server_.get(), &ready); } RETURN_AND_RESPOND_IF_ERR(req, err); evhtp_send_reply(req, ready ? EVHTP_RES_OK : EVHTP_RES_BADREQ); } void HTTPAPIServer::HandleRepositoryIndex( evhtp_request_t* req, const std::string& repository_name) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::MODEL_REPOSITORY, restricted_apis_); AddContentTypeHeader(req, "application/json"); if (req->method != htp_method_POST) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } TRITONSERVER_Error* err = nullptr; struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting registry index request body"); } } bool ready = false; if (err == nullptr) { // If no request json then just use all default values. size_t buffer_len = evbuffer_get_length(req->buffer_in); if (buffer_len > 0) { triton::common::TritonJson::Value index_request; err = EVBufferToJson(&index_request, v, &v_idx, buffer_len, n); if (err == nullptr) { triton::common::TritonJson::Value ready_json; if (index_request.Find("ready", &ready_json)) { err = ready_json.AsBool(&ready); } } } } if (err == nullptr) { uint32_t flags = 0; if (ready) { flags |= TRITONSERVER_INDEX_FLAG_READY; } TRITONSERVER_Message* message = nullptr; err = TRITONSERVER_ServerModelIndex(server_.get(), flags, &message); if (err == nullptr) { const char* buffer; size_t byte_size; err = TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size); if (err == nullptr) { evbuffer_add(req->buffer_out, buffer, byte_size); evhtp_send_reply(req, EVHTP_RES_OK); } TRITONSERVER_MessageDelete(message); } } RETURN_AND_RESPOND_IF_ERR(req, err); } void HTTPAPIServer::HandleRepositoryControl( evhtp_request_t* req, const std::string& repository_name, const std::string& model_name, const std::string& action) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::MODEL_REPOSITORY, restricted_apis_); AddContentTypeHeader(req, "application/json"); if (req->method != htp_method_POST) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } TRITONSERVER_Error* err = nullptr; if (!repository_name.empty()) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "'repository_name' specification is not supported"); } else { if (action == "load") { struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting load model request buffers")); } } static auto param_deleter = [](std::vector<TRITONSERVER_Parameter*>* params) { if (params != nullptr) { for (auto& param : *params) { TRITONSERVER_ParameterDelete(param); } delete params; } }; std::unique_ptr< std::vector<TRITONSERVER_Parameter*>, decltype(param_deleter)> params(new std::vector<TRITONSERVER_Parameter*>(), param_deleter); // local variables to store the decoded file content, the data must // be valid until TRITONSERVER_ServerLoadModelWithParameters returns. std::list<std::vector<char>> binary_files; // WAR for the const-ness check std::vector<const TRITONSERVER_Parameter*> const_params; size_t buffer_len = evbuffer_get_length(req->buffer_in); if (buffer_len > 0) { triton::common::TritonJson::Value request; RETURN_AND_RESPOND_IF_ERR( req, EVBufferToJson(&request, v, &v_idx, buffer_len, n)); // Parse request body for parameters triton::common::TritonJson::Value param_json; if (request.Find("parameters", ¶m_json)) { // Iterate over each member in 'param_json' std::vector<std::string> members; RETURN_AND_RESPOND_IF_ERR(req, param_json.Members(&members)); for (const auto& m : members) { const char* param_str = nullptr; size_t param_len = 0; RETURN_AND_RESPOND_IF_ERR( req, param_json.MemberAsString(m.c_str(), ¶m_str, ¶m_len)); TRITONSERVER_Parameter* param = nullptr; if (m == "config") { param = TRITONSERVER_ParameterNew( m.c_str(), TRITONSERVER_PARAMETER_STRING, param_str); } else if (m.rfind("file:", 0) == 0) { // Decode base64 base64_decodestate s; base64_init_decodestate(&s); // The decoded can not be larger than the input... binary_files.emplace_back(std::vector<char>(param_len + 1)); size_t decoded_size = base64_decode_block( param_str, param_len, binary_files.back().data(), &s); param = TRITONSERVER_ParameterBytesNew( m.c_str(), binary_files.back().data(), decoded_size); } if (param != nullptr) { params->emplace_back(param); const_params.emplace_back(param); } else { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error on creating Triton parameter")); } } } } RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ServerLoadModelWithParameters( server_.get(), model_name.c_str(), const_params.data(), const_params.size())); } else if (action == "unload") { // Check if the dependent models should be removed bool unload_dependents = false; { struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting model control request body"); } } size_t buffer_len = evbuffer_get_length(req->buffer_in); if (buffer_len > 0) { triton::common::TritonJson::Value control_request; err = EVBufferToJson(&control_request, v, &v_idx, buffer_len, n); if (err == nullptr) { triton::common::TritonJson::Value params_json; if (control_request.Find("parameters", ¶ms_json)) { triton::common::TritonJson::Value ud_json; if (params_json.Find("unload_dependents", &ud_json)) { auto parse_err = ud_json.AsBool(&unload_dependents); if (parse_err != nullptr) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ErrorCode(parse_err), (std::string("Unable to parse 'unload_dependents': ") + TRITONSERVER_ErrorMessage(parse_err)) .c_str()); TRITONSERVER_ErrorDelete(parse_err); } } } } } } if (unload_dependents) { err = TRITONSERVER_ServerUnloadModelAndDependents( server_.get(), model_name.c_str()); } else { err = TRITONSERVER_ServerUnloadModel(server_.get(), model_name.c_str()); } } } RETURN_AND_RESPOND_IF_ERR(req, err); evhtp_send_reply(req, EVHTP_RES_OK); } void HTTPAPIServer::HandleModelReady( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::HEALTH, restricted_apis_); if (req->method != htp_method_GET) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } if (model_name.empty()) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_BADREQ, "Missing model name in ModelReady request"); } bool ready = false; int64_t requested_model_version; auto err = GetModelVersionFromString(model_version_str, &requested_model_version); if (err == nullptr) { err = TRITONSERVER_ServerModelIsReady( server_.get(), model_name.c_str(), requested_model_version, &ready); } if (!ready && !err) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_BADREQ, "Model version not ready"); } RETURN_AND_RESPOND_IF_ERR(req, err); evhtp_send_reply(req, EVHTP_RES_OK); } void HTTPAPIServer::HandleModelMetadata( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::METADATA, restricted_apis_); AddContentTypeHeader(req, "application/json"); if (req->method != htp_method_GET) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } if (model_name.empty()) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_BADREQ, "Missing model name in ModelMetadata request"); } TRITONSERVER_Message* message = nullptr; int64_t requested_model_version; auto err = GetModelVersionFromString(model_version_str, &requested_model_version); if (err == nullptr) { err = TRITONSERVER_ServerModelMetadata( server_.get(), model_name.c_str(), requested_model_version, &message); if (err == nullptr) { const char* buffer; size_t byte_size; err = TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size); if (err == nullptr) { evbuffer_add(req->buffer_out, buffer, byte_size); evhtp_send_reply(req, EVHTP_RES_OK); } TRITONSERVER_MessageDelete(message); } } RETURN_AND_RESPOND_IF_ERR(req, err); } TRITONSERVER_Error* HTTPAPIServer::GetModelConfig( const std::string& model_name, int64_t requested_model_version, std::string* config_json) { if (model_name.empty()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Missing model name in ModelConfig request"); } TRITONSERVER_Message* message = nullptr; RETURN_IF_ERR(TRITONSERVER_ServerModelConfig( server_.get(), model_name.c_str(), requested_model_version, 1 /* config_version */, &message)); const char* buffer; size_t byte_size; TRITONSERVER_Error* err = nullptr; err = TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size); if (err == nullptr) { // Copy config into string for simplicity *config_json = std::string(buffer, byte_size); } if (message) { TRITONSERVER_MessageDelete(message); } return err; } void HTTPAPIServer::HandleModelConfig( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::MODEL_CONFIG, restricted_apis_); AddContentTypeHeader(req, "application/json"); if (req->method != htp_method_GET) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } int64_t requested_model_version; RETURN_AND_RESPOND_IF_ERR( req, GetModelVersionFromString(model_version_str, &requested_model_version)); std::string config_json_str = ""; RETURN_AND_RESPOND_IF_ERR( req, GetModelConfig(model_name, requested_model_version, &config_json_str)); evbuffer_add( req->buffer_out, config_json_str.c_str(), config_json_str.size()); evhtp_send_reply(req, EVHTP_RES_OK); } void HTTPAPIServer::HandleModelStats( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::STATISTICS, restricted_apis_); AddContentTypeHeader(req, "application/json"); if (req->method != htp_method_GET) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } #ifdef TRITON_ENABLE_STATS TRITONSERVER_Message* model_stats_message = nullptr; int64_t requested_model_version; auto err = GetModelVersionFromString(model_version_str, &requested_model_version); if (err == nullptr) { err = TRITONSERVER_ServerModelStatistics( server_.get(), model_name.c_str(), requested_model_version, &model_stats_message); if (err == nullptr) { const char* buffer; size_t byte_size; err = TRITONSERVER_MessageSerializeToJson( model_stats_message, &buffer, &byte_size); if (err == nullptr) { // Add the statistics to the response evbuffer_add(req->buffer_out, buffer, byte_size); evhtp_send_reply(req, EVHTP_RES_OK); } TRITONSERVER_MessageDelete(model_stats_message); } } #else auto err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "the server does not support model statistics"); #endif RETURN_AND_RESPOND_IF_ERR(req, err); } void HTTPAPIServer::HandleTrace(evhtp_request_t* req, const std::string& model_name) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::TRACE, restricted_apis_); AddContentTypeHeader(req, "application/json"); if ((req->method != htp_method_GET) && (req->method != htp_method_POST)) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); return; } #ifdef TRITON_ENABLE_TRACING TRITONSERVER_InferenceTraceLevel level = TRITONSERVER_TRACE_LEVEL_DISABLED; uint32_t rate; int32_t count; uint32_t log_frequency; std::string filepath; InferenceTraceMode trace_mode; TraceConfigMap config_map; if (!model_name.empty()) { bool ready = false; RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ServerModelIsReady( server_.get(), model_name.c_str(), -1 /* model version */, &ready)); if (!ready) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, ("Request for unknown model : " + model_name).c_str())); } } // Perform trace setting update if requested if (req->method == htp_method_POST) { struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting trace request buffers")); } } triton::common::TritonJson::Value request; size_t buffer_len = evbuffer_get_length(req->buffer_in); RETURN_AND_RESPOND_IF_ERR( req, EVBufferToJson(&request, v, &v_idx, buffer_len, n)); TraceManager::NewSetting new_setting; triton::common::TritonJson::Value setting_json; if (request.Find("trace_file", &setting_json)) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "trace file location can not be updated through network " "protocol")); } if (request.Find("trace_level", &setting_json)) { if (setting_json.IsNull()) { new_setting.clear_level_ = true; } else { triton::common::TritonJson::Value level_array; RETURN_AND_RESPOND_IF_ERR( req, request.MemberAsArray("trace_level", &level_array)); for (size_t i = 0; i < level_array.ArraySize(); ++i) { std::string level_str; RETURN_AND_RESPOND_IF_ERR( req, level_array.IndexAsString(i, &level_str)); if (level_str == "OFF") { if (level_array.ArraySize() == 1) { level = TRITONSERVER_TRACE_LEVEL_DISABLED; new_setting.level_ = &level; } else { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Expect only one trace level 'OFF' is specified")); } } else if (level_str == "TIMESTAMPS") { level = static_cast<TRITONSERVER_InferenceTraceLevel>( level | TRITONSERVER_TRACE_LEVEL_TIMESTAMPS); new_setting.level_ = &level; } else if (level_str == "TENSORS") { level = static_cast<TRITONSERVER_InferenceTraceLevel>( level | TRITONSERVER_TRACE_LEVEL_TENSORS); new_setting.level_ = &level; } } } } if (request.Find("trace_rate", &setting_json)) { if (setting_json.IsNull()) { new_setting.clear_rate_ = true; } else { std::string rate_str; RETURN_AND_RESPOND_IF_ERR(req, setting_json.AsString(&rate_str)); try { rate = std::stoi(rate_str); new_setting.rate_ = &rate; } catch (const std::invalid_argument& ia) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse 'trace_rate', got: ") + rate_str) .c_str())); } catch (const std::out_of_range& oor) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse 'trace_rate', value is out of " "range [ ") + std::to_string(std::numeric_limits<std::uint32_t>::min()) + ", " + std::to_string(std::numeric_limits<std::uint32_t>::max()) + " ], got: " + rate_str) .c_str())); } } } if (request.Find("trace_count", &setting_json)) { if (setting_json.IsNull()) { new_setting.clear_count_ = true; } else { std::string count_str; RETURN_AND_RESPOND_IF_ERR(req, setting_json.AsString(&count_str)); try { count = std::stoi(count_str); if (count < TraceManager::MIN_TRACE_COUNT_VALUE) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse 'trace_count'.") + " Expecting value >= " + std::to_string(TraceManager::MIN_TRACE_COUNT_VALUE) + ", got:" + count_str) .c_str())); } new_setting.count_ = &count; } catch (const std::invalid_argument& ia) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse 'trace_count', got: ") + count_str) .c_str())); } catch (const std::out_of_range& oor) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse 'trace_count', value is out of " "range [ ") + std::to_string(TraceManager::MIN_TRACE_COUNT_VALUE) + ", " + std::to_string(std::numeric_limits<std::int32_t>::max()) + " ], got: " + count_str) .c_str())); } } } if (request.Find("log_frequency", &setting_json)) { if (setting_json.IsNull()) { new_setting.clear_log_frequency_ = true; } else { std::string frequency_str; RETURN_AND_RESPOND_IF_ERR(req, setting_json.AsString(&frequency_str)); try { log_frequency = std::stoi(frequency_str); new_setting.log_frequency_ = &log_frequency; } catch (const std::invalid_argument& ia) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse 'log_frequency', got: ") + frequency_str) .c_str())); } catch (const std::out_of_range& oor) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "Unable to parse 'log_frequency', value is out of " "range [ ") + std::to_string(std::numeric_limits<std::uint32_t>::min()) + ", " + std::to_string(std::numeric_limits<std::uint32_t>::max()) + " ], got: " + frequency_str) .c_str())); } } } RETURN_AND_RESPOND_IF_ERR( req, trace_manager_->UpdateTraceSetting(model_name, new_setting)); } // Get current trace setting, this is needed even if the setting // has been updated above as some values may not be provided in the request. trace_manager_->GetTraceSetting( model_name, &level, &rate, &count, &log_frequency, &filepath, &trace_mode, &config_map); triton::common::TritonJson::Value trace_response( triton::common::TritonJson::ValueType::OBJECT); // level { triton::common::TritonJson::Value level_array( triton::common::TritonJson::ValueType::ARRAY); if (level == TRITONSERVER_TRACE_LEVEL_DISABLED) { RETURN_AND_RESPOND_IF_ERR(req, level_array.AppendString("OFF")); } else { if (level & TRITONSERVER_TRACE_LEVEL_TIMESTAMPS) { RETURN_AND_RESPOND_IF_ERR(req, level_array.AppendString("TIMESTAMPS")); } if (level & TRITONSERVER_TRACE_LEVEL_TENSORS) { RETURN_AND_RESPOND_IF_ERR(req, level_array.AppendString("TENSORS")); } } RETURN_AND_RESPOND_IF_ERR( req, trace_response.Add("trace_level", std::move(level_array))); } RETURN_AND_RESPOND_IF_ERR( req, trace_response.AddString("trace_rate", std::to_string(rate))); RETURN_AND_RESPOND_IF_ERR( req, trace_response.AddString("trace_count", std::to_string(count))); if (trace_mode == TRACE_MODE_TRITON) { RETURN_AND_RESPOND_IF_ERR( req, trace_response.AddString( "log_frequency", std::to_string(log_frequency))); RETURN_AND_RESPOND_IF_ERR( req, trace_response.AddString("trace_file", filepath)); } RETURN_AND_RESPOND_IF_ERR( req, trace_response.AddString( "trace_mode", trace_manager_->InferenceTraceModeString(trace_mode))); auto mode_key = std::to_string(trace_mode); auto trace_options_it = config_map.find(mode_key); if (trace_options_it != config_map.end()) { for (const auto& [key, value] : trace_options_it->second) { if ((key == "file") || (key == "log-frequency")) { continue; } std::string valueAsString; if (std::holds_alternative<std::string>(value)) { valueAsString = std::get<std::string>(value); } else if (std::holds_alternative<int>(value)) { valueAsString = std::to_string(std::get<int>(value)); } else if (std::holds_alternative<uint32_t>(value)) { valueAsString = std::to_string(std::get<uint32_t>(value)); } RETURN_AND_RESPOND_IF_ERR( req, trace_response.AddString(key.c_str(), valueAsString)); } } triton::common::TritonJson::WriteBuffer buffer; RETURN_AND_RESPOND_IF_ERR(req, trace_response.Write(&buffer)); evbuffer_add(req->buffer_out, buffer.Base(), buffer.Size()); evhtp_send_reply(req, EVHTP_RES_OK); #else RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "the server does not support tracing")); #endif } void HTTPAPIServer::HandleLogging(evhtp_request_t* req) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::LOGGING, restricted_apis_); AddContentTypeHeader(req, "application/json"); if ((req->method != htp_method_GET) && (req->method != htp_method_POST)) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } #ifdef TRITON_ENABLE_LOGGING // Perform log setting update if requested if (req->method == htp_method_POST) { struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting dynamic logging request buffers")); } } triton::common::TritonJson::Value request; size_t buffer_len = evbuffer_get_length(req->buffer_in); RETURN_AND_RESPOND_IF_ERR( req, EVBufferToJson(&request, v, &v_idx, buffer_len, n)); // Server and Core repos do not have the same Logger object // Each update must be applied to both server and core repo versions triton::common::TritonJson::Value setting_json; if (request.Find("log_file", &setting_json)) { if (!setting_json.IsNull()) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "log file location can not be updated through network " "protocol")); } } if (request.Find("log_info", &setting_json)) { if (!setting_json.IsNull()) { bool log_info_status; RETURN_AND_RESPOND_IF_ERR(req, setting_json.AsBool(&log_info_status)); LOG_ENABLE_INFO(log_info_status); TRITONSERVER_ServerOptionsSetLogInfo(nullptr, log_info_status); } } if (request.Find("log_warning", &setting_json)) { if (!setting_json.IsNull()) { bool log_warn_status; RETURN_AND_RESPOND_IF_ERR(req, setting_json.AsBool(&log_warn_status)); LOG_ENABLE_WARNING(log_warn_status); TRITONSERVER_ServerOptionsSetLogWarn(nullptr, log_warn_status); } } if (request.Find("log_error", &setting_json)) { if (!setting_json.IsNull()) { bool log_error_status; RETURN_AND_RESPOND_IF_ERR(req, setting_json.AsBool(&log_error_status)); LOG_ENABLE_ERROR(log_error_status); TRITONSERVER_ServerOptionsSetLogError(nullptr, log_error_status); } } if (request.Find("log_verbose_level", &setting_json)) { if (!setting_json.IsNull()) { uint64_t verbose_level; RETURN_AND_RESPOND_IF_ERR(req, setting_json.AsUInt(&verbose_level)); LOG_SET_VERBOSE(static_cast<int32_t>(verbose_level)); TRITONSERVER_ServerOptionsSetLogVerbose( nullptr, static_cast<int32_t>(verbose_level)); } } if (request.Find("log_format", &setting_json)) { if (!setting_json.IsNull()) { std::string log_format_parse; RETURN_AND_RESPOND_IF_ERR( req, setting_json.AsString(&log_format_parse)); triton::common::Logger::Format log_format_final = triton::common::Logger::Format::kDEFAULT; if (log_format_parse == "ISO8601") { log_format_final = triton::common::Logger::Format::kISO8601; } else if (log_format_parse != "default") { // Returns from function RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, ("invalid argument for --log_format, got: " + log_format_parse) .c_str())); } LOG_SET_FORMAT(log_format_final); switch (log_format_final) { case triton::common::Logger::Format::kDEFAULT: TRITONSERVER_ServerOptionsSetLogFormat( nullptr, TRITONSERVER_LOG_DEFAULT); break; case triton::common::Logger::Format::kISO8601: TRITONSERVER_ServerOptionsSetLogFormat( nullptr, TRITONSERVER_LOG_ISO8601); break; } } } } triton::common::TritonJson::Value log_setting_response( triton::common::TritonJson::ValueType::OBJECT); RETURN_AND_RESPOND_IF_ERR( req, log_setting_response.AddString("log_file", LOG_FILE)); RETURN_AND_RESPOND_IF_ERR( req, log_setting_response.AddBool("log_info", LOG_INFO_IS_ON)); RETURN_AND_RESPOND_IF_ERR( req, log_setting_response.AddBool("log_warning", LOG_WARNING_IS_ON)); RETURN_AND_RESPOND_IF_ERR( req, log_setting_response.AddBool("log_error", LOG_ERROR_IS_ON)); RETURN_AND_RESPOND_IF_ERR( req, log_setting_response.AddInt( "log_verbose_level", static_cast<uint64_t>(LOG_VERBOSE_LEVEL))); RETURN_AND_RESPOND_IF_ERR( req, log_setting_response.AddString("log_format", LOG_FORMAT_STRING)); triton::common::TritonJson::WriteBuffer buffer; RETURN_AND_RESPOND_IF_ERR(req, log_setting_response.Write(&buffer)); evbuffer_add(req->buffer_out, buffer.Base(), buffer.Size()); evhtp_send_reply(req, EVHTP_RES_OK); #else RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNAVAILABLE, "the server does not support dynamic logging")); #endif // TRITON_ENABLE_LOGGING } void HTTPAPIServer::HandleServerMetadata(evhtp_request_t* req) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::METADATA, restricted_apis_); AddContentTypeHeader(req, "application/json"); if (req->method != htp_method_GET) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } if (server_metadata_err_ == nullptr) { evbuffer_add( req->buffer_out, server_metadata_.c_str(), server_metadata_.size()); evhtp_send_reply(req, EVHTP_RES_OK); } else { // Not using RETURN_AND_RESPOND_IF_ERR macro as the Triton error can // be persistent, the macro will clean up the error object. EVBufferAddErrorJson(req->buffer_out, server_metadata_err_); evhtp_send_reply(req, HttpCodeFromError(server_metadata_err_)); } } void HTTPAPIServer::HandleSystemSharedMemory( evhtp_request_t* req, const std::string& region_name, const std::string& action) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::SHARED_MEMORY, restricted_apis_); AddContentTypeHeader(req, "application/json"); if ((action == "status") && (req->method != htp_method_GET)) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } else if ((action != "status") && (req->method != htp_method_POST)) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } TRITONSERVER_Error* err = nullptr; if (action == "status") { triton::common::TritonJson::Value shm_status( triton::common::TritonJson::ValueType::ARRAY); err = shm_manager_->GetStatus( region_name, TRITONSERVER_MEMORY_CPU, &shm_status); if (err == nullptr) { triton::common::TritonJson::WriteBuffer buffer; err = shm_status.Write(&buffer); if (err == nullptr) { evbuffer_add(req->buffer_out, buffer.Base(), buffer.Size()); } } } else if (action == "register") { if (region_name.empty()) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "'region name' is necessary to register system shared memory region"); } else { struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting register request buffers"); } } if (err == nullptr) { triton::common::TritonJson::Value register_request; size_t buffer_len = evbuffer_get_length(req->buffer_in); err = EVBufferToJson(®ister_request, v, &v_idx, buffer_len, n); if (err == nullptr) { triton::common::TritonJson::Value key_json; if (!register_request.Find("key", &key_json)) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Shared memory register request has no 'key' field"); } const char* shm_key = nullptr; if (err == nullptr) { size_t shm_key_len; err = key_json.AsString(&shm_key, &shm_key_len); } uint64_t offset = 0; if (err == nullptr) { triton::common::TritonJson::Value offset_json; if (register_request.Find("offset", &offset_json)) { err = offset_json.AsUInt(&offset); } } uint64_t byte_size = 0; if (err == nullptr) { triton::common::TritonJson::Value byte_size_json; if (!register_request.Find("byte_size", &byte_size_json)) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Shared memory register request has no 'byte_size' field"); } else { err = byte_size_json.AsUInt(&byte_size); } } if (err == nullptr) { err = shm_manager_->RegisterSystemSharedMemory( region_name, shm_key, offset, byte_size); } } } } } else if (action == "unregister") { if (region_name.empty()) { err = shm_manager_->UnregisterAll(TRITONSERVER_MEMORY_CPU); } else { err = shm_manager_->Unregister(region_name, TRITONSERVER_MEMORY_CPU); } } RETURN_AND_RESPOND_IF_ERR(req, err); evhtp_send_reply(req, EVHTP_RES_OK); } void HTTPAPIServer::HandleCudaSharedMemory( evhtp_request_t* req, const std::string& region_name, const std::string& action) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::SHARED_MEMORY, restricted_apis_); AddContentTypeHeader(req, "application/json"); if ((action == "status") && (req->method != htp_method_GET)) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } else if ((action != "status") && (req->method != htp_method_POST)) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } TRITONSERVER_Error* err = nullptr; if (action == "status") { triton::common::TritonJson::Value shm_status( triton::common::TritonJson::ValueType::ARRAY); err = shm_manager_->GetStatus( region_name, TRITONSERVER_MEMORY_GPU, &shm_status); if (err == nullptr) { triton::common::TritonJson::WriteBuffer buffer; err = shm_status.Write(&buffer); if (err == nullptr) { evbuffer_add(req->buffer_out, buffer.Base(), buffer.Size()); } } } else if (action == "register") { if (region_name.empty()) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "'region name' is necessary to register cuda shared memory region"); } else { #ifdef TRITON_ENABLE_GPU struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting register request buffers"); } } if (err == nullptr) { triton::common::TritonJson::Value register_request; size_t buffer_len = evbuffer_get_length(req->buffer_in); err = EVBufferToJson(®ister_request, v, &v_idx, buffer_len, n); if (err == nullptr) { const char* b64_handle = nullptr; size_t b64_handle_len = 0; triton::common::TritonJson::Value raw_handle_json; if (!register_request.Find("raw_handle", &raw_handle_json)) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Shared memory register request has no 'raw_handle' field"); } else { err = raw_handle_json.MemberAsString( "b64", &b64_handle, &b64_handle_len); } uint64_t byte_size = 0; if (err == nullptr) { err = register_request.MemberAsUInt("byte_size", &byte_size); } uint64_t device_id = 0; if (err == nullptr) { err = register_request.MemberAsUInt("device_id", &device_id); } if (err == nullptr) { base64_decodestate s; base64_init_decodestate(&s); // The decoded can not be larger than the input... std::vector<char> raw_handle(b64_handle_len + 1); size_t decoded_size = base64_decode_block( b64_handle, b64_handle_len, raw_handle.data(), &s); if (decoded_size != sizeof(cudaIpcMemHandle_t)) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "'raw_handle' must be a valid base64 encoded " "cudaIpcMemHandle_t"); } else { raw_handle.resize(sizeof(cudaIpcMemHandle_t)); err = shm_manager_->RegisterCUDASharedMemory( region_name.c_str(), reinterpret_cast<const cudaIpcMemHandle_t*>( raw_handle.data()), byte_size, device_id); } } } } #else err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "failed to register CUDA shared memory region: '" + region_name + "', GPUs not supported") .c_str()); #endif // TRITON_ENABLE_GPU } } else if (action == "unregister") { if (region_name.empty()) { err = shm_manager_->UnregisterAll(TRITONSERVER_MEMORY_GPU); } else { err = shm_manager_->Unregister(region_name, TRITONSERVER_MEMORY_GPU); } } RETURN_AND_RESPOND_IF_ERR(req, err); evhtp_send_reply(req, EVHTP_RES_OK); } TRITONSERVER_Error* HTTPAPIServer::GetContentLength( evhtp_request_t* req, evbuffer* decompressed_buffer, int32_t* content_length) { TRITONSERVER_Error* err = nullptr; // Set to body size in case there is no Content-Length to compare with int32_t lcontent_length = evbuffer_get_length(req->buffer_in); if (decompressed_buffer == nullptr) { const char* content_length_c_str = evhtp_kv_find(req->headers_in, kContentLengthHeader); if (content_length_c_str != nullptr) { try { lcontent_length = std::atoi(content_length_c_str); } catch (const std::invalid_argument& ia) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse ") + kContentLengthHeader + ", got: " + content_length_c_str) .c_str()); } } } else { // The Content-Length doesn't reflect the actual request body size // if compression is used, set 'content_length' to the decompressed size lcontent_length = evbuffer_get_length(decompressed_buffer); } *content_length = lcontent_length; return err; } TRITONSERVER_Error* HTTPAPIServer::GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, size_t* header_length) { // Set to content length in case that the header is not specified *header_length = content_length; // Find Inference-Header-Content-Length in header. const char* header_length_c_str = evhtp_kv_find(req->headers_in, kInferHeaderContentLengthHTTPHeader); if (header_length_c_str != NULL) { int parsed_value; try { parsed_value = std::atoi(header_length_c_str); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse ") + kInferHeaderContentLengthHTTPHeader + ", got: " + header_length_c_str) .c_str()); } // Check if the content length is in proper range if ((parsed_value < 0) || (parsed_value > content_length)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("inference header size should be in range (0, ") + std::to_string(content_length) + "), got: " + header_length_c_str) .c_str()); } *header_length = parsed_value; } return nullptr; } DataCompressor::Type HTTPAPIServer::GetRequestCompressionType(evhtp_request_t* req) { const char* content_encoding_c_str = evhtp_kv_find(req->headers_in, kContentEncodingHTTPHeader); if (content_encoding_c_str != NULL) { std::string content_encoding(content_encoding_c_str); if (content_encoding == "deflate") { return DataCompressor::Type::DEFLATE; } else if (content_encoding == "gzip") { return DataCompressor::Type::GZIP; } else if (!content_encoding.empty() && (content_encoding != "identity")) { return DataCompressor::Type::UNKNOWN; } } return DataCompressor::Type::IDENTITY; } DataCompressor::Type HTTPAPIServer::GetResponseCompressionType(evhtp_request_t* req) { // Find Accept-Encoding in header. Try to compress if found const char* accept_encoding_c_str = evhtp_kv_find(req->headers_in, kAcceptEncodingHTTPHeader); if (accept_encoding_c_str != NULL) { std::string accept_encoding = CompressionTypeUsed(accept_encoding_c_str); if (accept_encoding == "deflate") { return DataCompressor::Type::DEFLATE; } else if (accept_encoding == "gzip") { return DataCompressor::Type::GZIP; } } return DataCompressor::Type::IDENTITY; } // Helpers for parsing JSON requests for Triton-specific fields TRITONSERVER_Error* HTTPAPIServer::ParseJsonTritonIO( triton::common::TritonJson::Value& request_json, TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, const std::string& model_name, evbuffer_iovec* v, int* v_idx_ptr, size_t header_length, int n) { // Get the byte-size for each input and from that get the blocks // holding the data for that input triton::common::TritonJson::Value inputs_json; RETURN_MSG_IF_ERR( request_json.MemberAsArray("inputs", &inputs_json), "Unable to parse 'inputs'"); int& v_idx = *v_idx_ptr; for (size_t i = 0; i < inputs_json.ArraySize(); i++) { triton::common::TritonJson::Value request_input; RETURN_IF_ERR(inputs_json.At(i, &request_input)); RETURN_IF_ERR(ValidateInputContentType(request_input)); const char* input_name; size_t input_name_len; RETURN_MSG_IF_ERR( request_input.MemberAsString("name", &input_name, &input_name_len), "Unable to parse 'name'"); const char* datatype; size_t datatype_len; RETURN_MSG_IF_ERR( request_input.MemberAsString("datatype", &datatype, &datatype_len), "Unable to parse 'datatype'"); const TRITONSERVER_DataType dtype = TRITONSERVER_StringToDataType(datatype); triton::common::TritonJson::Value shape_json; RETURN_MSG_IF_ERR( request_input.MemberAsArray("shape", &shape_json), "Unable to parse 'shape'"); std::vector<int64_t> shape_vec; for (size_t i = 0; i < shape_json.ArraySize(); i++) { uint64_t d = 0; RETURN_MSG_IF_ERR( shape_json.IndexAsUInt(i, &d), "Unable to parse 'shape'"); shape_vec.push_back(d); } RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput( irequest, input_name, dtype, &shape_vec[0], shape_vec.size())); bool binary_input; size_t byte_size; RETURN_IF_ERR( CheckBinaryInputData(request_input, &binary_input, &byte_size)); if ((byte_size == 0) && binary_input) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( irequest, input_name, nullptr, 0 /* byte_size */, TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } else if (binary_input) { if (header_length == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "must specify valid 'Infer-Header-Content-Length' in request " "header and 'binary_data_size' when passing inputs in binary " "data format"); } // Process one block at a time while ((byte_size > 0) && (v_idx < n)) { char* base = static_cast<char*>(v[v_idx].iov_base); size_t base_size; if (v[v_idx].iov_len > byte_size) { base_size = byte_size; v[v_idx].iov_base = static_cast<void*>(base + byte_size); v[v_idx].iov_len -= byte_size; byte_size = 0; } else { base_size = v[v_idx].iov_len; byte_size -= v[v_idx].iov_len; v_idx++; } RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( irequest, input_name, base, base_size, TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } if (byte_size != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected size for input '" + std::string(input_name) + "', expecting " + std::to_string(byte_size) + " additional bytes for model '" + model_name + "'") .c_str()); } } else { // Process input if in shared memory. bool use_shm; uint64_t shm_offset; const char* shm_region; RETURN_IF_ERR(CheckSharedMemoryData( request_input, &use_shm, &shm_region, &shm_offset, reinterpret_cast<uint64_t*>(&byte_size))); if (use_shm) { void* base; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; RETURN_IF_ERR(shm_manager_->GetMemoryInfo( shm_region, shm_offset, byte_size, &base, &memory_type, &memory_type_id)); if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU cudaIpcMemHandle_t* cuda_handle; RETURN_IF_ERR(shm_manager_->GetCUDAHandle(shm_region, &cuda_handle)); TRITONSERVER_BufferAttributes* buffer_attributes; RETURN_IF_ERR(TRITONSERVER_BufferAttributesNew(&buffer_attributes)); auto buffer_attributes_del = [](TRITONSERVER_BufferAttributes* buffer_attributes) { TRITONSERVER_BufferAttributesDelete(buffer_attributes); }; std::unique_ptr< TRITONSERVER_BufferAttributes, decltype(buffer_attributes_del)> buffer_attrsl(buffer_attributes, buffer_attributes_del); RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetMemoryType( buffer_attributes, memory_type)); RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetMemoryTypeId( buffer_attributes, memory_type_id)); RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetCudaIpcHandle( buffer_attributes, reinterpret_cast<void*>(cuda_handle))); RETURN_IF_ERR(TRITONSERVER_BufferAttributesSetByteSize( buffer_attributes, byte_size)); RETURN_IF_ERR( TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes( irequest, input_name, base, buffer_attributes)); #endif } else { RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( irequest, input_name, base, byte_size, memory_type, memory_type_id)); } } else { const int64_t element_cnt = GetElementCount(shape_vec); // FIXME, element count should never be 0 or negative so // shouldn't we just return an error here? if (element_cnt == 0) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( irequest, input_name, nullptr, 0 /* byte_size */, TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } else { // JSON... presence of "data" already validated but still // checking here. Flow in this endpoint needs to be // reworked... triton::common::TritonJson::Value tensor_data; RETURN_MSG_IF_ERR( request_input.MemberAsArray("data", &tensor_data), "Unable to parse 'data'"); if (dtype == TRITONSERVER_TYPE_BYTES) { RETURN_IF_ERR(JsonBytesArrayByteSize(tensor_data, &byte_size)); } else { byte_size = element_cnt * TRITONSERVER_DataTypeByteSize(dtype); } infer_req->serialized_data_.emplace_back(); std::vector<char>& serialized = infer_req->serialized_data_.back(); serialized.resize(byte_size); RETURN_IF_ERR(ReadDataFromJson( input_name, tensor_data, &serialized[0], dtype, dtype == TRITONSERVER_TYPE_BYTES ? byte_size : element_cnt)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( irequest, input_name, &serialized[0], serialized.size(), TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } } } } if (v_idx != n) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected additional input data for model '" + model_name + "'") .c_str()); } // outputs is optional if (request_json.Find("outputs")) { triton::common::TritonJson::Value outputs_json; RETURN_MSG_IF_ERR( request_json.MemberAsArray("outputs", &outputs_json), "Unable to parse 'outputs'"); for (size_t i = 0; i < outputs_json.ArraySize(); i++) { triton::common::TritonJson::Value request_output; RETURN_IF_ERR(outputs_json.At(i, &request_output)); RETURN_IF_ERR(ValidateOutputParameter(request_output)); const char* output_name; size_t output_name_len; RETURN_MSG_IF_ERR( request_output.MemberAsString("name", &output_name, &output_name_len), "Unable to parse 'name'"); RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddRequestedOutput( irequest, output_name)); uint64_t class_size; RETURN_IF_ERR(CheckClassificationOutput(request_output, &class_size)); bool use_shm; uint64_t offset, byte_size; const char* shm_region; RETURN_IF_ERR(CheckSharedMemoryData( request_output, &use_shm, &shm_region, &offset, &byte_size)); // ValidateOutputParameter ensures that both shm and // classification cannot be true. if (use_shm) { void* base; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; RETURN_IF_ERR(shm_manager_->GetMemoryInfo( shm_region, offset, byte_size, &base, &memory_type, &memory_type_id)); if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU cudaIpcMemHandle_t* cuda_handle; RETURN_IF_ERR(shm_manager_->GetCUDAHandle(shm_region, &cuda_handle)); infer_req->alloc_payload_.output_map_.emplace( std::piecewise_construct, std::forward_as_tuple(output_name), std::forward_as_tuple(new AllocPayload::OutputInfo( base, byte_size, memory_type, memory_type_id, reinterpret_cast<char*>(cuda_handle)))); #endif } else { infer_req->alloc_payload_.output_map_.emplace( std::piecewise_construct, std::forward_as_tuple(output_name), std::forward_as_tuple(new AllocPayload::OutputInfo( base, byte_size, memory_type, memory_type_id, nullptr /* cuda ipc handle */))); } } else { bool use_binary; RETURN_IF_ERR(CheckBinaryOutputData(request_output, &use_binary)); infer_req->alloc_payload_.output_map_.emplace( std::piecewise_construct, std::forward_as_tuple(output_name), std::forward_as_tuple(new AllocPayload::OutputInfo( use_binary ? AllocPayload::OutputInfo::BINARY : AllocPayload::OutputInfo::JSON, class_size))); } } } return nullptr; // success } TRITONSERVER_Error* HTTPAPIServer::ParseJsonTritonParams( triton::common::TritonJson::Value& request_json, TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req) { // The default setting for returned outputs (JSON or BINARY). This // is needed for the case when outputs are not explicitly specified. AllocPayload::OutputInfo::Kind output_kind = AllocPayload::OutputInfo::JSON; triton::common::TritonJson::Value params_json; if (request_json.Find("parameters", ¶ms_json)) { std::vector<std::string> parameters; RETURN_MSG_IF_ERR( params_json.Members(¶meters), "failed to get request params."); uint32_t flags = 0; for (auto& parameter : parameters) { if (parameter == "sequence_id") { uint64_t seq_id; // Try to parse sequence_id as uint64_t TRITONSERVER_Error* err; if ((err = params_json.MemberAsUInt(parameter.c_str(), &seq_id)) != nullptr) { TRITONSERVER_ErrorDelete(err); // On failure try to parse as a string std::string seq_id; RETURN_MSG_IF_ERR( params_json.MemberAsString(parameter.c_str(), &seq_id), "Unable to parse 'sequence_id'"); RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetCorrelationIdString( irequest, seq_id.c_str())); } else { RETURN_IF_ERR( TRITONSERVER_InferenceRequestSetCorrelationId(irequest, seq_id)); } } else if (parameter == "sequence_start") { bool start; RETURN_MSG_IF_ERR( params_json.MemberAsBool(parameter.c_str(), &start), "Unable to parse 'sequence_start'"); if (start) { flags |= TRITONSERVER_REQUEST_FLAG_SEQUENCE_START; } } else if (parameter == "sequence_end") { bool end; RETURN_MSG_IF_ERR( params_json.MemberAsBool(parameter.c_str(), &end), "Unable to parse 'sequence_end'"); if (end) { flags |= TRITONSERVER_REQUEST_FLAG_SEQUENCE_END; } } else if (parameter == "priority") { uint64_t p; RETURN_MSG_IF_ERR( params_json.MemberAsUInt(parameter.c_str(), &p), "Unable to parse 'priority'"); RETURN_IF_ERR( TRITONSERVER_InferenceRequestSetPriorityUInt64(irequest, p)); } else if (parameter == "timeout") { uint64_t t; RETURN_MSG_IF_ERR( params_json.MemberAsUInt(parameter.c_str(), &t), "Unable to parse 'timeout'"); RETURN_IF_ERR( TRITONSERVER_InferenceRequestSetTimeoutMicroseconds(irequest, t)); } else if (parameter == "binary_data_output") { bool bdo; RETURN_MSG_IF_ERR( params_json.MemberAsBool(parameter.c_str(), &bdo), "Unable to parse 'binary_data_output'"); output_kind = (bdo) ? AllocPayload::OutputInfo::BINARY : AllocPayload::OutputInfo::JSON; } else if (parameter.rfind("triton_", 0) == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, ("parameter keys starting with 'triton_' are reserved for Triton " "usage " "and should not be specified.")); } else { RETURN_IF_ERR(SetTritonParameterFromJsonParameter( parameter, params_json, irequest)); } } RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetFlags(irequest, flags)); } // Set output kind to JSON by default, or BINARY if specified in parameters. infer_req->alloc_payload_.default_output_kind_ = output_kind; return nullptr; // Success } TRITONSERVER_Error* HTTPAPIServer::ParseJsonTritonRequestID( triton::common::TritonJson::Value& request_json, TRITONSERVER_InferenceRequest* irequest) { // Set InferenceRequest request_id triton::common::TritonJson::Value id_json; if (request_json.Find("id", &id_json)) { const char* id; size_t id_len; RETURN_MSG_IF_ERR(id_json.AsString(&id, &id_len), "Unable to parse 'id'"); RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetId(irequest, id)); } return nullptr; // Success } // TODO: Can refactor other non-inference routes to re-use this helper instead. TRITONSERVER_Error* HTTPAPIServer::EVRequestToJson( evhtp_request_t* req, triton::common::TritonJson::Value* request_json_ptr) { struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Unexpected error getting request buffers"); } } size_t buffer_len = evbuffer_get_length(req->buffer_in); RETURN_IF_ERR(EVBufferToJson(request_json_ptr, v, &v_idx, buffer_len, n)); return nullptr; // success } TRITONSERVER_Error* HTTPAPIServer::EVBufferToInput( const std::string& model_name, TRITONSERVER_InferenceRequest* irequest, evbuffer* input_buffer, InferRequestClass* infer_req, size_t header_length) { // Extract individual input data from HTTP body and register in // 'irequest'. The HTTP body is not necessarily stored in contiguous // memory. // // Get the addr and size of each chunk of memory holding the HTTP // body. struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(input_buffer, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(input_buffer, -1, NULL, v, n) != n) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting input buffers"); } } // Extract just the json header from the HTTP body. 'header_length == 0' means // that the entire HTTP body should be input data for a raw binary request. triton::common::TritonJson::Value request_json; RETURN_IF_ERR(EVBufferToJson(&request_json, v, &v_idx, header_length, n)); // Parse request JSON and fill related Triton fields RETURN_IF_ERR(ParseJsonTritonRequestID(request_json, irequest)); RETURN_IF_ERR(ParseJsonTritonParams(request_json, irequest, infer_req)); RETURN_IF_ERR(ParseJsonTritonIO( request_json, irequest, infer_req, model_name, v, &v_idx, header_length, n)); return nullptr; // success } TRITONSERVER_Error* HTTPAPIServer::EVBufferToRawInput( const std::string& model_name, TRITONSERVER_InferenceRequest* irequest, evbuffer* input_buffer, InferRequestClass* infer_req) { static const char* raw_input_name = "raw_input"; RETURN_IF_ERR( TRITONSERVER_InferenceRequestAddRawInput(irequest, raw_input_name)); size_t byte_size = evbuffer_get_length(input_buffer); // zero-shape tensor if (byte_size == 0) { RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( irequest, raw_input_name, nullptr, 0 /* byte_size */, TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } else { struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(input_buffer, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(input_buffer, -1, NULL, v, n) != n) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting input buffers"); } } // Process one block at a time while ((byte_size > 0) && (v_idx < n)) { char* base = static_cast<char*>(v[v_idx].iov_base); size_t base_size; if (v[v_idx].iov_len > byte_size) { base_size = byte_size; v[v_idx].iov_base = static_cast<void*>(base + byte_size); v[v_idx].iov_len -= byte_size; byte_size = 0; } else { base_size = v[v_idx].iov_len; byte_size -= v[v_idx].iov_len; v_idx++; } RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( irequest, raw_input_name, base, base_size, TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } } infer_req->alloc_payload_.default_output_kind_ = AllocPayload::OutputInfo::BINARY; return nullptr; // success } struct HeaderSearchPayload { HeaderSearchPayload( const re2::RE2& regex, TRITONSERVER_InferenceRequest* request) : regex_(regex), request_(request), error_(nullptr) { } const re2::RE2& regex_; TRITONSERVER_InferenceRequest* request_; TRITONSERVER_Error* error_; }; int ForEachHeader(evhtp_header_t* header, void* arg) { HeaderSearchPayload* header_search_payload = reinterpret_cast<HeaderSearchPayload*>(arg); TRITONSERVER_InferenceRequest* request = header_search_payload->request_; const re2::RE2& regex = header_search_payload->regex_; std::string matched_string; if (RE2::PartialMatch(std::string(header->key), regex)) { header_search_payload->error_ = TRITONSERVER_InferenceRequestSetStringParameter( request, header->key, header->val); if (header_search_payload->error_ != nullptr) { return 1; } } return 0; } TRITONSERVER_Error* HTTPAPIServer::CheckTransactionPolicy( evhtp_request_t* req, const std::string& model_name, int64_t requested_model_version) { uint32_t txn_flags; RETURN_IF_ERR(TRITONSERVER_ServerModelTransactionProperties( server_.get(), model_name.c_str(), requested_model_version, &txn_flags, nullptr /* voidp */)); if ((txn_flags & TRITONSERVER_TXN_DECOUPLED) != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "HTTP end point doesn't support models with decoupled " "transaction policy"); } return nullptr; // success } std::shared_ptr<TraceManager::Trace> HTTPAPIServer::StartTrace( evhtp_request_t* req, const std::string& model_name, TRITONSERVER_InferenceTrace** triton_trace) { #ifdef TRITON_ENABLE_TRACING HttpTextMapCarrier carrier(req->headers_in); auto start_options = trace_manager_->GetTraceStartOptions(carrier, model_name); std::shared_ptr<TraceManager::Trace> trace; trace = std::move(trace_manager_->SampleTrace(start_options)); if (trace != nullptr) { *triton_trace = trace->trace_; // Timestamps from evhtp are capture in 'req'. We record here // since this is the first place where we have access to trace // manager. trace->CaptureTimestamp("HTTP_RECV_START", req->recv_start_ns); trace->CaptureTimestamp("HTTP_RECV_END", req->recv_end_ns); } return trace; #else return nullptr; #endif // TRITON_ENABLE_TRACING } TRITONSERVER_Error* HTTPAPIServer::DecompressBuffer( evhtp_request_t* req, evbuffer** decompressed_buffer) { auto compression_type = GetRequestCompressionType(req); switch (compression_type) { case DataCompressor::Type::DEFLATE: case DataCompressor::Type::GZIP: { *decompressed_buffer = evbuffer_new(); RETURN_IF_ERR(DataCompressor::DecompressData( compression_type, req->buffer_in, *decompressed_buffer)); break; } case DataCompressor::Type::UNKNOWN: { // Encounter unsupported compressed type, send error with supported types // in Accept-Encoding evhtp_headers_add_header( req->headers_out, evhtp_header_new(kAcceptEncodingHTTPHeader, "gzip, deflate", 1, 1)); // FIXME: Map TRITONSERVER_ERROR_UNSUPPORTED to EVHTP_RES_UNSUPPORTED return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Unsupported compression type"); } case DataCompressor::Type::IDENTITY: // Do nothing break; } return nullptr; // success } TRITONSERVER_Error* HTTPAPIServer::EVRequestToTritonRequest( evhtp_request_t* req, const std::string& model_name, TRITONSERVER_InferenceRequest* irequest, evbuffer* decompressed_buffer, InferRequestClass* infer_req, size_t header_length) { if (header_length != 0) { RETURN_IF_ERR(EVBufferToInput( model_name, irequest, (decompressed_buffer == nullptr) ? req->buffer_in : decompressed_buffer, infer_req, header_length)); } else { RETURN_IF_ERR(EVBufferToRawInput( model_name, irequest, (decompressed_buffer == nullptr) ? req->buffer_in : decompressed_buffer, infer_req)); } return nullptr; // success } TRITONSERVER_Error* HTTPAPIServer::ForwardHeaders( evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest) { if (!header_forward_pattern_.empty()) { HeaderSearchPayload header_search_payload(header_forward_regex_, irequest); int status = evhtp_kvs_for_each( req->headers_in, ForEachHeader, reinterpret_cast<void*>(&header_search_payload)); if (status != 0) { return header_search_payload.error_; } } return nullptr; // success } void HTTPAPIServer::HandleGenerate( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str, bool streaming) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::INFERENCE, restricted_apis_); AddContentTypeHeader(req, "application/json"); if (req->method != htp_method_POST) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } int64_t requested_model_version; RETURN_AND_RESPOND_IF_ERR( req, GetModelVersionFromString(model_version_str, &requested_model_version)); // If tracing is enabled see if this request should be traced. TRITONSERVER_InferenceTrace* triton_trace = nullptr; std::shared_ptr<TraceManager::Trace> trace = StartTrace(req, model_name, &triton_trace); std::map<std::string, triton::common::TritonJson::Value> input_metadata; triton::common::TritonJson::Value meta_data_root; RETURN_AND_RESPOND_IF_ERR( req, ModelInputMetadata( model_name, requested_model_version, &input_metadata, &meta_data_root)); // [FIXME] decompression should have been done here. before parsing request // body if (GetRequestCompressionType(req) != DataCompressor::Type::IDENTITY) { RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Unsupported content-encoding, only 'identity' is supported.")); } // Create the inference request object which provides all information needed // for an inference. Make sure it is cleaned up on early error. TRITONSERVER_InferenceRequest* irequest = nullptr; RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_InferenceRequestNew( &irequest, server_.get(), model_name.c_str(), requested_model_version)); std::shared_ptr<TRITONSERVER_InferenceRequest> irequest_shared = { irequest, [](TRITONSERVER_InferenceRequest* request) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestDelete(request), "deleting HTTP/REST inference request"); }}; // HTTP request paused when creating inference request. Resume it on exit if // this function returns early due to error. Otherwise resumed in callback. std::unique_ptr<GenerateRequestClass> generate_request; if (streaming) { generate_request.reset(new GenerateRequestClass( server_.get(), req, GetResponseCompressionType(req), generate_stream_request_schema_.get(), generate_stream_response_schema_.get(), streaming, irequest_shared)); } else { generate_request.reset(new GenerateRequestClass( server_.get(), req, GetResponseCompressionType(req), generate_request_schema_.get(), generate_response_schema_.get(), streaming, irequest_shared)); } generate_request->trace_ = trace; const char* request_id = "<id_unknown>"; // Callback to cleanup on any errors encountered below. Capture everything // by reference to capture local updates, except for shared pointers which // should be captured by value in case of ref count issues. // The callback does not own the error object. auto error_callback = [&, trace](TRITONSERVER_Error* error) { if (error != nullptr) { // Get request ID for logging in case of error. if (irequest != nullptr) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestId(irequest, &request_id), "unable to retrieve request ID string"); } if (!strncmp(request_id, "", 1)) { request_id = "<id_unknown>"; } LOG_VERBOSE(1) << "[request id: " << request_id << "] " << "Infer failed: " << TRITONSERVER_ErrorMessage(error); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, error); evhtp_send_reply(req, HttpCodeFromError(error)); evhtp_request_resume(req); #ifdef TRITON_ENABLE_TRACING // If HTTP server still owns Triton trace if ((trace != nullptr) && (trace->trace_ != nullptr)) { TraceManager::TraceRelease(trace->trace_, trace->trace_userp_); } #endif // TRITON_ENABLE_TRACING } }; // Option 1: Form tensor-like JSON request and try to re-use HandleInfer // as much as possible. Probably need to do something like overwrite // req->buffer_in or create a new evhtp_request to pass and handle. // Option 2: Do inference logic directly here after parsing request. // Note: // Currently option 2 is selected. It is true that HandleInfer() includes // handling for features that will be requested for generate endpoints // (i.e. tracing), however, it is currently tied to infer endpoint logic and // some decoupling must be done to properly reuse it (for example, response // callback is tied to infer logic and inflexible for response streaming). // For the time being, it is less mental burden to support this endpoint // without early optimization for code reuse. // Also, there is limitation on Triton JSON library that makes forming // arbitrary JSON message convoluted (added key is reference to a string and // thus the string must live as long as the JSON message). triton::common::TritonJson::Value request; RETURN_AND_CALLBACK_IF_ERR(EVRequestToJson(req, &request), error_callback); RETURN_AND_CALLBACK_IF_ERR( ParseJsonTritonRequestID(request, irequest), error_callback); RETURN_AND_CALLBACK_IF_ERR( generate_request->ConvertGenerateRequest( input_metadata, generate_request->RequestSchema(), request), error_callback); auto request_release_payload = std::make_unique<RequestReleasePayload>(irequest_shared, nullptr); // [FIXME] decompression.. RETURN_AND_CALLBACK_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestClass::InferRequestComplete, request_release_payload.get()), error_callback); RETURN_AND_CALLBACK_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator_, reinterpret_cast<void*>(&generate_request->alloc_payload_), GenerateRequestClass::InferResponseComplete, reinterpret_cast<void*>(generate_request.get())), error_callback); RETURN_AND_CALLBACK_IF_ERR( TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace), error_callback); #ifdef TRITON_ENABLE_TRACING // Ownership of trace passed to Triton core, set trace to null to mark it // as no longer owned here. if (trace != nullptr) { trace->trace_ = nullptr; } #endif // TRITON_ENABLE_TRACING generate_request.release(); request_release_payload.release(); } TRITONSERVER_Error* HTTPAPIServer::ModelInputMetadata( const std::string& model_name, const int64_t model_version, std::map<std::string, triton::common::TritonJson::Value>* input_metadata, triton::common::TritonJson::Value* metadata_root) { { if (model_name.empty()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Missing model name in metadata request"); } TRITONSERVER_Message* message = nullptr; RETURN_IF_ERR(TRITONSERVER_ServerModelMetadata( server_.get(), model_name.c_str(), model_version, &message)); const char* buffer; size_t byte_size; TRITONSERVER_Error* err = nullptr; err = TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size); if (err == nullptr) { RETURN_IF_ERR(metadata_root->Parse(buffer, byte_size)); } if (message) { TRITONSERVER_MessageDelete(message); } } // input triton::common::TritonJson::Value inputs; RETURN_IF_ERR(metadata_root->MemberAsArray("inputs", &inputs)); for (size_t i = 0; i < inputs.ArraySize(); ++i) { triton::common::TritonJson::Value input; RETURN_IF_ERR(inputs.At(i, &input)); std::string name = ""; RETURN_IF_ERR(input.MemberAsString("name", &name)); (*input_metadata)[name] = std::move(input); } return nullptr; // success } TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::ConvertGenerateRequest( std::map<std::string, triton::common::TritonJson::Value>& input_metadata, const MappingSchema* schema, triton::common::TritonJson::Value& generate_request) { // First find all top-level keys in JSON std::vector<std::string> members; RETURN_IF_ERR(generate_request.Members(&members)); for (const auto& m : members) { auto it = schema->children_.find(m); if (it != schema->children_.end()) { switch (it->second->kind_) { case MappingSchema::Kind::EXACT_MAPPING: { // Read meta data RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata)); break; } case MappingSchema::Kind::MAPPING_SCHEMA: { // The key is nested schema if (input_metadata.find(m) != input_metadata.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string( "Keyword '" + m + "' for nested schema also given as input tensor name") .c_str())); } triton::common::TritonJson::Value nested_generate_request; RETURN_MSG_IF_ERR( generate_request.MemberAsObject( m.c_str(), &nested_generate_request), "Expected JSON object for keyword: '" + m + "'"); RETURN_MSG_IF_ERR( ConvertGenerateRequest( input_metadata, it->second.get(), nested_generate_request), "Converting keyword: '" + m + "'"); break; } default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Unsupported schema kind"); } } else if (schema->allow_unspecified_) { // Unspecified key follows EXACT_MAPPING RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata)); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "The schema disallow unspecified key"); } } return nullptr; // success } TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::ExactMappingInput( const std::string& name, triton::common::TritonJson::Value& generate_request, std::map<std::string, triton::common::TritonJson::Value>& input_metadata) { auto it = input_metadata.find(name); if (it == input_metadata.end()) { RETURN_IF_ERR(SetTritonParameterFromJsonParameter( name, generate_request, triton_request_.get())); } else { // Parse data type and shape std::string value; it->second.MemberAsString("datatype", &value); auto dtype = TRITONSERVER_StringToDataType(value.c_str()); // Perform shape validation, assume the value must be either // primitive type or 1-D array. triton::common::TritonJson::Value tensor_data; if (!generate_request.Find(name.c_str(), &tensor_data)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("unexpected key not found in generate request, " "expecting key '") + name + "'") .c_str()); } size_t element_cnt = tensor_data.IsArray() ? tensor_data.ArraySize() : 1; size_t byte_size = 0; if (dtype == TRITONSERVER_TYPE_BYTES) { RETURN_IF_ERR(JsonBytesArrayByteSize(tensor_data, &byte_size)); } else { byte_size = element_cnt * TRITONSERVER_DataTypeByteSize(dtype); } std::vector<int64_t> shape_vec; { triton::common::TritonJson::Value value; if (!it->second.Find("shape", &value)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string( "Unexpected 'shape' not found in model metadata for input '") + name) .c_str()); } for (size_t i = 0; i < value.ArraySize(); ++i) { int64_t d = 0; RETURN_IF_ERR(value.IndexAsInt(i, &d)); shape_vec.push_back(d); } // Because generate request don't carry too much shape information, using // a two-pass process to pad the request value to match input shape. // 1. iterate shape for fixed dimension to distribute 'element_cnt'. // 2. Set most inner dynamic shape to the remaining element count, // other dynamic shape to be 1. for (auto rit = shape_vec.rbegin(); rit != shape_vec.rend(); ++rit) { if (*rit != -1) { if (element_cnt % *rit) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("The schema can not convert input '") + name + "' to tensor with proper shape") .c_str()); } element_cnt /= *rit; } } for (auto rit = shape_vec.rbegin(); rit != shape_vec.rend(); ++rit) { if (*rit == -1) { *rit = element_cnt; element_cnt = 1; } } if (element_cnt != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("The schema can not convert input '") + name + "' to tensor with proper shape") .c_str()); } } serialized_data_.emplace_back(); std::vector<char>& serialized = serialized_data_.back(); serialized.resize(byte_size); RETURN_IF_ERR(ReadDataFromJson( name.c_str(), tensor_data, &serialized[0], dtype, dtype == TRITONSERVER_TYPE_BYTES ? byte_size : element_cnt)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput( triton_request_.get(), name.c_str(), dtype, &shape_vec[0], shape_vec.size())); RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( triton_request_.get(), name.c_str(), &serialized[0], serialized.size(), TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } return nullptr; // success } void HTTPAPIServer::HandleInfer( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str) { RETURN_AND_RESPOND_IF_RESTRICTED( req, RestrictedCategory::INFERENCE, restricted_apis_); if (req->method != htp_method_POST) { RETURN_AND_RESPOND_WITH_ERR( req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); } int64_t requested_model_version; RETURN_AND_RESPOND_IF_ERR( req, GetModelVersionFromString( model_version_str.c_str(), &requested_model_version)); RETURN_AND_RESPOND_IF_ERR( req, CheckTransactionPolicy(req, model_name, requested_model_version)); // If tracing is enabled see if this request should be traced. TRITONSERVER_InferenceTrace* triton_trace = nullptr; std::shared_ptr<TraceManager::Trace> trace = StartTrace(req, model_name, &triton_trace); // Decompress request body if it is compressed in supported type evbuffer* decompressed_buffer = nullptr; RETURN_AND_RESPOND_IF_ERR(req, DecompressBuffer(req, &decompressed_buffer)); // Get content length as a default header_length if no header specified int32_t content_length = 0; RETURN_AND_RESPOND_IF_ERR( req, GetContentLength(req, decompressed_buffer, &content_length)); // Get the header length size_t header_length = 0; RETURN_AND_RESPOND_IF_ERR( req, GetInferenceHeaderLength(req, content_length, &header_length)); // Create the inference request object which provides all information needed // for an inference. Make sure it is cleaned up on early error. TRITONSERVER_InferenceRequest* irequest = nullptr; RETURN_AND_RESPOND_IF_ERR( req, TRITONSERVER_InferenceRequestNew( &irequest, server_.get(), model_name.c_str(), requested_model_version)); std::shared_ptr<TRITONSERVER_InferenceRequest> irequest_shared( irequest, [](TRITONSERVER_InferenceRequest* request) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestDelete(request), "deleting HTTP/REST inference request"); }); // HTTP request paused when creating inference request. Resume it on exit if // this function returns early due to error. Otherwise resumed in callback. bool connection_paused = true; auto infer_request = CreateInferRequest(req, irequest_shared); infer_request->trace_ = trace; const char* request_id = "<id_unknown>"; // Callback to cleanup on any errors encountered below. Capture everything // by reference to capture local updates, except for shared pointers which // should be captured by value in case of ref count issues. auto error_callback = [&, trace](TRITONSERVER_Error* error) { if (error != nullptr) { LOG_VERBOSE(1) << "[request id: " << request_id << "] " << "Infer failed: " << TRITONSERVER_ErrorMessage(error); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, error); evhtp_send_reply(req, HttpCodeFromError(error)); if (connection_paused) { evhtp_request_resume(req); } #ifdef TRITON_ENABLE_TRACING // If HTTP server still owns Triton trace if ((trace != nullptr) && (trace->trace_ != nullptr)) { TraceManager::TraceRelease(trace->trace_, trace->trace_userp_); } #endif // TRITON_ENABLE_TRACING } }; // Parse EV request and fill Triton request fields from it RETURN_AND_CALLBACK_IF_ERR( EVRequestToTritonRequest( req, model_name, irequest, decompressed_buffer, infer_request.get(), header_length), error_callback); // Get request ID for logging in case of error. LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestId(irequest, &request_id), "unable to retrieve request ID string"); // Reset id to unknown if empty in core. if (!strncmp(request_id, "", 1)) { request_id = "<id_unknown>"; } RETURN_AND_CALLBACK_IF_ERR(ForwardHeaders(req, irequest), error_callback); auto request_release_payload = std::make_unique<RequestReleasePayload>( irequest_shared, decompressed_buffer); RETURN_AND_CALLBACK_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestClass::InferRequestComplete, request_release_payload.get()), error_callback); RETURN_AND_CALLBACK_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator_, reinterpret_cast<void*>(&infer_request->alloc_payload_), InferRequestClass::InferResponseComplete, reinterpret_cast<void*>(infer_request.get())), error_callback); auto err = TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace); #ifdef TRITON_ENABLE_TRACING // Ownership of trace passed to Triton core, set trace to null to mark it // as no longer owned here. if (trace != nullptr) { trace->trace_ = nullptr; } #endif // TRITON_ENABLE_TRACING RETURN_AND_CALLBACK_IF_ERR(err, error_callback); infer_request.release(); request_release_payload.release(); } void HTTPAPIServer::InferRequestClass::ReplyCallback( evthr_t* thr, void* arg, void* shared) { HTTPAPIServer::InferRequestClass* infer_request = reinterpret_cast<HTTPAPIServer::InferRequestClass*>(arg); evhtp_request_t* request = infer_request->EvHtpRequest(); if (request != nullptr) { evhtp_send_reply(request, infer_request->response_code_); evhtp_request_resume(request); } #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { infer_request->trace_->CaptureTimestamp( "HTTP_SEND_START", request->send_start_ns); infer_request->trace_->CaptureTimestamp( "HTTP_SEND_END", request->send_end_ns); } #endif // TRITON_ENABLE_TRACING delete infer_request; } evhtp_res HTTPAPIServer::InferRequestClass::RequestFiniHook( evhtp_request* request, void* arg) { HTTPAPIServer::InferRequestClass* infer_request = reinterpret_cast<HTTPAPIServer::InferRequestClass*>(arg); if (infer_request->req_ != request) { LOG_ERROR << "[INTERNAL] mismatched request in fini hook"; return EVHTP_RES_ERROR; } else { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestCancel( infer_request->triton_request_.get()), "cancelling request"); infer_request->req_ = nullptr; } return EVHTP_RES_OK; } HTTPAPIServer::InferRequestClass::InferRequestClass( TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr<TRITONSERVER_InferenceRequest>& triton_request) : server_(server), req_(req), response_compression_type_(response_compression_type), response_count_(0), triton_request_(triton_request) { evhtp_connection_t* htpconn = evhtp_request_get_connection(req); thread_ = htpconn->thread; evhtp_request_pause(req); evhtp_request_set_hook( req_, evhtp_hook_on_request_fini, (evhtp_hook)(void*)RequestFiniHook, reinterpret_cast<void*>(this)); } void HTTPAPIServer::InferRequestClass::InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { // FIXME need to manage the lifetime of InferRequestClass so that we // delete it here. RequestReleasePayload* request_release_payload = reinterpret_cast<RequestReleasePayload*>(userp); if ((flags & TRITONSERVER_REQUEST_RELEASE_ALL) != 0) { delete request_release_payload; } } void HTTPAPIServer::InferRequestClass::InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { // FIXME can't use InferRequestClass object here since it's lifetime // is different than response. For response we need to know how to // send each output (as json, shm, or binary) and that information // has to be maintained in a way that allows us to clean it up // appropriately if connection closed or last response sent. // // But for now userp is the InferRequestClass object and the end of // its life is in the ReplyCallback. HTTPAPIServer::InferRequestClass* infer_request = reinterpret_cast<HTTPAPIServer::InferRequestClass*>(userp); if (response != nullptr) { ++infer_request->response_count_; } TRITONSERVER_Error* err = nullptr; if (infer_request->response_count_ != 1) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "expected a single response, got " + std::to_string(infer_request->response_count_)) .c_str()); } else if (response != nullptr) { err = infer_request->FinalizeResponse(response); #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { infer_request->trace_->CaptureTimestamp( "INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp()); } #endif // TRITON_ENABLE_TRACING } LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); if (err != nullptr) { EVBufferAddErrorJson(infer_request->req_->buffer_out, err); infer_request->response_code_ = HttpCodeFromError(err); TRITONSERVER_ErrorDelete(err); } // Defer sending the response until FINAL flag is seen if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { return; } evthr_defer( infer_request->thread_, InferRequestClass::ReplyCallback, infer_request); } TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::FinalizeResponse( TRITONSERVER_InferenceResponse* response) { RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); triton::common::TritonJson::Value response_json( triton::common::TritonJson::ValueType::OBJECT); const char* request_id = ""; RETURN_IF_ERR(TRITONSERVER_InferenceResponseId(response, &request_id)); if (strncmp(request_id, "", 1)) { RETURN_IF_ERR(response_json.AddStringRef("id", request_id)); } const char* model_name; int64_t model_version; RETURN_IF_ERR(TRITONSERVER_InferenceResponseModel( response, &model_name, &model_version)); RETURN_IF_ERR(response_json.AddStringRef("model_name", model_name)); RETURN_IF_ERR(response_json.AddString( "model_version", std::move(std::to_string(model_version)))); // If the response has any parameters, convert them to JSON. uint32_t parameter_count; RETURN_IF_ERR( TRITONSERVER_InferenceResponseParameterCount(response, ¶meter_count)); if (parameter_count > 0) { triton::common::TritonJson::Value params_json( response_json, triton::common::TritonJson::ValueType::OBJECT); for (uint32_t pidx = 0; pidx < parameter_count; ++pidx) { const char* name; TRITONSERVER_ParameterType type; const void* vvalue; RETURN_IF_ERR(TRITONSERVER_InferenceResponseParameter( response, pidx, &name, &type, &vvalue)); switch (type) { case TRITONSERVER_PARAMETER_BOOL: RETURN_IF_ERR(params_json.AddBool( name, *(reinterpret_cast<const bool*>(vvalue)))); break; case TRITONSERVER_PARAMETER_INT: RETURN_IF_ERR(params_json.AddInt( name, *(reinterpret_cast<const int64_t*>(vvalue)))); break; case TRITONSERVER_PARAMETER_STRING: RETURN_IF_ERR(params_json.AddStringRef( name, reinterpret_cast<const char*>(vvalue))); break; case TRITONSERVER_PARAMETER_DOUBLE: RETURN_IF_ERR(params_json.AddDouble( name, *(reinterpret_cast<const double*>(vvalue)))); break; case TRITONSERVER_PARAMETER_BYTES: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Response parameter of type 'TRITONSERVER_PARAMETER_BYTES' is " "not currently supported"); break; } } RETURN_IF_ERR(response_json.Add("parameters", std::move(params_json))); } // Go through each response output and transfer information to JSON uint32_t output_count; RETURN_IF_ERR( TRITONSERVER_InferenceResponseOutputCount(response, &output_count)); std::vector<evbuffer*> ordered_buffers; ordered_buffers.reserve(output_count); triton::common::TritonJson::Value response_outputs( response_json, triton::common::TritonJson::ValueType::ARRAY); for (uint32_t idx = 0; idx < output_count; ++idx) { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput( response, idx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); triton::common::TritonJson::Value output_json( response_json, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_ERR(output_json.AddStringRef("name", cname)); // Handle data. SHM outputs will not have an info. auto info = reinterpret_cast<AllocPayload::OutputInfo*>(userp); size_t element_count = 1; uint32_t batch_size = 0; // If returning output as classification then need to set the // datatype and shape based on classification requirements. if ((info != nullptr) && (info->class_cnt_ > 0)) { // For classification need to determine the batch size, if any, // because need to use that to break up the response for each // batch entry. uint32_t batch_flags; RETURN_IF_ERR(TRITONSERVER_ServerModelBatchProperties( server_, model_name, model_version, &batch_flags, nullptr /* voidp */)); if ((dim_count > 0) && ((batch_flags & TRITONSERVER_BATCH_FIRST_DIM) != 0)) { batch_size = shape[0]; } // Determine the batch1 byte size of the output tensor... needed // when the response tensor batch-size > 1 so that we know how // to stride though the tensor data. size_t batch1_element_count = 1; for (size_t sidx = ((batch_size == 0) ? 0 : 1); sidx < dim_count; sidx++) { batch1_element_count *= shape[sidx]; } const size_t batch1_byte_size = batch1_element_count * TRITONSERVER_DataTypeByteSize(datatype); // Create the classification contents std::string serialized; size_t class_offset = 0; for (uint32_t bs = 0; bs < std::max((uint32_t)1, batch_size); ++bs) { std::vector<std::string> class_strs; RETURN_IF_ERR(TopkClassifications( response, idx, reinterpret_cast<const char*>(base) + class_offset, ((class_offset + batch1_byte_size) > byte_size) ? 0 : batch1_byte_size, datatype, info->class_cnt_, &class_strs)); // Serialize for binary representation... for (const auto& str : class_strs) { uint32_t len = str.size(); serialized.append(reinterpret_cast<const char*>(&len), sizeof(len)); if (len > 0) { serialized.append(str); } } class_offset += batch1_byte_size; } // Replace existing output with serialized classification output. const char* datatype_str = TRITONSERVER_DataTypeString(TRITONSERVER_TYPE_BYTES); RETURN_IF_ERR(output_json.AddStringRef("datatype", datatype_str)); triton::common::TritonJson::Value shape_json( response_json, triton::common::TritonJson::ValueType::ARRAY); if (batch_size > 0) { RETURN_IF_ERR(shape_json.AppendUInt(batch_size)); element_count *= batch_size; } size_t actual_class_count = std::min((size_t)info->class_cnt_, batch1_element_count); element_count *= actual_class_count; RETURN_IF_ERR(shape_json.AppendUInt(actual_class_count)); RETURN_IF_ERR(output_json.Add("shape", std::move(shape_json))); evbuffer_free(info->evbuffer_); info->evbuffer_ = nullptr; void* buffer; byte_size = serialized.size(); RETURN_IF_ERR(AllocEVBuffer(byte_size, &info->evbuffer_, &buffer)); memcpy(buffer, serialized.c_str(), byte_size); base = reinterpret_cast<const void*>(buffer); datatype = TRITONSERVER_TYPE_BYTES; } else { const char* datatype_str = TRITONSERVER_DataTypeString(datatype); RETURN_IF_ERR(output_json.AddStringRef("datatype", datatype_str)); triton::common::TritonJson::Value shape_json( response_json, triton::common::TritonJson::ValueType::ARRAY); for (size_t j = 0; j < dim_count; j++) { RETURN_IF_ERR(shape_json.AppendUInt(shape[j])); element_count *= shape[j]; } RETURN_IF_ERR(output_json.Add("shape", std::move(shape_json))); } // Add JSON data, or collect binary data. if (info->kind_ == AllocPayload::OutputInfo::BINARY) { triton::common::TritonJson::Value parameters_json; if (!output_json.Find("parameters", ¶meters_json)) { parameters_json = triton::common::TritonJson::Value( response_json, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_ERR(parameters_json.AddUInt("binary_data_size", byte_size)); RETURN_IF_ERR( output_json.Add("parameters", std::move(parameters_json))); } else { RETURN_IF_ERR(parameters_json.AddUInt("binary_data_size", byte_size)); } if (byte_size > 0) { ordered_buffers.push_back(info->evbuffer_); } } else if (info->kind_ == AllocPayload::OutputInfo::JSON) { triton::common::TritonJson::Value data_json( response_json, triton::common::TritonJson::ValueType::ARRAY); RETURN_IF_ERR(WriteDataToJson( &data_json, cname, datatype, base, byte_size, element_count)); RETURN_IF_ERR(output_json.Add("data", std::move(data_json))); } RETURN_IF_ERR(response_outputs.Append(std::move(output_json))); } RETURN_IF_ERR(response_json.Add("outputs", std::move(response_outputs))); evbuffer* response_placeholder = evbuffer_new(); // Write json metadata into response evbuffer triton::common::TritonJson::WriteBuffer buffer; RETURN_IF_ERR(response_json.Write(&buffer)); evbuffer_add(response_placeholder, buffer.Base(), buffer.Size()); // If there is binary data write it next in the appropriate // order... also need the HTTP header when returning binary data. if (!ordered_buffers.empty()) { for (evbuffer* b : ordered_buffers) { evbuffer_add_buffer(response_placeholder, b); } } evbuffer* response_body = response_placeholder; switch (response_compression_type_) { case DataCompressor::Type::DEFLATE: case DataCompressor::Type::GZIP: { auto compressed_buffer = evbuffer_new(); auto err = DataCompressor::CompressData( response_compression_type_, response_placeholder, compressed_buffer); if (err == nullptr) { response_body = compressed_buffer; evbuffer_free(response_placeholder); } else { // just log the compression error and return the uncompressed data LOG_VERBOSE(1) << "unable to compress response: " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); evbuffer_free(compressed_buffer); response_compression_type_ = DataCompressor::Type::IDENTITY; } break; } case DataCompressor::Type::IDENTITY: case DataCompressor::Type::UNKNOWN: // Do nothing for other cases break; } SetResponseHeader(!ordered_buffers.empty(), buffer.Size()); evbuffer_add_buffer(req_->buffer_out, response_body); // Destroy the evbuffer object as the data has been moved // to HTTP response buffer evbuffer_free(response_body); return nullptr; // success } void HTTPAPIServer::InferRequestClass::SetResponseHeader( bool has_binary_data, size_t header_length) { if (has_binary_data) { AddContentTypeHeader(req_, "application/octet-stream"); evhtp_headers_add_header( req_->headers_out, evhtp_header_new( kInferHeaderContentLengthHTTPHeader, std::to_string(header_length).c_str(), 1, 1)); } else { AddContentTypeHeader(req_, "application/json"); } switch (response_compression_type_) { case DataCompressor::Type::DEFLATE: evhtp_headers_add_header( req_->headers_out, evhtp_header_new(kContentEncodingHTTPHeader, "deflate", 1, 1)); break; case DataCompressor::Type::GZIP: evhtp_headers_add_header( req_->headers_out, evhtp_header_new(kContentEncodingHTTPHeader, "gzip", 1, 1)); break; case DataCompressor::Type::IDENTITY: case DataCompressor::Type::UNKNOWN: break; } } uint32_t HTTPAPIServer::InferRequestClass::IncrementResponseCount() { return response_count_++; } HTTPAPIServer::GenerateRequestClass::~GenerateRequestClass() { while (!pending_http_responses_.empty()) { evbuffer_free(pending_http_responses_.front()); pending_http_responses_.pop(); } } void HTTPAPIServer::GenerateRequestClass::InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { // FIXME can't use InferRequestClass object here since it's lifetime // is different than response. For response we need to know how to // send each output (as json, shm, or binary) and that information // has to be maintained in a way that allows us to clean it up // appropriately if connection closed or last response sent. // // But for now userp is the InferRequestClass object and the end of // its life is in the ReplyCallback. auto infer_request = reinterpret_cast<HTTPAPIServer::GenerateRequestClass*>(userp); // Assuming responses of the same request is sent in sequence. TRITONSERVER_Error* err = nullptr; if (response != nullptr) { err = infer_request->FinalizeResponse(response); } if (err != nullptr) { infer_request->AddErrorJson(err); } // First response starts the chunked response, the response code is set here // so user should check response body in case of error at later time. if (infer_request->IncrementResponseCount() == 0) { infer_request->response_code_ = HttpCodeFromError(err); evthr_defer(infer_request->thread_, StartResponse, infer_request); } #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { infer_request->trace_->CaptureTimestamp( "INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp()); } #endif // TRITON_ENABLE_TRACING // Final flag indicates there is no more responses, ending chunked response. if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) != 0) { evthr_defer(infer_request->thread_, EndResponseCallback, infer_request); } else { evthr_defer(infer_request->thread_, ChunkResponseCallback, infer_request); } LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); } void HTTPAPIServer::GenerateRequestClass::StartResponse( evthr_t* thr, void* arg, void* shared) { auto infer_request = reinterpret_cast<HTTPAPIServer::GenerateRequestClass*>(arg); auto req = infer_request->EvHtpRequest(); if (req == nullptr) { return; } if (infer_request->streaming_) { AddContentTypeHeader(req, "text/event-stream; charset=utf-8"); } else { AddContentTypeHeader(req, "application/json"); } evhtp_send_reply_chunk_start(req, infer_request->response_code_); evhtp_request_resume(req); } void HTTPAPIServer::GenerateRequestClass::ChunkResponseCallback( evthr_t* thr, void* arg, void* shared) { auto infer_request = reinterpret_cast<HTTPAPIServer::GenerateRequestClass*>(arg); if (infer_request->req_ == nullptr) { return; } infer_request->SendChunkResponse(false /* end */); } void HTTPAPIServer::GenerateRequestClass::EndResponseCallback( evthr_t* thr, void* arg, void* shared) { auto infer_request = reinterpret_cast<HTTPAPIServer::GenerateRequestClass*>(arg); if (infer_request->EvHtpRequest() != nullptr) { infer_request->SendChunkResponse(true /* end */); evhtp_send_reply_chunk_end(infer_request->EvHtpRequest()); } delete infer_request; } void HTTPAPIServer::GenerateRequestClass::SendChunkResponse(bool end) { // check if response count in the case of non-streaming if (!streaming_) { std::lock_guard<std::mutex> lk(res_mtx_); // For non-streaming, wait until end if (!end) { return; } if (pending_http_responses_.size() != 1) { EVBufferAddErrorJson( req_->buffer_out, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "generate expects model to produce exactly 1 " "response, use generate stream for model that " "generates various number of responses")); evhtp_send_reply_chunk(req_, req_->buffer_out); return; } } evbuffer* buffer = nullptr; { std::lock_guard<std::mutex> lk(res_mtx_); // This function may be called with no pending responses when // response complete callback is invoked with flag-only if (pending_http_responses_.empty()) { return; } buffer = pending_http_responses_.front(); pending_http_responses_.pop(); } evhtp_send_reply_chunk(req_, buffer); evbuffer_free(buffer); #ifdef TRITON_ENABLE_TRACING if (trace_ != nullptr) { // [FIXME] currently send_start_ns / send_end_ns is // not captured in evhtp when response is sent in chunks trace_->CaptureTimestamp("HTTP_SEND_START", req_->send_start_ns); trace_->CaptureTimestamp("HTTP_SEND_END", req_->send_end_ns); } #endif // TRITON_ENABLE_TRACING } TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::FinalizeResponse( TRITONSERVER_InferenceResponse* response) { triton_response_ = response; RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); triton::common::TritonJson::Value response_json( triton::common::TritonJson::ValueType::OBJECT); // Response metadata in addition to output tensor / parameter falls under // "unspecified field" with predefined name: // "id", "model_name", "model_version" std::map<std::string, TritonOutput> triton_outputs; const char* id = ""; RETURN_IF_ERR(TRITONSERVER_InferenceResponseId(response, &id)); if (strncmp(id, "", 1)) { triton_outputs.emplace( "id", TritonOutput(TritonOutput::Type::RESERVED, id)); } const char* model_name; int64_t model_version; RETURN_IF_ERR(TRITONSERVER_InferenceResponseModel( response, &model_name, &model_version)); triton_outputs.emplace( "model_name", TritonOutput(TritonOutput::Type::RESERVED, model_name)); triton_outputs.emplace( "model_version", TritonOutput( TritonOutput::Type::RESERVED, std::to_string(model_version))); // If the response has any parameters, convert them to JSON. uint32_t parameter_count; RETURN_IF_ERR( TRITONSERVER_InferenceResponseParameterCount(response, ¶meter_count)); if (parameter_count > 0) { for (uint32_t pidx = 0; pidx < parameter_count; ++pidx) { const char* name; TRITONSERVER_ParameterType type; const void* vvalue; RETURN_IF_ERR(TRITONSERVER_InferenceResponseParameter( response, pidx, &name, &type, &vvalue)); switch (type) { case TRITONSERVER_PARAMETER_BOOL: case TRITONSERVER_PARAMETER_INT: case TRITONSERVER_PARAMETER_STRING: case TRITONSERVER_PARAMETER_DOUBLE: triton_outputs.emplace( name, TritonOutput(TritonOutput::Type::PARAMETER, pidx)); break; case TRITONSERVER_PARAMETER_BYTES: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, (std::string("Response parameter '") + name + "' has type 'TRITONSERVER_PARAMETER_BYTES' which is " "not currently supported") .c_str()); break; } } } // Go through each response output and transfer information to JSON uint32_t output_count; RETURN_IF_ERR( TRITONSERVER_InferenceResponseOutputCount(response, &output_count)); for (uint32_t idx = 0; idx < output_count; ++idx) { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput( response, idx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); triton_outputs.emplace( cname, TritonOutput(TritonOutput::Type::TENSOR, idx)); } std::set<std::string> mapped_outputs; RETURN_IF_ERR(ConvertGenerateResponse( triton_outputs, response_schema_, &response_json, &mapped_outputs)); if (response_schema_->allow_unspecified_) { for (const auto& to : triton_outputs) { if (mapped_outputs.find(to.first) == mapped_outputs.end()) { RETURN_IF_ERR(ExactMappingOutput( to.first, to.second, &response_json, &mapped_outputs)); } } } // [FIXME] compression evbuffer* response_body = evbuffer_new(); if (streaming_) { static std::string sse_prefix = "data: "; evbuffer_add(response_body, sse_prefix.c_str(), sse_prefix.length()); } // Write json metadata into response evbuffer triton::common::TritonJson::WriteBuffer buffer; RETURN_IF_ERR(response_json.Write(&buffer)); evbuffer_add(response_body, buffer.Base(), buffer.Size()); if (streaming_) { static std::string sse_suffix = "\n\n"; evbuffer_add(response_body, sse_suffix.c_str(), sse_suffix.length()); } { std::lock_guard<std::mutex> lk(res_mtx_); pending_http_responses_.emplace(response_body); } return nullptr; // success } void HTTPAPIServer::GenerateRequestClass::AddErrorJson(TRITONSERVER_Error* error) { evbuffer* buffer = evbuffer_new(); if (streaming_) { static std::string sse_prefix = "data: "; evbuffer_add(buffer, sse_prefix.c_str(), sse_prefix.length()); } EVBufferAddErrorJson(buffer, error); if (streaming_) { static std::string sse_suffix = "\n\n"; evbuffer_add(buffer, sse_suffix.c_str(), sse_suffix.length()); } TRITONSERVER_ErrorDelete(error); { std::lock_guard<std::mutex> lk(res_mtx_); pending_http_responses_.emplace(buffer); } } TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::ConvertGenerateResponse( const std::map< std::string, HTTPAPIServer::GenerateRequestClass::TritonOutput>& output_metadata, const MappingSchema* schema, triton::common::TritonJson::Value* generate_response, std::set<std::string>* mapped_outputs) { for (auto& nested : schema->children_) { switch (nested.second->kind_) { case MappingSchema::Kind::MAPPING_SCHEMA: { triton::common::TritonJson::Value nested_response( *generate_response, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_ERR(ConvertGenerateResponse( output_metadata, nested.second.get(), &nested_response, mapped_outputs)); RETURN_IF_ERR(generate_response->Add( nested.first.c_str(), std::move(nested_response))); break; } case MappingSchema::Kind::EXACT_MAPPING: { auto it = output_metadata.find(nested.first); if (it == output_metadata.end()) { if (!nested.second->allow_unspecified_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("Schema requires output '") + nested.first + "' to be produced by the model.") .c_str()); } } else { RETURN_IF_ERR(ExactMappingOutput( nested.first, it->second, generate_response, mapped_outputs)); } break; } default: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Unsupported schema kind"); } } return nullptr; // success } TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::ExactMappingOutput( const std::string& name, const HTTPAPIServer::GenerateRequestClass::TritonOutput& triton_output, triton::common::TritonJson::Value* generate_response, std::set<std::string>* mapped_outputs) { mapped_outputs->emplace(name); switch (triton_output.type) { case TritonOutput::Type::RESERVED: { generate_response->AddStringRef( name.c_str(), triton_output.value.c_str()); break; } case TritonOutput::Type::PARAMETER: { const char* name; TRITONSERVER_ParameterType type; const void* vvalue; RETURN_IF_ERR(TRITONSERVER_InferenceResponseParameter( triton_response_, triton_output.index, &name, &type, &vvalue)); switch (type) { case TRITONSERVER_PARAMETER_BOOL: RETURN_IF_ERR(generate_response->AddBool( name, *(reinterpret_cast<const bool*>(vvalue)))); break; case TRITONSERVER_PARAMETER_INT: RETURN_IF_ERR(generate_response->AddInt( name, *(reinterpret_cast<const int64_t*>(vvalue)))); break; case TRITONSERVER_PARAMETER_STRING: RETURN_IF_ERR(generate_response->AddStringRef( name, reinterpret_cast<const char*>(vvalue))); break; case TRITONSERVER_PARAMETER_DOUBLE: RETURN_IF_ERR(generate_response->AddDouble( name, *(reinterpret_cast<const double*>(vvalue)))); break; case TRITONSERVER_PARAMETER_BYTES: return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, (std::string("Response parameter '") + name + "' has type 'TRITONSERVER_PARAMETER_BYTES' which is " "not currently supported") .c_str()); break; } break; } case TritonOutput::Type::TENSOR: { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput( triton_response_, triton_output.index, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); auto info = reinterpret_cast<AllocPayload::OutputInfo*>(userp); // sanity check if (info->kind_ != AllocPayload::OutputInfo::JSON) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, (std::string("non-JSON output response type is requested for '") + cname + "'") .c_str()); } size_t element_count = 1; for (size_t j = 0; j < dim_count; j++) { element_count *= shape[j]; } triton::common::TritonJson::Value data_json( *generate_response, triton::common::TritonJson::ValueType::ARRAY); RETURN_IF_ERR(WriteDataToJson( &data_json, cname, datatype, base, byte_size, element_count)); if (element_count == 1) { // if only 1 element, strip out the array triton::common::TritonJson::Value el; RETURN_IF_ERR(data_json.At(0, &el)); RETURN_IF_ERR(generate_response->Add(cname, std::move(el))); } else { RETURN_IF_ERR(generate_response->Add(cname, std::move(data_json))); } break; } } return nullptr; // success } void HTTPAPIServer::Handle(evhtp_request_t* req) { LOG_VERBOSE(1) << "HTTP request: " << req->method << " " << req->uri->path->full; if (std::string(req->uri->path->full) == "/v2/models/stats") { // model statistics HandleModelStats(req); return; } if (std::string(req->uri->path->full) == "/v2/logging") { // change logging HandleLogging(req); return; } std::string model_name, version, kind; if (RE2::FullMatch( std::string(req->uri->path->full), model_regex_, &model_name, &version, &kind)) { if (kind == "ready") { // model ready HandleModelReady(req, model_name, version); return; } else if (kind == "infer") { // model infer HandleInfer(req, model_name, version); return; } else if (kind == "generate") { // text generation HandleGenerate(req, model_name, version, false /* streaming */); return; } else if (kind == "generate_stream") { // text generation (streaming) HandleGenerate(req, model_name, version, true /* streaming */); return; } else if (kind == "config") { // model configuration HandleModelConfig(req, model_name, version); return; } else if (kind == "stats") { // model statistics HandleModelStats(req, model_name, version); return; } else if (kind == "trace/setting") { // Trace with specific model, there is no specification on versioning // so fall out and return bad request error if version is specified if (version.empty()) { HandleTrace(req, model_name); return; } } else if (kind == "") { // model metadata HandleModelMetadata(req, model_name, version); return; } } std::string region, action, rest, repo_name; if (std::string(req->uri->path->full) == "/v2") { // server metadata HandleServerMetadata(req); return; } else if (RE2::FullMatch( std::string(req->uri->path->full), server_regex_, &rest)) { // server health HandleServerHealth(req, rest); return; } else if (RE2::FullMatch( std::string(req->uri->path->full), systemsharedmemory_regex_, ®ion, &action)) { // system shared memory HandleSystemSharedMemory(req, region, action); return; } else if (RE2::FullMatch( std::string(req->uri->path->full), cudasharedmemory_regex_, ®ion, &action)) { // cuda shared memory HandleCudaSharedMemory(req, region, action); return; } else if (RE2::FullMatch( std::string(req->uri->path->full), modelcontrol_regex_, &repo_name, &kind, &model_name, &action)) { // model repository if (kind == "index") { HandleRepositoryIndex(req, repo_name); return; } else if (kind.find("models", 0) == 0) { HandleRepositoryControl(req, repo_name, model_name, action); return; } } else if (RE2::FullMatch(std::string(req->uri->path->full), trace_regex_)) { // trace request on global settings HandleTrace(req); return; } LOG_VERBOSE(1) << "HTTP error: " << req->method << " " << req->uri->path->full << " - " << static_cast<int>(EVHTP_RES_NOTFOUND); RETURN_AND_RESPOND_WITH_ERR(req, EVHTP_RES_NOTFOUND, "Not Found"); } TRITONSERVER_Error* HTTPAPIServer::Create( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const int32_t port, const bool reuse_port, const std::string& address, const std::string& header_forward_pattern, const int thread_cnt, const RestrictedFeatures& restricted_features, std::unique_ptr<HTTPServer>* http_server) { http_server->reset(new HTTPAPIServer( server, trace_manager, shm_manager, port, reuse_port, address, header_forward_pattern, thread_cnt, restricted_features)); const std::string addr = address + ":" + std::to_string(port); LOG_INFO << "Started HTTPService at " << addr; return nullptr; } bool HTTPAPIServer::RespondIfRestricted( evhtp_request_t* req, const Restriction& restriction) { auto header = restriction.first; auto expected_value = restriction.second; const char* actual_value = evhtp_kv_find(req->headers_in, header.c_str()); if ((actual_value == nullptr) || (actual_value != expected_value)) { EVBufferAddErrorJson( req->buffer_out, std::string("This API is restricted, expecting header '" + header + "'") .c_str()); evhtp_send_reply(req, EVHTP_RES_FORBIDDEN); return true; } return false; } }} // namespace triton::server ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/http_server.h�����������������������������������������������������������������0000664�0000000�0000000�00000057267�15270523712�0017163�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <evhtp/evhtp.h> #include <re2/re2.h> #include <list> #include <map> #include <memory> #include <mutex> #include <queue> #include <string> #include <thread> #include <unordered_map> #include "common.h" #include "data_compressor.h" #include "restricted_features.h" #include "shared_memory_manager.h" #include "tracer.h" #include "triton/common/logging.h" #include "triton/core/tritonserver.h" namespace triton { namespace server { class MappingSchema { public: enum class Kind { EXACT_MAPPING, // An object of this kind means it is a nested mapping schema. MAPPING_SCHEMA }; std::map<std::string, std::unique_ptr<MappingSchema>> children_; // Whether an unspecified key is allowed. If true, // * for requests, the unspecified key will be converted to Triton input // following the EXACT_MAPPING rule. // * for responses, the Triton output will be converted to JSON key-value // pairs at top level if the name is unspecified in the schema, // following the EXACT_MAPPING rule. const bool allow_unspecified_{true}; const Kind kind_{Kind::EXACT_MAPPING}; explicit MappingSchema( const MappingSchema::Kind& kind = Kind::EXACT_MAPPING, const bool& allow_unspecified = true) : allow_unspecified_(allow_unspecified), kind_(kind) { } private: }; // Generic HTTP server using evhtp class HTTPServer { public: virtual ~HTTPServer() { IGNORE_ERR(Stop()); } TRITONSERVER_Error* Start(); TRITONSERVER_Error* Stop( uint32_t* exit_timeout_secs = nullptr, const std::string& service_name = "HTTP"); protected: explicit HTTPServer( const int32_t port, const bool reuse_port, const std::string& address, const std::string& header_forward_pattern, const int thread_cnt) : port_(port), reuse_port_(reuse_port), address_(address), header_forward_pattern_(header_forward_pattern), thread_cnt_(thread_cnt), header_forward_regex_(header_forward_pattern_), conn_cnt_(0), accepting_new_conn_(true) { } static void Dispatch(evhtp_request_t* req, void* arg); protected: virtual void Handle(evhtp_request_t* req) = 0; static void StopCallback(evutil_socket_t sock, short events, void* arg); static evhtp_res NewConnection(evhtp_connection_t* conn, void* arg); static evhtp_res EndConnection(evhtp_connection_t* conn, void* arg); int32_t port_; bool reuse_port_; std::string address_; std::string header_forward_pattern_; int thread_cnt_; re2::RE2 header_forward_regex_; evhtp_t* htp_; struct event_base* evbase_; std::thread worker_; evutil_socket_t fds_[2]; event* break_ev_; std::mutex conn_mu_; uint32_t conn_cnt_; bool accepting_new_conn_; }; #ifdef TRITON_ENABLE_METRICS // Handle HTTP requests to obtain prometheus metrics class HTTPMetricsServer : public HTTPServer { public: static TRITONSERVER_Error* Create( const std::shared_ptr<TRITONSERVER_Server>& server, int32_t port, std::string address, int thread_cnt, std::unique_ptr<HTTPServer>* metrics_server); ~HTTPMetricsServer() = default; private: explicit HTTPMetricsServer( const std::shared_ptr<TRITONSERVER_Server>& server, const int32_t port, std::string address, const int thread_cnt) : HTTPServer( port, false /* reuse_port */, address, "" /* header_forward_pattern */, thread_cnt), server_(server), api_regex_(R"(/metrics/?)") { } void Handle(evhtp_request_t* req) override; std::shared_ptr<TRITONSERVER_Server> server_; re2::RE2 api_regex_; }; #endif // TRITON_ENABLE_METRICS #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING) class HttpTextMapCarrier : public otel_cntxt::propagation::TextMapCarrier { public: HttpTextMapCarrier(evhtp_kvs_t* headers) : headers_(headers) {} HttpTextMapCarrier() = default; virtual opentelemetry::nostd::string_view Get( opentelemetry::nostd::string_view key) const noexcept override { std::string key_to_compare = key.data(); auto it = evhtp_kv_find(headers_, key_to_compare.c_str()); if (it != NULL) { return opentelemetry::nostd::string_view(it); } return ""; } // Not required on server side virtual void Set( opentelemetry::nostd::string_view key, opentelemetry::nostd::string_view value) noexcept override { return; } evhtp_kvs_t* headers_; }; #else using HttpTextMapCarrier = void*; #endif // HTTP API server that implements KFServing community standard inference // protocols and extensions used by Triton. class HTTPAPIServer : public HTTPServer { public: static TRITONSERVER_Error* Create( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& smb_manager, const int32_t port, const bool reuse_port, const std::string& address, const std::string& header_forward_pattern, const int thread_cnt, const RestrictedFeatures& restricted_apis, std::unique_ptr<HTTPServer>* http_server); virtual ~HTTPAPIServer(); // // AllocPayload // // Simple structure that carries the userp payload needed for // allocation. struct AllocPayload { struct OutputInfo { enum Kind { JSON, BINARY, SHM }; Kind kind_; void* base_; uint64_t byte_size_; TRITONSERVER_MemoryType memory_type_; int64_t device_id_; uint32_t class_cnt_; evbuffer* evbuffer_; char* cuda_ipc_handle_; // For non-shared memory OutputInfo(Kind k, uint32_t class_cnt) : kind_(k), class_cnt_(class_cnt), evbuffer_(nullptr) { } // For shared memory OutputInfo( void* base, uint64_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t device_id, char* cuda_ipc_handle) : kind_(SHM), base_(base), byte_size_(byte_size), memory_type_(memory_type), device_id_(device_id), class_cnt_(0), evbuffer_(nullptr), cuda_ipc_handle_(cuda_ipc_handle) { } ~OutputInfo() { if (evbuffer_ != nullptr) { evbuffer_free(evbuffer_); } } }; ~AllocPayload() { for (auto it : output_map_) { delete it.second; } } AllocPayload() : default_output_kind_(OutputInfo::Kind::JSON){}; std::unordered_map<std::string, OutputInfo*> output_map_; AllocPayload::OutputInfo::Kind default_output_kind_; }; // Object associated with an inference request. This persists // information needed for the request and records the evhtp thread // that is bound to the request. This same thread must be used to // send the response. class InferRequestClass { public: // [FIXME] decompression / compression should be handled implicitly // within InferRequestClass. This alleviate the check for decompressed // buffer in HTTPServer code. explicit InferRequestClass( TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr<TRITONSERVER_InferenceRequest>& triton_request); virtual ~InferRequestClass() { if (req_ != nullptr) { evhtp_request_unset_hook(req_, evhtp_hook_on_request_fini); } req_ = nullptr; } evhtp_request_t* EvHtpRequest() const { return req_; } static void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp); static void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); virtual TRITONSERVER_Error* FinalizeResponse( TRITONSERVER_InferenceResponse* response); // Helper function to set infer response header in the form specified by // the endpoint protocol virtual void SetResponseHeader( const bool has_binary_data, const size_t header_length); uint32_t IncrementResponseCount(); // Only used if tracing enabled std::shared_ptr<TraceManager::Trace> trace_; AllocPayload alloc_payload_; // Data that cannot be used directly from the HTTP body is first // serialized. Hold that data here so that its lifetime spans the // lifetime of the request. std::list<std::vector<char>> serialized_data_; static void ReplyCallback(evthr_t* thr, void* arg, void* shared); protected: TRITONSERVER_Server* server_{nullptr}; evhtp_request_t* req_{nullptr}; evthr_t* thread_{nullptr}; DataCompressor::Type response_compression_type_{ DataCompressor::Type::IDENTITY}; // Counter to keep track of number of responses generated. std::atomic<uint32_t> response_count_{0}; // Event hook for called before request deletion static evhtp_res RequestFiniHook(evhtp_request* req, void* arg); // Pointer to associated Triton request, this class does not own the // request and must not reference it after a successful // TRITONSERVER_ServerInferAsync (except for cancellation). std::shared_ptr<TRITONSERVER_InferenceRequest> triton_request_{nullptr}; evhtp_res response_code_{EVHTP_RES_OK}; }; class GenerateRequestClass : public InferRequestClass { public: explicit GenerateRequestClass( TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const MappingSchema* request_schema, const MappingSchema* response_schema, bool streaming, const std::shared_ptr<TRITONSERVER_InferenceRequest>& triton_request) : InferRequestClass( server, req, response_compression_type, triton_request), request_schema_(request_schema), response_schema_(response_schema), streaming_(streaming) { } virtual ~GenerateRequestClass(); // [FIXME] Specialize response complete function for now, should have // been a dispatcher and call into object specific response function. static void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); static void ChunkResponseCallback(evthr_t* thr, void* arg, void* shared); static void EndResponseCallback(evthr_t* thr, void* arg, void* shared); // Return whether the response is ending void SendChunkResponse(bool end); // Response preparation TRITONSERVER_Error* FinalizeResponse( TRITONSERVER_InferenceResponse* response) override; void AddErrorJson(TRITONSERVER_Error* error); static void StartResponse(evthr_t* thr, void* arg, void* shared); // [DLIS-5551] currently always performs basic conversion, only maps schema // of EXACT_MAPPING kind. MAPPING_SCHEMA and upcoming kinds are for // customized conversion where a detailed schema will be provided. TRITONSERVER_Error* ConvertGenerateRequest( std::map<std::string, triton::common::TritonJson::Value>& input_metadata, const MappingSchema* schema, triton::common::TritonJson::Value& generate_request); const MappingSchema* RequestSchema() { return request_schema_; } const MappingSchema* ResponseSchema() { return response_schema_; } private: struct TritonOutput { enum class Type { RESERVED, TENSOR, PARAMETER }; TritonOutput(Type t, const std::string& val) : type(t), value(val) {} explicit TritonOutput(Type t, uint32_t i) : type(t), index(i) {} Type type; // RESERVED type std::string value; // TENSOR, PARAMETER type uint32_t index; }; TRITONSERVER_Error* ExactMappingInput( const std::string& name, triton::common::TritonJson::Value& value, std::map<std::string, triton::common::TritonJson::Value>& input_metadata); // [DLIS-5551] currently always performs basic conversion, only maps schema // of EXACT_MAPPING kind. MAPPING_SCHEMA and upcoming kinds are for // customized conversion where a detailed schema will be provided. TRITONSERVER_Error* ConvertGenerateResponse( const std::map<std::string, TritonOutput>& output_metadata, const MappingSchema* schema, triton::common::TritonJson::Value* generate_response, std::set<std::string>* mapped_outputs); TRITONSERVER_Error* ExactMappingOutput( const std::string& name, const TritonOutput& triton_output, triton::common::TritonJson::Value* generate_response, std::set<std::string>* mapped_outputs); const MappingSchema* request_schema_{nullptr}; const MappingSchema* response_schema_{nullptr}; const bool streaming_{false}; // Placeholder to completing response, this class does not own // the response. TRITONSERVER_InferenceResponse* triton_response_{nullptr}; // As InferResponseComplete and ChunkResponseCallback are called in // different threads, need to have dedicated buffers for each response and // ensure mutual exclusive access. std::mutex res_mtx_; std::queue<evbuffer*> pending_http_responses_; bool end_{false}; }; // Simple structure that carries the userp payload needed for // request release callback. struct RequestReleasePayload final { RequestReleasePayload( const std::shared_ptr<TRITONSERVER_InferenceRequest>& inference_request, evbuffer* buffer) : inference_request_(inference_request), buffer_(buffer){}; ~RequestReleasePayload() { if (buffer_ != nullptr) { evbuffer_free(buffer_); } }; private: std::shared_ptr<TRITONSERVER_InferenceRequest> inference_request_ = nullptr; evbuffer* buffer_ = nullptr; }; protected: explicit HTTPAPIServer( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const int32_t port, const bool reuse_port, const std::string& address, const std::string& header_forward_pattern, const int thread_cnt, const RestrictedFeatures& restricted_apis = {}); virtual void Handle(evhtp_request_t* req) override; // [FIXME] extract to "infer" class virtual std::unique_ptr<InferRequestClass> CreateInferRequest( evhtp_request_t* req, const std::shared_ptr<TRITONSERVER_InferenceRequest>& triton_request) { return std::unique_ptr<InferRequestClass>(new InferRequestClass( server_.get(), req, GetResponseCompressionType(req), triton_request)); } // Helper function to retrieve infer request header in the form specified by // the endpoint protocol // // Get the inference header length. Return 0 if the whole request body is // the inference header. virtual TRITONSERVER_Error* GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, size_t* header_length); virtual DataCompressor::Type GetRequestCompressionType(evhtp_request_t* req); virtual DataCompressor::Type GetResponseCompressionType(evhtp_request_t* req); TRITONSERVER_Error* GetModelConfig( const std::string& model_name, int64_t requested_model_version, std::string* config_json); TRITONSERVER_Error* GetContentLength( evhtp_request_t* req, evbuffer* decompressed_buffer, int32_t* content_length); TRITONSERVER_Error* DecompressBuffer( evhtp_request_t* req, evbuffer** decompressed_buffer); TRITONSERVER_Error* CheckTransactionPolicy( evhtp_request_t* req, const std::string& model_name, int64_t requested_model_version); std::shared_ptr<TraceManager::Trace> StartTrace( evhtp_request_t* req, const std::string& model_name, TRITONSERVER_InferenceTrace** triton_trace); TRITONSERVER_Error* ForwardHeaders( evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest); static TRITONSERVER_Error* InferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id); static TRITONSERVER_Error* OutputBufferQuery( TRITONSERVER_ResponseAllocator* allocator, void* userp, const char* tensor_name, size_t* byte_size, TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id); static TRITONSERVER_Error* OutputBufferAttributes( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, TRITONSERVER_BufferAttributes* buffer_attributes, void* userp, void* buffer_userp); static TRITONSERVER_Error* InferResponseFree( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id); void HandleServerHealth(evhtp_request_t* req, const std::string& kind); void HandleServerMetadata(evhtp_request_t* req); void HandleModelReady( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); void HandleModelMetadata( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); void HandleModelConfig( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); void HandleInfer( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); void HandleModelStats( evhtp_request_t* req, const std::string& model_name = "", const std::string& model_version_str = ""); void HandleRepositoryIndex( evhtp_request_t* req, const std::string& repository_name); void HandleRepositoryControl( evhtp_request_t* req, const std::string& repository_name, const std::string& model_name, const std::string& action); void HandleSystemSharedMemory( evhtp_request_t* req, const std::string& region_name, const std::string& action); void HandleCudaSharedMemory( evhtp_request_t* req, const std::string& region_name, const std::string& action); void HandleTrace(evhtp_request_t* req, const std::string& model_name = ""); void HandleLogging(evhtp_request_t* req); // Text Generation / LLM format //'streaming' selects the schema pair to convert request / response. // 'streaming' also controls the response convention, if true, // Server-Sent Events format will be used to send responses. void HandleGenerate( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str, bool streaming); // 'meta_data_root' is the root JSON document for 'input_metadata'. // In TritonJson, the Value objects are references to the root document. // Therefore the document must stay valid. TRITONSERVER_Error* ModelInputMetadata( const std::string& model_name, const int64_t model_version, std::map<std::string, triton::common::TritonJson::Value>* input_metadata, triton::common::TritonJson::Value* meta_data_root); // Parses full evhtp request and its evbuffers into JSON. TRITONSERVER_Error* EVRequestToJson( evhtp_request_t* req, triton::common::TritonJson::Value* request_json); // Parses evhtp request buffers into Triton Inference Request. TRITONSERVER_Error* EVRequestToTritonRequest( evhtp_request_t* req, const std::string& model_name, TRITONSERVER_InferenceRequest* irequest, evbuffer* decompressed_buffer, InferRequestClass* infer_req, size_t header_length); TRITONSERVER_Error* EVBufferToInput( const std::string& model_name, TRITONSERVER_InferenceRequest* irequest, evbuffer* input_buffer, InferRequestClass* infer_req, size_t header_length); TRITONSERVER_Error* EVBufferToRawInput( const std::string& model_name, TRITONSERVER_InferenceRequest* irequest, evbuffer* input_buffer, InferRequestClass* infer_req); // Helpers for parsing JSON requests for Triton-specific fields TRITONSERVER_Error* ParseJsonTritonIO( triton::common::TritonJson::Value& request_json, TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, const std::string& model_name, evbuffer_iovec* v, int* v_idx_ptr, size_t header_length, int n); TRITONSERVER_Error* ParseJsonTritonParams( triton::common::TritonJson::Value& request_json, TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req); TRITONSERVER_Error* ParseJsonTritonRequestID( triton::common::TritonJson::Value& request_json, TRITONSERVER_InferenceRequest* irequest); std::shared_ptr<TRITONSERVER_Server> server_; // Storing server metadata as it is consistent during server running TRITONSERVER_Error* server_metadata_err_; std::string server_metadata_; TraceManager* trace_manager_; std::shared_ptr<SharedMemoryManager> shm_manager_; // The allocator that will be used to allocate buffers for the // inference result tensors. TRITONSERVER_ResponseAllocator* allocator_; re2::RE2 server_regex_; re2::RE2 model_regex_; re2::RE2 modelcontrol_regex_; re2::RE2 systemsharedmemory_regex_; re2::RE2 cudasharedmemory_regex_; re2::RE2 trace_regex_; // [DLIS-5551] currently always performs basic conversion, only maps schema // of EXACT_MAPPING kind. MAPPING_SCHEMA and upcoming kinds are for // customized conversion where a detailed schema will be provided. std::unique_ptr<MappingSchema> generate_request_schema_{new MappingSchema()}; std::unique_ptr<MappingSchema> generate_response_schema_{new MappingSchema()}; std::unique_ptr<MappingSchema> generate_stream_response_schema_{ new MappingSchema()}; std::unique_ptr<MappingSchema> generate_stream_request_schema_{ new MappingSchema()}; // Provisional definition of generate mapping schema // to allow for parameters passing // // Note: subject to change void ConfigureGenerateMappingSchema() { // Reserved field parameters for generate // If present, parameters will be converted to tensors // or parameters based on model config const std::string parameters_field = "parameters"; generate_stream_request_schema_->children_.emplace( parameters_field, new MappingSchema(MappingSchema::Kind::MAPPING_SCHEMA, true)); generate_request_schema_->children_.emplace( parameters_field, new MappingSchema(MappingSchema::Kind::MAPPING_SCHEMA, true)); } RestrictedFeatures restricted_apis_{}; bool RespondIfRestricted( evhtp_request_t* req, const Restriction& restriction); }; }} // namespace triton::server �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/main.cc�����������������������������������������������������������������������0000664�0000000�0000000�00000040316�15270523712�0015663�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef _WIN32 #define NOMINMAX #define WIN32_LEAN_AND_MEAN #include <windows.h> #include <winsock2.h> #include <ws2tcpip.h> #pragma comment(lib, "ws2_32.lib") #endif #ifndef _WIN32 #include <getopt.h> #include <unistd.h> #endif #include <stdint.h> #include <algorithm> #include <cctype> #include <iomanip> #include <iostream> #include <sstream> #include <thread> #include "triton_signal.h" #ifdef TRITON_ENABLE_ASAN #include <sanitizer/lsan_interface.h> #endif // TRITON_ENABLE_ASAN #include "command_line_parser.h" #include "common.h" #include "shared_memory_manager.h" #include "tracer.h" #include "triton/common/logging.h" #include "triton/core/tritonserver.h" #if defined(TRITON_ENABLE_HTTP) || defined(TRITON_ENABLE_METRICS) #include "http_server.h" #endif // TRITON_ENABLE_HTTP|| TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_SAGEMAKER #include "sagemaker_server.h" #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI #include "vertex_ai_server.h" #endif // TRITON_ENABLE_VERTEX_AI #ifdef TRITON_ENABLE_GRPC #include "grpc/grpc_server.h" #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_GPU static_assert( TRITON_MIN_COMPUTE_CAPABILITY >= 1.0, "Invalid TRITON_MIN_COMPUTE_CAPABILITY specified"); #endif // TRITON_ENABLE_GPU namespace { #ifdef TRITON_ENABLE_HTTP std::unique_ptr<triton::server::HTTPServer> g_http_service; #endif // TRITON_ENABLE_HTTP #ifdef TRITON_ENABLE_GRPC std::unique_ptr<triton::server::grpc::Server> g_grpc_service; #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_METRICS std::unique_ptr<triton::server::HTTPServer> g_metrics_service; #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_SAGEMAKER std::unique_ptr<triton::server::HTTPServer> g_sagemaker_service; #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI std::unique_ptr<triton::server::HTTPServer> g_vertex_ai_service; #endif // TRITON_ENABLE_VERTEX_AI triton::server::TritonServerParameters g_triton_params; #ifdef TRITON_ENABLE_GRPC TRITONSERVER_Error* StartGrpcService( std::unique_ptr<triton::server::grpc::Server>* service, const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<triton::server::SharedMemoryManager>& shm_manager) { TRITONSERVER_Error* err = triton::server::grpc::Server::Create( server, trace_manager, shm_manager, g_triton_params.grpc_options_, service); if (err == nullptr) { err = (*service)->Start(); } if (err != nullptr) { service->reset(); } return err; } #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_HTTP TRITONSERVER_Error* StartHttpService( std::unique_ptr<triton::server::HTTPServer>* service, const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<triton::server::SharedMemoryManager>& shm_manager) { TRITONSERVER_Error* err = triton::server::HTTPAPIServer::Create( server, trace_manager, shm_manager, g_triton_params.http_port_, g_triton_params.reuse_http_port_, g_triton_params.http_address_, g_triton_params.http_forward_header_pattern_, g_triton_params.http_thread_cnt_, g_triton_params.http_restricted_apis_, service); if (err == nullptr) { err = (*service)->Start(); } if (err != nullptr) { service->reset(); } return err; } #endif // TRITON_ENABLE_HTTP #ifdef TRITON_ENABLE_METRICS TRITONSERVER_Error* StartMetricsService( std::unique_ptr<triton::server::HTTPServer>* service, const std::shared_ptr<TRITONSERVER_Server>& server) { TRITONSERVER_Error* err = triton::server::HTTPMetricsServer::Create( server, g_triton_params.metrics_port_, g_triton_params.metrics_address_, 1 /* HTTP thread count */, service); if (err == nullptr) { err = (*service)->Start(); } if (err != nullptr) { service->reset(); } return err; } #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_SAGEMAKER TRITONSERVER_Error* StartSagemakerService( std::unique_ptr<triton::server::HTTPServer>* service, const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<triton::server::SharedMemoryManager>& shm_manager) { TRITONSERVER_Error* err = triton::server::SagemakerAPIServer::Create( server, trace_manager, shm_manager, g_triton_params.sagemaker_port_, g_triton_params.sagemaker_address_, g_triton_params.sagemaker_thread_cnt_, service); if (err == nullptr) { err = (*service)->Start(); } if (err != nullptr) { service->reset(); } return err; } #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI TRITONSERVER_Error* StartVertexAiService( std::unique_ptr<triton::server::HTTPServer>* service, const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<triton::server::SharedMemoryManager>& shm_manager) { TRITONSERVER_Error* err = triton::server::VertexAiAPIServer::Create( server, trace_manager, shm_manager, g_triton_params.vertex_ai_port_, g_triton_params.vertex_ai_address_, g_triton_params.vertex_ai_thread_cnt_, g_triton_params.vertex_ai_default_model_, service); if (err == nullptr) { err = (*service)->Start(); } if (err != nullptr) { service->reset(); } return err; } #endif // TRITON_ENABLE_VERTEX_AI bool StartEndpoints( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<triton::server::SharedMemoryManager>& shm_manager) { #ifdef _WIN32 WSADATA wsaData; int wsa_ret = WSAStartup(MAKEWORD(2, 2), &wsaData); if (wsa_ret != 0) { LOG_ERROR << "Error in WSAStartup " << wsa_ret; return false; } #endif #ifdef TRITON_ENABLE_GRPC // Enable GRPC endpoints if requested... if (g_triton_params.allow_grpc_) { TRITONSERVER_Error* err = StartGrpcService(&g_grpc_service, server, trace_manager, shm_manager); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to start GRPC service"); return false; } } #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_HTTP // Enable HTTP endpoints if requested... if (g_triton_params.allow_http_) { TRITONSERVER_Error* err = StartHttpService(&g_http_service, server, trace_manager, shm_manager); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to start HTTP service"); return false; } } #endif // TRITON_ENABLE_HTTP #ifdef TRITON_ENABLE_SAGEMAKER // Enable Sagemaker endpoints if requested... if (g_triton_params.allow_sagemaker_) { TRITONSERVER_Error* err = StartSagemakerService( &g_sagemaker_service, server, trace_manager, shm_manager); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to start Sagemaker service"); return false; } } #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI // Enable Vertex AI endpoints if requested... if (g_triton_params.allow_vertex_ai_) { TRITONSERVER_Error* err = StartVertexAiService( &g_vertex_ai_service, server, trace_manager, shm_manager); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to start Vertex AI service"); return false; } } #endif // TRITON_ENABLE_VERTEX_AI #ifdef TRITON_ENABLE_METRICS // Enable metrics endpoint if requested... if (g_triton_params.allow_metrics_) { TRITONSERVER_Error* err = StartMetricsService(&g_metrics_service, server); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to start Metrics service"); return false; } } #endif // TRITON_ENABLE_METRICS return true; } bool StopEndpoints(uint32_t* exit_timeout_secs) { bool ret = true; #ifdef TRITON_ENABLE_HTTP if (g_http_service) { TRITONSERVER_Error* err = g_http_service->Stop(exit_timeout_secs); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to stop HTTP service"); ret = false; } g_http_service.reset(); } #endif // TRITON_ENABLE_HTTP return ret; } bool StopEndpoints() { bool ret = true; // TODO: Add support for 'exit_timeout_secs' to the endpoints below and move // them to the 'StopEndpoints(uint32_t* exit_timeout_secs)' function above. #ifdef TRITON_ENABLE_GRPC if (g_grpc_service) { TRITONSERVER_Error* err = g_grpc_service->Stop(); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to stop GRPC service"); ret = false; } g_grpc_service.reset(); } #endif // TRITON_ENABLE_GRPC #ifdef TRITON_ENABLE_METRICS if (g_metrics_service) { TRITONSERVER_Error* err = g_metrics_service->Stop(); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to stop Metrics service"); ret = false; } g_metrics_service.reset(); } #endif // TRITON_ENABLE_METRICS #ifdef TRITON_ENABLE_SAGEMAKER if (g_sagemaker_service) { TRITONSERVER_Error* err = g_sagemaker_service->Stop(); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to stop Sagemaker service"); ret = false; } g_sagemaker_service.reset(); } #endif // TRITON_ENABLE_SAGEMAKER #ifdef TRITON_ENABLE_VERTEX_AI if (g_vertex_ai_service) { TRITONSERVER_Error* err = g_vertex_ai_service->Stop(); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to stop Vertex AI service"); ret = false; } g_vertex_ai_service.reset(); } #endif // TRITON_ENABLE_VERTEX_AI #ifdef _WIN32 int wsa_ret = WSACleanup(); if (wsa_ret != 0) { LOG_ERROR << "Error in WSACleanup " << wsa_ret; ret = false; } #endif return ret; } bool StartTracing(triton::server::TraceManager** trace_manager) { *trace_manager = nullptr; #ifdef TRITON_ENABLE_TRACING TRITONSERVER_Error* err = triton::server::TraceManager::Create( trace_manager, g_triton_params.trace_level_, g_triton_params.trace_rate_, g_triton_params.trace_count_, g_triton_params.trace_log_frequency_, g_triton_params.trace_filepath_, g_triton_params.trace_mode_, g_triton_params.trace_config_map_); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "failed to configure tracing"); if (*trace_manager != nullptr) { delete (*trace_manager); } *trace_manager = nullptr; return false; } #endif // TRITON_ENABLE_TRACING return true; } bool StopTracing(triton::server::TraceManager** trace_manager) { #ifdef TRITON_ENABLE_TRACING // We assume that at this point Triton has been stopped gracefully, // so can delete the trace manager to finalize the output. delete (*trace_manager); *trace_manager = nullptr; #endif // TRITON_ENABLE_TRACING return true; } } // namespace int main(int argc, char** argv) { // Parse command-line to create the options for the inference // server. triton::server::TritonParser tp; try { auto res = tp.Parse(argc, argv); g_triton_params = res.first; g_triton_params.CheckPortCollision(); } catch (const triton::server::ParseException& pe) { std::cerr << "Usage: tritonserver [options]" << std::endl; std::cerr << tp.Usage() << std::endl; // Show error at bottom for immediate visibility std::cerr << pe.what() << std::endl; exit(1); } triton::server::TritonServerParameters::ManagedTritonServerOptionPtr triton_options(nullptr, TRITONSERVER_ServerOptionsDelete); try { triton_options = g_triton_params.BuildTritonServerOptions(); } catch (const triton::server::ParseException& pe) { std::cerr << "Failed to build Triton option:" << std::endl; std::cerr << pe.what() << std::endl; exit(1); } #ifdef TRITON_ENABLE_LOGGING // Initialize our own logging instance since it is used by GRPC and // HTTP endpoints. This logging instance is separate from the one in // libtritonserver so we must initialize explicitly. LOG_ENABLE_INFO(g_triton_params.log_info_); LOG_ENABLE_WARNING(g_triton_params.log_warn_); LOG_ENABLE_ERROR(g_triton_params.log_error_); LOG_SET_VERBOSE(g_triton_params.log_verbose_); LOG_SET_FORMAT(g_triton_params.log_format_); LOG_SET_OUT_FILE(g_triton_params.log_file_); #endif // TRITON_ENABLE_LOGGING // Trace manager. triton::server::TraceManager* trace_manager; // Manager for shared memory blocks. auto shm_manager = std::make_shared<triton::server::SharedMemoryManager>(); // Create the server... TRITONSERVER_Server* server_ptr = nullptr; FAIL_IF_ERR( TRITONSERVER_ServerNew(&server_ptr, triton_options.get()), "creating server"); std::shared_ptr<TRITONSERVER_Server> server( server_ptr, TRITONSERVER_ServerDelete); // Configure and start tracing if specified on the command line. if (!StartTracing(&trace_manager)) { exit(1); } // Trap SIGINT and SIGTERM to allow server to exit gracefully TRITONSERVER_Error* signal_err = triton::server::RegisterSignalHandler(); if (signal_err != nullptr) { LOG_TRITONSERVER_ERROR(signal_err, "failed to register signal handler"); exit(1); } // Start the HTTP, GRPC, and metrics endpoints. if (!StartEndpoints(server, trace_manager, shm_manager)) { exit(1); } // Wait until a signal terminates the server... while (!triton::server::signal_exiting_) { // If enabled, poll the model repository to see if there have been // any changes. if (g_triton_params.repository_poll_secs_ > 0) { LOG_TRITONSERVER_ERROR( TRITONSERVER_ServerPollModelRepository(server_ptr), "failed to poll model repository"); } // Wait for the polling interval (or a long time if polling is not // enabled). Will be woken if the server is exiting. std::unique_lock<std::mutex> lock(triton::server::signal_exit_mu_); std::chrono::seconds wait_timeout( (g_triton_params.repository_poll_secs_ == 0) ? 3600 : g_triton_params.repository_poll_secs_); triton::server::signal_exit_cv_.wait_for(lock, wait_timeout); } // Stop the HTTP[, gRPC, and metrics] endpoints, and update exit timeout. uint32_t exit_timeout_secs = g_triton_params.exit_timeout_secs_; StopEndpoints(&exit_timeout_secs); TRITONSERVER_ServerSetExitTimeout(server_ptr, exit_timeout_secs); TRITONSERVER_Error* stop_err = TRITONSERVER_ServerStop(server_ptr); // If unable to gracefully stop the server then Triton threads and // state are potentially in an invalid state, so just exit // immediately. if (stop_err != nullptr) { LOG_TRITONSERVER_ERROR(stop_err, "failed to stop server"); exit(1); } // Stop gRPC and metrics endpoints that do not yet support exit timeout. StopEndpoints(); // Stop tracing. StopTracing(&trace_manager); #ifdef TRITON_ENABLE_ASAN // Can invoke ASAN before exit though this is typically not very // useful since there are many objects that are not yet destructed. // __lsan_do_leak_check(); #endif // TRITON_ENABLE_ASAN return 0; } ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/memory_alloc.cc���������������������������������������������������������������0000664�0000000�0000000�00000101572�15270523712�0017423�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <cuda_runtime_api.h> #include <rapidjson/document.h> #include <rapidjson/error/en.h> #include <unistd.h> #include <chrono> #include <future> #include <iostream> #include <string> #include <thread> #include <vector> #include "common.h" #include "triton/core/tritonserver.h" static_assert( TRITON_MIN_COMPUTE_CAPABILITY >= 1.0, "Invalid TRITON_MIN_COMPUTE_CAPABILITY specified"); namespace ni = triton::server; namespace { struct IOSpec { TRITONSERVER_MemoryType input_type_; int64_t input_type_id_; TRITONSERVER_MemoryType output_type_; int64_t output_type_id_; }; // Meta data used for preparing input data and validate output data IOSpec io_spec; static auto gpu_data_deleter = [](void* data) { if (data != nullptr) { FAIL_IF_CUDA_ERR( cudaSetDevice(io_spec.input_type_id_), "setting CUDA device to release GPU memory on " + std::to_string(io_spec.input_type_id_)); FAIL_IF_CUDA_ERR(cudaFree(data), "releasing GPU memory"); } }; void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-i [input device ID]" << std::endl; std::cerr << "\t-out [output device ID]" << std::endl; std::cerr << "\t-v Enable verbose logging" << std::endl; std::cerr << "\t-r [model repository absolute path]" << std::endl; std::cerr << "\t-m [model name to be tested]" << std::endl; std::cerr << "\t-h [host policy name]" << std::endl; std::cerr << "\tFor '-h', if specify, the input will be set with different " << "host policy names, given that the specified value is the " << "host policy that the model under test is associated with." << std::endl; std::cerr << "\tFor device ID, -1 is used to stand for CPU device, " << "non-negative value is for GPU device." << std::endl; exit(1); } TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { // If 'byte_size' is zero just return 'buffer'==nullptr, we don't // need to do any other book-keeping. if (byte_size == 0) { *buffer = nullptr; *buffer_userp = nullptr; std::cout << "allocated " << byte_size << " bytes for result tensor " << tensor_name << std::endl; } else { void* allocated_ptr = nullptr; if (io_spec.output_type_ == TRITONSERVER_MEMORY_CPU) { allocated_ptr = malloc(byte_size); } else { auto err = cudaSetDevice(io_spec.output_type_id_); if (err == cudaSuccess) { err = cudaMalloc(&allocated_ptr, byte_size); } if (err != cudaSuccess) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "failed to allocate CUDA memory: " + std::string(cudaGetErrorString(err))) .c_str()); } } if (allocated_ptr == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "failed to allocate " + std::to_string(byte_size) + " bytes in " + TRITONSERVER_MemoryTypeString(io_spec.output_type_) + " for result tensor " + tensor_name) .c_str()); } // Pass the tensor name with buffer_userp so we can show it when // releasing the buffer. *buffer = allocated_ptr; *buffer_userp = new std::string(tensor_name); std::cout << "allocated " << byte_size << " bytes in " << TRITONSERVER_MemoryTypeString(io_spec.output_type_) << " for result tensor " << tensor_name << std::endl; } *actual_memory_type = io_spec.output_type_; *actual_memory_type_id = io_spec.output_type_id_; return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { std::unique_ptr<std::string> name; if (buffer_userp != nullptr) { name.reset(reinterpret_cast<std::string*>(buffer_userp)); } else { name.reset(new std::string("<unknown>")); } std::cout << "Releasing buffer " << buffer << " of size " << byte_size << " in " << TRITONSERVER_MemoryTypeString(memory_type) << " for result '" << *name << "'" << std::endl; if (memory_type == TRITONSERVER_MEMORY_CPU) { free(buffer); } else { auto err = cudaSetDevice(memory_type_id); if (err == cudaSuccess) { err = cudaFree(buffer); } if (err != cudaSuccess) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "failed to release CUDA memory: " + std::string(cudaGetErrorString(err))) .c_str()); } } return nullptr; // Success } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { if ((flags & TRITONSERVER_REQUEST_RELEASE_ALL) != 0) { TRITONSERVER_InferenceRequestDelete(request); } } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { if (response != nullptr) { // Send 'response' to the future. std::promise<TRITONSERVER_InferenceResponse*>* p = reinterpret_cast<std::promise<TRITONSERVER_InferenceResponse*>*>(userp); p->set_value(response); delete p; } } uint32_t OutputIndex(TRITONSERVER_InferenceResponse* response, const std::string& name) { uint32_t output_count; FAIL_IF_ERR( TRITONSERVER_InferenceResponseOutputCount(response, &output_count), "getting number of response outputs"); for (uint32_t idx = 0; idx < output_count; ++idx) { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; FAIL_IF_ERR( TRITONSERVER_InferenceResponseOutput( response, idx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp), "getting output info"); if (name == std::string(cname)) { return idx; } } FAIL("can't found output '" + name + "'"); return 0; } TRITONSERVER_Error* ParseModelConfig( const rapidjson::Document& model_metadata, TRITONSERVER_DataType* dtype, bool* is_torch_model) { *dtype = TRITONSERVER_TYPE_INVALID; for (const auto& input : model_metadata["inputs"].GetArray()) { if (strcmp(input["datatype"].GetString(), "INT32") && strcmp(input["datatype"].GetString(), "FP32") && strcmp(input["datatype"].GetString(), "BYTES")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "IO test utility only supports model with data type INT32, " "FP32 or BYTES"); } if (*dtype == TRITONSERVER_TYPE_INVALID) { *dtype = TRITONSERVER_StringToDataType(input["datatype"].GetString()); } else { auto dt = TRITONSERVER_StringToDataType(input["datatype"].GetString()); if (dt != *dtype) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "the model inputs must have the same data type"); } } } for (const auto& output : model_metadata["outputs"].GetArray()) { if (strcmp(output["datatype"].GetString(), "INT32") && strcmp(output["datatype"].GetString(), "FP32") && strcmp(output["datatype"].GetString(), "BYTES")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "IO test utility only supports model with data type INT32, " "FP32 or BYTES"); } else { auto dt = TRITONSERVER_StringToDataType(output["datatype"].GetString()); if (dt != *dtype) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "the model inputs and outputs must have the same data type"); } } } *is_torch_model = (model_metadata["platform"] == "pytorch_libtorch"); return nullptr; } template <typename T> void GenerateInputData( std::vector<char>* input0_data, std::vector<char>* input1_data) { input0_data->resize(16 * sizeof(T)); input1_data->resize(16 * sizeof(T)); for (size_t i = 0; i < 16; ++i) { ((T*)input0_data->data())[i] = i; ((T*)input1_data->data())[i] = 1; } } void GenerateStringInputData( std::vector<char>* input0_data, std::vector<char>* input1_data) { std::string input0_str = ""; std::string input1_str = ""; for (size_t i = 0; i < 16; ++i) { std::string i0 = std::to_string(i + 1); uint32_t i0_len = i0.size(); input0_str.append(reinterpret_cast<const char*>(&i0_len), sizeof(uint32_t)); input0_str.append(i0); std::string i1 = std::to_string(1); uint32_t i1_len = i1.size(); input1_str.append(reinterpret_cast<const char*>(&i1_len), sizeof(uint32_t)); input1_str.append(i1); } std::copy( input0_str.begin(), input0_str.end(), std::back_inserter(*input0_data)); std::copy( input1_str.begin(), input1_str.end(), std::back_inserter(*input1_data)); } void GenerateStringOutputData( std::vector<char>* output0_data, std::vector<char>* output1_data) { std::string output0_str = ""; std::string output1_str = ""; for (size_t i = 0; i < 16; ++i) { std::string o0 = std::to_string(i + 2); uint32_t o0_len = o0.size(); output0_str.append( reinterpret_cast<const char*>(&o0_len), sizeof(uint32_t)); output0_str.append(o0); std::string o1 = std::to_string(i); uint32_t o1_len = o1.size(); output1_str.append( reinterpret_cast<const char*>(&o1_len), sizeof(uint32_t)); output1_str.append(o1); } std::copy( output0_str.begin(), output0_str.end(), std::back_inserter(*output0_data)); std::copy( output1_str.begin(), output1_str.end(), std::back_inserter(*output1_data)); } template <typename T> void CompareResult( const std::string& output0_name, const std::string& output1_name, const void* input0, const void* input1, const void* output0, const void* output1) { for (size_t i = 0; i < 16; ++i) { std::cout << ((T*)input0)[i] << " + " << ((T*)input1)[i] << " = " << ((T*)output0)[i] << std::endl; std::cout << ((T*)input0)[i] << " - " << ((T*)input1)[i] << " = " << ((T*)output1)[i] << std::endl; if ((((T*)input0)[i] + ((T*)input1)[i]) != ((T*)output0)[i]) { FAIL("incorrect sum in " + output0_name); } if ((((T*)input0)[i] - ((T*)input1)[i]) != ((T*)output1)[i]) { FAIL("incorrect difference in " + output1_name); } } } void CompareStringResult( const std::string& output0_name, const std::string& output1_name, const void* input0, const void* input1, const void* output0, const void* output1) { // preprocess results from serialized buffer to integers std::vector<int> output0_numbers; std::vector<int> output1_numbers; size_t buf_offset0 = 0, buf_offset1 = 0; const uint8_t* base0 = reinterpret_cast<const uint8_t*>(output0); const uint8_t* base1 = reinterpret_cast<const uint8_t*>(output1); for (size_t i = 0; i < 16; ++i) { const uint32_t len0 = *(reinterpret_cast<const uint32_t*>(base0 + buf_offset0)); std::string o0_tmp( reinterpret_cast<const char*>(base0 + buf_offset0 + sizeof(len0)), len0); output0_numbers.push_back(std::atoi(o0_tmp.c_str())); buf_offset0 += sizeof(len0) + len0; const uint32_t len1 = *(reinterpret_cast<const uint32_t*>(base1 + buf_offset1)); std::string o1_tmp( reinterpret_cast<const char*>(base1 + buf_offset1 + sizeof(len1)), len1); output1_numbers.push_back(std::atoi(o1_tmp.c_str())); buf_offset1 += sizeof(len1) + len1; } for (int i = 0; i < 16; ++i) { std::cout << (i + 1) << " + " << 1 << " = " << output0_numbers[i] << std::endl; std::cout << (i + 1) << " - " << 1 << " = " << output1_numbers[i] << std::endl; if (((i + 1) + 1) != output0_numbers[i]) { FAIL("incorrect sum in " + output0_name); } if (((i + 1) - 1) != output1_numbers[i]) { FAIL("incorrect difference in " + output1_name); } } } } // namespace int main(int argc, char** argv) { std::string model_repository_path; std::string model_name; int verbose_level = 0; io_spec.input_type_ = TRITONSERVER_MEMORY_CPU; io_spec.input_type_id_ = 0; io_spec.output_type_ = TRITONSERVER_MEMORY_CPU; io_spec.output_type_id_ = 0; const char* host_policy_cstr = nullptr; std::string host_policy; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vi:o:r:m:h:")) != -1) { switch (opt) { case 'i': { int64_t raw_id = std::stoll(optarg); if (raw_id < 0) { io_spec.input_type_ = TRITONSERVER_MEMORY_CPU; io_spec.input_type_id_ = 0; } else { io_spec.input_type_ = TRITONSERVER_MEMORY_GPU; io_spec.input_type_id_ = raw_id; } break; } case 'o': { int64_t raw_id = std::stoll(optarg); if (raw_id < 0) { io_spec.output_type_ = TRITONSERVER_MEMORY_CPU; io_spec.output_type_id_ = 0; } else { io_spec.output_type_ = TRITONSERVER_MEMORY_GPU; io_spec.output_type_id_ = raw_id; } break; } case 'h': { host_policy = optarg; host_policy_cstr = host_policy.c_str(); break; } case 'r': model_repository_path = optarg; break; case 'm': model_name = optarg; break; case 'v': verbose_level = 1; break; case '?': Usage(argv); break; } } if (model_repository_path.empty()) { Usage(argv, "-r must be used to specify model repository path"); } if (model_name.empty()) { Usage(argv, "-m must be used to specify model being test"); } // Create the server... TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, model_repository_path.c_str()), "setting model repository path"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetModelControlMode( server_options, TRITONSERVER_MODEL_CONTROL_EXPLICIT), "setting model control mode"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetStartupModel( server_options, model_name.c_str()), "setting model to load"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetLogVerbose(server_options, verbose_level), "setting verbose logging level"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetBackendDirectory( server_options, "/opt/tritonserver/backends"), "setting backend directory"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetRepoAgentDirectory( server_options, "/opt/tritonserver/repoagents"), "setting repository agent directory"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), "setting strict model configuration"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( server_options, TRITON_MIN_COMPUTE_CAPABILITY), "setting minimum supported CUDA compute capability"); TRITONSERVER_Server* server_ptr = nullptr; FAIL_IF_ERR( TRITONSERVER_ServerNew(&server_ptr, server_options), "creating server"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); std::shared_ptr<TRITONSERVER_Server> server( server_ptr, TRITONSERVER_ServerDelete); // Wait until the server is both live and ready. size_t health_iters = 0; while (true) { bool live, ready; FAIL_IF_ERR( TRITONSERVER_ServerIsLive(server.get(), &live), "unable to get server liveness"); FAIL_IF_ERR( TRITONSERVER_ServerIsReady(server.get(), &ready), "unable to get server readiness"); std::cout << "Server Health: live " << live << ", ready " << ready << std::endl; if (live && ready) { break; } if (++health_iters >= 10) { FAIL("failed to find healthy inference server"); } std::this_thread::sleep_for(std::chrono::milliseconds(500)); } // Print status of the server. { TRITONSERVER_Message* server_metadata_message; FAIL_IF_ERR( TRITONSERVER_ServerMetadata(server.get(), &server_metadata_message), "unable to get server metadata message"); const char* buffer; size_t byte_size; FAIL_IF_ERR( TRITONSERVER_MessageSerializeToJson( server_metadata_message, &buffer, &byte_size), "unable to serialize server metadata message"); std::cout << "Server Status:" << std::endl; std::cout << std::string(buffer, byte_size) << std::endl; FAIL_IF_ERR( TRITONSERVER_MessageDelete(server_metadata_message), "deleting status metadata"); } // Wait for the model to become available. bool is_torch_model = false; TRITONSERVER_DataType dtype = TRITONSERVER_TYPE_INT32; bool is_ready = false; health_iters = 0; while (!is_ready) { FAIL_IF_ERR( TRITONSERVER_ServerModelIsReady( server.get(), model_name.c_str(), 1, &is_ready), "unable to get model readiness"); if (!is_ready) { if (++health_iters >= 10) { FAIL("model failed to be ready in 10 iterations"); } std::this_thread::sleep_for(std::chrono::milliseconds(500)); continue; } TRITONSERVER_Message* model_metadata_message; FAIL_IF_ERR( TRITONSERVER_ServerModelMetadata( server.get(), model_name.c_str(), 1, &model_metadata_message), "unable to get model metadata message"); const char* buffer; size_t byte_size; FAIL_IF_ERR( TRITONSERVER_MessageSerializeToJson( model_metadata_message, &buffer, &byte_size), "unable to serialize model status protobuf"); rapidjson::Document model_metadata; model_metadata.Parse(buffer, byte_size); if (model_metadata.HasParseError()) { FAIL( "error: failed to parse model metadata from JSON: " + std::string(GetParseError_En(model_metadata.GetParseError())) + " at " + std::to_string(model_metadata.GetErrorOffset())); } FAIL_IF_ERR( TRITONSERVER_MessageDelete(model_metadata_message), "deleting status protobuf"); if (strcmp(model_metadata["name"].GetString(), model_name.c_str())) { FAIL("unable to find metadata for model"); } bool found_version = false; if (model_metadata.HasMember("versions")) { for (const auto& version : model_metadata["versions"].GetArray()) { if (strcmp(version.GetString(), "1") == 0) { found_version = true; break; } } } if (!found_version) { FAIL("unable to find version 1 status for model"); } FAIL_IF_ERR( ParseModelConfig(model_metadata, &dtype, &is_torch_model), "parsing model metadata"); } // Create the allocator that will be used to allocate buffers for // the result tensors. TRITONSERVER_ResponseAllocator* allocator = nullptr; FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator, ResponseAlloc, ResponseRelease, nullptr /* start_fn */), "creating response allocator"); TRITONSERVER_InferenceRequest* irequest = nullptr; FAIL_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest, server.get(), model_name.c_str(), -1 /* model_version */), "creating inference request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetId(irequest, "123"), "setting ID for the request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Create 0 data that shouldn't be selected and used to test host policy // functionality std::vector<uint32_t> zero_data(16); // Create the data for the two input tensors. Initialize the first // to unique integers and the second to all ones. std::vector<char> input0_data; std::vector<char> input1_data; if (dtype == TRITONSERVER_TYPE_INT32) { GenerateInputData<int32_t>(&input0_data, &input1_data); } else if (dtype == TRITONSERVER_TYPE_FP32) { GenerateInputData<float>(&input0_data, &input1_data); } else { GenerateStringInputData(&input0_data, &input1_data); } auto input0 = "INPUT0"; auto input1 = "INPUT1"; // Get the size of the input tensors size_t input0_size = input0_data.size(); size_t input1_size = input1_data.size(); std::vector<int64_t> input0_shape({1, 16}); std::vector<int64_t> input1_shape({1, 16}); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest, input0, dtype, &input0_shape[0], input0_shape.size()), "setting input 0 meta-data for the request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest, input1, dtype, &input1_shape[0], input1_shape.size()), "setting input 1 meta-data for the request"); auto output0 = is_torch_model ? "OUTPUT__0" : "OUTPUT0"; auto output1 = is_torch_model ? "OUTPUT__1" : "OUTPUT1"; FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), "requesting output 0 for the request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), "requesting output 1 for the request"); const void* input0_base = &input0_data[0]; const void* input1_base = &input1_data[0]; bool gpu_input = (io_spec.input_type_ == TRITONSERVER_MEMORY_GPU); std::unique_ptr<void, decltype(gpu_data_deleter)> input0_gpu( nullptr, gpu_data_deleter); std::unique_ptr<void, decltype(gpu_data_deleter)> input1_gpu( nullptr, gpu_data_deleter); if (gpu_input) { FAIL_IF_CUDA_ERR( cudaSetDevice(io_spec.input_type_id_), "setting CUDA device to device " + std::to_string(io_spec.input_type_id_)); void* dst; FAIL_IF_CUDA_ERR( cudaMalloc(&dst, input0_size), "allocating GPU memory for INPUT0 data"); input0_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input0_data[0], input0_size, cudaMemcpyHostToDevice), "setting INPUT0 data in GPU memory"); FAIL_IF_CUDA_ERR( cudaMalloc(&dst, input1_size), "allocating GPU memory for INPUT1 data"); input1_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input1_data[0], input1_size, cudaMemcpyHostToDevice), "setting INPUT1 data in GPU memory"); } input0_base = gpu_input ? input0_gpu.get() : &input0_data[0]; input1_base = gpu_input ? input1_gpu.get() : &input1_data[0]; if (host_policy_cstr == nullptr) { FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input0, input0_base, input0_size, io_spec.input_type_, io_spec.input_type_id_), "assigning INPUT0 data"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input1, input1_base, input1_size, io_spec.input_type_, io_spec.input_type_id_), "assigning INPUT1 data"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( irequest, input0, zero_data.data(), zero_data.size() * sizeof(uint32_t), TRITONSERVER_MEMORY_CPU, 0, "fake_host_policy_name"), "assigning zero INPUT0 data with host policy 'fake_host_policy_name'"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( irequest, input1, zero_data.data(), zero_data.size() * sizeof(uint32_t), TRITONSERVER_MEMORY_CPU, 0, "fake_host_policy_name"), "assigning zero INPUT1 data with host policy 'fake_host_policy_name'"); } else { FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input0, zero_data.data(), zero_data.size() * sizeof(uint32_t), TRITONSERVER_MEMORY_CPU, 0), "assigning zero INPUT0 data"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input1, zero_data.data(), zero_data.size() * sizeof(uint32_t), TRITONSERVER_MEMORY_CPU, 0), "assigning zero INPUT1 data"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( irequest, input0, input0_base, input0_size, io_spec.input_type_, io_spec.input_type_id_, host_policy_cstr), "assigning INPUT0 data to provided host policy"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy( irequest, input1, input1_base, input1_size, io_spec.input_type_, io_spec.input_type_id_, host_policy_cstr), "assigning INPUT1 data to provided host policy"); } // Perform inference... auto p = new std::promise<TRITONSERVER_InferenceResponse*>(); std::future<TRITONSERVER_InferenceResponse*> completed = p->get_future(); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast<void*>(p)), "setting response callback"); FAIL_IF_ERR( TRITONSERVER_ServerInferAsync( server.get(), irequest, nullptr /* trace */), "running inference"); // Wait for the inference response and check the status. TRITONSERVER_InferenceResponse* response = completed.get(); FAIL_IF_ERR(TRITONSERVER_InferenceResponseError(response), "response status"); // Create the expected data for the two output tensors. std::vector<char> expected0_data; std::vector<char> expected1_data; if (dtype == TRITONSERVER_TYPE_BYTES) { GenerateStringOutputData(&expected0_data, &expected1_data); } // Check the output tensor values... // Note that depending on whether the backend supports outputs in GPU memory, // the output tensor may be in CPU memory even if -g flag is set. const void* output0_content; size_t output0_byte_size; TRITONSERVER_MemoryType output0_memory_type; int64_t output0_memory_type_id; { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; void* userp; FAIL_IF_ERR( TRITONSERVER_InferenceResponseOutput( response, OutputIndex(response, output0), &cname, &datatype, &shape, &dim_count, &output0_content, &output0_byte_size, &output0_memory_type, &output0_memory_type_id, &userp), "getting output0 info"); if (dtype == TRITONSERVER_TYPE_BYTES) { size_t expected0_size = expected0_data.size(); if (expected0_size != output0_byte_size) { FAIL( "unexpected output0 byte-size, expected " + std::to_string(expected0_size) + ", got " + std::to_string(output0_byte_size)); } } else if (output0_byte_size != input0_size) { FAIL( "unexpected output0 byte-size, expected " + std::to_string(input0_size) + ", got " + std::to_string(output0_byte_size)); } else if ( (io_spec.output_type_ != output0_memory_type) || (io_spec.output_type_id_ != output0_memory_type_id)) { FAIL( std::string("unexpected output0 memory type (id), expected to be " "allocated in ") + TRITONSERVER_MemoryTypeString(io_spec.output_type_) + " with id " + std::to_string(io_spec.output_type_id_) + ", got " + TRITONSERVER_MemoryTypeString(output0_memory_type) + " with id " + std::to_string(output0_memory_type_id)); } } const void* output1_content; size_t output1_byte_size; TRITONSERVER_MemoryType output1_memory_type; int64_t output1_memory_type_id; { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; void* userp; FAIL_IF_ERR( TRITONSERVER_InferenceResponseOutput( response, OutputIndex(response, output1), &cname, &datatype, &shape, &dim_count, &output1_content, &output1_byte_size, &output1_memory_type, &output1_memory_type_id, &userp), "getting output1 info"); if (dtype == TRITONSERVER_TYPE_BYTES) { size_t expected1_size = expected1_data.size(); if (expected1_size != output1_byte_size) { FAIL( "unexpected output1 byte-size, expected " + std::to_string(expected1_size) + ", got " + std::to_string(output1_byte_size)); } } else if (output1_byte_size != input1_size) { FAIL( "unexpected output1 byte-size, expected " + std::to_string(input1_size) + ", got " + std::to_string(output1_byte_size)); } else if ( (io_spec.output_type_ != output1_memory_type) || (io_spec.output_type_id_ != output1_memory_type_id)) { FAIL( std::string("unexpected output1 memory type (id), expected to be " "allocated in ") + TRITONSERVER_MemoryTypeString(io_spec.output_type_) + " with id " + std::to_string(io_spec.output_type_id_) + ", got " + TRITONSERVER_MemoryTypeString(output1_memory_type) + " with id " + std::to_string(output1_memory_type_id)); } } const void* output0_result = output0_content; const void* output1_result = output1_content; // Different from CPU memory, outputs in GPU memory must be copied to CPU // memory to be read directly. std::vector<char> output0_data(output0_byte_size); std::vector<char> output1_data(output1_byte_size); if (output0_memory_type == TRITONSERVER_MEMORY_CPU) { std::cout << "OUTPUT0 are stored in CPU memory" << std::endl; } else { std::cout << "OUTPUT0 are stored in GPU memory" << std::endl; FAIL_IF_CUDA_ERR( cudaMemcpy( &output0_data[0], output0_content, output0_byte_size, cudaMemcpyDeviceToHost), "setting INPUT0 data in GPU memory"); output0_result = &output0_data[0]; } if (output1_memory_type == TRITONSERVER_MEMORY_CPU) { std::cout << "OUTPUT1 are stored in CPU memory" << std::endl; } else { std::cout << "OUTPUT1 are stored in GPU memory" << std::endl; FAIL_IF_CUDA_ERR( cudaMemcpy( &output1_data[0], output1_content, output1_byte_size, cudaMemcpyDeviceToHost), "setting INPUT0 data in GPU memory"); output1_result = &output1_data[0]; } if (dtype == TRITONSERVER_TYPE_INT32) { CompareResult<int32_t>( output0, output1, &input0_data[0], &input1_data[0], output0_result, output1_result); } else if (dtype == TRITONSERVER_TYPE_FP32) { CompareResult<float>( output0, output1, &input0_data[0], &input1_data[0], output0_result, output1_result); } else { CompareStringResult( output0, output1, &input0_data[0], &input1_data[0], output0_result, output1_result); } FAIL_IF_ERR( TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorDelete(allocator), "deleting response allocator"); return 0; } ��������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/multi_server.cc���������������������������������������������������������������0000664�0000000�0000000�00000100646�15270523712�0017462�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <rapidjson/document.h> #include <rapidjson/error/en.h> #include <unistd.h> #include <chrono> #include <cstring> #include <future> #include <iostream> #include <string> #include <thread> #include <unordered_map> #include <vector> #include "common.h" #include "triton/core/tritonserver.h" #ifdef TRITON_ENABLE_GPU #include <cuda_runtime_api.h> #endif // TRITON_ENABLE_GPU namespace ni = triton::server; namespace { bool enforce_memory_type = false; TRITONSERVER_MemoryType requested_memory_type; #ifdef TRITON_ENABLE_GPU static auto cuda_data_deleter = [](void* data) { if (data != nullptr) { cudaPointerAttributes attr; auto cuerr = cudaPointerGetAttributes(&attr, data); if (cuerr != cudaSuccess) { std::cerr << "error: failed to get CUDA pointer attribute of " << data << ": " << cudaGetErrorString(cuerr) << std::endl; } if (attr.type == cudaMemoryTypeDevice) { cuerr = cudaFree(data); } else if (attr.type == cudaMemoryTypeHost) { cuerr = cudaFreeHost(data); } if (cuerr != cudaSuccess) { std::cerr << "error: failed to release CUDA pointer " << data << ": " << cudaGetErrorString(cuerr) << std::endl; } } }; #endif // TRITON_ENABLE_GPU void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-m <\"system\"|\"pinned\"|gpu>" << " Enforce the memory type for input and output tensors." << " If not specified, inputs will be in system memory and outputs" << " will be based on the model's preferred type." << std::endl; std::cerr << "\t-v Enable verbose logging" << std::endl; std::cerr << "\t-r [model repository absolute path]" << std::endl; std::cerr << "\t-t Thread count." << std::endl; std::cerr << "\t-l Number of loops to run in each thread." << std::endl; exit(1); } TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { // Initially attempt to make the actual memory type and id that we // allocate be the same as preferred memory type *actual_memory_type = preferred_memory_type; *actual_memory_type_id = preferred_memory_type_id; // If 'byte_size' is zero just return 'buffer' == nullptr, we don't // need to do any other book-keeping. if (byte_size == 0) { *buffer = nullptr; *buffer_userp = nullptr; std::cout << "allocated " << byte_size << " bytes for result tensor " << tensor_name << std::endl; } else { void* allocated_ptr = nullptr; if (enforce_memory_type) { *actual_memory_type = requested_memory_type; } switch (*actual_memory_type) { #ifdef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_CPU_PINNED: { auto err = cudaSetDevice(*actual_memory_type_id); if ((err != cudaSuccess) && (err != cudaErrorNoDevice) && (err != cudaErrorInsufficientDriver)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "unable to recover current CUDA device: " + std::string(cudaGetErrorString(err))) .c_str()); } err = cudaHostAlloc(&allocated_ptr, byte_size, cudaHostAllocPortable); if (err != cudaSuccess) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "cudaHostAlloc failed: " + std::string(cudaGetErrorString(err))) .c_str()); } break; } case TRITONSERVER_MEMORY_GPU: { auto err = cudaSetDevice(*actual_memory_type_id); if ((err != cudaSuccess) && (err != cudaErrorNoDevice) && (err != cudaErrorInsufficientDriver)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "unable to recover current CUDA device: " + std::string(cudaGetErrorString(err))) .c_str()); } err = cudaMalloc(&allocated_ptr, byte_size); if (err != cudaSuccess) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "cudaMalloc failed: " + std::string(cudaGetErrorString(err))) .c_str()); } break; } #endif // TRITON_ENABLE_GPU // Use CPU memory if the requested memory type is unknown // (default case). case TRITONSERVER_MEMORY_CPU: default: { *actual_memory_type = TRITONSERVER_MEMORY_CPU; allocated_ptr = malloc(byte_size); break; } } // Pass the tensor name with buffer_userp so we can show it when // releasing the buffer. if (allocated_ptr != nullptr) { *buffer = allocated_ptr; *buffer_userp = new std::string(tensor_name); std::cout << "allocated " << byte_size << " bytes in " << TRITONSERVER_MemoryTypeString(*actual_memory_type) << " for result tensor " << tensor_name << std::endl; } } return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { std::string* name = nullptr; if (buffer_userp != nullptr) { name = reinterpret_cast<std::string*>(buffer_userp); } else { name = new std::string("<unknown>"); } std::cout << "Releasing buffer " << buffer << " of size " << byte_size << " in " << TRITONSERVER_MemoryTypeString(memory_type) << " for result '" << *name << "'" << std::endl; switch (memory_type) { case TRITONSERVER_MEMORY_CPU: free(buffer); break; #ifdef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_CPU_PINNED: { auto err = cudaSetDevice(memory_type_id); if (err == cudaSuccess) { err = cudaFreeHost(buffer); } if (err != cudaSuccess) { std::cerr << "error: failed to cudaFree " << buffer << ": " << cudaGetErrorString(err) << std::endl; } break; } case TRITONSERVER_MEMORY_GPU: { auto err = cudaSetDevice(memory_type_id); if (err == cudaSuccess) { err = cudaFree(buffer); } if (err != cudaSuccess) { std::cerr << "error: failed to cudaFree " << buffer << ": " << cudaGetErrorString(err) << std::endl; } break; } #endif // TRITON_ENABLE_GPU default: std::cerr << "error: unexpected buffer allocated in CUDA managed memory" << std::endl; break; } delete name; return nullptr; // Success } void InferRequestComplete( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { // We reuse the request so we don't delete it here. } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { if (response != nullptr) { // Send 'response' to the future. std::promise<TRITONSERVER_InferenceResponse*>* p = reinterpret_cast<std::promise<TRITONSERVER_InferenceResponse*>*>(userp); p->set_value(response); delete p; } } TRITONSERVER_Error* ParseModelMetadata(const rapidjson::Document& model_metadata) { std::string seen_data_type; for (const auto& input : model_metadata["inputs"].GetArray()) { if (strcmp(input["datatype"].GetString(), "FP32")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "multi-server example only supports model with data type FP32"); } if (seen_data_type.empty()) { seen_data_type = input["datatype"].GetString(); } else if (strcmp(seen_data_type.c_str(), input["datatype"].GetString())) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "the inputs and outputs of model must have the data type"); } } for (const auto& output : model_metadata["outputs"].GetArray()) { if (strcmp(output["datatype"].GetString(), "FP32")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "multi-server example only supports model with data type FP32"); } else if (strcmp(seen_data_type.c_str(), output["datatype"].GetString())) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "the inputs and outputs of model must have the data type"); } } return nullptr; } void GenerateInputData( std::vector<float>* input0_data, std::vector<float>* input1_data) { input0_data->resize(16); input1_data->resize(16); for (size_t i = 0; i < 16; ++i) { input0_data->data()[i] = i; input1_data->data()[i] = 1; } } void CompareResult( const std::string& output0_name, const std::string& output1_name, const float* input0, const float* input1, const float* output0, const float* output1) { for (size_t i = 0; i < 16; ++i) { std::cout << input0[i] << " + " << input1[i] << " = " << output0[i] << std::endl; std::cout << input0[i] << " - " << input1[i] << " = " << output1[i] << std::endl; if ((input0[i] + input1[i]) != output0[i]) { FAIL("incorrect sum in " + output0_name); } if ((input0[i] - input1[i]) != output1[i]) { FAIL("incorrect difference in " + output1_name); } } } void Check( TRITONSERVER_InferenceResponse* response, const std::vector<float>& input0_data, const std::vector<float>& input1_data, const std::string& output0, const std::string& output1, const size_t expected_byte_size, const TRITONSERVER_DataType expected_datatype) { std::unordered_map<std::string, std::vector<float>> output_data; uint32_t output_count; FAIL_IF_ERR( TRITONSERVER_InferenceResponseOutputCount(response, &output_count), "getting number of response outputs"); if (output_count != 2) { FAIL("expecting 2 response outputs, got " + std::to_string(output_count)); } for (uint32_t idx = 0; idx < output_count; ++idx) { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; FAIL_IF_ERR( TRITONSERVER_InferenceResponseOutput( response, idx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp), "getting output info"); if (cname == nullptr) { FAIL("unable to get output name"); } std::string name(cname); if ((name != output0) && (name != output1)) { FAIL("unexpected output '" + name + "'"); } if ((dim_count != 2) || (shape[0] != 1) || (shape[1] != 16)) { FAIL("unexpected shape for '" + name + "'"); } if (datatype != expected_datatype) { FAIL( "unexpected datatype '" + std::string(TRITONSERVER_DataTypeString(datatype)) + "' for '" + name + "'"); } if (byte_size != expected_byte_size) { FAIL( "unexpected byte-size, expected " + std::to_string(expected_byte_size) + ", got " + std::to_string(byte_size) + " for " + name); } if (enforce_memory_type && (memory_type != requested_memory_type)) { FAIL( "unexpected memory type, expected to be allocated in " + std::string(TRITONSERVER_MemoryTypeString(requested_memory_type)) + ", got " + std::string(TRITONSERVER_MemoryTypeString(memory_type)) + ", id " + std::to_string(memory_type_id) + " for " + name); } // We make a copy of the data here... which we could avoid for // performance reasons but ok for this example. std::vector<float>& odata = output_data[name]; switch (memory_type) { case TRITONSERVER_MEMORY_CPU: { std::cout << name << " is stored in system memory" << std::endl; const float* cbase = reinterpret_cast<const float*>(base); odata.assign(cbase, cbase + (byte_size / sizeof(float))); break; } case TRITONSERVER_MEMORY_CPU_PINNED: { std::cout << name << " is stored in pinned memory" << std::endl; const float* cbase = reinterpret_cast<const float*>(base); odata.assign(cbase, cbase + (byte_size / sizeof(float))); break; } #ifdef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_GPU: { std::cout << name << " is stored in GPU memory" << std::endl; odata.reserve(byte_size); FAIL_IF_CUDA_ERR( cudaMemcpy(&odata[0], base, byte_size, cudaMemcpyDeviceToHost), "getting " + name + " data from GPU memory"); break; } #endif default: FAIL("unexpected memory type"); } } CompareResult( output0, output1, &input0_data[0], &input1_data[0], output_data[output0].data(), output_data[output1].data()); } } // namespace void SetServerOptions( TRITONSERVER_ServerOptions** server_options, bool verbose_level, std::string model_repository_path1, std::string model_repository_path2) { FAIL_IF_ERR( TRITONSERVER_ServerOptionsNew(server_options), "creating server options"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( *server_options, model_repository_path1.c_str()), "setting model repository path"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( *server_options, model_repository_path2.c_str()), "setting model repository path"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetLogVerbose(*server_options, verbose_level), "setting verbose logging level"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetMetrics(*server_options, true), "failed to enable metrics"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetStrictReadiness(*server_options, true), "failed to set strict readiness"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetStrictModelConfig(*server_options, true), "failed to set strict model config"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetModelControlMode( *server_options, TRITONSERVER_MODEL_CONTROL_EXPLICIT), "failed to set model control mode to explicit"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetBackendDirectory( *server_options, "/opt/tritonserver/backends"), "setting backend directory"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetRepoAgentDirectory( *server_options, "/opt/tritonserver/repoagents"), "setting repository agent directory"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetStrictModelConfig(*server_options, true), "setting strict model configuration"); #ifdef TRITON_ENABLE_GPU double min_compute_capability = TRITON_MIN_COMPUTE_CAPABILITY; #else double min_compute_capability = 0; #endif // TRITON_ENABLE_GPU FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( *server_options, min_compute_capability), "setting minimum supported CUDA compute capability"); } void CheckServerLiveAndReady(std::shared_ptr<TRITONSERVER_Server> server) { size_t wait_seconds = 0; while (true) { bool live, ready; FAIL_IF_ERR( TRITONSERVER_ServerIsLive(server.get(), &live), "unable to get server liveness"); FAIL_IF_ERR( TRITONSERVER_ServerIsReady(server.get(), &ready), "unable to get server readiness"); std::cout << "Server Health: live " << live << ", ready " << ready << std::endl; if (live && ready) { break; } if (++wait_seconds >= 10) { FAIL("failed to find healthy inference server"); } std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } } void PrintServerStatus(std::shared_ptr<TRITONSERVER_Server> server) { TRITONSERVER_Message* server_metadata_message; FAIL_IF_ERR( TRITONSERVER_ServerMetadata(server.get(), &server_metadata_message), "unable to get server metadata message"); const char* buffer; size_t byte_size; FAIL_IF_ERR( TRITONSERVER_MessageSerializeToJson( server_metadata_message, &buffer, &byte_size), "unable to serialize server metadata message"); std::cout << "Server Status:" << std::endl; std::cout << std::string(buffer, byte_size) << std::endl; FAIL_IF_ERR( TRITONSERVER_MessageDelete(server_metadata_message), "deleting status metadata"); } void AwaitModelReady( std::shared_ptr<TRITONSERVER_Server> server, const std::string model_name) { bool is_ready = false; size_t wait_seconds = 0; while (!is_ready) { FAIL_IF_ERR( TRITONSERVER_ServerModelIsReady( server.get(), model_name.c_str(), 1, &is_ready), "unable to get model readiness"); if (!is_ready) { if (++wait_seconds >= 5) { FAIL("model failed to be ready in 5 seconds"); } std::this_thread::sleep_for(std::chrono::milliseconds(1000)); continue; } TRITONSERVER_Message* model_metadata_message; FAIL_IF_ERR( TRITONSERVER_ServerModelMetadata( server.get(), model_name.c_str(), 1, &model_metadata_message), "unable to get model metadata message"); const char* buffer; size_t byte_size; FAIL_IF_ERR( TRITONSERVER_MessageSerializeToJson( model_metadata_message, &buffer, &byte_size), "unable to serialize model status protobuf"); rapidjson::Document model_metadata; model_metadata.Parse(buffer, byte_size); if (model_metadata.HasParseError()) { FAIL( "error: failed to parse model metadata from JSON: " + std::string(GetParseError_En(model_metadata.GetParseError())) + " at " + std::to_string(model_metadata.GetErrorOffset())); } FAIL_IF_ERR( TRITONSERVER_MessageDelete(model_metadata_message), "deleting status protobuf"); if (strcmp(model_metadata["name"].GetString(), model_name.c_str())) { FAIL("unable to find metadata for model"); } bool found_version = false; if (model_metadata.HasMember("versions")) { for (const auto& version : model_metadata["versions"].GetArray()) { if (strcmp(version.GetString(), "1") == 0) { found_version = true; break; } } } if (!found_version) { FAIL("unable to find version 1 status for model"); } FAIL_IF_ERR(ParseModelMetadata(model_metadata), "parsing model metadata"); } } void RunInferenceAndValidate( std::shared_ptr<TRITONSERVER_Server> server, TRITONSERVER_ResponseAllocator* allocator, const std::string model_name) { TRITONSERVER_InferenceRequest* irequest = nullptr; FAIL_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest, server.get(), model_name.c_str(), -1 /* model_version */), "creating inference request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetId(irequest, "my_request_id"), "setting ID for the request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestComplete, nullptr /* request_release_userp */), "setting request release callback"); // Inputs auto input0 = "INPUT0"; auto input1 = "INPUT1"; std::vector<int64_t> input0_shape({1, 16}); std::vector<int64_t> input1_shape({1, 16}); const TRITONSERVER_DataType datatype = TRITONSERVER_TYPE_FP32; FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest, input0, datatype, &input0_shape[0], input0_shape.size()), "setting input 0 meta-data for the request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest, input1, datatype, &input1_shape[0], input1_shape.size()), "setting input 1 meta-data for the request"); auto output0 = "OUTPUT0"; auto output1 = "OUTPUT1"; FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), "requesting output 0 for the request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), "requesting output 1 for the request"); // Create the data for the two input tensors. Initialize the first // to unique values and the second to all ones. std::vector<float> input0_data; std::vector<float> input1_data; GenerateInputData(&input0_data, &input1_data); size_t input0_size = input0_data.size() * 4; size_t input1_size = input1_data.size() * 4; const void* input0_base = &input0_data[0]; const void* input1_base = &input1_data[0]; #ifdef TRITON_ENABLE_GPU std::unique_ptr<void, decltype(cuda_data_deleter)> input0_gpu( nullptr, cuda_data_deleter); std::unique_ptr<void, decltype(cuda_data_deleter)> input1_gpu( nullptr, cuda_data_deleter); bool use_cuda_memory = (enforce_memory_type && (requested_memory_type != TRITONSERVER_MEMORY_CPU)); if (use_cuda_memory) { FAIL_IF_CUDA_ERR(cudaSetDevice(0), "setting CUDA device to device 0"); if (requested_memory_type != TRITONSERVER_MEMORY_CPU_PINNED) { void* dst; FAIL_IF_CUDA_ERR( cudaMalloc(&dst, input0_size), "allocating GPU memory for INPUT0 data"); input0_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input0_data[0], input0_size, cudaMemcpyHostToDevice), "setting INPUT0 data in GPU memory"); FAIL_IF_CUDA_ERR( cudaMalloc(&dst, input1_size), "allocating GPU memory for INPUT1 data"); input1_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input1_data[0], input1_size, cudaMemcpyHostToDevice), "setting INPUT1 data in GPU memory"); } else { void* dst; FAIL_IF_CUDA_ERR( cudaHostAlloc(&dst, input0_size, cudaHostAllocPortable), "allocating pinned memory for INPUT0 data"); input0_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input0_data[0], input0_size, cudaMemcpyHostToHost), "setting INPUT0 data in pinned memory"); FAIL_IF_CUDA_ERR( cudaHostAlloc(&dst, input1_size, cudaHostAllocPortable), "allocating pinned memory for INPUT1 data"); input1_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input1_data[0], input1_size, cudaMemcpyHostToHost), "setting INPUT1 data in pinned memory"); } } input0_base = use_cuda_memory ? input0_gpu.get() : &input0_data[0]; input1_base = use_cuda_memory ? input1_gpu.get() : &input1_data[0]; #endif // TRITON_ENABLE_GPU FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input0, input0_base, input0_size, requested_memory_type, 0 /* memory_type_id */), "assigning INPUT0 data"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input1, input1_base, input1_size, requested_memory_type, 0 /* memory_type_id */), "assigning INPUT1 data"); // Perform inference... { auto p = new std::promise<TRITONSERVER_InferenceResponse*>(); std::future<TRITONSERVER_InferenceResponse*> completed = p->get_future(); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast<void*>(p)), "setting response callback"); FAIL_IF_ERR( TRITONSERVER_ServerInferAsync( server.get(), irequest, nullptr /* trace */), "running inference"); // Wait for the inference to complete. TRITONSERVER_InferenceResponse* completed_response = completed.get(); FAIL_IF_ERR( TRITONSERVER_InferenceResponseError(completed_response), "response status"); Check( completed_response, input0_data, input1_data, output0, output1, input0_size, datatype); FAIL_IF_ERR( TRITONSERVER_InferenceResponseDelete(completed_response), "deleting inference response"); } FAIL_IF_ERR( TRITONSERVER_InferenceRequestDelete(irequest), "deleting inference request"); } void PrintModelStats( std::shared_ptr<TRITONSERVER_Server> server, const std::string model_name) { TRITONSERVER_Message* model_stats_message = nullptr; FAIL_IF_ERR( TRITONSERVER_ServerModelStatistics( server.get(), model_name.c_str(), -1 /* model_version */, &model_stats_message), "unable to get model stats message"); const char* buffer; size_t byte_size; FAIL_IF_ERR( TRITONSERVER_MessageSerializeToJson( model_stats_message, &buffer, &byte_size), "unable to serialize server metadata message"); std::cout << "Model '" << model_name << "' Stats:" << std::endl; std::cout << std::string(buffer, byte_size) << std::endl; FAIL_IF_ERR( TRITONSERVER_MessageDelete(model_stats_message), "deleting model stats message"); } void CreateAndRunTritonserverInstance( std::vector<std::string> model_repository_paths, size_t thread_id, bool verbose_level) { TRITONSERVER_ServerOptions* server_options = nullptr; SetServerOptions( &server_options, verbose_level, model_repository_paths[0], model_repository_paths[thread_id]); TRITONSERVER_Server* server_ptr = nullptr; FAIL_IF_ERR( TRITONSERVER_ServerNew(&server_ptr, server_options), "creating server instance no. " + std::to_string(thread_id)); FAIL_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); std::shared_ptr<TRITONSERVER_Server> server( server_ptr, TRITONSERVER_ServerDelete); // Wait and until the servers are both live and ready. CheckServerLiveAndReady(server); // Print status of the servers. PrintServerStatus(server); std::string model1 = "simple1", model2 = "simple" + std::to_string(thread_id + 1); // Load models in server. FAIL_IF_ERR( TRITONSERVER_ServerLoadModel(server.get(), model1.c_str()), "failed to load model"); FAIL_IF_ERR( TRITONSERVER_ServerLoadModel(server.get(), model2.c_str()), "failed to load model"); // Wait for the models to become available. AwaitModelReady(server, model1.c_str()); AwaitModelReady(server, model2.c_str()); // Create the allocator that will be used to allocate buffers for // the result tensors. TRITONSERVER_ResponseAllocator* allocator = nullptr; FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator, ResponseAlloc, ResponseRelease, nullptr /* start_fn */), "creating response allocator"); // Inference RunInferenceAndValidate(server, allocator, model1.c_str()); RunInferenceAndValidate(server, allocator, model2.c_str()); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorDelete(allocator), "deleting response allocator"); // Print Model Statistics for all models PrintModelStats(server, model1.c_str()); PrintModelStats(server, model2.c_str()); // Unload models in both servers. FAIL_IF_ERR( TRITONSERVER_ServerUnloadModel(server.get(), model1.c_str()), "failed to unload model"); FAIL_IF_ERR( TRITONSERVER_ServerUnloadModel(server.get(), model2.c_str()), "failed to unload model"); std::string wrong_model; if (thread_id == (model_repository_paths.size() + 1)) { wrong_model = "simple2"; } else { wrong_model = "simple" + std::to_string(thread_id + 2); } // Try to load wrong model. Expected to fail TRITONSERVER_Error* err = TRITONSERVER_ServerLoadModel(server.get(), wrong_model.c_str()); if (err == nullptr) { FAIL("Success when expected to failed to load wrong model"); } } static volatile std::atomic<int> counter(0); static std::mutex mutex; std::condition_variable cv; void RepeatedlyCreateAndRunInstance( std::vector<std::string> model_repository_paths, size_t thread_id, size_t loops, bool verbose_level) { std::unique_lock<std::mutex> lock(mutex); counter++; cv.wait(lock); for (size_t i = 0; i < loops; i++) { CreateAndRunTritonserverInstance( model_repository_paths, thread_id, verbose_level); } } int main(int argc, char** argv) { std::vector<std::string> model_repository_paths; int verbose_level = 0; int thread_count = 2; int loops = 1; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vm:r:t:l:")) != -1) { switch (opt) { case 'm': { enforce_memory_type = true; if (!strcmp(optarg, "system")) { requested_memory_type = TRITONSERVER_MEMORY_CPU; } else if (!strcmp(optarg, "pinned")) { requested_memory_type = TRITONSERVER_MEMORY_CPU_PINNED; } else if (!strcmp(optarg, "gpu")) { requested_memory_type = TRITONSERVER_MEMORY_GPU; } else { Usage( argv, "-m must be used to specify one of the following types:" " <\"system\"|\"pinned\"|gpu>"); } break; } case 'r': model_repository_paths.push_back(optarg); break; case 't': thread_count = std::stoi(optarg); break; case 'l': loops = std::stoi(optarg); break; case 'v': verbose_level = 1; break; case '?': Usage(argv); break; } } if ((thread_count < 1) && (loops < 1)) { Usage(argv, "thread_count and loops must be > 1"); } // model repository paths must be 'thread_count' + 1 if (int(model_repository_paths.size() - 1) != thread_count) { Usage( argv, "-r must be used to specify " + std::to_string(thread_count + 1) + " model repository paths, " + std::to_string(thread_count) + " unique paths and 1 common"); } for (const auto& repo_path : model_repository_paths) { if (repo_path.empty()) { Usage(argv, "model repository paths must not be empty"); } } #ifndef TRITON_ENABLE_GPU if (enforce_memory_type && requested_memory_type != TRITONSERVER_MEMORY_CPU) { Usage(argv, "-m can only be set to \"system\" without enabling GPU"); } #endif // TRITON_ENABLE_GPU // Check API version. uint32_t api_version_major, api_version_minor; FAIL_IF_ERR( TRITONSERVER_ApiVersion(&api_version_major, &api_version_minor), "getting Triton API version"); if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major) || (TRITONSERVER_API_VERSION_MINOR > api_version_minor)) { FAIL("triton server API version mismatch"); } // Create 'thread_count' number of instances of the server with 1 common and 1 // unique repo each std::thread tritonservers[thread_count]; for (int i = 0; i < thread_count; i++) { tritonservers[i] = std::thread( &RepeatedlyCreateAndRunInstance, model_repository_paths, size_t(i + 1), size_t(loops), verbose_level); } while (counter < thread_count) { usleep(50); } { std::unique_lock<std::mutex> lock(mutex); cv.notify_all(); } for (int i = 0; i < thread_count; ++i) { tritonservers[i].join(); } return 0; } ������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/restricted_features.h���������������������������������������������������������0000664�0000000�0000000�00000007740�15270523712�0020653�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <algorithm> #include <array> #include <string> namespace triton { namespace server { /// Header and Value pair for a restricted feature using Restriction = std::pair<std::string, std::string>; /// Restricted Categories enum RestrictedCategory : uint8_t { HEALTH, METADATA, INFERENCE, SHARED_MEMORY, MODEL_CONFIG, MODEL_REPOSITORY, STATISTICS, TRACE, LOGGING, INVALID, CATEGORY_COUNT = INVALID }; /// Restricted Category Names const std::array<const std::string, RestrictedCategory::CATEGORY_COUNT> RESTRICTED_CATEGORY_NAMES{ "health", "metadata", "inference", "shared-memory", "model-config", "model-repository", "statistics", "trace", "logging"}; /// Collection of restricted features /// /// Initially empty and all categories unrestricted class RestrictedFeatures { public: /// Returns RestrictedCategory enum from category name /// /// \param[in] category category name /// \return category enum returns INVALID if unknown static RestrictedCategory ToCategory(const std::string& category) { const auto found = std::find( begin(RESTRICTED_CATEGORY_NAMES), end(RESTRICTED_CATEGORY_NAMES), category); const auto offset = std::distance(begin(RESTRICTED_CATEGORY_NAMES), found); return RestrictedCategory(offset); } /// Insert restriction for given category /// /// \param[in] category category to restrict /// \param[in] restriction header, value pair void Insert(const RestrictedCategory& category, Restriction&& restriction) { restrictions_[category] = std::move(restriction); restricted_categories_[category] = true; } /// Get header,value pair for restricted category /// /// \param[in] category category to restrict /// \return restriction header, value pair const Restriction& Get(RestrictedCategory category) const { return restrictions_[category]; } /// Return true if a category is restricted /// /// \param[in] category category to restrict /// \return true if category is restricted, false otherwise const bool& IsRestricted(RestrictedCategory category) const { return restricted_categories_[category]; } RestrictedFeatures() = default; ~RestrictedFeatures() = default; private: std::array<Restriction, RestrictedCategory::CATEGORY_COUNT> restrictions_{}; std::array<bool, RestrictedCategory::CATEGORY_COUNT> restricted_categories_{}; }; }} // namespace triton::server ��������������������������������kytensor-2.49.0.7/src/sagemaker_server.cc�����������������������������������������������������������0000664�0000000�0000000�00000112235�15270523712�0020264�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "sagemaker_server.h" namespace triton { namespace server { #define HTTP_RESPOND_IF_ERR(REQ, X) \ do { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ EVBufferAddErrorJson((REQ)->buffer_out, err__); \ evhtp_send_reply((REQ), EVHTP_RES_BADREQ); \ TRITONSERVER_ErrorDelete(err__); \ return; \ } \ } while (false) namespace { void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { const char* message = TRITONSERVER_ErrorMessage(err); triton::common::TritonJson::Value response( triton::common::TritonJson::ValueType::OBJECT); response.AddStringRef("error", message, strlen(message)); triton::common::TritonJson::WriteBuffer buffer_json; response.Write(&buffer_json); evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); } TRITONSERVER_Error* EVBufferToJson( triton::common::TritonJson::Value* document, evbuffer_iovec* v, int* v_idx, const size_t length, int n) { size_t offset = 0, remaining_length = length; char* json_base; std::vector<char> json_buffer; // No need to memcpy when number of iovecs is 1 if ((n > 0) && (v[0].iov_len >= remaining_length)) { json_base = static_cast<char*>(v[0].iov_base); if (v[0].iov_len > remaining_length) { v[0].iov_base = static_cast<void*>(json_base + remaining_length); v[0].iov_len -= remaining_length; remaining_length = 0; } else if (v[0].iov_len == remaining_length) { remaining_length = 0; *v_idx += 1; } } else { json_buffer.resize(length); json_base = json_buffer.data(); while ((remaining_length > 0) && (*v_idx < n)) { char* base = static_cast<char*>(v[*v_idx].iov_base); size_t base_size; if (v[*v_idx].iov_len > remaining_length) { base_size = remaining_length; v[*v_idx].iov_base = static_cast<void*>(base + remaining_length); v[*v_idx].iov_len -= remaining_length; remaining_length = 0; } else { base_size = v[*v_idx].iov_len; remaining_length -= v[*v_idx].iov_len; *v_idx += 1; } memcpy(json_base + offset, base, base_size); offset += base_size; } } if (remaining_length != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "unexpected size for request JSON, expecting " + std::to_string(remaining_length) + " more bytes") .c_str()); } RETURN_IF_ERR(document->Parse(json_base, length)); return nullptr; // success } } // namespace const std::string SagemakerAPIServer::binary_mime_type_( "application/vnd.sagemaker-triton.binary+json;json-header-size="); TRITONSERVER_Error* SagemakerAPIServer::GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, size_t* header_length) { // Check mime type and set inference header length. // Set to content length in case that it is not specified *header_length = content_length; const char* content_type_c_str = evhtp_kv_find(req->headers_in, kContentTypeHeader); if (content_type_c_str != NULL) { std::string content_type(content_type_c_str); size_t pos = content_type.find(binary_mime_type_); if (pos != std::string::npos) { if (pos != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect MIME type for binary data starts with '") + binary_mime_type_ + "', got: " + content_type) .c_str()); } // Parse int32_t parsed_value; try { parsed_value = std::atoi(content_type_c_str + binary_mime_type_.length()); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse inference header size, got: ") + (content_type_c_str + binary_mime_type_.length())) .c_str()); } // Check if the content length is in proper range if ((parsed_value < 0) || (parsed_value > content_length)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("inference header size should be in range (0, ") + std::to_string(content_length) + "), got: " + (content_type_c_str + binary_mime_type_.length())) .c_str()); } *header_length = parsed_value; } } return nullptr; } void SagemakerAPIServer::SagemakeInferRequestClass::SetResponseHeader( bool has_binary_data, size_t header_length) { if (has_binary_data) { evhtp_headers_add_header( req_->headers_out, evhtp_header_new( kContentTypeHeader, (binary_mime_type_ + std::to_string(header_length)).c_str(), 1, 1)); } else { evhtp_headers_add_header( req_->headers_out, evhtp_header_new(kContentTypeHeader, "application/json", 1, 1)); } } void SagemakerAPIServer::Handle(evhtp_request_t* req) { LOG_VERBOSE(1) << "SageMaker request: " << req->method << " " << req->uri->path->full; if (RE2::FullMatch(std::string(req->uri->path->full), ping_regex_)) { HandleServerHealth(req, ping_mode_); return; } if (RE2::FullMatch(std::string(req->uri->path->full), invocations_regex_)) { HandleInfer(req, model_name_, model_version_str_); return; } std::string multi_model_name, action; if (RE2::FullMatch( std::string(req->uri->path->full), models_regex_, &multi_model_name, &action)) { switch (req->method) { case htp_method_GET: if (multi_model_name.empty()) { LOG_VERBOSE(1) << "SageMaker request: LIST ALL MODELS"; SageMakerMMEListModel(req); return; } else { LOG_VERBOSE(1) << "SageMaker request: GET MODEL"; SageMakerMMEGetModel(req, multi_model_name.c_str()); return; } case htp_method_POST: if (action == "/invoke") { LOG_VERBOSE(1) << "SageMaker request: INVOKE MODEL"; if (sagemaker_models_list_.find(multi_model_name.c_str()) == sagemaker_models_list_.end()) { evhtp_send_reply(req, EVHTP_RES_NOTFOUND); /* 404*/ return; } LOG_VERBOSE(1) << "SageMaker MME Custom Invoke Model Path"; /* Extract targetModel to log the associated archive */ const char* target_model = evhtp_kv_find(req->headers_in, "X-Amzn-SageMaker-Target-Model"); /* If target_model is not available (e.g., in local testing) use * model_name_hash as target_model) */ if (target_model == nullptr) { target_model = multi_model_name.c_str(); } LOG_INFO << "Invoking SageMaker TargetModel: " << target_model; SageMakerMMEHandleInfer(req, target_model, model_version_str_); return; } if (action.empty()) { LOG_VERBOSE(1) << "SageMaker request: LOAD MODEL"; std::unordered_map<std::string, std::string> parse_load_map; ParseSageMakerRequest(req, &parse_load_map, "load"); SageMakerMMELoadModel(req, parse_load_map); return; } break; case htp_method_DELETE: { // UNLOAD MODEL LOG_VERBOSE(1) << "SageMaker request: UNLOAD MODEL"; req->method = htp_method_POST; SageMakerMMEUnloadModel(req, multi_model_name.c_str()); return; } default: LOG_VERBOSE(1) << "SageMaker error: " << req->method << " " << req->uri->path->full << " - " << static_cast<int>(EVHTP_RES_BADREQ); evhtp_send_reply(req, EVHTP_RES_BADREQ); return; } } LOG_VERBOSE(1) << "SageMaker error: " << req->method << " " << req->uri->path->full << " - " << static_cast<int>(EVHTP_RES_BADREQ); evhtp_send_reply(req, EVHTP_RES_BADREQ); } TRITONSERVER_Error* SagemakerAPIServer::Create( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const int32_t port, const std::string address, const int thread_cnt, std::unique_ptr<HTTPServer>* http_server) { http_server->reset(new SagemakerAPIServer( server, trace_manager, shm_manager, port, address, thread_cnt)); const std::string addr = address + ":" + std::to_string(port); LOG_INFO << "Started Sagemaker HTTPService at " << addr; return nullptr; } void SagemakerAPIServer::ParseSageMakerRequest( evhtp_request_t* req, std::unordered_map<std::string, std::string>* parse_map, const std::string& action) { struct evbuffer_iovec* v = nullptr; int v_idx = 0; int n = evbuffer_peek(req->buffer_in, -1, NULL, NULL, 0); if (n > 0) { v = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * n)); if (evbuffer_peek(req->buffer_in, -1, NULL, v, n) != n) { HTTP_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error getting load model request buffers")); } } std::string model_name_string; std::string url_string; size_t buffer_len = evbuffer_get_length(req->buffer_in); if (buffer_len > 0) { triton::common::TritonJson::Value request; HTTP_RESPOND_IF_ERR( req, EVBufferToJson(&request, v, &v_idx, buffer_len, n)); triton::common::TritonJson::Value url; triton::common::TritonJson::Value model_name; if (request.Find("model_name", &model_name)) { HTTP_RESPOND_IF_ERR(req, model_name.AsString(&model_name_string)); LOG_VERBOSE(1) << "Received model_name: " << model_name_string.c_str(); } if ((action == "load") && (request.Find("url", &url))) { HTTP_RESPOND_IF_ERR(req, url.AsString(&url_string)); LOG_VERBOSE(1) << "Received url: " << url_string.c_str(); } } if (action == "load") { (*parse_map)["url"] = url_string.c_str(); } (*parse_map)["model_name_hash"] = model_name_string.c_str(); /* Extract target_model, specified in header, to log the associated archive */ const char* target_model = evhtp_kv_find(req->headers_in, "X-Amzn-SageMaker-Target-Model"); /* If target_model is not available (e.g., in local testing) use * model_name_hash as target_model) */ if (target_model != nullptr) { (*parse_map)["target_model"] = target_model; } else { (*parse_map)["target_model"] = model_name_string.c_str(); } LOG_INFO << "Loading SageMaker TargetModel: " << target_model; return; } void SagemakerAPIServer::SagemakeInferRequestClass::InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { // FIXME can't use InferRequestClass object here since it's lifetime // is different than response. For response we need to know how to // send each output (as json, shm, or binary) and that information // has to be maintained in a way that allows us to clean it up // appropriately if connection closed or last response sent. // // But for now userp is the InferRequestClass object and the end of // its life is in the ReplyCallback. SagemakerAPIServer::SagemakeInferRequestClass* infer_request = reinterpret_cast<SagemakerAPIServer::SagemakeInferRequestClass*>(userp); if (response != nullptr) { ++infer_request->response_count_; } TRITONSERVER_Error* err = nullptr; if (infer_request->response_count_ != 1) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "expected a single response, got " + std::to_string(infer_request->response_count_)) .c_str()); } else if (response != nullptr) { err = infer_request->FinalizeResponse(response); #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { infer_request->trace_->CaptureTimestamp( "INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp()); } #endif // TRITON_ENABLE_TRACING } LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); if (err != nullptr) { EVBufferAddErrorJson(infer_request->req_->buffer_out, err); // [FIXME] In http_server.cc, error handling is enhanced to reporting // different error code according to the Triton error code, holding // the change from SageMaker endpoint as it may not fit their SLA. infer_request->response_code_ = EVHTP_RES_BADREQ; if (SageMakerMMECheckOOMError(err) == true) { LOG_VERBOSE(1) << "Received an OOM error during INVOKE MODEL. Returning a 507." << std::endl; infer_request->response_code_ = 507; } TRITONSERVER_ErrorDelete(err); } // Defer sending the response until FINAL flag is seen if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { return; } evthr_defer(infer_request->thread_, ReplyCallback, infer_request); } void SagemakerAPIServer::SageMakerMMEHandleInfer( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str) { if (req->method != htp_method_POST) { evhtp_send_reply(req, EVHTP_RES_METHNALLOWED); return; } bool connection_paused = false; int64_t requested_model_version; auto err = GetModelVersionFromString( model_version_str.c_str(), &requested_model_version); if (err == nullptr) { uint32_t txn_flags; err = TRITONSERVER_ServerModelTransactionProperties( server_.get(), model_name.c_str(), requested_model_version, &txn_flags, nullptr /* voidp */); if ((err == nullptr) && (txn_flags & TRITONSERVER_TXN_DECOUPLED) != 0) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "HTTP end point doesn't support models with decoupled " "transaction policy"); } } // If tracing is enabled see if this request should be traced. TRITONSERVER_InferenceTrace* triton_trace = nullptr; std::shared_ptr<TraceManager::Trace> trace = StartTrace(req, model_name, &triton_trace); // Create the inference request object which provides all information needed // for an inference. TRITONSERVER_InferenceRequest* irequest = nullptr; std::shared_ptr<TRITONSERVER_InferenceRequest> irequest_shared = nullptr; if (err == nullptr) { err = TRITONSERVER_InferenceRequestNew( &irequest, server_.get(), model_name.c_str(), requested_model_version); } if (err == nullptr) { irequest_shared = std::shared_ptr<TRITONSERVER_InferenceRequest>( irequest, [](TRITONSERVER_InferenceRequest* request) { LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceRequestDelete(request), "deleting HTTP/REST inference request"); }); } // Decompress request body if it is compressed in supported type evbuffer* decompressed_buffer = nullptr; if (err == nullptr) { auto compression_type = GetRequestCompressionType(req); switch (compression_type) { case DataCompressor::Type::DEFLATE: case DataCompressor::Type::GZIP: { decompressed_buffer = evbuffer_new(); err = DataCompressor::DecompressData( compression_type, req->buffer_in, decompressed_buffer); break; } case DataCompressor::Type::UNKNOWN: { // Encounter unsupported compressed type, // send 415 error with supported types in Accept-Encoding evhtp_headers_add_header( req->headers_out, evhtp_header_new(kAcceptEncodingHTTPHeader, "gzip, deflate", 1, 1)); evhtp_send_reply(req, EVHTP_RES_UNSUPPORTED); return; } case DataCompressor::Type::IDENTITY: // Do nothing break; } } // Get the header length size_t header_length; if (err == nullptr) { // Set to body size in case there is no Content-Length to compare with int32_t content_length = evbuffer_get_length(req->buffer_in); if (decompressed_buffer == nullptr) { const char* content_length_c_str = evhtp_kv_find(req->headers_in, kContentLengthHeader); if (content_length_c_str != nullptr) { try { content_length = std::atoi(content_length_c_str); } catch (const std::invalid_argument& ia) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse ") + kContentLengthHeader + ", got: " + content_length_c_str) .c_str()); } } } else { // The Content-Length doesn't reflect the actual request body size // if compression is used, set 'content_length' to the decompressed size content_length = evbuffer_get_length(decompressed_buffer); } if (err == nullptr) { err = GetInferenceHeaderLength(req, content_length, &header_length); } } if (err == nullptr) { connection_paused = true; auto infer_request = CreateInferRequest(req, irequest_shared); auto request_release_payload = std::make_unique<RequestReleasePayload>( irequest_shared, decompressed_buffer); #ifdef TRITON_ENABLE_TRACING infer_request->trace_ = trace; #endif // TRITON_ENABLE_TRACING if (err == nullptr) { if (header_length != 0) { err = EVBufferToInput( model_name, irequest, (decompressed_buffer == nullptr) ? req->buffer_in : decompressed_buffer, infer_request.get(), header_length); } else { err = EVBufferToRawInput( model_name, irequest, (decompressed_buffer == nullptr) ? req->buffer_in : decompressed_buffer, infer_request.get()); } } if (err == nullptr) { err = TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestClass::InferRequestComplete, request_release_payload.get()); if (err == nullptr) { err = TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator_, reinterpret_cast<void*>(&infer_request->alloc_payload_), SagemakerAPIServer::SagemakeInferRequestClass:: InferResponseComplete, reinterpret_cast<void*>(infer_request.get())); LOG_VERBOSE(1) << std::endl; } if (err == nullptr) { err = TRITONSERVER_ServerInferAsync( server_.get(), irequest, triton_trace); #ifdef TRITON_ENABLE_TRACING if (trace != nullptr) { trace->trace_ = nullptr; } #endif // TRITON_ENABLE_TRACING } if (err == nullptr) { infer_request.release(); request_release_payload.release(); } } } if (err != nullptr) { LOG_VERBOSE(1) << "Infer failed: " << TRITONSERVER_ErrorMessage(err); evhtp_headers_add_header( req->headers_out, evhtp_header_new(kContentTypeHeader, "application/json", 1, 1)); SageMakerMMEHandleOOMError(req, err); if (connection_paused) { evhtp_request_resume(req); } TRITONSERVER_ErrorDelete(err); #ifdef TRITON_ENABLE_TRACING // If HTTP server still owns Triton trace if ((trace != nullptr) && (trace->trace_ != nullptr)) { TraceManager::TraceRelease(trace->trace_, trace->trace_userp_); } #endif // TRITON_ENABLE_TRACING } } TRITONSERVER_Error* SagemakerAPIServer::SageMakerMMECheckUnloadedModelIsUnavailable( const char* model_name, bool* is_model_unavailable) { /* Use the RepositoryIndex API to check if the model state has become UNAVAILABLE i.e. model is no longer in the 'in-the-process-of' being UNLOADED. Consequently, the reason field should be 'unloaded'.*/ TRITONSERVER_Message* server_model_index_message = nullptr; uint32_t ready_flag = 0; // value of 1 should be set if only the 'ready' // models are required from the index. In this case, // we need all models. TRITONSERVER_ServerModelIndex( server_.get(), ready_flag, &server_model_index_message); std::shared_ptr<TRITONSERVER_Message> shared_ptr_msg( server_model_index_message, [](TRITONSERVER_Message* msg) { TRITONSERVER_MessageDelete(msg); }); const char* index_buffer; size_t index_byte_size; RETURN_IF_ERR(TRITONSERVER_MessageSerializeToJson( server_model_index_message, &index_buffer, &index_byte_size)); /* Read into json buffer*/ triton::common::TritonJson::Value server_model_index_json; server_model_index_json.Parse(index_buffer, index_byte_size); const char* name; const char* state; const char* reason; const char* version; size_t name_len; size_t state_len; size_t reason_len; size_t version_len; for (size_t id = 0; id < server_model_index_json.ArraySize(); ++id) { triton::common::TritonJson::Value index_json; server_model_index_json.IndexAsObject(id, &index_json); RETURN_IF_ERR(index_json.MemberAsString("name", &name, &name_len)); if (std::string(name) == std::string(model_name)) { RETURN_IF_ERR(index_json.MemberAsString("state", &state, &state_len)); if (std::string(state) == UNLOAD_EXPECTED_STATE_) { RETURN_IF_ERR( index_json.MemberAsString("reason", &reason, &reason_len)); if (std::string(reason) == UNLOAD_EXPECTED_REASON_) { *is_model_unavailable = true; RETURN_IF_ERR( index_json.MemberAsString("version", &version, &version_len)); LOG_VERBOSE(1) << "Discovered model: " << name << ", version: " << version << " in state: " << state << " for the reason: " << reason; break; } } } } return nullptr; } void SagemakerAPIServer::SageMakerMMEUnloadModel( evhtp_request_t* req, const char* model_name_hash) { /* Extract targetModel to log the associated archive */ const char* target_model = evhtp_kv_find(req->headers_in, "X-Amzn-SageMaker-Target-Model"); /* If target_model is not available (e.g., in local testing) use * model_name_hash as target_model) */ if (target_model == nullptr) { target_model = model_name_hash; } if (sagemaker_models_list_.find(model_name_hash) == sagemaker_models_list_.end()) { LOG_VERBOSE(1) << "Model " << target_model << " with model hash " << model_name_hash << " is not loaded." << std::endl; evhtp_send_reply(req, EVHTP_RES_NOTFOUND); /* 404*/ return; } LOG_INFO << "Unloading SageMaker TargetModel: " << target_model << std::endl; auto start_time = std::chrono::high_resolution_clock::now(); /* Always unload dependents as well - this is required to unload dependents in * ensemble */ TRITONSERVER_Error* unload_err = nullptr; unload_err = TRITONSERVER_ServerUnloadModelAndDependents(server_.get(), target_model); if (unload_err != nullptr) { EVBufferAddErrorJson(req->buffer_out, unload_err); evhtp_send_reply(req, EVHTP_RES_BADREQ); LOG_ERROR << "Error when unloading SageMaker Model with dependents for model: " << target_model << std::endl; TRITONSERVER_ErrorDelete(unload_err); return; } /*Note: Model status check is repo-specific and therefore must be run before * unregistering the repo, else the model information is lost*/ bool is_model_unavailable = false; int64_t unload_time_in_secs = 0; /* Wait for the model to be completely unloaded. SageMaker waits a maximum of 360 seconds for the UNLOAD request to timeout. Setting a limit of 350 seconds for Triton unload. This should be run only if above UNLOAD call has succeeded.*/ if (unload_err == nullptr) { LOG_VERBOSE(1) << "Using Model Repository Index during UNLOAD to check for " "status of model hash: " << model_name_hash << " for model: " << target_model; while (is_model_unavailable == false && unload_time_in_secs < UNLOAD_TIMEOUT_SECS_) { LOG_VERBOSE(1) << "In the loop to wait for model to be unavailable"; unload_err = SageMakerMMECheckUnloadedModelIsUnavailable( target_model, &is_model_unavailable); if (unload_err != nullptr) { LOG_ERROR << "Error: Received non-zero exit code on checking for " "model unavailability. " << TRITONSERVER_ErrorMessage(unload_err); break; } std::this_thread::sleep_for( std::chrono::milliseconds(UNLOAD_SLEEP_MILLISECONDS_)); auto end_time = std::chrono::high_resolution_clock::now(); unload_time_in_secs = std::chrono::duration_cast<std::chrono::seconds>( end_time - start_time) .count(); } LOG_INFO << "UNLOAD for model " << target_model << " completed in " << unload_time_in_secs << " seconds."; TRITONSERVER_ErrorDelete(unload_err); } if ((is_model_unavailable == false) && (unload_time_in_secs >= UNLOAD_TIMEOUT_SECS_)) { LOG_ERROR << "Error: UNLOAD did not complete within expected " << UNLOAD_TIMEOUT_SECS_ << " seconds. This may " "result in SageMaker UNLOAD timeout."; } std::string repo_parent_path = sagemaker_models_list_.at(model_name_hash); TRITONSERVER_Error* unregister_err = nullptr; unregister_err = TRITONSERVER_ServerUnregisterModelRepository( server_.get(), repo_parent_path.c_str()); if (unregister_err != nullptr) { EVBufferAddErrorJson(req->buffer_out, unload_err); evhtp_send_reply(req, EVHTP_RES_BADREQ); LOG_ERROR << "Unable to unregister model repository for path: " << repo_parent_path << std::endl; } else { evhtp_send_reply(req, EVHTP_RES_OK); } TRITONSERVER_ErrorDelete(unregister_err); std::lock_guard<std::mutex> lock(models_list_mutex_); sagemaker_models_list_.erase(model_name_hash); } void SagemakerAPIServer::SageMakerMMEGetModel( evhtp_request_t* req, const char* model_name) { std::lock_guard<std::mutex> lock(models_list_mutex_); if (sagemaker_models_list_.find(model_name) == sagemaker_models_list_.end()) { evhtp_send_reply(req, EVHTP_RES_NOTFOUND); /* 404*/ return; } triton::common::TritonJson::Value sagemaker_get_json( triton::common::TritonJson::ValueType::OBJECT); sagemaker_get_json.AddString("modelName", model_name); sagemaker_get_json.AddString( "modelUrl", sagemaker_models_list_.at(model_name)); const char* buffer; size_t byte_size; triton::common::TritonJson::WriteBuffer json_buffer_; json_buffer_.Clear(); sagemaker_get_json.Write(&json_buffer_); byte_size = json_buffer_.Size(); buffer = json_buffer_.Base(); evbuffer_add(req->buffer_out, buffer, byte_size); evhtp_send_reply(req, EVHTP_RES_OK); } void SagemakerAPIServer::SageMakerMMEListModel(evhtp_request_t* req) { std::lock_guard<std::mutex> lock(models_list_mutex_); triton::common::TritonJson::Value sagemaker_list_json( triton::common::TritonJson::ValueType::OBJECT); triton::common::TritonJson::Value models_array( sagemaker_list_json, triton::common::TritonJson::ValueType::ARRAY); for (auto it = sagemaker_models_list_.begin(); it != sagemaker_models_list_.end(); it++) { triton::common::TritonJson::Value model_url_pair( models_array, triton::common::TritonJson::ValueType::OBJECT); bool ready = false; TRITONSERVER_ServerModelIsReady( server_.get(), it->first.c_str(), 1, &ready); /* Add to return list only if model is ready to be served */ if (ready) { model_url_pair.AddString("modelName", it->first); model_url_pair.AddString("modelUrl", it->second); } models_array.Append(std::move(model_url_pair)); } sagemaker_list_json.Add("models", std::move(models_array)); const char* buffer; size_t byte_size; triton::common::TritonJson::WriteBuffer json_buffer_; json_buffer_.Clear(); sagemaker_list_json.Write(&json_buffer_); byte_size = json_buffer_.Size(); buffer = json_buffer_.Base(); evbuffer_add(req->buffer_out, buffer, byte_size); evhtp_send_reply(req, EVHTP_RES_OK); } bool SagemakerAPIServer::SageMakerMMECheckOOMError(TRITONSERVER_Error* err) { const char* message = TRITONSERVER_ErrorMessage(err); std::string error_string(message); LOG_VERBOSE(1) << "Logging Verbose Error: " << std::endl << error_string.c_str() << std::endl; const std::vector<std::string> error_messages{ "CUDA out of memory", /* pytorch */ "CUDA_OUT_OF_MEMORY", /* tensorflow */ "Out of memory", /* generic */ "Out Of Memory", "out of memory", "MemoryError", "OutOfMemory", "OOM", "Dst tensor is not initialized", "Src tensor is not initialized", "CNMEM_STATUS_OUT_OF_MEMORY", "CUDNN_STATUS_NOT_INITIALIZED", "CUBLAS_STATUS_ALLOC_FAILED", "CUBLAS_STATUS_NOT_INITIALIZED", "Failed to allocate memory", "failed to allocate memory", "No space left on device"}; /* TODO: Improve the search to do pattern match on whole words only */ for (long unsigned int i = 0; i < error_messages.size(); i++) { if (error_string.find(error_messages[i]) != std::string::npos) { LOG_VERBOSE(1) << "OOM string '" << error_messages[i].c_str() << "' detected in logs."; return true; } } return false; } void SagemakerAPIServer::SageMakerMMEHandleOOMError( evhtp_request_t* req, TRITONSERVER_Error* err) { EVBufferAddErrorJson(req->buffer_out, err); if (SageMakerMMECheckOOMError(err) == true) { /* Return a 507*/ evhtp_send_reply(req, 507); LOG_VERBOSE(1) << "Received an OOM error during LOAD MODEL. Returning a 507."; return; } /* Return a 400*/ evhtp_send_reply(req, EVHTP_RES_BADREQ); return; } void SagemakerAPIServer::SageMakerMMELoadModel( evhtp_request_t* req, const std::unordered_map<std::string, std::string> parse_map) { std::string repo_path = parse_map.at("url"); std::string model_name_hash = parse_map.at("model_name_hash"); std::string target_model = parse_map.at("target_model"); /* Check subdirs for models and find ensemble model within the repo_path * If only 1 model, that will be selected as model_subdir * Else ensemble model directory is set as model_subdir */ DIR* dir; struct dirent* ent; int dir_count = 0; std::string model_subdir, ensemble_model_subdir; if ((dir = opendir(repo_path.c_str())) != NULL) { while ((ent = readdir(dir)) != NULL) { if ((ent->d_type == DT_DIR) && (!strcmp(ent->d_name, ".") == 0) && (!strcmp(ent->d_name, "..") == 0)) { dir_count += 1; model_subdir = std::string(ent->d_name); } if (dir_count >= 2) { LOG_VERBOSE(1) << "More than one model detected in archive. " "Checking if it is an ensemble." << std::endl; } LOG_VERBOSE(1) << "Reading model sub-directory: " << model_subdir.c_str() << std::endl; // Read the config.pbtxt file at each path, if available std::string ensemble_config_path = repo_path + "/" + model_subdir + "/" + "config.pbtxt"; std::ifstream config_fstream(ensemble_config_path); std::stringstream ensemble_config_content; if (config_fstream.is_open()) { ensemble_config_content << config_fstream.rdbuf(); } else { continue; // A valid config.pbtxt does not exist at this path, or // cannot be read } /* Compare matched string with `platform: "ensemble"` or * `platform:"ensemble"`. If present, we break, and use the model_subdir * to load the ensemble model */ std::string detected_ensemble_regex; if (RE2::PartialMatch( ensemble_config_content.str(), platform_ensemble_regex_, &detected_ensemble_regex)) { LOG_INFO << "SageMaker front-end detected an Ensemble config at path: " << ensemble_config_path << std::endl; ensemble_model_subdir = model_subdir; } if (dir_count > 5) { LOG_WARNING << "Several model directories found. If using ensemble, smaller " "ensembles are recommended for better memory management." << std::endl; } } closedir(dir); } if (!strcmp(ensemble_model_subdir.c_str(), "") == 0) { model_subdir = ensemble_model_subdir; } std::vector<const TRITONSERVER_Parameter*> subdir_modelname_map; /* Split repo path into three parts: * /opt/ml/models/<hash>/model/optional_customer_subdir * 1st repo_parent_path: /opt/ml/models/<hash> * 2nd subdir: model * 3rd customer_subdir: optional_customer_subdir */ std::string repo_parent_path, subdir, customer_subdir; RE2::FullMatch( repo_path, model_path_regex_, &repo_parent_path, &subdir, &customer_subdir); std::string config_path = repo_path + "/config.pbtxt"; struct stat buffer; /* If config.pbtxt is at repo root, * then repo_parent_path = /opt/ml/models/<hash>/, and model_subdir = model * else repo_parent_path = /opt/ml/models/<hash>/model and * model_subdir = dir under model/ */ if (stat(config_path.c_str(), &buffer) == 0) { model_subdir = subdir; } else { repo_parent_path = repo_path; } auto param = TRITONSERVER_ParameterNew( model_subdir.c_str(), TRITONSERVER_PARAMETER_STRING, target_model.c_str()); if (param != nullptr) { subdir_modelname_map.emplace_back(param); } else { HTTP_RESPOND_IF_ERR( req, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "unexpected error on creating Triton parameter")); } /* Register repository with model mapping */ TRITONSERVER_Error* err = nullptr; err = TRITONSERVER_ServerRegisterModelRepository( server_.get(), repo_parent_path.c_str(), subdir_modelname_map.data(), subdir_modelname_map.size()); TRITONSERVER_ParameterDelete(param); // If a model_name is reused i.e. model_name is already mapped, return a 409 if ((err != nullptr) && (TRITONSERVER_ErrorCode(err) == TRITONSERVER_ERROR_ALREADY_EXISTS)) { EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, EVHTP_RES_CONFLICT); /* 409 */ TRITONSERVER_ErrorDelete(err); return; } else if (err != nullptr) { EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, EVHTP_RES_BADREQ); TRITONSERVER_ErrorDelete(err); return; } err = TRITONSERVER_ServerLoadModel(server_.get(), target_model.c_str()); /* Unlikely after duplicate repo check, but in case Load Model also returns * ALREADY_EXISTS error */ if ((err != nullptr) && (TRITONSERVER_ErrorCode(err) == TRITONSERVER_ERROR_ALREADY_EXISTS)) { EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, EVHTP_RES_CONFLICT); /* 409 */ TRITONSERVER_ErrorDelete(err); return; } else if (err != nullptr) { SageMakerMMEHandleOOMError(req, err); } else { std::lock_guard<std::mutex> lock(models_list_mutex_); /* Use model name hash as expected in SageMaker MME contract */ sagemaker_models_list_.emplace(model_name_hash, repo_parent_path); evhtp_send_reply(req, EVHTP_RES_OK); } /* Unregister model repository in case of load failure*/ if (err != nullptr) { err = TRITONSERVER_ServerUnregisterModelRepository( server_.get(), repo_parent_path.c_str()); LOG_VERBOSE(1) << "Unregistered model repository due to load failure for model: " << target_model << std::endl; } if (err != nullptr) { EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, EVHTP_RES_BADREQ); TRITONSERVER_ErrorDelete(err); } return; } }} // namespace triton::server �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/sagemaker_server.h������������������������������������������������������������0000664�0000000�0000000�00000015116�15270523712�0020126�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <sys/stat.h> #include <fstream> #include <mutex> #include "common.h" #include "dirent.h" #include "http_server.h" #include "triton/core/tritonserver.h" namespace triton { namespace server { // Handle Sagemaker HTTP requests to inference server APIs class SagemakerAPIServer : public HTTPAPIServer { public: static TRITONSERVER_Error* Create( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& smb_manager, const int32_t port, const std::string address, const int thread_cnt, std::unique_ptr<HTTPServer>* sagemaker_server); class SagemakeInferRequestClass : public InferRequestClass { public: explicit SagemakeInferRequestClass( TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr<TRITONSERVER_InferenceRequest>& triton_request) : InferRequestClass( server, req, response_compression_type, triton_request) { } using InferRequestClass::InferResponseComplete; static void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); void SetResponseHeader( const bool has_binary_data, const size_t header_length) override; }; private: explicit SagemakerAPIServer( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const int32_t port, const std::string address, const int thread_cnt) : HTTPAPIServer( server, trace_manager, shm_manager, port, false /* reuse_port */, address, "" /* header_forward_pattern */, thread_cnt), ping_regex_(R"(/ping)"), invocations_regex_(R"(/invocations)"), models_regex_(R"(/models(?:/)?([^/]+)?(/invoke)?)"), model_path_regex_( R"((\/opt\/ml\/models\/[0-9A-Za-z._]+)\/(model)\/?([0-9A-Za-z._]+)?)"), platform_ensemble_regex_(R"(platform:(\s)*\"ensemble\")"), ping_mode_(GetEnvironmentVariableOrDefault( "SAGEMAKER_TRITON_PING_MODE", "ready")), model_name_(GetEnvironmentVariableOrDefault( "SAGEMAKER_TRITON_DEFAULT_MODEL_NAME", "unspecified_SAGEMAKER_TRITON_DEFAULT_MODEL_NAME")), model_version_str_("") { } void ParseSageMakerRequest( evhtp_request_t* req, std::unordered_map<std::string, std::string>* parse_map, const std::string& action); void SageMakerMMEHandleInfer( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); void SageMakerMMELoadModel( evhtp_request_t* req, const std::unordered_map<std::string, std::string> parse_map); void SageMakerMMEHandleOOMError( evhtp_request_t* req, TRITONSERVER_Error* load_err); static bool SageMakerMMECheckOOMError(TRITONSERVER_Error* load_err); void SageMakerMMEUnloadModel(evhtp_request_t* req, const char* model_name); TRITONSERVER_Error* SageMakerMMECheckUnloadedModelIsUnavailable( const char* model_name, bool* is_model_unavailable); void SageMakerMMEListModel(evhtp_request_t* req); void SageMakerMMEGetModel(evhtp_request_t* req, const char* model_name); void Handle(evhtp_request_t* req) override; std::unique_ptr<InferRequestClass> CreateInferRequest( evhtp_request_t* req, const std::shared_ptr<TRITONSERVER_InferenceRequest>& triton_request) override { return std::unique_ptr<InferRequestClass>(new SagemakeInferRequestClass( server_.get(), req, GetResponseCompressionType(req), triton_request)); } TRITONSERVER_Error* GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, size_t* header_length) override; // Currently the compression schema hasn't been defined, // assume identity compression type is used for both request and response DataCompressor::Type GetRequestCompressionType(evhtp_request_t* req) override { return DataCompressor::Type::IDENTITY; } DataCompressor::Type GetResponseCompressionType(evhtp_request_t* req) override { return DataCompressor::Type::IDENTITY; } re2::RE2 ping_regex_; re2::RE2 invocations_regex_; re2::RE2 models_regex_; re2::RE2 model_path_regex_; re2::RE2 platform_ensemble_regex_; const std::string ping_mode_; /* For single model mode, assume that only one version of "model" is presented */ const std::string model_name_; const std::string model_version_str_; static const std::string binary_mime_type_; /* Maintain list of loaded models */ std::unordered_map<std::string, std::string> sagemaker_models_list_; /* Mutex to handle concurrent updates */ std::mutex models_list_mutex_; /* Constants */ const uint32_t UNLOAD_TIMEOUT_SECS_ = 350; const uint32_t UNLOAD_SLEEP_MILLISECONDS_ = 500; const std::string UNLOAD_EXPECTED_STATE_ = "UNAVAILABLE"; const std::string UNLOAD_EXPECTED_REASON_ = "unloaded"; }; }} // namespace triton::server ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/shared_memory_manager.cc������������������������������������������������������0000664�0000000�0000000�00000056105�15270523712�0021272�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "shared_memory_manager.h" // Not supporting shared memory for now #ifdef _WIN32 namespace triton { namespace server { SharedMemoryManager::~SharedMemoryManager() {} TRITONSERVER_Error* SharedMemoryManager::RegisterSystemSharedMemory( const std::string& name, const std::string& shm_key, const size_t offset, const size_t byte_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string("Shared memory feature is currently not supported on Windows") .c_str()); } #ifdef TRITON_ENABLE_GPU TRITONSERVER_Error* SharedMemoryManager::RegisterCUDASharedMemory( const std::string& name, const cudaIpcMemHandle_t* cuda_shm_handle, const size_t byte_size, const int device_id) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string("Shared memory feature is currently not supported on Windows") .c_str()); } TRITONSERVER_Error* SharedMemoryManager::GetCUDAHandle( const std::string& name, cudaIpcMemHandle_t** cuda_mem_handle) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string("Shared memory feature is currently not supported on Windows") .c_str()); } #endif // TRITON_ENABLE_GPU TRITONSERVER_Error* SharedMemoryManager::GetMemoryInfo( const std::string& name, size_t offset, size_t byte_size, void** shm_mapped_addr, TRITONSERVER_MemoryType* memory_type, int64_t* device_id) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string("Shared memory feature is currently not supported on Windows") .c_str()); } TRITONSERVER_Error* SharedMemoryManager::GetStatus( const std::string& name, TRITONSERVER_MemoryType memory_type, triton::common::TritonJson::Value* shm_status) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string("Shared memory feature is currently not supported on Windows") .c_str()); } TRITONSERVER_Error* SharedMemoryManager::Unregister( const std::string& name, TRITONSERVER_MemoryType memory_type) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string("Shared memory feature is currently not supported on Windows") .c_str()); } TRITONSERVER_Error* SharedMemoryManager::UnregisterAll(TRITONSERVER_MemoryType memory_type) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string("Shared memory feature is currently not supported on Windows") .c_str()); } TRITONSERVER_Error* SharedMemoryManager::UnregisterHelper( const std::string& name, TRITONSERVER_MemoryType memory_type) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, std::string("Shared memory feature is currently not supported on Windows") .c_str()); } }} // namespace triton::server #else #include <errno.h> #include <fcntl.h> #include <sys/mman.h> #include <sys/stat.h> #include <unistd.h> #include "common.h" #include "triton/common/logging.h" namespace triton { namespace server { namespace { TRITONSERVER_Error* OpenSharedMemoryRegion(const std::string& shm_key, int* shm_fd) { // get shared memory region descriptor *shm_fd = shm_open(shm_key.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (*shm_fd == -1) { LOG_VERBOSE(1) << "shm_open failed, errno: " << errno; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string("Unable to open shared memory region: '" + shm_key + "'") .c_str()); } return nullptr; } TRITONSERVER_Error* GetSharedMemoryRegionSize( const std::string& shm_key, int shm_fd, size_t* shm_region_size) { struct stat file_status; if (fstat(shm_fd, &file_status) == -1) { LOG_VERBOSE(1) << "fstat on shm_fd failed, errno: " << errno; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string("Invalid shared memory region: '" + shm_key + "'").c_str()); } // According to POSIX standard, type off_t can be negative, so for sake of // catching possible under/overflows, assert that the size is non-negative. if (file_status.st_size < 0) { LOG_VERBOSE(1) << "File size of shared memory region must be non-negative"; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string("Invalid shared memory region: '" + shm_key + "'").c_str()); } *shm_region_size = static_cast<size_t>(file_status.st_size); return nullptr; // success } TRITONSERVER_Error* CheckSharedMemoryRegionSize( const std::string& name, const std::string& shm_key, int shm_fd, size_t offset, size_t byte_size) { size_t shm_region_size = 0; RETURN_IF_ERR(GetSharedMemoryRegionSize(shm_key, shm_fd, &shm_region_size)); // User-provided offset and byte_size should not go out-of-bounds. if ((offset + byte_size) > shm_region_size) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "failed to register shared memory region '" + name + "': invalid args") .c_str()); } return nullptr; // success } TRITONSERVER_Error* MapSharedMemory( const int shm_fd, const size_t offset, const size_t byte_size, void** mapped_addr) { // map shared memory to process address space *mapped_addr = mmap(NULL, byte_size, PROT_WRITE | PROT_READ, MAP_SHARED, shm_fd, offset); if (*mapped_addr == MAP_FAILED) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "unable to process address space" + std::string(std::strerror(errno))) .c_str()); } return nullptr; } TRITONSERVER_Error* CloseSharedMemoryRegion(int shm_fd) { int status = close(shm_fd); if (status == -1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "unable to close shared memory descriptor, errno: " + std::string(std::strerror(errno))) .c_str()); } return nullptr; } TRITONSERVER_Error* UnmapSharedMemory(void* mapped_addr, size_t byte_size) { int status = munmap(mapped_addr, byte_size); if (status == -1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "unable to munmap shared memory region, errno: " + std::string(std::strerror(errno))) .c_str()); } return nullptr; } #ifdef TRITON_ENABLE_GPU TRITONSERVER_Error* OpenCudaIPCRegion( const cudaIpcMemHandle_t* cuda_shm_handle, void** data_ptr, int device_id) { // Set to device curres cudaSetDevice(device_id); // Open CUDA IPC handle and read data from it cudaError_t err = cudaIpcOpenMemHandle( data_ptr, *cuda_shm_handle, cudaIpcMemLazyEnablePeerAccess); if (err != cudaSuccess) { // Log detailed error message and send generic error to client LOG_ERROR << "failed to open CUDA IPC handle: " << cudaGetErrorString(err); return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("failed to register shared memory region: invalid args") .c_str()); } return nullptr; } // Using `cudaGetDriverEntryPoint` from CUDA runtime API to get CUDA driver // entry point. This approach is used to avoid linking against CUDA driver // library so that when Triton is built with GPU support, it can still be run on // CPU-only environments. TRITONSERVER_Error* GetCudaDriverEntryPoint(const char* name, void** func_ptr) { cudaError_t err = cudaGetDriverEntryPoint(name, func_ptr, cudaEnableDefault); if (err != cudaSuccess) { LOG_ERROR << "Failed to get CUDA driver entry point for " << name << ": " << cudaGetErrorString(err); return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string("Failed to get CUDA driver entry point").c_str()); } return nullptr; } TRITONSERVER_Error* GetCudaSharedMemoryRegionSize(CUdeviceptr data_ptr, size_t& shm_region_size) { void* cu_mem_get_address_range = nullptr; void* cu_get_error_string = nullptr; RETURN_IF_ERR(GetCudaDriverEntryPoint( "cuMemGetAddressRange", &cu_mem_get_address_range)); RETURN_IF_ERR( GetCudaDriverEntryPoint("cuGetErrorString", &cu_get_error_string)); CUdeviceptr* base = nullptr; CUresult result = (( CUresult(*)(CUdeviceptr*, size_t*, CUdeviceptr))cu_mem_get_address_range)( base, &shm_region_size, data_ptr); if (result != CUDA_SUCCESS) { const char* errorString; if (((CUresult(*)(CUresult, const char**))cu_get_error_string)( result, &errorString) != CUDA_SUCCESS) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Failed to get CUDA error string"); } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "Failed to get CUDA address range: " + std::string(errorString)) .c_str()); } return nullptr; } TRITONSERVER_Error* CheckCudaSharedMemoryRegionSize( const std::string& name, CUdeviceptr data_ptr, size_t byte_size) { size_t shm_region_size = 0; auto err = GetCudaSharedMemoryRegionSize(data_ptr, shm_region_size); // User-provided offset and byte_size should not go out-of-bounds. if (err != nullptr || byte_size > shm_region_size) { if (err != nullptr) { // Log detailed error message and send generic error to client LOG_ERROR << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); } return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "failed to register shared memory region '" + name + "': invalid args") .c_str()); } return nullptr; } #endif // TRITON_ENABLE_GPU } // namespace SharedMemoryManager::~SharedMemoryManager() { UnregisterAll(TRITONSERVER_MEMORY_CPU); UnregisterAll(TRITONSERVER_MEMORY_GPU); } TRITONSERVER_Error* SharedMemoryManager::RegisterSystemSharedMemory( const std::string& name, const std::string& shm_key, const size_t offset, const size_t byte_size) { std::lock_guard<std::mutex> lock(mu_); if (shared_memory_map_.find(name) != shared_memory_map_.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_ALREADY_EXISTS, std::string("shared memory region '" + name + "' already in manager") .c_str()); } // register void* mapped_addr; int shm_fd = -1; // don't re-open if shared memory is already open for (auto itr = shared_memory_map_.begin(); itr != shared_memory_map_.end(); ++itr) { if (itr->second->shm_key_ == shm_key) { // FIXME: Consider invalid file descriptors after close shm_fd = itr->second->shm_fd_; break; } } // open and set new shm_fd if new shared memory key if (shm_fd == -1) { RETURN_IF_ERR(OpenSharedMemoryRegion(shm_key, &shm_fd)); } // Enforce that registered region is in-bounds of shm file object. RETURN_IF_ERR( CheckSharedMemoryRegionSize(name, shm_key, shm_fd, offset, byte_size)); // Mmap and then close the shared memory descriptor TRITONSERVER_Error* err_mmap = MapSharedMemory(shm_fd, offset, byte_size, &mapped_addr); TRITONSERVER_Error* err_close = CloseSharedMemoryRegion(shm_fd); if (err_mmap != nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "failed to register shared memory region '" + name + "': " + TRITONSERVER_ErrorMessage(err_mmap)) .c_str()); } if (err_close != nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "failed to register shared memory region '" + name + "': " + TRITONSERVER_ErrorMessage(err_close)) .c_str()); } shared_memory_map_.insert(std::make_pair( name, std::unique_ptr<SharedMemoryInfo>(new SharedMemoryInfo( name, shm_key, offset, byte_size, shm_fd, mapped_addr, TRITONSERVER_MEMORY_CPU, 0)))); return nullptr; // success } #ifdef TRITON_ENABLE_GPU TRITONSERVER_Error* SharedMemoryManager::RegisterCUDASharedMemory( const std::string& name, const cudaIpcMemHandle_t* cuda_shm_handle, const size_t byte_size, const int device_id) { // Serialize all operations that write/read current shared memory regions std::lock_guard<std::mutex> lock(mu_); // If name is already in shared_memory_map_ then return error saying already // registered if (shared_memory_map_.find(name) != shared_memory_map_.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_ALREADY_EXISTS, std::string("shared memory region '" + name + "' already in manager") .c_str()); } // register void* mapped_addr; // Get CUDA shared memory base address RETURN_IF_ERR(OpenCudaIPCRegion(cuda_shm_handle, &mapped_addr, device_id)); // Enforce that registered region is in-bounds of shm file object. RETURN_IF_ERR(CheckCudaSharedMemoryRegionSize( name, reinterpret_cast<CUdeviceptr>(mapped_addr), byte_size)); shared_memory_map_.insert(std::make_pair( name, std::unique_ptr<CUDASharedMemoryInfo>(new CUDASharedMemoryInfo( name, "", 0, byte_size, 0, mapped_addr, TRITONSERVER_MEMORY_GPU, device_id, cuda_shm_handle)))); return nullptr; // success } #endif // TRITON_ENABLE_GPU TRITONSERVER_Error* SharedMemoryManager::GetMemoryInfo( const std::string& name, size_t offset, size_t byte_size, void** shm_mapped_addr, TRITONSERVER_MemoryType* memory_type, int64_t* device_id) { // protect shared_memory_map_ from concurrent access std::lock_guard<std::mutex> lock(mu_); auto it = shared_memory_map_.find(name); if (it == shared_memory_map_.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, std::string("Unable to find shared memory region: '" + name + "'") .c_str()); } // validate offset size_t shm_region_end = 0; if (it->second->kind_ == TRITONSERVER_MEMORY_CPU) { shm_region_end = it->second->offset_; } if (it->second->byte_size_ > 0) { shm_region_end += it->second->byte_size_ - 1; } if (offset > shm_region_end) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("Invalid offset for shared memory region: '" + name + "'") .c_str()); } // validate byte_size + offset is within memory bounds size_t total_req_shm = offset + byte_size - 1; if (total_req_shm > shm_region_end) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "Invalid offset + byte size for shared memory region: '" + name + "'") .c_str()); } if (it->second->kind_ == TRITONSERVER_MEMORY_CPU) { *shm_mapped_addr = (void*)((uint8_t*)it->second->mapped_addr_ + it->second->offset_ + offset); } else { *shm_mapped_addr = (void*)((uint8_t*)it->second->mapped_addr_ + offset); } *memory_type = it->second->kind_; *device_id = it->second->device_id_; return nullptr; } #ifdef TRITON_ENABLE_GPU TRITONSERVER_Error* SharedMemoryManager::GetCUDAHandle( const std::string& name, cudaIpcMemHandle_t** cuda_mem_handle) { // protect shared_memory_map_ from concurrent access std::lock_guard<std::mutex> lock(mu_); auto it = shared_memory_map_.find(name); if (it == shared_memory_map_.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, std::string("Unable to find shared memory region: '" + name + "'") .c_str()); } CUDASharedMemoryInfo& shm_info = reinterpret_cast<CUDASharedMemoryInfo&>(*(it->second)); *cuda_mem_handle = &(shm_info.cuda_ipc_handle_); return nullptr; } #endif TRITONSERVER_Error* SharedMemoryManager::GetStatus( const std::string& name, TRITONSERVER_MemoryType memory_type, triton::common::TritonJson::Value* shm_status) { std::lock_guard<std::mutex> lock(mu_); if (name.empty()) { for (const auto& shm_info : shared_memory_map_) { if (shm_info.second->kind_ == memory_type) { triton::common::TritonJson::Value shm_region( *shm_status, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_ERR(shm_region.AddString( "name", shm_info.first.c_str(), shm_info.first.size())); if (memory_type == TRITONSERVER_MEMORY_CPU) { RETURN_IF_ERR(shm_region.AddString( "key", shm_info.second->shm_key_.c_str(), shm_info.second->shm_key_.size())); RETURN_IF_ERR(shm_region.AddUInt("offset", shm_info.second->offset_)); } else { RETURN_IF_ERR( shm_region.AddUInt("device_id", shm_info.second->device_id_)); } RETURN_IF_ERR( shm_region.AddUInt("byte_size", shm_info.second->byte_size_)); RETURN_IF_ERR(shm_status->Append(std::move(shm_region))); } } } else { auto it = shared_memory_map_.find(name); if (it == shared_memory_map_.end()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, std::string( "Unable to find system shared memory region: '" + name + "'") .c_str()); } if (it->second->kind_ != memory_type) { if (it->second->kind_ == TRITONSERVER_MEMORY_GPU) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, std::string( "The region named '" + name + "' is registered as CUDA shared " "memory, not system shared memory") .c_str()); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_NOT_FOUND, std::string( "The region named '" + name + "' is registered as system shared " "memory, not CUDA shared memory") .c_str()); } } triton::common::TritonJson::Value shm_region( *shm_status, triton::common::TritonJson::ValueType::OBJECT); RETURN_IF_ERR(shm_region.AddString( "name", it->second->name_.c_str(), it->second->name_.size())); if (memory_type == TRITONSERVER_MEMORY_CPU) { RETURN_IF_ERR(shm_region.AddString( "key", it->second->shm_key_.c_str(), it->second->shm_key_.size())); RETURN_IF_ERR(shm_region.AddUInt("offset", it->second->offset_)); } else { RETURN_IF_ERR(shm_region.AddUInt("device_id", it->second->device_id_)); } RETURN_IF_ERR(shm_region.AddUInt("byte_size", it->second->byte_size_)); RETURN_IF_ERR(shm_status->Append(std::move(shm_region))); } return nullptr; // success } TRITONSERVER_Error* SharedMemoryManager::Unregister( const std::string& name, TRITONSERVER_MemoryType memory_type) { // Serialize all operations that write/read current shared memory regions std::lock_guard<std::mutex> lock(mu_); return UnregisterHelper(name, memory_type); } TRITONSERVER_Error* SharedMemoryManager::UnregisterAll(TRITONSERVER_MemoryType memory_type) { std::lock_guard<std::mutex> lock(mu_); std::string error_message = "Failed to unregister the following "; std::vector<std::string> unregister_fails; if (memory_type == TRITONSERVER_MEMORY_CPU) { // Serialize all operations that write/read current shared memory regions error_message += "system shared memory regions: "; for (auto it = shared_memory_map_.cbegin(), next_it = it; it != shared_memory_map_.cend(); it = next_it) { ++next_it; if (it->second->kind_ == TRITONSERVER_MEMORY_CPU) { TRITONSERVER_Error* err = UnregisterHelper(it->first, memory_type); if (err != nullptr) { unregister_fails.push_back(it->first); } } } } else if (memory_type == TRITONSERVER_MEMORY_GPU) { error_message += "cuda shared memory regions: "; for (auto it = shared_memory_map_.cbegin(), next_it = it; it != shared_memory_map_.cend(); it = next_it) { ++next_it; if (it->second->kind_ == TRITONSERVER_MEMORY_GPU) { TRITONSERVER_Error* err = UnregisterHelper(it->first, memory_type); ; if (err != nullptr) { unregister_fails.push_back(it->first); } } } } if (!unregister_fails.empty()) { for (auto unreg_fail : unregister_fails) { error_message += unreg_fail + " ,"; } LOG_ERROR << error_message; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, error_message.c_str()); } return nullptr; } TRITONSERVER_Error* SharedMemoryManager::UnregisterHelper( const std::string& name, TRITONSERVER_MemoryType memory_type) { // Must hold the lock on register_mu_ while calling this function. auto it = shared_memory_map_.find(name); if (it != shared_memory_map_.end() && it->second->kind_ == memory_type) { if (it->second->kind_ == TRITONSERVER_MEMORY_CPU) { RETURN_IF_ERR( UnmapSharedMemory(it->second->mapped_addr_, it->second->byte_size_)); } else { #ifdef TRITON_ENABLE_GPU cudaError_t err = cudaIpcCloseMemHandle(it->second->mapped_addr_); if (err != cudaSuccess) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "failed to close CUDA IPC handle: " + std::string(cudaGetErrorString(err))) .c_str()); } #else return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string( "failed to unregister CUDA shared memory region: '" + name + "', GPUs not supported") .c_str()); #endif // TRITON_ENABLE_GPU } // Remove region information from shared_memory_map_ shared_memory_map_.erase(it); } return nullptr; } }} // namespace triton::server #endif �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/shared_memory_manager.h�������������������������������������������������������0000664�0000000�0000000�00000020426�15270523712�0021131�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <cstring> #include <map> #include <memory> #include <mutex> #include "triton/core/tritonserver.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr #include "triton/common/triton_json.h" #ifdef TRITON_ENABLE_GPU #include <cuda.h> #include <cuda_runtime_api.h> #endif // TRITON_ENABLE_GPU namespace triton { namespace server { class SharedMemoryManager { public: SharedMemoryManager() = default; ~SharedMemoryManager(); /// Add a shared memory block representing shared memory in system /// (CPU) memory to the manager. Return TRITONSERVER_ERROR_ALREADY_EXISTS /// if a shared memory block of the same name already exists in the manager. /// \param name The name of the memory block. /// \param shm_key The name of the posix shared memory object /// containing the block of memory. /// \param offset The offset within the shared memory object to the /// start of the block. /// \param byte_size The size, in bytes of the block. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* RegisterSystemSharedMemory( const std::string& name, const std::string& shm_key, const size_t offset, const size_t byte_size); #ifdef TRITON_ENABLE_GPU /// Add a shared memory block representing shared memory in CUDA /// (GPU) memory to the manager. Return TRITONSERVER_ERROR_ALREADY_EXISTS /// if a shared memory block of the same name already exists in the manager. /// \param name The name of the memory block. /// \param cuda_shm_handle The unique memory handle to the cuda shared /// memory block. /// \param byte_size The size, in bytes of the block. /// \param device id The GPU number the shared memory region is in. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* RegisterCUDASharedMemory( const std::string& name, const cudaIpcMemHandle_t* cuda_shm_handle, const size_t byte_size, const int device_id); #endif // TRITON_ENABLE_GPU /// Get the access information for the shared memory block /// with the specified name. Return TRITONSERVER_ERROR_NOT_FOUND /// if named block doesn't exist. /// \param name The name of the shared memory block to get. /// \param offset The offset in the block /// \param byte_size The byte size to request for the shm region /// \param shm_mapped_addr Returns the pointer to the shared /// memory block with the specified name and offset /// \param memory_type Returns the type of the memory /// \param device_id Returns the device id associated with the /// memory block /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* GetMemoryInfo( const std::string& name, size_t offset, size_t byte_size, void** shm_mapped_addr, TRITONSERVER_MemoryType* memory_type, int64_t* device_id); #ifdef TRITON_ENABLE_GPU /// Get the CUDA memory handle associated with the block name. /// Return TRITONSERVER_ERROR_NOT_FOUND if named block doesn't exist. /// \param name The name of the shared memory block to get. /// \param cuda_mem_handle Returns the cuda memory handle with the memory /// block. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* GetCUDAHandle( const std::string& name, cudaIpcMemHandle_t** cuda_mem_handle); #endif /// Populates the status of active system/CUDA shared memory regions /// in the status JSON. If 'name' is empty then return status of all /// active system/CUDA shared memory regions as specified by 'memory_type'. /// \param name The name of the shared memory block to get the status of. /// \param memory_type The type of memory to get the status of. /// \param shm_status Returns status of active shared memory blocks in JSON. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* GetStatus( const std::string& name, TRITONSERVER_MemoryType memory_type, triton::common::TritonJson::Value* shm_status); /// Removes the named shared memory block of the specified type from /// the manager. Any future attempt to get the details of this block /// will result in an array till another block with the same name is /// added to the manager. /// \param name The name of the shared memory block to remove. /// \param memory_type The type of memory to unregister. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* Unregister( const std::string& name, TRITONSERVER_MemoryType memory_type); /// Unregister all shared memory blocks of specified type from the manager. /// \param memory_type The type of memory to unregister. /// \return a TRITONSERVER_Error indicating success or failure. TRITONSERVER_Error* UnregisterAll(TRITONSERVER_MemoryType memory_type); private: /// A helper function to remove the named shared memory blocks of /// specified type TRITONSERVER_Error* UnregisterHelper( const std::string& name, TRITONSERVER_MemoryType memory_type); /// A struct that records the shared memory regions registered by the shared /// memory manager. struct SharedMemoryInfo { SharedMemoryInfo( const std::string& name, const std::string& shm_key, const size_t offset, const size_t byte_size, int shm_fd, void* mapped_addr, const TRITONSERVER_MemoryType kind, const int64_t device_id) : name_(name), shm_key_(shm_key), offset_(offset), byte_size_(byte_size), shm_fd_(shm_fd), mapped_addr_(mapped_addr), kind_(kind), device_id_(device_id) { } std::string name_; std::string shm_key_; size_t offset_; size_t byte_size_; int shm_fd_; void* mapped_addr_; TRITONSERVER_MemoryType kind_; int64_t device_id_; }; #ifdef TRITON_ENABLE_GPU struct CUDASharedMemoryInfo : SharedMemoryInfo { CUDASharedMemoryInfo( const std::string& name, const std::string& shm_key, const size_t offset, const size_t byte_size, int shm_fd, void* mapped_addr, const TRITONSERVER_MemoryType kind, const int64_t device_id, const cudaIpcMemHandle_t* cuda_ipc_handle) : SharedMemoryInfo( name, shm_key, offset, byte_size, shm_fd, mapped_addr, kind, device_id), cuda_ipc_handle_(*cuda_ipc_handle) { } cudaIpcMemHandle_t cuda_ipc_handle_; }; #endif using SharedMemoryStateMap = std::map<std::string, std::unique_ptr<SharedMemoryInfo>>; // A map between the name and the details of the associated // shared memory block SharedMemoryStateMap shared_memory_map_; // A mutex to protect the concurrent access to shared_memory_map_ std::mutex mu_; }; }} // namespace triton::server ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/simple.cc���������������������������������������������������������������������0000664�0000000�0000000�00000106544�15270523712�0016236�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <rapidjson/document.h> #include <rapidjson/error/en.h> #include <unistd.h> #include <chrono> #include <cstring> #include <future> #include <iostream> #include <string> #include <thread> #include <unordered_map> #include <vector> #include "common.h" #include "triton/core/tritonserver.h" #ifdef TRITON_ENABLE_GPU #include <cuda_runtime_api.h> #endif // TRITON_ENABLE_GPU namespace ni = triton::server; namespace { bool enforce_memory_type = false; TRITONSERVER_MemoryType requested_memory_type; #ifdef TRITON_ENABLE_GPU static auto cuda_data_deleter = [](void* data) { if (data != nullptr) { cudaPointerAttributes attr; auto cuerr = cudaPointerGetAttributes(&attr, data); if (cuerr != cudaSuccess) { std::cerr << "error: failed to get CUDA pointer attribute of " << data << ": " << cudaGetErrorString(cuerr) << std::endl; } if (attr.type == cudaMemoryTypeDevice) { cuerr = cudaFree(data); } else if (attr.type == cudaMemoryTypeHost) { cuerr = cudaFreeHost(data); } if (cuerr != cudaSuccess) { std::cerr << "error: failed to release CUDA pointer " << data << ": " << cudaGetErrorString(cuerr) << std::endl; } } }; #endif // TRITON_ENABLE_GPU void Usage(char** argv, const std::string& msg = std::string()) { if (!msg.empty()) { std::cerr << msg << std::endl; } std::cerr << "Usage: " << argv[0] << " [options]" << std::endl; std::cerr << "\t-m <\"system\"|\"pinned\"|gpu>" << " Enforce the memory type for input and output tensors." << " If not specified, inputs will be in system memory and outputs" << " will be based on the model's preferred type." << std::endl; std::cerr << "\t-v Enable verbose logging" << std::endl; std::cerr << "\t-r [model repository absolute path]" << std::endl; exit(1); } TRITONSERVER_Error* ResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, int64_t preferred_memory_type_id, void* userp, void** buffer, void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type, int64_t* actual_memory_type_id) { // Initially attempt to make the actual memory type and id that we // allocate be the same as preferred memory type *actual_memory_type = preferred_memory_type; *actual_memory_type_id = preferred_memory_type_id; // If 'byte_size' is zero just return 'buffer' == nullptr, we don't // need to do any other book-keeping. if (byte_size == 0) { *buffer = nullptr; *buffer_userp = nullptr; std::cout << "allocated " << byte_size << " bytes for result tensor " << tensor_name << std::endl; } else { void* allocated_ptr = nullptr; if (enforce_memory_type) { *actual_memory_type = requested_memory_type; } switch (*actual_memory_type) { #ifdef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_CPU_PINNED: { auto err = cudaSetDevice(*actual_memory_type_id); if ((err != cudaSuccess) && (err != cudaErrorNoDevice) && (err != cudaErrorInsufficientDriver)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "unable to recover current CUDA device: " + std::string(cudaGetErrorString(err))) .c_str()); } err = cudaHostAlloc(&allocated_ptr, byte_size, cudaHostAllocPortable); if (err != cudaSuccess) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "cudaHostAlloc failed: " + std::string(cudaGetErrorString(err))) .c_str()); } break; } case TRITONSERVER_MEMORY_GPU: { auto err = cudaSetDevice(*actual_memory_type_id); if ((err != cudaSuccess) && (err != cudaErrorNoDevice) && (err != cudaErrorInsufficientDriver)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "unable to recover current CUDA device: " + std::string(cudaGetErrorString(err))) .c_str()); } err = cudaMalloc(&allocated_ptr, byte_size); if (err != cudaSuccess) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( "cudaMalloc failed: " + std::string(cudaGetErrorString(err))) .c_str()); } break; } #endif // TRITON_ENABLE_GPU // Use CPU memory if the requested memory type is unknown // (default case). case TRITONSERVER_MEMORY_CPU: default: { *actual_memory_type = TRITONSERVER_MEMORY_CPU; allocated_ptr = malloc(byte_size); break; } } // Pass the tensor name with buffer_userp so we can show it when // releasing the buffer. if (allocated_ptr != nullptr) { *buffer = allocated_ptr; *buffer_userp = new std::string(tensor_name); std::cout << "allocated " << byte_size << " bytes in " << TRITONSERVER_MemoryTypeString(*actual_memory_type) << " for result tensor " << tensor_name << std::endl; } } return nullptr; // Success } TRITONSERVER_Error* ResponseRelease( TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp, size_t byte_size, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id) { std::string* name = nullptr; if (buffer_userp != nullptr) { name = reinterpret_cast<std::string*>(buffer_userp); } else { name = new std::string("<unknown>"); } std::cout << "Releasing buffer " << buffer << " of size " << byte_size << " in " << TRITONSERVER_MemoryTypeString(memory_type) << " for result '" << *name << "'" << std::endl; switch (memory_type) { case TRITONSERVER_MEMORY_CPU: free(buffer); break; #ifdef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_CPU_PINNED: { auto err = cudaSetDevice(memory_type_id); if (err == cudaSuccess) { err = cudaFreeHost(buffer); } if (err != cudaSuccess) { std::cerr << "error: failed to cudaFree " << buffer << ": " << cudaGetErrorString(err) << std::endl; } break; } case TRITONSERVER_MEMORY_GPU: { auto err = cudaSetDevice(memory_type_id); if (err == cudaSuccess) { err = cudaFree(buffer); } if (err != cudaSuccess) { std::cerr << "error: failed to cudaFree " << buffer << ": " << cudaGetErrorString(err) << std::endl; } break; } #endif // TRITON_ENABLE_GPU default: std::cerr << "error: unexpected buffer allocated in CUDA managed memory" << std::endl; break; } delete name; return nullptr; // Success } void InferRequestRelease( TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp) { std::promise<void>* barrier = reinterpret_cast<std::promise<void>*>(userp); barrier->set_value(); } void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { if (response != nullptr) { // Send 'response' to the future. std::promise<TRITONSERVER_InferenceResponse*>* p = reinterpret_cast<std::promise<TRITONSERVER_InferenceResponse*>*>(userp); p->set_value(response); delete p; } } TRITONSERVER_Error* ParseModelMetadata( const rapidjson::Document& model_metadata, bool* is_int, bool* is_torch_model) { std::string seen_data_type; for (const auto& input : model_metadata["inputs"].GetArray()) { if (strcmp(input["datatype"].GetString(), "INT32") && strcmp(input["datatype"].GetString(), "FP32")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "simple lib example only supports model with data type INT32 or " "FP32"); } if (seen_data_type.empty()) { seen_data_type = input["datatype"].GetString(); } else if (strcmp(seen_data_type.c_str(), input["datatype"].GetString())) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "the inputs and outputs of 'simple' model must have the data type"); } } for (const auto& output : model_metadata["outputs"].GetArray()) { if (strcmp(output["datatype"].GetString(), "INT32") && strcmp(output["datatype"].GetString(), "FP32")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "simple lib example only supports model with data type INT32 or " "FP32"); } else if (strcmp(seen_data_type.c_str(), output["datatype"].GetString())) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "the inputs and outputs of 'simple' model must have the data type"); } } *is_int = (strcmp(seen_data_type.c_str(), "INT32") == 0); *is_torch_model = (strcmp(model_metadata["platform"].GetString(), "pytorch_libtorch") == 0); return nullptr; } template <typename T> void GenerateInputData( std::vector<char>* input0_data, std::vector<char>* input1_data) { input0_data->resize(16 * sizeof(T)); input1_data->resize(16 * sizeof(T)); for (size_t i = 0; i < 16; ++i) { ((T*)input0_data->data())[i] = i; ((T*)input1_data->data())[i] = 1; } } template <typename T> void CompareResult( const std::string& output0_name, const std::string& output1_name, const void* input0, const void* input1, const char* output0, const char* output1) { for (size_t i = 0; i < 16; ++i) { std::cout << ((T*)input0)[i] << " + " << ((T*)input1)[i] << " = " << ((T*)output0)[i] << std::endl; std::cout << ((T*)input0)[i] << " - " << ((T*)input1)[i] << " = " << ((T*)output1)[i] << std::endl; if ((((T*)input0)[i] + ((T*)input1)[i]) != ((T*)output0)[i]) { FAIL("incorrect sum in " + output0_name); } if ((((T*)input0)[i] - ((T*)input1)[i]) != ((T*)output1)[i]) { FAIL("incorrect difference in " + output1_name); } } } void Check( TRITONSERVER_InferenceResponse* response, const std::vector<char>& input0_data, const std::vector<char>& input1_data, const std::string& output0, const std::string& output1, const size_t expected_byte_size, const TRITONSERVER_DataType expected_datatype, const bool is_int) { std::unordered_map<std::string, std::vector<char>> output_data; uint32_t output_count; FAIL_IF_ERR( TRITONSERVER_InferenceResponseOutputCount(response, &output_count), "getting number of response outputs"); if (output_count != 2) { FAIL("expecting 2 response outputs, got " + std::to_string(output_count)); } for (uint32_t idx = 0; idx < output_count; ++idx) { const char* cname; TRITONSERVER_DataType datatype; const int64_t* shape; uint64_t dim_count; const void* base; size_t byte_size; TRITONSERVER_MemoryType memory_type; int64_t memory_type_id; void* userp; FAIL_IF_ERR( TRITONSERVER_InferenceResponseOutput( response, idx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp), "getting output info"); if (cname == nullptr) { FAIL("unable to get output name"); } std::string name(cname); if ((name != output0) && (name != output1)) { FAIL("unexpected output '" + name + "'"); } if ((dim_count != 2) || (shape[0] != 1) || (shape[1] != 16)) { FAIL("unexpected shape for '" + name + "'"); } if (datatype != expected_datatype) { FAIL( "unexpected datatype '" + std::string(TRITONSERVER_DataTypeString(datatype)) + "' for '" + name + "'"); } if (byte_size != expected_byte_size) { FAIL( "unexpected byte-size, expected " + std::to_string(expected_byte_size) + ", got " + std::to_string(byte_size) + " for " + name); } if (enforce_memory_type && (memory_type != requested_memory_type)) { FAIL( "unexpected memory type, expected to be allocated in " + std::string(TRITONSERVER_MemoryTypeString(requested_memory_type)) + ", got " + std::string(TRITONSERVER_MemoryTypeString(memory_type)) + ", id " + std::to_string(memory_type_id) + " for " + name); } // We make a copy of the data here... which we could avoid for // performance reasons but ok for this simple example. std::vector<char>& odata = output_data[name]; switch (memory_type) { case TRITONSERVER_MEMORY_CPU: { std::cout << name << " is stored in system memory" << std::endl; const char* cbase = reinterpret_cast<const char*>(base); odata.assign(cbase, cbase + byte_size); break; } case TRITONSERVER_MEMORY_CPU_PINNED: { std::cout << name << " is stored in pinned memory" << std::endl; const char* cbase = reinterpret_cast<const char*>(base); odata.assign(cbase, cbase + byte_size); break; } #ifdef TRITON_ENABLE_GPU case TRITONSERVER_MEMORY_GPU: { std::cout << name << " is stored in GPU memory" << std::endl; odata.reserve(byte_size); FAIL_IF_CUDA_ERR( cudaMemcpy(&odata[0], base, byte_size, cudaMemcpyDeviceToHost), "getting " + name + " data from GPU memory"); break; } #endif default: FAIL("unexpected memory type"); } } if (is_int) { CompareResult<int32_t>( output0, output1, &input0_data[0], &input1_data[0], output_data[output0].data(), output_data[output1].data()); } else { CompareResult<float>( output0, output1, &input0_data[0], &input1_data[0], output_data[output0].data(), output_data[output1].data()); } } } // namespace int main(int argc, char** argv) { std::string model_repository_path; int verbose_level = 0; // Parse commandline... int opt; while ((opt = getopt(argc, argv, "vm:r:")) != -1) { switch (opt) { case 'm': { enforce_memory_type = true; if (!strcmp(optarg, "system")) { requested_memory_type = TRITONSERVER_MEMORY_CPU; } else if (!strcmp(optarg, "pinned")) { requested_memory_type = TRITONSERVER_MEMORY_CPU_PINNED; } else if (!strcmp(optarg, "gpu")) { requested_memory_type = TRITONSERVER_MEMORY_GPU; } else { Usage( argv, "-m must be used to specify one of the following types:" " <\"system\"|\"pinned\"|gpu>"); } break; } case 'r': model_repository_path = optarg; break; case 'v': verbose_level = 1; break; case '?': Usage(argv); break; } } if (model_repository_path.empty()) { Usage(argv, "-r must be used to specify model repository path"); } #ifndef TRITON_ENABLE_GPU if (enforce_memory_type && requested_memory_type != TRITONSERVER_MEMORY_CPU) { Usage(argv, "-m can only be set to \"system\" without enabling GPU"); } #endif // TRITON_ENABLE_GPU // Check API version. This compares the API version of the // triton-server library linked into this application against the // API version of the header file used when compiling this // application. The API version of the shared library must be >= the // API version used when compiling this application. uint32_t api_version_major, api_version_minor; FAIL_IF_ERR( TRITONSERVER_ApiVersion(&api_version_major, &api_version_minor), "getting Triton API version"); if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major) || (TRITONSERVER_API_VERSION_MINOR > api_version_minor)) { FAIL("triton server API version mismatch"); } // Create the option setting to use when creating the inference // server object. TRITONSERVER_ServerOptions* server_options = nullptr; FAIL_IF_ERR( TRITONSERVER_ServerOptionsNew(&server_options), "creating server options"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetModelRepositoryPath( server_options, model_repository_path.c_str()), "setting model repository path"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetLogVerbose(server_options, verbose_level), "setting verbose logging level"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetBackendDirectory( server_options, "/opt/tritonserver/backends"), "setting backend directory"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetRepoAgentDirectory( server_options, "/opt/tritonserver/repoagents"), "setting repository agent directory"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetStrictModelConfig(server_options, true), "setting strict model configuration"); #ifdef TRITON_ENABLE_GPU double min_compute_capability = TRITON_MIN_COMPUTE_CAPABILITY; #else double min_compute_capability = 0; #endif // TRITON_ENABLE_GPU FAIL_IF_ERR( TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability( server_options, min_compute_capability), "setting minimum supported CUDA compute capability"); // Create the server object using the option settings. The server // object encapsulates all the functionality of the Triton server // and allows access to the Triton server API. Typically only a // single server object is needed by an application, but it is // allowed to create multiple server objects within a single // application. After the server object is created the server // options can be deleted. TRITONSERVER_Server* server_ptr = nullptr; FAIL_IF_ERR( TRITONSERVER_ServerNew(&server_ptr, server_options), "creating server object"); FAIL_IF_ERR( TRITONSERVER_ServerOptionsDelete(server_options), "deleting server options"); // Use a shared_ptr to manage the lifetime of the server object. std::shared_ptr<TRITONSERVER_Server> server( server_ptr, TRITONSERVER_ServerDelete); // Wait until the server is both live and ready. The server will not // appear "ready" until all models are loaded and ready to receive // inference requests. size_t health_iters = 0; while (true) { bool live, ready; FAIL_IF_ERR( TRITONSERVER_ServerIsLive(server.get(), &live), "unable to get server liveness"); FAIL_IF_ERR( TRITONSERVER_ServerIsReady(server.get(), &ready), "unable to get server readiness"); std::cout << "Server Health: live " << live << ", ready " << ready << std::endl; if (live && ready) { break; } if (++health_iters >= 10) { FAIL("failed to find healthy inference server"); } std::this_thread::sleep_for(std::chrono::milliseconds(500)); } // Server metadata can be accessed using the server object. The // metadata is returned as an abstract TRITONSERVER_Message that can // be converted to JSON for further processing. { TRITONSERVER_Message* server_metadata_message; FAIL_IF_ERR( TRITONSERVER_ServerMetadata(server.get(), &server_metadata_message), "unable to get server metadata message"); const char* buffer; size_t byte_size; FAIL_IF_ERR( TRITONSERVER_MessageSerializeToJson( server_metadata_message, &buffer, &byte_size), "unable to serialize server metadata message"); std::cout << "Server Metadata:" << std::endl; std::cout << std::string(buffer, byte_size) << std::endl; FAIL_IF_ERR( TRITONSERVER_MessageDelete(server_metadata_message), "deleting server metadata message"); } const std::string model_name("simple"); // We already waited for the server to be ready, above, so we know // that all models are also ready. But as an example we also wait // for a specific model to become available. bool is_torch_model = false; bool is_int = true; bool is_ready = false; health_iters = 0; while (!is_ready) { FAIL_IF_ERR( TRITONSERVER_ServerModelIsReady( server.get(), model_name.c_str(), 1 /* model_version */, &is_ready), "unable to get model readiness"); if (!is_ready) { if (++health_iters >= 10) { FAIL("model failed to be ready in 10 iterations"); } std::this_thread::sleep_for(std::chrono::milliseconds(500)); continue; } TRITONSERVER_Message* model_metadata_message; FAIL_IF_ERR( TRITONSERVER_ServerModelMetadata( server.get(), model_name.c_str(), 1, &model_metadata_message), "unable to get model metadata message"); const char* buffer; size_t byte_size; FAIL_IF_ERR( TRITONSERVER_MessageSerializeToJson( model_metadata_message, &buffer, &byte_size), "unable to serialize model metadata"); // Parse the JSON string that represents the model metadata into a // JSON document. We use rapidjson for this parsing but any JSON // parser can be used. rapidjson::Document model_metadata; model_metadata.Parse(buffer, byte_size); if (model_metadata.HasParseError()) { FAIL( "error: failed to parse model metadata from JSON: " + std::string(GetParseError_En(model_metadata.GetParseError())) + " at " + std::to_string(model_metadata.GetErrorOffset())); } FAIL_IF_ERR( TRITONSERVER_MessageDelete(model_metadata_message), "deleting model metadata message"); // Now that we have a document representation of the model // metadata, we can query it to extract some information about the // model. if (strcmp(model_metadata["name"].GetString(), model_name.c_str())) { FAIL("unable to find metadata for model"); } bool found_version = false; if (model_metadata.HasMember("versions")) { for (const auto& version : model_metadata["versions"].GetArray()) { if (strcmp(version.GetString(), "1") == 0) { found_version = true; break; } } } if (!found_version) { FAIL("unable to find version 1 status for model"); } FAIL_IF_ERR( ParseModelMetadata(model_metadata, &is_int, &is_torch_model), "parsing model metadata"); } // When triton needs a buffer to hold an output tensor, it will ask // us to provide the buffer. In this way we can have any buffer // management and sharing strategy that we want. To communicate to // triton the functions that we want it to call to perform the // allocations, we create a "response allocator" object. We pass // this response allocate object to triton when requesting // inference. We can reuse this response allocate object for any // number of inference requests. TRITONSERVER_ResponseAllocator* allocator = nullptr; FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorNew( &allocator, ResponseAlloc, ResponseRelease, nullptr /* start_fn */), "creating response allocator"); // Create an inference request object. The inference request object // is where we set the name of the model we want to use for // inference and the input tensors. TRITONSERVER_InferenceRequest* irequest = nullptr; FAIL_IF_ERR( TRITONSERVER_InferenceRequestNew( &irequest, server.get(), model_name.c_str(), -1 /* model_version */), "creating inference request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetId(irequest, "my_request_id"), "setting ID for the request"); std::unique_ptr<std::promise<void>> barrier = std::make_unique<std::promise<void>>(); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestRelease, reinterpret_cast<void*>(barrier.get())), "setting request release callback"); std::future<void> request_release_future = barrier->get_future(); // Add the 2 input tensors to the request... auto input0 = "INPUT0"; auto input1 = "INPUT1"; std::vector<int64_t> input0_shape({1, 16}); std::vector<int64_t> input1_shape({1, 16}); const TRITONSERVER_DataType datatype = (is_int) ? TRITONSERVER_TYPE_INT32 : TRITONSERVER_TYPE_FP32; FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest, input0, datatype, &input0_shape[0], input0_shape.size()), "setting input 0 meta-data for the request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddInput( irequest, input1, datatype, &input1_shape[0], input1_shape.size()), "setting input 1 meta-data for the request"); auto output0 = is_torch_model ? "OUTPUT__0" : "OUTPUT0"; auto output1 = is_torch_model ? "OUTPUT__1" : "OUTPUT1"; // Indicate that we want both output tensors calculated and returned // for the inference request. These calls are optional, if no // output(s) are specifically requested then all outputs defined by // the model will be calculated and returned. FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output0), "requesting output 0 for the request"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output1), "requesting output 1 for the request"); // Create the data for the two input tensors. Initialize the first // to unique values and the second to all ones. std::vector<char> input0_data; std::vector<char> input1_data; if (is_int) { GenerateInputData<int32_t>(&input0_data, &input1_data); } else { GenerateInputData<float>(&input0_data, &input1_data); } size_t input0_size = input0_data.size(); size_t input1_size = input1_data.size(); const void* input0_base = &input0_data[0]; const void* input1_base = &input1_data[0]; #ifdef TRITON_ENABLE_GPU std::unique_ptr<void, decltype(cuda_data_deleter)> input0_gpu( nullptr, cuda_data_deleter); std::unique_ptr<void, decltype(cuda_data_deleter)> input1_gpu( nullptr, cuda_data_deleter); bool use_cuda_memory = (enforce_memory_type && (requested_memory_type != TRITONSERVER_MEMORY_CPU)); if (use_cuda_memory) { FAIL_IF_CUDA_ERR(cudaSetDevice(0), "setting CUDA device to device 0"); if (requested_memory_type != TRITONSERVER_MEMORY_CPU_PINNED) { void* dst; FAIL_IF_CUDA_ERR( cudaMalloc(&dst, input0_size), "allocating GPU memory for INPUT0 data"); input0_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input0_data[0], input0_size, cudaMemcpyHostToDevice), "setting INPUT0 data in GPU memory"); FAIL_IF_CUDA_ERR( cudaMalloc(&dst, input1_size), "allocating GPU memory for INPUT1 data"); input1_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input1_data[0], input1_size, cudaMemcpyHostToDevice), "setting INPUT1 data in GPU memory"); } else { void* dst; FAIL_IF_CUDA_ERR( cudaHostAlloc(&dst, input0_size, cudaHostAllocPortable), "allocating pinned memory for INPUT0 data"); input0_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input0_data[0], input0_size, cudaMemcpyHostToHost), "setting INPUT0 data in pinned memory"); FAIL_IF_CUDA_ERR( cudaHostAlloc(&dst, input1_size, cudaHostAllocPortable), "allocating pinned memory for INPUT1 data"); input1_gpu.reset(dst); FAIL_IF_CUDA_ERR( cudaMemcpy(dst, &input1_data[0], input1_size, cudaMemcpyHostToHost), "setting INPUT1 data in pinned memory"); } } input0_base = use_cuda_memory ? input0_gpu.get() : &input0_data[0]; input1_base = use_cuda_memory ? input1_gpu.get() : &input1_data[0]; #endif // TRITON_ENABLE_GPU FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input0, input0_base, input0_size, requested_memory_type, 0 /* memory_type_id */), "assigning INPUT0 data"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input1, input1_base, input1_size, requested_memory_type, 0 /* memory_type_id */), "assigning INPUT1 data"); // Perform inference by calling TRITONSERVER_ServerInferAsync. This // call is asynchronous and therefore returns immediately. The // completion of the inference and delivery of the response is done // by triton by calling the "response complete" callback functions // (InferResponseComplete in this case). { auto p = new std::promise<TRITONSERVER_InferenceResponse*>(); std::future<TRITONSERVER_InferenceResponse*> completed = p->get_future(); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast<void*>(p)), "setting response callback"); FAIL_IF_ERR( TRITONSERVER_ServerInferAsync( server.get(), irequest, nullptr /* trace */), "running inference"); // The InferResponseComplete function sets the std::promise so // that this thread will block until the response is returned. TRITONSERVER_InferenceResponse* completed_response = completed.get(); FAIL_IF_ERR( TRITONSERVER_InferenceResponseError(completed_response), "response status"); Check( completed_response, input0_data, input1_data, output0, output1, input0_size, datatype, is_int); FAIL_IF_ERR( TRITONSERVER_InferenceResponseDelete(completed_response), "deleting inference response"); } // The TRITONSERVER_InferenceRequest object can be reused for // multiple (sequential) inference requests. For example, if we have // multiple requests where the inference request is the same except // for different input tensor data, then we can just change the // input data buffers. Below some input data is changed in place and // then another inference request is issued. For simplicity we only // do this when the input tensors are in non-pinned system memory. if (!enforce_memory_type || (requested_memory_type == TRITONSERVER_MEMORY_CPU)) { if (is_int) { int32_t* input0_base = reinterpret_cast<int32_t*>(&input0_data[0]); input0_base[0] = 27; } else { float* input0_base = reinterpret_cast<float*>(&input0_data[0]); input0_base[0] = 27.0; } auto p = new std::promise<TRITONSERVER_InferenceResponse*>(); std::future<TRITONSERVER_InferenceResponse*> completed = p->get_future(); // Using a new promise so have to re-register the callback to set // the promise as the userp. FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast<void*>(p)), "setting response callback"); // We need to make sure that the previous request was released before // reusing it. request_release_future.get(); // Register a new promise for the request callback barrier. barrier = std::make_unique<std::promise<void>>(); request_release_future = barrier->get_future(); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestRelease, reinterpret_cast<void*>(barrier.get())), "setting request release callback"); FAIL_IF_ERR( TRITONSERVER_ServerInferAsync( server.get(), irequest, nullptr /* trace */), "running inference"); TRITONSERVER_InferenceResponse* completed_response = completed.get(); FAIL_IF_ERR( TRITONSERVER_InferenceResponseError(completed_response), "response status"); Check( completed_response, input0_data, input1_data, output0, output1, input0_size, datatype, is_int); FAIL_IF_ERR( TRITONSERVER_InferenceResponseDelete(completed_response), "deleting inference response"); } // There are other TRITONSERVER_InferenceRequest APIs that allow // other in-place modifications so that the object can be reused for // multiple (sequential) inference requests. For example, we can // assign a new data buffer for an input by first removing the // existing data with // TRITONSERVER_InferenceRequestRemoveAllInputData. { FAIL_IF_ERR( TRITONSERVER_InferenceRequestRemoveAllInputData(irequest, input0), "removing INPUT0 data"); FAIL_IF_ERR( TRITONSERVER_InferenceRequestAppendInputData( irequest, input0, input1_base, input1_size, requested_memory_type, 0 /* memory_type_id */), "assigning INPUT1 data to INPUT0"); auto p = new std::promise<TRITONSERVER_InferenceResponse*>(); std::future<TRITONSERVER_InferenceResponse*> completed = p->get_future(); // Using a new promise so have to re-register the callback to set // the promise as the userp. FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetResponseCallback( irequest, allocator, nullptr /* response_allocator_userp */, InferResponseComplete, reinterpret_cast<void*>(p)), "setting response callback"); // Register a new promise for the request callback barrier. barrier = std::make_unique<std::promise<void>>(); request_release_future.get(); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( irequest, InferRequestRelease, reinterpret_cast<void*>(barrier.get())), "setting request release callback"); FAIL_IF_ERR( TRITONSERVER_ServerInferAsync( server.get(), irequest, nullptr /* trace */), "running inference"); TRITONSERVER_InferenceResponse* completed_response = completed.get(); FAIL_IF_ERR( TRITONSERVER_InferenceResponseError(completed_response), "response status"); // Both inputs are using input1_data... Check( completed_response, input1_data, input1_data, output0, output1, input0_size, datatype, is_int); FAIL_IF_ERR( TRITONSERVER_InferenceResponseDelete(completed_response), "deleting inference response"); } FAIL_IF_ERR( TRITONSERVER_InferenceRequestDelete(irequest), "deleting inference request"); FAIL_IF_ERR( TRITONSERVER_ResponseAllocatorDelete(allocator), "deleting response allocator"); return 0; } ������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/�������������������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0015403�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/CMakeLists.txt�����������������������������������������������������������0000664�0000000�0000000�00000005660�15270523712�0020152�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required (VERSION 3.16) # # Unit tests # find_package(GTest REQUIRED) # # Unit test for DataCompressor # if(${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) add_executable( data_compressor_test data_compressor_test.cc ../data_compressor.h ../common.h ) set_target_properties( data_compressor_test PROPERTIES SKIP_BUILD_RPATH TRUE BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH FALSE INSTALL_RPATH "" ) target_include_directories( data_compressor_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. ${GTEST_INCLUDE_DIRS} ${LIBEVENT_INCLUDE_DIRS} ) target_link_libraries( data_compressor_test PRIVATE triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core GTest::gtest GTest::gtest_main libevent.a -lz ) install( TARGETS data_compressor_test RUNTIME DESTINATION bin ) endif() add_subdirectory(repoagent/relocation_repoagent repoagent/relocation_repoagent) add_subdirectory(distributed_addsub distributed_addsub) add_subdirectory(dyna_sequence dyna_sequence) add_subdirectory(iterative_sequence iterative_sequence) add_subdirectory(query_backend query_backend) if(${TRITON_ENABLE_GPU}) add_subdirectory(sequence sequence) add_subdirectory(implicit_state implicit_state) endif() ��������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/data_compressor_test.cc��������������������������������������������������0000664�0000000�0000000�00000054111�15270523712�0022140�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "gtest/gtest.h" // Undefine the FAIL() macro inside Triton code to avoid redefine error // from gtest. Okay as FAIL() is not used in data_compressor #ifdef FAIL #undef FAIL #endif #include <event2/buffer.h> #include <chrono> #include <condition_variable> #include <fstream> #include <future> #include <limits> #include <mutex> #include <random> #include <string> #include <thread> #include <vector> #include "data_compressor.h" namespace ni = triton::server; namespace { struct TritonServerError { TritonServerError(TRITONSERVER_Error_Code code, const char* msg) : code_(code), msg_(msg) { } TRITONSERVER_Error_Code code_; std::string msg_; }; void WriteEVBufferToFile(const std::string& file_name, evbuffer* evb) { std::ofstream fs(file_name); struct evbuffer_iovec* buffer_array = nullptr; int buffer_count = evbuffer_peek(evb, -1, NULL, NULL, 0); if (buffer_count > 0) { buffer_array = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * buffer_count)); ASSERT_EQ( evbuffer_peek(evb, -1, NULL, buffer_array, buffer_count), buffer_count) << "unexpected error getting buffers for result"; } for (int idx = 0; idx < buffer_count; ++idx) { fs.write( reinterpret_cast<const char*>(buffer_array[idx].iov_base), buffer_array[idx].iov_len); } } void EVBufferToContiguousBuffer(evbuffer* evb, std::vector<char>* buffer) { *buffer = std::vector<char>(evbuffer_get_length(evb)); { struct evbuffer_iovec* buffer_array = nullptr; int buffer_count = evbuffer_peek(evb, -1, NULL, NULL, 0); if (buffer_count > 0) { buffer_array = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * buffer_count)); ASSERT_EQ( evbuffer_peek(evb, -1, NULL, buffer_array, buffer_count), buffer_count) << "unexpected error getting buffers for result"; } size_t offset = 0; for (int idx = 0; idx < buffer_count; ++idx) { memcpy( buffer->data() + offset, buffer_array[idx].iov_base, buffer_array[idx].iov_len); offset += buffer_array[idx].iov_len; } } } } // namespace #ifdef __cplusplus extern "C" { #endif TRITONSERVER_Error* TRITONSERVER_ErrorNew(TRITONSERVER_Error_Code code, const char* msg) { return reinterpret_cast<TRITONSERVER_Error*>( new TritonServerError(code, msg)); } TRITONSERVER_Error_Code TRITONSERVER_ErrorCode(TRITONSERVER_Error* error) { return (reinterpret_cast<TritonServerError*>(error))->code_; } const char* TRITONSERVER_ErrorMessage(TRITONSERVER_Error* error) { return (reinterpret_cast<TritonServerError*>(error))->msg_.c_str(); } #ifdef __cplusplus } #endif namespace { class DataCompressorTest : public ::testing::Test { public: DataCompressorTest() : raw_data_length_(0), deflate_compressed_length_(0), gzip_compressed_length_(0) { std::vector<std::string> files{ "raw_data", "deflate_compressed_data", "gzip_compressed_data"}; for (const auto& file : files) { std::fstream fs(file); // get length of file fs.seekg(0, fs.end); int length = fs.tellg(); fs.seekg(0, fs.beg); // allocate memory char* data = nullptr; if (file == "raw_data") { raw_data_.reset(new char[length]); data = raw_data_.get(); raw_data_length_ = length; } else if (file == "deflate_compressed_data") { deflate_compressed_data_.reset(new char[length]); data = deflate_compressed_data_.get(); deflate_compressed_length_ = length; } else { gzip_compressed_data_.reset(new char[length]); data = gzip_compressed_data_.get(); gzip_compressed_length_ = length; } fs.read(data, length); } } std::unique_ptr<char[]> raw_data_; size_t raw_data_length_; std::unique_ptr<char[]> deflate_compressed_data_; size_t deflate_compressed_length_; std::unique_ptr<char[]> gzip_compressed_data_; size_t gzip_compressed_length_; }; TEST_F(DataCompressorTest, DeflateOneBuffer) { // Convert the raw data into evbuffer format auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; ASSERT_EQ(evbuffer_add(source, raw_data_.get(), raw_data_length_), 0) << "Failed to initialize source evbuffer"; auto compressed = evbuffer_new(); ASSERT_TRUE((compressed != nullptr)) << "Failed to create compressed evbuffer"; auto decompressed = evbuffer_new(); ASSERT_TRUE((decompressed != nullptr)) << "Failed to create decompressed evbuffer"; auto err = ni::DataCompressor::CompressData( ni::DataCompressor::Type::DEFLATE, source, compressed); ASSERT_TRUE((err == nullptr)) << "Failed to compress data: " << TRITONSERVER_ErrorMessage(err); err = ni::DataCompressor::DecompressData( ni::DataCompressor::Type::DEFLATE, compressed, decompressed); ASSERT_TRUE((err == nullptr)) << "Failed to decompress data: " << TRITONSERVER_ErrorMessage(err); size_t destination_byte_size = evbuffer_get_length(decompressed); ASSERT_EQ(destination_byte_size, raw_data_length_) << "Mismatched byte size"; std::vector<char> res; EVBufferToContiguousBuffer(decompressed, &res); for (size_t idx = 0; idx < raw_data_length_; ++idx) { ASSERT_TRUE(raw_data_[idx] == res[idx]); } } TEST_F(DataCompressorTest, GzipOneBuffer) { // Convert the raw data into evbuffer format auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; ASSERT_EQ(evbuffer_add(source, raw_data_.get(), raw_data_length_), 0) << "Failed to initialize source evbuffer"; auto compressed = evbuffer_new(); ASSERT_TRUE((compressed != nullptr)) << "Failed to create compressed evbuffer"; auto decompressed = evbuffer_new(); ASSERT_TRUE((decompressed != nullptr)) << "Failed to create decompressed evbuffer"; auto err = ni::DataCompressor::CompressData( ni::DataCompressor::Type::GZIP, source, compressed); ASSERT_TRUE((err == nullptr)) << "Failed to compress data: " << TRITONSERVER_ErrorMessage(err); err = ni::DataCompressor::DecompressData( ni::DataCompressor::Type::GZIP, compressed, decompressed); ASSERT_TRUE((err == nullptr)) << "Failed to decompress data: " << TRITONSERVER_ErrorMessage(err); size_t destination_byte_size = evbuffer_get_length(decompressed); ASSERT_EQ(destination_byte_size, raw_data_length_) << "Mismatched byte size"; std::vector<char> res; EVBufferToContiguousBuffer(decompressed, &res); for (size_t idx = 0; idx < raw_data_length_; ++idx) { ASSERT_TRUE(raw_data_[idx] == res[idx]); } } TEST_F(DataCompressorTest, DeflateTwoBuffer) { // Convert the raw data into evbuffer format with two buffers auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; size_t half_length = raw_data_length_ / 2; ASSERT_EQ(evbuffer_add(source, raw_data_.get(), half_length), 0) << "Failed to initialize source evbuffer"; // verify evbuffer has two extend { auto second_source = evbuffer_new(); ASSERT_EQ( evbuffer_add( second_source, raw_data_.get() + half_length, raw_data_length_ - half_length), 0) << "Failed to initialize source evbuffer"; ASSERT_EQ(evbuffer_add_buffer(source, second_source), 0) << "Failed to initialize source evbuffer"; int buffer_count = evbuffer_peek(source, -1, NULL, NULL, 0); ASSERT_EQ(buffer_count, 2) << "Expect two buffers as source"; } auto compressed = evbuffer_new(); ASSERT_TRUE((compressed != nullptr)) << "Failed to create compressed evbuffer"; // Reconstruct the compressed buffer to be two buffers if (evbuffer_peek(compressed, -1, NULL, NULL, 0) == 1) { struct evbuffer_iovec* buffer_array = nullptr; int buffer_count = evbuffer_peek(compressed, -1, NULL, NULL, 0); if (buffer_count > 0) { buffer_array = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * buffer_count)); ASSERT_EQ( evbuffer_peek(compressed, -1, NULL, buffer_array, buffer_count), buffer_count) << "unexpected error getting buffers for result"; } auto first_compressed = evbuffer_new(); auto second_compressed = evbuffer_new(); size_t half_length = buffer_array[0].iov_len / 2; ASSERT_EQ( evbuffer_add(first_compressed, buffer_array[0].iov_base, half_length), 0) << "Failed to split compressed buffer"; ASSERT_EQ( evbuffer_add( second_compressed, reinterpret_cast<char*>(buffer_array[0].iov_base) + half_length, buffer_array[0].iov_len - half_length), 0) << "Failed to split compressed buffer"; ASSERT_EQ(evbuffer_add_buffer(first_compressed, second_compressed), 0) << "Failed to initialize source evbuffer"; compressed = first_compressed; } auto decompressed = evbuffer_new(); ASSERT_TRUE((decompressed != nullptr)) << "Failed to create decompressed evbuffer"; auto err = ni::DataCompressor::CompressData( ni::DataCompressor::Type::DEFLATE, source, compressed); ASSERT_TRUE((err == nullptr)) << "Failed to compress data: " << TRITONSERVER_ErrorMessage(err); err = ni::DataCompressor::DecompressData( ni::DataCompressor::Type::DEFLATE, compressed, decompressed); ASSERT_TRUE((err == nullptr)) << "Failed to decompress data: " << TRITONSERVER_ErrorMessage(err); size_t destination_byte_size = evbuffer_get_length(decompressed); ASSERT_EQ(destination_byte_size, raw_data_length_) << "Mismatched byte size"; std::vector<char> res; EVBufferToContiguousBuffer(decompressed, &res); for (size_t idx = 0; idx < raw_data_length_; ++idx) { ASSERT_TRUE(raw_data_[idx] == res[idx]); } } TEST_F(DataCompressorTest, GzipTwoBuffer) { // Convert the raw data into evbuffer format with two buffers auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; size_t half_length = raw_data_length_ / 2; ASSERT_EQ(evbuffer_add(source, raw_data_.get(), half_length), 0) << "Failed to initialize source evbuffer"; // verify evbuffer has two extend { auto second_source = evbuffer_new(); ASSERT_EQ( evbuffer_add( second_source, raw_data_.get() + half_length, raw_data_length_ - half_length), 0) << "Failed to initialize source evbuffer"; ASSERT_EQ(evbuffer_add_buffer(source, second_source), 0) << "Failed to initialize source evbuffer"; int buffer_count = evbuffer_peek(source, -1, NULL, NULL, 0); ASSERT_EQ(buffer_count, 2) << "Expect two buffers as source"; } auto compressed = evbuffer_new(); ASSERT_TRUE((compressed != nullptr)) << "Failed to create compressed evbuffer"; // Reconstruct the compressed buffer to be two buffers if (evbuffer_peek(compressed, -1, NULL, NULL, 0) == 1) { struct evbuffer_iovec* buffer_array = nullptr; int buffer_count = evbuffer_peek(compressed, -1, NULL, NULL, 0); if (buffer_count > 0) { buffer_array = static_cast<struct evbuffer_iovec*>( alloca(sizeof(struct evbuffer_iovec) * buffer_count)); ASSERT_EQ( evbuffer_peek(compressed, -1, NULL, buffer_array, buffer_count), buffer_count) << "unexpected error getting buffers for result"; } auto first_compressed = evbuffer_new(); auto second_compressed = evbuffer_new(); size_t half_length = buffer_array[0].iov_len / 2; ASSERT_EQ( evbuffer_add(first_compressed, buffer_array[0].iov_base, half_length), 0) << "Failed to split compressed buffer"; ASSERT_EQ( evbuffer_add( second_compressed, reinterpret_cast<char*>(buffer_array[0].iov_base) + half_length, buffer_array[0].iov_len - half_length), 0) << "Failed to split compressed buffer"; ASSERT_EQ(evbuffer_add_buffer(first_compressed, second_compressed), 0) << "Failed to initialize source evbuffer"; compressed = first_compressed; } auto decompressed = evbuffer_new(); ASSERT_TRUE((decompressed != nullptr)) << "Failed to create decompressed evbuffer"; auto err = ni::DataCompressor::CompressData( ni::DataCompressor::Type::GZIP, source, compressed); ASSERT_TRUE((err == nullptr)) << "Failed to compress data: " << TRITONSERVER_ErrorMessage(err); err = ni::DataCompressor::DecompressData( ni::DataCompressor::Type::GZIP, compressed, decompressed); ASSERT_TRUE((err == nullptr)) << "Failed to decompress data: " << TRITONSERVER_ErrorMessage(err); size_t destination_byte_size = evbuffer_get_length(decompressed); ASSERT_EQ(destination_byte_size, raw_data_length_) << "Mismatched byte size"; std::vector<char> res; EVBufferToContiguousBuffer(decompressed, &res); for (size_t idx = 0; idx < raw_data_length_; ++idx) { ASSERT_TRUE(raw_data_[idx] == res[idx]); } } TEST_F(DataCompressorTest, DeflateOneLargeBuffer) { // Duplicate raw data 2^20 times { std::unique_ptr<char[]> extended_raw_data( new char[raw_data_length_ * (1 << 20)]); memcpy(extended_raw_data.get(), raw_data_.get(), raw_data_length_); size_t filled_size = raw_data_length_; for (size_t i = 1; i < 20; ++i) { memcpy( extended_raw_data.get() + filled_size, extended_raw_data.get(), filled_size); filled_size += filled_size; } raw_data_length_ = filled_size; raw_data_.swap(extended_raw_data); } // Convert the raw data into evbuffer format auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; ASSERT_EQ(evbuffer_add(source, raw_data_.get(), raw_data_length_), 0) << "Failed to initialize source evbuffer"; auto compressed = evbuffer_new(); ASSERT_TRUE((compressed != nullptr)) << "Failed to create compressed evbuffer"; ASSERT_GE(raw_data_length_ / 2, evbuffer_get_length(compressed)) << "Compression should be desired for large data"; auto decompressed = evbuffer_new(); ASSERT_TRUE((decompressed != nullptr)) << "Failed to create decompressed evbuffer"; auto err = ni::DataCompressor::CompressData( ni::DataCompressor::Type::DEFLATE, source, compressed); ASSERT_TRUE((err == nullptr)) << "Failed to compress data: " << TRITONSERVER_ErrorMessage(err); err = ni::DataCompressor::DecompressData( ni::DataCompressor::Type::DEFLATE, compressed, decompressed); ASSERT_TRUE((err == nullptr)) << "Failed to decompress data: " << TRITONSERVER_ErrorMessage(err); size_t destination_byte_size = evbuffer_get_length(decompressed); ASSERT_EQ(destination_byte_size, raw_data_length_) << "Mismatched byte size"; std::vector<char> res; EVBufferToContiguousBuffer(decompressed, &res); for (size_t idx = 0; idx < raw_data_length_; ++idx) { ASSERT_TRUE(raw_data_[idx] == res[idx]); } } TEST_F(DataCompressorTest, GzipOneLargeBuffer) { // Duplicate raw data 2^20 times { std::unique_ptr<char[]> extended_raw_data( new char[raw_data_length_ * (1 << 20)]); memcpy(extended_raw_data.get(), raw_data_.get(), raw_data_length_); size_t filled_size = raw_data_length_; for (size_t i = 1; i < 20; ++i) { memcpy( extended_raw_data.get() + filled_size, extended_raw_data.get(), filled_size); filled_size += filled_size; } raw_data_length_ = filled_size; raw_data_.swap(extended_raw_data); } // Convert the raw data into evbuffer format auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; ASSERT_EQ(evbuffer_add(source, raw_data_.get(), raw_data_length_), 0) << "Failed to initialize source evbuffer"; auto compressed = evbuffer_new(); ASSERT_TRUE((compressed != nullptr)) << "Failed to create compressed evbuffer"; ASSERT_GE(raw_data_length_ / 2, evbuffer_get_length(compressed)) << "Compression should be desired for large data"; auto decompressed = evbuffer_new(); ASSERT_TRUE((decompressed != nullptr)) << "Failed to create decompressed evbuffer"; auto err = ni::DataCompressor::CompressData( ni::DataCompressor::Type::GZIP, source, compressed); ASSERT_TRUE((err == nullptr)) << "Failed to compress data: " << TRITONSERVER_ErrorMessage(err); err = ni::DataCompressor::DecompressData( ni::DataCompressor::Type::GZIP, compressed, decompressed); ASSERT_TRUE((err == nullptr)) << "Failed to decompress data: " << TRITONSERVER_ErrorMessage(err); size_t destination_byte_size = evbuffer_get_length(decompressed); ASSERT_EQ(destination_byte_size, raw_data_length_) << "Mismatched byte size"; std::vector<char> res; EVBufferToContiguousBuffer(decompressed, &res); for (size_t idx = 0; idx < raw_data_length_; ++idx) { ASSERT_TRUE(raw_data_[idx] == res[idx]); } } TEST_F(DataCompressorTest, DecompressDeflateBuffer) { // Convert the compressed data into evbuffer format auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; ASSERT_EQ( evbuffer_add( source, deflate_compressed_data_.get(), deflate_compressed_length_), 0) << "Failed to initialize source evbuffer"; auto decompressed = evbuffer_new(); ASSERT_TRUE((decompressed != nullptr)) << "Failed to create decompressed evbuffer"; auto err = ni::DataCompressor::DecompressData( ni::DataCompressor::Type::DEFLATE, source, decompressed); ASSERT_TRUE((err == nullptr)) << "Failed to decompress data: " << TRITONSERVER_ErrorMessage(err); size_t destination_byte_size = evbuffer_get_length(decompressed); ASSERT_EQ(destination_byte_size, raw_data_length_) << "Mismatched byte size"; std::vector<char> res; EVBufferToContiguousBuffer(decompressed, &res); for (size_t idx = 0; idx < raw_data_length_; ++idx) { ASSERT_TRUE(raw_data_[idx] == res[idx]); } } TEST_F(DataCompressorTest, DecompressGzipBuffer) { // Convert the compressed data into evbuffer format auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; ASSERT_EQ( evbuffer_add( source, gzip_compressed_data_.get(), gzip_compressed_length_), 0) << "Failed to initialize source evbuffer"; auto decompressed = evbuffer_new(); ASSERT_TRUE((decompressed != nullptr)) << "Failed to create decompressed evbuffer"; auto err = ni::DataCompressor::DecompressData( ni::DataCompressor::Type::GZIP, source, decompressed); ASSERT_TRUE((err == nullptr)) << "Failed to decompress data: " << TRITONSERVER_ErrorMessage(err); size_t destination_byte_size = evbuffer_get_length(decompressed); ASSERT_EQ(destination_byte_size, raw_data_length_) << "Mismatched byte size"; std::vector<char> res; EVBufferToContiguousBuffer(decompressed, &res); for (size_t idx = 0; idx < raw_data_length_; ++idx) { ASSERT_TRUE(raw_data_[idx] == res[idx]); } } TEST_F(DataCompressorTest, CompressDeflateBuffer) { // Convert the raw data into evbuffer format auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; ASSERT_EQ(evbuffer_add(source, raw_data_.get(), raw_data_length_), 0) << "Failed to initialize source evbuffer"; auto compressed = evbuffer_new(); ASSERT_TRUE((compressed != nullptr)) << "Failed to create compressed evbuffer"; auto err = ni::DataCompressor::CompressData( ni::DataCompressor::Type::DEFLATE, source, compressed); ASSERT_TRUE((err == nullptr)) << "Failed to compress data: " << TRITONSERVER_ErrorMessage(err); // Write compressed data to file which will be validated by other compression // tool WriteEVBufferToFile("generated_deflate_compressed_data", compressed); } TEST_F(DataCompressorTest, CompressGzipBuffer) { // Convert the raw data into evbuffer format auto source = evbuffer_new(); ASSERT_TRUE((source != nullptr)) << "Failed to create source evbuffer"; ASSERT_EQ(evbuffer_add(source, raw_data_.get(), raw_data_length_), 0) << "Failed to initialize source evbuffer"; auto compressed = evbuffer_new(); ASSERT_TRUE((compressed != nullptr)) << "Failed to create compressed evbuffer"; auto err = ni::DataCompressor::CompressData( ni::DataCompressor::Type::GZIP, source, compressed); ASSERT_TRUE((err == nullptr)) << "Failed to compress data: " << TRITONSERVER_ErrorMessage(err); // Write compressed data to file which will be validated by other compression // tool WriteEVBufferToFile("generated_gzip_compressed_data", compressed); } } // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/distributed_addsub/������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021247�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/distributed_addsub/CMakeLists.txt����������������������������������������0000664�0000000�0000000�00000010123�15270523712�0024004�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritondistributedaddsubbackend LANGUAGES C CXX) # # libtriton_distributed_addsub.so # Shared library implementing the Triton Distributed Addsub Backend API # configure_file(src/libtriton_distributed_addsub.ldscript libtriton_distributed_addsub.ldscript COPYONLY) add_library( triton-distributed-addsub-backend SHARED src/distributed_addsub.cc ) add_library( TritonDistributedAddsubBackend::triton-distributed-addsub-backend ALIAS triton-distributed-addsub-backend ) target_compile_features(triton-distributed-addsub-backend PRIVATE cxx_std_11) target_compile_options( triton-distributed-addsub-backend PRIVATE $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> ) target_link_libraries( triton-distributed-addsub-backend PRIVATE triton-backend-utils # from repo-backend triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core ) set_target_properties( triton-distributed-addsub-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_distributed_addsub LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_distributed_addsub.ldscript LINK_FLAGS "-Wl,--version-script libtriton_distributed_addsub.ldscript" ) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonDistributedAddsubBackend) install( TARGETS triton-distributed-addsub-backend EXPORT triton-distributed-addsub-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/distributed_addsub ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/distributed_addsub ) install( EXPORT triton-distributed-addsub-backend-targets FILE TritonDistributedAddsubBackendTargets.cmake NAMESPACE TritonDistributedAddsubBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonDistributedAddsubBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonDistributedAddsubBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonDistributedAddsubBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-distributed-addsub-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonDistributedAddsubBackendTargets.cmake NAMESPACE TritonDistributedAddsubBackend:: ) export(PACKAGE TritonDistributedAddsubBackend) ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/distributed_addsub/cmake/������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022327�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/distributed_addsub/cmake/TritonDistributedAddsubBackendConfig.cmake.in���0000664�0000000�0000000�00000004004�15270523712�0033137�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONDISTRIBUTEDADDSUBBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONDISTRIBUTEDADDSUBBACKEND_CMAKE_DIR}) if(NOT TARGET TritonDistributedAddsubBackend::triton-distributed-addsub-backend) include("${TRITONDISTRIBUTEDADDSUBBACKEND_CMAKE_DIR}/TritonDistributedAddsubBackendTargets.cmake") endif() set(TRITONDISTRIBUTEDADDSUBBACKEND_LIBRARIES TritonDistributedAddsubBackend::triton-distributed-addsub-backend) ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/distributed_addsub/src/��������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022036�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/distributed_addsub/src/distributed_addsub.cc�����������������������������0000664�0000000�0000000�00000071636�15270523712�0026226�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <atomic> #include <memory> #include <thread> #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" namespace triton { namespace backend { namespace distributed_addsub { // Addsub backend that distributes partial computation to different model // instances and gather the results to form the response internally. // This backend is designed in the way that CPU instance will perform add task // and GPU instance will perform sub task, and only the CPU instances will // accept inference request from Triton core. Note that GPU instance has // different meaning in this backend. // // The backend supports models that take two input tensors, two variable-size // INT32 [ -1 ] value inputs INPUT0 and INPUT1; and produces two output tensors: // OUTPUT0 as the element-wise sum of INPUT0 and INPUT1, OUTPUT1 as // the element-wise difference of INPUT0 and INPUT1 // #define GUARDED_RESPOND_IF_ERROR(RESPONSES, IDX, X) \ do { \ if ((RESPONSES)[IDX] != nullptr) { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (RESPONSES)[IDX], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ err__), \ "failed to send error response"); \ (RESPONSES)[IDX] = nullptr; \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. // class ModelInstanceState; class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; // Validate that model configuration is supported by this backend. TRITONSERVER_Error* ValidateModelConfig(); // Keep track of the model instance that will only accept works distributed // from within the model (instance) void AddSubInstance(ModelInstanceState* instance) { instance_ = instance; } ModelInstanceState* SubInstance() { return instance_; } std::atomic<size_t> instance_counter_; private: ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model), instance_counter_(0), instance_(nullptr) { } ModelInstanceState* instance_; }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } TRITONSERVER_Error* ModelState::ValidateModelConfig() { // We have the json DOM for the model configuration... common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(model_config_.PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model configuration:\n") + buffer.Contents()).c_str()); common::TritonJson::Value inputs, outputs; RETURN_IF_ERROR(model_config_.MemberAsArray("input", &inputs)); RETURN_IF_ERROR(model_config_.MemberAsArray("output", &outputs)); // There must be one INT32 input called INPUT defined in the model // configuration and it must be a 1D vector (of any length). RETURN_ERROR_IF_FALSE( inputs.ArraySize() == 2, TRITONSERVER_ERROR_INVALID_ARG, std::string("model must have two inputs")); RETURN_ERROR_IF_FALSE( outputs.ArraySize() == 2, TRITONSERVER_ERROR_INVALID_ARG, std::string("model must have two outputs")); int64_t dim_value = 0; for (size_t idx = 0; idx < 2; ++idx) { common::TritonJson::Value input; RETURN_IF_ERROR(inputs.IndexAsObject(idx, &input)); std::vector<int64_t> input_shape; RETURN_IF_ERROR(backend::ParseShape(input, "dims", &input_shape)); RETURN_ERROR_IF_FALSE( input_shape.size() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string("model must have input of one-dimensional shape")); if (idx == 0) { dim_value = input_shape[0]; } else { RETURN_ERROR_IF_FALSE( dim_value == input_shape[0], TRITONSERVER_ERROR_INVALID_ARG, std::string("model must have consistent shape for all tensors")); } std::string input_dtype; RETURN_IF_ERROR(input.MemberAsString("data_type", &input_dtype)); RETURN_ERROR_IF_FALSE( input_dtype == "TYPE_INT32", TRITONSERVER_ERROR_INVALID_ARG, std::string("model input must have TYPE_INT32 data-type")); const char* input_name; size_t input_name_len; RETURN_IF_ERROR(input.MemberAsString("name", &input_name, &input_name_len)); auto expected_name = (std::string("INPUT") + std::to_string(idx)); RETURN_ERROR_IF_FALSE( expected_name == input_name, TRITONSERVER_ERROR_INVALID_ARG, std::string("model input must be named '") + expected_name + "' at index " + std::to_string(idx)); common::TritonJson::Value output; RETURN_IF_ERROR(outputs.IndexAsObject(idx, &output)); std::vector<int64_t> output_shape; RETURN_IF_ERROR(backend::ParseShape(output, "dims", &output_shape)); RETURN_ERROR_IF_FALSE( (output_shape.size() == 1) && (output_shape[0] == input_shape[0]), TRITONSERVER_ERROR_INVALID_ARG, std::string("model must have consistent shape for all tensors")); std::string output_dtype; RETURN_IF_ERROR(output.MemberAsString("data_type", &output_dtype)); RETURN_ERROR_IF_FALSE( output_dtype == "TYPE_INT32", TRITONSERVER_ERROR_INVALID_ARG, std::string("model output must have TYPE_INT32 data-type")); const char* output_name; size_t output_name_len; RETURN_IF_ERROR( output.MemberAsString("name", &output_name, &output_name_len)); expected_name = (std::string("OUTPUT") + std::to_string(idx)); RETURN_ERROR_IF_FALSE( expected_name == output_name, TRITONSERVER_ERROR_INVALID_ARG, std::string("model output must be named '") + expected_name + "' at index " + std::to_string(idx)); } return nullptr; // success } // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each TRITONBACKEND_ModelInstance. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState() = default; // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } bool IsPassive() const { return passive_; } TRITONSERVER_Error* Add( const size_t element_count, const int32_t* input_0, const int32_t* input_1, int32_t* output); TRITONSERVER_Error* Sub( const size_t element_count, const int32_t* input_0, const int32_t* input_1, int32_t* output); private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance); ModelState* model_state_; bool passive_; }; ModelInstanceState::ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state) { // Check if the setup is correct THROW_IF_BACKEND_INSTANCE_ERROR( TRITONBACKEND_ModelInstanceIsPassive(triton_model_instance, &passive_)); switch (kind_) { case TRITONSERVER_INSTANCEGROUPKIND_CPU: { if (passive_) { throw BackendModelInstanceException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("CPU instance should not be passive").c_str())); } break; } case TRITONSERVER_INSTANCEGROUPKIND_GPU: if (!passive_) { throw BackendModelInstanceException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("GPU instance should be passive").c_str())); } break; default: throw BackendModelInstanceException(TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, std::string("instance kind must be CPU or GPU").c_str())); break; } } TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } TRITONSERVER_Error* ModelInstanceState::Add( const size_t element_count, const int32_t* input_0, const int32_t* input_1, int32_t* output) { if (kind_ != TRITONSERVER_INSTANCEGROUPKIND_CPU) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string("Add operation must be done by CPU instance").c_str()); } for (size_t i = 0; i < element_count; ++i) { output[i] = input_0[i] + input_1[i]; } return nullptr; // success } TRITONSERVER_Error* ModelInstanceState::Sub( const size_t element_count, const int32_t* input_0, const int32_t* input_1, int32_t* output) { if (kind_ != TRITONSERVER_INSTANCEGROUPKIND_GPU) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string("Sub operation must be done by GPU instance").c_str()); } for (size_t i = 0; i < element_count; ++i) { output[i] = input_0[i] - input_1[i]; } return nullptr; // success } ///////////// extern "C" { // Implementing TRITONBACKEND_Initialize is optional. The backend // should initialize any global state that is intended to be shared // across all models and model instances that use the backend. TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); // We should check the backend API version that Triton supports // vs. what this backend was compiled against. uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Triton TRITONBACKEND API version: ") + std::to_string(api_version_major) + "." + std::to_string(api_version_minor)) .c_str()); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("'") + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "triton backend API version does not support this backend"); } // The backend configuration may contain information needed by the // backend, such a command-line arguments. This backend doesn't use // any such configuration but we print whatever is available. TRITONSERVER_Message* backend_config_message; RETURN_IF_ERROR( TRITONBACKEND_BackendConfig(backend, &backend_config_message)); const char* buffer; size_t byte_size; RETURN_IF_ERROR(TRITONSERVER_MessageSerializeToJson( backend_config_message, &buffer, &byte_size)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("backend configuration:\n") + buffer).c_str()); return nullptr; // success } // Implementing TRITONBACKEND_ModelInitialize is optional. The backend // should initialize any state that is intended to be shared across // all instances of the model. TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); // With each model we create a ModelState object and associate it // with the TRITONBACKEND_Model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast<void*>(model_state))); // One of the primary things to do in ModelInitialize is to examine // the model configuration to ensure that it is something that this // backend can support. If not, returning an error from this // function will prevent the model from loading. RETURN_IF_ERROR(model_state->ValidateModelConfig()); return nullptr; // success } // Implementing TRITONBACKEND_ModelFinalize is optional unless state // is set using TRITONBACKEND_ModelSetState. The backend must free // this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vstate); TRITONSERVER_Error* err = nullptr; if (model_state->instance_counter_ != 0) { err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Unexpected unfinalized model instance(s)"); } LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return err; } // Implementing TRITONBACKEND_ModelInstanceInitialize is optional. The // backend should initialize any state that is required for a model // instance. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); // The instance can access the corresponding model as well... here // we get the model and from that get the model's state. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vmodelstate); // With each instance we create a ModelInstanceState object and // associate it with the TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast<void*>(instance_state))); if (instance_state->IsPassive()) { model_state->AddSubInstance(instance_state); } return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceFinalize is optional unless // state is set using TRITONBACKEND_ModelInstanceSetState. The backend // must free this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast<ModelInstanceState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceExecute is required. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Suggested practice for this is to use only // function-local and model-instance-specific state (obtained from // 'instance'), which is what we do here. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast<void**>(&instance_state))); ModelState* model_state = instance_state->StateForModel(); // This backend specifies BLOCKING execution policy. That means that // we should not return from this function until execution is complete. Triton // will automatically release 'instance' on return from this function so that // it is again available to be used for another call to // TRITONBACKEND_ModelInstanceExecute. LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model ") + model_state->Name() + ", instance " + instance_state->Name() + " (" + TRITONSERVER_InstanceGroupKindString(instance_state->Kind()) + " device " + std::to_string(instance_state->DeviceId()) + ")" + ", executing " + std::to_string(request_count) + " requests") .c_str()); if (instance_state->Kind() != TRITONSERVER_INSTANCEGROUPKIND_CPU) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "Unexpected inference request sent to non-CPU instance"); } auto sub_instance_state = model_state->SubInstance(); // 'responses' is initialized with the response objects below and // if/when an error response is sent the corresponding entry in // 'responses' is set to nullptr to indicate that that response has // already been sent. std::vector<TRITONBACKEND_Response*> responses; responses.reserve(request_count); // Create a single response object for each request. If something // goes wrong when attempting to create the response objects just // fail all of the requests by returning an error. for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); responses.push_back(response); } // After this point we take ownership of 'requests', which means that a // response must be sent for every request. If something does go wrong in // processing a particular request then we send an error response just for the // specific request. // The way we collect these batch timestamps is not entirely accurate. // Normally, in a performant backend you would execute all the requests at the // same time, and so there would be a single compute-start / compute-end // time-range. But here we execute each request separately so there is no // single range. As a result we just show the entire execute time as being the // compute time as well. uint64_t batch_exec_start_ns = 0; SET_TIMESTAMP(batch_exec_start_ns); uint64_t batch_exec_end_ns = 0; uint64_t total_batch_size = 0; // For simplicity we just process each request separately... in // general a backend should try to operate on the entire batch of // requests at the same time for improved performance. std::vector<uint8_t> start_buffer, ready_buffer, input_buffer; for (uint32_t r = 0; r < request_count; ++r) { uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); TRITONBACKEND_Request* request = requests[r]; uint32_t input_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInputCount(request, &input_count)); uint32_t requested_output_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestOutputCount(request, &requested_output_count)); // If an error response was sent for the above then display an error // message and move on to next request. if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read request input/output counts, error response " "sent") .c_str()); continue; } TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "INPUT0", &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input, error response sent") .c_str()); continue; } TRITONSERVER_DataType input_datatype; const int64_t* input_shape; uint32_t input_dims_count; uint64_t input_byte_size; uint32_t input_buffer_count; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputProperties( input, nullptr /* input_name */, &input_datatype, &input_shape, &input_dims_count, &input_byte_size, &input_buffer_count)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input properties, error response sent") .c_str()); continue; } if (input_dims_count > 1) { total_batch_size += input_shape[0]; } else { ++total_batch_size; } std::vector<char> input_0(input_byte_size); std::vector<char> input_1(input_byte_size); uint64_t input_0_byte_size = input_byte_size; uint64_t input_1_byte_size = input_byte_size; GUARDED_RESPOND_IF_ERROR( responses, r, ReadInputTensor( request, "INPUT0", input_0.data(), reinterpret_cast<size_t*>(&input_0_byte_size))); GUARDED_RESPOND_IF_ERROR( responses, r, ReadInputTensor( request, "INPUT1", input_1.data(), reinterpret_cast<size_t*>(&input_1_byte_size))); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer in CPU memory, error " "response sent") .c_str()); continue; } // Compute... Get GPU instance from model state and let it compute // the subtraction, while the CPU instance computes the addition. // In real world some parallelization should be used, but here just // serialize the "distributed" work. TRITONBACKEND_Response* response = responses[r]; uint64_t compute_start_ns = 0; SET_TIMESTAMP(compute_start_ns); for (size_t out_idx = 0; out_idx < requested_output_count; ++out_idx) { const char* output_name; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestOutputName(request, out_idx, &output_name)); TRITONBACKEND_Output* output; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_ResponseOutput( response, &output, output_name, input_datatype, input_shape, input_dims_count)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create response output, error response sent") .c_str()); break; } // Get the output buffer. We request a buffer in CPU memory but we have // to handle any returned type. If we get back a buffer in GPU memory we // just fail the request. void* output_buffer; TRITONSERVER_MemoryType output_memory_type = TRITONSERVER_MEMORY_CPU; int64_t output_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_OutputBuffer( output, &output_buffer, input_byte_size, &output_memory_type, &output_memory_type_id)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create output buffer in CPU memory, error " "response sent") .c_str()); break; } if (output_memory_type == TRITONSERVER_MEMORY_GPU) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create output buffer in CPU memory")); break; } static std::string output_0_name("OUTPUT0"); if (output_0_name == output_name) { instance_state->Add( (input_byte_size / sizeof(int32_t)), reinterpret_cast<int32_t*>(input_0.data()), reinterpret_cast<int32_t*>(input_1.data()), reinterpret_cast<int32_t*>(output_buffer)); } else { sub_instance_state->Sub( (input_byte_size / sizeof(int32_t)), reinterpret_cast<int32_t*>(input_0.data()), reinterpret_cast<int32_t*>(input_1.data()), reinterpret_cast<int32_t*>(output_buffer)); } } uint64_t compute_end_ns = 0; SET_TIMESTAMP(compute_end_ns); uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); batch_exec_end_ns = exec_end_ns; // Send all the responses that haven't already been sent because of an // earlier error. if (responses[r] != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( responses[r], TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr /* success */), "failed sending response"); } // Report statistics for each request. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_state->TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns), "failed reporting request statistics"); LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } // Report the entire batch statistics. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( instance_state->TritonModelInstance(), total_batch_size, batch_exec_start_ns, batch_exec_start_ns, batch_exec_end_ns, batch_exec_end_ns), "failed reporting batch request statistics"); return nullptr; // success } } // extern "C" }}} // namespace triton::backend::distributed_addsub ��������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/distributed_addsub/src/libtriton_distributed_addsub.ldscript�������������0000664�0000000�0000000�00000003055�15270523712�0031541�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/dyna_sequence/�����������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020226�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/dyna_sequence/CMakeLists.txt���������������������������������������������0000664�0000000�0000000�00000007656�15270523712�0023004�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritondynasequencebackend LANGUAGES C CXX) # # libtriton_dyna_sequence.so # Shared library implementing the Triton Sequence Backend API # configure_file(src/libtriton_dyna_sequence.ldscript libtriton_dyna_sequence.ldscript COPYONLY) add_library( triton-dyna-sequence-backend SHARED src/dyna_sequence.cc ) add_library( TritonDynaSequenceBackend::triton-dyna-sequence-backend ALIAS triton-dyna-sequence-backend ) target_compile_features(triton-dyna-sequence-backend PRIVATE cxx_std_11) target_compile_options( triton-dyna-sequence-backend PRIVATE $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> ) target_link_libraries( triton-dyna-sequence-backend PRIVATE triton-backend-utils # from repo-backend triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core ) set_target_properties( triton-dyna-sequence-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_dyna_sequence LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_dyna_sequence.ldscript LINK_FLAGS "-Wl,--version-script libtriton_dyna_sequence.ldscript" ) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonDynaSequenceBackend) install( TARGETS triton-dyna-sequence-backend EXPORT triton-dyna-sequence-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/dyna_sequence ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/dyna_sequence ) install( EXPORT triton-dyna-sequence-backend-targets FILE TritonDynaSequenceBackendTargets.cmake NAMESPACE TritonDynaSequenceBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonDynaSequenceBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonDynaSequenceBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonDynaSequenceBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-dyna-sequence-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonDynaSequenceBackendTargets.cmake NAMESPACE TritonDynaSequenceBackend:: ) export(PACKAGE TritonDynaSequenceBackend) ����������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/dyna_sequence/cmake/�����������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021306�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/dyna_sequence/cmake/TritonDynaSequenceBackendConfig.cmake.in�������������0000664�0000000�0000000�00000003674�15270523712�0031111�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONSEQUENCEBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONSEQUENCEBACKEND_CMAKE_DIR}) if(NOT TARGET TritonDynaSequenceBackend::triton-sequence-backend) include("${TRITONSEQUENCEBACKEND_CMAKE_DIR}/TritonDynaSequenceBackendTargets.cmake") endif() set(TRITONSEQUENCEBACKEND_LIBRARIES TritonDynaSequenceBackend::triton-sequence-backend)��������������������������������������������������������������������kytensor-2.49.0.7/src/test/dyna_sequence/src/�������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021015�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/dyna_sequence/src/dyna_sequence.cc���������������������������������������0000664�0000000�0000000�00000125220�15270523712�0024151�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <algorithm> #include <memory> #include <thread> #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" namespace triton { namespace backend { namespace dyna_sequence { // Simple dynamic sequence backend that demonstrates the TRITONBACKEND API for a // blocking backend. A blocking backend completes execution of the // inference before returning from TRITONBACKEND_ModelInstanceExecute. // // The backend supports models that take 5 input tensors, three INT32 [ 1 ] // control values, one UINT64 [ 1 ] correlation ID control, and one // variable-size INT32 [ -1 ] value input; and produces an output // tensor with the same shape as the input tensor. The input tensors // must be named "START", "END", "READY", "CORRID" and "INPUT". The // output tensor must be named "OUTPUT". // // The model maintains an INT32 accumulator for each sequence which // is updated based on the control values in "START", "END", "READY" // and "CORRID": // // READY=0, START=x, END=x: Ignore value input, do not change // accumulator value. // // READY=1, START=1, END=x: Start accumulating. Set accumulator // equal to sum of INPUT tensor elements. // // READY=1, START=0, END=x: Add INPUT tensor elements to // accumulator. // // In addition to the above, when END=1 CORRID is added to the accumulator. // // When READY=1, the accumulator is returned in every element of the // OUTPUT tensor. // #define GUARDED_RESPOND_IF_ERROR(RESPONSES, IDX, X) \ do { \ if ((RESPONSES)[IDX] != nullptr) { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (RESPONSES)[IDX], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ err__), \ "failed to send error response"); \ (RESPONSES)[IDX] = nullptr; \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; // Get accumulator size and execution delay size_t AccumulatorSize() const { return accumulator_size_; } int ExecDelay() const { return execute_delay_ms_; } const std::string& CorrelationIdType() const { return corrid_dtype_; } // Validate that model configuration is supported by this backend. TRITONSERVER_Error* ValidateModelConfig(); private: ModelState(TRITONBACKEND_Model* triton_model); // Delay to introduce into execution, in milliseconds. int execute_delay_ms_; // Accumulator size size_t accumulator_size_; // Correlation id type std::string corrid_dtype_; }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelState::ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model), execute_delay_ms_(0), accumulator_size_(0), corrid_dtype_("TYPE_UINT64") { } TRITONSERVER_Error* ModelState::ValidateModelConfig() { // We have the json DOM for the model configuration... common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(model_config_.PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model configuration:\n") + buffer.Contents()).c_str()); triton::common::TritonJson::Value params; if (model_config_.Find("parameters", ¶ms)) { common::TritonJson::Value exec_delay; if (params.Find("execute_delay_ms", &exec_delay)) { std::string exec_delay_str; RETURN_IF_ERROR( exec_delay.MemberAsString("string_value", &exec_delay_str)); execute_delay_ms_ = std::stoi(exec_delay_str); } } int64_t max_batch_size = 0; RETURN_IF_ERROR(model_config_.MemberAsInt("max_batch_size", &max_batch_size)); accumulator_size_ = (size_t)(std::max((int64_t)1, max_batch_size)); // The model configuration must specify the sequence batcher and // must use the START, END, READY and CORRID input to indicate // control values. triton::common::TritonJson::Value sequence_batching; RETURN_IF_ERROR( model_config_.MemberAsObject("sequence_batching", &sequence_batching)); common::TritonJson::Value control_inputs; RETURN_IF_ERROR( sequence_batching.MemberAsArray("control_input", &control_inputs)); RETURN_ERROR_IF_FALSE( control_inputs.ArraySize() == 4, TRITONSERVER_ERROR_INVALID_ARG, std::string("'START', 'END, 'READY' and 'CORRID' must be configured as " "the control inputs")); std::vector<std::string> control_input_names; for (size_t io_index = 0; io_index < control_inputs.ArraySize(); io_index++) { common::TritonJson::Value control_input; RETURN_IF_ERROR(control_inputs.IndexAsObject(io_index, &control_input)); const char* input_name; size_t input_name_len; RETURN_IF_ERROR( control_input.MemberAsString("name", &input_name, &input_name_len)); control_input_names.push_back(input_name); } RETURN_ERROR_IF_FALSE( (std::find( control_input_names.begin(), control_input_names.end(), "START") != control_input_names.end()) || (std::find( control_input_names.begin(), control_input_names.end(), "END") != control_input_names.end()) || (std::find( control_input_names.begin(), control_input_names.end(), "READY") != control_input_names.end()) || (std::find( control_input_names.begin(), control_input_names.end(), "CORRID") != control_input_names.end()), TRITONSERVER_ERROR_INVALID_ARG, std::string("'START', 'END, 'READY' and 'CORRID' must be configured as " "the control inputs")); // The CORRID input must be UINT64 type. auto itr = std::find( control_input_names.begin(), control_input_names.end(), "CORRID"); size_t corrid_pos = std::distance(control_input_names.begin(), itr); triton::common::TritonJson::Value corrid_input; RETURN_IF_ERROR(control_inputs.IndexAsObject(corrid_pos, &corrid_input)); triton::common::TritonJson::Value corrid_control; RETURN_IF_ERROR(corrid_input.MemberAsArray("control", &corrid_control)); common::TritonJson::Value control_item; RETURN_IF_ERROR(corrid_control.IndexAsObject(0 /* index */, &control_item)); std::string corrid_dtype; RETURN_IF_ERROR(control_item.MemberAsString("data_type", &corrid_dtype)); RETURN_ERROR_IF_FALSE( ((corrid_dtype == "TYPE_UINT64") || (corrid_dtype == "TYPE_STRING")), TRITONSERVER_ERROR_INVALID_ARG, std::string("model CORRID control input must have TYPE_UINT64 " "or TYPE_STRING data-type")); corrid_dtype_ = corrid_dtype; common::TritonJson::Value inputs, outputs; RETURN_IF_ERROR(model_config_.MemberAsArray("input", &inputs)); RETURN_IF_ERROR(model_config_.MemberAsArray("output", &outputs)); // There must be one INT32 input called INPUT defined in the model // configuration and it must be a 1D vector (of any length). RETURN_ERROR_IF_FALSE( inputs.ArraySize() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string( "model must have input 'INPUT' with vector shape, any length")); common::TritonJson::Value input; RETURN_IF_ERROR(inputs.IndexAsObject(0 /* index */, &input)); std::vector<int64_t> input_shape; RETURN_IF_ERROR(backend::ParseShape(input, "dims", &input_shape)); RETURN_ERROR_IF_FALSE( input_shape.size() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string( "model must have one input 'INPUT' with vector shape, any length")); std::string input_dtype; RETURN_IF_ERROR(input.MemberAsString("data_type", &input_dtype)); RETURN_ERROR_IF_FALSE( input_dtype == "TYPE_INT32", TRITONSERVER_ERROR_INVALID_ARG, std::string("model input must have TYPE_INT32 data-type")); const char* input_name; size_t input_name_len; RETURN_IF_ERROR(input.MemberAsString("name", &input_name, &input_name_len)); RETURN_ERROR_IF_FALSE( strcmp(input_name, "INPUT") == 0, TRITONSERVER_ERROR_INVALID_ARG, std::string("model input must be named 'INPUT'")); // There must be one INT32 output with shape that matches the // input. The output must be named OUTPUT. RETURN_ERROR_IF_FALSE( outputs.ArraySize() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string( "model must have one output 'OUTPUT' with vector shape, any length")); common::TritonJson::Value output; RETURN_IF_ERROR(outputs.IndexAsObject(0 /* index */, &output)); std::vector<int64_t> output_shape; RETURN_IF_ERROR(backend::ParseShape(output, "dims", &output_shape)); RETURN_ERROR_IF_FALSE( (output_shape.size() == 1) && (output_shape[0] == input_shape[0]), TRITONSERVER_ERROR_INVALID_ARG, std::string( "model must have output 'OUTPUT' with shape matching 'INPUT'")); std::string output_dtype; RETURN_IF_ERROR(output.MemberAsString("data_type", &output_dtype)); RETURN_ERROR_IF_FALSE( output_dtype == "TYPE_INT32", TRITONSERVER_ERROR_INVALID_ARG, std::string("model output must have TYPE_INT32 data-type")); const char* output_name; size_t output_name_len; RETURN_IF_ERROR( output.MemberAsString("name", &output_name, &output_name_len)); RETURN_ERROR_IF_FALSE( strcmp(output_name, "OUTPUT") == 0, TRITONSERVER_ERROR_INVALID_ARG, std::string("model output must be named 'OUTPUT'")); return nullptr; // success } // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each TRITONBACKEND_ModelInstance. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState(); // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } // Modify/get accumulator values for this instance int32_t GetAccumulatorVal(uint64_t corrid); void SetAccumulatorVal(uint64_t corrid, int32_t value); void AddAccumulatorVal(uint64_t corrid, int32_t value); void EraseAccumulatorKey(uint64_t corrid); private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance); ModelState* model_state_; // Accumulators maintained by this context, as a map from // correlation ID to the accumulator. std::unordered_map<uint64_t, int32_t> accumulator_; }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelInstanceState::ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state) { } int32_t ModelInstanceState::GetAccumulatorVal(uint64_t corrid) { return accumulator_[corrid]; } void ModelInstanceState::SetAccumulatorVal(uint64_t corrid, int32_t value) { accumulator_[corrid] = value; } void ModelInstanceState::AddAccumulatorVal(uint64_t corrid, int32_t value) { accumulator_[corrid] += value; } void ModelInstanceState::EraseAccumulatorKey(uint64_t corrid) { accumulator_.erase(corrid); } ModelInstanceState::~ModelInstanceState() { accumulator_.clear(); } ///////////// extern "C" { // Implementing TRITONBACKEND_Initialize is optional. The backend // should initialize any global state that is intended to be shared // across all models and model instances that use the backend. TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); // We should check the backend API version that Triton supports // vs. what this backend was compiled against. uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Triton TRITONBACKEND API version: ") + std::to_string(api_version_major) + "." + std::to_string(api_version_minor)) .c_str()); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("'") + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "triton backend API version does not support this backend"); } // The backend configuration may contain information needed by the // backend, such a command-line arguments. This backend doesn't use // any such configuration but we print whatever is available. TRITONSERVER_Message* backend_config_message; RETURN_IF_ERROR( TRITONBACKEND_BackendConfig(backend, &backend_config_message)); const char* buffer; size_t byte_size; RETURN_IF_ERROR(TRITONSERVER_MessageSerializeToJson( backend_config_message, &buffer, &byte_size)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("backend configuration:\n") + buffer).c_str()); return nullptr; // success } // Implementing TRITONBACKEND_ModelInitialize is optional. The backend // should initialize any state that is intended to be shared across // all instances of the model. TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); // With each model we create a ModelState object and associate it // with the TRITONBACKEND_Model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast<void*>(model_state))); // One of the primary things to do in ModelInitialize is to examine // the model configuration to ensure that it is something that this // backend can support. If not, returning an error from this // function will prevent the model from loading. RETURN_IF_ERROR(model_state->ValidateModelConfig()); return nullptr; // success } // Implementing TRITONBACKEND_ModelFinalize is optional unless state // is set using TRITONBACKEND_ModelSetState. The backend must free // this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceInitialize is optional. The // backend should initialize any state that is required for a model // instance. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); // The instance can access the corresponding model as well... here // we get the model and from that get the model's state. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vmodelstate); // With each instance we create a ModelInstanceState object and // associate it with the TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast<void*>(instance_state))); // Because this backend just copies IN -> OUT and requires that // input and output be in CPU memory, we fail if a GPU instances is // requested. RETURN_ERROR_IF_FALSE( instance_state->Kind() == TRITONSERVER_INSTANCEGROUPKIND_CPU, TRITONSERVER_ERROR_INVALID_ARG, std::string("'dyna_sequence' backend only supports CPU instances")); return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceFinalize is optional unless // state is set using TRITONBACKEND_ModelInstanceSetState. The backend // must free this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast<ModelInstanceState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceExecute is required. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Suggested practice for this is to use only // function-local and model-instance-specific state (obtained from // 'instance'), which is what we do here. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast<void**>(&instance_state))); ModelState* model_state = instance_state->StateForModel(); // This backend specifies BLOCKING execution policy. That means that // we should not return from this function until execution is // complete. Triton will automatically release 'instance' on return // from this function so that it is again available to be used for // another call to TRITONBACKEND_ModelInstanceExecute. LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model ") + model_state->Name() + ", instance " + instance_state->Name() + ", executing " + std::to_string(request_count) + " requests") .c_str()); bool supports_batching = false; RETURN_IF_ERROR(model_state->SupportsFirstDimBatching(&supports_batching)); // Each request represents a different sequence, which corresponds // to the accumulator at the same index. Each request must have // batch-size 1 inputs which is the next timestep for that // sequence. The total number of requests will not exceed the // max-batch-size specified in the model configuration. if (request_count > model_state->AccumulatorSize()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "unable to execute batch larger than max-batch-size"); } // Delay if requested... if (model_state->ExecDelay() > 0) { std::this_thread::sleep_for( std::chrono::milliseconds(model_state->ExecDelay())); } // 'responses' is initialized with the response objects below and // if/when an error response is sent the corresponding entry in // 'responses' is set to nullptr to indicate that that response has // already been sent. std::vector<TRITONBACKEND_Response*> responses; responses.reserve(request_count); // Create a single response object for each request. If something // goes wrong when attempting to create the response objects just // fail all of the requests by returning an error. for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); responses.push_back(response); } // The way we collect these batch timestamps is not entirely // accurate. Normally, in a performant backend you would execute all // the requests at the same time, and so there would be a single // compute-start / compute-end time-range. But here we execute each // request separately so there is no single range. As a result we // just show the entire execute time as being the compute time as // well. uint64_t min_exec_start_ns = std::numeric_limits<uint64_t>::max(); uint64_t max_exec_end_ns = 0; uint64_t total_batch_size = 0; // After this point we take ownership of 'requests', which means // that a response must be sent for every request. If something does // go wrong in processing a particular request then we send an error // response just for the specific request. // For simplicity we just process each request separately... in // general a backend should try to operate on the entire batch of // requests at the same time for improved performance. std::vector<uint8_t> start_buffer, end_buffer, ready_buffer, corrid_buffer, input_buffer; for (uint32_t r = 0; r < request_count; ++r) { uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); min_exec_start_ns = std::min(min_exec_start_ns, exec_start_ns); TRITONBACKEND_Request* request = requests[r]; const char* request_id = ""; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestId(request, &request_id)); uint64_t correlation_id = 0; if (model_state->CorrelationIdType() == "TYPE_UINT64") { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestCorrelationId(request, &correlation_id)); } else if (model_state->CorrelationIdType() == "TYPE_STRING") { const char* correlation_id_str; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestCorrelationIdString( request, &correlation_id_str)); // Require that the string be decodable into an unsigned int. try { correlation_id = std::stoi(correlation_id_str); } catch (const std::invalid_argument& ia) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "dyna sequence backend expects correlation ID to be decodable " "into an integer")); } } uint32_t input_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInputCount(request, &input_count)); uint32_t requested_output_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestOutputCount(request, &requested_output_count)); // If an error response was sent for the above then display an error // message and move on to next request. if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read request input/output counts, error response " "sent") .c_str()); continue; } LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("request ") + std::to_string(r) + ": id = \"" + request_id + "\", correlation_id = " + std::to_string(correlation_id) + ", input_count = " + std::to_string(input_count) + ", requested_output_count = " + std::to_string(requested_output_count)) .c_str()); // For statistics we need to collect the total batch size of all the // requests. If the model doesn't support batching then each request is // necessarily batch-size 1. If the model does support batching then the // first dimension of the shape is the batch size. We only the first input // for this. if (supports_batching) { TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInputByIndex(request, 0 /* index */, &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input, error response sent") .c_str()); continue; } const int64_t* input_shape; uint32_t input_dims_count; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputProperties( input, nullptr, nullptr, &input_shape, &input_dims_count, nullptr, nullptr)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input properties, error response sent") .c_str()); continue; } if (input_dims_count > 0) { if (input_shape[0] != 1) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "unable to execute more than one timestep at a time")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": unable to execute more than one timestep at a time, error " "response sent") .c_str()); continue; } total_batch_size += input_shape[0]; } } else { total_batch_size++; } LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("total_batch_size: ") + std::to_string(total_batch_size)) .c_str()); std::set<uint64_t> seen_corrids; // Get the input tensors. TRITONBACKEND_Input* start_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "START", &start_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'START', error response sent") .c_str()); continue; } TRITONBACKEND_Input* end_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "END", &end_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'END', error response sent") .c_str()); continue; } TRITONBACKEND_Input* ready_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "READY", &ready_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'READY', error response sent") .c_str()); continue; } TRITONBACKEND_Input* corrid_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "CORRID", &corrid_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'CORRID', error response sent") .c_str()); continue; } const void* start_buffer = nullptr; uint64_t buffer_byte_size = 0; TRITONSERVER_MemoryType input_memory_type = TRITONSERVER_MEMORY_CPU; int64_t input_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( start_input, 0 /* input_buffer_count */, &start_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } const void* end_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( end_input, 0 /* input_buffer_count */, &end_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } const void* ready_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( ready_input, 0 /* input_buffer_count */, &ready_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } const void* corrid_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( corrid_input, 0 /* input_buffer_count */, &corrid_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "INPUT", &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'INPUT', error response sent") .c_str()); continue; } const void* input_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( input, 0 /* input_buffer_count */, &input_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if ((responses[r] == nullptr) || (input_memory_type == TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer in CPU memory, error " "response sent") .c_str()); continue; } TRITONSERVER_DataType input_datatype; const int64_t* input_shape; uint32_t input_dims_count; uint64_t input_byte_size; uint32_t input_buffer_count; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputProperties( input, nullptr /* input_name */, &input_datatype, &input_shape, &input_dims_count, &input_byte_size, &input_buffer_count)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input properties, error response sent") .c_str()); continue; } int64_t input_element_cnt = input_byte_size / sizeof(int32_t); const int32_t start = *reinterpret_cast<const int32_t*>(start_buffer); const int32_t end = *reinterpret_cast<const int32_t*>(end_buffer); const int32_t ready = *reinterpret_cast<const int32_t*>(ready_buffer); uint64_t corrid; if (model_state->CorrelationIdType() == "TYPE_STRING") { // interpret buffer as const char* where first 4 bytes are string length const char* corrid_p = reinterpret_cast<const char*>(corrid_buffer); const std::string corrid_str( corrid_p + sizeof(uint32_t), *((uint32_t*)corrid_p)); // String sequence ID must be decodable into int for dyna sequence backend try { corrid = std::stoi(corrid_str); } catch (const std::invalid_argument& ia) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "dyna sequence backend expects correlation ID to be decodable " "into an integer")); } } else { corrid = *reinterpret_cast<const uint64_t*>(corrid_buffer); } const int32_t* ipbuffer_int = reinterpret_cast<const int32_t*>(input_buffer); // Sequence batcher should never send us a batch of payloads where // a given correlation ID occurs more that once. Check that here // and fail if it happens. if (seen_corrids.find(corrid) != seen_corrids.end()) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Execute() called with batch containing multiple inferences " "requests for the same Correlation ID")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": Execute() called with batch containing " "multiple inferences requests for the same Correlation ID, error " "response sent") .c_str()); continue; } seen_corrids.insert(corrid); // Update the accumulator value based on START/END/READY/CORRID // and calculate the output value. if (ready != 0) { if (start == 0) { // Update accumulator. for (int64_t e = 0; e < input_element_cnt; ++e) { instance_state->AddAccumulatorVal(corrid, ipbuffer_int[e]); } } else { // Set accumulator. instance_state->SetAccumulatorVal(corrid, ipbuffer_int[0]); for (int64_t e = 1; e < input_element_cnt; ++e) { instance_state->AddAccumulatorVal(corrid, ipbuffer_int[e]); } } if (end != 0) { // Add CORRID (truncated to 32 bits) to accumulator. instance_state->AddAccumulatorVal(corrid, (int32_t)corrid); } const int32_t output_val = instance_state->GetAccumulatorVal(corrid); // If sequence has ended remove CORRID from the accumulator map. if (end != 0) { instance_state->EraseAccumulatorKey(corrid); } TRITONBACKEND_Response* response = responses[r]; // If the output is requested, copy the calculated output value // into the output buffer. if (requested_output_count > 0) { // The output shape is [1, input_element_cnt] if the model // configuration supports batching, or just // [input_element_cnt] if the model configuration does not // support batching. std::vector<int64_t> shape; if (supports_batching) { shape.push_back(1); } shape.push_back(input_element_cnt); TRITONBACKEND_Output* output; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_ResponseOutput( response, &output, "OUTPUT", input_datatype, input_shape, input_dims_count)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create response output, error response sent") .c_str()); continue; } // Step 2. Get the output buffer. We request a buffer in CPU // memory but we have to handle any returned type. If we get // back a buffer in GPU memory we just fail the request. void* output_buffer; TRITONSERVER_MemoryType output_memory_type = TRITONSERVER_MEMORY_CPU; int64_t output_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_OutputBuffer( output, &output_buffer, buffer_byte_size, &output_memory_type, &output_memory_type_id)); if ((responses[r] == nullptr) || (output_memory_type == TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create output buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create output buffer in CPU memory, error " "response sent") .c_str()); continue; } int32_t* obuffer_int = reinterpret_cast<int32_t*>(output_buffer); for (int64_t i = 0; i < input_element_cnt; ++i) { obuffer_int[i] = output_val; } } } uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); max_exec_end_ns = std::max(max_exec_end_ns, exec_end_ns); // Send all the responses that haven't already been sent because of // an earlier error. if (responses[r] != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( responses[r], TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr /* success */), "failed sending response"); } // Report statistics for each request. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_state->TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, exec_start_ns, exec_end_ns, exec_end_ns), "failed reporting request statistics"); LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } // Report the entire batch statistics. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( instance_state->TritonModelInstance(), total_batch_size, min_exec_start_ns, min_exec_start_ns, max_exec_end_ns, max_exec_end_ns), "failed reporting batch request statistics"); return nullptr; // success } } // extern "C" }}} // namespace triton::backend::dyna_sequence ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/dyna_sequence/src/libtriton_dyna_sequence.ldscript�����������������������0000664�0000000�0000000�00000003054�15270523712�0027476�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; };������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/implicit_state/����������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020415�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/implicit_state/CMakeLists.txt��������������������������������������������0000664�0000000�0000000�00000007724�15270523712�0023167�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonimplicitsequencebackend LANGUAGES C CXX) # # libtriton_implicit_state.so # Shared library implementing the Triton Implicit Sequence Backend API # configure_file(src/libtriton_implicit_state.ldscript libtriton_implicit_state.ldscript COPYONLY) add_library( triton-implicit-state-backend SHARED src/implicit_state.cc ) add_library( TritonImplicitStateBackend::triton-implicit-state-backend ALIAS triton-implicit-state-backend ) target_compile_features(triton-implicit-state-backend PRIVATE cxx_std_11) target_compile_options( triton-implicit-state-backend PRIVATE $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> ) target_link_libraries( triton-implicit-state-backend PRIVATE triton-backend-utils # from repo-backend triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core ) set_target_properties( triton-implicit-state-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_implicit_state LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_implicit_state.ldscript LINK_FLAGS "-Wl,--version-script libtriton_implicit_state.ldscript" ) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonImplicitStateBackend) install( TARGETS triton-implicit-state-backend EXPORT triton-implicit-state-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/implicit_state ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/implicit_state ) install( EXPORT triton-implicit-state-backend-targets FILE TritonImplicitStateBackendTargets.cmake NAMESPACE TritonImplicitStateBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonImplicitStateBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonImplicitStateBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonImplicitStateBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-implicit-state-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonImplicitStateBackendTargets.cmake NAMESPACE TritonImplicitStateBackend:: ) export(PACKAGE TritonImplicitStateBackend) ��������������������������������������������kytensor-2.49.0.7/src/test/implicit_state/cmake/����������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021475�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/implicit_state/cmake/TritonImplicitStateBackendConfig.cmake.in�����������0000664�0000000�0000000�00000003760�15270523712�0031463�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONIMPLICITSTATEBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONIMPLICITSTATEBACKEND_CMAKE_DIR}) if(NOT TARGET TritonImplicitStateBackend::triton-implicit-state-backend) include("${TRITONIMPLICITSEQUENCEBACKEND_CMAKE_DIR}/TritonImplicitStateBackendTargets.cmake") endif() set(TRITONIMPLICITSTATEBACKEND_LIBRARIES TritonImplicitStateBackend::triton-implicit-state-backend) ����������������kytensor-2.49.0.7/src/test/implicit_state/src/������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021204�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/implicit_state/src/implicit_state.cc�������������������������������������0000664�0000000�0000000�00000131572�15270523712�0024536�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <algorithm> #include <vector> #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" namespace triton { namespace backend { namespace implicit { // Implicit state backend that is solely used with testing implicit state // management functionality in the backend API. // // The backend supports models that take 4 input tensors, three INT32 [ 1 ] // control values, one UINT64 [ 1 ] correlation ID control, one INT32 [ 1 ] // value input, and one INT32 [ 1 ] input indicating the test case. The input // tensors must be named "START", "END", "READY", "CORRID", "UPDATE", "INPUT", // and "TEST_CASE". The output tensor must be named "OUTPUT". // // The list of accepted values for the "TEST_CASE" field are: // // * STATE_NEW_NON_EXISTENT = 0: This tests calling the TRITONBACKEND_StateNew // for a non existent state or a model that doesn't have states section in // sequence batching. // // * STATE_UPDATE_FALSE = 1: Tests not calling the state update and expecting // the implicit state to not be updated. // // * USE_SINGLE_STATE_BUFFER = 2: For this scenario we will be using the same // buffer for both input and output state. In total there will be 3 requests // sent in a sequence. // // * USE_GROWABLE_STATE_BUFFER = 3: In this test case we use growable state // buffer. Currently, growable state buffer only supports CUDA memory. #define GUARDED_RESPOND_IF_ERROR(RESPONSES, IDX, REQUEST, X) \ do { \ if ((RESPONSES)[IDX] != nullptr) { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (RESPONSES)[IDX], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ err__), \ "failed to send error response"); \ LOG_IF_ERROR( \ TRITONBACKEND_RequestRelease( \ REQUEST, TRITONSERVER_REQUEST_RELEASE_ALL), \ "failed to release the request."); \ (RESPONSES)[IDX] = nullptr; \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; // Validate that model configuration is supported by this backend. TRITONSERVER_Error* ValidateModelConfig(); private: ModelState(TRITONBACKEND_Model* triton_model); }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelState::ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model) { } TRITONSERVER_Error* ModelState::ValidateModelConfig() { // We have the json DOM for the model configuration... common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(model_config_.PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model configuration:\n") + buffer.Contents()).c_str()); // The model configuration must specify the sequence batcher and // must use the START, END, READ and CORRID input to indicate // control values. triton::common::TritonJson::Value sequence_batching; RETURN_IF_ERROR( model_config_.MemberAsObject("sequence_batching", &sequence_batching)); common::TritonJson::Value control_inputs; RETURN_IF_ERROR( sequence_batching.MemberAsArray("control_input", &control_inputs)); RETURN_ERROR_IF_FALSE( control_inputs.ArraySize() == 3, TRITONSERVER_ERROR_INVALID_ARG, std::string("'START', 'END, and 'READY' must be configured as " "the control inputs")); std::vector<std::string> control_input_names; for (size_t io_index = 0; io_index < control_inputs.ArraySize(); io_index++) { common::TritonJson::Value control_input; RETURN_IF_ERROR(control_inputs.IndexAsObject(io_index, &control_input)); const char* input_name = nullptr; size_t input_name_len; RETURN_IF_ERROR( control_input.MemberAsString("name", &input_name, &input_name_len)); control_input_names.push_back(input_name); } RETURN_ERROR_IF_FALSE( ((std::find( control_input_names.begin(), control_input_names.end(), "START") != control_input_names.end()) || (std::find( control_input_names.begin(), control_input_names.end(), "END") != control_input_names.end()) || (std::find( control_input_names.begin(), control_input_names.end(), "READY") != control_input_names.end())), TRITONSERVER_ERROR_INVALID_ARG, std::string("'START', 'END, and 'READY' must be configured as " "the control inputs")); return nullptr; // success } // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each TRITONBACKEND_ModelInstance. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } void* state_ = nullptr; // Index of the request in the sequence uint32_t request_index_ = 0; private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance); ModelState* model_state_; }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelInstanceState::ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state) { } extern "C" { // Implementing TRITONBACKEND_Initialize is optional. The backend // should initialize any global state that is intended to be shared // across all models and model instances that use the backend. TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); // We should check the backend API version that Triton supports // vs. what this backend was compiled against. uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Triton TRITONBACKEND API version: ") + std::to_string(api_version_major) + "." + std::to_string(api_version_minor)) .c_str()); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("'") + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "triton backend API version does not support this backend"); } // The backend configuration may contain information needed by the // backend, such a command-line arguments. This backend doesn't use // any such configuration but we print whatever is available. TRITONSERVER_Message* backend_config_message; RETURN_IF_ERROR( TRITONBACKEND_BackendConfig(backend, &backend_config_message)); const char* buffer; size_t byte_size; RETURN_IF_ERROR(TRITONSERVER_MessageSerializeToJson( backend_config_message, &buffer, &byte_size)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("backend configuration:\n") + buffer).c_str()); return nullptr; // success } // Implementing TRITONBACKEND_ModelInitialize is optional. The backend // should initialize any state that is intended to be shared across // all instances of the model. TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); // With each model we create a ModelState object and associate it // with the TRITONBACKEND_Model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast<void*>(model_state))); // One of the primary things to do in ModelInitialize is to examine // the model configuration to ensure that it is something that this // backend can support. If not, returning an error from this // function will prevent the model from loading. RETURN_IF_ERROR(model_state->ValidateModelConfig()); return nullptr; // success } // Implementing TRITONBACKEND_ModelFinalize is optional unless state // is set using TRITONBACKEND_ModelSetState. The backend must free // this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceInitialize is optional. The // backend should initialize any state that is required for a model // instance. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); // The instance can access the corresponding model as well... here // we get the model and from that get the model's state. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vmodelstate); // With each instance we create a ModelInstanceState object and // associate it with the TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast<void*>(instance_state))); return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceFinalize is optional unless // state is set using TRITONBACKEND_ModelInstanceSetState. The backend // must free this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast<ModelInstanceState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceExecute is required. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Suggested practice for this is to use only // function-local and model-instance-specific state (obtained from // 'instance'), which is what we do here. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast<void**>(&instance_state))); ModelState* model_state = instance_state->StateForModel(); // This backend specifies BLOCKING execution policy. That means that // we should not return from this function until execution is // complete. Triton will automatically release 'instance' on return // from this function so that it is again available to be used for // another call to TRITONBACKEND_ModelInstanceExecute. LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model ") + model_state->Name() + ", instance " + instance_state->Name() + ", executing " + std::to_string(request_count) + " requests") .c_str()); bool supports_batching = false; RETURN_IF_ERROR(model_state->SupportsFirstDimBatching(&supports_batching)); // 'responses' is initialized with the response objects below and // if/when an error response is sent the corresponding entry in // 'responses' is set to nullptr to indicate that that response has // already been sent. std::vector<TRITONBACKEND_Response*> responses; responses.reserve(request_count); // Create a single response object for each request. If something // goes wrong when attempting to create the response objects just // fail all of the requests by returning an error. for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); responses.push_back(response); } // The way we collect these batch timestamps is not entirely // accurate. Normally, in a performant backend you would execute all // the requests at the same time, and so there would be a single // compute-start / compute-end time-range. But here we execute each // request separately so there is no single range. As a result we // just show the entire execute time as being the compute time as // well. uint64_t min_exec_start_ns = std::numeric_limits<uint64_t>::max(); uint64_t max_exec_end_ns = 0; uint64_t total_batch_size = 0; // After this point we take ownership of 'requests', which means // that a response must be sent for every request. If something does // go wrong in processing a particular request then we send an error // response just for the specific request. // For simplicity we just process each request separately... in // general a backend should try to operate on the entire batch of // requests at the same time for improved performance. std::vector<uint8_t> start_buffer, end_buffer, ready_buffer, corrid_buffer, input_buffer; for (uint32_t r = 0; r < request_count; ++r) { uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); min_exec_start_ns = std::min(min_exec_start_ns, exec_start_ns); TRITONBACKEND_Request* request = requests[r]; const char* request_id = ""; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestId(request, &request_id)); uint32_t input_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestInputCount(request, &input_count)); uint32_t requested_output_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestOutputCount(request, &requested_output_count)); // If an error response was sent for the above then display an error // message and move on to next request. if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read request input/output counts, error response " "sent") .c_str()); continue; } LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("request ") + std::to_string(r) + ": id = \"" + request_id + "\", input_count = " + std::to_string(input_count) + ", requested_output_count = " + std::to_string(requested_output_count)) .c_str()); // For statistics we need to collect the total batch size of all the // requests. If the model doesn't support batching then each request is // necessarily batch-size 1. If the model does support batching then the // first dimension of the shape is the batch size. We only the first input // for this. if (supports_batching) { TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestInputByIndex(request, 0 /* index */, &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input, error response sent") .c_str()); continue; } const int64_t* input_shape; uint32_t input_dims_count; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_InputProperties( input, nullptr, nullptr, &input_shape, &input_dims_count, nullptr, nullptr)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input properties, error response sent") .c_str()); continue; } if (input_dims_count > 0) { if (input_shape[0] != 1) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "unable to execute more than one timestep at a time")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": unable to execute more than one timestep at a time, error " "response sent") .c_str()); continue; } total_batch_size += input_shape[0]; } } else { total_batch_size++; } LOG_MESSAGE( TRITONSERVER_LOG_VERBOSE, (std::string("total_batch_size: ") + std::to_string(total_batch_size)) .c_str()); std::set<uint64_t> seen_corrids; // Get the input tensors. TRITONBACKEND_Input* start_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestInput(request, "START", &start_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'START', error response sent") .c_str()); continue; } TRITONBACKEND_Input* end_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestInput(request, "END", &end_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'END', error response sent") .c_str()); continue; } TRITONBACKEND_Input* ready_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestInput(request, "READY", &ready_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'READY', error response sent") .c_str()); continue; } const void* start_buffer = nullptr; uint64_t buffer_byte_size = 0; TRITONSERVER_MemoryType input_memory_type = TRITONSERVER_MEMORY_CPU; int64_t input_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_InputBuffer( start_input, 0 /* input_buffer_count */, &start_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } const float* lstart_buffer = reinterpret_cast<const float*>(start_buffer); if (*lstart_buffer == 1) { instance_state->request_index_ = 0; instance_state->state_ = nullptr; } const void* end_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_InputBuffer( end_input, 0 /* input_buffer_count */, &end_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } const void* ready_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_InputBuffer( ready_input, 0 /* input_buffer_count */, &ready_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestInput(request, "INPUT", &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'INPUT', error response sent") .c_str()); continue; } const void* input_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_InputBuffer( input, 0 /* input_buffer_count */, &input_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if ((responses[r] == nullptr) || (input_memory_type == TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer in CPU memory, error " "response sent") .c_str()); continue; } TRITONBACKEND_Input* test_case = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestInput(request, "TEST_CASE", &test_case)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'TEST_CASE', error response sent") .c_str()); continue; } const void* test_case_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_InputBuffer( test_case, 0 /* test_case_buffer_count */, &test_case_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if ((responses[r] == nullptr) || (input_memory_type == TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer in CPU memory, error " "response sent") .c_str()); continue; } const int32_t test_case_buffer_int = *reinterpret_cast<const int32_t*>(test_case_buffer); const int32_t ipbuffer_int = *reinterpret_cast<const int32_t*>(input_buffer); int32_t ipbuffer_state_int = 0; if (test_case_buffer_int != 0) { TRITONBACKEND_Input* input_state = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_RequestInput(request, "INPUT_STATE", &input_state)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'INPUT_STATE', error response sent") .c_str()); continue; } const void* input_state_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_InputBuffer( input_state, 0 /* input_buffer_count */, &input_state_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if ((responses[r] == nullptr) || (test_case_buffer_int == 3 && input_memory_type != TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "growable memory should always provide memory in GPU")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer in GPU memory, error " "response sent") .c_str()); continue; } else if ( (responses[r] == nullptr) || (input_memory_type == TRITONSERVER_MEMORY_GPU && test_case_buffer_int != 3)) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer in CPU memory, error " "response sent") .c_str()); continue; } // When using single state buffer, input/output tensors should point to // the buffer. if ((test_case_buffer_int == 2 || test_case_buffer_int == 3) && instance_state->state_ != nullptr) { if (input_state_buffer != instance_state->state_) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "Input and output state are using different buffers.")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": input and output state are using different buffers, error " "response sent") .c_str()); continue; } } if (test_case_buffer_int == 2 || test_case_buffer_int == 1 || test_case_buffer_int == 0) { const int32_t ipbuffer_state = *reinterpret_cast<const int32_t*>(input_state_buffer); ipbuffer_state_int = ipbuffer_state; } } switch (test_case_buffer_int) { // STATE_NEW_NON_EXISTENT. The behavior for both of the test cases is // the same. case 0: { TRITONBACKEND_State* response_state; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateNew( &response_state, request, "undefined_state", TRITONSERVER_TYPE_INT32, nullptr /* shape */, 0 /* dim_count */)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create the output state 'OUTPUT_STATE', error " "response sent") .c_str()); continue; } } break; // STATE_UPDATE_FALSE case 1: { TRITONBACKEND_State* response_state; TRITONBACKEND_Output* response_output; std::vector<int64_t> shape{1}; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateNew( &response_state, request, "OUTPUT_STATE", TRITONSERVER_TYPE_INT32, shape.data() /* data */, shape.size() /* dim_count */)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create the output state 'OUTPUT_STATE', error " "response sent") .c_str()); continue; } TRITONSERVER_MemoryType actual_memory_type = TRITONSERVER_MEMORY_GPU; int64_t actual_memory_type_id = 0; char* buffer; // Request an output buffer in GPU. This is only for testing purposes // to make sure that GPU output buffers can be requested. GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateBuffer( response_state, reinterpret_cast<void**>(&buffer), sizeof(int32_t), &actual_memory_type, &actual_memory_type_id)); if ((responses[r] == nullptr) || (actual_memory_type == TRITONSERVER_MEMORY_CPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create the state buffer in GPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create the state buffer in GPU memory, error " "response sent") .c_str()); continue; } actual_memory_type = TRITONSERVER_MEMORY_CPU; actual_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateBuffer( response_state, reinterpret_cast<void**>(&buffer), sizeof(int32_t), &actual_memory_type, &actual_memory_type_id)); if ((responses[r] == nullptr) || (actual_memory_type == TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create the state buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create the state buffer in CPU memory, error " "response sent") .c_str()); continue; } TRITONSERVER_BufferAttributes* buffer_attributes; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateBufferAttributes( response_state, &buffer_attributes)); // Testing for the StateBuffer attributes TRITONSERVER_MemoryType ba_memory_type; int64_t ba_memory_type_id; size_t ba_byte_size; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_BufferAttributesMemoryType( buffer_attributes, &ba_memory_type)); GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_BufferAttributesMemoryTypeId( buffer_attributes, &ba_memory_type_id)); GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_BufferAttributesByteSize( buffer_attributes, &ba_byte_size)); if (!((actual_memory_type == ba_memory_type) && (sizeof(int32_t) == ba_byte_size) && (ba_memory_type_id == actual_memory_type_id)) || responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "State buffer attributes are not set correctly.")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": State buffer attributes are not set correctly., error " "response sent") .c_str()); continue; } // Put the new state in the output buffer but intentionally do not // call the TRITONBACKEND_StateUpdate function. int32_t* lbuffer = reinterpret_cast<int32_t*>(buffer); *lbuffer = ipbuffer_int + ipbuffer_state_int; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_ResponseOutput( responses[r], &response_output, "OUTPUT", TRITONSERVER_TYPE_INT32, shape.data() /* data */, shape.size() /* dim_count */)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create the output state 'OUTPUT_STATE', error " "response sent") .c_str()); continue; } actual_memory_type = TRITONSERVER_MEMORY_CPU; actual_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_OutputBuffer( response_output, reinterpret_cast<void**>(&buffer), sizeof(int32_t), &actual_memory_type, &actual_memory_type_id)); if ((responses[r] == nullptr) || (actual_memory_type == TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create the state buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create the state buffer in CPU memory, error " "response sent") .c_str()); continue; } lbuffer = reinterpret_cast<int32_t*>(buffer); *lbuffer = ipbuffer_int + ipbuffer_state_int; } break; // USE_SINGLE_BUFFER case 2: { TRITONBACKEND_State* response_state; std::vector<int64_t> shape{1}; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateNew( &response_state, request, "OUTPUT_STATE", TRITONSERVER_TYPE_INT32, shape.data() /* data */, shape.size() /* dim_count */)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create the output state 'OUTPUT_STATE', error " "response sent") .c_str()); continue; } TRITONSERVER_MemoryType actual_memory_type = TRITONSERVER_MEMORY_CPU; int64_t actual_memory_type_id = 0; char* buffer; // Request an output buffer in GPU. This is only for testing purposes // to make sure that GPU output buffers can be requested. GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateBuffer( response_state, reinterpret_cast<void**>(&buffer), sizeof(int32_t), &actual_memory_type, &actual_memory_type_id)); instance_state->state_ = buffer; } break; case 3: { TRITONBACKEND_State* response_state; size_t block_size = sizeof(int8_t) * 1024 * 1024; int64_t current_elements = (instance_state->request_index_ + 1) * 1024 * 1024; std::cout << "current elements are " << (instance_state->request_index_ + 1) << std::endl; std::vector<int64_t> shape{current_elements}; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateNew( &response_state, request, "OUTPUT_STATE", TRITONSERVER_TYPE_INT8, shape.data() /* data */, shape.size() /* dim_count */)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create the output state 'OUTPUT_STATE', error " "response sent") .c_str()); continue; } TRITONSERVER_MemoryType actual_memory_type = TRITONSERVER_MEMORY_GPU; int64_t actual_memory_type_id = 0; char* buffer; // Request an output buffer in GPU. This is only for testing purposes // to make sure that GPU output buffers can be requested. GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_StateBuffer( response_state, reinterpret_cast<void**>(&buffer), block_size * (instance_state->request_index_ + 1), &actual_memory_type, &actual_memory_type_id)); // Only write the new data to the portion of the state buffer that // has been grown. cudaMemset( buffer + block_size * (instance_state->request_index_), instance_state->request_index_, block_size); TRITONBACKEND_Output* response_output; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_ResponseOutput( responses[r], &response_output, "OUTPUT_STATE", TRITONSERVER_TYPE_INT8, shape.data() /* data */, shape.size() /* dim_count */)); actual_memory_type = TRITONSERVER_MEMORY_CPU; actual_memory_type_id = 0; char* output_buffer; GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONBACKEND_OutputBuffer( response_output, reinterpret_cast<void**>(&output_buffer), block_size * (instance_state->request_index_ + 1), &actual_memory_type, &actual_memory_type_id)); if ((responses[r] == nullptr) || (actual_memory_type != TRITONSERVER_MEMORY_CPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, request, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "the backend can only handle CPU tensors")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + "the backend can only handle CPU tensors" "response sent") .c_str()); continue; } cudaMemcpy( output_buffer, buffer, block_size * (instance_state->request_index_ + 1), cudaMemcpyDeviceToHost); instance_state->state_ = buffer; } break; } const float* lend_buffer = reinterpret_cast<const float*>(end_buffer); if (*lend_buffer == 1) { instance_state->request_index_ = 0; } else { instance_state->request_index_ += 1; } uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); max_exec_end_ns = std::max(max_exec_end_ns, exec_end_ns); // Send all the responses that haven't already been sent because of // an earlier error. if (responses[r] != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( responses[r], TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr /* success */), "failed sending response"); } // Report statistics for each request. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_state->TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, exec_start_ns, exec_end_ns, exec_end_ns), "failed reporting request statistics"); LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } // Report the entire batch statistics. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( instance_state->TritonModelInstance(), total_batch_size, min_exec_start_ns, min_exec_start_ns, max_exec_end_ns, max_exec_end_ns), "failed reporting batch request statistics"); return nullptr; // success } } // extern "C" }}} // namespace triton::backend::implicit ��������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/implicit_state/src/libtriton_implicit_state.ldscript���������������������0000664�0000000�0000000�00000003072�15270523712�0030054�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/iterative_sequence/������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021267�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/iterative_sequence/CMakeLists.txt����������������������������������������0000664�0000000�0000000�00000010104�15270523712�0024023�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritoniterativesequencebackend LANGUAGES C CXX) # # libtriton_iterative_sequence.so # Shared library implementing the Triton Sequence Backend API # configure_file(src/libtriton_iterative_sequence.ldscript libtriton_iterative_sequence.ldscript COPYONLY) add_library( triton-iterative-sequence-backend SHARED src/iterative_sequence.cc ) add_library( TritonIterativeSequenceBackend::triton-iterative-sequence-backend ALIAS triton-iterative-sequence-backend ) target_compile_features(triton-iterative-sequence-backend PRIVATE cxx_std_11) target_compile_options( triton-iterative-sequence-backend PRIVATE $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> ) target_link_libraries( triton-iterative-sequence-backend PRIVATE triton-backend-utils # from repo-backend triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core ) set_target_properties( triton-iterative-sequence-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_iterative_sequence LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_iterative_sequence.ldscript LINK_FLAGS "-Wl,--version-script libtriton_iterative_sequence.ldscript" ) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonIterativeSequenceBackend) install( TARGETS triton-iterative-sequence-backend EXPORT triton-iterative-sequence-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/iterative_sequence ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/iterative_sequence ) install( EXPORT triton-iterative-sequence-backend-targets FILE TritonIterativeSequenceBackendTargets.cmake NAMESPACE TritonIterativeSequenceBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonIterativeSequenceBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonIterativeSequenceBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonIterativeSequenceBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-iterative-sequence-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonIterativeSequenceBackendTargets.cmake NAMESPACE TritonIterativeSequenceBackend:: ) export(PACKAGE TritonIterativeSequenceBackend) ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/iterative_sequence/cmake/������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022347�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/iterative_sequence/cmake/TritonIterativeSequenceBackendConfig.cmake.in���0000664�0000000�0000000�00000003731�15270523712�0033205�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONSEQUENCEBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONSEQUENCEBACKEND_CMAKE_DIR}) if(NOT TARGET TritonIterativeSequenceBackend::triton-sequence-backend) include("${TRITONSEQUENCEBACKEND_CMAKE_DIR}/TritonIterativeSequenceBackendTargets.cmake") endif() set(TRITONSEQUENCEBACKEND_LIBRARIES TritonIterativeSequenceBackend::triton-sequence-backend) ���������������������������������������kytensor-2.49.0.7/src/test/iterative_sequence/src/��������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022056�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/iterative_sequence/src/iterative_sequence.cc�����������������������������0000664�0000000�0000000�00000052556�15270523712�0026266�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <algorithm> #include <memory> #include <thread> #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" namespace triton { namespace backend { namespace iterative_sequence { // Simple iterative sequence backend that demonstrates the use of // TRITONSERVER_REQUEST_RELEASE_RESCHEDULE flag to iteratively produce // sequence response. // // The backend supports models that take 1 input tensor, an INT32 [ 1 ] // input named "INPUT"; and produces an output tensor "OUTPUT" with the same // shape as the input tensor. The input value indicates the total number of // responses to be generated and the output value indicates the number of // remaining responses. For example, if the request input has value 2, // the backend will: // - Send a response with value 1. // - Release request with RESCHEDULE flag. // - When execute on the same request, send the last response with value 0. // - Release request with ALL flag. // #define GUARDED_RESPOND_IF_ERROR(RESPONSES, IDX, X) \ do { \ if ((RESPONSES)[IDX] != nullptr) { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (RESPONSES)[IDX], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ err__), \ "failed to send error response"); \ (RESPONSES)[IDX] = nullptr; \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; private: ModelState(TRITONBACKEND_Model* triton_model); }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelState::ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model) { } // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each TRITONBACKEND_ModelInstance. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState() = default; // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } // return output value on receiving request, initialize remainder // if the corrid hasn't been recorded. int32_t GetOutput(uint64_t corrid, int32_t init_value); private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance); ModelState* model_state_; // A map from correlation ID to the remaining responses. std::unordered_map<uint64_t, int32_t> remainders_; }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelInstanceState::ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state) { } int32_t ModelInstanceState::GetOutput(uint64_t corrid, int32_t init_value) { auto it = remainders_.find(corrid); if (it == remainders_.end()) { it = remainders_.emplace(corrid, init_value).first; } auto res = --it->second; if (res <= 0) { remainders_.erase(it); } return res; } ///////////// extern "C" { // Implementing TRITONBACKEND_Initialize is optional. The backend // should initialize any global state that is intended to be shared // across all models and model instances that use the backend. TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); // We should check the backend API version that Triton supports // vs. what this backend was compiled against. uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "triton backend API version does not support this backend"); } return nullptr; // success } // Implementing TRITONBACKEND_ModelInitialize is optional. The backend // should initialize any state that is intended to be shared across // all instances of the model. TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); // With each model we create a ModelState object and associate it // with the TRITONBACKEND_Model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast<void*>(model_state))); return nullptr; // success } // Implementing TRITONBACKEND_ModelFinalize is optional unless state // is set using TRITONBACKEND_ModelSetState. The backend must free // this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceInitialize is optional. The // backend should initialize any state that is required for a model // instance. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); // The instance can access the corresponding model as well... here // we get the model and from that get the model's state. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vmodelstate); // With each instance we create a ModelInstanceState object and // associate it with the TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast<void*>(instance_state))); // Because this backend just copies IN -> OUT and requires that // input and output be in CPU memory, we fail if a GPU instances is // requested. RETURN_ERROR_IF_FALSE( instance_state->Kind() == TRITONSERVER_INSTANCEGROUPKIND_CPU, TRITONSERVER_ERROR_INVALID_ARG, std::string("'iterative_sequence' backend only supports CPU instances")); return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceFinalize is optional unless // state is set using TRITONBACKEND_ModelInstanceSetState. The backend // must free this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast<ModelInstanceState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceExecute is required. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Suggested practice for this is to use only // function-local and model-instance-specific state (obtained from // 'instance'), which is what we do here. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast<void**>(&instance_state))); ModelState* model_state = instance_state->StateForModel(); // This backend specifies BLOCKING execution policy. That means that // we should not return from this function until execution is // complete. Triton will automatically release 'instance' on return // from this function so that it is again available to be used for // another call to TRITONBACKEND_ModelInstanceExecute. LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model ") + model_state->Name() + ", instance " + instance_state->Name() + ", executing " + std::to_string(request_count) + " requests") .c_str()); bool supports_batching = false; RETURN_IF_ERROR(model_state->SupportsFirstDimBatching(&supports_batching)); // 'responses' is initialized with the response objects below and // if/when an error response is sent the corresponding entry in // 'responses' is set to nullptr to indicate that that response has // already been sent. std::vector<TRITONBACKEND_Response*> responses; responses.reserve(request_count); // Create a single response object for each request. If something // goes wrong when attempting to create the response objects just // fail all of the requests by returning an error. for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); responses.push_back(response); } // The way we collect these batch timestamps is not entirely // accurate. Normally, in a performant backend you would execute all // the requests at the same time, and so there would be a single // compute-start / compute-end time-range. But here we execute each // request separately so there is no single range. As a result we // just show the entire execute time as being the compute time as // well. uint64_t min_exec_start_ns = std::numeric_limits<uint64_t>::max(); uint64_t max_exec_end_ns = 0; uint64_t total_batch_size = 0; // After this point we take ownership of 'requests', which means // that a response must be sent for every request. If something does // go wrong in processing a particular request then we send an error // response just for the specific request. // For simplicity we just process each request separately... in // general a backend should try to operate on the entire batch of // requests at the same time for improved performance. std::vector<uint8_t> start_buffer, end_buffer, ready_buffer, corrid_buffer, input_buffer; for (uint32_t r = 0; r < request_count; ++r) { ++total_batch_size; uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); min_exec_start_ns = std::min(min_exec_start_ns, exec_start_ns); TRITONBACKEND_Request* request = requests[r]; uint64_t correlation_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestCorrelationId(request, &correlation_id)); // If an error response was sent for the above then display an error // message and move on to next request. if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read request input/output counts, error response " "sent") .c_str()); continue; } TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "INPUT", &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'INPUT', error response sent") .c_str()); continue; } const void* input_buffer = nullptr; uint64_t buffer_byte_size = 0; TRITONSERVER_MemoryType input_memory_type = TRITONSERVER_MEMORY_CPU; int64_t input_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( input, 0 /* input_buffer_count */, &input_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if ((responses[r] == nullptr) || (input_memory_type == TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer in CPU memory, error " "response sent") .c_str()); continue; } const int32_t init_value = *reinterpret_cast<const int32_t*>(input_buffer); auto output_value = instance_state->GetOutput(correlation_id, init_value); TRITONBACKEND_Response* response = responses[r]; // The output shape is [1, 1] if the model // configuration supports batching, or just // [1] if the model configuration does not // support batching. std::vector<int64_t> shape; if (supports_batching) { shape.push_back(1); } shape.push_back(1); TRITONBACKEND_Output* output; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_ResponseOutput( response, &output, "OUTPUT", TRITONSERVER_TYPE_INT32, shape.data(), shape.size())); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create response output, error response sent") .c_str()); continue; } // Step 2. Get the output buffer. We request a buffer in CPU // memory but we have to handle any returned type. If we get // back a buffer in GPU memory we just fail the request. void* output_buffer; TRITONSERVER_MemoryType output_memory_type = TRITONSERVER_MEMORY_CPU; int64_t output_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_OutputBuffer( output, &output_buffer, sizeof(int32_t), &output_memory_type, &output_memory_type_id)); if ((responses[r] == nullptr) || (output_memory_type == TRITONSERVER_MEMORY_GPU)) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create output buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create output buffer in CPU memory, error " "response sent") .c_str()); continue; } reinterpret_cast<int32_t*>(output_buffer)[0] = output_value; // Set response flag and request flag correctly based on whether this // is the last response of the sequence. uint32_t res_flag = (output_value <= 0) ? TRITONSERVER_RESPONSE_COMPLETE_FINAL : 0; uint32_t req_flag = (output_value <= 0) ? TRITONSERVER_REQUEST_RELEASE_ALL : TRITONSERVER_REQUEST_RELEASE_RESCHEDULE; uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); max_exec_end_ns = std::max(max_exec_end_ns, exec_end_ns); // wait for 0.5 second before rescheduling the request. std::this_thread::sleep_for(std::chrono::milliseconds(500)); // Release the request first as the testing backend may be configured to // receive error on request release, in such a case, the error will be // propagated back through error response. auto err = TRITONBACKEND_RequestRelease(request, req_flag); if (err) { // Release request with ALL flag LOG_IF_ERROR( TRITONBACKEND_RequestRelease( request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); res_flag = TRITONSERVER_RESPONSE_COMPLETE_FINAL; } // Send all the responses that haven't already been sent because of // an earlier error. if (responses[r] != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend(responses[r], res_flag, err), "failed sending response"); } TRITONSERVER_ErrorDelete(err); // Report statistics for each request. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_state->TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, exec_start_ns, exec_end_ns, exec_end_ns), "failed reporting request statistics"); } // Report the entire batch statistics. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( instance_state->TritonModelInstance(), total_batch_size, min_exec_start_ns, min_exec_start_ns, max_exec_end_ns, max_exec_end_ns), "failed reporting batch request statistics"); return nullptr; // success } } // extern "C" }}} // namespace triton::backend::iterative_sequence ��������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/iterative_sequence/src/libtriton_iterative_sequence.ldscript�������������0000664�0000000�0000000�00000003072�15270523712�0031600�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/models/������������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0016666�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/models/identity_fp32/����������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021351�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/models/identity_fp32/config.pbtxt����������������������������������������0000664�0000000�0000000�00000003426�15270523712�0023706�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "identity_fp32" backend: "identity" max_batch_size: 1 input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ -1 ] } ] output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ -1 ] } ] instance_group [ { count: 1 kind : KIND_CPU } ] ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/models/repeat_int32/�����������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021165�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/models/repeat_int32/config.pbtxt�����������������������������������������0000664�0000000�0000000�00000003717�15270523712�0023525�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "repeat_int32" backend: "repeat" max_batch_size: 0 model_transaction_policy { decoupled: True } input [ { name: "IN" data_type: TYPE_INT32 dims: [ -1 ] }, { name: "DELAY" data_type: TYPE_UINT32 dims: [ -1 ] }, { name: "WAIT" data_type: TYPE_UINT32 dims: [ 1 ] } ] output [ { name: "OUT" data_type: TYPE_INT32 dims: [ 1 ] }, { name: "IDX" data_type: TYPE_UINT32 dims: [ 1 ] } ] �������������������������������������������������kytensor-2.49.0.7/src/test/models/square_int32/�����������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021205�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/models/square_int32/config.pbtxt�����������������������������������������0000664�0000000�0000000�00000003376�15270523712�0023546�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "square_int32" backend: "square" max_batch_size: 0 model_transaction_policy { decoupled: True } input [ { name: "IN" data_type: TYPE_INT32 dims: [ 1 ] } ] output [ { name: "OUT" data_type: TYPE_INT32 dims: [ 1 ] } ] ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/query_backend/�����������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020217�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/query_backend/CMakeLists.txt���������������������������������������������0000664�0000000�0000000�00000007276�15270523712�0022773�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonquerybackend LANGUAGES C CXX) # # libtriton_query.so # Shared library implementing the Triton Query Backend API # configure_file(src/libtriton_query.ldscript libtriton_query.ldscript COPYONLY) add_library( triton-query-backend SHARED src/query.cc ) add_library( TritonQueryBackend::triton-query-backend ALIAS triton-query-backend ) target_compile_features(triton-query-backend PRIVATE cxx_std_11) target_compile_options( triton-query-backend PRIVATE $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> ) target_link_libraries( triton-query-backend PRIVATE triton-backend-utils # from repo-backend triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core ) set_target_properties( triton-query-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_query LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_query.ldscript LINK_FLAGS "-Wl,--version-script libtriton_query.ldscript" ) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonQueryBackend) install( TARGETS triton-query-backend EXPORT triton-query-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/query ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/query ) install( EXPORT triton-query-backend-targets FILE TritonQueryBackendTargets.cmake NAMESPACE TritonQueryBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonQueryBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonQueryBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonQueryBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-query-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonQueryBackendTargets.cmake NAMESPACE TritonQueryBackend:: ) export(PACKAGE TritonQueryBackend) ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/query_backend/cmake/�����������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021277�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/query_backend/cmake/TritonQueryBackendConfig.cmake.in��������������������0000664�0000000�0000000�00000003660�15270523712�0027616�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONQUERYBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONQUERYBACKEND_CMAKE_DIR}) if(NOT TARGET TritonQueryBackend::triton-distributed-addsub-backend) include("${TRITONQUERYBACKEND_CMAKE_DIR}/TritonQueryBackendTargets.cmake") endif() set(TRITONQUERYBACKEND_LIBRARIES TritonQueryBackend::triton-query-backend) ��������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/query_backend/src/�������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021006�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/query_backend/src/libtriton_query.ldscript�������������������������������0000664�0000000�0000000�00000003072�15270523712�0026011�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/query_backend/src/query.cc�����������������������������������������������0000664�0000000�0000000�00000016666�15270523712�0022501�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <vector> #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" namespace triton { namespace backend { namespace query { // Query backend that is solely used with unit-testing query functionality // in both the server API and backend API. // // The backend will call the backend query API in setting below: // name, byte_size, memory_type, memory_type_id (refer backend API for detail) // "OUTPUT0", nullptr, CPU_PINNED, 1 // "OUTPUT1", nullptr, CPU_PINNED, 1 // Then it will call the alloc function (TRITONBACKEND_OutputBuffer) with // the returned value accordingly. If 'byte_size' is nullptr, it creates the // outputs with UINT8 type and shape [2]. // The backend will read environment variables for different query behavior // 'TEST_ANONYMOUS': the backend will call the query API only once with 'name' // set to nullptr // 'TEST_BYTE_SIZE': the backend will call the query API once with 'byte_size' // set to the variable value, and the outputs will be created // with UINT8 and shape [byte_size]. If 'TEST_ANONYMOUS' is // also specified, the outputs will have shape [byte_size / 2] // 'TEST_FAIL_WITH_QUERY_RESULT' : the query results will be formatted to string // and returned as error message. #define RESPOND_IF_ERROR(RESPONSE, X) \ do { \ if (RESPONSE != nullptr) { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ RESPONSE, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err__), \ "failed to send error response"); \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) extern "C" { // Implementing TRITONBACKEND_ModelInstanceExecute is required. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Read environment variables const char* anonymous_str = getenv("TEST_ANONYMOUS"); const char* byte_size_str = getenv("TEST_BYTE_SIZE"); const char* fail_str = getenv("TEST_FAIL_WITH_QUERY_RESULT"); bool anonymous = (anonymous_str != nullptr); size_t byte_size = 2; size_t query_byte_size = byte_size; size_t* byte_size_ptr = nullptr; if (byte_size_str != nullptr) { byte_size = atoi(byte_size_str); query_byte_size = byte_size; if (anonymous) { byte_size /= 2; } byte_size_ptr = &query_byte_size; } for (uint32_t r = 0; r < request_count; ++r) { std::string log_message; TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response = nullptr; // Query before creating output std::vector<const char*> names; if (anonymous) { names.emplace_back(nullptr); } else { names = {"OUTPUT0", "OUTPUT1"}; } std::vector<TRITONSERVER_MemoryType> types{ TRITONSERVER_MEMORY_CPU_PINNED, TRITONSERVER_MEMORY_CPU_PINNED}; std::vector<int64_t> type_ids{1, 1}; for (size_t i = 0; i < names.size(); ++i) { auto err = TRITONBACKEND_RequestOutputBufferProperties( request, names[i], byte_size_ptr, &types[i], &type_ids[i]); if (err != nullptr) { RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); RESPOND_IF_ERROR(response, err); break; } if (fail_str != nullptr) { log_message += ((names[i] == nullptr) ? "NULL" : names[i]); switch (types[i]) { case TRITONSERVER_MEMORY_CPU: log_message += " CPU "; break; case TRITONSERVER_MEMORY_CPU_PINNED: log_message += " CPU_PINNED "; break; case TRITONSERVER_MEMORY_GPU: log_message += " GPU "; break; } log_message += (std::to_string(type_ids[i]) + "; "); } } // If response is not nullptr, some error is returned from query API and // the response has been sent if (response == nullptr) { if (names.size() == 1) { names = {"OUTPUT0", "OUTPUT1"}; types[1] = types[0]; type_ids[1] = type_ids[0]; } std::vector<int64_t> shape{(int64_t)byte_size}; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); TRITONSERVER_Error* err = nullptr; if (fail_str == nullptr) { for (size_t i = 0; i < names.size(); ++i) { TRITONBACKEND_Output* output; err = TRITONBACKEND_ResponseOutput( response, &output, names[i], TRITONSERVER_TYPE_UINT8, shape.data(), 1); if (err != nullptr) { break; } void* output_buffer; err = TRITONBACKEND_OutputBuffer( output, &output_buffer, byte_size, &types[i], &type_ids[i]); if (err != nullptr) { break; } // Do nothing with the buffer as we don't care } } else { // Use an uncommon error code err = TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNKNOWN, log_message.c_str()); } TRITONBACKEND_ResponseSend( response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err); } TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL); } return nullptr; // success } } // extern "C" }}} // namespace triton::backend::query ��������������������������������������������������������������������������kytensor-2.49.0.7/src/test/repoagent/���������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0017367�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/repoagent/relocation_repoagent/������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0023572�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/repoagent/relocation_repoagent/CMakeLists.txt����������������������������0000664�0000000�0000000�00000007312�15270523712�0026335�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonrelocationrepoagent LANGUAGES C CXX) configure_file( src/libtritonrepoagent_relocation.ldscript libtritonrepoagent_relocation.ldscript COPYONLY) add_library( triton-relocation-repoagent SHARED src/relocation.cc ) add_library( TritonRelocationRepoAgent::triton-relocation-repoagent ALIAS triton-relocation-repoagent ) target_compile_features(triton-relocation-repoagent PRIVATE cxx_std_11) target_compile_options( triton-relocation-repoagent PRIVATE $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> ) target_link_libraries( triton-relocation-repoagent PRIVATE triton-core-serverapi # from repo-core triton-core-repoagentapi # from repo-core triton-core-serverstub # from repo-core ) set_target_properties( triton-relocation-repoagent PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME tritonrepoagent_relocation LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtritonrepoagent_relocation.ldscript LINK_FLAGS "-Wl,--version-script libtritonrepoagent_relocation.ldscript" ) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonRelocationRepoAgent) install( TARGETS triton-relocation-repoagent EXPORT triton-relocation-repoagent-targets LIBRARY DESTINATION lib ) install( EXPORT triton-relocation-repoagent-targets FILE TritonRelocationRepoAgentTargets.cmake NAMESPACE TritonRelocationRepoAgent:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonRelocationRepoAgentConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonRelocationRepoAgentConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonRelocationRepoAgentConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-relocation-repoagent-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonRelocationRepoAgentTargets.cmake NAMESPACE TritonRelocationRepoAgent:: ) export(PACKAGE TritonRelocationRepoAgent) ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/repoagent/relocation_repoagent/cmake/������������������������������������0000775�0000000�0000000�00000000000�15270523712�0024652�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������TritonRelocationRepoAgentConfig.cmake.in������������������������������������������������������������0000664�0000000�0000000�00000003725�15270523712�0034443�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000�kytensor-2.49.0.7/src/test/repoagent/relocation_repoagent/cmake��������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONRELOCATIONREPOAGENT_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONRELOCATIONREPOAGENT_CMAKE_DIR}) if(NOT TARGET TritonRelocationRepoAgent::triton-relocation-repoagent) include("${TRITONRELOCATIONREPOAGENT_CMAKE_DIR}/TritonRelocationRepoAgentTargets.cmake") endif() set(TRITONRELOCATIONREPOAGENT_LIBRARIES TritonRelocationRepoAgent::triton-relocation-repoagent) �������������������������������������������kytensor-2.49.0.7/src/test/repoagent/relocation_repoagent/src/��������������������������������������0000775�0000000�0000000�00000000000�15270523712�0024361�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/repoagent/relocation_repoagent/src/libtritonrepoagent_relocation.ldscript0000664�0000000�0000000�00000003057�15270523712�0034266�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONREPOAGENT_*; local: *; }; ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/repoagent/relocation_repoagent/src/relocation.cc�������������������������0000664�0000000�0000000�00000014751�15270523712�0027037�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <cctype> #include <cstring> #include <iomanip> #include <stdexcept> #include <string> #include "triton/core/tritonrepoagent.h" #include "triton/core/tritonserver.h" // // Relocation Repository Agent that is for test only. // namespace triton { namespace repoagent { namespace relocation { namespace { // // ErrorException // // Exception thrown if error occurs while running RelocationRepoAgent // struct ErrorException { ErrorException(TRITONSERVER_Error* err) : err_(err) {} TRITONSERVER_Error* err_; }; #define THROW_IF_TRITON_ERROR(X) \ do { \ TRITONSERVER_Error* tie_err__ = (X); \ if (tie_err__ != nullptr) { \ throw triton::repoagent::relocation::ErrorException(tie_err__); \ } \ } while (false) #define THROW_TRITON_ERROR(CODE, MSG) \ do { \ TRITONSERVER_Error* tie_err__ = TRITONSERVER_ErrorNew(CODE, MSG); \ throw triton::repoagent::relocation::ErrorException(tie_err__); \ } while (false) #define RETURN_IF_ERROR(X) \ do { \ TRITONSERVER_Error* rie_err__ = (X); \ if (rie_err__ != nullptr) { \ return rie_err__; \ } \ } while (false) } // namespace ///////////// extern "C" { TRITONSERVER_Error* TRITONREPOAGENT_ModelFinalize( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model) { const char* location; RETURN_IF_ERROR(TRITONREPOAGENT_ModelState(model, (void**)&location)); RETURN_IF_ERROR( TRITONREPOAGENT_ModelRepositoryLocationRelease(agent, model, location)); return nullptr; } TRITONSERVER_Error* TRITONREPOAGENT_ModelAction( TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model, const TRITONREPOAGENT_ActionType action_type) { // Return success (nullptr) if the agent does not handle the action if (action_type != TRITONREPOAGENT_ACTION_LOAD) { return nullptr; } // Check the agent parameters for the relocation configuration of the model uint32_t parameter_count = 0; RETURN_IF_ERROR( TRITONREPOAGENT_ModelParameterCount(agent, model, ¶meter_count)); if (parameter_count != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Relocation repoagent expects 1 parameter for relocation agent"); } const char* key = nullptr; const char* value = nullptr; RETURN_IF_ERROR( TRITONREPOAGENT_ModelParameter(agent, model, 0, &key, &value)); if (std::string(key) != "empty_config") { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Relocation repoagent expects parameter with key 'empty_config' for " "relocation agent"); } else if ( (std::string(value) != "true") && (std::string(value) != "false")) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Relocation repoagent expects 'empty_config' parameter with value " "'true' or 'false' for relocation agent"); } TRITONSERVER_Message* model_config; RETURN_IF_ERROR(TRITONREPOAGENT_ModelConfig(agent, model, 1, &model_config)); const char* base; size_t byte_size; auto err = TRITONSERVER_MessageSerializeToJson(model_config, &base, &byte_size); if (err == nullptr) { // hack to check if proper model config is passed by knowing that only // the original config will contain 'model_repository_agents' setting auto pos = std::string(base, byte_size).find("model_repository_agents"); if ((std::string(value) == "true") && (pos != std::string::npos)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Relocation repoagent expects config does not contain " "'model_repository_agents' field when 'empty_config' has value " "'true' for relocation agent"); } else if ((std::string(value) == "false") && (pos == std::string::npos)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Relocation repoagent expects config contains " "'model_repository_agents' field when 'empty_config' has value " "'false' for relocation agent"); } } RETURN_IF_ERROR(TRITONSERVER_MessageDelete(model_config)); RETURN_IF_ERROR(err); // Point to a new model repository const char* location; RETURN_IF_ERROR(TRITONREPOAGENT_ModelRepositoryLocationAcquire( agent, model, TRITONREPOAGENT_ARTIFACT_FILESYSTEM, &location)); RETURN_IF_ERROR(TRITONREPOAGENT_ModelRepositoryUpdate( agent, model, TRITONREPOAGENT_ARTIFACT_FILESYSTEM, location)); RETURN_IF_ERROR(TRITONREPOAGENT_ModelSetState(model, (void*)location)); return nullptr; // success } } // extern "C" }}} // namespace triton::repoagent::relocation �����������������������kytensor-2.49.0.7/src/test/sequence/����������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0017213�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/sequence/CMakeLists.txt��������������������������������������������������0000664�0000000�0000000�00000007436�15270523712�0021765�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonsequencebackend LANGUAGES C CXX) # # libtriton_sequence.so # Shared library implementing the Triton Sequence Backend API # configure_file(src/libtriton_sequence.ldscript libtriton_sequence.ldscript COPYONLY) add_library( triton-sequence-backend SHARED src/sequence.cc ) add_library( TritonSequenceBackend::triton-sequence-backend ALIAS triton-sequence-backend ) target_compile_features(triton-sequence-backend PRIVATE cxx_std_11) target_compile_options( triton-sequence-backend PRIVATE $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>: -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror> ) target_link_libraries( triton-sequence-backend PRIVATE triton-backend-utils # from repo-backend triton-core-serverapi # from repo-core triton-core-backendapi # from repo-core triton-core-serverstub # from repo-core ) set_target_properties( triton-sequence-backend PROPERTIES POSITION_INDEPENDENT_CODE ON OUTPUT_NAME triton_sequence LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_sequence.ldscript LINK_FLAGS "-Wl,--version-script libtriton_sequence.ldscript" ) # # Install # include(GNUInstallDirs) set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonSequenceBackend) install( TARGETS triton-sequence-backend EXPORT triton-sequence-backend-targets LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/sequence ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/sequence ) install( EXPORT triton-sequence-backend-targets FILE TritonSequenceBackendTargets.cmake NAMESPACE TritonSequenceBackend:: DESTINATION ${INSTALL_CONFIGDIR} ) include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonSequenceBackendConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TritonSequenceBackendConfig.cmake INSTALL_DESTINATION ${INSTALL_CONFIGDIR} ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/TritonSequenceBackendConfig.cmake DESTINATION ${INSTALL_CONFIGDIR} ) # # Export from build tree # export( EXPORT triton-sequence-backend-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonSequenceBackendTargets.cmake NAMESPACE TritonSequenceBackend:: ) export(PACKAGE TritonSequenceBackend) ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/sequence/cmake/����������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020273�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/sequence/cmake/TritonSequenceBackendConfig.cmake.in����������������������0000664�0000000�0000000�00000003661�15270523712�0027256�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include(CMakeFindDependencyMacro) get_filename_component( TRITONSEQUENCEBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) list(APPEND CMAKE_MODULE_PATH ${TRITONSEQUENCEBACKEND_CMAKE_DIR}) if(NOT TARGET TritonSequenceBackend::triton-sequence-backend) include("${TRITONSEQUENCEBACKEND_CMAKE_DIR}/TritonSequenceBackendTargets.cmake") endif() set(TRITONSEQUENCEBACKEND_LIBRARIES TritonSequenceBackend::triton-sequence-backend) �������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/sequence/src/������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020002�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/sequence/src/libtriton_sequence.ldscript���������������������������������0000664�0000000�0000000�00000003055�15270523712�0025451�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { global: TRITONBACKEND_*; local: *; }; �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/test/sequence/src/sequence.cc�������������������������������������������������0000664�0000000�0000000�00000107732�15270523712�0022133�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <memory> #include <thread> #include "triton/backend/backend_common.h" #include "triton/backend/backend_model.h" #include "triton/backend/backend_model_instance.h" namespace triton { namespace backend { namespace sequence { // Simple sequence backend that demonstrates the TRITONBACKEND API for a // blocking backend. A blocking backend completes execution of the inference // before returning from TRITONBACKEND_ModelInstanceExecute. // // The backend supports models that take three input tensors, two INT32 [ 1 ] // control values and one variable-size INT32 [ -1 ] value input; and produces // an output tensor with the same shape as the input tensor. The input tensors // must be named "START", "READY" and "INPUT". The output tensor must be named // "OUTPUT". // // The model maintains an INT32 accumulator which is updated based on the // control values in "START" and "READY": // // READY=0, START=x: Ignore value input, do not change accumulator value. // // READY=1, START=1: Start accumulating. Set accumulator equal to sum of input // tensor. // // READY=1, START=0: Add input tensor values to accumulator. // // When READY=1, the accumulator is returned in every element of the output. // #define GUARDED_RESPOND_IF_ERROR(RESPONSES, IDX, X) \ do { \ if ((RESPONSES)[IDX] != nullptr) { \ TRITONSERVER_Error* err__ = (X); \ if (err__ != nullptr) { \ LOG_IF_ERROR( \ TRITONBACKEND_ResponseSend( \ (RESPONSES)[IDX], TRITONSERVER_RESPONSE_COMPLETE_FINAL, \ err__), \ "failed to send error response"); \ (RESPONSES)[IDX] = nullptr; \ TRITONSERVER_ErrorDelete(err__); \ } \ } \ } while (false) // // ModelState // // State associated with a model that is using this backend. An object // of this class is created and associated with each // TRITONBACKEND_Model. // class ModelState : public BackendModel { public: static TRITONSERVER_Error* Create( TRITONBACKEND_Model* triton_model, ModelState** state); virtual ~ModelState() = default; // Get accumulator size and execution delay size_t AccumulatorSize() const { return accumulator_size_; } int ExecDelay() const { return execute_delay_ms_; } // Validate that model configuration is supported by this backend. TRITONSERVER_Error* ValidateModelConfig(); private: ModelState(TRITONBACKEND_Model* triton_model); // Delay to introduce into execution, in milliseconds. int execute_delay_ms_; // Accumulator size size_t accumulator_size_; }; TRITONSERVER_Error* ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state) { try { *state = new ModelState(triton_model); } catch (const BackendModelException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelState::ModelState(TRITONBACKEND_Model* triton_model) : BackendModel(triton_model), execute_delay_ms_(0) { } TRITONSERVER_Error* ModelState::ValidateModelConfig() { // We have the json DOM for the model configuration... common::TritonJson::WriteBuffer buffer; RETURN_IF_ERROR(model_config_.PrettyWrite(&buffer)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model configuration:\n") + buffer.Contents()).c_str()); triton::common::TritonJson::Value params; if (model_config_.Find("parameters", ¶ms)) { common::TritonJson::Value exec_delay; if (params.Find("execute_delay_ms", &exec_delay)) { std::string exec_delay_str; RETURN_IF_ERROR( exec_delay.MemberAsString("string_value", &exec_delay_str)); execute_delay_ms_ = std::stoi(exec_delay_str); } } int64_t max_batch_size = 0; RETURN_IF_ERROR(model_config_.MemberAsInt("max_batch_size", &max_batch_size)); accumulator_size_ = (size_t)(std::max((int64_t)1, max_batch_size)); // The model configuration must specify the sequence batcher and must use the // START and READY input to indicate control values. triton::common::TritonJson::Value sequence_batching; RETURN_IF_ERROR( model_config_.MemberAsObject("sequence_batching", &sequence_batching)); common::TritonJson::Value control_inputs; RETURN_IF_ERROR( sequence_batching.MemberAsArray("control_input", &control_inputs)); RETURN_ERROR_IF_FALSE( control_inputs.ArraySize() == 2, TRITONSERVER_ERROR_INVALID_ARG, std::string( "'START' and 'READY' must be configured as the control inputs")); std::vector<std::string> control_input_names; for (size_t io_index = 0; io_index < control_inputs.ArraySize(); io_index++) { common::TritonJson::Value control_input; RETURN_IF_ERROR(control_inputs.IndexAsObject(io_index, &control_input)); const char* input_name; size_t input_name_len; RETURN_IF_ERROR( control_input.MemberAsString("name", &input_name, &input_name_len)); control_input_names.push_back(input_name); } RETURN_ERROR_IF_FALSE( ((control_input_names[0] == "START") && (control_input_names[1] == "READY")) || ((control_input_names[0] == "READY") && (control_input_names[1] == "START")), TRITONSERVER_ERROR_INVALID_ARG, std::string( "'START' and 'READY' must be configured as the control inputs")); common::TritonJson::Value inputs, outputs; RETURN_IF_ERROR(model_config_.MemberAsArray("input", &inputs)); RETURN_IF_ERROR(model_config_.MemberAsArray("output", &outputs)); // There must be one INT32 input called INPUT defined in the model // configuration and it must be a 1D vector (of any length). RETURN_ERROR_IF_FALSE( inputs.ArraySize() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string( "model must have input 'INPUT' with vector shape, any length")); common::TritonJson::Value input; RETURN_IF_ERROR(inputs.IndexAsObject(0 /* index */, &input)); std::vector<int64_t> input_shape; RETURN_IF_ERROR(backend::ParseShape(input, "dims", &input_shape)); RETURN_ERROR_IF_FALSE( input_shape.size() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string( "model must have one input 'INPUT' with vector shape, any length")); std::string input_dtype; RETURN_IF_ERROR(input.MemberAsString("data_type", &input_dtype)); RETURN_ERROR_IF_FALSE( input_dtype == "TYPE_INT32", TRITONSERVER_ERROR_INVALID_ARG, std::string("model input must have TYPE_INT32 data-type")); const char* input_name; size_t input_name_len; RETURN_IF_ERROR(input.MemberAsString("name", &input_name, &input_name_len)); RETURN_ERROR_IF_FALSE( strcmp(input_name, "INPUT") == 0, TRITONSERVER_ERROR_INVALID_ARG, std::string("model input must be named 'INPUT'")); // There must be one INT32 output with shape that matches the input. The // output must be named OUTPUT. RETURN_ERROR_IF_FALSE( outputs.ArraySize() == 1, TRITONSERVER_ERROR_INVALID_ARG, std::string( "model must have one output 'OUTPUT' with vector shape, any length")); common::TritonJson::Value output; RETURN_IF_ERROR(outputs.IndexAsObject(0 /* index */, &output)); std::vector<int64_t> output_shape; RETURN_IF_ERROR(backend::ParseShape(output, "dims", &output_shape)); RETURN_ERROR_IF_FALSE( (output_shape.size() == 1) && (output_shape[0] == input_shape[0]), TRITONSERVER_ERROR_INVALID_ARG, std::string( "model must have output 'OUTPUT' with shape matching 'INPUT'")); std::string output_dtype; RETURN_IF_ERROR(output.MemberAsString("data_type", &output_dtype)); RETURN_ERROR_IF_FALSE( output_dtype == "TYPE_INT32", TRITONSERVER_ERROR_INVALID_ARG, std::string("model output must have TYPE_INT32 data-type")); const char* output_name; size_t output_name_len; RETURN_IF_ERROR( output.MemberAsString("name", &output_name, &output_name_len)); RETURN_ERROR_IF_FALSE( strcmp(output_name, "OUTPUT") == 0, TRITONSERVER_ERROR_INVALID_ARG, std::string("model output must be named 'OUTPUT'")); return nullptr; // success } // // ModelInstanceState // // State associated with a model instance. An object of this class is // created and associated with each TRITONBACKEND_ModelInstance. // class ModelInstanceState : public BackendModelInstance { public: static TRITONSERVER_Error* Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state); virtual ~ModelInstanceState(); // Get the state of the model that corresponds to this instance. ModelState* StateForModel() const { return model_state_; } // Get accumulator for this instance int32_t GetAccumulatorAt(size_t idx); void SetAccumulatorAt(size_t idx, int32_t value); void AddAccumulatorAt(size_t idx, int32_t value); private: ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance); ModelState* model_state_; // Accumulators maintained by this instance, one for each batch slot. std::vector<int32_t> accumulator_; }; TRITONSERVER_Error* ModelInstanceState::Create( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance, ModelInstanceState** state) { try { *state = new ModelInstanceState(model_state, triton_model_instance); } catch (const BackendModelInstanceException& ex) { RETURN_ERROR_IF_TRUE( ex.err_ == nullptr, TRITONSERVER_ERROR_INTERNAL, std::string("unexpected nullptr in BackendModelInstanceException")); RETURN_IF_ERROR(ex.err_); } return nullptr; // success } ModelInstanceState::ModelInstanceState( ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(model_state, triton_model_instance), model_state_(model_state) { accumulator_.resize(model_state->AccumulatorSize()); } int32_t ModelInstanceState::GetAccumulatorAt(size_t idx) { return accumulator_[idx]; } void ModelInstanceState::SetAccumulatorAt(size_t idx, int32_t value) { accumulator_[idx] = value; } void ModelInstanceState::AddAccumulatorAt(size_t idx, int32_t value) { accumulator_[idx] += value; } ModelInstanceState::~ModelInstanceState() { accumulator_.clear(); } ///////////// extern "C" { // Implementing TRITONBACKEND_Initialize is optional. The backend // should initialize any global state that is intended to be shared // across all models and model instances that use the backend. TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_BackendName(backend, &cname)); std::string name(cname); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); // We should check the backend API version that Triton supports // vs. what this backend was compiled against. uint32_t api_version_major, api_version_minor; RETURN_IF_ERROR( TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("Triton TRITONBACKEND API version: ") + std::to_string(api_version_major) + "." + std::to_string(api_version_minor)) .c_str()); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("'") + name + "' TRITONBACKEND API version: " + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) .c_str()); if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) || (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "triton backend API version does not support this backend"); } // The backend configuration may contain information needed by the // backend, such a command-line arguments. This backend doesn't use // any such configuration but we print whatever is available. TRITONSERVER_Message* backend_config_message; RETURN_IF_ERROR( TRITONBACKEND_BackendConfig(backend, &backend_config_message)); const char* buffer; size_t byte_size; RETURN_IF_ERROR(TRITONSERVER_MessageSerializeToJson( backend_config_message, &buffer, &byte_size)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("backend configuration:\n") + buffer).c_str()); return nullptr; // success } // Implementing TRITONBACKEND_ModelInitialize is optional. The backend // should initialize any state that is intended to be shared across // all instances of the model. TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelName(model, &cname)); std::string name(cname); uint64_t version; RETURN_IF_ERROR(TRITONBACKEND_ModelVersion(model, &version)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); // With each model we create a ModelState object and associate it // with the TRITONBACKEND_Model. ModelState* model_state; RETURN_IF_ERROR(ModelState::Create(model, &model_state)); RETURN_IF_ERROR( TRITONBACKEND_ModelSetState(model, reinterpret_cast<void*>(model_state))); // One of the primary things to do in ModelInitialize is to examine // the model configuration to ensure that it is something that this // backend can support. If not, returning an error from this // function will prevent the model from loading. RETURN_IF_ERROR(model_state->ValidateModelConfig()); return nullptr; // success } // Implementing TRITONBACKEND_ModelFinalize is optional unless state // is set using TRITONBACKEND_ModelSetState. The backend must free // this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceInitialize is optional. The // backend should initialize any state that is required for a model // instance. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) { const char* cname; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceName(instance, &cname)); std::string name(cname); int32_t device_id; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceDeviceId(instance, &device_id)); TRITONSERVER_InstanceGroupKind kind; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceKind(instance, &kind)); LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + " device " + std::to_string(device_id) + ")") .c_str()); // The instance can access the corresponding model as well... here // we get the model and from that get the model's state. TRITONBACKEND_Model* model; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceModel(instance, &model)); void* vmodelstate; RETURN_IF_ERROR(TRITONBACKEND_ModelState(model, &vmodelstate)); ModelState* model_state = reinterpret_cast<ModelState*>(vmodelstate); // With each instance we create a ModelInstanceState object and // associate it with the TRITONBACKEND_ModelInstance. ModelInstanceState* instance_state; RETURN_IF_ERROR( ModelInstanceState::Create(model_state, instance, &instance_state)); RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceSetState( instance, reinterpret_cast<void*>(instance_state))); // Because this backend just copies IN -> OUT and requires that // input and output be in CPU memory, we fail if a GPU instances is // requested. RETURN_ERROR_IF_FALSE( instance_state->Kind() == TRITONSERVER_INSTANCEGROUPKIND_CPU, TRITONSERVER_ERROR_INVALID_ARG, std::string("'sequence' backend only supports CPU instances")); return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceFinalize is optional unless // state is set using TRITONBACKEND_ModelInstanceSetState. The backend // must free this state and perform any other cleanup. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) { void* vstate; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(instance, &vstate)); ModelInstanceState* instance_state = reinterpret_cast<ModelInstanceState*>(vstate); LOG_MESSAGE( TRITONSERVER_LOG_INFO, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; return nullptr; // success } // Implementing TRITONBACKEND_ModelInstanceExecute is required. TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests, const uint32_t request_count) { // Triton will not call this function simultaneously for the same // 'instance'. But since this backend could be used by multiple // instances from multiple models the implementation needs to handle // multiple calls to this function at the same time (with different // 'instance' objects). Suggested practice for this is to use only // function-local and model-instance-specific state (obtained from // 'instance'), which is what we do here. ModelInstanceState* instance_state; RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState( instance, reinterpret_cast<void**>(&instance_state))); ModelState* model_state = instance_state->StateForModel(); // This backend specifies BLOCKING execution policy. That means that // we should not return from this function until execution is complete. Triton // will automatically release 'instance' on return from this function so that // it is again available to be used for another call to // TRITONBACKEND_ModelInstanceExecute. LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("model ") + model_state->Name() + ", instance " + instance_state->Name() + ", executing " + std::to_string(request_count) + " requests") .c_str()); bool supports_batching = false; RETURN_IF_ERROR(model_state->SupportsFirstDimBatching(&supports_batching)); // Each request represents a different sequence, which corresponds // to the accumulator at the same index. Each request must have // batch-size 1 inputs which is the next timestep for that sequence. The total // number of requests will not exceed the max-batch-size specified in the // model configuration. if (request_count > model_state->AccumulatorSize()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "unable to execute batch larger than max-batch-size"); } // Delay if requested... if (model_state->ExecDelay() > 0) { std::this_thread::sleep_for( std::chrono::milliseconds(model_state->ExecDelay())); } // 'responses' is initialized with the response objects below and // if/when an error response is sent the corresponding entry in // 'responses' is set to nullptr to indicate that that response has // already been sent. std::vector<TRITONBACKEND_Response*> responses; responses.reserve(request_count); // Create a single response object for each request. If something // goes wrong when attempting to create the response objects just // fail all of the requests by returning an error. for (uint32_t r = 0; r < request_count; ++r) { TRITONBACKEND_Request* request = requests[r]; TRITONBACKEND_Response* response; RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&response, request)); responses.push_back(response); } // After this point we take ownership of 'requests', which means that a // response must be sent for every request. If something does go wrong in // processing a particular request then we send an error response just for the // specific request. // The way we collect these batch timestamps is not entirely accurate. // Normally, in a performant backend you would execute all the requests at the // same time, and so there would be a single compute-start / compute-end // time-range. But here we execute each request separately so there is no // single range. As a result we just show the entire execute time as being the // compute time as well. uint64_t min_exec_start_ns = std::numeric_limits<uint64_t>::max(); uint64_t max_exec_end_ns = 0; uint64_t total_batch_size = 0; // For simplicity we just process each request separately... in // general a backend should try to operate on the entire batch of // requests at the same time for improved performance. std::vector<uint8_t> start_buffer, ready_buffer, input_buffer; for (uint32_t r = 0; r < request_count; ++r) { uint64_t exec_start_ns = 0; SET_TIMESTAMP(exec_start_ns); min_exec_start_ns = std::min(min_exec_start_ns, exec_start_ns); TRITONBACKEND_Request* request = requests[r]; const char* request_id = ""; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestId(request, &request_id)); uint64_t correlation_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestCorrelationId(request, &correlation_id)); uint32_t input_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInputCount(request, &input_count)); uint32_t requested_output_count = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestOutputCount(request, &requested_output_count)); // If an error response was sent for the above then display an error // message and move on to next request. if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read request input/output counts, error response " "sent") .c_str()); continue; } LOG_MESSAGE( TRITONSERVER_LOG_INFO, (std::string("request ") + std::to_string(r) + ": id = \"" + request_id + "\", correlation_id = " + std::to_string(correlation_id) + ", input_count = " + std::to_string(input_count) + ", requested_output_count = " + std::to_string(requested_output_count)) .c_str()); // For statistics we need to collect the total batch size of all the // requests. If the model doesn't support batching then each request is // necessarily batch-size 1. If the model does support batching then the // first dimension of the shape is the batch size. We only the first input // for this. if (supports_batching) { TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInputByIndex(request, 0 /* index */, &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input, error response sent") .c_str()); continue; } const int64_t* input_shape; uint32_t input_dims_count; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputProperties( input, nullptr, nullptr, &input_shape, &input_dims_count, nullptr, nullptr)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input properties, error response sent") .c_str()); continue; } if (input_dims_count > 0) { if (input_shape[0] != 1) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "unable to execute more than one timestep at a time")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": unable to execute more than one timestep at a time, error " "response sent") .c_str()); continue; } total_batch_size += input_shape[0]; } } else { total_batch_size++; } LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("total_batch_size: ") + std::to_string(total_batch_size)) .c_str()); // Get the input tensors. TRITONBACKEND_Input* start_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "START", &start_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'START', error response sent") .c_str()); continue; } TRITONBACKEND_Input* ready_input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "READY", &ready_input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input 'READY', error response sent") .c_str()); continue; } const void* start_buffer = nullptr; uint64_t buffer_byte_size = 0; TRITONSERVER_MemoryType input_memory_type = TRITONSERVER_MEMORY_CPU; int64_t input_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( start_input, 0 /* input_buffer_count */, &start_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } const void* ready_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( ready_input, 0 /* input_buffer_count */, &ready_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer, error " "response sent") .c_str()); continue; } TRITONBACKEND_Input* input = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_RequestInput(request, "INPUT", &input)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input, error response sent") .c_str()); continue; } const void* input_buffer = nullptr; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputBuffer( input, 0 /* input_buffer_count */, &input_buffer, &buffer_byte_size, &input_memory_type, &input_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to get input buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to get input buffer in CPU memory, error " "response sent") .c_str()); continue; } TRITONSERVER_DataType input_datatype; const int64_t* input_shape; uint32_t input_dims_count; uint64_t input_byte_size; uint32_t input_buffer_count; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_InputProperties( input, nullptr /* input_name */, &input_datatype, &input_shape, &input_dims_count, &input_byte_size, &input_buffer_count)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to read input properties, error response sent") .c_str()); continue; } int64_t input_element_cnt = input_byte_size / sizeof(int32_t); const int32_t* start = reinterpret_cast<const int32_t*>(start_buffer); const int32_t* ready = reinterpret_cast<const int32_t*>(ready_buffer); const int32_t* ipbuffer_int = nullptr; std::vector<int32_t> ipbuffer_vec; if (input_memory_type == TRITONSERVER_MEMORY_GPU) { ipbuffer_vec.resize(input_element_cnt); ipbuffer_int = ipbuffer_vec.data(); cudaMemcpy( const_cast<int32_t*>(ipbuffer_int), input_buffer, input_byte_size, cudaMemcpyDeviceToHost); } else { ipbuffer_int = reinterpret_cast<const int32_t*>(input_buffer); } // Update the accumulator value based on START/READY and calculate the // output value. if (ready[0] != 0) { if (start[0] == 0) { // Update accumulator. for (int64_t e = 0; e < input_element_cnt; ++e) { instance_state->AddAccumulatorAt(r, ipbuffer_int[e]); } } else { // Set accumulator. instance_state->SetAccumulatorAt(r, ipbuffer_int[0]); for (int64_t e = 1; e < input_element_cnt; ++e) { instance_state->AddAccumulatorAt(r, ipbuffer_int[e]); } } TRITONBACKEND_Response* response = responses[r]; // If the output is requested, copy the calculated output value // into the output buffer. if (requested_output_count > 0) { // The output shape is [1, input_element_cnt] if the model configuration // supports batching, or just [input_element_cnt] if the model // configuration does not support batching. std::vector<int64_t> shape; if (supports_batching) { shape.push_back(1); } shape.push_back(input_element_cnt); TRITONBACKEND_Output* output; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_ResponseOutput( response, &output, "OUTPUT", input_datatype, input_shape, input_dims_count)); if (responses[r] == nullptr) { LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create response output, error response sent") .c_str()); continue; } // Get the output buffer. We request a buffer in CPU memory but we have // to handle any returned type. If we get back a buffer in GPU memory we // just fail the request. void* output_buffer; TRITONSERVER_MemoryType output_memory_type = TRITONSERVER_MEMORY_CPU; int64_t output_memory_type_id = 0; GUARDED_RESPOND_IF_ERROR( responses, r, TRITONBACKEND_OutputBuffer( output, &output_buffer, buffer_byte_size, &output_memory_type, &output_memory_type_id)); if (responses[r] == nullptr) { GUARDED_RESPOND_IF_ERROR( responses, r, TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, "failed to create output buffer in CPU memory")); LOG_MESSAGE( TRITONSERVER_LOG_ERROR, (std::string("request ") + std::to_string(r) + ": failed to create output buffer in CPU memory, error " "response sent") .c_str()); continue; } int32_t* obuffer_int = nullptr; std::vector<int32_t> obuffer_vec; if (output_memory_type == TRITONSERVER_MEMORY_GPU) { obuffer_vec.resize(buffer_byte_size / sizeof(int32_t)); obuffer_int = obuffer_vec.data(); } else { obuffer_int = reinterpret_cast<int32_t*>(output_buffer); } for (int64_t i = 0; i < input_element_cnt; ++i) { obuffer_int[i] = instance_state->GetAccumulatorAt(r); } if (output_memory_type == TRITONSERVER_MEMORY_GPU) { cudaMemcpy( output_buffer, const_cast<int32_t*>(obuffer_int), buffer_byte_size, cudaMemcpyHostToDevice); } } } uint64_t exec_end_ns = 0; SET_TIMESTAMP(exec_end_ns); max_exec_end_ns = std::max(max_exec_end_ns, exec_end_ns); // Send all the responses that haven't already been sent because of an // earlier error. if (responses[r] != nullptr) { LOG_IF_ERROR( TRITONBACKEND_ResponseSend( responses[r], TRITONSERVER_RESPONSE_COMPLETE_FINAL, nullptr /* success */), "failed sending response"); } // Report statistics for each request. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportStatistics( instance_state->TritonModelInstance(), request, (responses[r] != nullptr) /* success */, exec_start_ns, exec_start_ns, exec_end_ns, exec_end_ns), "failed reporting request statistics"); LOG_IF_ERROR( TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL), "failed releasing request"); } // Report the entire batch statistics. LOG_IF_ERROR( TRITONBACKEND_ModelInstanceReportBatchStatistics( instance_state->TritonModelInstance(), total_batch_size, min_exec_start_ns, min_exec_start_ns, max_exec_end_ns, max_exec_end_ns), "failed reporting batch request statistics"); return nullptr; // success } } // extern "C" }}} // namespace triton::backend::sequence ��������������������������������������kytensor-2.49.0.7/src/tracer.cc���������������������������������������������������������������������0000664�0000000�0000000�00000127023�15270523712�0016220�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "tracer.h" #include <stdlib.h> #include "common.h" #include "triton/common/logging.h" #ifdef TRITON_ENABLE_GPU #include <cuda_runtime_api.h> #endif // TRITON_ENABLE_GPU #ifndef _WIN32 #include "opentelemetry/sdk/resource/semantic_conventions.h" #include "opentelemetry/sdk/trace/batch_span_processor_factory.h" namespace otel_common = opentelemetry::common; #endif namespace triton { namespace server { TRITONSERVER_Error* TraceManager::Create( TraceManager** manager, const TRITONSERVER_InferenceTraceLevel level, const uint32_t rate, const int32_t count, const uint32_t log_frequency, const std::string& filepath, const InferenceTraceMode mode, const triton::server::TraceConfigMap& config_map) { // Always create TraceManager regardless of the global setting as they // can be updated at runtime even if tracing is not enable at start. // No trace should be sampled if the setting is not valid. *manager = new TraceManager( level, rate, count, log_frequency, filepath, mode, config_map); return nullptr; // success } TraceManager::TraceManager( const TRITONSERVER_InferenceTraceLevel level, const uint32_t rate, const int32_t count, const uint32_t log_frequency, const std::string& filepath, const InferenceTraceMode mode, const TraceConfigMap& config_map) { std::shared_ptr<TraceFile> file(new TraceFile(filepath)); global_default_.reset(new TraceSetting( level, rate, count, log_frequency, file, mode, config_map, false /*level_specified*/, false /*rate_specified*/, false /*count_specified*/, false /*log_frequency_specified*/, false /*filepath_specified*/, false /*mode_specified*/, false /*config_map_specified*/)); global_setting_.reset(new TraceSetting( level, rate, count, log_frequency, file, mode, config_map, false /*level_specified*/, false /*rate_specified*/, false /*count_specified*/, false /*log_frequency_specified*/, false /*filepath_specified*/, false /*mode_specified*/, false /*config_map_specified*/)); trace_files_.emplace(filepath, file); InitTracer(config_map); } TRITONSERVER_Error* TraceManager::UpdateTraceSetting( const std::string& model_name, const NewSetting& new_setting) { std::lock_guard<std::mutex> w_lk(w_mu_); RETURN_IF_ERR(UpdateTraceSettingInternal(model_name, new_setting)); // If updating global setting, must check and update the model settings // that are (partially) mirroring global setting. if (model_name.empty()) { // Default constructed setting means no active update, // only the unspecified fields will be checked and updated. NewSetting setting; // Make a copy of the set as UpdateTraceSettingInternal() may modify // 'fallback_used_models_' auto fallback_models = fallback_used_models_; for (const auto& name : fallback_models) { RETURN_IF_ERR(UpdateTraceSettingInternal(name, setting)); } } return nullptr; } TRITONSERVER_Error* TraceManager::UpdateTraceSettingInternal( const std::string& model_name, const NewSetting& new_setting) { // First try to get the current setting and fallback setting, // current setting may be 'nullptr' if the setting is newly added const TraceSetting* current_setting = nullptr; const TraceSetting* fallback_setting = nullptr; if (!model_name.empty()) { auto it = model_settings_.find(model_name); if (it != model_settings_.end()) { current_setting = it->second.get(); } fallback_setting = global_setting_.get(); } else { current_setting = global_setting_.get(); fallback_setting = global_default_.get(); } // Prepare the updated setting, use two passes for simplicity: // 1. Set all fields based on 'fallback_setting' // 2. If there are specified fields based on current and new setting, // use the specified value TRITONSERVER_InferenceTraceLevel level = fallback_setting->level_; uint32_t rate = fallback_setting->rate_; int32_t count = fallback_setting->count_; uint32_t log_frequency = fallback_setting->log_frequency_; std::string filepath = fallback_setting->file_->FileName(); InferenceTraceMode mode = fallback_setting->mode_; TraceConfigMap config_map = fallback_setting->config_map_; // Whether the field value is specified: // if clear then it is not specified, otherwise, // it is specified if it is being updated, or it was previously specified const bool level_specified = (new_setting.clear_level_ ? false : (((current_setting != nullptr) && current_setting->level_specified_) || (new_setting.level_ != nullptr))); const bool rate_specified = (new_setting.clear_rate_ ? false : (((current_setting != nullptr) && current_setting->rate_specified_) || (new_setting.rate_ != nullptr))); const bool count_specified = (new_setting.clear_count_ ? false : (((current_setting != nullptr) && current_setting->count_specified_) || (new_setting.count_ != nullptr))); const bool log_frequency_specified = (new_setting.clear_log_frequency_ ? false : (((current_setting != nullptr) && current_setting->log_frequency_specified_) || (new_setting.log_frequency_ != nullptr))); const bool filepath_specified = (((current_setting != nullptr) && current_setting->filepath_specified_)); if (level_specified) { level = (new_setting.level_ != nullptr) ? *new_setting.level_ : current_setting->level_; } if (rate_specified) { rate = (new_setting.rate_ != nullptr) ? *new_setting.rate_ : current_setting->rate_; } if (count_specified) { count = (new_setting.count_ != nullptr) ? *new_setting.count_ : current_setting->count_; } if (log_frequency_specified) { log_frequency = (new_setting.log_frequency_ != nullptr) ? *new_setting.log_frequency_ : current_setting->log_frequency_; } if (filepath_specified) { filepath = current_setting->file_->FileName(); } // Some special case when updating model setting if (!model_name.empty()) { bool all_specified = (level_specified & rate_specified & count_specified & log_frequency_specified & filepath_specified); bool none_specified = !(level_specified | rate_specified | count_specified | log_frequency_specified | filepath_specified); if (all_specified) { fallback_used_models_.erase(model_name); } else if (none_specified) { // Simply let the model uses global setting std::lock_guard<std::mutex> r_lk(r_mu_); model_settings_.erase(model_name); return nullptr; } else { fallback_used_models_.emplace(model_name); } } // Create TraceSetting object with the updated setting std::shared_ptr<TraceFile> file; const auto it = trace_files_.find(filepath); if (it != trace_files_.end()) { file = it->second.lock(); // The TraceFile object is no longer valid if (file == nullptr) { trace_files_.erase(it); } } if (file == nullptr) { file.reset(new TraceFile(filepath)); trace_files_.emplace(filepath, file); } std::shared_ptr<TraceSetting> lts(new TraceSetting( level, rate, count, log_frequency, file, mode, config_map, level_specified, rate_specified, count_specified, log_frequency_specified, filepath_specified, false /*mode_specified*/, false /*config_map_specified*/)); // The only invalid setting allowed is if it disables tracing if ((!lts->Valid()) && (level != TRITONSERVER_TRACE_LEVEL_DISABLED)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Attempting to set invalid trace setting :") + lts->Reason()) .c_str()); } // Update / Init the setting in read lock to exclude reader access, // we replace the object instead of modifying the existing object in case // of there are ongoing traces. This makes sure those traces are referring // to the setting when the traces are sampled. { std::lock_guard<std::mutex> r_lk(r_mu_); if (model_name.empty()) { // global update global_setting_ = std::move(lts); } else { auto it = model_settings_.find(model_name); if (it != model_settings_.end()) { // Model update it->second = std::move(lts); } else { // Model init model_settings_.emplace(model_name, lts); } } } return nullptr; } void TraceManager::GetTraceSetting( const std::string& model_name, TRITONSERVER_InferenceTraceLevel* level, uint32_t* rate, int32_t* count, uint32_t* log_frequency, std::string* filepath, InferenceTraceMode* trace_mode, TraceConfigMap* config_map) { std::shared_ptr<TraceSetting> trace_setting; { std::lock_guard<std::mutex> r_lk(r_mu_); auto m_it = model_settings_.find(model_name); trace_setting = (m_it == model_settings_.end()) ? global_setting_ : m_it->second; } *level = trace_setting->level_; *rate = trace_setting->rate_; *count = trace_setting->count_; *log_frequency = trace_setting->log_frequency_; *filepath = trace_setting->file_->FileName(); *trace_mode = trace_setting->mode_; *config_map = trace_setting->config_map_; } void TraceManager::GetTraceSetting( const std::string& model_name, std::shared_ptr<TraceSetting>& trace_setting) { std::lock_guard<std::mutex> r_lk(r_mu_); auto m_it = model_settings_.find(model_name); trace_setting = (m_it == model_settings_.end()) ? global_setting_ : m_it->second; } TraceManager::TraceStartOptions TraceManager::GetTraceStartOptions( AbstractCarrier& carrier, const std::string& model_name) { TraceManager::TraceStartOptions start_options; GetTraceSetting(model_name, start_options.trace_setting); if (!start_options.trace_setting->level_ == TRITONSERVER_TRACE_LEVEL_DISABLED && start_options.trace_setting->mode_ == TRACE_MODE_OPENTELEMETRY) { #ifndef _WIN32 auto prop = otel_cntxt::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); auto ctxt = otel_cntxt::Context(); ctxt = prop->Extract(carrier, ctxt); otel_trace_api::SpanContext span_context = otel_trace_api::GetSpan(ctxt)->GetContext(); if (span_context.IsValid()) { start_options.propagated_context = ctxt; start_options.force_sample = true; } #else LOG_ERROR << "Unsupported trace mode: " << TraceManager::InferenceTraceModeString( start_options.trace_setting->mode_); #endif // _WIN32 } return start_options; } std::shared_ptr<TraceManager::Trace> TraceManager::SampleTrace(const TraceStartOptions& start_options) { std::shared_ptr<Trace> ts = start_options.trace_setting->SampleTrace(start_options.force_sample); if (ts != nullptr) { ts->setting_ = start_options.trace_setting; if (ts->setting_->mode_ == TRACE_MODE_OPENTELEMETRY) { #ifndef _WIN32 auto steady_timestamp_ns = std::chrono::duration_cast<std::chrono::nanoseconds>( std::chrono::steady_clock::now().time_since_epoch()) .count(); if (ts->span_stacks_.find(ts->trace_id_) == ts->span_stacks_.end()) { std::unique_ptr< std::stack<opentelemetry::nostd::shared_ptr<otel_trace_api::Span>>> st(new std::stack< opentelemetry::nostd::shared_ptr<otel_trace_api::Span>>()); ts->span_stacks_.emplace(ts->trace_id_, std::move(st)); } auto active_span = otel_trace_api::GetSpan(start_options.propagated_context); if (active_span->GetContext().IsValid()) { ts->span_stacks_[ts->trace_id_]->emplace(active_span); } // Storing "InferRequest" span as a root span // to keep it alive for the duration of the request. ts->root_span_ = ts->StartSpan("InferRequest", steady_timestamp_ns, ts->trace_id_); ts->span_stacks_[ts->trace_id_]->emplace(ts->root_span_); #else LOG_ERROR << "Unsupported trace mode: " << TraceManager::InferenceTraceModeString(ts->setting_->mode_); #endif } } return ts; } TraceManager::Trace::~Trace() { if (setting_->mode_ == TRACE_MODE_TRITON) { // Write trace now setting_->WriteTrace(streams_); } else if (setting_->mode_ == TRACE_MODE_OPENTELEMETRY) { #ifndef _WIN32 EndSpan(trace_id_); #else LOG_ERROR << "Unsupported trace mode: " << TraceManager::InferenceTraceModeString(setting_->mode_); #endif } } void TraceManager::Trace::CaptureTimestamp( const std::string& name, uint64_t timestamp_ns) { if (setting_->level_ & TRITONSERVER_TRACE_LEVEL_TIMESTAMPS) { if (setting_->mode_ == TRACE_MODE_TRITON) { std::lock_guard<std::mutex> lk(mtx_); std::stringstream* ss = nullptr; { if (streams_.find(trace_id_) == streams_.end()) { std::unique_ptr<std::stringstream> stream(new std::stringstream()); ss = stream.get(); streams_.emplace(trace_id_, std::move(stream)); } else { ss = streams_[trace_id_].get(); // If the string stream is not newly created, add "," as there is // already content in the string stream *ss << ","; } } *ss << "{\"id\":" << trace_id_ << ",\"timestamps\":[" << "{\"name\":\"" << name << "\",\"ns\":" << timestamp_ns << "}]}"; } else if (setting_->mode_ == TRACE_MODE_OPENTELEMETRY) { #ifndef _WIN32 root_span_->AddEvent( name, time_offset_ + std::chrono::nanoseconds{timestamp_ns}); #else LOG_ERROR << "Unsupported trace mode: " << TraceManager::InferenceTraceModeString(setting_->mode_); #endif } } } std::string TraceManager::Trace::RetrieveActivityName( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns) { std::string activity_name = TRITONSERVER_InferenceTraceActivityString(activity); if (activity == TRITONSERVER_TRACE_CUSTOM_ACTIVITY) { const char* val = nullptr; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceContext(trace, &val), "Failed to retrieve trace context"); std::string context_str = (val != nullptr) ? std::string(val) : ""; triton::common::TritonJson::Value context; LOG_TRITONSERVER_ERROR( context.Parse(context_str), "Failed to parse trace context"); std::string look_for_key = std::to_string(timestamp_ns); if (context.Find(look_for_key.c_str())) { context.MemberAsString(look_for_key.c_str(), &activity_name); } } return activity_name; } void TraceManager::InitTracer(const triton::server::TraceConfigMap& config_map) { switch (global_setting_->mode_) { case TRACE_MODE_OPENTELEMETRY: { #ifndef _WIN32 otlp::OtlpHttpExporterOptions exporter_options; otel_resource::ResourceAttributes attributes = {}; otel_trace_sdk::BatchSpanProcessorOptions processor_options; ProcessOpenTelemetryParameters( config_map, exporter_options, attributes, processor_options); auto exporter = otlp::OtlpHttpExporterFactory::Create(exporter_options); auto processor = otel_trace_sdk::BatchSpanProcessorFactory::Create( std::move(exporter), processor_options); auto resource = otel_resource::Resource::Create(attributes); std::shared_ptr<otel_trace_api::TracerProvider> provider = otel_trace_sdk::TracerProviderFactory::Create( std::move(processor), resource); otel_trace_api::Provider::SetTracerProvider(provider); otel_cntxt::propagation::GlobalTextMapPropagator::SetGlobalPropagator( opentelemetry::nostd::shared_ptr< otel_cntxt::propagation::TextMapPropagator>( new otel_trace_api::propagation::HttpTraceContext())); break; #else LOG_ERROR << "Unsupported trace mode: " << TraceManager::InferenceTraceModeString( global_setting_->mode_); break; #endif } default: return; } } void TraceManager::CleanupTracer() { switch (global_setting_->mode_) { case TRACE_MODE_OPENTELEMETRY: { #ifndef _WIN32 std::shared_ptr<otel_trace_api::TracerProvider> none; otel_trace_api::Provider::SetTracerProvider(none); break; #else LOG_ERROR << "Unsupported trace mode: " << TraceManager::InferenceTraceModeString( global_setting_->mode_); break; #endif } default: return; } } #ifndef _WIN32 void TraceManager::ProcessOpenTelemetryParameters( const triton::server::TraceConfigMap& config_map, otlp::OtlpHttpExporterOptions& exporter_options, otel_resource::ResourceAttributes& attributes, otel_trace_sdk::BatchSpanProcessorOptions& processor_options) { attributes[otel_resource::SemanticConventions::kServiceName] = std::string("triton-inference-server"); auto mode_key = std::to_string(TRACE_MODE_OPENTELEMETRY); auto otel_options_it = config_map.find(mode_key); if (otel_options_it == config_map.end()) { return; } for (const auto& [setting, value] : otel_options_it->second) { // FIXME add more configuration options of OTLP HTTP Exporter if (setting == "url") { exporter_options.url = std::get<std::string>(value); } if (setting == "resource") { auto user_setting = std::get<std::string>(value); auto pos = user_setting.find('='); auto key = user_setting.substr(0, pos); auto value = user_setting.substr(pos + 1); attributes[key] = value; } if (setting == "bsp_max_queue_size") { processor_options.max_queue_size = std::get<uint32_t>(value); } if (setting == "bsp_schedule_delay") { processor_options.schedule_delay_millis = std::chrono::milliseconds(std::get<uint32_t>(value)); } if (setting == "bsp_max_export_batch_size") { processor_options.max_export_batch_size = std::get<uint32_t>(value); } } } void TraceManager::Trace::StartSpan( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, uint64_t trace_id, std::string display_name) { uint64_t parent_id; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceParentId(trace, &parent_id), "getting trace parent id"); auto span_parent_id = parent_id; // Currently, only 2 types of sub-spans are supported: // request span and compute span. Compute span is a leaf span // and can not be a parent of any sub-span. If parent_id==0, // then current model is either a standalone model, or an ensemble model. // In both cases, the parent of the new request sub-span is the kRootSpan. // A request span with trace id = `trace_id` is a parent of a compute span, // started in the same trace. // If parent_id > 0, then this is a child trace, spawned from // the ensamble's main request. For this instance, the parent // span is the ensembles's request span. if ((parent_id == 0 && activity == TRITONSERVER_TRACE_REQUEST_START) || (activity == TRITONSERVER_TRACE_COMPUTE_START) || (activity == TRITONSERVER_TRACE_CUSTOM_ACTIVITY)) { span_parent_id = trace_id; } auto span = StartSpan(display_name, timestamp_ns, span_parent_id); if (activity == TRITONSERVER_TRACE_REQUEST_START) { int64_t model_version; const char* request_id; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceModelVersion(trace, &model_version), "getting model version"); LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceRequestId(trace, &request_id), "getting request id"); span->SetAttribute("triton.model_name", display_name); span->SetAttribute("triton.model_version", model_version); span->SetAttribute("triton.trace_id", trace_id); span->SetAttribute("triton.trace_parent_id", parent_id); if (std::string(request_id) != "") { span->SetAttribute("triton.request_id", request_id); } triton::common::TritonJson::WriteBuffer buffer; PrepareTraceContext(span, &buffer); TRITONSERVER_InferenceTraceSetContext(trace, buffer.Contents().c_str()); } span_stacks_[trace_id]->emplace(span); } opentelemetry::nostd::shared_ptr<otel_trace_api::Span> TraceManager::Trace::StartSpan( std::string display_name, const uint64_t& raw_timestamp_ns, uint64_t trace_id) { otel_trace_api::StartSpanOptions options; options.kind = otel_trace_api::SpanKind::kServer; options.start_system_time = time_offset_ + std::chrono::nanoseconds{raw_timestamp_ns}; options.start_steady_time = otel_common::SteadyTimestamp{std::chrono::nanoseconds{raw_timestamp_ns}}; // If the new span is a child span, we need to retrieve its parent and // provide it through StartSpanOptions to the child span if (span_stacks_.find(trace_id) != span_stacks_.end() && !span_stacks_[trace_id]->empty()) { options.parent = span_stacks_[trace_id]->top()->GetContext(); } auto provider = opentelemetry::trace::Provider::GetTracerProvider(); return provider->GetTracer(kTritonTracer)->StartSpan(display_name, options); } void TraceManager::Trace::EndSpan(uint64_t trace_id) { auto timestamp_ns = std::chrono::duration_cast<std::chrono::nanoseconds>( std::chrono::steady_clock::now().time_since_epoch()) .count(); EndSpan(timestamp_ns, trace_id); } void TraceManager::Trace::EndSpan( const uint64_t& raw_timestamp_ns, uint64_t trace_id) { if (span_stacks_.find(trace_id) != span_stacks_.end() && !span_stacks_[trace_id]->empty()) { otel_trace_api::EndSpanOptions end_options; end_options.end_steady_time = otel_common::SteadyTimestamp{ std::chrono::nanoseconds{raw_timestamp_ns}}; span_stacks_[trace_id]->top()->End(end_options); span_stacks_[trace_id]->pop(); } } void TraceManager::Trace::ReportToOpenTelemetry( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns) { uint64_t id; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceId(trace, &id), "getting trace id"); if (span_stacks_.find(id) == span_stacks_.end()) { std::unique_ptr< std::stack<opentelemetry::nostd::shared_ptr<otel_trace_api::Span>>> st(new std::stack< opentelemetry::nostd::shared_ptr<otel_trace_api::Span>>()); span_stacks_.emplace(id, std::move(st)); } AddEvent(trace, activity, timestamp_ns, id); } void TraceManager::Trace::AddEvent( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, uint64_t trace_id) { std::string activity_name = RetrieveActivityName(trace, activity, timestamp_ns); static std::string start = "_START"; static std::string end = "_END"; if (activity == TRITONSERVER_TRACE_REQUEST_START || activity == TRITONSERVER_TRACE_COMPUTE_START || (activity == TRITONSERVER_TRACE_CUSTOM_ACTIVITY && activity_name.length() > start.length() && std::equal(start.rbegin(), start.rend(), activity_name.rbegin()))) { std::string span_name = activity_name; if (activity == TRITONSERVER_TRACE_CUSTOM_ACTIVITY) { span_name = activity_name.substr(0, activity_name.length() - start.length()); } else if (activity == TRITONSERVER_TRACE_REQUEST_START) { const char* model_name; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceModelName(trace, &model_name), "getting model name"); span_name = model_name; } else if (activity == TRITONSERVER_TRACE_COMPUTE_START) { span_name = "compute"; } StartSpan(trace, activity, timestamp_ns, trace_id, span_name); } AddEvent(activity_name, timestamp_ns, trace_id); if (activity == TRITONSERVER_TRACE_REQUEST_END || activity == TRITONSERVER_TRACE_COMPUTE_END || (activity == TRITONSERVER_TRACE_CUSTOM_ACTIVITY && activity_name.length() > end.length() && std::equal(end.rbegin(), end.rend(), activity_name.rbegin()))) { EndSpan(timestamp_ns, trace_id); } } void TraceManager::Trace::AddEvent( const std::string& event, uint64_t timestamp, uint64_t trace_id) { if (span_stacks_.find(trace_id) != span_stacks_.end() && !span_stacks_[trace_id]->empty()) { span_stacks_[trace_id]->top()->AddEvent( event, time_offset_ + std::chrono::nanoseconds{timestamp}); } } void TraceManager::Trace::PrepareTraceContext( opentelemetry::nostd::shared_ptr<otel_trace_api::Span> span, triton::common::TritonJson::WriteBuffer* buffer) { triton::common::TritonJson::Value json( triton::common::TritonJson::ValueType::OBJECT); char trace_id[32] = {0}; char span_id[16] = {0}; char trace_flags[2] = {0}; span->GetContext().span_id().ToLowerBase16(span_id); span->GetContext().trace_id().ToLowerBase16(trace_id); span->GetContext().trace_flags().ToLowerBase16(trace_flags); std::string kTraceParent = std::string("traceparent"); std::string kTraceState = std::string("tracestate"); std::string traceparent = std::string("00-") + std::string(trace_id, 32) + std::string("-") + std::string(span_id, 16) + std::string("-") + std::string(trace_flags, 2); std::string tracestate = span->GetContext().trace_state()->ToHeader(); json.SetStringObject(kTraceParent.c_str(), traceparent); if (!tracestate.empty()) { json.SetStringObject(kTraceState.c_str(), tracestate); } json.Write(buffer); } #endif void TraceManager::TraceRelease(TRITONSERVER_InferenceTrace* trace, void* userp) { uint64_t id; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceId(trace, &id), "getting trace id"); auto ts = reinterpret_cast<std::shared_ptr<TraceManager::Trace>*>(userp); std::lock_guard<std::mutex> lk((*ts)->mtx_); (*ts)->spawned_traces_tracker_.erase(id); // The userp will be shared with the trace children, so only delete it // if no more TraceRelease calls are expected if ((*ts)->spawned_traces_tracker_.empty()) { delete ts; } LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceDelete(trace), "deleting trace"); } const char* TraceManager::InferenceTraceModeString(InferenceTraceMode mode) { switch (mode) { case TRACE_MODE_TRITON: return "triton"; case TRACE_MODE_OPENTELEMETRY: return "opentelemetry"; } return "<unknown>"; } void TraceManager::TraceActivity( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, void* userp) { uint64_t id; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceId(trace, &id), "getting trace id"); // The function may be called with different traces but the same 'userp', // group the activity of the same trace together for more readable output. auto ts = reinterpret_cast<std::shared_ptr<TraceManager::Trace>*>(userp)->get(); std::lock_guard<std::mutex> lk(ts->mtx_); if (ts->spawned_traces_tracker_.find(id) == ts->spawned_traces_tracker_.end()) { ts->spawned_traces_tracker_.emplace(id); } if (ts->setting_->mode_ == TRACE_MODE_OPENTELEMETRY) { #ifndef _WIN32 ts->ReportToOpenTelemetry(trace, activity, timestamp_ns); #else LOG_ERROR << "Unsupported trace mode: " << TraceManager::InferenceTraceModeString(ts->setting_->mode_); #endif return; } std::stringstream* ss = nullptr; { if (ts->streams_.find(id) == ts->streams_.end()) { std::unique_ptr<std::stringstream> stream(new std::stringstream()); ss = stream.get(); ts->streams_.emplace(id, std::move(stream)); } else { ss = ts->streams_[id].get(); // If the string stream is not newly created, add "," as there is // already content in the string stream *ss << ","; } } // If 'activity' is TRITONSERVER_TRACE_REQUEST_START then collect // and serialize trace details. if (activity == TRITONSERVER_TRACE_REQUEST_START) { const char* model_name; int64_t model_version; uint64_t parent_id; const char* request_id; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceModelName(trace, &model_name), "getting model name"); LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceModelVersion(trace, &model_version), "getting model version"); LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceParentId(trace, &parent_id), "getting trace parent id"); LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceRequestId(trace, &request_id), "getting request id"); *ss << "{\"id\":" << id << ",\"model_name\":\"" << model_name << "\",\"model_version\":" << model_version; if (std::string(request_id) != "") { *ss << ",\"request_id\":\"" << request_id << "\""; } if (parent_id != 0) { *ss << ",\"parent_id\":" << parent_id; } *ss << "},"; } *ss << "{\"id\":" << id << ",\"timestamps\":[" << "{\"name\":\"" << ts->RetrieveActivityName(trace, activity, timestamp_ns) << "\",\"ns\":" << timestamp_ns << "}]}"; } void TraceManager::TraceTensorActivity( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, const char* name, TRITONSERVER_DataType datatype, const void* base, size_t byte_size, const int64_t* shape, uint64_t dim_count, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* userp) { if ((activity != TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT) && (activity != TRITONSERVER_TRACE_TENSOR_BACKEND_INPUT) && (activity != TRITONSERVER_TRACE_TENSOR_BACKEND_OUTPUT)) { LOG_ERROR << "Unsupported activity: " << TRITONSERVER_InferenceTraceActivityString(activity); return; } void* buffer_base = const_cast<void*>(base); if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU buffer_base = malloc(byte_size); if (buffer_base == nullptr) { LOG_ERROR << "Failed to malloc CPU buffer"; return; } FAIL_IF_CUDA_ERR( cudaMemcpy(buffer_base, base, byte_size, cudaMemcpyDeviceToHost), "copying buffer into CPU memory"); #else LOG_ERROR << "GPU buffer is unsupported"; return; #endif // TRITON_ENABLE_GPU } uint64_t id; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceId(trace, &id), "getting trace id"); // The function may be called with different traces but the same 'userp', // group the activity of the same trace together for more readable output. auto ts = reinterpret_cast<std::shared_ptr<TraceManager::Trace>*>(userp)->get(); if (ts->setting_->mode_ == TRACE_MODE_OPENTELEMETRY) { LOG_ERROR << "Tensor level tracing is not supported by the mode: " << TraceManager::InferenceTraceModeString(ts->setting_->mode_); } else if (ts->setting_->mode_ == TRACE_MODE_TRITON) { std::lock_guard<std::mutex> lk(ts->mtx_); std::stringstream* ss = nullptr; { if (ts->streams_.find(id) == ts->streams_.end()) { std::unique_ptr<std::stringstream> stream(new std::stringstream()); ss = stream.get(); ts->streams_.emplace(id, std::move(stream)); ts->spawned_traces_tracker_.emplace(id); } else { ss = ts->streams_[id].get(); // If the string stream is not newly created, add "," as there is // already content in the string stream *ss << ","; } } // collect and serialize trace details. *ss << "{\"id\":" << id << ",\"activity\":\"" << TRITONSERVER_InferenceTraceActivityString(activity) << "\""; // collect tensor *ss << ",\"tensor\":{"; // collect tensor name *ss << "\"name\":\"" << std::string(name) << "\""; // collect tensor data *ss << ",\"data\":\""; size_t element_count = 1; for (uint64_t i = 0; i < dim_count; i++) { element_count *= shape[i]; } switch (datatype) { case TRITONSERVER_TYPE_BOOL: { const uint8_t* bool_base = reinterpret_cast<const uint8_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << ((bool_base[e] == 0) ? false : true); if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_UINT8: { const uint8_t* cbase = reinterpret_cast<const uint8_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_UINT16: { const uint16_t* cbase = reinterpret_cast<const uint16_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_UINT32: { const uint32_t* cbase = reinterpret_cast<const uint32_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_UINT64: { const uint64_t* cbase = reinterpret_cast<const uint64_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_INT8: { const int8_t* cbase = reinterpret_cast<const int8_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_INT16: { const int16_t* cbase = reinterpret_cast<const int16_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_INT32: { const int32_t* cbase = reinterpret_cast<const int32_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_INT64: { const int64_t* cbase = reinterpret_cast<const int64_t*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } // FP16 / BF16 already handled as binary blobs, no need to manipulate // here case TRITONSERVER_TYPE_FP16: { break; } case TRITONSERVER_TYPE_BF16: { break; } case TRITONSERVER_TYPE_FP32: { const float* cbase = reinterpret_cast<const float*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_FP64: { const double* cbase = reinterpret_cast<const double*>(buffer_base); for (size_t e = 0; e < element_count; ++e) { *ss << cbase[e]; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_BYTES: { const char* cbase = reinterpret_cast<const char*>(buffer_base); size_t offset = 0; for (size_t e = 0; e < element_count; ++e) { if ((offset + sizeof(uint32_t)) > byte_size) { return; } const size_t len = *(reinterpret_cast<const uint32_t*>(cbase + offset)); offset += sizeof(uint32_t); if ((offset + len) > byte_size) { return; } std::string str(cbase + offset, len); *ss << "\\\"" << str << "\\\""; offset += len; if (e < (element_count - 1)) *ss << ","; } break; } case TRITONSERVER_TYPE_INVALID: { return; } } *ss << "\",\"shape\":\""; for (uint64_t i = 0; i < dim_count; i++) { *ss << shape[i]; if (i < (dim_count - 1)) { *ss << ","; } } *ss << "\",\"dtype\":\"" << TRITONSERVER_DataTypeString(datatype) << "\"}"; *ss << "}"; } if (memory_type == TRITONSERVER_MEMORY_GPU) { #ifdef TRITON_ENABLE_GPU if (buffer_base != nullptr) { free(buffer_base); } #endif // TRITON_ENABLE_GPU } } TraceManager::TraceFile::~TraceFile() { if (!first_write_) { trace_file_ << "]"; } } void TraceManager::TraceFile::SaveTraces( std::stringstream& trace_stream, const bool to_index_file) { try { if (to_index_file) { std::string file_name = file_name_ + "." + std::to_string(index_.fetch_add(1)); std::ofstream file_stream; file_stream.open(file_name); file_stream << "["; file_stream << trace_stream.rdbuf(); file_stream << "]"; } else { std::lock_guard<std::mutex> lock(mu_); if (first_write_) { trace_file_.open(file_name_); trace_file_ << "["; first_write_ = false; } else { trace_file_ << ","; } trace_file_ << trace_stream.rdbuf(); } } catch (const std::ofstream::failure& e) { LOG_ERROR << "failed creating trace file: " << e.what(); } catch (...) { LOG_ERROR << "failed creating trace file: reason unknown"; } } std::shared_ptr<TraceManager::Trace> TraceManager::TraceSetting::SampleTrace(bool force_sample) { bool count_rate_hit = false; { std::lock_guard<std::mutex> lk(mu_); // [FIXME: DLIS-6033] // A current WAR for initiating trace based on propagated context only // Currently this is implemented through setting trace rate as 0 if (rate_ != 0) { // If `count_` hits 0, `Valid()` returns false for this and all // following requests (unless `count_` is updated by a user). // At this point we only trace requests for which // `force_sample` is true. if (!Valid() && !force_sample) { return nullptr; } // `sample_` counts all requests, coming to server. count_rate_hit = (((++sample_) % rate_) == 0); if (count_rate_hit && (count_ > 0)) { --count_; ++created_; } else if (count_rate_hit && (count_ == 0)) { // This condition is reached, when `force_sample` is true, // `count_rate_hit` is true, but `count_` is 0. Due to the // latter, we explicitly set `count_rate_hit` to false. count_rate_hit = false; } } } if (count_rate_hit || force_sample) { std::shared_ptr<TraceManager::Trace> lts(new Trace()); // Split 'Trace' management to frontend and Triton trace separately // to avoid dependency between frontend request and Triton trace's // liveness auto trace_userp = new std::shared_ptr<TraceManager::Trace>(lts); TRITONSERVER_InferenceTrace* trace; TRITONSERVER_Error* err = TRITONSERVER_InferenceTraceTensorNew( &trace, level_, 0 /* parent_id */, TraceActivity, TraceTensorActivity, TraceRelease, trace_userp); if (err != nullptr) { LOG_TRITONSERVER_ERROR(err, "creating inference trace object"); delete trace_userp; return nullptr; } lts->trace_ = trace; lts->trace_userp_ = trace_userp; LOG_TRITONSERVER_ERROR( TRITONSERVER_InferenceTraceId(trace, <s->trace_id_), "getting trace id"); return lts; } return nullptr; } void TraceManager::TraceSetting::WriteTrace( const std::unordered_map<uint64_t, std::unique_ptr<std::stringstream>>& streams) { std::unique_lock<std::mutex> lock(mu_); if (sample_in_stream_ != 0) { trace_stream_ << ","; } ++sample_in_stream_; ++collected_; size_t stream_count = 0; for (const auto& stream : streams) { trace_stream_ << stream.second->rdbuf(); // Need to add ',' unless it is the last trace in the group ++stream_count; if (stream_count != streams.size()) { trace_stream_ << ","; } } // Write to file with index when one of the following is true // 1. trace_count is specified and that number of traces has been collected // 2. log_frequency is specified and that number of traces has been // collected if (((count_ == 0) && (collected_ == sample_)) || ((log_frequency_ != 0) && (sample_in_stream_ >= log_frequency_))) { // Reset variables and release lock before saving to file sample_in_stream_ = 0; std::stringstream stream; trace_stream_.swap(stream); lock.unlock(); file_->SaveTraces(stream, true /* to_index_file */); } } TraceManager::TraceSetting::TraceSetting( const TRITONSERVER_InferenceTraceLevel level, const uint32_t rate, const int32_t count, const uint32_t log_frequency, const std::shared_ptr<TraceFile>& file, const InferenceTraceMode mode, const TraceConfigMap& config_map, const bool level_specified, const bool rate_specified, const bool count_specified, const bool log_frequency_specified, const bool filepath_specified, const bool mode_specified, const bool config_map_specified) : level_(level), rate_(rate), count_(count), log_frequency_(log_frequency), file_(file), mode_(mode), config_map_(config_map), level_specified_(level_specified), rate_specified_(rate_specified), count_specified_(count_specified), log_frequency_specified_(log_frequency_specified), filepath_specified_(filepath_specified), mode_specified_(mode_specified), config_map_specified_(config_map_specified), sample_(0), created_(0), collected_(0), sample_in_stream_(0) { if (level_ == TRITONSERVER_TRACE_LEVEL_DISABLED) { invalid_reason_ = "tracing is disabled"; } else if (rate_ == 0) { invalid_reason_ = "sample rate must be non-zero"; } else if (mode_ == TRACE_MODE_TRITON && file_->FileName().empty()) { invalid_reason_ = "trace file name is not given"; } } TraceManager::TraceSetting::~TraceSetting() { // If log frequency is set, should log the remaining traces to indexed file. if (mode_ == TRACE_MODE_TRITON && sample_in_stream_ != 0) { file_->SaveTraces(trace_stream_, (log_frequency_ != 0)); } } }} // namespace triton::server �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/tracer.h����������������������������������������������������������������������0000664�0000000�0000000�00000052621�15270523712�0016063�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <atomic> #include <condition_variable> #include <fstream> #include <memory> #include <mutex> #include <set> #include <sstream> #include <stack> #include <string> #include <unordered_map> #include <variant> #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING) #include "opentelemetry/context/propagation/global_propagator.h" #include "opentelemetry/exporters/otlp/otlp_http_exporter_factory.h" #include "opentelemetry/nostd/shared_ptr.h" #include "opentelemetry/sdk/resource/resource.h" #include "opentelemetry/sdk/trace/batch_span_processor_options.h" #include "opentelemetry/sdk/trace/processor.h" #include "opentelemetry/sdk/trace/tracer_provider_factory.h" #include "opentelemetry/trace/context.h" #include "opentelemetry/trace/propagation/http_trace_context.h" #include "opentelemetry/trace/provider.h" namespace otlp = opentelemetry::exporter::otlp; namespace otel_trace_sdk = opentelemetry::sdk::trace; namespace otel_trace_api = opentelemetry::trace; namespace otel_cntxt = opentelemetry::context; namespace otel_resource = opentelemetry::sdk::resource; #endif #include "triton/core/tritonserver.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSSUCCESS nullptr #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #include "triton/common/triton_json.h" namespace triton { namespace server { using TraceConfig = std::vector< std::pair<std::string, std::variant<std::string, int, uint32_t>>>; // Key is trace mode, using TraceConfigMap = std::unordered_map<std::string, TraceConfig>; #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING) using AbstractCarrier = otel_cntxt::propagation::TextMapCarrier; #else using AbstractCarrier = void*; #endif // Common OTel span keys to store in OTel context // with the corresponding trace id. constexpr char kRootSpan[] = "root_span"; constexpr char kRequestSpan[] = "request_span"; constexpr char kComputeSpan[] = "compute_span"; // OTel tracer name constexpr char kTritonTracer[] = "triton-server"; /// Trace modes. typedef enum tracemode_enum { /// Default is Triton tracing API TRACE_MODE_TRITON = 0, /// OpenTelemetry API for tracing TRACE_MODE_OPENTELEMETRY = 1 } InferenceTraceMode; // // Manager for tracing to a file. // class TraceManager { private: class TraceSetting; public: static constexpr int32_t MIN_TRACE_COUNT_VALUE{-1}; // The new field values for a setting, 'clear_xxx_' indicates // whether to clear the previously specified filed value. // If false, 'xxx_' will be used as the new field value. // If 'xxx_' is nullptr, the field value will not be updated. struct NewSetting { NewSetting() : clear_level_(false), level_(nullptr), clear_rate_(false), rate_(nullptr), clear_count_(false), count_(nullptr), clear_log_frequency_(false), log_frequency_(nullptr), mode_(nullptr), config_map_(nullptr) { } bool clear_level_; const TRITONSERVER_InferenceTraceLevel* level_; bool clear_rate_; const uint32_t* rate_; bool clear_count_; const int32_t* count_; bool clear_log_frequency_; const uint32_t* log_frequency_; const InferenceTraceMode* mode_; const TraceConfigMap* config_map_; }; struct Trace; // Create a trace manager that appends trace information // to a specified file as global setting. static TRITONSERVER_Error* Create( TraceManager** manager, const TRITONSERVER_InferenceTraceLevel level, const uint32_t rate, const int32_t count, const uint32_t log_frequency, const std::string& filepath, const InferenceTraceMode mode, const TraceConfigMap& config_map); ~TraceManager() { CleanupTracer(); } /// Options required at Trace initialization struct TraceStartOptions { #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING) otel_cntxt::Context propagated_context{otel_cntxt::Context{}}; #else void* propagated_context{nullptr}; #endif std::shared_ptr<TraceSetting> trace_setting{nullptr}; bool force_sample{false}; }; // Returns TraceStartOptions for specified model TraceStartOptions GetTraceStartOptions( AbstractCarrier& carriers, const std::string& model_name); // Return a trace that should be used to collected trace activities // for an inference request. Return nullptr if no tracing should occur. std::shared_ptr<Trace> SampleTrace(const TraceStartOptions& start_options); // Update global setting if 'model_name' is empty, otherwise, model setting is // updated. TRITONSERVER_Error* UpdateTraceSetting( const std::string& model_name, const NewSetting& new_setting); void GetTraceSetting( const std::string& model_name, TRITONSERVER_InferenceTraceLevel* level, uint32_t* rate, int32_t* count, uint32_t* log_frequency, std::string* filepath, InferenceTraceMode* mode, TraceConfigMap* config_map); // Sets provided TraceSetting with correct trace settings for provided model. void GetTraceSetting( const std::string& model_name, std::shared_ptr<TraceSetting>& trace_setting); // Return the current timestamp. static uint64_t CaptureTimestamp() { return std::chrono::duration_cast<std::chrono::nanoseconds>( std::chrono::steady_clock::now().time_since_epoch()) .count(); } static void TraceRelease(TRITONSERVER_InferenceTrace* trace, void* userp); static const char* InferenceTraceModeString(InferenceTraceMode mode); /// In OpenTelemetry trace mode initializes Opentelemetry exporter, processor, /// and sets the global trace provider. /// In Triton trace mode is a no-op. /// /// \param config_map A config map, which stores all parameters, specified /// by user. void InitTracer(const TraceConfigMap& config_map); /// In OpenTelemetry trace mode cleans global tracer provider, /// set by InitTracer. /// In Triton trace mode is a no-op. void CleanupTracer(); #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING) void ProcessOpenTelemetryParameters( const triton::server::TraceConfigMap& config_map, otlp::OtlpHttpExporterOptions& exporter_options, otel_resource::ResourceAttributes& attributes, otel_trace_sdk::BatchSpanProcessorOptions& processor_options); #endif struct Trace { Trace() : trace_(nullptr), trace_id_(0) {} ~Trace(); std::shared_ptr<TraceSetting> setting_; // Group the spawned traces by trace ID for better formatting std::mutex mtx_; std::unordered_map<uint64_t, std::unique_ptr<std::stringstream>> streams_; // We use the set to track the number of spawned traces, so that // when TraceManager::TraceRelease() with 'trace_userp_' is called // we can safely release 'trace_userp_' std::set<uint64_t> spawned_traces_tracker_; // Triton trace object that this trace is assosicated with, // 'Trace' object does not take ownership of 'trace_'. The caller of // SampleTrace() must call TraceManager::TraceRelease() with 'trace_userp_' // to properly release the resources if 'trace_' is not passed to a // TRITONSERVER_ServerInferAsync() call. TRITONSERVER_InferenceTrace* trace_; void* trace_userp_; uint64_t trace_id_; // Capture a timestamp generated outside of triton and associate it // with this trace. void CaptureTimestamp(const std::string& name, uint64_t timestamp_ns); /// Returns activity name. For custom activities, retrieves the name from /// the trace context. For other activities, returns default name. /// /// \param trace TRITONSERVER_InferenceTrace instance. /// \param activity Trace activity. /// \param timestamp_ns Steady timestamp, which is used to calculate /// OpenTelemetry SystemTimestamp to display span on a timeline, and /// OpenTelemetry SteadyTimestamp to calculate the duration on the span /// with better precision. std::string RetrieveActivityName( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns); #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING) /// Reports TRITONSERVER_InferenceTraceActivity as event to /// the currently active span. If activity is an instance of /// `TRITONSERVER_TRACE_REQUEST_START` or /// `TRITONSERVER_TRACE_COMPUTE_START`, /// it starts a new request or compute span. For the request span it /// adds some triton related attributes, and adds this span to /// a span stack, corresponding to the current trace. Alternatively, /// if activity is `TRITONSERVER_TRACE_REQUEST_END` or /// `TRITONSERVER_TRACE_COMPUTE_END`, it ends the corresponding span. /// /// \param trace TRITONSERVER_InferenceTrace instance. /// \param activity Trace activity. /// \param timestamp_ns Steady timestamp, which is used to calculate /// OpenTelemetry SystemTimestamp to display span on a timeline, and /// OpenTelemetry SteadyTimestamp to calculate the duration on the span /// with better precision. void ReportToOpenTelemetry( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns); /// Starts a span with the provided timestamp and name. /// /// \param display_name Span's name, which will be shown in the trace. /// \param raw_timestamp_ns Steady timestamp, which is used to calculate /// OpenTelemetry SystemTimestamp to display span on a timeline, and /// OpenTelemetry SteadyTimestamp to calculate the duration on the span /// with better precision. /// \param trace_id Trace id. /// \return A shared pointer to a newly created OpenTelemetry span. opentelemetry::nostd::shared_ptr<otel_trace_api::Span> StartSpan( std::string display_name, const uint64_t& raw_timestamp_ns, uint64_t trace_id); // A map to hold spans. Any trace can spawn any amount of child traces, // e.g. ensemble model and BLS. This map holds // ( trace id, stack of started spans ) pair and for each trase keeps // started spans alive for the duration of the traced // event and helps to preserve parent-child relationship. std::unordered_map< uint64_t, std::unique_ptr<std::stack< opentelemetry::nostd::shared_ptr<otel_trace_api::Span>>>> span_stacks_; // Root span. Some events should be recorded in the root span, while // request span is still alive and present in the stack. opentelemetry::nostd::shared_ptr<otel_trace_api::Span> root_span_; /// Prepares trace context to propagate to TRITONSERVER_InferenceTrace. /// Trace context follows W3C Trace Context specification. /// Ref. https://www.w3.org/TR/trace-context/. /// OpenTelemetry ref: /// https://github.com/open-telemetry/opentelemetry-cpp/blob/4bd64c9a336fd438d6c4c9dad2e6b61b0585311f/api/include/opentelemetry/trace/propagation/http_trace_context.h#L94-L113 /// /// \param span An OpenTelemetry span, which is used to extract /// OpenTelemetry's trace_id and span_id. /// \param buffer Buffer used when writing JSON representation of /// OpenTelemetry's context. void PrepareTraceContext( opentelemetry::nostd::shared_ptr<otel_trace_api::Span> span, triton::common::TritonJson::WriteBuffer* buffer); private: // OpenTelemetry SDK relies on system's clock for event timestamps. // Triton Tracing records timestamps using steady_clock. This is a // monotonic clock, i.e. time is always moving forward. It is not related // to wall clock time (for example, it can be time since last reboot). // `time_offset_` is recorded when the trace instance is created, // and further used to calculate `opentelemetry::common::SystemTimestamp` // as `time_offset_` + std::chrono:nanoseconds{temestamp_ns}. This way, // every event recorded timestamp will receive a timestamp of // <time when the trace started> + <nanoseconds passed since the start> // FIXME: add steady clock timestamps to Triton OpenTelemetry SDK, // when created const std::chrono::time_point<std::chrono::system_clock> time_offset_ = std::chrono::system_clock::now() - std::chrono::duration_cast<std::chrono::nanoseconds>( std::chrono::steady_clock::now().time_since_epoch()); /// Starts a compute or request span based on `activity`. /// For request spans, it will add the following attributes to the span: /// `model_name`, `model_version`, `trace_id`, `parent_id`. /// /// \param trace TRITONSERVER_InferenceTrace, used to request model's name, /// version, trace parent_id from the backend. /// \param activity Trace activity. /// \param timestamp_ns Steady timestamp, which is used to calculate /// OpenTelemetry SystemTimestamp to display span on a timeline, and /// OpenTelemetry SteadyTimestamp to calculate the duration on the span /// with better precision. /// \param trace_id Trace id. /// \param display_name Span name. void StartSpan( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, uint64_t trace_id, std::string display_name); /// Ends the span on the top of the stack, related to trace with `trace_id`. /// /// \param trace_id Trace id. void EndSpan(uint64_t trace_id); /// Ends the span on the top of the stack, related to trace with `trace_id` /// at specified steady timestamp. /// /// \param raw_timestamp_ns Steady timestamp to use as /// `EndSpanOptions::end_steady_time`. /// \param trace_id Trace id. void EndSpan(const uint64_t& raw_timestamp_ns, uint64_t trace_id); /// Adds an event to the span on the top of the stack, related to trace /// with `trace_id`. If activity is TRITONSERVER_TRACE_REQUEST_START, /// or TRITONSERVER_TRACE_COMPUTE_START, starts a new span and adds it /// to the span's stack. /// /// \param trace TRITONSERVER_InferenceTrace, used to request model's name, /// version, trace parent_id from the backend. /// \param activity Trace activity. /// \param timestamp_ns Timestamp of the provided event. /// \param trace_id Trace id. void AddEvent( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, uint64_t trace_id); /// Adds an event to the OpenTelemetry span. /// /// \param event An event to add to the span. /// \param timestamp_ns Timestamp of the provided event. /// \param trace_id Trace id. void AddEvent( const std::string& event, uint64_t timestamp_ns, uint64_t trace_id); #endif }; private: TraceManager( const TRITONSERVER_InferenceTraceLevel level, const uint32_t rate, const int32_t count, const uint32_t log_frequency, const std::string& filepath, const InferenceTraceMode mode, const TraceConfigMap& config_map); static void TraceActivity( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns, void* userp); static void TraceTensorActivity( TRITONSERVER_InferenceTrace* trace, TRITONSERVER_InferenceTraceActivity activity, const char* name, TRITONSERVER_DataType datatype, const void* base, size_t byte_size, const int64_t* shape, uint64_t dim_count, TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* userp); // Helper function for UpdateTraceSetting() as recursive update may be needed // if global setting is being updated TRITONSERVER_Error* UpdateTraceSettingInternal( const std::string& model_name, const NewSetting& new_setting); class TraceFile { public: TraceFile(const std::string& file_name) : file_name_(file_name), index_(0), first_write_(true) { } ~TraceFile(); // Save the traces stored in 'trace_stream' into the file. 'to_index_file' // specifies whether the file name should be indexed, if true, the traces // will be written to 'file_name.index' where index will be incremented // every time the traces are written to a file with index. If false, the // trace will be written to 'file_name'. void SaveTraces(std::stringstream& trace_stream, const bool to_index_file); const std::string& FileName() { return file_name_; } private: const std::string file_name_; // The file index for the next index file write. std::atomic<uint32_t> index_; // Multiple traces may be finished and write to the trace file at the same // time std::mutex mu_; std::ofstream trace_file_; bool first_write_; }; class TraceSetting { public: TraceSetting() : level_(TRITONSERVER_TRACE_LEVEL_DISABLED), rate_(0), count_(-1), log_frequency_(0), mode_(TRACE_MODE_TRITON), level_specified_(false), rate_specified_(false), count_specified_(false), log_frequency_specified_(false), filepath_specified_(false), mode_specified_(false), config_map_specified_(false), sample_(0), created_(0), collected_(0), sample_in_stream_(0) { invalid_reason_ = "Setting hasn't been initialized"; } TraceSetting( const TRITONSERVER_InferenceTraceLevel level, const uint32_t rate, const int32_t count, const uint32_t log_frequency, const std::shared_ptr<TraceFile>& file, const InferenceTraceMode mode, const TraceConfigMap& config_map, const bool level_specified, const bool rate_specified, const bool count_specified, const bool log_frequency_specified, const bool filepath_specified, const bool mode_specified, const bool config_map_specified); ~TraceSetting(); bool Valid() { return invalid_reason_.empty() && (count_ != 0); } const std::string& Reason() { return invalid_reason_; } void WriteTrace( const std::unordered_map<uint64_t, std::unique_ptr<std::stringstream>>& streams); // Pass `force_sample` = true, when trace needs to be initiated // no matter what `rate` and `count` is. // For example, in OpenTelemetry tracing mode, we always initiate tracing // when OpenTelemetry context was propagated from client. std::shared_ptr<Trace> SampleTrace(bool force_sample = false); const TRITONSERVER_InferenceTraceLevel level_; const uint32_t rate_; int32_t count_; const uint32_t log_frequency_; const std::shared_ptr<TraceFile> file_; const InferenceTraceMode mode_; const TraceConfigMap config_map_; // Whether the field value is specified or mirror from upper level setting const bool level_specified_; const bool rate_specified_; const bool count_specified_; const bool log_frequency_specified_; const bool filepath_specified_; const bool mode_specified_; const bool config_map_specified_; private: std::string invalid_reason_; std::mutex mu_; // use to sample a trace based on sampling rate. uint64_t sample_; // use to track the status of trace count feature uint64_t created_; uint64_t collected_; // Tracking traces that haven't been saved to file uint32_t sample_in_stream_; std::stringstream trace_stream_; }; // Trace settings // Note that 'global_default_' doesn't use for actual trace sampling, // it is used to revert the field values when clearing fields in // 'global_setting_' std::unique_ptr<TraceSetting> global_default_; std::shared_ptr<TraceSetting> global_setting_; std::unordered_map<std::string, std::shared_ptr<TraceSetting>> model_settings_; // The collection of models that have their own trace setting while // some of the fields are mirroring global setting. std::set<std::string> fallback_used_models_; // The collection of files that are used in trace settings, use to // avoid creating duplicate TraceFile objects for the same file path. std::unordered_map<std::string, std::weak_ptr<TraceFile>> trace_files_; // lock for accessing trace setting. 'w_mu_' for write and // 'r_mu_' for read / write std::mutex w_mu_; std::mutex r_mu_; }; }} // namespace triton::server ���������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/triton_signal.cc��������������������������������������������������������������0000664�0000000�0000000�00000007132�15270523712�0017612�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "triton_signal.h" #include <iostream> #ifdef _WIN32 #include <windows.h> #else #include <csignal> #endif #define BOOST_STACKTRACE_USE_ADDR2LINE #include <boost/stacktrace.hpp> namespace triton { namespace server { // Exit mutex and cv used to signal the main thread that it should // close the server and exit. bool signal_exiting_ = false; std::mutex signal_exit_mu_; std::condition_variable signal_exit_cv_; namespace { void CommonSignalHandler() { { std::unique_lock<std::mutex> lock(signal_exit_mu_); // Do nothing if already exiting... if (signal_exiting_) return; signal_exiting_ = true; } signal_exit_cv_.notify_all(); } } // namespace #ifdef _WIN32 // Windows BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) { switch (fdwCtrlType) { // Handle these events... case CTRL_C_EVENT: case CTRL_CLOSE_EVENT: case CTRL_BREAK_EVENT: case CTRL_LOGOFF_EVENT: case CTRL_SHUTDOWN_EVENT: break; default: return FALSE; } CommonSignalHandler(); return TRUE; } TRITONSERVER_Error* RegisterSignalHandler() { if (!SetConsoleCtrlHandler(CtrlHandler, TRUE)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, "SetConsoleCtrlHandler failed"); } return nullptr; // success } #else namespace { // Non-Windows void SignalHandler(int signum) { std::cout << "Signal (" << signum << ") received." << std::endl; CommonSignalHandler(); } void ErrorSignalHandler(int signum) { std::cerr << "Signal (" << signum << ") received." << std::endl; std::cerr << boost::stacktrace::stacktrace() << std::endl; // Trigger the core dump signal(signum, SIG_DFL); raise(signum); } } // namespace TRITONSERVER_Error* RegisterSignalHandler() { // Trap SIGINT and SIGTERM to allow server to exit gracefully signal(SIGINT, SignalHandler); signal(SIGTERM, SignalHandler); // Trap SIGSEGV and SIGABRT to exit when server crashes signal(SIGSEGV, ErrorSignalHandler); signal(SIGABRT, ErrorSignalHandler); return nullptr; // success } #endif }} // namespace triton::server ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/triton_signal.h���������������������������������������������������������������0000664�0000000�0000000�00000004010�15270523712�0017444�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include <condition_variable> #include <mutex> #include "triton/core/tritonserver.h" namespace triton { namespace server { // Exit mutex and cv used to signal the main thread that it should // close the server and exit. extern bool signal_exiting_; extern std::mutex signal_exit_mu_; extern std::condition_variable signal_exit_cv_; // Register signal handler. Return true if success, false if failure. TRITONSERVER_Error* RegisterSignalHandler(); }} // namespace triton::server ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/vertex_ai_server.cc�����������������������������������������������������������0000664�0000000�0000000�00000031224�15270523712�0020311�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "vertex_ai_server.h" #include <memory> #include "common.h" namespace triton { namespace server { const std::string VertexAiAPIServer::binary_mime_type_( "application/vnd.vertex-ai-triton.binary+json;json-header-size="); const std::string VertexAiAPIServer::redirect_header_( "X-Vertex-Ai-Triton-Redirect"); VertexAiAPIServer::VertexAiAPIServer( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const int32_t port, const std::string address, const int thread_cnt, const std::string& prediction_route, const std::string& health_route, const std::string& default_model_name) : HTTPAPIServer( server, trace_manager, shm_manager, port, false /* reuse_port */, address, "" /* header_forward_pattern */, thread_cnt), prediction_regex_(prediction_route), health_regex_(health_route), health_mode_("ready"), model_name_(default_model_name), model_version_str_("") { } TRITONSERVER_Error* VertexAiAPIServer::GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, size_t* header_length) { // Check mime type and set inference header length. // Set to content length in case that it is not specified *header_length = content_length; const char* content_type_c_str = evhtp_kv_find(req->headers_in, kContentTypeHeader); if (content_type_c_str != NULL) { std::string content_type(content_type_c_str); size_t pos = content_type.find(binary_mime_type_); if (pos != std::string::npos) { if (pos != 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("expect MIME type for binary data starts with '") + binary_mime_type_ + "', got: " + content_type) .c_str()); } // Parse int32_t parsed_value; try { parsed_value = std::atoi(content_type_c_str + binary_mime_type_.length()); } catch (const std::invalid_argument& ia) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Unable to parse inference header size, got: ") + (content_type_c_str + binary_mime_type_.length())) .c_str()); } // Check if the content length is in proper range if ((parsed_value < 0) || (parsed_value > content_length)) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("inference header size should be in range (0, ") + std::to_string(content_length) + "), got: " + (content_type_c_str + binary_mime_type_.length())) .c_str()); } *header_length = parsed_value; } } return nullptr; } void VertexAiAPIServer::Handle(evhtp_request_t* req) { LOG_VERBOSE(1) << "Vertex AI request: " << req->method << " " << req->uri->path->full; if (RE2::FullMatch(std::string(req->uri->path->full), health_regex_)) { HandleServerHealth(req, health_mode_); return; } if (RE2::FullMatch(std::string(req->uri->path->full), prediction_regex_)) { // Secondary regex matching if redirection is requested const char* redirect_c_str = evhtp_kv_find(req->headers_in, redirect_header_.c_str()); if (redirect_c_str == nullptr) { // Infer the default model HandleInfer(req, model_name_, model_version_str_); return; } else { // Endpoint redirection is requested // Prepend the header value with "/" to form the regex expected by // Triton endpoints std::string redirect_endpoint("/"); redirect_endpoint += redirect_c_str; LOG_VERBOSE(1) << "Redirecting Vertex AI request: " << redirect_endpoint; // The endpoint handlers in base class expects specific HTTP methods // while the Vertex AI endpoint only accepts "POST", so the method will // be set to endpoint expected one before invoking the handlers if (req->method != htp_method_POST) { evhtp_send_reply(req, EVHTP_RES_METHNALLOWED); return; } if (redirect_endpoint == "/metrics") { req->method = htp_method_GET; HandleMetrics(req); return; } if (redirect_endpoint == "/v2/models/stats") { // model statistics req->method = htp_method_GET; HandleModelStats(req); return; } std::string model_name, version, kind; if (RE2::FullMatch( redirect_endpoint, model_regex_, &model_name, &version, &kind)) { if (kind == "ready") { // model ready req->method = htp_method_GET; HandleModelReady(req, model_name, version); return; } else if (kind == "infer") { // model infer HandleInfer(req, model_name, version); return; } else if (kind == "config") { // model configuration req->method = htp_method_GET; HandleModelConfig(req, model_name, version); return; } else if (kind == "stats") { // model statistics req->method = htp_method_GET; HandleModelStats(req, model_name, version); return; } else if (kind == "") { // model metadata req->method = htp_method_GET; HandleModelMetadata(req, model_name, version); return; } } std::string region, action, rest, repo_name; if (redirect_endpoint == "/v2") { // server metadata req->method = htp_method_GET; HandleServerMetadata(req); return; } else if (RE2::FullMatch(redirect_endpoint, server_regex_, &rest)) { // server health req->method = htp_method_GET; HandleServerHealth(req, rest); return; } else if (RE2::FullMatch( redirect_endpoint, systemsharedmemory_regex_, ®ion, &action)) { // system shared memory if (action == "status") { req->method = htp_method_GET; } HandleSystemSharedMemory(req, region, action); return; } else if (RE2::FullMatch( redirect_endpoint, cudasharedmemory_regex_, ®ion, &action)) { // cuda shared memory if (action == "status") { req->method = htp_method_GET; } HandleCudaSharedMemory(req, region, action); return; } else if (RE2::FullMatch( redirect_endpoint, modelcontrol_regex_, &repo_name, &kind, &model_name, &action)) { // model repository if (kind == "index") { HandleRepositoryIndex(req, repo_name); return; } else if (kind.find("models", 0) == 0) { HandleRepositoryControl(req, repo_name, model_name, action); return; } } } } LOG_VERBOSE(1) << "Vertex AI error: " << req->method << " " << req->uri->path->full << " - " << static_cast<int>(EVHTP_RES_BADREQ); evhtp_send_reply(req, EVHTP_RES_BADREQ); } void VertexAiAPIServer::HandleMetrics(evhtp_request_t* req) { // Mirror of HTTPMetricsServer::Handle() if (req->method != htp_method_GET) { evhtp_send_reply(req, EVHTP_RES_METHNALLOWED); return; } evhtp_res res = EVHTP_RES_BADREQ; // Call to metric endpoint should not have any trailing string TRITONSERVER_Metrics* metrics = nullptr; TRITONSERVER_Error* err = TRITONSERVER_ServerMetrics(server_.get(), &metrics); if (err == nullptr) { const char* base; size_t byte_size; err = TRITONSERVER_MetricsFormatted( metrics, TRITONSERVER_METRIC_PROMETHEUS, &base, &byte_size); if (err == nullptr) { res = EVHTP_RES_OK; evbuffer_add(req->buffer_out, base, byte_size); } } TRITONSERVER_MetricsDelete(metrics); TRITONSERVER_ErrorDelete(err); evhtp_send_reply(req, res); } TRITONSERVER_Error* VertexAiAPIServer::Create( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const int32_t port, const std::string address, const int thread_cnt, std::string default_model_name, std::unique_ptr<HTTPServer>* http_server) { auto predict_route = GetEnvironmentVariableOrDefault("AIP_PREDICT_ROUTE", ""); auto health_route = GetEnvironmentVariableOrDefault("AIP_HEALTH_ROUTE", ""); if (predict_route.empty()) return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "API_PREDICT_ROUTE is not defined for Vertex AI endpoint"); else if (health_route.empty()) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "AIP_HEALTH_ROUTE is not defined for Vertex AI endpoint"); } // Set default model { TRITONSERVER_Message* model_index_message = nullptr; RETURN_IF_ERR(TRITONSERVER_ServerModelIndex( server.get(), TRITONSERVER_INDEX_FLAG_READY, &model_index_message)); // avoid memory leak when return early std::shared_ptr<TRITONSERVER_Message> managed_msg( model_index_message, [](TRITONSERVER_Message* msg) { TRITONSERVER_MessageDelete(msg); }); const char* buffer; size_t byte_size; RETURN_IF_ERR(TRITONSERVER_MessageSerializeToJson( model_index_message, &buffer, &byte_size)); triton::common::TritonJson::Value model_index_json; RETURN_IF_ERR(model_index_json.Parse(buffer, byte_size)); if (default_model_name.empty()) { if (model_index_json.ArraySize() != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "Expect the model repository contains only a single model if " "default model is not specified"); } triton::common::TritonJson::Value index_json; RETURN_IF_ERR(model_index_json.IndexAsObject(0, &index_json)); const char* name; size_t namelen; RETURN_IF_ERR(index_json.MemberAsString("name", &name, &namelen)); default_model_name = std::string(name, namelen); } // Check if default model is loaded else { bool found = false; for (size_t idx = 0; idx < model_index_json.ArraySize(); ++idx) { triton::common::TritonJson::Value index_json; RETURN_IF_ERR(model_index_json.IndexAsObject(idx, &index_json)); const char* name; size_t namelen; RETURN_IF_ERR(index_json.MemberAsString("name", &name, &namelen)); if (default_model_name == std::string(name, namelen)) { found = true; break; } } if (!found) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("Expect the default model '") + default_model_name + "' is loaded") .c_str()); } } } http_server->reset(new VertexAiAPIServer( server, trace_manager, shm_manager, port, address, thread_cnt, predict_route, health_route, default_model_name)); const std::string addr = address + ":" + std::to_string(port); LOG_INFO << "Started Vertex AI HTTPService at " << addr; return nullptr; } }} // namespace triton::server ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/src/vertex_ai_server.h������������������������������������������������������������0000664�0000000�0000000�00000006737�15270523712�0020166�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of NVIDIA CORPORATION nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY // OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once #include "http_server.h" namespace triton { namespace server { // Handle Vertex HTTP requests to inference server APIs class VertexAiAPIServer : public HTTPAPIServer { public: static TRITONSERVER_Error* Create( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& smb_manager, const int32_t port, const std::string address, const int thread_cnt, std::string default_model_name, std::unique_ptr<HTTPServer>* vertex_ai_server); private: explicit VertexAiAPIServer( const std::shared_ptr<TRITONSERVER_Server>& server, triton::server::TraceManager* trace_manager, const std::shared_ptr<SharedMemoryManager>& shm_manager, const int32_t port, const std::string address, const int thread_cnt, const std::string& prediction_route, const std::string& health_route, const std::string& default_model_name); void Handle(evhtp_request_t* req) override; void HandleMetrics(evhtp_request_t* req); TRITONSERVER_Error* GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, size_t* header_length) override; // Currently the compression schema hasn't been defined, // assume identity compression type is used for both request and response DataCompressor::Type GetRequestCompressionType(evhtp_request_t* req) override { return DataCompressor::Type::IDENTITY; } DataCompressor::Type GetResponseCompressionType(evhtp_request_t* req) override { return DataCompressor::Type::IDENTITY; } re2::RE2 prediction_regex_; re2::RE2 health_regex_; const std::string health_mode_; // For default model, assume that only one version of "model" is presented const std::string model_name_; const std::string model_version_str_; static const std::string binary_mime_type_; static const std::string redirect_header_; }; }} // namespace triton::server ���������������������������������kytensor-2.49.0.7/third_party/����������������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0016166�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/CMakeLists.txt��������������������������������������������������������0000664�0000000�0000000�00000016763�15270523712�0020743�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of NVIDIA CORPORATION nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmake_minimum_required(VERSION 3.16) project(tritonthirdparty LANGUAGES C CXX) # # ExternalProject Dependencies # # For each ExternalProject dependency we use PATCH_COMMAND to copy the # project source into TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX. If # TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX is not set then source will # not be installed. Note that PATCH_COMMAND is executed after the # source is downloaded/updated and is not defined by default so it # allows a good point to perform the source install. # include(ExternalProject) if(NOT TRITON_THIRD_PARTY_INSTALL_PREFIX) message(FATAL_ERROR "TRITON_THIRD_PARTY_INSTALL_PREFIX must be set") endif() # TRITON_THIRD_PARTY_INSTALL_PREFIX if(NOT TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX) message(STATUS "TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX not set") set(INSTALL_SRC_DEST_ARG "") else() set(INSTALL_SRC_DEST_ARG --dest ${TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX}) endif() # TRITON_THIRD_PARTY_INSTALL_PREFIX # If CMAKE_TOOLCHAIN_FILE is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "") if (CMAKE_TOOLCHAIN_FILE) set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "-DCMAKE_TOOLCHAIN_FILE:PATH=${CMAKE_TOOLCHAIN_FILE}") endif() # If VCPKG_TARGET_TRIPLET is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "") if (VCPKG_TARGET_TRIPLET) set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "-DVCPKG_TARGET_TRIPLET:STRING=${VCPKG_TARGET_TRIPLET}") endif() # If OPENSSL_ROOT_DIR is set, propagate that hint path to the external # projects with OpenSSL dependency. set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "") if (OPENSSL_ROOT_DIR) set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") endif() # Some libs are installed to ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib64 instead # of ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib on Centos set (LIB_DIR "lib") if(LINUX) file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE") if(${DISTRO_ID_LIKE} MATCHES "rhel|centos") set (LIB_DIR "lib64") endif(${DISTRO_ID_LIKE} MATCHES "rhel|centos") endif(LINUX) set(TRITON_CORE_HEADERS_ONLY OFF) # C++ standard set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard whose features are requested to build this target.") # # Build libcurl # set(LIBCURL_EXTRA_BUILD_ARGS "") if (WIN32) set(LIBCURL_EXTRA_BUILD_ARGS "-DCURL_USE_SCHANNEL:BOOL=ON") endif() #ExternalProject_Add(curl # PREFIX curl # #GIT_REPOSITORY "https://github.com/curl/curl.git" # #GIT_TAG "curl-7_86_0" # SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/curl" # EXCLUDE_FROM_ALL ON # CMAKE_CACHE_ARGS # ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} # ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} # -DCMAKE_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} # -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON # -DBUILD_TESTING:BOOL=OFF # -DBUILD_CURL_EXE:BOOL=OFF # -DBUILD_SHARED_LIBS:BOOL=OFF # -DCURL_STATICLIB:BOOL=ON # -DHTTP_ONLY:BOOL=ON # -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} # -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/curl # ${LIBCURL_EXTRA_BUILD_ARGS} # PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src <SOURCE_DIR> ${INSTALL_SRC_DEST_ARG} #) # # # Build libevent # ExternalProject_Add(libevent PREFIX libevent #GIT_REPOSITORY "https://github.com/libevent/libevent.git" #GIT_TAG "release-2.1.8-stable" #SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/libevent/src/libevent" SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/libevent" EXCLUDE_FROM_ALL ON CMAKE_CACHE_ARGS ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DCMAKE_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DEVENT__DISABLE_SAMPLES:BOOL=ON -DEVENT__DISABLE_TESTS:BOOL=ON -DEVENT__DISABLE_BENCHMARK:BOOL=ON -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_C_FLAGS:STRING=-Wno-error=implicit-function-declaration -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevent PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src <SOURCE_DIR> ${INSTALL_SRC_DEST_ARG} ) # # Build patched libevhtp # if (WIN32) # FIXME evhtp can't be built successfully in native Windows environment # CMAKE_PREFIX_PATH is needed to find libevent ExternalProject_Add(libevhtp PREFIX libevhtp SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/libevhtp/libevhtp" EXCLUDE_FROM_ALL ON DOWNLOAD_COMMAND "" CMAKE_CACHE_ARGS ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DCMAKE_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DEVHTP_DISABLE_REGEX:BOOL=ON -DEVHTP_DISABLE_SSL:BOOL=ON -DEVHTP_TRITON_ENABLE_HTTP_CONTIGUOUS:BOOL=ON -DEVHTP_TRITON_ENABLE_TRACING:BOOL=${TRITON_ENABLE_TRACING} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_PREFIX_PATH:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevent -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevhtp PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src <SOURCE_DIR> ${INSTALL_SRC_DEST_ARG} DEPENDS libevent ) else() ExternalProject_Add(libevhtp PREFIX libevhtp SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/libevhtp/libevhtp" EXCLUDE_FROM_ALL ON DOWNLOAD_COMMAND "" CMAKE_CACHE_ARGS ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE} ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET} -DCMAKE_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DEVHTP_DISABLE_REGEX:BOOL=ON -DEVHTP_DISABLE_SSL:BOOL=ON -DEVHTP_TRITON_ENABLE_HTTP_CONTIGUOUS:BOOL=ON -DEVHTP_TRITON_ENABLE_TRACING:BOOL=${TRITON_ENABLE_TRACING} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DLibevent_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevent/lib/cmake/libevent -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevhtp PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src <SOURCE_DIR> ${INSTALL_SRC_DEST_ARG} DEPENDS libevent ) endif() �������������kytensor-2.49.0.7/third_party/dlpack/���������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0017424�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/.github/�������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020764�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/.github/workflows/���������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0023021�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/.github/workflows/deploy.yml�����������������������������������0000664�0000000�0000000�00000001754�15270523712�0025047�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������name: CI on: push: branches: - main jobs: test_linux: name: Deploy Docs runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: submodules: recursive - name: Configuring Test Environment run: | sudo apt-get update sudo apt-get -y install build-essential doxygen ghp-import python3 -m pip install -U pip wheel - name: Installing dependencies run: | python3 -m pip install -r doc_requirements.txt - name: Generating Docs run: | make doc - name: Deploying on GitHub Pages run: | touch docs/build/.nojekyll git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY git config --global user.email "dlpack-gh-actions-bot@nomail" git config --global user.name "dlpack-gh-actions-bot" ghp-import -m "Generate DLPack website" -b gh-pages docs/build git push origin gh-pages -f ��������������������kytensor-2.49.0.7/third_party/dlpack/.github/workflows/docs.yaml������������������������������������0000664�0000000�0000000�00000001746�15270523712�0024645�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������name: Build Doc on: push: branches: - main pull_request: branches: - main jobs: test_linux: name: Linux runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: submodules: recursive - name: Configuring Test Environment run: | sudo apt-get update sudo apt-get -y install build-essential doxygen python3 -m pip install -U pip wheel python3 -m pip install cmake ninja python3 --version python3 -m pip --version doxygen --version make --version cmake --version ninja --version - name: Installing dependencies run: | python3 -m pip install -r doc_requirements.txt - name: Testing CMakeLists.txt run: | mkdir build cd build cmake .. -G Ninja -DCMAKE_INSTALL_PREFIX=./install -DBUILD_DOCS=ON ninja ninja install - name: Testing Makefile run: | make doc ��������������������������kytensor-2.49.0.7/third_party/dlpack/.github/workflows/main.yaml������������������������������������0000664�0000000�0000000�00000001210�15270523712�0024623�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������name: CI on: push: branches: - main pull_request: branches: - main jobs: Build: strategy: matrix: os: [ubuntu-latest, macOS-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - name: Setup Python run: | python3 -m pip install cpplint - name: Setup@Ubuntu if: startsWith(matrix.os, 'ubuntu') run: | sudo apt-get install -y doxygen wget graphviz unzip - name: Lint if: startsWith(matrix.os, 'ubuntu') run: | ./tests/scripts/task_lint.sh - name: Test run: | ./tests/scripts/task_build.sh ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/CMakeLists.txt�������������������������������������������������0000664�0000000�0000000�00000007251�15270523712�0022171�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������### # Set minimum version of CMake. Since command 'project' use # VERSION sub-option we need at least 3.0. # Note: If you use 2.6 or 2.4, God kills a kitten. Seriously. cmake_minimum_required(VERSION 3.10 FATAL_ERROR) #### # Set variables: # * PROJECT_NAME # * PROJECT_VERSION project(dlpack VERSION 0.6 LANGUAGES C CXX) ##### # Change the default build type from Debug to Release, while still # supporting overriding the build type. # # The CACHE STRING logic here and elsewhere is needed to force CMake # to pay attention to the value of these variables. if(NOT CMAKE_BUILD_TYPE) message(STATUS "No build type specified; defaulting to CMAKE_BUILD_TYPE=Release.") set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE) else(NOT CMAKE_BUILD_TYPE) if(CMAKE_BUILD_TYPE STREQUAL "Debug") message("==========================================================================================") message(STATUS "Build type: Debug. Performance will be terrible!") message(STATUS "Add -DCMAKE_BUILD_TYPE=Release to the CMake command line to get an optimized build.") message("==========================================================================================") endif(CMAKE_BUILD_TYPE STREQUAL "Debug") endif(NOT CMAKE_BUILD_TYPE) #### # Setup the compiler options # set c++ standard to c++11. # Note: not working on CMake 2.8. We assume that user has # a compiler with C++11 support. set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) message(STATUS "C++11 support has been enabled by default.") option(BUILD_DOCS "Set to ON to build documentation" OFF) option(BUILD_MOCK "Build mock executable" ON) if(BUILD_DOCS) add_subdirectory(docs) endif(BUILD_DOCS) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin) if(BUILD_MOCK) set(DLPACK_MOCK_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/mock_main.cc ${CMAKE_CURRENT_SOURCE_DIR}/contrib/mock_c.c) add_executable(mock ${DLPACK_MOCK_SRCS}) endif() add_library(dlpack INTERFACE) add_library(${PROJECT_NAME}::dlpack ALIAS dlpack) target_include_directories( dlpack INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/contrib> ) if(BUILD_MOCK) target_link_libraries(mock PRIVATE dlpack) endif() # Installation (https://github.com/forexample/package-example) { # Introduce variables: # * CMAKE_INSTALL_LIBDIR # * CMAKE_INSTALL_BINDIR # * CMAKE_INSTALL_INCLUDEDIR include(GNUInstallDirs) set(config_install_dir "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake") set(TARGETS_EXPORT_NAME "${PROJECT_NAME}Targets") set(namespace "${PROJECT_NAME}::") include(CMakePackageConfigHelpers) # Use: # * PROJECT_VERSION write_basic_package_version_file( "${version_config}" COMPATIBILITY SameMajorVersion ) # Use: # * TARGETS_EXPORT_NAME # * PROJECT_NAME configure_package_config_file( "cmake/template/Config.cmake.in" "${project_config}" INSTALL_DESTINATION "${config_install_dir}" ) install( TARGETS dlpack EXPORT "${TARGETS_EXPORT_NAME}" INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" ) install( DIRECTORY include/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" ) install( FILES "${project_config}" "${version_config}" DESTINATION "${config_install_dir}" ) install( EXPORT "${TARGETS_EXPORT_NAME}" NAMESPACE "${namespace}" DESTINATION "${config_install_dir}" ) # } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/LICENSE��������������������������������������������������������0000664�0000000�0000000�00000026121�15270523712�0020433�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2017 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/Makefile�������������������������������������������������������0000664�0000000�0000000�00000001666�15270523712�0021075�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������.PHONY: clean all test doc lint show_docs all: bin/mock LDFLAGS = CFLAGS = -Wall -O3 -Iinclude -Icontrib CXXFLAGS = -std=c++11 $(CFLAGS) SRC = $(wildcard contrib/*.cc contrib/*.c) ALL_CXX_OBJ = $(patsubst contrib/%.cc, build/%.o, $(SRC)) ALL_C_OBJ = $(patsubst contrib/%.c, build/%.o, $(SRC)) ALL_OBJ = $(ALL_CC_OBJ) $(ALL_CXX_OBJ) doc: doxygen docs/Doxyfile $(MAKE) -C docs html show_docs: @python3 -m http.server --directory docs/build/latest lint: ./tests/scripts/task_lint.sh build/%.o: contrib/%.cc @mkdir -p $(@D) $(CXX) $(CXXFLAGS) -MM -MT build/$*.o $< >build/$*.d $(CXX) -c $(CXXFLAGS) -c $< -o $@ build/%.o: contrib/%.c @mkdir -p $(@D) $(CC) $(CFLAGS) -MM -MT build/$*.o $< >build/$*.d $(CC) -c $(CFLAGS) -c $< -o $@ bin/mock: $(ALL_OBJ) @mkdir -p $(@D) $(CXX) $(CXXFLAGS) -o $@ $(filter %.o %.a, $^) $(LDFLAGS) clean: $(RM) -rf build */*/*/*~ */*.o */*/*.o */*/*/*.o */*.d */*/*.d */*/*/*.d docs/build docs/doxygen ��������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/NEWS.md��������������������������������������������������������0000664�0000000�0000000�00000002220�15270523712�0020516�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������DLPack Change Log ================= This file records the changes in DLPack in reverse chronological order. ## v0.8 - Add kDLBool ## v0.7 - Add kDLHexagon - Add kDLOneAPI - Add DLPACK_VERSION and DLPACK_ABI_VERSION ## v0.6 - Add kDLROCMHost - Add kDLCUDAManaged ## v0.5 Rename enum values - kDLGPU -> kDLCUDA - kDLCPUPinned -> kDLCUDAHost The ABI is backward compatible, as it is only change of constant name, exchange can still happen between the new version and old version. ## v0.4 - OpaqueHandle type - Complex support - Rename DLContext -> DLDevice - This requires dependent frameworks to upgrade the type name. - The ABI is backward compatible, as it is only change of constant name. ## v0.3 - Add bfloat16 - Vulkan support ## v0.2 - New device types - kDLMetal for Apple Metal device - kDLVPI for verilog simulator memory - kDLROCM for AMD GPUs - Add prefix DL to all enum constant values - This requires dependent frameworks to upgrade their reference to these constant - The ABI is compatible, as it is only change of constant name. - Add DLManagedTensor structure for borrowing tensors ## v0.1 - Finalize DLTensor structure ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/README.md������������������������������������������������������0000664�0000000�0000000�00000002505�15270523712�0020705�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# DLPack: Open In Memory Tensor Structure [![Build Status](https://github.com/dmlc/dlpack/actions/workflows/main.yaml/badge.svg?branch=main)](https://github.com/dmlc/dlpack/actions/workflows/main.yaml) Documentation: [https://dmlc.github.io/dlpack/latest](https://dmlc.github.io/dlpack/latest) DLPack is an open in-memory tensor structure for sharing tensors among frameworks. DLPack enables - Easier sharing of operators between deep learning frameworks. - Easier wrapping of vendor level operator implementations, allowing collaboration when introducing new devices/ops. - Quick swapping of backend implementations, like different version of BLAS - For final users, this could bring more operators, and possibility of mixing usage between frameworks. We do not intend to implement Tensor and Ops, but instead use this as common bridge to reuse tensor and ops across frameworks. ## Proposal Procedure RFC proposals are opened as issues. The major release will happen as a vote issue to make sure the participants agree on the changes. ## Project Structure There are two major components so far - include: stabilized headers - contrib: in progress unstable libraries ## People Here are list of people who have been involved in DLPack RFC design proposals: @soumith @piiswrong @Yangqing @naibaf7 @bhack @edgarriba @tqchen @prigoyal @zdevito �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/����������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020367�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/���������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0023055�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/README.md������������������������������������0000664�0000000�0000000�00000002117�15270523712�0024335�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Numpy DLPack Array Conversion Example This example demonstrates how a underlying array memory can be handed off between two DLPack compatible frameworks without requiring any copies. In this case, we demonstrate how to convert numpy to TVM's NDArray and vice-versa with proper memory handling. We hope that not only is this directly useful for TVM users, but also a solid example for how similar efficient copies can be implemented in other array frameworks. ## File Breakdown [dlpack.py](dlpack/dlpack.py): Contains the definition of common DLPack structures shared between frameworks. Mirrors the official C++ definitions. [from_numpy.py](dlpack/from_numpy.py): Demonstrates how to convert a numpy array into a PyCapsule containing a DLPack Tensor. [to_numpy.py](dlpack/to_numpy.py): Demonstrates how to take a PyCapsule with a DLPack Tensor and convert it into a numpy array. [test.py](dlpack/test.py): Shows how to_numpy and from_numpy can be used to convert tensor formats without copies. ## Authors [Josh Fromm](https://github.com/jwfromm) [Junru Shao](https://github.com/junrushao1994) �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/dlpack/��������������������������������������0000775�0000000�0000000�00000000000�15270523712�0024313�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/dlpack/__init__.py���������������������������0000664�0000000�0000000�00000000102�15270523712�0026415�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������from .from_numpy import from_numpy from .to_numpy import to_numpy ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/dlpack/dlpack.py�����������������������������0000664�0000000�0000000�00000010256�15270523712�0026127�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������import ctypes _c_str_dltensor = b"dltensor" class DLDeviceType(ctypes.c_int): """The enum that encodes the type of the device where DLTensor memory is allocated. """ kDLCPU = 1 kDLCUDA = 2 kDLCUDAHost = 3 kDLOpenCL = 4 kDLVulkan = 7 kDLMetal = 8 kDLVPI = 9 kDLROCM = 10 kDLROCMHost = 11 kDLCUDAManaged = 13 kDLOneAPI = 14 def __str__(self): return { self.kDLCPU : "CPU", self.kDLCUDA: "CUDA", self.kDLCUDAHost: "CUDAHost", self.kDLOpenCL: "OpenCL", self.kDLVulkan: "Vulkan", self.kDLMetal: "Metal", self.kDLVPI: "VPI", self.kDLROCM: "ROCM", self.kDLROCMHost: "ROMCHost", self.kDLCUDAManaged: "CUDAManaged", self.kDLOneAPI: "oneAPI", }[self.value] class DLDevice(ctypes.Structure): """Represents the device where DLTensor memory is allocated. The device is represented by the pair of fields: device_type: DLDeviceType device_id: c_int """ _fields_ = [ ("device_type", DLDeviceType), ("device_id", ctypes.c_int), ] class DLDataTypeCode(ctypes.c_uint8): """An integer that encodes the category of DLTensor elements' data type.""" kDLInt = 0 kDLUInt = 1 kDLFloat = 2 kDLOpaquePointer = 3 kDLBfloat = 4 kDLComplex = 5 def __str__(self): return { self.kDLInt: "int", self.kDLUInt: "uint", self.kDLFloat: "float", self.kDLBfloat: "bfloat", self.kDLComplex: "complex", self.kDLOpaquePointer: "void_p" }[self.value] class DLDataType(ctypes.Structure): """Descriptor of data type for elements of DLTensor. The data type is described by a triple, `DLDataType.type_code`, `DLDataType.bits`, and `DLDataType.lanes`. The element is understood as packed `lanes` repetitions of elements from `type_code` data-category of width `bits`. """ _fields_ = [ ("type_code", DLDataTypeCode), ("bits", ctypes.c_uint8), ("lanes", ctypes.c_uint16), ] TYPE_MAP = { "bool": (DLDataTypeCode.kDLUInt, 1, 1), "int8": (DLDataTypeCode.kDLInt, 8, 1), "int16": (DLDataTypeCode.kDLInt, 16, 1), "int32": (DLDataTypeCode.kDLInt, 32, 1), "int64": (DLDataTypeCode.kDLInt, 64, 1), "uint8": (DLDataTypeCode.kDLUInt, 8, 1), "uint16": (DLDataTypeCode.kDLUInt, 16, 1), "uint32": (DLDataTypeCode.kDLUInt, 32, 1), "uint64": (DLDataTypeCode.kDLUInt, 64, 1), "float16": (DLDataTypeCode.kDLFloat, 16, 1), "float32": (DLDataTypeCode.kDLFloat, 32, 1), "float64": (DLDataTypeCode.kDLFloat, 64, 1), "complex64": (DLDataTypeCode.kDLComplex, 64, 1), "complex128": (DLDataTypeCode.kDLComplex, 128, 1) } class DLTensor(ctypes.Structure): """Structure describing strided layout of DLTensor. Fields are: data: void pointer device: DLDevice ndim: number of indices needed to reference an element of the tensor dtype: data type descriptor shape: tuple with lengths of the corresponding tensor dimensions strides: tuple of numbers of array elements to step in each dimension when traversing the tensor byte_offset: data + byte_offset gives the address of tensor element with index (0,) * ndim """ _fields_ = [ ("data", ctypes.c_void_p), ("device", DLDevice), ("ndim", ctypes.c_int), ("dtype", DLDataType), ("shape", ctypes.POINTER(ctypes.c_int64)), ("strides", ctypes.POINTER(ctypes.c_int64)), ("byte_offset", ctypes.c_uint64), ] class DLManagedTensor(ctypes.Structure): """Structure storing the pointer to the tensor descriptor, deleter callable for the tensor descriptor, and pointer to some additional data. These are stored in fields `dl_tensor`, `deleter`, and `manager_ctx`.""" _fields_ = [ ("dl_tensor", DLTensor), ("manager_ctx", ctypes.c_void_p), ("deleter", ctypes.CFUNCTYPE(None, ctypes.c_void_p)), ] ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/dlpack/from_numpy.py�������������������������0000664�0000000�0000000�00000007172�15270523712�0027067�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������from typing import Callable import numpy as np import ctypes from .dlpack import DLManagedTensor, DLDevice, DLDataType, _c_str_dltensor ctypes.pythonapi.PyMem_RawMalloc.restype = ctypes.c_void_p ctypes.pythonapi.PyMem_RawFree.argtypes = [ctypes.c_void_p] ctypes.pythonapi.PyCapsule_New.restype=ctypes.py_object ctypes.pythonapi.PyCapsule_New.argtypes=[ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p] class _Holder: """A wrapper around a numpy array to keep track of references to the underlying memory. Parameters ---------- np_array : np.ndarray The numpy array that will be converted to a DLPack tensor and must be managed. """ def __init__(self, np_array: np.ndarray) -> None: self.np_array = np_array self.data = np_array.ctypes.data_as(ctypes.c_void_p) self.shape = np_array.ctypes.shape_as(ctypes.c_int64) self.strides = np_array.ctypes.strides_as(ctypes.c_int64) for i in range(np_array.ndim): self.strides[i] //= np_array.itemsize def _as_manager_ctx(self) -> ctypes.c_void_p: py_obj = ctypes.py_object(self) py_obj_ptr = ctypes.pointer(py_obj) ctypes.pythonapi.Py_IncRef(py_obj) ctypes.pythonapi.Py_IncRef(ctypes.py_object(py_obj_ptr)) return ctypes.cast(py_obj_ptr, ctypes.c_void_p) @ctypes.CFUNCTYPE(None, ctypes.c_void_p) def _numpy_array_deleter(handle: ctypes.c_void_p) -> None: """A function to deallocate the memory of a numpy array.""" dl_managed_tensor = DLManagedTensor.from_address(handle) py_obj_ptr = ctypes.cast( dl_managed_tensor.manager_ctx, ctypes.POINTER(ctypes.py_object) ) py_obj = py_obj_ptr.contents ctypes.pythonapi.Py_DecRef(py_obj) ctypes.pythonapi.Py_DecRef(ctypes.py_object(py_obj_ptr)) ctypes.pythonapi.PyMem_RawFree(handle) @ctypes.CFUNCTYPE(None, ctypes.c_void_p) def _numpy_pycapsule_deleter(handle: ctypes.c_void_p) -> None: """A function to deallocate a pycapsule that wraps a numpy array.""" pycapsule: ctypes.py_object = ctypes.cast(handle, ctypes.py_object) if ctypes.pythonapi.PyCapsule_IsValid(pycapsule, _c_str_dltensor): dl_managed_tensor = ctypes.pythonapi.PyCapsule_GetPointer( pycapsule, _c_str_dltensor ) _numpy_array_deleter(dl_managed_tensor) ctypes.pythonapi.PyCapsule_SetDestructor(pycapsule, None) def from_numpy(np_array: np.ndarray): """Convert a numpy array to another type of dlpack compatible array. Parameters ---------- np_array : np.ndarray The source numpy array that will be converted. Returns ------- pycapsule : PyCapsule A pycapsule containing a DLManagedTensor that can be converted to other array formats without copying the underlying memory. """ holder = _Holder(np_array) size = ctypes.c_size_t(ctypes.sizeof(DLManagedTensor)) dl_managed_tensor = DLManagedTensor.from_address( ctypes.pythonapi.PyMem_RawMalloc(size) ) dl_managed_tensor.dl_tensor.data = holder.data dl_managed_tensor.dl_tensor.device = DLDevice(1, 0) dl_managed_tensor.dl_tensor.ndim = np_array.ndim dl_managed_tensor.dl_tensor.dtype = DLDataType.TYPE_MAP[str(np_array.dtype)] dl_managed_tensor.dl_tensor.shape = holder.shape dl_managed_tensor.dl_tensor.strides = holder.strides dl_managed_tensor.dl_tensor.byte_offset = 0 dl_managed_tensor.manager_ctx = holder._as_manager_ctx() dl_managed_tensor.deleter = _numpy_array_deleter pycapsule = ctypes.pythonapi.PyCapsule_New( ctypes.byref(dl_managed_tensor), _c_str_dltensor, _numpy_pycapsule_deleter, ) return pycapsule ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/dlpack/to_numpy.py���������������������������0000664�0000000�0000000�00000005163�15270523712�0026544�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������import ctypes import numpy as np from .dlpack import _c_str_dltensor, DLManagedTensor, DLTensor ctypes.pythonapi.PyCapsule_IsValid.restype = ctypes.c_int ctypes.pythonapi.PyCapsule_IsValid.argtypes = [ctypes.py_object, ctypes.c_char_p] ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object, ctypes.c_char_p] def _array_interface_from_dl_tensor(dlt): """Constructs NumPy's array_interface dictionary from `dlpack.DLTensor` descriptor.""" assert isinstance(dlt, DLTensor) shape = tuple(dlt.shape[dim] for dim in range(dlt.ndim)) itemsize = dlt.dtype.lanes * dlt.dtype.bits // 8 if dlt.strides: strides = tuple( dlt.strides[dim] * itemsize for dim in range(dlt.ndim) ) else: # Array is compact, make it numpy compatible. strides = [] for i, s in enumerate(shape): cumulative = 1 for e in range(i + 1, dlt.ndim): cumulative *= shape[e] strides.append(cumulative * itemsize) strides = tuple(strides) typestr = "|" + str(dlt.dtype.type_code)[0] + str(itemsize) return dict( version=3, shape=shape, strides=strides, data=(dlt.data, True), offset=dlt.byte_offset, typestr=typestr, ) class _Holder: """A wrapper that combines a pycapsule and array_interface for consumption by numpy. Parameters ---------- array_interface : dict A description of the underlying memory. pycapsule : PyCapsule A wrapper around the dlpack tensor that will be converted to numpy. """ def __init__(self, array_interface, pycapsule) -> None: self.__array_interface__ = array_interface self._pycapsule = pycapsule def to_numpy(pycapsule) -> np.ndarray: """Convert a dlpack tensor into a numpy array without copying. Parameters ---------- pycapsule : PyCapsule A pycapsule wrapping a dlpack tensor that will be converted. Returns ------- np_array : np.ndarray A new numpy array that uses the same underlying memory as the input pycapsule. """ assert ctypes.pythonapi.PyCapsule_IsValid(pycapsule, _c_str_dltensor) dl_managed_tensor = ctypes.pythonapi.PyCapsule_GetPointer( pycapsule, _c_str_dltensor ) dl_managed_tensor_ptr = ctypes.cast(dl_managed_tensor, ctypes.POINTER(DLManagedTensor)) dl_managed_tensor = dl_managed_tensor_ptr.contents holder = _Holder(_array_interface_from_dl_tensor(dl_managed_tensor.dl_tensor), pycapsule) return np.ctypeslib.as_array(holder) �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/test.py��������������������������������������0000664�0000000�0000000�00000002070�15270523712�0024405�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������import tvm import numpy as np from dlpack import from_numpy, to_numpy def test_from_numpy(): """Test the copy free conversion of numpy to a tvm ndarray.""" np_array = np.random.normal(size=[10, 10]) np_array_ref = np_array.copy() tvm_array = tvm.nd.from_dlpack(from_numpy(np_array)) del np_array np.testing.assert_equal(actual=tvm_array.numpy(), desired=np_array_ref) del tvm_array def test_to_numpy(): """Test the copy free conversion of a tvm ndarray to a numpy array""" tvm_array = tvm.nd.array(np.random.normal(size=[10, 10])) np_array_ref = tvm_array.numpy() np_array = to_numpy(tvm_array.__dlpack__()) del tvm_array np.testing.assert_equal(actual=np_array, desired=np_array_ref) del np_array if __name__ == "__main__": """ Run both tests a bunch of times to make sure the conversions and memory management are stable. """ print("### Testing from_numpy") for i in range(10000): test_from_numpy() print("### Testing to_numpy") for i in range(10000): test_to_numpy() ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/apps/numpy_dlpack/test_pure_numpy.py���������������������������0000664�0000000�0000000�00000002415�15270523712�0026673�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������import numpy as np from dlpack import from_numpy, to_numpy def test_to_from_numpy_zero_copy(): """Test the copy free conversion of numpy array via DLPack.""" np_ary = np.random.normal(size=[10, 10]) np_ary_big = np.random.normal(size=[12, 10]) dlpack_capsule = from_numpy(np_ary_big) reconstructed_ary = to_numpy(dlpack_capsule) del dlpack_capsule np_ary_big[1:11] = np_ary del np_ary_big np.testing.assert_equal(actual=reconstructed_ary[1:11], desired=np_ary) def test_to_from_numpy_memory(): """Test that DLPack capsule keeps the source array alive""" source_array = np.random.normal(size=[10, 10]) np_array_ref = source_array.copy() dlpack_capsule = from_numpy(source_array) del source_array reconstructed_array = to_numpy(dlpack_capsule) del dlpack_capsule np.testing.assert_equal(actual=reconstructed_array, desired=np_array_ref) if __name__ == "__main__": """ Run both tests a bunch of times to make sure the conversions and memory management are stable. """ print("### Running `test_to_from_numpy_zero_copy`") for i in range(10000): test_to_from_numpy_zero_copy() print("### Running `test_to_from_numpy_memory`") for i in range(10000): test_to_from_numpy_memory() ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/cmake/���������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020504�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/cmake/template/������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022317�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/cmake/template/Config.cmake.in���������������������������������0000664�0000000�0000000�00000000175�15270523712�0025136�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������@PACKAGE_INIT@ include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake") check_required_components("@PROJECT_NAME@") ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/contrib/�������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021064�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/contrib/dlpack/������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022322�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/contrib/dlpack/dlpackcpp.h�������������������������������������0000664�0000000�0000000�00000003116�15270523712�0024435�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*! * Copyright (c) 2017 by Contributors * \file dlpackcpp.h * \brief Example C++ wrapper of DLPack */ #ifndef DLPACK_DLPACKCPP_H_ #define DLPACK_DLPACKCPP_H_ #include <dlpack/dlpack.h> #include <cstdint> // for int64_t etc #include <cstdlib> // for free() #include <functional> // for std::multiplies #include <memory> #include <numeric> #include <vector> namespace dlpack { // Example container wrapping of DLTensor. class DLTContainer { public: DLTContainer() { // default to float32 handle_.data = nullptr; handle_.dtype.code = kDLFloat; handle_.dtype.bits = 32U; handle_.dtype.lanes = 1U; handle_.device.device_type = kDLCPU; handle_.device.device_id = 0; handle_.shape = nullptr; handle_.strides = nullptr; handle_.byte_offset = 0; } ~DLTContainer() { if (origin_ == nullptr) { free(handle_.data); } } operator DLTensor() { return handle_; } operator DLTensor*() { return &(handle_); } void Reshape(const std::vector<int64_t>& shape) { shape_ = shape; int64_t sz = std::accumulate(std::begin(shape), std::end(shape), int64_t(1), std::multiplies<int64_t>()); int ret = posix_memalign(&handle_.data, 256, sz); if (ret != 0) throw std::bad_alloc(); handle_.shape = &shape_[0]; handle_.ndim = static_cast<uint32_t>(shape.size()); } private: DLTensor handle_; std::vector<int64_t> shape_; std::vector<int64_t> strides_; // original space container, if std::shared_ptr<DLTContainer> origin_; }; } // namespace dlpack #endif // DLPACK_DLPACKCPP_H_ ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/contrib/mock_c.c�����������������������������������������������0000664�0000000�0000000�00000000246�15270523712�0022465�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright by contributors // This file is used to make sure the package is C compatible #include <dlpack/dlpack.h> int GetNDim(DLTensor *t) { return t->ndim; } ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/contrib/mock_main.cc�������������������������������������������0000664�0000000�0000000�00000000214�15270523712�0023325�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������// Copyright by contributors #include <dlpack/dlpack.h> #include <dlpack/dlpackcpp.h> int main() { dlpack::DLTContainer c; return 0; } ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/doc_requirements.txt�������������������������������������������0000664�0000000�0000000�00000000071�15270523712�0023533�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Sphinx==4.4.0 pydata-sphinx-theme==0.7.1 breathe==4.31.0 �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/����������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020354�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/CMakeLists.txt��������������������������������������������0000664�0000000�0000000�00000002343�15270523712�0023116�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������find_package(Doxygen QUIET) if(NOT DOXYGEN_FOUND) message(FATAL_ERROR "Doxygen is needed to build the documentation.") endif() # TODO: add config file set(doxyfile_in ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in) set(doxyfile ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) set(doxygen_output_dir ${CMAKE_CURRENT_BINARY_DIR}/doxygen) configure_file(${doxyfile_in} ${doxyfile} @ONLY) file(MAKE_DIRECTORY ${doxygen_output_dir}) add_custom_target(Doxygen ALL COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating API documentation with Doxygen" VERBATIM) find_program(SPHINX_EXECUTABLE NAMES sphinx-build DOC "Path to sphinx-build executable") set(sphinx_source ${CMAKE_CURRENT_SOURCE_DIR}/source) set(sphinx_build ${CMAKE_CURRENT_BINARY_DIR}/build/latest) set(doxygen_xml_builddir ${doxygen_output_dir}/xml) add_custom_target(Sphinx ALL COMMAND ${SPHINX_EXECUTABLE} -b html -Dbreathe_projects.dlpack=${doxygen_xml_builddir} ${sphinx_source} ${sphinx_build} -WT --keep-going WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating documentation with Sphinx" VERBATIM) install(DIRECTORY ${sphinx_build} DESTINATION share/${PROJECT_NAME}/docs) ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/Doxyfile��������������������������������������������������0000664�0000000�0000000�00000306554�15270523712�0022077�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Doxyfile 1.8.8 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. # # All text after a double hash (##) is considered a comment and is placed in # front of the TAG it is preceding. # # All text after a single hash (#) is considered a comment and will be ignored. # The format is: # TAG = value [value, ...] # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the config file # that follow. The default is UTF-8 which is also the encoding used for all text # before the first occurrence of this tag. Doxygen uses libiconv (or the iconv # built into libc) for the transcoding. See http://www.gnu.org/software/libiconv # for the list of possible encodings. # The default value is: UTF-8. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded by # double-quotes, unless you are using Doxywizard) that should identify the # project for which the documentation is generated. This name is used in the # title of most generated pages and in a few other places. # The default value is: My Project. PROJECT_NAME = "dlpack" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version # control system is used. PROJECT_NUMBER = 0.6 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. PROJECT_BRIEF = "Common in-memory tensor structure and operator interface for deep learning and other systems" # With the PROJECT_LOGO tag one can specify an logo or icon that is included in # the documentation. The maximum height of the logo should not exceed 55 pixels # and the maximum width should not exceed 200 pixels. Doxygen will copy the logo # to the output directory. PROJECT_LOGO = # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. OUTPUT_DIRECTORY = docs/doxygen # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and # will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes # performance problems for the file system. # The default value is: NO. CREATE_SUBDIRS = NO # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode # U+3044. # The default value is: NO. #ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, # Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), # Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, # Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), # Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, # Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, # Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, # Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English # If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. # The default value is: YES. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief # description of a member or function before the detailed description # # Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. # The default value is: YES. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator that is # used to form the text in various listings. Each string in this list, if found # as the leading text of the brief description, will be stripped from the text # and the result, after processing the whole list, is used as the annotated # text. Otherwise, the brief description is used as-is. If left blank, the # following values are used ($name is automatically replaced with the name of # the entity):The $name class, The $name widget, The $name file, is, provides, # specifies, contains, represents, a, an and the. ABBREVIATE_BRIEF = # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # doxygen will generate a detailed section even if there is only a brief # description. # The default value is: NO. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. # The default value is: NO. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path # before files name in the file list and in the header files. If set to NO the # shortest path that makes the file name unique will be used # The default value is: YES. FULL_PATH_NAMES = YES # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. # Stripping is only done if one of the specified strings matches the left-hand # part of the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the path to # strip. # # Note that you can specify absolute paths here, but also relative paths, which # will be relative from the directory where doxygen is started. # This tag requires that the tag FULL_PATH_NAMES is set to YES. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the # path mentioned in the documentation of a class, which tells the reader which # header file to include in order to use a class. If left blank only the name of # the header file containing the class definition is used. Otherwise one should # specify the list of include paths that are normally passed to the compiler # using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but # less readable) file names. This can be useful is your file systems doesn't # support long names like on DOS, Mac, or CD-ROM. # The default value is: NO. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the # first line (until the first dot) of a Javadoc-style comment as the brief # description. If set to NO, the Javadoc-style will behave just like regular Qt- # style comments (thus requiring an explicit @brief command for a brief # description.) # The default value is: NO. JAVADOC_AUTOBRIEF = NO # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If # set to NO, the Qt-style will behave just like regular Qt-style comments (thus # requiring an explicit \brief command for a brief description.) # The default value is: NO. QT_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a # multi-line C++ special comment block (i.e. a block of //! or /// comments) as # a brief description. This used to be the default behavior. The new default is # to treat a multi-line C++ comment block as a detailed description. Set this # tag to YES if you prefer the old behavior instead. # # Note that setting this tag to YES also means that rational rose comments are # not recognized any more. # The default value is: NO. MULTILINE_CPP_IS_BRIEF = NO # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the # documentation from any documented member that it re-implements. # The default value is: YES. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a # new page for each member. If set to NO, the documentation of a member will be # part of the file/class/namespace that contains it. # The default value is: NO. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen # uses this value to replace tabs by spaces in code fragments. # Minimum value: 1, maximum value: 16, default value: 4. TAB_SIZE = 8 # This tag can be used to specify a number of aliases that act as commands in # the documentation. An alias has the form: # name=value # For example adding # "sideeffect=@par Side Effects:\n" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading # "Side Effects:". You can put \n's in the value part of an alias to insert # newlines. ALIASES = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For # instance, some of the names that are used will be different. The list of all # members will be omitted, etc. # The default value is: NO. OPTIMIZE_OUTPUT_FOR_C = NO # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or # Python sources only. Doxygen will then generate output that is more tailored # for that language. For instance, namespaces will be presented as packages, # qualified scopes will look different, etc. # The default value is: NO. OPTIMIZE_OUTPUT_JAVA = NO # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran # sources. Doxygen will then generate output that is tailored for Fortran. # The default value is: NO. OPTIMIZE_FOR_FORTRAN = NO # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL # sources. Doxygen will then generate output that is tailored for VHDL. # The default value is: NO. OPTIMIZE_OUTPUT_VHDL = NO # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, Javascript, # C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: # FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: # Fortran. In the later case the parser tries to guess whether the code is fixed # or free formatted code, this is the default for Fortran type files), VHDL. For # instance to make doxygen treat .inc files as Fortran files (default is PHP), # and .f files as C (default is Fortran), use: inc=Fortran f=C. # # Note For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise # the files are not read by doxygen. EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable # documentation. See http://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you can # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. # The default value is: YES. #MARKDOWN_SUPPORT = YES # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by by putting a % sign in front of the word # or globally by setting AUTOLINK_SUPPORT to NO. # The default value is: YES. #AUTOLINK_SUPPORT = YES # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should set this # tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); # versus func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. # The default value is: NO. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. # The default value is: NO. CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: # http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen # will parse them like normal C++ but will assume all classes use public instead # of private inheritance when no explicit protection keyword is present. # The default value is: NO. SIP_SUPPORT = NO # For Microsoft's IDL there are propget and propput attributes to indicate # getter and setter methods for a property. Setting this option to YES will make # doxygen to replace the get and set methods by a property in the documentation. # This will only work if the methods are indeed getting or setting a simple # type. If this is not the case, or you want to show the methods anyway, you # should set this option to NO. # The default value is: YES. IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES, then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. # The default value is: NO. DISTRIBUTE_GROUP_DOC = NO # Set the SUBGROUPING tag to YES to allow class member groups of the same type # (for instance a group of public functions) to be put as a subgroup of that # type (e.g. under the Public Functions section). Set it to NO to prevent # subgrouping. Alternatively, this can be done per class using the # \nosubgrouping command. # The default value is: YES. SUBGROUPING = YES # When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions # are shown inside the group in which they are included (e.g. using \ingroup) # instead of on a separate page (for HTML and Man pages) or section (for LaTeX # and RTF). # # Note that this feature does not work in combination with # SEPARATE_MEMBER_PAGES. # The default value is: NO. INLINE_GROUPED_CLASSES = NO # When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions # with only public data fields or simple typedef fields will be shown inline in # the documentation of the scope in which they are defined (i.e. file, # namespace, or group documentation), provided this scope is documented. If set # to NO, structs, classes, and unions are shown on a separate page (for HTML and # Man pages) or section (for LaTeX and RTF). # The default value is: NO. INLINE_SIMPLE_STRUCTS = NO # When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or # enum is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, # namespace, or class. And the struct will be named TypeS. This can typically be # useful for C code in case the coding convention dictates that all compound # types are typedef'ed and only the typedef is referenced, never the tag name. # The default value is: NO. TYPEDEF_HIDES_STRUCT = NO # The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This # cache is used to resolve symbols given their name and scope. Since this can be # an expensive process and often the same symbol appears multiple times in the # code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small # doxygen will become slower. If the cache is too large, memory is wasted. The # cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range # is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 # symbols. At the end of a run doxygen will report the cache usage and suggest # the optimal cache size from a speed point of view. # Minimum value: 0, maximum value: 9, default value: 0. LOOKUP_CACHE_SIZE = 0 #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in # documentation are documented, even if no documentation was available. Private # class members and static file members will be hidden unless the # EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. # Note: This will also disable the warnings about undocumented members that are # normally produced when WARNINGS is set to YES. # The default value is: NO. EXTRACT_ALL = YES # If the EXTRACT_PRIVATE tag is set to YES all private members of a class will # be included in the documentation. # The default value is: NO. EXTRACT_PRIVATE = NO # If the EXTRACT_PACKAGE tag is set to YES all members with package or internal # scope will be included in the documentation. # The default value is: NO. #EXTRACT_PACKAGE = NO # If the EXTRACT_STATIC tag is set to YES all static members of a file will be # included in the documentation. # The default value is: NO. EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined # locally in source files will be included in the documentation. If set to NO # only classes defined in header files are included. Does not have any effect # for Java sources. # The default value is: YES. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. When set to YES local methods, # which are defined in the implementation section but not in the interface are # included in the documentation. If set to NO only methods in the interface are # included. # The default value is: NO. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be # extracted and appear in the documentation as a namespace called # 'anonymous_namespace{file}', where file will be replaced with the base name of # the file that contains the anonymous namespace. By default anonymous namespace # are hidden. # The default value is: NO. EXTRACT_ANON_NSPACES = NO # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation # section is generated. This option has no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO these classes will be included in the various overviews. This option has # no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend # (class|struct|union) declarations. If set to NO these declarations will be # included in the documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any # documentation blocks found inside the body of a function. If set to NO these # blocks will be appended to the function's detailed documentation block. # The default value is: NO. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation that is typed after a # \internal command is included. If the tag is set to NO then the documentation # will be excluded. Set it to YES to include the internal documentation. # The default value is: NO. INTERNAL_DOCS = NO # If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file # names in lower-case letters. If set to YES upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. # The default value is: system dependent. CASE_SENSE_NAMES = YES # If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with # their full class and namespace scopes in the documentation. If set to YES the # scope will be hidden. # The default value is: NO. HIDE_SCOPE_NAMES = NO # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. SHOW_INCLUDE_FILES = YES # If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each # grouped member an include statement to the documentation, telling the reader # which file to include in order to use the member. # The default value is: NO. #SHOW_GROUPED_MEMB_INC = NO # If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include # files with double quotes in the documentation rather than with sharp brackets. # The default value is: NO. FORCE_LOCAL_INCLUDES = NO # If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the # documentation for inline members. # The default value is: YES. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the # (detailed) documentation of file and class members alphabetically by member # name. If set to NO the members will appear in declaration order. # The default value is: YES. SORT_MEMBER_DOCS = YES # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief # descriptions of file, namespace and class members alphabetically by member # name. If set to NO the members will appear in declaration order. Note that # this will also influence the order of the classes in the class list. # The default value is: NO. SORT_BRIEF_DOCS = NO # If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the # (brief and detailed) documentation of class members so that constructors and # destructors are listed first. If set to NO the constructors will appear in the # respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. # Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief # member documentation. # Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting # detailed member documentation. # The default value is: NO. SORT_MEMBERS_CTORS_1ST = NO # If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy # of group names into alphabetical order. If set to NO the group names will # appear in their defined order. # The default value is: NO. SORT_GROUP_NAMES = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by # fully-qualified names, including namespaces. If set to NO, the class list will # be sorted only by class name, not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the alphabetical # list. # The default value is: NO. SORT_BY_SCOPE_NAME = NO # If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper # type resolution of all parameters of a function it will reject a match between # the prototype and the implementation of a member function even if there is # only one candidate or it is obvious which candidate to choose by doing a # simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still # accept a match between prototype and implementation in such cases. # The default value is: NO. STRICT_PROTO_MATCHING = NO # The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the # todo list. This list is created by putting \todo commands in the # documentation. # The default value is: YES. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the # test list. This list is created by putting \test commands in the # documentation. # The default value is: YES. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug # list. This list is created by putting \bug commands in the documentation. # The default value is: YES. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO) # the deprecated list. This list is created by putting \deprecated commands in # the documentation. # The default value is: YES. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional documentation # sections, marked by \if <section_label> ... \endif and \cond <section_label> # ... \endcond blocks. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the # initial value of a variable or macro / define can have for it to appear in the # documentation. If the initializer consists of more lines than specified here # it will be hidden. Use a value of 0 to hide initializers completely. The # appearance of the value of individual variables and macros / defines can be # controlled using \showinitializer or \hideinitializer command in the # documentation regardless of this setting. # Minimum value: 0, maximum value: 10000, default value: 30. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated at # the bottom of the documentation of classes and structs. If set to YES the list # will mention the files that were used to generate the documentation. # The default value is: YES. SHOW_USED_FILES = YES # Set the SHOW_FILES tag to NO to disable the generation of the Files page. This # will remove the Files entry from the Quick Index and from the Folder Tree View # (if specified). # The default value is: YES. SHOW_FILES = YES # Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces # page. This will remove the Namespaces entry from the Quick Index and from the # Folder Tree View (if specified). # The default value is: YES. SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via # popen()) the command command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided # by doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. FILE_VERSION_FILTER = # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed # by doxygen. The layout file controls the global structure of the generated # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml # will be used as the name of the layout file. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE # tag is left empty. LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool # to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. See also \cite for info how to create references. CITE_BIB_FILES = #--------------------------------------------------------------------------- # Configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated to # standard output by doxygen. If QUIET is set to YES this implies that the # messages are off. # The default value is: NO. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES # this implies that the warnings are on. # # Tip: Turn warnings on while writing the documentation. # The default value is: YES. WARNINGS = YES # If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate # warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag # will automatically be disabled. # The default value is: YES. WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some parameters # in a documented function, or documenting parameters that don't exist or using # markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return # value. If set to NO doxygen will only warn about wrong or incomplete parameter # documentation, but not about the absence of documentation. # The default value is: NO. WARN_NO_PARAMDOC = YES # The WARN_FORMAT tag determines the format of the warning messages that doxygen # can produce. The string should contain the $file, $line, and $text tags, which # will be replaced by the file and line number from which the warning originated # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard # error (stderr). WARN_LOGFILE = #--------------------------------------------------------------------------- # Configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag is used to specify the files and/or directories that contain # documented source files. You may enter file names like myfile.cpp or # directories like /usr/src/myproject. Separate the files or directories with # spaces. # Note: If this tag is empty the current directory is searched. INPUT = include/dlpack # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv # documentation (see: http://www.gnu.org/software/libiconv) for the list of # possible encodings. # The default value is: UTF-8. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank the # following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii, # *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, # *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, # *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, # *.qsf, *.as and *.js. FILE_PATTERNS = *.h # The RECURSIVE tag can be used to specify whether or not subdirectories should # be searched for input files as well. # The default value is: NO. RECURSIVE = YES # The EXCLUDE tag can be used to specify files and/or directories that should be # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. # # Note that relative paths are relative to the directory from which doxygen is # run. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded # from the input. # The default value is: NO. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* EXCLUDE_PATTERNS = */test/* \ logging.h # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # AClass::ANamespace, ANamespace::*Test # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories use the pattern */test/* EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or directories # that contain example code fragments that are included (see the \include # command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank all # files are included. EXAMPLE_PATTERNS = # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude commands # irrespective of the value of the RECURSIVE tag. # The default value is: NO. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or directories # that contain images that are to be included in the documentation (see the # \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command: # # <filter> <input-file> # # where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the # name of an input file. Doxygen will then use the output that the filter # program writes to standard output. If FILTER_PATTERNS is specified, this tag # will be ignored. # # Note that the filter must not add or remove lines; it is applied before the # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: pattern=filter # (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how # filters are used. If the FILTER_PATTERNS tag is empty or if none of the # patterns match the file name, INPUT_FILTER is applied. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER ) will also be used to filter the input files that are used for # producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). # The default value is: NO. FILTER_SOURCE_FILES = NO # The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file # pattern. A pattern will override the setting for FILTER_PATTERN (if any) and # it is also possible to disable source filtering for a specific pattern using # *.ext= (so without naming a filter). # This tag requires that the tag FILTER_SOURCE_FILES is set to YES. FILTER_SOURCE_PATTERNS = # If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that # is part of the input, its contents will be placed on the main page # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. #USE_MDFILE_AS_MAINPAGE = #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will be # generated. Documented entities will be cross-referenced with these sources. # # Note: To get rid of all source code in the generated output, make sure that # also VERBATIM_HEADERS is set to NO. # The default value is: NO. SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body of functions, # classes and enums directly into the documentation. # The default value is: NO. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any # special comment blocks from generated source code fragments. Normal C, C++ and # Fortran comments will always remain visible. # The default value is: YES. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES then for each documented # function all documented functions referencing it will be listed. # The default value is: NO. REFERENCED_BY_RELATION = NO # If the REFERENCES_RELATION tag is set to YES then for each documented function # all documented entities called/used by that function will be listed. # The default value is: NO. REFERENCES_RELATION = NO # If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set # to YES, then the hyperlinks from functions in REFERENCES_RELATION and # REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will # link to the documentation. # The default value is: YES. REFERENCES_LINK_SOURCE = YES # If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the # source code will show a tooltip with additional information such as prototype, # brief description and links to the definition and documentation. Since this # will make the HTML file larger and loading of large files a bit slower, you # can opt to disable this feature. # The default value is: YES. # This tag requires that the tag SOURCE_BROWSER is set to YES. #SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system # (see http://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: # - Install the latest version of global # - Enable SOURCE_BROWSER and USE_HTAGS in the config file # - Make sure the INPUT points to the root of the source tree # - Run doxygen as normal # # Doxygen will invoke htags (and that will in turn invoke gtags), so these # tools must be available from the command line (i.e. in the search path). # # The result: instead of the source browser generated by doxygen, the links to # source code will now point to the output of htags. # The default value is: NO. # This tag requires that the tag SOURCE_BROWSER is set to YES. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a # verbatim copy of the header file for each class for which an include is # specified. Set to NO to disable this. # See also: Section \class. # The default value is: YES. VERBATIM_HEADERS = YES # If the CLANG_ASSISTED_PARSING tag is set to YES, then doxygen will use the # clang parser (see: http://clang.llvm.org/) for more accurate parsing at the # cost of reduced performance. This can be particularly helpful with template # rich C++ code for which doxygen's built-in parser lacks the necessary type # information. # Note: The availability of this option depends on whether or not doxygen was # compiled with the --with-libclang option. # The default value is: NO. #CLANG_ASSISTED_PARSING = NO # If clang assisted parsing is enabled you can provide the compiler with command # line options that you would normally use when invoking the compiler. Note that # the include paths will already be set by doxygen for the files and directories # specified with INPUT and INCLUDE_PATH. # This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. #CLANG_OPTIONS = #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all # compounds will be generated. Enable this if the project contains a lot of # classes, structs, unions or interfaces. # The default value is: YES. ALPHABETICAL_INDEX = YES # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored # while generating the index headers. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES doxygen will generate HTML output # The default value is: YES. GENERATE_HTML = NO # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). # The default value is: .html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a user-defined HTML header file for # each generated HTML page. If the tag is left blank doxygen will generate a # standard header. # # To get valid HTML the header file that includes any scripts and style sheets # that doxygen needs, which is dependent on the configuration options used (e.g. # the setting GENERATE_TREEVIEW). It is highly recommended to start with a # default header using # doxygen -w html new_header.html new_footer.html new_stylesheet.css # YourConfigFile # and then modify the file new_header.html. See also section "Doxygen usage" # for information on how to generate the default header that doxygen normally # uses. # Note: The header is subject to change so you typically have to regenerate the # default header when upgrading to a newer version of doxygen. For a description # of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each # generated HTML page. If the tag is left blank doxygen will generate a standard # footer. See HTML_HEADER for more information on how to generate a default # footer and what special commands can be used inside the footer. See also # section "Doxygen usage" for information on how to generate the default footer # that doxygen normally uses. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading style # sheet that is used by each HTML page. It can be used to fine-tune the look of # the HTML output. If left blank doxygen will generate a default style sheet. # See also section "Doxygen usage" for information on how to generate the style # sheet that doxygen normally uses. # Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as # it is more robust and this tag (HTML_STYLESHEET) will in the future become # obsolete. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_STYLESHEET = # The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined # cascading style sheets that are included after the standard style sheets # created by doxygen. Using this option one can overrule certain style aspects. # This is preferred over using HTML_STYLESHEET since it does not replace the # standard style sheet and is therefor more robust against future updates. # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra stylesheet files is of importance (e.g. the last # stylesheet in the list overrules the setting of the previous ones in the # list). For an example see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. #HTML_EXTRA_STYLESHEET = # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or # other source files which should be copied to the HTML output directory. Note # that these files will be copied to the base HTML output directory. Use the # $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these # files. In the HTML_STYLESHEET file, use the file name only. Also note that the # files will be copied as-is; there are no commands or markers available. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the stylesheet and background images according to # this color. Hue is specified as an angle on a colorwheel, see # http://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors # in the HTML output. For a value of 0 the output will use grayscales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_SAT = 100 # The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the # luminance component of the colors in the HTML output. Values below 100 # gradually make the output lighter, whereas values above 100 make the output # darker. The value divided by 100 is the actual gamma applied, so 80 represents # a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not # change the gamma. # Minimum value: 40, maximum value: 240, default value: 80. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_GAMMA = 80 # If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML # page will contain the date and time when the page was generated. Setting this # to NO can help when comparing the output of multiple runs. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_TIMESTAMP = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_DYNAMIC_SECTIONS = NO # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to # such a level that at most the specified number of entries are visible (unless # a fully collapsed tree already exceeds this amount). So setting the number of # entries 1 will produce a full collapsed tree by default. 0 is a special value # representing an infinite number of entries and will result in a full expanded # tree by default. # Minimum value: 0, maximum value: 9999, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. #HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development # environment (see: http://developer.apple.com/tools/xcode/), introduced with # OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a # Makefile in the HTML output directory. Running make will produce the docset in # that directory and running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html # for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_DOCSET = NO # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider # (such as a company or product suite) can be grouped. # The default value is: Doxygen generated docs. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_FEEDNAME = "Doxygen generated docs" # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_BUNDLE_ID = org.doxygen.Project # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style # string, e.g. com.mycompany.MyDocSet.documentation. # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_ID = org.doxygen.Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop # (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on # Windows. # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML # files are now used as the Windows 98 help format, and will replace the old # Windows help format (.hlp) on all Windows platforms in the future. Compressed # HTML files also contain an index, a table of contents, and you can search for # words in the documentation. The HTML workshop also contains a viewer for # compressed HTML files. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_HTMLHELP = NO # The CHM_FILE tag can be used to specify the file name of the resulting .chm # file. You can add a path in front of the file if the result should not be # written to the html output directory. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_FILE = # The HHC_LOCATION tag can be used to specify the location (absolute path # including file name) of the HTML help compiler ( hhc.exe). If non-empty # doxygen will try to run the HTML help compiler on the generated index.hhp. # The file has to be specified with full path. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. HHC_LOCATION = # The GENERATE_CHI flag controls if a separate .chi index file is generated ( # YES) or that it should be included in the main .chm file ( NO). # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. GENERATE_CHI = NO # The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc) # and project file content. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_INDEX_ENCODING = # The BINARY_TOC flag controls whether a binary table of contents is generated ( # YES) or a normal table of contents ( NO) in the .chm file. Furthermore it # enables the Previous and Next buttons. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members to # the table of contents of the HTML help documentation and to the tree view. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. TOC_EXPAND = NO # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help # (.qch) of the generated HTML documentation. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_QHP = NO # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to # the HTML output folder. # This tag requires that the tag GENERATE_QHP is set to YES. QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace # (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual # Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- # folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom # Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom # Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: # http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = # The QHG_LOCATION tag can be used to specify the location of Qt's # qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the # generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be # generated, together with the HTML files, they form an Eclipse help plugin. To # install this plugin and make it available under the help contents menu in # Eclipse, the contents of the directory containing the HTML and XML files needs # to be copied into the plugins directory of eclipse. The name of the directory # within the plugins directory should be the same as the ECLIPSE_DOC_ID value. # After copying Eclipse needs to be restarted before the help appears. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_ECLIPSEHELP = NO # A unique identifier for the Eclipse help plugin. When installing the plugin # the directory name containing the HTML and XML files should also have this # name. Each documentation set should have its own identifier. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. ECLIPSE_DOC_ID = org.doxygen.Project # If you want full control over the layout of the generated HTML pages it might # be necessary to disable the index and replace it with your own. The # DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top # of each HTML page. A value of NO enables the index and the value YES disables # it. Since the tabs in the index contain the same information as the navigation # tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. DISABLE_INDEX = NO # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag # value is set to YES, a side panel will be generated containing a tree-like # index structure (just like the one that is generated for HTML Help). For this # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can # further fine-tune the look of the index. As an example, the default style # sheet generated by doxygen has an example that shows how to put an image at # the root of the tree instead of the PROJECT_NAME. Since the tree basically has # the same information as the tab index, you could consider setting # DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # # Note that a value of 0 will completely suppress the enum values from appearing # in the overview section. # Minimum value: 0, maximum value: 20, default value: 4. # This tag requires that the tag GENERATE_HTML is set to YES. ENUM_VALUES_PER_LINE = 4 # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used # to set the initial width (in pixels) of the frame in which the tree is shown. # Minimum value: 0, maximum value: 1500, default value: 250. # This tag requires that the tag GENERATE_HTML is set to YES. TREEVIEW_WIDTH = 250 # When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to # external symbols imported via tag files in a separate window. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. EXT_LINKS_IN_WINDOW = NO # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful # doxygen run you need to manually remove any form_*.png images from the HTML # output directory to force them to be regenerated. # Minimum value: 8, maximum value: 50, default value: 10. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_FONTSIZE = 10 # Use the FORMULA_TRANPARENT tag to determine whether or not the images # generated for formulas are transparent PNGs. Transparent PNGs are not # supported properly for IE 6.0, but are supported on all modern browsers. # # Note that when changing this option you need to delete any form_*.png files in # the HTML output directory before the changes have effect. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_TRANSPARENT = YES # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see # http://www.mathjax.org) which uses client side Javascript for the rendering # instead of using prerendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path # to it using the MATHJAX_RELPATH option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. USE_MATHJAX = NO # When MathJax is enabled you can set the default output format to be used for # the MathJax output. See the MathJax site (see: # http://docs.mathjax.org/en/latest/output.html) for more details. # Possible values are: HTML-CSS (which is slower, but has the best # compatibility), NativeMML (i.e. MathML) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. #MATHJAX_FORMAT = HTML-CSS # When MathJax is enabled you need to specify the location relative to the HTML # output directory using the MATHJAX_RELPATH option. The destination directory # should contain the MathJax.js script. For instance, if the mathjax directory # is located at the same level as the HTML output directory, then # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of # MathJax from http://www.mathjax.org before deployment. # The default value is: http://cdn.mathjax.org/mathjax/latest. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://www.mathjax.org/mathjax # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site # (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. #MATHJAX_CODEFILE = # When the SEARCHENGINE tag is enabled doxygen will generate a search box for # the HTML output. The underlying search engine uses javascript and DHTML and # should work on any modern browser. Note that when using HTML help # (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) # there is already a search function so this one should typically be disabled. # For large projects the javascript based search engine can be slow, then # enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to # search using the keyboard; to jump to the search box use <access key> + S # (what the <access key> is depends on the OS and browser, but it is typically # <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down # key> to jump into the search results window, the results can be navigated # using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel # the search. The filter options can be selected when the cursor is inside the # search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys> # to select a filter and <Enter> or <escape> to activate or cancel the filter # option. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. SEARCHENGINE = YES # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using Javascript. There # are two flavors of web server based searching depending on the EXTERNAL_SEARCH # setting. When disabled, doxygen will generate a PHP script for searching and # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing # and searching needs to be provided by external tools. See the section # "External Indexing and Searching" for details. # The default value is: NO. # This tag requires that the tag SEARCHENGINE is set to YES. SERVER_BASED_SEARCH = NO # When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP # script for searching. Instead the search results are written to an XML file # which needs to be processed by an external indexer. Doxygen will invoke an # external search engine pointed to by the SEARCHENGINE_URL option to obtain the # search results. # # Doxygen ships with an example indexer ( doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library # Xapian (see: http://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. # This tag requires that the tag SEARCHENGINE is set to YES. #EXTERNAL_SEARCH = NO # The SEARCHENGINE_URL should point to a search engine hosted by a web server # which will return the search results when EXTERNAL_SEARCH is enabled. # # Doxygen ships with an example indexer ( doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library # Xapian (see: http://xapian.org/). See the section "External Indexing and # Searching" for details. # This tag requires that the tag SEARCHENGINE is set to YES. #SEARCHENGINE_URL = # When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed # search data is written to a file for indexing by an external tool. With the # SEARCHDATA_FILE tag the name of this file can be specified. # The default file is: searchdata.xml. # This tag requires that the tag SEARCHENGINE is set to YES. #SEARCHDATA_FILE = searchdata.xml # When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the # EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is # useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple # projects and redirect the results back to the right project. # This tag requires that the tag SEARCHENGINE is set to YES. #EXTERNAL_SEARCH_ID = # The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen # projects other than the one defined by this configuration file, but that are # all added to the same external search index. Each project needs to have a # unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of # to a relative location where the documentation can be found. The format is: # EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ... # This tag requires that the tag SEARCHENGINE is set to YES. #EXTRA_SEARCH_MAPPINGS = #--------------------------------------------------------------------------- # Configuration options related to the LaTeX output #--------------------------------------------------------------------------- # If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output. # The default value is: YES. GENERATE_LATEX = YES # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. # The default directory is: latex. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. # # Note that when enabling USE_PDFLATEX this option is only used for generating # bitmaps for formulas in the HTML output, but not in the Makefile that is # written to the output directory. # The default file is: latex. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate # index for LaTeX. # The default file is: makeindex. # This tag requires that the tag GENERATE_LATEX is set to YES. MAKEINDEX_CMD_NAME = makeindex # If the COMPACT_LATEX tag is set to YES doxygen generates more compact LaTeX # documents. This may be useful for small projects and may help to save some # trees in general. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. COMPACT_LATEX = NO # The PAPER_TYPE tag can be used to set the paper type that is used by the # printer. # Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x # 14 inches) and executive (7.25 x 10.5 inches). # The default value is: a4. # This tag requires that the tag GENERATE_LATEX is set to YES. PAPER_TYPE = a4 # The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names # that should be included in the LaTeX output. To get the times font for # instance you can specify # EXTRA_PACKAGES=times # If left blank no extra packages will be included. # This tag requires that the tag GENERATE_LATEX is set to YES. EXTRA_PACKAGES = # The LATEX_HEADER tag can be used to specify a personal LaTeX header for the # generated LaTeX document. The header should contain everything until the first # chapter. If it is left blank doxygen will generate a standard header. See # section "Doxygen usage" for information on how to let doxygen write the # default header to a separate file. # # Note: Only use a user-defined header if you know what you are doing! The # following commands have a special meaning inside the header: $title, # $datetime, $date, $doxygenversion, $projectname, $projectnumber, # $projectbrief, $projectlogo. Doxygen will replace $title with the empy string, # for the replacement values of the other commands the user is refered to # HTML_HEADER. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = # The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the # generated LaTeX document. The footer should contain everything after the last # chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what # special commands can be used inside the footer. # # Note: Only use a user-defined footer if you know what you are doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = # The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or # other source files which should be copied to the LATEX_OUTPUT output # directory. Note that the files will be copied as-is; there are no commands or # markers available. # This tag requires that the tag GENERATE_LATEX is set to YES. #LATEX_EXTRA_FILES = # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is # prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will # contain links (just like the HTML output) instead of page references. This # makes the output suitable for online browsing using a PDF viewer. # The default value is: YES. # This tag requires that the tag GENERATE_LATEX is set to YES. PDF_HYPERLINKS = YES # If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate # the PDF file directly from the LaTeX files. Set this option to YES to get a # higher quality PDF documentation. # The default value is: YES. # This tag requires that the tag GENERATE_LATEX is set to YES. USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode # command to the generated LaTeX files. This will instruct LaTeX to keep running # if errors occur, instead of asking the user for help. This option is also used # when generating formulas in HTML. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_BATCHMODE = NO # If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the # index chapters (such as File Index, Compound Index, etc.) in the output. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HIDE_INDICES = NO # If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source # code with syntax highlighting in the LaTeX output. # # Note that which sources are shown also depends on other settings such as # SOURCE_BROWSER. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_SOURCE_CODE = NO # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # http://en.wikipedia.org/wiki/BibTeX and \cite for more info. # The default value is: plain. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_BIB_STYLE = plain #--------------------------------------------------------------------------- # Configuration options related to the RTF output #--------------------------------------------------------------------------- # If the GENERATE_RTF tag is set to YES doxygen will generate RTF output. The # RTF output is optimized for Word 97 and may not look too pretty with other RTF # readers/editors. # The default value is: NO. GENERATE_RTF = NO # The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. # The default directory is: rtf. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_OUTPUT = rtf # If the COMPACT_RTF tag is set to YES doxygen generates more compact RTF # documents. This may be useful for small projects and may help to save some # trees in general. # The default value is: NO. # This tag requires that the tag GENERATE_RTF is set to YES. COMPACT_RTF = NO # If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will # contain hyperlink fields. The RTF file will contain links (just like the HTML # output) instead of page references. This makes the output suitable for online # browsing using Word or some other Word compatible readers that support those # fields. # # Note: WordPad (write) and others do not support links. # The default value is: NO. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_HYPERLINKS = NO # Load stylesheet definitions from file. Syntax is similar to doxygen's config # file, i.e. a series of assignments. You only have to provide replacements, # missing definitions are set to their default value. # # See also section "Doxygen usage" for information on how to generate the # default style sheet that doxygen normally uses. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an RTF document. Syntax is # similar to doxygen's config file. A template extensions file can be generated # using doxygen -e rtf extensionFile. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_EXTENSIONS_FILE = #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- # If the GENERATE_MAN tag is set to YES doxygen will generate man pages for # classes and files. # The default value is: NO. GENERATE_MAN = NO # The MAN_OUTPUT tag is used to specify where the man pages will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. A directory man3 will be created inside the directory specified by # MAN_OUTPUT. # The default directory is: man. # This tag requires that the tag GENERATE_MAN is set to YES. MAN_OUTPUT = man # The MAN_EXTENSION tag determines the extension that is added to the generated # man pages. In case the manual section does not start with a number, the number # 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is # optional. # The default value is: .3. # This tag requires that the tag GENERATE_MAN is set to YES. MAN_EXTENSION = .3 # The MAN_SUBDIR tag determines the name of the directory created within # MAN_OUTPUT in which the man pages are placed. If defaults to man followed by # MAN_EXTENSION with the initial . removed. # This tag requires that the tag GENERATE_MAN is set to YES. #MAN_SUBDIR = # If the MAN_LINKS tag is set to YES and doxygen generates man output, then it # will generate one additional man file for each entity documented in the real # man page(s). These additional files only source the real man page, but without # them the man command would be unable to find the correct page. # The default value is: NO. # This tag requires that the tag GENERATE_MAN is set to YES. MAN_LINKS = NO #--------------------------------------------------------------------------- # Configuration options related to the XML output #--------------------------------------------------------------------------- # If the GENERATE_XML tag is set to YES doxygen will generate an XML file that # captures the structure of the code including all documentation. # The default value is: NO. GENERATE_XML = YES # The XML_OUTPUT tag is used to specify where the XML pages will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. # The default directory is: xml. # This tag requires that the tag GENERATE_XML is set to YES. XML_OUTPUT = xml # If the XML_PROGRAMLISTING tag is set to YES doxygen will dump the program # listings (including syntax highlighting and cross-referencing information) to # the XML output. Note that enabling this will significantly increase the size # of the XML output. # The default value is: YES. # This tag requires that the tag GENERATE_XML is set to YES. XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- # Configuration options related to the DOCBOOK output #--------------------------------------------------------------------------- # If the GENERATE_DOCBOOK tag is set to YES doxygen will generate Docbook files # that can be used to generate PDF. # The default value is: NO. #GENERATE_DOCBOOK = NO # The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be put in # front of it. # The default directory is: docbook. # This tag requires that the tag GENERATE_DOCBOOK is set to YES. #DOCBOOK_OUTPUT = docbook # If the DOCBOOK_PROGRAMLISTING tag is set to YES doxygen will include the # program listings (including syntax highlighting and cross-referencing # information) to the DOCBOOK output. Note that enabling this will significantly # increase the size of the DOCBOOK output. # The default value is: NO. # This tag requires that the tag GENERATE_DOCBOOK is set to YES. #DOCBOOK_PROGRAMLISTING = NO #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES doxygen will generate an AutoGen # Definitions (see http://autogen.sf.net) file that captures the structure of # the code including all documentation. Note that this feature is still # experimental and incomplete at the moment. # The default value is: NO. GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # Configuration options related to the Perl module output #--------------------------------------------------------------------------- # If the GENERATE_PERLMOD tag is set to YES doxygen will generate a Perl module # file that captures the structure of the code including all documentation. # # Note that this feature is still experimental and incomplete at the moment. # The default value is: NO. GENERATE_PERLMOD = NO # If the PERLMOD_LATEX tag is set to YES doxygen will generate the necessary # Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI # output from the Perl module output. # The default value is: NO. # This tag requires that the tag GENERATE_PERLMOD is set to YES. PERLMOD_LATEX = NO # If the PERLMOD_PRETTY tag is set to YES the Perl module output will be nicely # formatted so it can be parsed by a human reader. This is useful if you want to # understand what is going on. On the other hand, if this tag is set to NO the # size of the Perl module output will be much smaller and Perl will parse it # just the same. # The default value is: YES. # This tag requires that the tag GENERATE_PERLMOD is set to YES. PERLMOD_PRETTY = YES # The names of the make variables in the generated doxyrules.make file are # prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful # so different doxyrules.make files included by the same Makefile don't # overwrite each other's variables. # This tag requires that the tag GENERATE_PERLMOD is set to YES. PERLMOD_MAKEVAR_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- # If the ENABLE_PREPROCESSING tag is set to YES doxygen will evaluate all # C-preprocessor directives found in the sources and include files. # The default value is: YES. ENABLE_PREPROCESSING = YES # If the MACRO_EXPANSION tag is set to YES doxygen will expand all macro names # in the source code. If set to NO only conditional compilation will be # performed. Macro expansion can be done in a controlled way by setting # EXPAND_ONLY_PREDEF to YES. # The default value is: NO. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. MACRO_EXPANSION = NO # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then # the macro expansion is limited to the macros specified with the PREDEFINED and # EXPAND_AS_DEFINED tags. # The default value is: NO. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. EXPAND_ONLY_PREDEF = NO # If the SEARCH_INCLUDES tag is set to YES the includes files in the # INCLUDE_PATH will be searched if a #include is found. # The default value is: YES. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the # preprocessor. # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard # patterns (like *.h and *.hpp) to filter out the header-files in the # directories. If left blank, the patterns specified with FILE_PATTERNS will be # used. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. INCLUDE_FILE_PATTERNS = # The PREDEFINED tag can be used to specify one or more macro names that are # defined before the preprocessor is started (similar to the -D option of e.g. # gcc). The argument of the tag is a list of macros of the form: name or # name=definition (no spaces). If the definition and the "=" are omitted, "=1" # is assumed. To prevent a macro definition from being undefined via #undef or # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. PREDEFINED = DMLC_USE_CXX11 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The # macro definition that is found in the sources will be used. Use the PREDEFINED # tag if you want to use a different macro definition that overrules the # definition found in the source code. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. EXPAND_AS_DEFINED = # If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will # remove all references to function-like macros that are alone on a line, have # an all uppercase name, and do not end with a semicolon. Such function macros # are typically used for boiler-plate code, and will confuse the parser if not # removed. # The default value is: YES. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration options related to external references #--------------------------------------------------------------------------- # The TAGFILES tag can be used to specify one or more tag files. For each tag # file the location of the external documentation should be added. The format of # a tag file without this location is as follows: # TAGFILES = file1 file2 ... # Adding location for the tag files is done as follows: # TAGFILES = file1=loc1 "file2 = loc2" ... # where loc1 and loc2 can be relative or absolute paths or URLs. See the # section "Linking to external documentation" for more information about the use # of tag files. # Note: Each tag file must have a unique name (where the name does NOT include # the path). If a tag file is not located in the directory in which doxygen is # run, you must also specify the path to the tagfile here. TAGFILES = # When a file name is specified after GENERATE_TAGFILE, doxygen will create a # tag file that is based on the input files it reads. See section "Linking to # external documentation" for more information about the usage of tag files. GENERATE_TAGFILE = # If the ALLEXTERNALS tag is set to YES all external class will be listed in the # class index. If set to NO only the inherited external classes will be listed. # The default value is: NO. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed in # the modules index. If set to NO, only the current project's groups will be # listed. # The default value is: YES. EXTERNAL_GROUPS = YES # If the EXTERNAL_PAGES tag is set to YES all external pages will be listed in # the related pages index. If set to NO, only the current project's pages will # be listed. # The default value is: YES. #EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- # If the CLASS_DIAGRAMS tag is set to YES doxygen will generate a class diagram # (in HTML and LaTeX) for classes with base or super classes. Setting the tag to # NO turns the diagrams off. Note that this option also works with HAVE_DOT # disabled, but it is recommended to install and use dot, since it yields more # powerful graphs. # The default value is: YES. CLASS_DIAGRAMS = YES # You can include diagrams made with dia in doxygen documentation. Doxygen will # then run dia to produce the diagram and insert it in the documentation. The # DIA_PATH tag allows you to specify the directory where the dia binary resides. # If left empty dia is assumed to be found in the default search path. #DIA_PATH = # If set to YES, the inheritance and collaboration graphs will hide inheritance # and usage relations if the target is undocumented or is not a class. # The default value is: YES. HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: # http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO # The default value is: YES. HAVE_DOT = YES # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed # to run in parallel. When set to 0 doxygen will base this on the number of # processors available in the system. You can set it explicitly to a value # larger than 0 to get control over the balance between CPU load and processing # speed. # Minimum value: 0, maximum value: 32, default value: 0. # This tag requires that the tag HAVE_DOT is set to YES. DOT_NUM_THREADS = 0 # When you want a differently looking font in the dot files that doxygen # generates you can specify the font name using DOT_FONTNAME. You need to make # sure dot is able to find the font, which can be done by putting it in a # standard location or by setting the DOTFONTPATH environment variable or by # setting DOT_FONTPATH to the directory containing the font. # The default value is: Helvetica. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTNAME = Helvetica # The DOT_FONTSIZE tag can be used to set the size (in points) of the font of # dot graphs. # Minimum value: 4, maximum value: 24, default value: 10. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTSIZE = 10 # By default doxygen will tell dot to use the default font as specified with # DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set # the path where dot can find it using this tag. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTPATH = # If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for # each documented class showing the direct and indirect inheritance relations. # Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a # graph for each documented class showing the direct and indirect implementation # dependencies (inheritance, containment, and class references variables) of the # class with other documented classes. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for # groups, showing the direct groups dependencies. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GROUP_GRAPHS = YES # If the UML_LOOK tag is set to YES doxygen will generate inheritance and # collaboration diagrams in a style similar to the OMG's Unified Modeling # Language. # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. UML_LOOK = YES # If the UML_LOOK tag is enabled, the fields and methods are shown inside the # class node. If there are many fields or methods and many nodes the graph may # become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the # number of items for each type to make the size more manageable. Set this to 0 # for no limit. Note that the threshold may be exceeded by 50% before the limit # is enforced. So when you set the threshold to 10, up to 15 fields may appear, # but if the number exceeds 15, the total amount of fields shown is limited to # 10. # Minimum value: 0, maximum value: 100, default value: 10. # This tag requires that the tag HAVE_DOT is set to YES. #UML_LIMIT_NUM_FIELDS = 10 # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and # collaboration graphs will show the relations between templates and their # instances. # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. TEMPLATE_RELATIONS = NO # If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to # YES then doxygen will generate a graph for each documented file showing the # direct and indirect include dependencies of the file with other documented # files. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. INCLUDE_GRAPH = YES # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are # set to YES then doxygen will generate a graph for each documented file showing # the direct and indirect include dependencies of the file with other documented # files. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. INCLUDED_BY_GRAPH = YES # If the CALL_GRAPH tag is set to YES then doxygen will generate a call # dependency graph for every global function or class method. # # Note that enabling this option will significantly increase the time of a run. # So in most cases it will be better to enable call graphs for selected # functions only using the \callgraph command. # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. CALL_GRAPH = NO # If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller # dependency graph for every global function or class method. # # Note that enabling this option will significantly increase the time of a run. # So in most cases it will be better to enable caller graphs for selected # functions only using the \callergraph command. # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. CALLER_GRAPH = NO # If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical # hierarchy of all classes instead of a textual one. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the # dependencies a directory has on other directories in a graphical way. The # dependency relations are determined by the #include relations between the # files in the directories. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. DIRECTORY_GRAPH = YES # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order # to make the SVG files visible in IE 9+ (other browsers do not have this # requirement). # Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd, # png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo, # gif:cairo:gd, gif:gd, gif:gd:gd and svg. # The default value is: png. # This tag requires that the tag HAVE_DOT is set to YES. DOT_IMAGE_FORMAT = png # If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to # enable generation of interactive SVG images that allow zooming and panning. # # Note that this requires a modern browser other than Internet Explorer. Tested # and working are Firefox, Chrome, Safari, and Opera. # Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make # the SVG files visible. Older versions of IE do not have SVG support. # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. INTERACTIVE_SVG = NO # The DOT_PATH tag can be used to specify the path where the dot tool can be # found. If left blank, it is assumed the dot tool can be found in the path. # This tag requires that the tag HAVE_DOT is set to YES. DOT_PATH = # The DOTFILE_DIRS tag can be used to specify one or more directories that # contain dot files that are included in the documentation (see the \dotfile # command). # This tag requires that the tag HAVE_DOT is set to YES. DOTFILE_DIRS = # The MSCFILE_DIRS tag can be used to specify one or more directories that # contain msc files that are included in the documentation (see the \mscfile # command). MSCFILE_DIRS = # The DIAFILE_DIRS tag can be used to specify one or more directories that # contain dia files that are included in the documentation (see the \diafile # command). #DIAFILE_DIRS = # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the # path where java can find the plantuml.jar file. If left blank, it is assumed # PlantUML is not used or called during a preprocessing step. Doxygen will # generate a warning when it encounters a \startuml command in this case and # will not generate output for the diagram. # This tag requires that the tag HAVE_DOT is set to YES. #PLANTUML_JAR_PATH = # The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes # that will be shown in the graph. If the number of nodes in a graph becomes # larger than this value, doxygen will truncate the graph, which is visualized # by representing a node as a red box. Note that doxygen if the number of direct # children of the root node in a graph is already larger than # DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that # the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. # Minimum value: 0, maximum value: 10000, default value: 50. # This tag requires that the tag HAVE_DOT is set to YES. DOT_GRAPH_MAX_NODES = 50 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs # generated by dot. A depth value of 3 means that only nodes reachable from the # root by following a path via at most 3 edges will be shown. Nodes that lay # further from the root node will be omitted. Note that setting this option to 1 # or 2 may greatly reduce the computation time needed for large code bases. Also # note that the size of a graph can be further restricted by # DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. # Minimum value: 0, maximum value: 1000, default value: 0. # This tag requires that the tag HAVE_DOT is set to YES. MAX_DOT_GRAPH_DEPTH = 0 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent # background. This is disabled by default, because dot on Windows does not seem # to support this out of the box. # # Warning: Depending on the platform used, enabling this option may lead to # badly anti-aliased labels on the edges of a graph (i.e. they become hard to # read). # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. DOT_TRANSPARENT = NO # Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) support # this, this feature is disabled by default. # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. DOT_MULTI_TARGETS = YES # If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page # explaining the meaning of the various boxes and arrows in the dot generated # graphs. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES doxygen will remove the intermediate dot # files that are used to generate the various graphs. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. DOT_CLEANUP = YES ����������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/Doxyfile.in�����������������������������������������������0000664�0000000�0000000�00000001425�15270523712�0022471�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������PROJECT_NAME = @PROJECT_NAME@ PROJECT_NUMBER = @PROJECT_VERSION@ PROJECT_BRIEF = "Common in-memory tensor structure and operator interface for deep learning and other systems" STRIP_FROM_PATH = @PROJECT_SOURCE_DIR@ \ @PROJECT_BINARY_DIR@ OUTPUT_LANGUAGE = English FILE_PATTERNS = *.h *.md RECURSIVE = YES INPUT = @CMAKE_SOURCE_DIR@/include/dlpack IMAGE_PATH = @CMAKE_SOURCE_DIR@/docs USE_MDFILE_AS_MAINPAGE = @CMAKE_SOURCE_DIR@/docs/readme.md JAVADOC_AUTOBRIEF = YES GENERATE_HTML = NO GENERATE_LATEX = NO GENERATE_XML = YES XML_OUTPUT = xml XML_PROGRAMLISTING = YES OUTPUT_DIRECTORY = @CMAKE_BINARY_DIR@/docs/doxygen �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/Makefile��������������������������������������������������0000664�0000000�0000000�00000001277�15270523712�0022023�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= -Dbreathe_projects.dlpack=../doxygen/xml -WT --keep-going SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build/latest # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -b $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/README.md�������������������������������������������������0000664�0000000�0000000�00000001323�15270523712�0021632�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Documentation for DLPack ## Building Locally The following dependencies must be downloaded in order to build docs: - Doxygen (On debian distros, simply run `sudo apt -y install doxygen`) - Python dependencies present in the `doc_requirements.txt` file in the root directory of the project. Run `python3 -m pip install -r doc_requirements.txt` to install them. Once the dependencies are installed, docs can be built using either CMake or the Makefile from the root directory of the project. - Using Makefile: Run `make doc` to build the HTML pages. Run `make show_docs` to serve the website locally. - Using CMake: Build with `BUILD_DOCS` option `ON`: `mkdir -p build && cd build && cmake .. -DBUILD_DOCS=ON && make` �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/make.bat��������������������������������������������������0000664�0000000�0000000�00000001565�15270523712�0021770�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������@ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=source set SPHINXOPTS=-Dbreathe_projects.dlpack=../doxygen/xml -WT --keep-going set BUILDDIR=build/latest if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.https://www.sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -b %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd �������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/source/���������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021654�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/source/_static/�������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0023302�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/source/_static/images/������������������������������������0000775�0000000�0000000�00000000000�15270523712�0024547�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/source/_static/images/DLPack_diagram.png������������������0000664�0000000�0000000�00000056617�15270523712�0030056�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������PNG  ��� IHDR��w��,���O'���gAMA�� a��� cHRM��z&���������u0��`��:��pQ<��PLTEٯzzz]]]ꈈmmm÷ۆ3333;F-?V6f!W$NĴpru(Fm(Jt<l*]$Y3dpa㏪T}Ht!W38@,?W*@]5W?HVJ_z@KW@JWdusvzY|]zw;CKKa~Nh@JV;AJ;EP;BKmZh���bKGD�H��� pHYs����Iu���tIME�~S��[ZIDATx {ؕ9��8JiͶ,g.U R?R}[ rOK���������������������������������������������������AP��R0J , Ө-/ �8&JQפm##! x�%F±J] "[�Að-T+Qw��P2ᗾwdx�5- ;5��R2 +cL��Q`|yGa�ؕvV.{Z2MlL6�@t&Y\iTk;fЮlŢ}GWR 5��6Rֱ<Q $%T2l ��Q(ʲNzu(%nx%wL~ ��"0xʆQ+-Wkk}w��Z{VVRSe"7Iת��D6jVM/ngU1G� 2͕,��H #v�� %,ۨ;��N Yi;��N 1�dL�lh�r�@Ƙn�|��;Yՠv<XĠ �KF1YϼS6Mz?G! n2�z"Kg=Ee/�;z0뵭&��'wgYۆ- zB��X!wgU,{R]Tqe�Z"zg%"aͲwlhMm ,�wFMhY�!w+,[/Գ[4m,�;,6/ڕ ^A2�` Q?6* 8A?zE*6Z�k<np9i asCUINV Z�kY%1)h[ :T*Pȯ^e�o.Ϣ%R^5lfj5,�sKv޿=�|�w��Y���d �x�5� k�@lgUL$�lg>;�6zg�'YSw�8%2ɟ%Ө%ndlV]kZBk-�<2 i?-Psu2U �pJd?4-RWkUѮ5En)I,YVG9�$6ʙ:EYjpII,՟Ew=<qj $; ]3 c;�Zx'ܾX4IIwLd@rMG1�c Y!lJӤS(mYN�NLg#t,cI^uziC,k_�p`/�@;�w��Y���d �x�5� kNbc!߿;:~xoL6IYيzhAB`bB y?fˀӻO?cHO1CU(h=ZǪ[v!=~g/yiBz1gь;&ˑdZ4w~Ǽ}xN2Q *R* eYMST?U.WΛ,>˫秼)fL &)l.M"2jUhc93CQO?Sa;ϟ1.IEJ'=" ;\lVb2 YLvg5Eow_[GLiDCޠYIj?ZTYk`wM׼ $aLwNkB!Tk;i�aL׳*\VcI%H[C[\ivwIx'ØFoWzU6M坪b;hIx' cS?:ǡUCXNЏ. TwIx'Ø2n&eJQ)CEX,q%awIx'ØFN¦YfLe+6ϪCX%~"P/IL% @x;tBKPcZ(;(R"\_ޑ;prfWiK@x;tYmRF˪n;qFr;)&N1qޯwR ?<M c "N ;9Dw2ޱ]vx'0;חUV7ty\__1O8Hr޹\|ײI6'W}#]h:wÿV"'p㟂\J:Hr9WGxp~WIxssCo~xnQ/^d ?*1;:i[;*rJ{i0u3~;Z y$w:;OwB|NAږQ>{+~ob;)DF#Wa;y|7x*^SEO7ums}M//U޹yЫ{0n?\?\>=!6)w/z\$Mó %vwL]~ywW ]3D\?ݧkOOw\Y-,h-OS)c5{/n/Ws~+.ՉxwnrQ'W2 ȻH_7n߯zᖷxſxO\ Nۜ絾lӬ5VPb_E/yAW=W@<^)T\r2&L|L UxE V^j3] 㥾cz)^\ν{#ORw7W䩝?ˏjGu<J\^ u/ˇG #>tPu~/7|_Ez##Q,&qÐg)sΕM(*2HIZZ>Rb~4<_ZQ>ǛQ,/{Ë..[I`ӶZ-:P\;R{{<Fy#Q}٪w՗jBV>u\4BvYMb^a\Zs)�^BoK*&zwi3״w8vIQL] ]/c}uS0 f(Űw*fI4g&.V0zGv;ZȀ/keNs}|T;P*x :LC٩Kn2\yԽ/a?GUU΃ރn$N1 yۍf ⨯C~NKW*FHI* SyvFٴyj¦Q3l=[O.l_ޑZxvΥ{UVgڕ|r?ͭ<8('>׹k uۭ9Y{'Mb^+均zUιAR[^Ww`{'.wn_/|W#r'n9>ow\'rޑFa=FuwvȪS# pZm ʒM;ە(^Eoȯ[>y uU~&#eF{ZR7iy*Ռv?c^;W/zz9>ڏܾR͋+~r3#}/׋JUh grd|4(;*N# <AOè+/ȒWsh~qθoT5w.95K<?+\?}jz+(wHK*ҥ _妊=N~1]Ϻ~?]PRP ho<}1rQwɬVdԼ)Q˭5<w {uvVI<>qwn>7{Z.뫕ZwXޡ+fW\Z| zGwӹ wc4~%+f)Pj\W9___'^jL,I崹w =SjpCt~}]/{Z,sC|f;Fy|yF乾xgNuĿ?=<}V7K޹5z^j|)xfL]$s޻xj+ x;.#S?@4zWCp޼beR S;U+[;w.wQǫk>W_BV<.uokY L^ZP$q=W>>Pϲw)򍏗KޑNxӏ~#Mq=_V [g-ks!οP{dcJ;B&VW#_}8T|!J7}t߶k=FLxNl:^i XGP].A*l&̺'\/yG| R8W_zUJeީD-q $5J=ڔ1Ͼ⓿G;qn?-+/XWh;p=^sn:1; ᵩeQ6u5:Sa\fb݇ B?͢޸|tOsâ6p~}?rŵ@wi c1ί1wv["wCxNBkR�te>A{5H# އ ϯxBI1.DZ({Ї@Woz1;m觴db;j)FllG9WiaL<4w9?2kby1.<^R+i<nPvn7{"޹b!�aLw_R<bgF;)�dӓ+nu3;)�dSxwR�0!�aLCIx'Ø;N1wx' c "N ;9Dw2i)8xg=G^�0Q4 Zuz6"U]6YH xg=G^�0[x,c; cu[x']w2itz3wR�0ѽSsԌ{eچ)]FiJU;�aL{jsQ7;cT*{n@T(wIx'Øn oZ2˦vB9CiSޭ;�aLN'umAi)O*ӮS/w%ǿ;diM)wٔK3W9u/??XȟOfi2NQԍ{ �ij*uxٟ{Zqu?ŸmcwFݢT䝊a ;%X1ﰧ#,G4ØnYi Y1嶡ny?XK}Z%ΥnӭS4lJNi")JYJ*O/osd}8ډySALVޑ *٘:'n5ʗE{jQw>sޑ1w"t;QM<d)ڦZuj=۳/:Gȷ/ճS4 ʲl&_|Z|#,1M"g<7gG-B|˫1ELiO=||5ﰧ_vK{1M"&7%~;V ,cj-f<:~kQOALbڤ6G #i1l~8wփ{19C]z;=Ӫ.=JN'/{=[73#˾G!]r8.?&E0t9w;ɱ6('ԺS1 {5kA2H!j'A X3"z˞ng&io`=grL5Î<V{Y?+< ꓔwOȓL?<OqgT;N6Ӟ G|A_3H>VGTtpt띷,K4մI/?Pݟ}_ݙL/{(\6]WNML~{]f\KqL{ rYn丄l1,X[ԋA tw:uOMz<TM+iF;{#>h6IK:=gV Gѝ 4veNAuG|?y޾ /p*tU Y\M8,יּ }]+q Ud\y#Oǯb&W<-NnzVcZ,lFX(TZ*ŢY坪Y6j%n6(y5]'ykӣ:,>љ'/Fy(-g><w V1B'cxKf(fc?b=R#O5r'x?Myg"E72._>rw/Kiv6:KLx'NC9:Eۉg)JfخIW$a EC-wd1ʖЬ< ۰ Z$.5q9'%L;]>=ھ|nG=y130 gp1ތ4-XǤ$}<iPޡ/s=rc4NwuE\Jڤ=y_]]44БiCL+&On[YQVA6J J4RuѮlFUՌ ,Qxn*9qW6D}ӏ66#fݕQV驘0)FѢ(~SoS1%M䱾OriNx#i9c8X,zʤ[Tء-5;+r5U2-5Æy4ޙYzR x(ʛMԷH$=Rm}gx?S7ӓvJϿI;|p|ʲu8.l6Qr#7'=*s4W}]ΖF(X(W]-Q,*~.;eA:J8D<I{ʩ뽣g7Rk+8CbyǬU\CwwB?)8/ճt91 `&~ UAhޙJ`}|vN;X9qp jE4;ڕ{ϼ^3|ͶTIhP֮ޱC'3n-r}PO;ʵޡ@".wSwFkSu5U٩ ?KpFU Rg]J*K坶NK-6;ÉnMatNs}ļӮ@;nLcxghT{G)Tz}g;?!kkY)G/f6,jwnhq}ŏwQ(slm4,w׋USqt瞺7gQU_uzvTUyă(wi h  eQ7I9]uC"?kF wx5!{/^gwfBq ]w\GպԟLnqgЙo Ye 0eUX6ӪS)C;Oez}34>q�;2LN9O;]G$<R8Kw>VnwʡqA+!?%Q]As ?' 1tjd_3tui83[=EH"\Ke;Ubs'WӠlZ?~pf!4KC<\gܡJiH֙<uN;<?`إgttO+�RG"[>1;ldxH7/֘a&py"LG3WKr< 49nL~9kSw|F%zGeժ%֬wSYgd5WLw@~M~(j5 SEIxgbz\ޙsbzIw-+ei53<z;P;ۍNTWոJ!ux'ywƎnZhLS'A=fF)|N=/1=&dӴgY9fɁw(;[;sH-cz9{$Ζ1*x'yN�eLqfo~w;W1wal=O^~_kQOӵi4wzZQwQL&5FݦgvXҼ9wtb;)97l^32r,S:^n;O|>ψ^4wN3ѽa7KqǪ6,=g_ۗ^4wN2zG m -~?n-JժfKK9V~>o?2ܴ'w(#?^)Ј]aCRmMJ:.|9o ψϣ㿒%XYĹJ*/7 ʢU7(V)COSԳitRQ(:ϥl,oKۼ_^S+omUPzR3 7iO"y4wJj7ֳ쥣:Y91nmceNE5(7UhnU;xN"yNsLfĶ1;| - T;x.4qsYti5 7!.@x'.yl6;Hs9B9LӼG6lӭӶUUޭrEׯRm x'.yl6qw{h_NاU۽?z2R:qS:}Nٜ n2qN;Mn֬t =!^옒;?OA{Gfp*^Eps yqD/ދ&gx'CNْw5pޙf+dY79ygq{g[]|<-BsowZNtqo"wx;b=Ie?xtC.p3+#Dstя('1}h*hp&r"\)瞬|Zz%מv Ą9¯o8|Ng8 go_<Urpuܹ^S!o5Ix'q%y='%(# ٕASA(_Á,oz+A՝ήz2c/Չi-?]Uqߑk<ֈY;"Oԕ[syu❮ĽCgcw&SN\ \KÐΝGʷg<{d#8 D"(13SƁ9su)x\a'<##fC x)'Ò"2bH_]:p{|VGKwG>o7)nWrRzg o.{H )inEmDO<=z,tT^jҽ="pJڦ+щ\"t";{g"p]y;}.wRY.\х;RV碫n~G\;䝩Zd U_0rpsB-{o{CQv8љ :@ohO"tĸG6Q1S5`"zts5b6Z?]ٕoFT㇣C](jV|8sbFrѕ+Y!BѧL>Y}w/{gzzq_Μ˄ GbyvY7n`;ƴڨ&SHO/6 ۅ>꣎ۥKvzgJ=/+U\Dy'hNܑt㳦QMSmn~wun889t+ywfVMEUiA&m~bjJ=΀Db:Z{L1Bײ/|w7޴+ϟ +ْWPNt;#nZ;88г#kܮ(z~t8~N;sygHp5w6ʮޱV1].,ODng鄖(Wy'~i$}wQ]!8|;K;gEF ǣaǯj桊֪wE{ }v)yŔuޙP-tn-'q%\<ͦdz;ދjO8}g5H}WFA}'~)Ϊw<T̨FAPI՝rHgSѝa0;ԕIU'Nqyg8uTOF|8S7>^:2X#U0<1AW(٧ǽ>ϢɾygY;RiTK3YviRKhMϮG,3sĤoy(܉Ȣu Fxcw6wVL(&U3etޑo (w@G];΀ EK\K1#zk@J.Ec7;ǔĘBeRXK<~[oc{mY)Y4cXٔ|+{P9c2gNg*/p|w͞T“CY;|$4>xyU(i=]*ꁁ1u:E}pM žOf#]zpO3ֽPFl2Wiy")3x ;VE'o O_t.?**kÑ:ݽQo=m*. 5!5F%%vcdPG,;3]Ns4f?ӮMvx+k/)ww,c]AA&tQ4JRJ3N_:'kMwӌ tgg%zSk@)h DWtKO<;it7V;RiwӼGC#1ȟNS-6Thw8rb|L˟N?ժH`*?> $i#i1.V;BVLO ]j4dNN=&gC hES-N=&gv>I+=﨧w#kQOS4wޜ};i b=x#>mg1^{ٗ&-b;#~>q9[1M)_/__{;S!ףWˎ;'Y~EL.'{$ӃIwR5ѦC=tALA2l"= 2dUgy ՛C5,GE{p4Ǹ~G杲I/'JCRl$4읒2d{')QOU6$o;G υ&1{'&Qf#D晕iE]TLR+[%6nvU.v\imT3E^ŲRݒZ]%-axxbzD" wd9?m+H0:OE,,5~W0WZeʽU3ڶŕybm#Ew@2DNmij eX4Nx=L\IhO- 9CUSq<EuK4lNYlS-x9&cK5ټ#e#!lj{n,ZMǽ+LYʱUcSUyͥbt~mq[|3wK1#Yzg ߜczD-l"dڋ]LɆQ V[Ŧi?%3(a`Ojjq]13=wf2=,J=K5,V'-˦Q 'R#,wt gQ#YzgE IT2d/j7"Ԯl:͢h;En<V#Ydp"c)YywAk|vIsr.IT,2dʼGm-~n!4)VZ+ivmB宰Ba;SOxu?=!1(X}i"~q{qDRP?K8{$ASei%,㤽cm&s-і;R[:]ئj\tBNE;R53OH C坮+oiggԓ]zY^�43Ew\!MXMw2%w8CVI?'Ar1hL2KH7*V1^-˫ޡ:\%,i1,1eJ'j]ưsTxvNř.)3tΌ $<]=b.ti8qb'-#=I4|UQ3V[pJmzaA-Ftw,XTYxSiՊ3ySpϥZ,AQ~Ľ3 GF.n .x#Q[K͎I;TiHv8, Yj{-}z|ĆVX(3W;g&G]c <G2+wIɷIw'?Γwi'N.x'Ktޯݠϖ1R {;c>;pv Αޡf9\iw&,EI%I D`;g9b;܌m1nV ;";xZxg3sw$V;)gOGgVN՟Ouz0bމFo)u�8;{'&ߙbxIx]\d ;pۇwN(αlss8b2U#ox! t;=D-7ެ߁w"6| +!r|x'KC9&y\Vl^K~i<Hmla|96mze?B~ZE~Vǽ\=m~XTXm&?dm`рw;TN_- eTSީ$Uh/m3RX4 YV. x'y QcU,چuk_X.�Ó2ךz S1Inmf yGM x'y Q *R!`V rYUdbX)_*YDA .ӴR1iUeZX;`Ս;d&hҥp4f2- X6MY)_p!%:K:&tjnD {g׀w=DT )U(rJ䠄iV剽SnIx$CDs *4u~ |sӔWwLU\BRej;-˿~JN4;u_ 4M`ӨQ~ Q9Tʾ  YxgXu;5%/\ީF/_/\-? ׳*\۟S ְjI-T;a3^NYDٷ/v&1]9=WʁwdyG'!5Vw򻔃FBu|/\S;[a4B%? x; T)wi赅#{ٗvm^ b aq y6$UtU5T m³qzO4~>GgsEdLߏkLOIؔ]wre6Kto̘RkTu3jzNl:='QSD/!gx'cx. {@ƔETTB.$XuN hpo 7Pe&imEŃ yЮ !y[4,D qЏ!k;~A xl6s r_P;㘽Bxudw@t ;="xun#xI;su@(;j_ w<^F_9x"zώY~"Ф5fE?Ph6דR$wf#0z Og3^ԑbB?'wJzZcC4w L'SIN[*61CZ>Iq Wτ5á.uuagg,蝢?U~.~1NFXzDS;MLHb;L$DGnңoƮpgOK1$| >BxxKbx a6g,rmd !%a;3ޑ>CEHѴ3tx wTLœ6Tg"559x�ڮ&lm~g*Q.a4ٴjw{y;9p{'e;M xcY.)J:jN'Z1\D1x'gzMӖ)i#dӁ8V27Ij4IC&wBzaw\R!itY9j o3ϐǵ;JSS5ΠG?T49NqMDer&w|8N;q&+S[Г}LT_uYljTSެGj흪?A`0Y 63}%KޙStɲwck#dY)%wfNf`Ad䤶M-̦*ꘋGuҏNW,Zߏ^IYJK!;ڕԮ<;Iyꝩ Tޙ󎤎i%wZY6aTJUXwDUpC$wF;C&3 2+ޑ[K;diWwL�9 h5F4dU%51>-<vLhx;Yaiˬ3 eиpjqY]7i-ήQs[yaFY@.UQ,*Xu~bg;Mèȝ9,@='9tؕbbLxsϑ SOKfoM亙j\�1#Wxa(W؝άg劭Zo'ڪ' wxQPR3á :]z;hȒm&H]¢qD#wچ\G(,mT 9mW7ml  {3wjlqӏnlMP4NX!Bd 9T;]GfO(�6{ x@' {<g0wZ8�) cr�^Y5krpr;w4 "z(�lYcwx'cQkKy!={\hZD]U 4<]ޜ}[iqwvCxTwV|xgG(ٞ|�۳/GYm;/WrϪwXmӔ5rI=vaUzkeƥ)Qf{v|~dqĠ]lkGxnf͠y@{f\kJՔ�beۨNx$7INF�Fz/GNT+4' =2?j+|6)K%0?(OeM&}*Ƭ YkwҢ>;;BmjSiƪrI3z/NKRs2tjfp]y-NZ4;QY*zßT\\[tzʨn wb}^&za;uu ;΁qwb;QsgNCT$;΁$Ūw}g*wZ. #Ber[.)S g5(Qq/msN:�Y-Y+wj;FQ(K6Fd%,Rn--H;Uέ8zX!uN:sYLd򄚆R;q<CY5; u66&4<R; m6 6Eq(vbc#Y/Ќ>62sYx] g1i󱑊w3yɈ 1ک#7%G)鐊w<)+;1󄐊 w<pń GfER=i`?I;}HĜdCӢףLP5'4H;EE-ŘVSΌ㈙zG1ZDTH5ŜQ_qzF#kW읹PnIp 4H;Ñ'=읁\xK7R?;� i.Uޑ%jqB ׳Ew�WGH) LDھJ:wX wdlCY-ux]/b~w dB;!w);�Գ4TС6w3?uF#$U3'%r[w<O᷀w�SjW_yIc˼_�' xw�_y=� =;�xw��{_Cz|e;�Ǜoy!-=.�)h+Z_> @=r%o_ @|=2s;�_zǟO?}c_VDZ% hW^K,%G_O,㧼/Xp`zbyv|y_s坼&=+ܞ s;Hxg=Y;n94�zJW2.; U 9`�zy.a;]xw@#;y:c{#%>/gޜvzrq9�{g(2 1uFBzhޑ&rgKCW.΄t:O � wb,rGB%3;1-DVzvfJ=o �H_ee!KH%^ԁwǤоzӼ`ZݥEWL'F5MA=aNWwd)i䛽N;Ue|onu$Uԝ-,N_w;<UT}}^ZysWWkww~w4V!LU΢|u7Gw>оwks+{}|.U~;`?I;n1 G{&䝱pބ“ =uz6;Yg>3[MVK747:; )NaxN;=W;RO™ybF ;) <s/zH܊[姻p/I1W-vQ޹P;m/ontEˍR!zbLcrp$'h=4Se(wswޝ <Oy <.{Gnxy/WmnI<xQ-; {iWɫTՊ㝊aFܲiF8ċ#W=w^| ޻|wnŧe\+] )^n<BTzX/V^W480zQ%ygE=qs!ćsCo>.>P鞬܆J4^.6)NxjFiC]jUq#_}+>&TiգVѹ\oo;-0fN U)JyÅw*Xyz>'/sywZlV nuQu<0gw,۰m`((v?cxF/\s# s%#||wWQ[L;uxZ)y^ Jsx?jJuν|{;[wz.lNv�B 9AWn%Zz`8~[q΃ {ܷEޡekѪ[oI;-cFSmVۍ:ķe5YĀwd }k{w8s;ō@Ã}Mzq4J"VfC�%~kj.=_&-=5M.o4%ն/dEV Hp9߹?@T2?x<Ry0VѰ>c̼Ak<.+]!.nC//xk8|֭UwVŅKaKwrіI׼_"<'6._9|XT.?W4IOs~A:ҟr:[YE4ίyܼS7wN7gCZ|;{SPwSA%zB1b9Q>mEbl.n>坪ԳNg_ۼbn|#o}M;{9vWHI#Flg%On8(N +'OnkED@?z 톽�7 y{H�=瘽sk s s s sbzgLw߹;sp8}x`w@NBB?HF;O{1;{:YR,D4=6s<A*% W ; ,ZW|z/߃w4;M<u*YnqČ+ӟ*Ww,z#U#WG2Y Ob ;^⊘ [OڑyB :=)Gr+C{MzZ9{Oy!=ϯ雽 9mJEr*JJ6h> b{G+=1֚\{z\r�4m=3jTzˡ'R{ͅR!x秼?mӣ̓!0Ԕ5NZWP9 w:옑tX.MDOyg(axl'cOW%{Ǽ?nwQ.aXrs*~:C*tޑ&pۍ19e)ծBu[iM.ޡ533sYA/Yo;Dn=-u?ۿwlzY!΀*Y.n]qVtz<* >X|5FzMxH}vT߾3j}ΐ\2u99zMwNYR =Ո3Kv>mI[ޙF@b. {~ܼssVGN9ib{;T!qxd$f =.;cf=c*A_./՛S-cs@$0^QuͨZ]wǐP=k0-m-;S:¸A,u1 : +J;5chӜ՛cix`uw 7x*$x gqd>Qj7Z5k)V {d"7\FYcw�H#Ipb|LSGri' ^PNQ& 5g]!#Qwr)Oi&&3U MOsѳν^"Qԧ)W'Zv)�SGF3Ǖ&2ȒwșH6W{cshJyπw@;9T)C#K,u8wћtÇY�bߩPs'blNb (w6q-\}SWøC]ɧ=ᎧaF1U=@\~=yPsՐqzc#EdȒdw+ +]2MwzjJp#1Q9RP4b3;s'Z.ÿ́`J)6 <28]Nƴ.k3Nazt= &fY]Y뒾Vp, t(GOQĄgNr7%ap i;5凞]M?hOxg'y+HO9^Zf^T_؜_jZ x6׼cvUiw/yd23T+ G3ðwW5בq_D..cP-!&iir^N-D'GLX"t'<˽#kPs֜Yi8%8zVPyG;M(h F,5x4+ӡztS;ڕl.p|;$ڎ;@t͊xY.wHؔܤY0TNgڨx�ؖޙ,k39;ck=-WC_:In,5jY߾m2 im2JVa8sV(d`mQ-=�`K"zii{;*h)O/! >^41{u{͜!epiϒeߧbaޱFyStQeU&)`">%$\+ud.9yKxMZ=-yG^Iq#DC'T(F"K'<~GK={NUO.I6w�؅h񜑚{z7qf#5fHwt;cg?ϼ-p#1Oٮ咞%Y=Jg&u&Գ 5èlN;5^Qix]b6w%j%oN򎪓w侍j �+Y{gN+Q~SPmf.,Գ�ؙ3 ; Em-&U@n\�lIYLlwKȳY< KujRY%FE<H!SqsڟUp[25Em5hqXC�lzKMѱ(SΚUh5M9V�tC<yG䄽ӎ0p�'aIr4g5W Nyjx|89�r笆w�ȉ /~>YsVoN(0�I ytW"1C#|J<2�g;5NwjxP ]-1l!^ *G}jR';Yj٫MBiT}<v@$0n-<r!H%cmz]mTj"�so!S-ӤU MXV4gj,�rމT21B;jSVؠ|�"D6 HieY*6K-x;oR? T9GmՃTZ�w^ߢ~1{ jVnVExH;}gfp[z6A= H;=NZ ;M-: y‚\iq?LDL�dwP1ŦWwaJud=QkjK9ZHdCmJ7jh%*[M2D*-�"wZANϑ?NjY?Y*? n5A*-ZVH@d+ܱސ3l 1ۍt�H#N"P=k.f򭹗rtx$PH:gᝩp~,A=~Ldkr- `_ɨ̕=w�8I)!�H.;j@@x�H6Y,۰wԕ|Fܫ�prd<~™#7~�@ޑd1^YpiiL6[Z�D wREw�W0W;�D�yV3e5fvb6ۢZ1i:vh56m#߮7 BTsO4W;i얔KMe*n%Z[+{<X͟SkDa��x5*F(Z4i5b4U-hTgiz^*4 a61/�;Q\5MY2xM RI:};*ޔ)E0 �yX֥2iݚF7v5Ty�;P%1U!Hۧi m`aS�B5(j;TV;4)7Lۘe;gjONlp=wYaW~/QI4\9sh˰uޡS7BT*xUޜ}[if! ,mh51̖9;mJժ5KTqmi磭h}OP᱁5JUkC MZ BUSU] l}+ޞ}9Ϸ/go7Q*@VRLI7KX-N%G)�K|~dϛX `[w~rt;;�l ׳zPN\�ەF20`rS�E�� 9��)?'j`P�s\C`�2Ľ#4)i;�iqޑy�N x�5iygw[;�{ �Ț3V.3Ȏ�`"d.uNb2zǤL.MF�'E o0bN\w&}ӡy•HȵsxS"Mtr?1g:CWLeQh,rOcxS"nOy1Z.i;�1zg>pGװdk$ë;UR�ZϚH񎏬mͩyy�43&38 swywƂB51 bx@O�N43nӛp;CW j@c~!wdAud y;T�rgN3O4p`?I?ko6A;#zjL&ԣNq`O}aJҁw�[rN;�'� k�@;�w��Y���d �Țcwx䇯y!=wt�xs-o=ŷ7yG�G[9�Qx}9{wl�/=2= �ǼݯG/h��������������������������������������������������������������p]/$U���%tEXtdate:create�2022-02-08T20:00:28+00:00ϻ{���%tEXtdate:modify�2022-02-08T20:00:28+00:00Ē����IENDB`�����������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/source/c_api.rst������������������������������������������0000664�0000000�0000000�00000000710�15270523712�0023457�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������.. _c_api: C API (``dlpack.h``) ==================== Macros ~~~~~~ .. doxygendefine:: DLPACK_EXTERN_C .. doxygendefine:: DLPACK_VERSION .. doxygendefine:: DLPACK_DLL Enumerations ~~~~~~~~~~~~ .. doxygenenum:: DLDeviceType .. doxygenenum:: DLDataTypeCode Structs ~~~~~~~ .. doxygenstruct:: DLDevice :members: .. doxygenstruct:: DLDataType :members: .. doxygenstruct:: DLTensor :members: .. doxygenstruct:: DLManagedTensor :members: ��������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/source/conf.py��������������������������������������������0000664�0000000�0000000�00000005222�15270523712�0023154�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # # import os # import sys # sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- project = 'DLPack' copyright = '2022, DLPack contributors' author = 'DLPack contributors' # The full version, including alpha/beta/rc tags release = '0.6.0' # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.intersphinx', 'breathe'] # Add any paths that contain templates here, relative to this directory. # templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] intersphinx_mapping = { "array_api": ("https://data-apis.org/array-api/latest", None), } html_use_index = True html_domain_indices = True html_theme_options = { "github_url": "https://github.com/dmlc/dlpack", "use_edit_page_button": True, "show_toc_level": 1, } html_context = { "github_user": "dmlc", "github_repo": "dlpack", "github_version": "main", "doc_path": "docs/source", } # -- Breathe configuration --------------------------------------------------- # Tell breathe what to call the project. This is used to link XML files # generated by Doxygen by passing a command line option: # -Dbreathe_projects.<breathe_default_project>=</path/to/xml/index/directory> breathe_default_project = "dlpack" # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'pydata_sphinx_theme' # 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/source/index.rst������������������������������������������0000664�0000000�0000000�00000006277�15270523712�0023531�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Welcome to DLPack's documentation! ================================== Purpose ~~~~~~~ In order for an ndarray system to interact with a variety of frameworks, a stable in-memory data structure is needed. DLPack is one such data structure that allows exchange between major frameworks. It is developed with inputs from many deep learning system core developers. Highlights include: * Minimum and stable: :ref:`simple header <c_api>` * Designed for cross hardware: CPU, CUDA, OpenCL, Vulkan, Metal, VPI, ROCm, WebGPU, Hexagon * Already a standard with wide community adoption and support: * `NumPy <https://numpy.org/doc/stable/release/1.22.0-notes.html#add-nep-47-compatible-dlpack-support>`_ * `CuPy <https://docs.cupy.dev/en/stable/reference/generated/cupy.fromDlpack.html>`_ * `PyTorch <https://pytorch.org/docs/stable/dlpack.html>`_ * `Tensorflow <https://www.tensorflow.org/api_docs/python/tf/experimental/dlpack/from_dlpack>`_ * `MXNet <https://mxnet.apache.org/versions/master/api/python/docs/_modules/mxnet/dlpack.html>`_ * `TVM <https://tvm.apache.org/docs/reference/api/python/contrib.html#module-tvm.contrib.dlpack>`_ * `mpi4py <https://mpi4py.readthedocs.io/en/stable/overview.html#support-for-gpu-aware-mpi>`_ * Clean C ABI compatible. * Means you can create and access it from any language. * It is also essential for building JIT and AOT compilers to support these data types. Scope ~~~~~ The main design rationale of DLPack is the minimalism. DLPack drops the consideration of allocator, device API and focus on the minimum data structure. While still considering the need for cross hardware support (e.g. the data field is opaque for platforms that does not support normal addressing). It also simplifies some of the design to remove legacy issues (e.g. everything assumes to be row major, strides can be used to support other case, and avoid the complexity to consider more layouts). Roadmap ~~~~~~~ * C API that could be exposed as a new Python attribute ``__dlpack_info__`` for returning API and ABI versions. (see `#34 <https://github.com/dmlc/dlpack/issues/34>`_, `#72 <https://github.com/dmlc/dlpack/pull/72>`_) * Clarify alignment requirements. (see `data-apis/array-api#293 <https://github.com/data-apis/array-api/issues/293>`_, `numpy/numpy#20338 <https://github.com/numpy/numpy/issues/20338>`_, `data-apis/array-api#293 (comment) <https://github.com/data-apis/array-api/issues/293#issuecomment-964434449>`_) * Adding support for boolean data type (see `#75 <https://github.com/dmlc/dlpack/issues/75>`_) * Adding a read-only flag (ABI break) or making it a hard requirement in the spec that imported arrays should be treated as read-only. (see `data-apis/consortium-feedback#1 (comment) <https://github.com/data-apis/consortium-feedback/issues/1#issuecomment-675857753>`_, `data-apis/array-api#191 <https://github.com/data-apis/array-api/issues/191>`_) * Standardize C interface for stream exchange. (see `#74 <https://github.com/dmlc/dlpack/issues/74>`_, `#65 <https://github.com/dmlc/dlpack/issues/65>`_) DLPack Documentation ~~~~~~~~~~~~~~~~~~~~ .. toctree:: :maxdepth: 2 c_api python_spec Indices and tables ================== * :ref:`genindex` * :ref:`search` ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/docs/source/python_spec.rst������������������������������������0000664�0000000�0000000�00000016403�15270523712�0024745�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������.. _python-spec: Python Specification for DLPack =============================== The Python specification for DLPack is a part of the `Python array API standard <https://data-apis.org/array-api/latest/index.html>`_. More details about the spec can be found under the :ref:`data-interchange` page. Syntax for data interchange with DLPack ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The array API will offer the following syntax for data interchange: 1. A ``from_dlpack(x)`` function, which accepts (array) objects with a ``__dlpack__`` method and uses that method to construct a new array containing the data from ``x``. 2. ``__dlpack__(self, stream=None)`` and ``__dlpack_device__`` methods on the array object, which will be called from within ``from_dlpack``, to query what device the array is on (may be needed to pass in the correct stream, e.g. in the case of multiple GPUs) and to access the data. Semantics ~~~~~~~~~ DLPack describes the memory layout of strided, n-dimensional arrays. When a user calls ``y = from_dlpack(x)``, the library implementing ``x`` (the "producer") will provide access to the data from ``x`` to the library containing ``from_dlpack`` (the "consumer"). If possible, this must be zero-copy (i.e. ``y`` will be a *view* on ``x``). If not possible, that library may make a copy of the data. In both cases: - The producer keeps owning the memory - ``y`` may or may not be a view, therefore the user must keep the recommendation to avoid mutating ``y`` in mind - see :ref:`copyview-mutability`. - Both ``x`` and ``y`` may continue to be used just like arrays created in other ways. If an array that is accessed via the interchange protocol lives on a device that the requesting library does not support, it is recommended to raise a ``TypeError``. Stream handling through the ``stream`` keyword applies to CUDA and ROCm (perhaps to other devices that have a stream concept as well, however those haven't been considered in detail). The consumer must pass the stream it will use to the producer; the producer must synchronize or wait on the stream when necessary. In the common case of the default stream being used, synchronization will be unnecessary so asynchronous execution is enabled. Implementation ~~~~~~~~~~~~~~ *Note that while this API standard largely tries to avoid discussing implementation details, some discussion and requirements are needed here because data interchange requires coordination between implementers on, e.g., memory management.* .. image:: /_static/images/DLPack_diagram.png :alt: Diagram of DLPack structs *DLPack diagram. Dark blue are the structs it defines, light blue struct members, gray text enum values of supported devices and data types.* The ``__dlpack__`` method will produce a ``PyCapsule`` containing a ``DLManagedTensor``, which will be consumed immediately within ``from_dlpack`` - therefore it is consumed exactly once, and it will not be visible to users of the Python API. The producer must set the ``PyCapsule`` name to ``"dltensor"`` so that it can be inspected by name, and set ``PyCapsule_Destructor`` that calls the ``deleter`` of the ``DLManagedTensor`` when the ``"dltensor"``-named capsule is no longer needed. The consumer must transer ownership of the ``DLManangedTensor`` from the capsule to its own object. It does so by renaming the capsule to ``"used_dltensor"`` to ensure that ``PyCapsule_Destructor`` will not get called (ensured if ``PyCapsule_Destructor`` calls ``deleter`` only for capsules whose name is ``"dltensor"``), but the ``deleter`` of the ``DLManagedTensor`` will be called by the destructor of the consumer library object created to own the ``DLManagerTensor`` obtained from the capsule. Below is an example of the capsule deleter written in the Python C API which is called either when the refcount on the capsule named ``"dltensor"`` reaches zero or the consumer decides to deallocate its array: .. code-block:: C static void dlpack_capsule_deleter(PyObject *self){ if (PyCapsule_IsValid(self, "used_dltensor")) { return; /* Do nothing if the capsule has been consumed. */ } /* an exception may be in-flight, we must save it in case we create another one */ PyObject *type, *value, *traceback; PyErr_Fetch(&type, &value, &traceback); DLManagedTensor *managed = (DLManagedTensor *)PyCapsule_GetPointer(self, "dltensor"); if (managed == NULL) { PyErr_WriteUnraisable(self); goto done; } /* the spec says the deleter can be NULL if there is no way for the caller to provide a reasonable destructor. */ if (managed->deleter) { managed->deleter(managed); /* TODO: is the deleter allowed to set a python exception? */ assert(!PyErr_Occurred()); } done: PyErr_Restore(type, value, traceback); } Note: the capsule names ``"dltensor"`` and ``"used_dltensor"`` must be statically allocated. When the ``strides`` field in the ``DLTensor`` struct is ``NULL``, it indicates a row-major compact array. If the array is of size zero, the data pointer in ``DLTensor`` should be set to either ``NULL`` or ``0``. DLPack version used must be ``0.2 <= DLPACK_VERSION < 1.0``. For further details on DLPack design and how to implement support for it, refer to `github.com/dmlc/dlpack <https://github.com/dmlc/dlpack>`_. .. warning:: DLPack contains a ``device_id``, which will be the device ID (an integer, ``0, 1, ...``) which the producer library uses. In practice this will likely be the same numbering as that of the consumer, however that is not guaranteed. Depending on the hardware type, it may be possible for the consumer library implementation to look up the actual device from the pointer to the data - this is possible for example for CUDA device pointers. It is recommended that implementers of this array API consider and document whether the ``.device`` attribute of the array returned from ``from_dlpack`` is guaranteed to be in a certain order or not. Reference Implementations ~~~~~~~~~~~~~~~~~~~~~~~~~ Several Python libraries have adopted this standard using Python C API, C++, Cython, ctypes, cffi, etc: * NumPy: `Python C API <https://github.com/numpy/numpy/blob/main/numpy/core/src/multiarray/dlpack.c>`__ * CuPy: `Cython <https://github.com/cupy/cupy/blob/master/cupy/_core/dlpack.pyx>`__ * Tensorflow: `C++ <https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/eager/dlpack.cc>`__, `Python wrapper using Python C API <https://github.com/tensorflow/tensorflow/blob/a97b01a4ff009ed84a571c138837130a311e74a7/tensorflow/python/tfe_wrapper.cc#L1562>`__, `XLA <https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/python/dlpack.cc>`__ * PyTorch: `C++ <https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/DLConvertor.cpp>`__, `Python wrapper using Python C API <https://github.com/pytorch/pytorch/blob/c22b8a42e6038ed2f6a161114cf3d8faac3f6e9a/torch/csrc/Module.cpp#L355>`__ * MXNet: `ctypes <https://github.com/apache/incubator-mxnet/blob/master/python/mxnet/dlpack.py>`__ * TVM: `ctypes <https://github.com/apache/tvm/blob/main/python/tvm/_ffi/_ctypes/ndarray.py>`__, `Cython <https://github.com/apache/tvm/blob/main/python/tvm/_ffi/_cython/ndarray.pxi>`__ * mpi4py: `Cython <https://github.com/mpi4py/mpi4py/blob/master/src/mpi4py/MPI/asdlpack.pxi>`_ �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/include/�������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021047�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/include/dlpack/������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022305�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/include/dlpack/dlpack.h����������������������������������������0000664�0000000�0000000�00000015272�15270523712�0023723�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*! * Copyright (c) 2017 by Contributors * \file dlpack.h * \brief The common header of DLPack. */ #ifndef DLPACK_DLPACK_H_ #define DLPACK_DLPACK_H_ /** * \brief Compatibility with C++ */ #ifdef __cplusplus #define DLPACK_EXTERN_C extern "C" #else #define DLPACK_EXTERN_C #endif /*! \brief The current version of dlpack */ #define DLPACK_VERSION 80 /*! \brief The current ABI version of dlpack */ #define DLPACK_ABI_VERSION 1 /*! \brief DLPACK_DLL prefix for windows */ #ifdef _WIN32 #ifdef DLPACK_EXPORTS #define DLPACK_DLL __declspec(dllexport) #else #define DLPACK_DLL __declspec(dllimport) #endif #else #define DLPACK_DLL #endif #include <stdint.h> #include <stddef.h> #ifdef __cplusplus extern "C" { #endif /*! * \brief The device type in DLDevice. */ #ifdef __cplusplus typedef enum : int32_t { #else typedef enum { #endif /*! \brief CPU device */ kDLCPU = 1, /*! \brief CUDA GPU device */ kDLCUDA = 2, /*! * \brief Pinned CUDA CPU memory by cudaMallocHost */ kDLCUDAHost = 3, /*! \brief OpenCL devices. */ kDLOpenCL = 4, /*! \brief Vulkan buffer for next generation graphics. */ kDLVulkan = 7, /*! \brief Metal for Apple GPU. */ kDLMetal = 8, /*! \brief Verilog simulator buffer */ kDLVPI = 9, /*! \brief ROCm GPUs for AMD GPUs */ kDLROCM = 10, /*! * \brief Pinned ROCm CPU memory allocated by hipMallocHost */ kDLROCMHost = 11, /*! * \brief Reserved extension device type, * used for quickly test extension device * The semantics can differ depending on the implementation. */ kDLExtDev = 12, /*! * \brief CUDA managed/unified memory allocated by cudaMallocManaged */ kDLCUDAManaged = 13, /*! * \brief Unified shared memory allocated on a oneAPI non-partititioned * device. Call to oneAPI runtime is required to determine the device * type, the USM allocation type and the sycl context it is bound to. * */ kDLOneAPI = 14, /*! \brief GPU support for next generation WebGPU standard. */ kDLWebGPU = 15, /*! \brief Qualcomm Hexagon DSP */ kDLHexagon = 16, } DLDeviceType; /*! * \brief A Device for Tensor and operator. */ typedef struct { /*! \brief The device type used in the device. */ DLDeviceType device_type; /*! * \brief The device index. * For vanilla CPU memory, pinned memory, or managed memory, this is set to 0. */ int32_t device_id; } DLDevice; /*! * \brief The type code options DLDataType. */ typedef enum { /*! \brief signed integer */ kDLInt = 0U, /*! \brief unsigned integer */ kDLUInt = 1U, /*! \brief IEEE floating point */ kDLFloat = 2U, /*! * \brief Opaque handle type, reserved for testing purposes. * Frameworks need to agree on the handle data type for the exchange to be well-defined. */ kDLOpaqueHandle = 3U, /*! \brief bfloat16 */ kDLBfloat = 4U, /*! * \brief complex number * (C/C++/Python layout: compact struct per complex number) */ kDLComplex = 5U, /*! \brief boolean */ kDLBool = 6U, } DLDataTypeCode; /*! * \brief The data type the tensor can hold. The data type is assumed to follow the * native endian-ness. An explicit error message should be raised when attempting to * export an array with non-native endianness * * Examples * - float: type_code = 2, bits = 32, lanes = 1 * - float4(vectorized 4 float): type_code = 2, bits = 32, lanes = 4 * - int8: type_code = 0, bits = 8, lanes = 1 * - std::complex<float>: type_code = 5, bits = 64, lanes = 1 * - bool: type_code = 6, bits = 8, lanes = 1 (as per common array library convention, the underlying storage size of bool is 8 bits) */ typedef struct { /*! * \brief Type code of base types. * We keep it uint8_t instead of DLDataTypeCode for minimal memory * footprint, but the value should be one of DLDataTypeCode enum values. * */ uint8_t code; /*! * \brief Number of bits, common choices are 8, 16, 32. */ uint8_t bits; /*! \brief Number of lanes in the type, used for vector types. */ uint16_t lanes; } DLDataType; /*! * \brief Plain C Tensor object, does not manage memory. */ typedef struct { /*! * \brief The data pointer points to the allocated data. This will be CUDA * device pointer or cl_mem handle in OpenCL. It may be opaque on some device * types. This pointer is always aligned to 256 bytes as in CUDA. The * `byte_offset` field should be used to point to the beginning of the data. * * Note that as of Nov 2021, multiply libraries (CuPy, PyTorch, TensorFlow, * TVM, perhaps others) do not adhere to this 256 byte aligment requirement * on CPU/CUDA/ROCm, and always use `byte_offset=0`. This must be fixed * (after which this note will be updated); at the moment it is recommended * to not rely on the data pointer being correctly aligned. * * For given DLTensor, the size of memory required to store the contents of * data is calculated as follows: * * \code{.c} * static inline size_t GetDataSize(const DLTensor* t) { * size_t size = 1; * for (tvm_index_t i = 0; i < t->ndim; ++i) { * size *= t->shape[i]; * } * size *= (t->dtype.bits * t->dtype.lanes + 7) / 8; * return size; * } * \endcode */ void* data; /*! \brief The device of the tensor */ DLDevice device; /*! \brief Number of dimensions */ int32_t ndim; /*! \brief The data type of the pointer*/ DLDataType dtype; /*! \brief The shape of the tensor */ int64_t* shape; /*! * \brief strides of the tensor (in number of elements, not bytes) * can be NULL, indicating tensor is compact and row-majored. */ int64_t* strides; /*! \brief The offset in bytes to the beginning pointer to data */ uint64_t byte_offset; } DLTensor; /*! * \brief C Tensor object, manage memory of DLTensor. This data structure is * intended to facilitate the borrowing of DLTensor by another framework. It is * not meant to transfer the tensor. When the borrowing framework doesn't need * the tensor, it should call the deleter to notify the host that the resource * is no longer needed. */ typedef struct DLManagedTensor { /*! \brief DLTensor which is being memory managed */ DLTensor dl_tensor; /*! \brief the context of the original host framework of DLManagedTensor in * which DLManagedTensor is used in the framework. It can also be NULL. */ void * manager_ctx; /*! \brief Destructor signature void (*)(void*) - this should be called * to destruct manager_ctx which holds the DLManagedTensor. It can be NULL * if there is no way for the caller to provide a reasonable destructor. * The destructors deletes the argument self as well. */ void (*deleter)(struct DLManagedTensor * self); } DLManagedTensor; #ifdef __cplusplus } // DLPACK_EXTERN_C #endif #endif // DLPACK_DLPACK_H_ ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/tests/���������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0020566�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/tests/scripts/�������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022255�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/tests/scripts/task_build.sh������������������������������������0000775�0000000�0000000�00000000130�15270523712�0024727�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/bash make || exit -1 mkdir -p build cd build cmake .. || exit -1 make || exit -1 ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/dlpack/tests/scripts/task_lint.sh�������������������������������������0000775�0000000�0000000�00000001206�15270523712�0024603�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/bash mkdir -p bin if [ ! -f bin/lint.py ]; then echo "Grab linter ..." wget https://raw.githubusercontent.com/dmlc/dmlc-core/main/scripts/lint.py mv lint.py bin/lint.py fi echo "Check codestyle of c++ code..." python bin/lint.py dlpack cpp include contrib echo "Check doxygen generation..." make doc 2>log.txt (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt echo "---------Error Log----------" cat logclean.txt echo "----------------------------" (cat logclean.txt|grep warning) && exit -1 (cat logclean.txt|grep error) && exit -1 rm logclean.txt rm log.txt echo "All checks passed..." ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/�������������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0017776�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/.clang-format������������������������������������������������0000664�0000000�0000000�00000003130�15270523712�0022346�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������--- Language: Cpp BasedOnStyle: LLVM AccessModifierOffset: -4 AlignAfterOpenBracket: DontAlign AlignEscapedNewlinesLeft: true # AlignOperands: true AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: All AllowShortIfStatementsOnASingleLine: false AllowShortLoopsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: All AlwaysBreakBeforeMultilineStrings: false AlwaysBreakTemplateDeclarations: false # BinPackArguments: false # BinPackParameters: true BreakBeforeBinaryOperators: false BreakBeforeBraces: Custom BraceWrapping: { AfterFunction: true } BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: true ColumnLimit: 80 ContinuationIndentWidth: 4 DerivePointerAlignment: false #XXX DisableFormat: false ExperimentalAutoDetectBinPacking: false #XXX ForEachMacros: [ LIST_FOREACH, SIMPLEQ_FOREACH, CIRCLEQ_FOREACH, TAILQ_FOREACH, TAILQ_FOREACH_REVERSE, HT_FOREACH ] IndentCaseLabels: false IndentFunctionDeclarationAfterType: false IndentWidth: 4 IndentWrappedFunctionNames: false KeepEmptyLinesAtTheStartOfBlocks: true MaxEmptyLinesToKeep: 2 PointerAlignment: Right #XXX # SpaceAfterCStyleCast: false SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 1 SpacesInAngles: false SpacesInCStyleCastParentheses: false SpacesInParentheses: false Standard: Cpp03 TabWidth: 4 UseTab: Always SortIncludes: false ... ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/.travis.yml��������������������������������������������������0000664�0000000�0000000�00000005461�15270523712�0022115�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������os: - linux - osx sudo: false dist: trusty env: matrix: - EVENT_BUILD_METHOD=cmake EVENT_CMAKE_OPTIONS="-DEVENT__COVERAGE=ON -DCMAKE_BUILD_TYPE=debug" COVERALLS=yes - EVENT_BUILD_METHOD=cmake EVENT_CMAKE_OPTIONS="" - EVENT_BUILD_METHOD=cmake EVENT_CMAKE_OPTIONS="-DEVENT__DISABLE_OPENSSL=ON" - EVENT_BUILD_METHOD=cmake EVENT_CMAKE_OPTIONS="-DEVENT__DISABLE_THREAD_SUPPORT=ON" - EVENT_BUILD_METHOD=cmake EVENT_CMAKE_OPTIONS="-DEVENT__DISABLE_DEBUG_MODE=ON" - EVENT_BUILD_METHOD=cmake EVENT_CMAKE_OPTIONS="-DEVENT__DISABLE_MM_REPLACEMENT=ON" - EVENT_BUILD_METHOD=cmake EVENT_CMAKE_OPTIONS="-DEVENT__ENABLE_VERBOSE_DEBUG=ON" - EVENT_BUILD_METHOD=autotools EVENT_CONFIGURE_OPTIONS="" - EVENT_BUILD_METHOD=autotools EVENT_CONFIGURE_OPTIONS="--disable-openssl" - EVENT_BUILD_METHOD=autotools EVENT_CONFIGURE_OPTIONS="--disable-thread-support" - EVENT_BUILD_METHOD=autotools EVENT_CONFIGURE_OPTIONS="--disable-debug-mode" - EVENT_BUILD_METHOD=autotools EVENT_CONFIGURE_OPTIONS="--disable-malloc-replacement" language: c compiler: - gcc - clang before_install: - if [ -n "$COVERALLS" ]; then pip install --user cpp-coveralls; fi - export JOBS=20 - export TIMEOUT=50 - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew update; brew uninstall libtool && brew install libtool; brew install openssl; brew install lcov; if [ "$CC" == "gcc" ]; then export CC=$(ls -t /usr/local/bin/gcc-?.?); fi export OPENSSL_ROOT=$(echo /usr/local/Cellar/openssl/*); export CMAKE_INCLUDE_PATH=$OPENSSL_ROOT/include CMAKE_LIBRARY_PATH=$OPENSSL_ROOT/lib; export CFLAGS=-I$CMAKE_INCLUDE_PATH LDFLAGS=-L$CMAKE_LIBRARY_PATH; export JOBS=4; fi addons: apt: sources: - xenial - sourceline: 'deb http://archive.ubuntu.com/ubuntu xenial main' packages: - zlib1g-dev - libssl-dev - build-essential - automake - autoconf - cmake - lcov script: - if [ "$EVENT_BUILD_METHOD" = "autotools" ]; then ./autogen.sh && ./configure $EVENT_CONFIGURE_OPTIONS && make && travis_wait $TIMEOUT make -j $JOBS verify; fi - if [ "$EVENT_BUILD_METHOD" = "cmake" ]; then export CTEST_PARALLEL_LEVEL=$JOBS CTEST_OUTPUT_ON_FAILURE=1; mkdir build && cd build && cmake .. $EVENT_CMAKE_OPTIONS && travis_wait $TIMEOUT cmake --build . --target verify; fi after_script: - if [ -n "$COVERALLS" ]; then coveralls --build-root . --root .. --exclude test --exclude sample --exclude cmake --exclude build/CMakeFiles/CheckTypeSize --exclude build/CMakeFiles/CompilerIdC --gcov-options '\-lp'; fi notifications: irc: "irc.oftc.net#libevent" ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/.uncrustify��������������������������������������������������0000664�0000000�0000000�00000005163�15270523712�0022217�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������input_tab_size = 8 output_tab_size = 8 indent_with_tabs = 2 indent_cmt_with_tabs = false indent_brace_parent = false indent_func_call_param = true indent_func_def_param = true sp_enum_before_assign = add sp_enum_after_assign = add sp_inside_paren = remove sp_paren_brace = add sp_before_ptr_star = add sp_before_unnamed_ptr_star = add sp_between_ptr_star = remove sp_after_ptr_star = remove sp_after_ptr_star_func = add sp_before_ptr_star_func = add sp_before_sparen = add sp_inside_sparen = remove sp_inside_sparen_close = remove sp_after_sparen = add sp_sparen_brace = add sp_special_semi = remove sp_before_semi_for = remove sp_after_comma = add sp_after_cast = remove sp_inside_braces_struct = add sp_type_func = remove sp_func_def_paren = remove sp_inside_fparen = remove sp_fparen_brace = add sp_func_call_paren = remove sp_else_brace = add sp_after_oc_block_caret = remove align_keep_tabs = true align_with_tabs = true align_on_tabstop = true nl_fcall_brace = remove nl_enum_brace = remove nl_struct_brace = remove nl_union_brace = remove nl_if_brace = remove nl_brace_else = remove nl_elseif_brace = remove nl_else_brace = remove nl_else_if = remove nl_for_brace = remove sp_after_semi_for_empty = remove nl_while_brace = remove nl_do_brace = remove nl_brace_while = remove nl_switch_brace = remove nl_func_type_name = add nl_fdef_brace = add mod_paren_on_return = ignore �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/CMakeLists.txt�����������������������������������������������0000664�0000000�0000000�00000135137�15270523712�0022550�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# # Libevent CMake project # # Based on initial work by: # Alexey Ozeritsky # # Additional changes: # Brodie Thiesfield # Joakim Soderberg # Trond Norbye # Sergei Nikulov # # Build example: # # cd libevent # md build # cd build # cmake -G "Visual Studio 10" .. # start libevent.sln # if (WIN32) cmake_minimum_required(VERSION 3.1 FATAL_ERROR) else() cmake_minimum_required(VERSION 3.10 FATAL_ERROR) endif() if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Set build type to Debug o Release (default Release)" FORCE) endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LOWER) # get rid of the extra default configurations # what? why would you get id of other useful build types? - Ellzey set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "Limited configurations" FORCE) project(libevent C) set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/") string(REGEX MATCH "SunOS" SOLARIS "${CMAKE_SYSTEM_NAME}") include(CheckTypeSize) include(CheckFunctionExistsEx) include(CheckFileOffsetBits) include(CheckFunctionExists) include(CheckIncludeFile) include(CheckIncludeFiles) include(CheckVariableExists) include(CheckSymbolExists) include(CheckStructHasMember) include(CheckCSourceCompiles) include(CheckPrototypeDefinition) include(CheckFunctionKeywords) include(AddCompilerFlags) include(VersionViaGit) event_fuzzy_version_from_git() set(EVENT_VERSION_MAJOR 2) set(EVENT_VERSION_MINOR 1) set(EVENT_VERSION_PATCH 8) set(EVENT_VERSION_STAGE beta) set(EVENT_ABI_MAJOR ${EVENT_VERSION_MAJOR}) set(EVENT_ABI_MINOR ${EVENT_VERSION_MINOR}) set(EVENT_ABI_PATCH ${EVENT_VERSION_PATCH}) set(EVENT_ABI_LIBVERSION "${EVENT_ABI_MAJOR}.${EVENT_ABI_MINOR}.${EVENT_ABI_PATCH}") set(EVENT_PACKAGE_VERSION "${EVENT_VERSION_MAJOR}.${EVENT_VERSION_MINOR}.${EVENT_VERSION_PATCH}") set(EVENT_NUMERIC_VERSION 0x02010800) # only a subset of names can be used, defaults to "beta" set(EVENT_STAGE_NAME ${EVENT_VERSION_STAGE} CACHE STRING "set the stage name (beta|alpha|release)") # a list that defines what can set for EVENT_STAGE_VERSION set(EVENT__ALLOWED_STAGE_NAMES rc beta alpha release) # attempt to find the EVENT__STAGE_VERSION in the allowed list # of accepted stage names, the return value is stord in # EVENT__STAGE_RET list(FIND EVENT__ALLOWED_STAGE_NAMES ${EVENT_STAGE_NAME} EVENT__STAGE_RET) if (EVENT__STAGE_RET EQUAL "-1") set(EVENT_STAGE_NAME beta) endif() set(EVENT_VERSION "${EVENT_VERSION_MAJOR}.${EVENT_VERSION_MINOR}.${EVENT_VERSION_PATCH}-${EVENT_STAGE_NAME}") option(EVENT__BUILD_SHARED_LIBRARIES "Define if libevent should be built with shared libraries instead of archives" OFF) option(EVENT__DISABLE_DEBUG_MODE "Define if libevent should build without support for a debug mode" OFF) option(EVENT__ENABLE_VERBOSE_DEBUG "Enables verbose debugging" OFF) option(EVENT__DISABLE_MM_REPLACEMENT "Define if libevent should not allow replacing the mm functions" OFF) option(EVENT__DISABLE_THREAD_SUPPORT "Define if libevent should not be compiled with thread support" OFF) option(EVENT__DISABLE_OPENSSL "Define if libevent should build without support for OpenSSL encrpytion" OFF) option(EVENT__DISABLE_BENCHMARK "Defines if libevent should build without the benchmark exectuables" OFF) option(EVENT__DISABLE_TESTS "If tests should be compiled or not" OFF) option(EVENT__DISABLE_REGRESS "Disable the regress tests" OFF) option(EVENT__DISABLE_SAMPLES "Disable sample files" OFF) option(EVENT__DISABLE_CLOCK_GETTIME "Do not use clock_gettime even if it is available" OFF) option(EVENT__FORCE_KQUEUE_CHECK "When crosscompiling forces running a test program that verifies that Kqueue works with pipes. Note that this requires you to manually run the test program on the the cross compilation target to verify that it works. See cmake documentation for try_run for more details" OFF) # TODO: Add --disable-largefile omit support for large files option(EVENT__COVERAGE "Enable running gcov to get a test coverage report (only works with GCC/CLang). Make sure to enable -DCMAKE_BUILD_TYPE=Debug as well." OFF) # Put the libaries and binaries that get built into directories at the # top of the build tree rather than in hard-to-find leaf directories. set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) if (EVENT__ENABLE_VERBOSE_DEBUG) add_definitions(-DUSE_DEBUG=1) endif() # Setup compiler flags for coverage. if (EVENT__COVERAGE) if (NOT "${CMAKE_BUILD_TYPE_LOWER}" STREQUAL "debug") message(FATAL_ERROR "Coverage requires -DCMAKE_BUILD_TYPE=Debug") endif() message(STATUS "Setting coverage compiler flags") set(CMAKE_REQUIRED_LIBRARIES "--coverage") add_compiler_flags(-g -O0 --coverage) set(CMAKE_REQUIRED_LIBRARIES "") endif() # GCC specific options. if (CMAKE_COMPILER_IS_GNUCC) option(EVENT__DISABLE_GCC_WARNINGS "Disable verbose warnings with GCC" OFF) option(EVENT__ENABLE_GCC_HARDENING "Enable compiler security checks" OFF) option(EVENT__ENABLE_GCC_FUNCTION_SECTIONS "Enable gcc function sections" OFF) option(EVENT__ENABLE_GCC_WARNINGS "Make all GCC warnings into errors" OFF) list(APPEND __FLAGS -Wall -Wswitch) if (EVENT__DISABLE_GCC_WARNINGS) list(APPEND __FLAGS -w) endif() if (EVENT__ENABLE_GCC_HARDENING) list(APPEND __FLAGS -fstack-protector-all -fwrapv -fPIE -Wstack-protector "--param ssp-buffer-size=1") add_definitions(-D_FORTIFY_SOURCE=2) endif() if (EVENT__ENABLE_GCC_FUNCTION_SECTIONS) list(APPEND __FLAGS -ffunction-sections) # TODO: Add --gc-sections support. We need some checks for NetBSD to ensure this works. endif() if (EVENT__ENABLE_GCC_WARNINGS) list(APPEND __FLAGS -Werror) endif() # We need to test for at least gcc 2.95 here, because older versions don't # have -fno-strict-aliasing list(APPEND __FLAGS -fno-strict-aliasing) add_compiler_flags(${__FLAGS}) endif() if (APPLE) # Get rid of deprecated warnings for OpenSSL on OSX 10.7 and greater. add_compiler_flags( -Wno-error=deprecated-declarations -Qunused-arguments ) endif() # Winsock. if(WIN32) set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h ws2tcpip.h) set(CMAKE_REQUIRED_LIBRARIES ws2_32.lib) set(CMAKE_REQUIRED_DEFINITIONS -FIwinsock2.h -FIws2tcpip.h) endif() if (SOLARIS) set(CMAKE_REQUIRED_LIBRARIES socket nsl) endif() # Check if _GNU_SOURCE is available. CHECK_SYMBOL_EXISTS(__GNU_LIBRARY__ "features.h" _GNU_SOURCE) if (_GNU_SOURCE) add_definitions(-D_GNU_SOURCE) endif() CHECK_INCLUDE_FILE(sys/types.h EVENT__HAVE_SYS_TYPES_H) if(EVENT__HAVE_SYS_TYPES_H) list(APPEND CMAKE_EXTRA_INCLUDE_FILES sys/types.h) endif() CHECK_INCLUDE_FILE(sys/socket.h EVENT__HAVE_SYS_SOCKET_H) if(EVENT__HAVE_SYS_SOCKET_H) list(APPEND CMAKE_EXTRA_INCLUDE_FILES sys/socket.h) endif() CHECK_INCLUDE_FILE(netinet/in.h EVENT__HAVE_NETINET_IN_H) if(EVENT__HAVE_NETINET_IN_H) list(APPEND CMAKE_EXTRA_INCLUDE_FILES netinet/in.h) endif() CHECK_INCLUDE_FILE(netinet/in6.h EVENT__HAVE_NETINET_IN6_H) if(EVENT__HAVE_NETINET_IN6_H) list(APPEND CMAKE_EXTRA_INCLUDE_FILES netinet/in6.h) endif() CHECK_INCLUDE_FILE(unistd.h EVENT__HAVE_UNISTD_H) CHECK_INCLUDE_FILE(netdb.h EVENT__HAVE_NETDB_H) CHECK_INCLUDE_FILE(dlfcn.h EVENT__HAVE_DLFCN_H) CHECK_INCLUDE_FILE(arpa/inet.h EVENT__HAVE_ARPA_INET_H) CHECK_INCLUDE_FILE(fcntl.h EVENT__HAVE_FCNTL_H) if(EVENT__HAVE_FCNTL_H) list(APPEND CMAKE_EXTRA_INCLUDE_FILES fcntl.h) endif() CHECK_INCLUDE_FILE(inttypes.h EVENT__HAVE_INTTYPES_H) CHECK_INCLUDE_FILE(memory.h EVENT__HAVE_MEMORY_H) CHECK_INCLUDE_FILE(poll.h EVENT__HAVE_POLL_H) CHECK_INCLUDE_FILE(port.h EVENT__HAVE_PORT_H) if(EVENT__HAVE_PORT_H) list(APPEND CMAKE_EXTRA_INCLUDE_FILES port.h) endif() CHECK_INCLUDE_FILE(signal.h EVENT__HAVE_SIGNAL_H) CHECK_INCLUDE_FILE(stdarg.h EVENT__HAVE_STDARG_H) CHECK_INCLUDE_FILE(stddef.h EVENT__HAVE_STDDEF_H) CHECK_INCLUDE_FILE(stdint.h EVENT__HAVE_STDINT_H) CHECK_INCLUDE_FILE(stdlib.h EVENT__HAVE_STDLIB_H) CHECK_INCLUDE_FILE(strings.h EVENT__HAVE_STRINGS_H) CHECK_INCLUDE_FILE(string.h EVENT__HAVE_STRING_H) CHECK_INCLUDE_FILE(sys/devpoll.h EVENT__HAVE_SYS_DEVPOLL_H) CHECK_INCLUDE_FILE(sys/epoll.h EVENT__HAVE_SYS_EPOLL_H) CHECK_INCLUDE_FILE(sys/eventfd.h EVENT__HAVE_SYS_EVENTFD_H) CHECK_INCLUDE_FILE(sys/event.h EVENT__HAVE_SYS_EVENT_H) CHECK_INCLUDE_FILE(sys/ioctl.h EVENT__HAVE_SYS_IOCTL_H) CHECK_INCLUDE_FILE(sys/mman.h EVENT__HAVE_SYS_MMAN_H) CHECK_INCLUDE_FILE(sys/param.h EVENT__HAVE_SYS_PARAM_H) CHECK_INCLUDE_FILE(sys/queue.h EVENT__HAVE_SYS_QUEUE_H) CHECK_INCLUDE_FILE(sys/select.h EVENT__HAVE_SYS_SELECT_H) CHECK_INCLUDE_FILE(sys/sendfile.h EVENT__HAVE_SYS_SENDFILE_H) CHECK_INCLUDE_FILE(sys/stat.h EVENT__HAVE_SYS_STAT_H) CHECK_INCLUDE_FILE(sys/time.h EVENT__HAVE_SYS_TIME_H) if(EVENT__HAVE_SYS_TIME_H) list(APPEND CMAKE_EXTRA_INCLUDE_FILES sys/time.h) endif() CHECK_INCLUDE_FILE(sys/uio.h EVENT__HAVE_SYS_UIO_H) CHECK_INCLUDE_FILES("sys/types.h;ifaddrs.h" EVENT__HAVE_IFADDRS_H) CHECK_INCLUDE_FILE(mach/mach_time.h EVENT__HAVE_MACH_MACH_TIME_H) CHECK_INCLUDE_FILE(netinet/tcp.h EVENT__HAVE_NETINET_TCP_H) CHECK_INCLUDE_FILE(sys/wait.h EVENT__HAVE_SYS_WAIT_H) CHECK_INCLUDE_FILE(sys/resource.h EVENT__HAVE_SYS_RESOURCE_H) CHECK_INCLUDE_FILE(sys/sysctl.h EVENT__HAVE_SYS_SYSCTL_H) CHECK_INCLUDE_FILE(sys/timerfd.h EVENT__HAVE_SYS_TIMERFD_H) CHECK_INCLUDE_FILE(errno.h EVENT__HAVE_ERRNO_H) CHECK_FUNCTION_EXISTS_EX(epoll_create EVENT__HAVE_EPOLL) CHECK_FUNCTION_EXISTS_EX(epoll_ctl EVENT__HAVE_EPOLL_CTL) CHECK_FUNCTION_EXISTS_EX(eventfd EVENT__HAVE_EVENTFD) if(NOT EVENT__DISABLE_CLOCK_GETTIME) CHECK_FUNCTION_EXISTS_EX(clock_gettime EVENT__HAVE_CLOCK_GETTIME) endif() CHECK_FUNCTION_EXISTS_EX(fcntl EVENT__HAVE_FCNTL) CHECK_FUNCTION_EXISTS_EX(getaddrinfo EVENT__HAVE_GETADDRINFO) CHECK_FUNCTION_EXISTS_EX(getnameinfo EVENT__HAVE_GETNAMEINFO) CHECK_FUNCTION_EXISTS_EX(gettimeofday EVENT__HAVE_GETTIMEOFDAY) CHECK_FUNCTION_EXISTS_EX(getprotobynumber EVENT__HAVE_GETPROTOBYNUMBER) CHECK_FUNCTION_EXISTS_EX(getservbyname EVENT__HAVE_GETSERVBYNAME) CHECK_FUNCTION_EXISTS_EX(inet_ntop EVENT__HAVE_INET_NTOP) CHECK_FUNCTION_EXISTS_EX(inet_pton EVENT__HAVE_INET_PTON) CHECK_FUNCTION_EXISTS_EX(kqueue EVENT__HAVE_KQUEUE) CHECK_FUNCTION_EXISTS_EX(mmap EVENT__HAVE_MMAP) CHECK_FUNCTION_EXISTS_EX(pipe EVENT__HAVE_PIPE) CHECK_FUNCTION_EXISTS_EX(pipe2 EVENT__HAVE_PIPE2) CHECK_FUNCTION_EXISTS_EX(poll EVENT__HAVE_POLL) CHECK_FUNCTION_EXISTS_EX(port_create EVENT__HAVE_PORT_CREATE) CHECK_FUNCTION_EXISTS_EX(sendfile EVENT__HAVE_SENDFILE) CHECK_FUNCTION_EXISTS_EX(sigaction EVENT__HAVE_SIGACTION) CHECK_FUNCTION_EXISTS_EX(signal EVENT__HAVE_SIGNAL) CHECK_FUNCTION_EXISTS_EX(splice EVENT__HAVE_SPLICE) CHECK_FUNCTION_EXISTS_EX(strlcpy EVENT__HAVE_STRLCPY) CHECK_FUNCTION_EXISTS_EX(strsep EVENT__HAVE_STRSEP) CHECK_FUNCTION_EXISTS_EX(strtok_r EVENT__HAVE_STRTOK_R) CHECK_FUNCTION_EXISTS_EX(strtoll EVENT__HAVE_STRTOLL) CHECK_FUNCTION_EXISTS_EX(vasprintf EVENT__HAVE_VASPRINTF) CHECK_FUNCTION_EXISTS_EX(sysctl EVENT__HAVE_SYSCTL) CHECK_FUNCTION_EXISTS_EX(accept4 EVENT__HAVE_ACCEPT4) CHECK_FUNCTION_EXISTS_EX(arc4random EVENT__HAVE_ARC4RANDOM) CHECK_FUNCTION_EXISTS_EX(arc4random_buf EVENT__HAVE_ARC4RANDOM_BUF) CHECK_FUNCTION_EXISTS_EX(epoll_create1 EVENT__HAVE_EPOLL_CREATE1) CHECK_FUNCTION_EXISTS_EX(getegid EVENT__HAVE_GETEGID) CHECK_FUNCTION_EXISTS_EX(geteuid EVENT__HAVE_GETEUID) CHECK_FUNCTION_EXISTS_EX(getifaddrs EVENT__HAVE_GETIFADDRS) CHECK_FUNCTION_EXISTS_EX(issetugid EVENT__HAVE_ISSETUGID) CHECK_FUNCTION_EXISTS_EX(mach_absolute_time EVENT__HAVE_MACH_ABSOLUTE_TIME) CHECK_FUNCTION_EXISTS_EX(nanosleep EVENT__HAVE_NANOSLEEP) CHECK_FUNCTION_EXISTS_EX(usleep EVENT__HAVE_USLEEP) CHECK_FUNCTION_EXISTS_EX(timeradd EVENT__HAVE_TIMERADD) CHECK_FUNCTION_EXISTS_EX(timerclear EVENT__HAVE_TIMERCLEAR) CHECK_FUNCTION_EXISTS_EX(timercmp EVENT__HAVE_TIMERCMP) CHECK_FUNCTION_EXISTS_EX(timerfd_create HAVE_TIMERFD_CREATE) CHECK_FUNCTION_EXISTS_EX(timerisset EVENT__HAVE_TIMERISSET) CHECK_FUNCTION_EXISTS_EX(putenv EVENT__HAVE_PUTENV) CHECK_FUNCTION_EXISTS_EX(setenv EVENT__HAVE_SETENV) CHECK_FUNCTION_EXISTS_EX(setrlimit EVENT__HAVE_SETRLIMIT) CHECK_FUNCTION_EXISTS_EX(umask EVENT__HAVE_UMASK) CHECK_FUNCTION_EXISTS_EX(unsetenv EVENT__HAVE_UNSETENV) # Get the gethostbyname_r prototype. CHECK_FUNCTION_EXISTS_EX(gethostbyname_r EVENT__HAVE_GETHOSTBYNAME_R) if(EVENT__HAVE_GETHOSTBYNAME_R) CHECK_PROTOTYPE_DEFINITION(gethostbyname_r "int gethostbyname_r(const char *name, struct hostent *hp, struct hostent_data *hdata)" "0" "netdb.h" EVENT__HAVE_GETHOSTBYNAME_R_3_ARG) CHECK_PROTOTYPE_DEFINITION(gethostbyname_r "struct hostent *gethostbyname_r(const char *name, struct hostent *hp, char *buf, size_t buflen, int *herr)" "NULL" "netdb.h" EVENT__HAVE_GETHOSTBYNAME_R_5_ARG) CHECK_PROTOTYPE_DEFINITION(gethostbyname_r "int gethostbyname_r(const char *name, struct hostent *hp, char *buf, size_t buflen, struct hostent **result, int *herr)" "0" "netdb.h" EVENT__HAVE_GETHOSTBYNAME_R_6_ARG) endif() if(HAVE_PORT_H AND HAVE_PORT_CREATE) set(EVENT__HAVE_EVENT_PORTS 1) endif() if(NOT WIN32) CHECK_FUNCTION_EXISTS_EX(select EVENT__HAVE_SELECT) endif() CHECK_TYPE_SIZE("uint8_t" EVENT__HAVE_UINT8_T) CHECK_TYPE_SIZE("uint16_t" EVENT__HAVE_UINT16_T) CHECK_TYPE_SIZE("uint32_t" EVENT__HAVE_UINT32_T) CHECK_TYPE_SIZE("uint64_t" EVENT__HAVE_UINT64_T) CHECK_TYPE_SIZE("short" EVENT__SIZEOF_SHORT BUILTIN_TYPES_ONLY) CHECK_TYPE_SIZE("int" EVENT__SIZEOF_INT BUILTIN_TYPES_ONLY) CHECK_TYPE_SIZE("unsigned" EVENT__SIZEOF_UNSIGNED BUILTIN_TYPES_ONLY) CHECK_TYPE_SIZE("unsigned int" EVENT__SIZEOF_UNSIGNED_INT BUILTIN_TYPES_ONLY) CHECK_TYPE_SIZE("long" EVENT__SIZEOF_LONG BUILTIN_TYPES_ONLY) CHECK_TYPE_SIZE("long long" EVENT__SIZEOF_LONG_LONG BUILTIN_TYPES_ONLY) if(WIN32) # These aren't available until Windows Vista. # But you can still link them. They just won't be found when running the exe. set(EVENT__HAVE_INET_NTOP 0) set(EVENT__HAVE_INET_PTON 0) endif() # Check for different inline keyword versions. check_function_keywords("inline" "__inline" "__inline__") if (HAVE_INLINE) set (EVENT__inline inline) elseif (HAVE___INLINE) set(EVENT__inline __inline) elseif(HAVE___INLINE__) set(EVENT__inline __inline__) else() set(EVENT__inline) endif() CHECK_SYMBOL_EXISTS(TAILQ_FOREACH sys/queue.h EVENT__HAVE_TAILQFOREACH) CHECK_SYMBOL_EXISTS(CTL_KERN sys/sysctl.h EVENT__HAVE_DECL_CTL_KERN) CHECK_SYMBOL_EXISTS(KERN_ARND sys/sysctl.h EVENT__HAVE_DECL_KERN_ARND) CHECK_SYMBOL_EXISTS(KERN_RANDOM sys/sysctl.h EVENT__HAVE_DECL_KERN_RANDOM) CHECK_SYMBOL_EXISTS(RANDOM_UUID sys/sysctl.h EVENT__HAVE_DECL_RANDOM_UUID) CHECK_SYMBOL_EXISTS(F_SETFD fcntl.h EVENT__HAVE_SETFD) CHECK_TYPE_SIZE(fd_mask EVENT__HAVE_FD_MASK) CHECK_TYPE_SIZE(size_t EVENT__SIZEOF_SIZEE_T) if(NOT EVENT__SIZEOF_SIZE_T) set(EVENT__size_t "unsigned") set(EVENT__SIZEOF_SIZE_T ${EVENT__SIZEOF_UNSIGNED}) else() set(EVENT__size_t size_t) endif() CHECK_TYPE_SIZE("off_t" EVENT__SIZEOF_OFF_T LANGUAGE C) # XXX we should functionalize these size and type sets. --elley # Winssck. if (_MSC_VER) list(APPEND CMAKE_EXTRA_INCLUDE_FILES BaseTsd.h) endif() CHECK_TYPE_SIZE("ssize_t" EVENT__SIZEOF_SSIZE_T_LOWER LANGUAGE C) CHECK_TYPE_SIZE("SSIZE_T" EVENT__SIZEOF_SSIZE_T_UPPER LANGUAGE C) if (EVENT__SIZEOF_SSIZE_T_LOWER) set(EVENT__ssize_t "ssize_t") set(EVENT__SIZEOF_SSIZE_T ${EVENT__SIZEOF_SSIZE_T_LOWER}) elseif (EVENT__SIZEOF_SSIZE_T_UPPER) set(EVENT__ssize_t "SSIZE_T") set(EVENT__SIZEOF_SSIZE_T ${EVENT__SIZEOF_SSIZE_T_UPPER}) else() set(EVENT__ssize_t "int") set(EVENT__SIZEOF_SSIZE_T ${EVENT__SIZEOF_INT}) endif() CHECK_TYPE_SIZE(socklen_t EVENT__SIZEOF_SOCKLEN_T) if(NOT EVENT__SIZEOF_SOCKLEN_T) set(EVENT__socklen_t "unsigned int") set(EVENT__SIZEOF_SOCKLEN_T ${EVENT__SIZEOF_UNSIGNED_INT}) else() set(EVENT__socklen_t "socklen_t") endif() CHECK_TYPE_SIZE(pid_t EVENT__SIZEOF_PID_T) if(NOT EVENT__SIZEOF_PID_T) set(EVENT__pid_t "int") set(EVENT__SIZEOF_PID_T ${EVENT__SIZEOF_INT}) else() set(EVENT__pid_t "pid_t") set(EVENT__SIZEOF_PID_T EVENT__SIZEOF_PID_T) endif() if (NOT EVENT__DISABLE_THREAD_SUPPORT) CHECK_TYPE_SIZE(pthread_t EVENT__SIZEOF_PTHREAD_T) endif() if(EVENT__HAVE_CLOCK_GETTIME) set(EVENT__DNS_USE_CPU_CLOCK_FOR_ID 1) endif() # we're just getting lazy now. CHECK_TYPE_SIZE("uintptr_t" EVENT__HAVE_UINTPTR_T) CHECK_TYPE_SIZE("void *" EVENT__SIZEOF_VOID_P) # Tests file offset bits. # TODO: Add AIX test for if -D_LARGE_FILES is needed. # XXX: Why is this here? we don't even use it. Well, we don't even use it # on top of that, why is it set in the config.h?! IT_MAKES_NO_SENSE # I'm commenting it out for now. # - ellzey #CHECK_FILE_OFFSET_BITS() #set(EVENT___FILE_OFFSET_BITS _FILE_OFFSET_BITS) include(CheckWaitpidSupportWNOWAIT) # Verify kqueue works with pipes. if (EVENT__HAVE_KQUEUE) if (CMAKE_CROSSCOMPILING AND NOT EVENT__FORCE_KQUEUE_CHECK) message(WARNING "Cannot check if kqueue works with pipes when crosscompiling, use EVENT__FORCE_KQUEUE_CHECK to be sure (this requires manually running a test program on the cross compilation target)") set(EVENT__HAVE_WORKING_KQUEUE 1) else() message(STATUS "Checking if kqueue works with pipes...") include(CheckWorkingKqueue) endif() endif() CHECK_SYMBOL_EXISTS(_MINIX "stdio.h" EVENT___MINIX) CHECK_SYMBOL_EXISTS(_POSIX_1_SOURCE "stdio.h" EVENT___POSIX_1_SOURCE) CHECK_SYMBOL_EXISTS(_POSIX_SOURCE "stdio.h" EVENT___POSIX_SOURCE) if(EVENT__HAVE_NETDB_H) list(APPEND CMAKE_EXTRA_INCLUDE_FILES netdb.h) CHECK_TYPE_SIZE("struct addrinfo" EVENT__HAVE_STRUCT_ADDRINFO) elseif(WIN32) list(APPEND CMAKE_EXTRA_INCLUDE_FILES ws2tcpip.h) CHECK_TYPE_SIZE("struct addrinfo" EVENT__HAVE_STRUCT_ADDRINFO) endif() # Check for sockaddr structure sizes. set(SOCKADDR_HEADERS) if (WIN32) set(CMAKE_REQUIRED_DEFINITIONS "-DWIN32_LEAN_AND_MEAN") if (_MSC_VER LESS 1300) set(SOCKADDR_HEADERS winsock.h) else() set(SOCKADDR_HEADERS winsock2.h ws2tcpip.h) endif() else() if (EVENT__HAVE_NETINET_IN_H) set(SOCKADDR_HEADERS ${SOCKADDR_HEADERS} netinet/in.h) endif() if (EVENT__HAVE_NETINET_IN6_H) set(SOCKADDR_HEADERS ${SOCKADDR_HEADERS} netinet/in6.h) endif() if (EVENT__HAVE_SYS_SOCKET_H) set(SOCKADDR_HEADERS ${SOCKADDR_HEADERS} sys/socket.h) endif() if (EVENT__HAVE_NETDB_H) set(SOCKADDR_HEADERS ${SOCKADDR_HEADERS} netdb.h) endif() endif() CHECK_TYPE_SIZE("struct in6_addr" EVENT__HAVE_STRUCT_IN6_ADDR) if(EVENT__HAVE_STRUCT_IN6_ADDR) CHECK_STRUCT_HAS_MEMBER("struct in6_addr" s6_addr16 "${SOCKADDR_HEADERS}" EVENT__HAVE_STRUCT_IN6_ADDR_S6_ADDR16) CHECK_STRUCT_HAS_MEMBER("struct in6_addr" s6_addr32 "${SOCKADDR_HEADERS}" EVENT__HAVE_STRUCT_IN6_ADDR_S6_ADDR32) endif() CHECK_TYPE_SIZE("sa_family_t" EVENT__HAVE_SA_FAMILY_T) CHECK_TYPE_SIZE("struct sockaddr_in6" EVENT__HAVE_STRUCT_SOCKADDR_IN6) if(EVENT__HAVE_STRUCT_SOCKADDR_IN6) CHECK_STRUCT_HAS_MEMBER("struct sockaddr_in6" sin6_len "${SOCKADDR_HEADERS}" EVENT__HAVE_STRUCT_SOCKADDR_IN6_SIN6_LEN) CHECK_STRUCT_HAS_MEMBER("struct sockaddr_in6" sin_len "${SOCKADDR_HEADERS}" EVENT__HAVE_STRUCT_SOCKADDR_IN_SIN_LEN) endif() CHECK_TYPE_SIZE("struct sockaddr_storage" EVENT__HAVE_STRUCT_SOCKADDR_STORAGE) if(EVENT__HAVE_STRUCT_SOCKADDR_STORAGE) CHECK_STRUCT_HAS_MEMBER("struct sockaddr_storage" ss_family "${SOCKADDR_HEADERS}" EVENT__HAVE_STRUCT_SOCKADDR_STORAGE_SS_FAMILY) CHECK_STRUCT_HAS_MEMBER("struct sockaddr_storage" __ss_family "${SOCKADDR_HEADERS}" EVENT__HAVE_STRUCT_SOCKADDR_STORAGE___SS_FAMILY) endif() # Group the source files. set(HDR_PRIVATE bufferevent-internal.h changelist-internal.h defer-internal.h epolltable-internal.h evbuffer-internal.h event-internal.h evmap-internal.h evrpc-internal.h evsignal-internal.h evthread-internal.h ht-internal.h http-internal.h iocp-internal.h ipv6-internal.h log-internal.h minheap-internal.h mm-internal.h ratelim-internal.h strlcpy-internal.h util-internal.h evconfig-private.h compat/sys/queue.h) set(HDR_COMPAT include/evdns.h include/evrpc.h include/event.h include/evhttp.h include/evutil.h) set(HDR_PUBLIC include/event2/buffer.h include/event2/bufferevent.h include/event2/bufferevent_compat.h include/event2/bufferevent_struct.h include/event2/buffer_compat.h include/event2/dns.h include/event2/dns_compat.h include/event2/dns_struct.h include/event2/event.h include/event2/event_compat.h include/event2/event_struct.h include/event2/http.h include/event2/http_compat.h include/event2/http_struct.h include/event2/keyvalq_struct.h include/event2/listener.h include/event2/rpc.h include/event2/rpc_compat.h include/event2/rpc_struct.h include/event2/tag.h include/event2/tag_compat.h include/event2/thread.h include/event2/util.h include/event2/visibility.h ${PROJECT_BINARY_DIR}/include/event2/event-config.h) set(SRC_CORE buffer.c bufferevent.c bufferevent_filter.c bufferevent_pair.c bufferevent_ratelim.c bufferevent_sock.c event.c evmap.c evthread.c evutil.c evutil_rand.c evutil_time.c listener.c log.c signal.c strlcpy.c) if(EVENT__HAVE_SELECT) list(APPEND SRC_CORE select.c) endif() if(EVENT__HAVE_POLL) list(APPEND SRC_CORE poll.c) endif() if(EVENT__HAVE_KQUEUE) list(APPEND SRC_CORE kqueue.c) endif() if(EVENT__HAVE_DEVPOLL) list(APPEND SRC_CORE devpoll.c) endif() if(EVENT__HAVE_EPOLL) list(APPEND SRC_CORE epoll_sub.c epoll.c) endif() if(EVENT__HAVE_EVENT_PORTS) list(APPEND SRC_CORE evport.c) endif() if (NOT EVENT__DISABLE_OPENSSL) find_package(OpenSSL REQUIRED) set(EVENT__HAVE_OPENSSL 1) message(STATUS "OpenSSL include: ${OPENSSL_INCLUDE_DIR}") message(STATUS "OpenSSL lib: ${OPENSSL_LIBRARIES}") include_directories(${OPENSSL_INCLUDE_DIR}) list(APPEND SRC_CORE bufferevent_openssl.c) list(APPEND HDR_PUBLIC include/event2/bufferevent_ssl.h) list(APPEND LIB_APPS ${OPENSSL_LIBRARIES}) endif() if (NOT EVENT__DISABLE_THREAD_SUPPORT) if (WIN32) list(APPEND SRC_CORE evthread_win32.c) else() find_package(Threads REQUIRED) if (NOT CMAKE_USE_PTHREADS_INIT) message(FATAL_ERROR "Failed to find Pthreads, set EVENT__DISABLE_THREAD_SUPPORT to disable") endif() set(EVENT__HAVE_PTHREADS 1) list(APPEND SRC_CORE evthread_pthread.c) list(APPEND LIB_APPS ${CMAKE_THREAD_LIBS_INIT}) endif() endif() if (NOT EVENT__DISABLE_TESTS) # Zlib is only used for testing. find_package(ZLIB) if (ZLIB_LIBRARY AND ZLIB_INCLUDE_DIR) include_directories(${ZLIB_INCLUDE_DIRS}) set(EVENT__HAVE_LIBZ 1) list(APPEND LIB_APPS ${ZLIB_LIBRARIES}) endif() endif() set(SRC_EXTRA event_tagging.c http.c evdns.c evrpc.c) add_definitions(-DHAVE_CONFIG_H) # We use BEFORE here so we don't accidentally look in system directories # first for some previous versions of the headers that are installed. include_directories(BEFORE ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/compat ${PROJECT_SOURCE_DIR}/include) if(WIN32) list(APPEND SRC_CORE buffer_iocp.c bufferevent_async.c event_iocp.c win32select.c) list(APPEND HDR_PRIVATE WIN32-Code/getopt.h) set(EVENT__DNS_USE_FTIME_FOR_ID 1) set(LIB_PLATFORM ws2_32) add_definitions( -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE) include_directories(./WIN32-Code) endif() if (UNIX) list(APPEND LIB_PLATFORM m) endif() if (SOLARIS) list(APPEND LIB_PLATFORM socket nsl) endif() source_group("Headers Private" FILES ${HDR_PRIVATE}) source_group("Header Compat" FILES ${HDR_COMPAT}) source_group("Headers Public" FILES ${HDR_PUBLIC}) source_group("Source Core" FILES ${SRC_CORE}) source_group("Source Extra" FILES ${SRC_EXTRA}) # Generate the configure headers. # (Place them in the build dir so we don't polute the source tree with generated files). include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/include) if (EVENT__BUILD_SHARED_LIBRARIES) set(EVENT__LIBRARY_TYPE SHARED) if ((CMAKE_COMPILER_IS_GNUCC) OR (${CMAKE_C_COMPILER_ID} STREQUAL "Clang")) add_compiler_flags(-fvisibility=hidden) elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "SunPro") add_compiler_flags(-xldscope=hidden) endif() set(EVENT__NEED_DLLIMPORT 1) else (EVENT__BUILD_SHARED_LIBRARIES) set(EVENT__LIBRARY_TYPE STATIC) endif (EVENT__BUILD_SHARED_LIBRARIES) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/event-config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/event2/event-config.h NEWLINE_STYLE UNIX) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/evconfig-private.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/evconfig-private.h) # # Create the libraries. # # TODO: Add dynamic versions of the libraries as well. add_library(event_core ${EVENT__LIBRARY_TYPE} ${HDR_PRIVATE} ${HDR_COMPAT} ${HDR_PUBLIC} ${SRC_CORE}) add_library(event_extra ${EVENT__LIBRARY_TYPE} ${HDR_PRIVATE} ${HDR_COMPAT} ${HDR_PUBLIC} ${SRC_CORE} ${SRC_EXTRA}) # library exists for historical reasons; it contains the contents of # both libevent_core and libevent_extra. You shouldn’t use it; it may # go away in a future version of Libevent. add_library(event ${EVENT__LIBRARY_TYPE} ${HDR_PRIVATE} ${HDR_COMPAT} ${HDR_PUBLIC} ${SRC_CORE} ${SRC_EXTRA}) if (EVENT__BUILD_SHARED_LIBRARIES) # Prepare static library to be linked to tests that need hidden symbols add_library(event_extra_static STATIC ${HDR_PRIVATE} ${HDR_COMPAT} ${HDR_PUBLIC} ${SRC_CORE} ${SRC_EXTRA}) set(EVENT_EXTRA_FOR_TEST event_extra_static) target_link_libraries(event_core ${OPENSSL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${LIB_PLATFORM}) target_link_libraries(event ${OPENSSL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${LIB_PLATFORM}) target_link_libraries(event_extra ${OPENSSL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${LIB_PLATFORM}) set_target_properties(event PROPERTIES SOVERSION ${EVENT_ABI_LIBVERSION}) set_target_properties(event_core PROPERTIES SOVERSION ${EVENT_ABI_LIBVERSION}) set_target_properties(event_extra PROPERTIES SOVERSION ${EVENT_ABI_LIBVERSION}) else (EVENT__BUILD_SHARED_LIBRARIES) set(EVENT_EXTRA_FOR_TEST event_extra) endif (EVENT__BUILD_SHARED_LIBRARIES) # # Samples. # if (NOT EVENT__DISABLE_SAMPLES) set(SAMPLES dns-example event-read-fifo hello-world signal-test http-server http-connect time-test) if (NOT EVENT__DISABLE_OPENSSL AND OPENSSL_LIBRARIES) set(CMAKE_REQUIRED_LIBRARIES ${OPENSSL_LIBRARIES}) CHECK_FUNCTION_EXISTS_EX(ERR_remove_thread_state EVENT__HAVE_ERR_REMOVE_THREAD_STATE) set(CMAKE_REQUIRED_LIBRARIES "") # Special sample with more than one file. add_executable(https-client sample/https-client.c sample/openssl_hostname_validation.c sample/hostcheck.c) target_link_libraries(https-client event_extra ${LIB_APPS} ${LIB_PLATFORM}) add_dependencies(https-client event_extra) # Requires OpenSSL. list(APPEND SAMPLES le-proxy) endif() foreach(SAMPLE ${SAMPLES}) add_executable(${SAMPLE} sample/${SAMPLE}.c) target_link_libraries(${SAMPLE} event_extra ${LIB_APPS} ${LIB_PLATFORM}) add_dependencies(${SAMPLE} event_extra) endforeach() if (WIN32) target_sources(dns-example PUBLIC WIN32-Code/getopt.c WIN32-Code/getopt_long.c) endif() endif() if (NOT EVENT__DISABLE_BENCHMARK) foreach (BENCHMARK bench bench_cascade bench_http bench_httpclient) set(BENCH_SRC test/${BENCHMARK}.c) if (WIN32) list(APPEND BENCH_SRC WIN32-Code/getopt.c WIN32-Code/getopt_long.c) endif() add_executable(${BENCHMARK} ${BENCH_SRC}) target_link_libraries(${BENCHMARK} event_extra ${LIB_PLATFORM}) add_dependencies(${BENCHMARK} event_extra) endforeach() endif() if (NOT EVENT__DISABLE_TESTS) # # Generate Regress tests. # if (NOT EVENT__DISABLE_REGRESS) # (We require python2 to generate the regress tests) foreach (PY python2.6 python2.7 python2) unset(FIND_PYTHON2 CACHE) find_program(FIND_PYTHON2 ${PY}) if (FIND_PYTHON2) set(PYTHON_EXECUTABLE "${PY}") break() endif() endforeach() find_package(PythonInterp) if (PYTHONINTERP_FOUND AND PYTHON_VERSION_STRING VERSION_LESS "3.0.0") set(__FOUND_USABLE_PYTHON 1) endif() if (__FOUND_USABLE_PYTHON) message(STATUS "Generating regress tests...") add_definitions(-DTINYTEST_LOCAL) add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/test/regress.gen.c ${CMAKE_CURRENT_SOURCE_DIR}/test/regress.gen.h DEPENDS event_rpcgen.py test/regress.rpc COMMAND ${PYTHON_EXECUTABLE} ../event_rpcgen.py regress.rpc WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test) list(APPEND SRC_REGRESS test/regress.c test/regress.gen.c test/regress.gen.h test/regress_buffer.c test/regress_bufferevent.c test/regress_dns.c test/regress_et.c test/regress_finalize.c test/regress_http.c test/regress_listener.c test/regress_main.c test/regress_minheap.c test/regress_rpc.c test/regress_testutils.c test/regress_testutils.h test/regress_util.c test/tinytest.c ${SRC_CORE} ${SRC_EXTRA}) if (WIN32) list(APPEND SRC_REGRESS test/regress_iocp.c) if (NOT EVENT__DISABLE_THREAD_SUPPORT) list(APPEND SRC_REGRESS test/regress_thread.c) endif() endif() if (CMAKE_USE_PTHREADS_INIT) list(APPEND SRC_REGRESS test/regress_thread.c) endif() if (ZLIB_LIBRARY AND ZLIB_INCLUDE_DIR) list(APPEND SRC_REGRESS test/regress_zlib.c) endif() if (OPENSSL_LIBRARIES) list(APPEND SRC_REGRESS test/regress_ssl.c) endif() add_executable(regress ${SRC_REGRESS}) # While building the test suite we don't want the visibility # header trying to "dllimport" the symbols on windows (it # generates a ton of warnings due to different link # attributes for all of the symbols) SET_TARGET_PROPERTIES(regress PROPERTIES COMPILE_DEFINITIONS "EVENT_BUILDING_REGRESS_TEST=1") target_link_libraries(regress ${LIB_APPS} ${LIB_PLATFORM}) else() message(WARNING "No suitable Python interpreter found, cannot generate regress tests!") endif() endif() # # Test programs. # # all of these, including the cmakelists.txt should be moved # into the dirctory 'tests' first. # # doing this, we can remove all the DISABLE_TESTS stuff, and simply # do something like: # # add_custom_targets(tests) # add_executable(... EXCLUDE_FROM_ALL ...c) # add_dependencis(tests testa testb testc) # add_test(....) # # then you can just run 'make tests' instead of them all # auto-compile|running # - ellzey set(TESTPROGS test-changelist test-eof test-fdleak test-init test-time test-weof) set(ALL_TESTPROGS ${TESTPROGS} test-dumpevents test-ratelim) # Create test program executables. foreach (TESTPROG ${ALL_TESTPROGS}) add_executable(${TESTPROG} test/${TESTPROG}.c) target_link_libraries(${TESTPROG} ${EVENT_EXTRA_FOR_TEST} ${LIB_PLATFORM}) add_dependencies(${TESTPROG} ${EVENT_EXTRA_FOR_TEST}) endforeach() # # We run all tests with the different backends turned on one at a time. # # Add event backends based on system introspection result. set(BACKENDS "") if (EVENT__HAVE_EPOLL) list(APPEND BACKENDS EPOLL) endif() if (EVENT__HAVE_SELECT) list(APPEND BACKENDS SELECT) endif() if (EVENT__HAVE_POLL) list(APPEND BACKENDS POLL) endif() if (EVENT__HAVE_KQUEUE) list(APPEND BACKENDS KQUEUE) endif() if (EVENT__HAVE_EVENT_PORTS) list(APPEND BACKENDS EVPORT) endif() if (EVENT__HAVE_DEVPOLL) list(APPEND BACKENDS DEVPOLL) endif() if (WIN32) list(APPEND BACKENDS WIN32) endif() # Default environment variables turns off all event systems, # then we enable each one, one at a time when creating the tests. set(DEFAULT_TEST_ENV_VARS "EVENT_SHOW_METHOD=1;") foreach(BACKEND ${BACKENDS}) set(BACKEND_ENV_VAR "EVENT_NO${BACKEND}=1") list(APPEND DEFAULT_TEST_ENV_VARS "${BACKEND_ENV_VAR}") endforeach() # Macro that creates the ctest test for a backend. macro(add_backend_test BACKEND_TEST_NAME ENV_VARS) set(TEST_NAMES "") foreach (TESTPROG ${TESTPROGS}) set(TEST_NAME ${TESTPROG}__${BACKEND_TEST_NAME}) add_test(${TEST_NAME} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TESTPROG}) list(APPEND TEST_NAMES ${TEST_NAME}) set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT "${ENV_VARS}") endforeach() # Dump events test. if (__FOUND_USABLE_PYTHON) set(TEST_NAME test-dumpevents__${BACKEND_TEST_NAME}) add_test(${TEST_NAME} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test-dumpevents | ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/check-dumpevents.py) set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT "${ENV_VARS}") else() message(WARNING "test-dumpevents will be run without output check since python was not found!") set(TEST_NAME test-dumpevents__${BACKEND_TEST_NAME}_no_check) add_test(${TEST_NAME} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test-dumpevents) set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT "${ENV_VARS}") endif() # Regress tests. if (NOT EVENT__DISABLE_REGRESS AND __FOUND_USABLE_PYTHON) set(TEST_NAME regress__${BACKEND_TEST_NAME}) add_test(${TEST_NAME} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/regress) set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT "${ENV_VARS}") add_test(${TEST_NAME}_debug ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/regress) set_tests_properties(${TEST_NAME}_debug PROPERTIES ENVIRONMENT "${ENV_VARS};EVENT_DEBUG_MODE=1") endif() endmacro() # Add the tests for each backend. foreach(BACKEND ${BACKENDS}) # Enable this backend only. set(BACKEND_ENV_VARS ${DEFAULT_TEST_ENV_VARS}) list(REMOVE_ITEM BACKEND_ENV_VARS EVENT_NO${BACKEND}=1) # Epoll has some extra settings. if (${BACKEND} STREQUAL "EPOLL") add_backend_test(timerfd_${BACKEND} "${BACKEND_ENV_VARS};EVENT_PRECISE_TIMER=1") add_backend_test(changelist_${BACKEND} "${BACKEND_ENV_VARS};EVENT_EPOLL_USE_CHANGELIST=yes") add_backend_test(timerfd_changelist_${BACKEND} "${BACKEND_ENV_VARS};EVENT_EPOLL_USE_CHANGELIST=yes;EVENT_PRECISE_TIMER=1") else() add_backend_test(${BACKEND} "${BACKEND_ENV_VARS}") endif() endforeach() # # Rate limiter tests. # # Group limits, no connection limit. set(RL_BIN ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test-ratelim) add_test(test-ratelim__group_lim ${RL_BIN} -g 30000 -n 30 -t 100 --check-grouplimit 1000 --check-stddev 100) # Connection limit, no group limit. add_test(test-ratelim__con_lim ${RL_BIN} -c 1000 -n 30 -t 100 --check-connlimit 50 --check-stddev 50) # Connection limit and group limit. add_test(test-ratelim__group_con_lim ${RL_BIN} -c 1000 -g 30000 -n 30 -t 100 --check-grouplimit 1000 --check-connlimit 50 --check-stddev 50) # Connection limit and group limit with independent drain. add_test(test-ratelim__group_con_lim_drain ${RL_BIN} -c 1000 -g 35000 -n 30 -t 100 -G 500 --check-grouplimit 1000 --check-connlimit 50 --check-stddev 50) # Add a "make verify" target, same as for autoconf. # (Important! This will unset all EVENT_NO* environment variables. # If they are set in the shell the tests are running using simply "ctest" or "make test" will fail) if (WIN32) # Windows doesn't have "unset". But you can use "set VAR=" instead. # We need to guard against the possibility taht EVENT_NOWIN32 is set, and all test failing # since no event backend being available. file(TO_NATIVE_PATH ${CMAKE_CTEST_COMMAND} WINDOWS_CTEST_COMMAND) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp/verify_tests.bat " set EVENT_NOWIN32= \"${WINDOWS_CTEST_COMMAND}\" ") message(STATUS "${WINDOWS_CTEST_COMMAND}") file(COPY ${CMAKE_CURRENT_BINARY_DIR}/tmp/verify_tests.bat DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) file(TO_NATIVE_PATH "${CMAKE_CURRENT_BINARY_DIR}/verify_tests.bat" VERIFY_PATH) add_custom_target(verify COMMAND "${VERIFY_PATH}" DEPENDS event ${ALL_TESTPROGS}) else() # On some platforms doing exec(unset) as CMake does won't work, so make sure # we run the unset command in a shell instead. # First we write the script contents. file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/tmp/verify_tests.sh " #!/bin/bash unset EVENT_NOEPOLL; unset EVENT_NOPOLL; unset EVENT_NOSELECT; unset EVENT_NOWIN32; unset EVENT_NOEVPORT; unset EVENT_NOKQUEUE; unset EVENT_NODEVPOLL ${CMAKE_CTEST_COMMAND} ") # Then we copy the file (this allows us to set execute permission on it) file(COPY ${CMAKE_CURRENT_BINARY_DIR}/tmp/verify_tests.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) # Create the target that runs the script. add_custom_target(verify COMMAND ${CMAKE_CURRENT_BINARY_DIR}/verify_tests.sh DEPENDS event ${ALL_TESTPROGS}) endif() if (NOT EVENT__DISABLE_REGRESS AND __FOUND_USABLE_PYTHON) add_dependencies(verify regress) endif() if (EVENT__COVERAGE) include(CodeCoverage) setup_target_for_coverage( verify_coverage # Coverage target name "make verify_coverage" make # Test runner. coverage # Output directory. verify) # Arguments passed to test runner. "make verify" endif() enable_testing() include(CTest) endif() # # Installation preparation. # # Allow the user to override installation directories. set(EVENT_INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries") set(EVENT_INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables") set(EVENT_INSTALL_INCLUDE_DIR include CACHE PATH "Installation directory for header files") if(WIN32 AND NOT CYGWIN) set(DEF_INSTALL_CMAKE_DIR cmake) else() set(DEF_INSTALL_CMAKE_DIR lib/cmake/libevent) endif() set(EVENT_INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH "Installation directory for CMake files") # Make sure the paths are absolute. foreach(p LIB BIN INCLUDE CMAKE) set(var EVENT_INSTALL_${p}_DIR) if(NOT IS_ABSOLUTE "${${var}}") set(${var} "${CMAKE_INSTALL_PREFIX}/${${var}}") endif() endforeach() # Export targets (This is used for other CMake projects to easily find the libraries and include files). export(TARGETS event event_extra event_core FILE "${PROJECT_BINARY_DIR}/LibeventTargets.cmake") export(PACKAGE libevent) # Generate the config file for the build-tree. set(EVENT__INCLUDE_DIRS "${PROJECT_SOURCE_DIR}/include" "${PROJECT_BINARY_DIR}/include") set(LIBEVENT_INCLUDE_DIRS ${EVENT__INCLUDE_DIRS} CACHE PATH "Libevent include directories") configure_file(${PROJECT_SOURCE_DIR}/cmake/LibeventConfigBuildTree.cmake.in ${PROJECT_BINARY_DIR}/LibeventConfig.cmake @ONLY) # Generate the config file for the installation tree. file(RELATIVE_PATH REL_INCLUDE_DIR "${EVENT_INSTALL_CMAKE_DIR}" "${EVENT_INSTALL_INCLUDE_DIR}") # Calculate the relative directory from the Cmake dir. # Note the EVENT_CMAKE_DIR is defined in LibeventConfig.cmake.in, # we escape it here so it's evaluated when it is included instead # so that the include dirs are givenrelative to where the # config file is located. set(EVENT__INCLUDE_DIRS "\${EVENT_CMAKE_DIR}/${REL_INCLUDE_DIR}") configure_file(${PROJECT_SOURCE_DIR}/cmake/LibeventConfig.cmake.in ${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/LibeventConfig.cmake @ONLY) # Generate version info for both build-tree and install-tree. configure_file(${PROJECT_SOURCE_DIR}/cmake/LibeventConfigVersion.cmake.in ${PROJECT_BINARY_DIR}/LibeventConfigVersion.cmake @ONLY) # Define the public headers. set_target_properties(event event_core event_extra PROPERTIES PUBLIC_HEADER "${HDR_PUBLIC}") # # Install targets. # install(TARGETS event event_core event_extra EXPORT LibeventTargets RUNTIME DESTINATION "${EVENT_INSTALL_BIN_DIR}" COMPONENT bin LIBRARY DESTINATION "${EVENT_INSTALL_LIB_DIR}" COMPONENT lib ARCHIVE DESTINATION "${EVENT_INSTALL_LIB_DIR}" COMPONENT lib PUBLIC_HEADER DESTINATION "${EVENT_INSTALL_INCLUDE_DIR}/event2" COMPONENT dev) # Install compat headers install(FILES ${HDR_COMPAT} DESTINATION "${EVENT_INSTALL_INCLUDE_DIR}" COMPONENT dev) # Install the configs. install(FILES ${PROJECT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/LibeventConfig.cmake ${PROJECT_BINARY_DIR}/LibeventConfigVersion.cmake DESTINATION "${EVENT_INSTALL_CMAKE_DIR}" COMPONENT dev) # Install exports for the install-tree. install(EXPORT LibeventTargets DESTINATION "${EVENT_INSTALL_CMAKE_DIR}" COMPONENT dev) set(LIBEVENT_LIBRARIES event event_core event_extra CACHE STRING "Libevent libraries") message(STATUS "") message(STATUS " ---( Libevent " ${EVENT_VERSION} " )---") message(STATUS "") message(STATUS "Available event backends: ${BACKENDS}") message(STATUS "CMAKE_BINARY_DIR: " ${CMAKE_BINARY_DIR}) message(STATUS "CMAKE_CURRENT_BINARY_DIR: " ${CMAKE_CURRENT_BINARY_DIR}) message(STATUS "CMAKE_SOURCE_DIR: " ${CMAKE_SOURCE_DIR}) message(STATUS "CMAKE_CURRENT_SOURCE_DIR: " ${CMAKE_CURRENT_SOURCE_DIR}) message(STATUS "PROJECT_BINARY_DIR: " ${PROJECT_BINARY_DIR}) message(STATUS "PROJECT_SOURCE_DIR: " ${PROJECT_SOURCE_DIR}) message(STATUS "CMAKE_MODULE_PATH: " ${CMAKE_MODULE_PATH}) message(STATUS "CMAKE_COMMAND: " ${CMAKE_COMMAND}) message(STATUS "CMAKE_ROOT: " ${CMAKE_ROOT} ) message(STATUS "CMAKE_SYSTEM: " ${CMAKE_SYSTEM} ) message(STATUS "CMAKE_SYSTEM_NAME: " ${CMAKE_SYSTEM_NAME} ) message(STATUS "CMAKE_SYSTEM_VERSION: " ${CMAKE_SYSTEM_VERSION} ) message(STATUS "CMAKE_SYSTEM_PROCESSOR: " ${CMAKE_SYSTEM_PROCESSOR} ) message(STATUS "CMAKE_SKIP_RPATH: " ${CMAKE_SKIP_RPATH} ) message(STATUS "CMAKE_VERBOSE_MAKEFILE: " ${CMAKE_VERBOSE_MAKEFILE} ) message(STATUS "CMAKE_C_FLAGS: " ${CMAKE_C_FLAGS} ) message(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE} ) message(STATUS "CMAKE_C_COMPILER: " ${CMAKE_C_COMPILER} ) message(STATUS "CMAKE_AR: " ${CMAKE_AR} ) message(STATUS "CMAKE_RANLIB: " ${CMAKE_RANLIB} ) message(STATUS "") ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/CONTRIBUTING.md����������������������������������������������0000664�0000000�0000000�00000001300�15270523712�0022221�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Contributing to the libevent ## Coding style First and most generic rule: **just look around**. But, we have a script for checking patches/files/git-refs: ```shell # Chech HEAD git ref ./checkpatch.sh -r ./checkpatch.sh -r HEAD # Check patch git format-patch --stdout -1 | ./checkpatch.sh -p git show -1 | ./checkpatch.sh -p # Or via regular files git format-patch --stdout -2 ./checkpatch.sh *.patch # Over a file ./checkpatch.sh -d event.c ./checkpatch.sh -d < event.c # And print the whole file not only summary ./checkpatch.sh -f event.c ./checkpatch.sh -f < event.c # See ./checkpatch.sh -h ``` ## Testing - Write new unit test in `test/regress_{MORE_SUITABLE_FOR_YOU}.c` - `make verify` ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/ChangeLog����������������������������������������������������0000664�0000000�0000000�00000305466�15270523712�0021566�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Changes in version 2.1.8-stable (22 January 2017) Libevent 2.1.8-stable, it contains openssl fixes for resetting fd and using bufferevent_openssl_filter_new(). vagrant fixes, some build fixes, increased timeout for some tests (to reduce number of failures due to timing issues), date in RFC1123 format and running tests in parallel. There are highlighted changes above. Build fixes: o Fix _FILE_OFFSET_BITS redinition (solaris/autotools) (336f3b11 Azat Khuzhin) o util-internal: fix __func__ redefinition (netbsd) (253e7fa9 Azat Khuzhin) o Fix signedness differ for iov_base (solaris) (2c62062e Azat Khuzhin) o evutil_time: include <unistd.h> when there is only sleep()/usleep() (3e75194c Azat Khuzhin) o http: fix formatter for pritnf for req->ntoread (osx) (1cbf26f6 Azat Khuzhin) Testing environment: o Merge branch 'automake-tests-parallel-v4' (*includes ci bits also*) (59e217df Azat Khuzhin) Vagrant env fixes: o vagrant/netbsd: missing libtool (9c9be399 Azat Khuzhin) o vagrant/netbsd: more reliable way of installing packages (36da6877 Azat Khuzhin) o vagrant/osx: use make instead of gmake (there is no gmake) (f7c70aef Azat Khuzhin) o vagrant: add centos box (ca591c5b Azat Khuzhin) Tests: o test/dns: replace servname since solaris does not have "http" (d6bafbbe Azat Khuzhin) o test/thread: netbsd is too slow, increase timeout for conditions_simple (3c7422fc Azat Khuzhin) o test/dns: run async resolving after sync one (to avoid timeouts) (07862531 Azat Khuzhin) o test/http: turn off some tests that based on backlog filling (falky) (26f416c1 Azat Khuzhin) Bugfixes: o Merge branch 'openssl-filter-fixes-v4' (83e0f43b Azat Khuzhin) o Merge branch 'date-rfc1123' (68def435,4798de6c,4545807d Azat Khuzhin) o Merge branch 'be-openssl-fd-reset-fix-v2' (86fa0070,32adf434 Azat Khuzhin) o Merge branch 'openssl-1.1-init-fixes-v2' (18a161f0 Azat Khuzhin) o Fix incorrect MIME type (23f9a20e johnsonlee) Trivial fixes: Documentation updates: o Update README.md (3821cca1 Breaker) Changes in version 2.1.7-rc (2 Novemer 2016) Libevent 2.1.7-rc contains openssl 1.1 support, build fixes, CI improvements and plus Vagrantfile for testing under multiple OS'es. Continious Integration: o Use coveralls.io via travis (9ac000c Azat Khuzhin) o travis-ci: use container-based infrastructure (7e12e96 Azat Khuzhin) o travis-ci/osx: fix compiling/linking openssl libraries (9d2f8d4 Azat Khuzhin) o travis-ci: use gcc-5 (fixes osx|gcc failures) (d7ceae5 Azat Khuzhin) o Testing with vagrant for 6 OS and cmake+autoconf (9585338 Azat Khuzhin) o travis-ci/osx: install lcov (e4e099b Azat Khuzhin) Build Improvements/Fixes: o Fix cmake -DEVENT__COVERAGE=ON (40fbffc Azat Khuzhin) o autogen.sh: learn about gmake (9376ac4 Azat Khuzhin) o autogen.sh: remove all autoconf/automake caches, if any (69cce25 Azat Khuzhin) o cmake: fix finding python2, and check that it is really 2 (3453c08 Azat Khuzhin) o cmake: fix CheckFunctionExistsEx/CheckPrototypeDefinition (CMP0054) (43b69b2 Azat Khuzhin) o cmake: cleanup (dc624ad Zonr Chang) o cmake/win32: fix running regress, but fixing finding python2 interpreter (bcb990a Azat Khuzhin) o cmake: use PYTHON_EXECUTABLE to find python2 (a4d044c Azat Khuzhin) o Merge branch 'force-disable-clockgettime' (83c7cdf Azat Khuzhin) Code Improvements (core) o use ev_uint16_t instead of unsigned short for port (e983712 Thomas Bernard) o Merge branch 'contrib-guide-v2' (b9c5077 Azat Khuzhin) o poll: Prevent libevent from spinning if POLLNVAL occurs (675974c Tim Hentenaar) Testing: o test/regress: cover a polling of invalid fd (cb0df5c Tim Hentenaar) Code Improvements (bufferevent_openssl) o Make it build using OpenSSL 1.1.0 (3e9e0a0 Kurt Roeckx) o Don't call BIO_number_{read|written} on NULL BIOs. (6702da1 Adam Langley) o Switch from a 512 to 2048-bit RSA key. (f9803a6 Adam Langley) Trivial fixes: o Ignore temporary configure files (8fb08ae Azat Khuzhin) o README.md: fix typo: ar -> are (2361616 Simone Basso) o be: just a simple mistake, reinclude the <errno.h> (7521664 Seven) Changes in version 2.1.6-beta (4 July 2016) Libevent 2.1.6-beta contains mostly bug fixes (evbuffers, evthread, evdns, bufferevents, core, http, samples), improvements but mostly to fix some possible issues (EVHTTP_CON_LINGERING_CLOSE), a lot of new unit tests and new appveyor integration. Security Fixes (utils) o evutil_parse_sockaddr_port(): fix buffer overflow (329acc1 Azat Khuzhin) Security Fixes (evdns) o evdns: name_parse(): fix remote stack overread (96f64a0 Azat Khuzhin) o evdns: fix searching empty hostnames (ec65c42 Azat Khuzhin) New APIs (evdns) o New function to get address for nameserver. (537177d Nick Mathewson) New APIs (bufferevents) o expose bufferevent_incref/decref (with fewer modifications) (1ed6718 Mark Ellzey) New APIs (internal) o evdns: export cancel via callbacks in util (like async lib core/extra issues) (8cbe65d Azat Khuzhin) New APIs/Improvements (http) o http: take EVHTTP_CON_LINGERING_CLOSE into account for "Expect: 100-Continue" (ac448a7 Azat Khuzhin) o http: lingering close (like nginx have) for entity-too-large (9fde518 Azat Khuzhin) o http: read server response even after server closed the connection (680742e Azat Khuzhin) o http: export evhttp_connection_set_family() (714fc70 Azat Khuzhin) o http: reuse connected address only with EVHTTP_CON_REUSE_CONNECTED_ADDR (a50f5f0 Azat Khuzhin) o http: use IP address that we got before (if any) during retrying (54c887d Azat Khuzhin) Bugfixes (core) o Fix getaddrinfo under solaris (for multiprotocol case) (40730ae Azat Khuzhin) o Check for Mac OS X 10.4 kqueue bug properly (df6f99e Mark Mentovai) o event_reinit: make signals works after fork() without evsig_add() (88640aa Nicholas Marriott) o event_reinit: always re-init signal's socketpair (ad0c237 Nicholas Marriott) o Free event queues even for recursive finalizers (7c8d015 Azat Khuzhin) o Fix checking for make_base_notifiable() (f337296 Azat Khuzhin) o Set correct socklen for PF_INET6 sockaddr len (3499ad9 Mark Ellzey) o Fix garbage value in socketpair util function, stdint? (043ae74 Mark Ellzey) o fix the return value of event_deferred_cb_schedule_ (38cef64 Greg Hazel) o event_free_debug_globals_locks(): disable lock debugging (e5c87d1 Azat Khuzhin) o event: call event_disable_debug_mode() in libevent_global_shutdown() (941faae Azat Khuzhin) o ht-internal: don't reset hth_table_length explicitly in name_##HT_CLEAR (597c7b2 Azat Khuzhin) Bugfixes (evthread) o evthread: fix evthread_setup_global_lock_() for debug-lock with a real-lock case (e4556fc Azat Khuzhin) o evthread: evthreadimpl_disable_lock_debugging_() for libevent_global_shutdown() (ccc5593 Azat Khuzhin) Bugfixes (evdns) o evdns: avoid double-free in evdns_base_free() for probing requests (4db15e0 Azat Khuzhin) o evdns: evdns_base_free(): fix UAF of evdns_base with @fail_requests (00313c5 Azat Khuzhin) o evdns: evdns_base_free(): free requests before namservers (14f84bb Azat Khuzhin) o evdns: fix randomize-case by make case-insensitive as required (9c238de Azat Khuzhin) Bugfixes (bufferevents) o be_sock: handle readv() returns ECONNREFUSED (freebsd 9.2) (3189eb0 Azat Khuzhin) o be_filter: avoid data stuck under active watermarks (b627ad8 Eduardo Panisset) o Fix bufferevent_pair to properly set BEV_EVENT_{READING,WRITING} on flush. (2851889 David Paschich) o be_openssl: clear all pending errors before SSL_*() calls (38e0f4a Azat Khuzhin) o be_sock: cancel in-progress dns requests (86dfd2c Azat Khuzhin) o be_sock: unfreeze buffers on fd changing (255525d Azat Khuzhin) o be_sock: bufferevent_socket_connect_hostname(): make it thread-safe (809bb39 Azat Khuzhin) o be_openssl: don't call do_write() directly from outbuf_cb (da52933 Azat Khuzhin) o be_openssl: use bufferevent_enable() instead of bufferevent_add_event_() (0c66d32 Azat Khuzhin) o be_openssl: don't add events during bev creation (like be_sock) (f4b6284 Azat Khuzhin) o Fix lock leak in be_pair_flush() if flush type is BEV_NORMAL (f45d39d Bill Vaughan) o be_openssl: don't use *_auto() in do_handshake() we can't have fd == -1 there (877280d Azat Khuzhin) o be_openssl: don't call set_open_callbacks() if fd == -1 (e8a2da9 Azat Khuzhin) o be_openssl: get rid off hackish "fd_is_set", to fix some corner cases (40b0379 Azat Khuzhin) o be: we don't need to use getpeername() we we have conn_address (2c271e2 Azat Khuzhin) o Call underlying bev ctrl SET_FD on filtered bufferevents (c2aa7dc Mark Ellzey) o be_pair: release shared lock with the latest of bufferevent_pair (92a359e Azat Khuzhin) Bugfixes (http) o [Issue #313] set method to ASCII "NULL" if evhttp_method() returns NULL (17cc636 Mark Ellzey) o evhttp_have_expect(): fix -Wlogical-not-parentheses (24b5214 Azat Khuzhin) o http: set fd to -1 unconditioally, to avoid leaking of DNS requests (7a4b472 Azat Khuzhin) o http: avoid leaking of fd in evhttp_connection_free() (f0e1341 Azat Khuzhin) o http: get fd from be layer during connection reset (4a53c54 Azat Khuzhin) o http: fix EVHTTP_CON_READ_ON_WRITE_ERROR when it doesn't supported by OS (2ff164a Azat Khuzhin) o http: do not do function calls under EVUTIL_ASSERT() to fix NDEBUG builds (7c89999 Azat Khuzhin) o http: fix leaking of response_code_line (8f18a62 Azat Khuzhin) o http: fix "Expect: 100-continue" client side (0b46b39 Azat Khuzhin) o http: fix conflicts EVHTTP_CON_AUTOFREE and EVHTTP_CON_REUSE_CONNECTED_ADDR (4dc0979 Azat Khuzhin) o http: avoid epoll_ctl() on already closed fd (triggers by http/chunk_out) (ab3bc69 Azat Khuzhin) o http: install timeout for read too during connect for ssl (040000d Azat Khuzhin) o http: fix evhttp_request_own() by checking EVHTTP_USER_OWNED in more cases (b0d3964 Azat Khuzhin) o http: fix detecting EOF without write (7ed02ac Azat Khuzhin) o evhttp: Fix failure to send all output data for POST/PUT requests (24eea0d John Ohl) o Fix evhttp_uriencode() regression. (c6b1ec1 Mark Ellzey) o removed unused vars (e94250c Mark Ellzey) o pointer overflow checks for evhttp_uriencode (72afe4c Zonr Chang) Bugfixes (evbuffers) o buffer: fix overflow check in evbuffer_expand_singlechain() (a3f4ccd Azat Khuzhin) o buffer: evbuffer_add_buffer(): clean empty chains from destination buffer (26fd932 Azat Khuzhin) o Fix n_add_for_cb in evbuffer_prepend() in case of new buffer required (0abd039 Azat Khuzhin) o be_filter: actually disable output_filter during processing output (c031215 Simon Perreault) o evbuffer_add: Use last_with_datap if set, not last. (a8769ef Marcus Sundberg) o EVBUFFER_PTR_SET -> EVBUFFER_PTR_ADD (8674e4f jer-gentoo) Bugfixes (evconnlistener) o listener: unlock lev on error in listener_read_cb() (2a71b33 Azat Khuzhin) o Fix potential fd leak in listener_read_cb() (a695a72 Mark Ellzey) Testing o tests: use waitpid(..., WNOWAIT) to fix failing of main/fork under solaris (43eb56c Azat Khuzhin) o test: replace sleeping with syncing pair in main/fork (16d220c Azat Khuzhin) o test/http: do not run tests that based on backlog filling (freebsd) (500b6b7 Azat Khuzhin) o test/bufferevent/iocp: fix test name for "bufferevent_connect_fail_eventcb" (4410e9d Azat Khuzhin) o test/ssl: use send()/recv()/EVUTIL_ERR_RW_RETRIABLE()/EVUTIL_SOCKET_ERROR() to fix win32 (a9e8cd6 Azat Khuzhin) o test/https_basic: increase timeout for complete write (fixes win32) (d5a2f2f Azat Khuzhin) o test: fix building with --disable-thread-support under win32 (a487706 Azat Khuzhin) o test/buffer: evbuffer_add_buffer() with empty chains (a272bc4 Azat Khuzhin) o test/buffer: evbuffer_remove_buffer() with empty chains (prepend) (f0cfa14 Azat Khuzhin) o test/buffer: evbuffer_remove_buffer() with empty chains (evbuffer_add_buffer()) (2880ce6 Azat Khuzhin) o test/buffer: cover evbuffer_expand() for overflow (48dab7a Azat Khuzhin) o test/be_filter: creating test case for data stuck with active watermarks (766194b Eduardo Panisset) o test/http: avoid using conditionals with omitted operands (fixes VS2015) (2a4bf29 Azat Khuzhin) o test/http: don't mix declarations and code (fixes -Wdeclaration-after-statement) (aabf1c2 Azat Khuzhin) o test/buffer: fix leak in test_evbuffer_prepend() (c08d90b Azat Khuzhin) o test/buffer: avoid errors with --no-fork (reinitialize static vars) (e7d1e39 Azat Khuzhin) o test/buffer: cover n_add_for_cb when evbuffer_prepend() need to allocate buffer (e77ff41 Azat Khuzhin) o test/tinytest_macros: add new one tt_nstr_op() (bd19a28 Azat Khuzhin) o test/bufferevent: check that output_filter disabled during processing output (ae28812 Azat Khuzhin) o test/listener: regression for missing unlock in listener_read_cb() (7d85651 Azat Khuzhin) o test/regress: add tests for evbuffer_add() breakage on empty last chain (d5ee739 Marcus Sundberg) o test/http: fix running some tests sequential (with --no-fork) (bddad71 Azat Khuzhin) o test/http: localize evhttp server structure (cbc3209 Azat Khuzhin) o test/dns: regression for empty hostname (d7348ba Azat Khuzhin) o test/http: fix SERVER_TIMEOUT tests under win32 (d49a658 Azat Khuzhin) o test/http: add a helper for creating timedout/failed request (376f107 Azat Khuzhin) o test/http: adopt for C90 (mixed code and declarations) (d02a285 Azat Khuzhin) o test/http: cover NS timed out during request cancellations separatelly (0c343af Azat Khuzhin) o test/http: request cancellation with resolving/{conn,write}-timeouts in progress (334340d Azat Khuzhin) o test/http: exit from the loop in the errorcb to wait cancellation (927ab33 Azat Khuzhin) o regress_clean_dnsserver(): reset global port vars (351207f Azat Khuzhin) o test/http: read_on_write_error: fix it for win32 (3b58169 Azat Khuzhin) o test/http: separate coverage for EVHTTP_CON_READ_ON_WRITE_ERROR (5c2b4c1 Azat Khuzhin) o test/http: cover "Expect: 100-continue" client-server interaction (31d8116 Azat Khuzhin) o test/http: *lingering tests shouldn't have "Expect: 100-continue" (ed469ab Azat Khuzhin) o test: use EVUTIL_SHUT_WR (04fc82f Azat Khuzhin) o test/http: avoid huge stack allocations to fix win32 builds (3166765 Azat Khuzhin) o test: http/lingering_close: cover EVHTTP_SERVER_LINGERING_CLOSE (e122ca1 Azat Khuzhin) o test: http/non_lingering_close: cover ~EVHTTP_SERVER_LINGERING_CLOSE (f41e1b0 Azat Khuzhin) o test: http/*: update expected HTTP codes for body exceeds `max_body_size` (addf2b9 Azat Khuzhin) o test: http/data_length_constrains: set EVHTTP_CON_READ_ON_WRITE_ERROR (d38a723 Azat Khuzhin) o test: increase buffer size for http/data_length_constraints to trigger EPIPE (0792e1e Azat Khuzhin) o test/tinytest_demo: include <windows.h> for win32 to fix tdm-gcc (f062bbe Azat Khuzhin) o test/regress: cover event_del() waiting mechanism (5b58b70 Azat Khuzhin) o test/regress: cover existing signal callbacks and fork() + event_reinit() (ceddc60 Azat Khuzhin) o test/regress: cover signals after fork() + event_reinit() (b075b81 Azat Khuzhin) o test/regress: main/fork: rewrite assertions by just removing event in callback (088d8b3 Azat Khuzhin) o test/dns: check exit code of evdns_getaddrinfo() (0b9d432 Azat Khuzhin) o test/dns: cover evdns_getaddrinfo() and evdns_base_free() with @fail_requests (4ad3483 Azat Khuzhin) o test/dns: cover @fail_requests for evdns_base_free() (d6c6fb4 Azat Khuzhin) o test/dns: more graceful coverage of @fail_requests (123d372 Azat Khuzhin) o test/ssl: cover busy-loop (i.e. {read,write}-blocked-on-{write,read} stuff) (da0ea7a Azat Khuzhin) o test/http: write_during_read for https (23c77b6 Azat Khuzhin) o test/http: connection_fail for https (7ea26f7 Azat Khuzhin) o test/http: stream_out for https (ac04968 Azat Khuzhin) o test/http: chunk_out for https (a71ffb9 Azat Khuzhin) o test/regress: fix ssl-less builds (need to make this prettier) (3160716 Azat Khuzhin) o test/http: allow dirty shutdown for ssl to fix https_incomplete (1ede326 Azat Khuzhin) o test/http: https basic (59714b4 Azat Khuzhin) o test/http: incomplete{,_timeout} for https (615490d Azat Khuzhin) o test/http: add simplest test for http/https/https_dirty_shutdown (93b19dc Azat Khuzhin) o test/http: https: retry coverage (7c2d24a Azat Khuzhin) o test/http: https server support (plus some helpers) (a7088ad Azat Khuzhin) o test/http: more sanity checks (a27c53c Azat Khuzhin) o test/ssl: export getkey()/getcert()/get_ssl_ctx()/init_ssl() for https (0c4c387 Azat Khuzhin) o test/regress_be: basic coverage bufferevent_flush() for pair/sock layers (ad52602 Azat Khuzhin) o test/regress_be: socket_filter_inactive: check bufferevent after creation (f8081af Azat Khuzhin) o test/regress_be: cover finalizers from inactive to active queue (337684b Azat Khuzhin) o test/regress_buffer: fix clang compilation warnings (d8fd4c0 Azat Khuzhin) o test/regress_http: fix compilation warnings (-Wmissing-field-initializers) (cd422e0 Azat Khuzhin) o test/regress_dns: fix compilation warnings (-Wmissing-field-initializers/for) (f55db98 Azat Khuzhin) o tests/regress_dns: cover that randomize-case works case-insensitive (1e8bfbc Azat Khuzhin) o test: fix bufferevent/bufferevent_pair_release_lock in debug mode (3f749e9 Azat Khuzhin) o test: fix bufferevent/bufferevent_pair_release_lock for freebsd (79f9ace Azat Khuzhin) o test/regress_be: bufferevent_enable() shouldn't call eventcb by it's own (a0f308d Azat Khuzhin) o test/regress_be: introduce fake_listener_create() (37dc9e0 Azat Khuzhin) o test/regress_http: cover evhttp_request_own() (6f6fa0d Azat Khuzhin) o test/regress_http: cover write during read (3d15aeb Azat Khuzhin) o test/regress_http: verify that closecb will be called without multiple write (4be6c70 Azat Khuzhin) o test/regress: fix bufferevent_pair_release_lock with EVENT_DEBUG_MODE (6ea6655 Azat Khuzhin) o test/regress_ssl: check events fd/pending after timeout triggered (cdafdf0 Azat Khuzhin) o test/regress_ssl: cover case when server didn't up (failed with timeout) (74845f1 Azat Khuzhin) o test/regress_ssl: covert that we can't change fd with underlying (df507af Azat Khuzhin) o test/regress_ssl: cover that events (read/write) at finish not pending (762edb4 Azat Khuzhin) o test/regress_ssl: cover fd manipulations (b78a829 Azat Khuzhin) o test/regress_ssl: convert open_ssl_bufevs() to mask (46bba73 Azat Khuzhin) o test/regress_ssl: convert client/server to mask too (3455991 Azat Khuzhin) o test/regress_ssl: cover "allow_dirty_shutdown" (0430327 Azat Khuzhin) o test/regress_ssl: convert regress_bufferevent_openssl() to bitmask (342e116 Azat Khuzhin) o tests/regress_ssl: drop duplicated assert (25e56fd Azat Khuzhin) o test/regress_http: initialize "dns_base" to avoid reading trash (9f0bff3 Azat Khuzhin) o test/http: cover retrying with saved conn_address by shutting down dns server (f4874d8 Azat Khuzhin) o be_pair/regress: cover use of shared lock (lock/unlock/free) (a558fcd Azat Khuzhin) o regress_dns: drop hack for event_debug_map_HT_GROW in leak tests (3540a19 Azat Khuzhin) Sample code o Fix memory leak in signal-test.c (666db91 basavesh.as) o sample/hello-world: exAmple, not eXMple (2d3cd35 kirillDanshin) o dns-example: allow to set ns from args (df19a97 Azat Khuzhin) o dns-example: convert to getopt() (32f8592 Azat Khuzhin) o http-connect: make it win32 compilable (1bf7595 Azat Khuzhin) o sample/https-client: allow to change path to ca-certificates (fdf713a Azat Khuzhin) o sample/https-client: check for ERR_remove_thread_state() existence (c4e9d9b Azat Khuzhin) o sample/https-client: replace ERR_remove_state() by ERR_remove_thread_state() (77ad68a Azat Khuzhin) o sample/https-client: add -timeout option (4637aa8 Azat Khuzhin) o sample/https-client: don't try to free uninitialized SSL (f3d7ff5 Azat Khuzhin) o sample/https-client: graceful exit with freeing memory (to make valgrind happy) (24a1f25 Azat Khuzhin) o https-client: correctly handle URLs with no path (like "https://host:port") (29a0482 Andrey Skriabin) o sample/http-connect: don't use assert() to make it work with NDEBUG (6dc71e7 Azat Khuzhin) o sample/http-connect: made it compatible with C90 (f976d43 Azat Khuzhin) o sample: add HTTP CONNECT tunnelling example using libevent http layer (1d34498 Azat Khuzhin) o Update dns-example. (620ff24 Mark Ellzey) Documentation o Update README.md (b8ec70c Mark Ellzey) o Update README.md (80faee9 Mark Ellzey) o Update README.md (ad4a897 Mark Ellzey) o Update README.md (a2b2e1e Mark Ellzey) o Update README.md (0dfa5dc Mark Ellzey) Code Improvements (evthread) o evthread: add evthread_get_{lock,condition}_callbacks() helpers (c0b34f6 Azat Khuzhin) Code Improvements (core) o util: make @sa const for evutil_socket_connect_() (a8d32c2 Azat Khuzhin) Code Improvements (http) o http: assert's that evbuffer_drain() success on connection reset (2185e63 Azat Khuzhin) o http: introduce evhttp_request_free_() helper (22061ac Azat Khuzhin) o http: introduce evhttp_is_request_connection_close() helper (6540da3 Azat Khuzhin) Code Improvements (bufferevents) o be_sock: bufferevent_socket_set_conn_address(): assert instead of silent no-op (0ab88c2 Azat Khuzhin) o be_sock: sanity check in bufferevent_socket_set_conn_address() (eedbeff Azat Khuzhin) o be: replace sockaddr_storage with sockaddr_in6 for conn_address (3889612 Azat Khuzhin) o be: replace conn_address by full struct instead of pointer (e5615aa Azat Khuzhin) o bufferevent: move conn_address out from http into bufferevent (8bb3842 Azat Khuzhin) o be: make @sa const for bufferevent_socket_connect() (dc33c78 Azat Khuzhin) Cleanups (core) o Refactoring conditional directives that break parts of statements. (4b41eeb lzmths) o epoll: introduce PRINT_CHANGES() macro to avoid copy-pasting (a1b142b Azat Khuzhin) o tab (6e7a580 Greg Hazel) Cleanups (evbuffers) o buffer_compat: fix comment -- we have EVBUFFER_EOL_ANY not EOL_STYLE_ANY (575ff67 Azat Khuzhin) Cleanups (bufferevents) o be_sock: evutil_getaddrinfo_async_() always return 0 (dbff101 Azat Khuzhin) o be_sock: drop be_sock_add() macro (useless and debug unfriendly) (fad5fe2 Azat Khuzhin) o be: introduce bufferevent_generic_adj_existing_timeouts_() (3c1f58f Azat Khuzhin) o be: add_event: use evutil_timerisset() (a96b73b Azat Khuzhin) o be_openssl: introduce be_openssl_auto_fd() helper (2a8a711 Azat Khuzhin) o be_openssl: introduce set_open_callbacks_auto() (510da71 Azat Khuzhin) Cleanups (http) o http: make fallback for EVHTTP_CON_READ_ON_WRITE_ERROR more cleaner (d405492 Azat Khuzhin) o http: coding style issue (365f181 Azat Khuzhin) Cleanups (evdns) o evnds: inline TEST_NAME macro to make debuggin easier (0c615f4 Azat Khuzhin) Portability Fixes o [#372] check for errno.h (3031617 Mark Ellzey) o Fixed Unicode issue in error messages. (e8b7895 Mattes D) o Assume that ke_udata is an integer type on CloudABI. (5602e45 Ed Schouten) o Add missing include of <netinet/in.h>. (b2c68bc Ed Schouten) o Include <sys/ioctl.h>, <sys/resource.h> and <sys/wait.h> optionally. (c1404b5 Ed Schouten) o Test against SO_REUSEADDR (along with _WIN32). (ce1776c Ed Schouten) o Always define missing TAILQ functions from sys/queue.h (2828bdb Christopher Wiley) o Don't use BSD u_* types. (fd36647 Ed Schouten) o Remove BSD-ism: TIMEVAL_TO_TIMESPEC(). (193c7de Ed Schouten) o be: include all variations of headers for sockaddr_in6 struct (c212291 Azat Khuzhin) o be: fix sockaddr_in6 type definition for win32 (c42bc6b Azat Khuzhin) Continious Integration: o travis: split long lines, and make it cleaner (685a6a1 Azat Khuzhin) o travis: fix autotools on osx by reinstalling libtool (088ea5e Azat Khuzhin) o appveyor/autotools: link with openssl by passing LDFLAGS/CFLAGS (6fcfa25 Azat Khuzhin) o appveyor: image already had openssl installed (4634b85 Azat Khuzhin) o appveyor: check -DUNICODE -D_UNICODE according to ReleaseChecklist (cmake only) (e9acc44 Azat Khuzhin) o appveyor: ignore failure of mingw-get (1810857 Azat Khuzhin) o appveyor: drop shallow_clone, since we use tags for detecting version in cmake (ac90133 Azat Khuzhin) o appveyor: support cmake & autotools using build matrix (like travis-ci has) (8f95015 Azat Khuzhin) o travis-ci/osx: relink gcc/g++ instead of clang (481481d Azat Khuzhin) o travis-ci: enable multi-os mode (osx, linux) (79917e4 Azat Khuzhin) o travis-ci: increase matrix (--disable-foo) (59649f7 Azat Khuzhin) o travis-ci: adjust alignment (c8be339 Azat Khuzhin) o travis: add builds without debug mode into matrix (3e56da2 Azat Khuzhin) o test: run regress with EVENT_DEBUG_MODE=1 and without (cf2cf2a Azat Khuzhin) o Update travis config for status updates (37453ab Mark Ellzey) o Use autotools for appveyor until cmake is fixed. (1cc2e29 Mark Ellzey) o Fix the link for appveyor OpenSSL installer (WIN32) (107d565 Mark Ellzey) o Forgot to install OpenSSL for appveyor (26164a5 Joakim Söderberg) o Add support for appveyor.com windows CI (5f89c37 Joakim Söderberg) Build Improvements/Fixes: o evutil: mark ai_find_protocol() static (prototype-less) (5a157c8 Azat Khuzhin) o cmake/solaris: set CMAKE_REQUIRED_LIBRARIES to fix functions detections (dc95823 Azat Khuzhin) o cmake/solaris: fix building (link with socket,nsl) (050bfc7 Azat Khuzhin) o cmake: check for ZLIB_INCLUDE_DIR, since we can have only library without headers (c4dfb93 Azat Khuzhin) o autotools/win32: fix searching ssl library (671a24f Azat Khuzhin) o cmake/win32: do not compile regress_thread on -DEVENT__DISABLE_THREAD_SUPPORT=ON (de0c196 Azat Khuzhin) o cmake/win32: do not compile evthread_win32 on -DEVENT__DISABLE_THREAD_SUPPORT=ON (ecb0ec8 Azat Khuzhin) o cmake: fix -DEVENT__ENABLE_VERBOSE_DEBUG (typo on -DUSE_DEBUG) (e35f224 Azat Khuzhin) o cmake: do not use stderr for notifications/version-info (38716c6 Azat Khuzhin) o autoconf: fix --disable-thread-support build under win32 (bb09535 Azat Khuzhin) o buffer: don't mix code and declarations (8892f4c Azat Khuzhin) o Update gitignore file to ignore cscope gen'ed files (0aaa4fb Neeraj Badlani) o For non GCC/clang on OSX the -Wno-deprecated-declarations may not be valid (b5ca365 Rainer Keller) o automake: define serial-tests only if automake have this option (61179de Azat Khuzhin) o test/automake: don't use paralell test harness (since automake 1.12) (44d755e Azat Khuzhin) o Ignore all pkgconfig generated stuff (ce38993 Azat Khuzhin) o libevent_core and libevent_extra also deserve a pkgconfig file (b8d7c62 Jan Heylen) o Ignore verify_tests.bat (win32 version) (0f2de10 Azat Khuzhin) o cmake: require 3.1 only for win32 to make it work under ubunty precise (87f7238 Azat Khuzhin) o cmake: require at least 3.1 for target_sources() (c46ead5 Azat Khuzhin) o cmake: fix adding of compiler flags, and now it will (36588e1 Azat Khuzhin) o Replace -Wswitch-enum with -Wswitch, and add it into cmake rules too (f29f59e Azat Khuzhin) o test/regress_ssl: Fix compile problems for win32 (73d0360 Trond Norbye) o util: fix "%zu" format on TDM-gcc/MinGW-w64 (79b69d8 Azat Khuzhin) o cmake: don't define EVENT__NEED_DLLIMPORT always (fixes VS2013 static build) (49bd790 Azat Khuzhin) o Add missing return statement to del_wait_thread so libevent can build. (4f778ab Nick Mathewson) o cmake: fix building dns-example under win32 (missing getopt) (a1609a8 Azat Khuzhin) o visibility: align it to make it more readable (bb6b53d Azat Khuzhin) o cmake: Fix detection of ssize_t/SSIZE_T (7707f6b Azat Khuzhin) o Ignore more configure stuff (configure.lineno) (8d34302 Azat Khuzhin) o Fixed issue with cmake version generation (d56efd9 Mark Ellzey) o Cmake is now officially working. (7f9646d Mark Ellzey) o More cmake updates, lot's of missing definitions (49a5381 Mark Ellzey) o CMake syntax fixes fo .in files (6aad23d Mark Ellzey) o Revert "The Windows socket type is defined as SOCKET." (a264da8 Mark Ellzey) o CMAKE CMAKE CMAKE CLEANUPS (a9db46a Mark Ellzey) o Lot's of cmake updates (8b228e2 Mark Ellzey) o Provide a mechanism for building the library on Windows with different compiler flags. Add a batch file that builds it for the M[DT][d] options and performs a hunt and gather of the different output libraries. (ded8086 billsegall) o The Windows socket type is defined as SOCKET. (c9e6c3d billsegall) o autotools: fix getservbyname() detection (959a4c2 Azat Khuzhin) o Add missing <string.h> for openssl_hostname_validation module (3316a21 Azat Khuzhin) o make test/regress_ssl.c compile without warnings (9f02a44 Thomas Bernard) o test/regress_be: drop debug __asm__(int3) to fix arm build (8240379 Azat Khuzhin) o event_debug_created_threadable_ctx_: fix compilation without debug mode (a068f2e Azat Khuzhin) o Add a prototype for event_disable_debug_mode() (bfcedee Sebastian Hahn) o http: eliminate warning about "socklen" in evhttp_connection_connect_() (dfad1a4 Azat Khuzhin) o Updated gitignore (1dbb55d Mark Ellzey) o Update bench_httpclient.c (cb96931 Seungmo Koo) o *fix: bench_httpclient to support win32 (4e9325e zeliard) o Commented out a WIN32 threading / timing test for now (e84e269 Mark Ellzey) o Fix mixed declarations and code (forbidden by ISO C90) (0c7f217 Thomas Bernard) o Fix "function declaration isn’t a prototype" (746d2c5 Thomas Bernard) o This fixes a bug introduced in 27bd9faf498b91923296cc91643e03ec4055c230 (19ba454 Joakim Söderberg) o changed strtotimeval signature as per #211 (bdbc823 Xiao Bao Clark) o Added cmake-generated files to ignore list. (6c12bfe Matyas Dolak) o Ignore `make dist` generated files (8a2c6c7 Azat Khuzhin) Debugging o Debug mode option to error on evthread init AFTER other event calls. (dcfb19a Mark Ellzey) Changes in version 2.1.5-beta (5 January 2015) Security Fixes (evbuffers) o Avoid integer overflow bugs in evbuffer_add() and related functions. See CVE-2014-6272 advisory for more information. (d49bc0e88b81a5812116074dc007f1db0ca1eecd) New APIs (evconnlistener) o Provide support for SO_REUSEPORT through LEV_OPT_REUSABLE_PORT (b625361 Maciej Soltysiak) Bugfixes (core) o Fix use-after-free error in EV_CLOSURE_EVENT callback (3cc0eac John Ohl) o Fix race caused by event_active (3c7d6fc vjpai) Bugfixes (evbuffer) o Fix evbuffer_peek() with len==-1 and start_at non-NULL. (ba59923) o Consistently check for failure from evbuffer_pullup() (60f8f72) o Fix evbuffer_peek() with len==-1 and start_at non-NULL. (fb7e76a) Bugfixes (windows, IOCP) o be async: avoid double close() (f133b86 Azat Khuzhin) Bugfixes (bufferevents) o Fix issue #127, double free for filterevents that use BEV_OPT_CLOSE_ON_FREE (2c82aa0 John Ohl) o make bufferevent_getwatermark api more robust (a21e510 ufo2243) o [Bugfix] fix bufferevent setwatermark suspend_read (b34e4ac ufo2243) o bufferevent_openssl: reset fd_is_set when setfd with -1 is called (3da84c2 Azat Khuzhin) o Fix compilation for older OpenSSL versions. (5c7282f Joakim Soderberg) New APIs (evhttp) o Add evhttp_connection_set_family() to set addrinfo->family for DNS requests (12c29b0 Azat Khuzhin) o Implement interface that provides the ability to have an outbound evhttp_connection free itself once all requests have completed (2b9ec4c,10fe4f John Ohl) New APIs (core) o Implement new/free for struct evutil_monotonic_timer and export monotonic time functions (f2645f8 Andrea Shepard) Bugfixes (evdns) o Load hosts file on Windows. (a0b247c Vilmos Nebehaj) o Don't truncate hosts file path on Windows. (d0dc861 Vilmos Nebehaj) o Fix a crash in evdns related to shutting down evdns (9f39c88,e8fe749) o evdns: avoid read-after-free in evdns_request_timeout_callback() (61262a0 Azat Khuzhin) o Correctly handle allocation failures in evdns_getaddrinfo (6a53d15) o evdns: fix EVDNS_BASE_DISABLE_WHEN_INACTIVE in case retransmit/retry (74d0eee Azat Khuzhin) o evdns: add retry/reissue tests for EVDNS_BASE_DISABLE_WHEN_INACTIVE (3ca9d43 Azat Khuzhin) o evdns: fail ns after we are failing/retrasmitting request (97c750d Azat Khuzhin) Bugfixes (evhttp) o http: reset connection before installing retry timer (fix http retries handling) (bc79cc5 Azat Khuzhin) Testing o regress_dns: fix leaks in getaddrinfo_async{,_cancel_stress} tests (2fdc5f2 Azat Khuzhin) o test: add family argument for http_connection_test_() (177b8a7 Azat Khuzhin) o test: add regress for evhttp_connection_set_family() with AF_INET and AF_UNSPEC (42aefeb Azat Khuzhin) o test/http: add regress test for set family to AF_INET6 (3fbf3cc Azat Khuzhin) o Update to a more recent tinytest_macros. (8da5a18) o test/regress: add simplestsignal: to track reorder bugs separately (b897bef Azat Khuzhin) o test/evbuffer_peek: add regress in case we have first buffer greater (e2d139d Azat Khuzhin) o More evbuffer_peek() test cases (154006a) o use correct tt macro for pointer compare (08c88ea) o regress_buffer: fix 'memcmp' compare size (79800df Maks Naumov) o Fix a use-after-free in unit tests. CID 752027 (3739057) o Fix a dead-code warning in unit tests. CID 1193548 (c119f24) o Use evutil_weakrand() in unit tests. (a677b72, 364c110) o Use a more precise calculation for max in time-ratelim.c (ca5b5c7) o Make a buffer larger in the tests to avoid a scary evbuffer_copyout_from() (fb57b8b) o Fix several memory leaks in the unit tests. (89c1a3b) o Add test for evhttp_connection_free_on_completion (b0e9924 John Ohl) o Fix annoying heisenbug in test-time.c (cb73704) Sample code o Make http-server.c output into good html5 (6d72bdc) o Use FindClose for handle from FindFirstFile in http-server.c (6466e88) o https-client: add -retries argument, for connection retries (d9da844 Azat Khuzhin) Bugfixes (build) o Add missing headerfile for cmake (15d90cc Trond Norbye) o ignore one more test binary (b6593aa Michael Richardson) o ignore config.cache/test-driver files (c83f333 Mike Frysinger) o add a --disable-samples configure flag (0c492b3 Mike Frysinger) o Add a few files created by "make verify" to .gitignore. (1a8295a Pierre Phaneuf) o updates in cmake build (27bd9fa Sergey Nikulov) o Fix cmake error when the Module path has more than one entry. (befbd13 Acer Yang) o Fix CMake shared library build (e69d910 Nobuaki Sukegawa) o Fix warnings when compiling with clang 3.5 (f5b4765 John Ohl) o Fix mixed declarations and code (forbidden by ISO C90) (8afbdbc Thomas Bernard) Bugfixes (miscellaneous) o tree.h: drop duplicated content of tree.h (6193187 Azat Khuzhin) o evdns: disable probing with EVDNS_BASE_DISABLE_WHEN_INACTIVE (610410b,ad0493e,fea86a6,d83b337,5ca9e97 Azat Khuzhin) o [Bugfix] fix grammer error (3a4d249 ufo2243) o Change return type of evutil_load_windows_system_library_ to HMODULE (f691389) o Fix a c90 warning (76643dd) o Fix a typo in a doxygen comment. Reported by 亦得. (be1aeff) o remove trailing comma from enum (b361b8a Jean-Philippe Ouellet) Bugfixes (FreeBSD) o Handle ENOTCAPABLE from FreeBSD - this is returned if an event in the changelist is for an FD that has been closed. (6fd7394 Adrian Chadd) Changes in version 2.1.4-alpha (21 Mar 2014) Libevent 2.1.4-alpha adds a number of new miscellaneous APIs to make Libevent more useful, including support for early close detection with epoll via EPOLLRDHUP, triggering bufferevent callbacks, adding more evhttp callbacks, and more. There are also numerous bugfixes, including a number for finalize-related issues from 2.1.3-alpha; and an alternative (non-primary!) cmake-based build mechanism. New APIs (core) o Added event_base_get_num_events() (0fa107d Mobai Zhang) o New event_base_active_by_fd API (865a142 Greg Hazel, 5c9da9a, 87fa2b0) o Add event_base_active_by_signal by analogy (4865943) o Add access to max event count stats (5173bef, efbd3dc, 26230a2 Andrew Sweeney) o Implemented EV_CLOSED event for epoll backend (EPOLLRDHUP). (b1b69ac Diego Giagio, 53d2793, 43ffcf6, dfe1e52 Marcin Juszkiewicz, ff26633 Joakim Soderberg, 3908a5e) New APIs (evutil_secure_rng) o Add evutil_secure_rng_set_urandom_device_file (2bbb5d7) New APIs (bufferevents) o Add function to fetch underlying ratelimit cfg (4b3d5af Mark Ellzey) o Pass and return const for bufferevent_get_token_bucket_cfg (1c77fbb Mark Ellzey) o Add watermark introspection (4ce242b Ondřej Kuzník) o Add an option to trigger bufferevent I/O callbacks (61ee18b Ondřej Kuzník) o Add an option to trigger bufferevent event callbacks (a7384c7 Ondřej Kuzník) o Clarifications in response to merge req. comments (bd41947 Ondřej Kuzník) o Minor optimizations on bufferevent_trigger options (a3172a4) New APIs (evhttp) o Add evhttp_connection_get_server(). (a7f82a3 Maxime Henrion) o add a http default content type option (5a5acd9 Nicolas Martyanoff) o http: implement new evhttp_connection_get_addr() api. (0c7f040 Azat Khuzhin) o Add a variant of evhttp_send_reply_chunk() with a callback on evhttp_write_buffer() (8d8decf Julien BLACHE) o Allow registering callback for parsing HTTP headers (b0bd7fe Balint Reczey) o Provide on request complete callback facility (b083ca0 Andrew Sweeney) o evhttp_request_set_on_complete_cb to be more specific about what the function actually does and usage (da86dda Andrew Sweeney) o Update unit test to make sure that the callback happens after the output data is written (b85f398 Andrew Sweeney) Features (evdns) o bug fix for issues #293 evdns_base_load_hosts doesn't remove outdated addresses (954d2f9, f03d353, 45eba6f Kuldeep Gupta) Features: (cmake build support) o Initial CMake commit. (e415196 Joakim Soderberg) o Add all tests and benchmarks to CMake project. (e9fc014 Joakim Soderberg) o More work on adding tests to CMake project (99c1dc3 Joakim Soderberg) o Generate a dummy evconfig-private.h so things build properly. (ce14def Joakim Soderberg) o Link libm on unix platforms. (58fcd42 Joakim Soderberg) o Added some GCC specific options. (19222e5 Joakim Soderberg) o Use evutil_closesocket instead. (dbf2b51 Joakim Soderberg) o Add copyright and licensing files for CMake modules. (c259d53 Joakim Soderberg) o Only include WIN32 getopt where it is used. (9bbce0b Joakim Soderberg) o Fix bench_cascade program on Windows. (78da644 Joakim Soderberg) o Don't segfault on no found event backend. (8f2af50 Joakim Soderberg) o Only test the event backends available on the system. (7ea4159 Joakim Soderberg) o Added a "make verify" target. (e053c4f Joakim Soderberg) o Fix the make "verify" target on Windows. (67e5d74 Joakim Soderberg) o Get rid of deprecation warnings for OpenSSL on OSX 10.7+ (69c3516 Joakim Söderberg) o Fix kqueue support. (a831f2f Joakim Söderberg) o Added a test for testing if kqueue works with pipes. (2799b35 Joakim Söderberg) o Change the BSD license from 4 to 3-clause. (86df3ed Joakim Soderberg) o Minimum required python version is 2.4. (968e97b Joakim Soderberg) o Get rid of unknown pragma warnings. (0ef1d04 Joakim Soderberg) o Add a "make verify_coverage" target generation coverage info. (f2483f8 Joakim Soderberg) o Fix the "make verify" target on NetBSD (4ac086a Joakim Soderberg) o Only look for ZLib when it is used (if tests are included). (f780593 Joakim Soderberg) o Added EVENT__ENABLE_GCC_WARNINGS, turns all warnings into errors. (dd413bd Joakim Soderberg) o Add CMake config and install targets. (f3446ed Joakim Soderberg) o Fix typo (4b754df Joakim Soderberg) o Some work on making it possible to simply do add_subdirectory() on the project. (49ab363 Joakim Soderberg) o Set USE_DEBUG=1 on EVENT__ENABLE_VERBOSE_DEBUG (fd42e70 Joakim Soderberg) o Fix so that old nmake project still builds. (24d6466 Joakim Soderberg) o Rename README to README.md and use markdown to format. (d2bc39a Joakim Soderberg) o Update README with CMake build instructions. (604b8cc Joakim Soderberg) o Clean up the README some. (8d4cb35 JoakimSoderberg) o Forgotten headers for old nmake project compatability. (8697b99 Joakim Soderberg) o Change all uses of WIN32 to _WIN32 (4e14395 Joakim Söderberg) o Fix include bug. (2024467 Joakim Söderberg) o Check if we're on OSX before disabling deprecation in le-proxy (8b40a5b Joakim Söderberg) o Fix broken autotools build. (ae1bd82 Joakim Söderberg) o Disclaimerize cmake a little in the README (d03b5bf) o Fix CMake compile when OpenSSL is disabled. (e423d42 Joakim Söderberg) o CMake: Get rid of python not found warning when regress tests turned off. (d38d798 Joakim Söderberg) o Fix https-client compilation on Windows. (d7be788 Joakim Soderberg) o Guard against EVENT_NOWIN32 being set during testing. (f1715b4 Joakim Soderberg) o Check for OSX when checking for clang. (e212c54 Joakim Soderberg) o Added a Travis-CI configuration file. (8c0f0a9 Joakim Soderberg) o Added -Qunused-arguments for clang on macosx (ed99d92 Trond Norbye) o Rename event_extras to event_extra (a0dd5df Trond Norbye) o Add option to build shared library (4545fa9 Trond Norbye) o Add -Qunused-arguments for clang on macos (b56611d Trond Norbye) o Add cmake-related files to .gitignore (e061321 Trond Norbye) o Export event_extra not event_extras. (2b41bcf Joakim Söderberg) Bugfixes (core) o If evsel->del() fails, don't leave the evmap in an inconsistent state (9b5a527 Maxime Henrion) o Move event_debug_note_teardown_ before mm_free. (69b5c64) o Check CLOCK_MONOTONIC_* at runtime if needed. (911abf3) o Fix reinit of fds with EV_WRITE but not EV_READ. (ebfd8a8 maksqwe) o Tweaked callbacks to prevent race condition (https://github.com/libevent/libevent/issues/104) (40830f1, 2ea15ed John Ohl) o Move assert(ev) to before we use ev in EV_CLOSURE_EVENT_FINALIZE case (9805972) Bugfixes (evhttp) o Fix a double close() bug in evhttp when the underlying bufferevent uses BEV_OPT_CLOSE_ON_FREE. (31db8a0 Maxime Henrion) o Fix an unlikely but possible error case for http connections (f22049e) o Avoid racy bufferevent activation (5eb1788 Nate Rosenblum) Bugfixes on 2.0 (Windows) o Use windows vsnprintf fixup logic on all windows environments (e826f19) o libevent/win32_dealloc() : fix sizeof(pointer) vs sizeof(*pointer) (b8f5980 Frank Denis) Bugfixes (evutil_secure_rng) o When we seed from /proc/sys/kernel/random/uuid, count it as success (e35b540) o We should return after arc4random_buf() (1ea1f26 Makoto Kato) o Avoid other RNG initialization FS reads when urandom file is specified (9695e9c) o Really remove RNG seeds from the stack (f5ced88) o Fix another arc4random_buf-related warning (e64a2b0) Bugfixes (bufferevents) o Initialize async bufferevent timeout CBs unconditionally (af9b2a7) Bugfixes (evdns) o Checking request nameserver for NULL, before using it. (5c710c0 Belobrov Andrey) o Fix SEGFAULT after evdns_base_resume if no nameservers installed. (14971a8 Azat Khuzhin) o Actually use the log facility for reporting evdns problems. (e1766a1) o Fix SEGFAULT after evdns_base_resume if no nameservers installed. (f8d7df8 Azat Khuzhin) o fix for ServFail from RIPE Atlas release (62f596b Antony Antony) Bugfixes (compilation) o Fix test compilation with nmake: add the gdi.lib dependency (5ba8ab7) o Whoops. It is gdi.lib, not gdi32.lib. (github issue #61) (8ab612e) o Don't use return since return type is void and build error occurs using clang (838161d Makoto Kato) o Use void casts to suppress some "unchecked return value" warns (7080d55) o rpcgen: Generate regress.gen.[c,h] in build rather than src dir (243386c Ross Lagerwall) o Fix a compiler warning when checking for arc4random_buf linker breakage. (5cb3865) o Fix 'make distcheck' by adding regress.gen.[ch] to DISTCLEANFILES (239d834) o Fix a c90 warning (c207682) o Fix consts in WIN32-Code/getopt*.[ch] (57abb35) Bugfixes (locks, synchronization) o Missed lock acquire/release in event_base_cancel_single_callback_() (d3d999a Azat Khuzhin) o Fix locking in bufferevent_get_options_(). (dbc9cd4 Maxime Henrion) Bugfixes (leaks) o Avoid leaking segment mappings when offset is not a page multiple (d409514) Testing o Add tests for evdns_base_resume(). (1cd9ff5 Azat Khuzhin) o Fix dns/leak_resume_send_err test. (7e876df Azat Khuzhin) o Add checks for evhttp_connection_get_server() in unit tests. (fbc323b Maxime Henrion) o Fix a (failure-only) null dereference in the unit tests (1104d0b) o Fix a logic error in test_evbuffer_freeze (7765884) o Add missing check to test_evbuffer_file_segment_add_cleanup_cb (eba4506) o Fix some crash-on-fail cases in DNS regression tests (87cd6f0) o DNS tests: add a missing check (f314900) o Finalize tests: add a missing check (82b6956) o test_evutil_rtrim: add another missing check. (e193c95) o regress_main: logging all if env EVENT_DEBUG_LOGGING_ALL isset (611e28b Azat Khuzhin) o regress_http: add tests for evhttp_connection_get_addr() (4dd500c Azat Khuzhin) o Update to the latest version of tinytest (7a80476) o Heap-allocate zlib data structure in regress_zlib tests (4947c18) Performance tweaks (core) o Avoid redundant syscall to make a nonblocking socket nonblocking (42c03da Maxime Henrion) o Avoid redundant syscall if making a socket cloexec twice (1f29b18) o Avoid redundant invocations of init_extension_functions for IOCP (3b77d62) Documentation o Document that arc4random is not a great cryptographic PRNG. (6e49696) o Small doxygen tweaks (6e67b51) o Try another doxygen tweak (ccf432b) o Clarify event_base_loop exit conditions (031a803) o Fix a typo (be7bf2c Ondřej Kuzník) o Document deferred eventcb behaviour (13a9a02 Ondřej Kuzník) o Typo fixes from Linus Nordberg (cec62cb, 8cd695b) o Fix duplicate paragraph in evbuffer_ptr documentation (58408ee) Code Improvements (coverity) o Fix a pile of coverity warnings in the unit tests (867f401) o Fix coverity warnings in benchmark tools. (ff7f739) o Whoops; fix compilation in bench.c (544cf88) o Remove spurious checks in evrpc.c error cases (coverity) (991b362) o Fix a couple of compilation warnings in regress_http.c (860767e) o Fix even more coverity warnings. (d240328) o Stop checking for inet_aton; we don't use it. (f665d5c) o Add an include to evrpc-internal to fix openbsd compilation warning (5e161c6) Cleanups o Remove an unreachable return statement in minheap-internal.h (e639a9e) o Refactor evmap_{io,signal}_active_() to tolerate bad inputs (974c60e) o Fix needless bufferevent includes in evdns.c (254c04e) o Fix a couple of "#ifdef WIN32" instances (88ecda3) o Remove unneeded declaration in bufferevent-internal.h (4c8ebcd) Sample code o le-proxy: Fail more gracefully if opening listener fails (44b2491) o http-server: drop uri_root from base_url in http-server. (6171e1c Azat Khuzhin) o https-client: POST supported, args supported (c5887f7 Alexey Ozeritsky) o https-client: code cleanup (29af65e Alexey Ozeritsky) o https-client: Small tweaks to https-client.c (90786eb) o https-client: Set hostname for SNI extension (by f69m) (d1976f8) o https-client: add a cast to https-client.c (462e6b6) Changes in version 2.1.3-alpha (1 May 2013) Libevent 2.1.3-alpha fixes various bugs, adds new unit tests, and cleans up the code in a couple of places. It has a new callback in evhttp for reporting errors during a request, a new feature for allowing evdns to not keep the event_base looping when there are no requests inflight, and example code for writing an https client. Libevent 2.1.3-alpha also has an important new (experimental) event finalization feature to allow safe event teardown in multithreaded programs. This ought to fix the longstanding bug with deadlocks in multithreaded use of SSL-based bufferevents that some people have been experiencing since Libevent 2.0. Core (event finalization) o Implement event_finalize() and related functions to avoid certain deadlocks (8eedeab) o Use finalization feature so bufferevents can avoid deadlocks (02fbf68) o Always run pending finalizers when event_base_free() is called (e9ebef8) o Remove bufferevent_del_generic_timeout_cbs as now unused (4ea4c6a) o More documentation for finalization feature (a800b91) o Make the event_finalize* functions return an error code (5d11f4f) o Mark the finalize stuff as experiemental in case it needs to change (23e2e29) Evdns o evdns: New flag to make evdns not prevent the event loop from exiting (6b7fa62 Azat Khuzhin) Bugfixes (Core) o Make event_remove_timer behave correctly with persistent timers (5623e80) o Unit test for event_remove_timer with EV_PERSIST. (96150dd) o Double-check next timeout when adding events (9443868 Nate Rosenblum) o event_base_update_cache_time should be a no-op if the loop isn't running (5e6fa2a) Bugfixes (evhttp, crash fix, from 2.0) o fix #73 and fix http_connection_fail_test to catch it (b618204 Greg Hazel) Bugfixes (compilation and portability, from 2.0) o Fix compilation with WIN32_HAVE_CONDITION_VARIABLES enabled (7e45739) o Fix missing AC_PROG_SED on older Autoconfs (9ab2b3f Tay Ray Chuan) o Backport libevent to vanilla Autoconf 2.59 (as used in RHEL5) (74d4c44 Kevin Bowling) o Use AC_CONFIG_HEADERS in place of AM_CONFIG_HEADERS for autmake 1.13 compat (817ea36) o Rename configure.in to configure.ac to appease newer autoconfs (0c79787) o Avoid using top_srcdir in TESTS: new automakes do not like this (a55514e) Bugfixes (resource leaks/lock errors on error, from 2.0) o Avoid leaking fds on evconnlistener with no callback set (69db261) o Avoid double-close on getsockname error in evutil_ersatz_socketpair (0a822a6) o Fix a locking error in bufferevent_socket_get_dns_error. (0a5eb2e) Documentation Fixes (from 2.0) o Fix a mistake in evbuffer_remove() arguments in example http server code (c322c20 Gyepi Sam) o Fix a typo in a comment in buffer.h. Spotted by Alt_F4 (773b0a5) Documentation Fixes o minor documentation typos (809586a Patrick Pelletier) o Fix cut-and-paste err in whatsnew-2.1 (49905ac) o Fix comment to refer to sample/include.am correctly (9e8cdf3 Sebastian Hahn) o Fix typo : Dispatching instead of Dispaching (0c2bacc Volker Lendecke) o fix some hinky indentation in evhttp_make_request (80e220e Patrick Pelletier) o "buffer" spelling (a452811 Patrick Pelletier) o Specify return behavior in header for evbuffer_pullup() in corner case (cf8d1cd Dan Petro) o Clarify an important point about event_base_foreach_event() (920a5e6) Compilation Fixes/Tool Support o avoid valgrind false positive by zeroing epoll_event (1258614 Patrick Pelletier) o Fix harmless clang enum warning (b452a43 Sebastian Hahn) o remove all exes on "make clean", not just regress.exe (974bfa0 Patrick Pelletier) o Make --disable-libevent-regress work again (787fd74) o Do not build strlcpy.c when it will have no code. (4914620) Portability Fixes o When EWOULDBLOCK is not EAGAIN, treat it as equivalent to it (bf7a0ff) o Preliminary changes for Minix3. (0dda56a Nicholas Heath) o Use AC_CONFIG_HEADERS in place of AM_CONFIG_HEADERS for autmake 1.13 compat (bf278b) o Avoid using $(top_srcdir) in TESTS. (2863c83) o build test/test-script.sh on systems with a less-featureful $< (f935e21) o Implement EVUTIL_ERR_IS_EAGAIN on windows. (42aaf4d) Evhttp changes: o Fix ipv6 support for http. When URL contain domain, not IP address. (71e709c Azat Khuzhin) o uri decode: fix for warning "use of uninitialised value" (64b6ece Azat Khuzhin) o uri decode: changed the test for the existence of the next character (e1903e3 Azat Khuzhin) o Move prototype of evhttp_decode_uri_internal() to http-internal.h (de8101a Azat Khuzhin) o Test: decoding just part of string with evhttp_decode_uri_internal() (1367653 Azat Khuzhin) o Add new error_cb for actual reporting of HTTP request errors. (7b07719 Azat Khuzhin) o Add test for EVREQ_HTTP_REQUEST_CANCEL into http_cancel_test() (862c217 Azat Khuzhin) o Drop extra header http_struct.h from regress_http.c (54cc800 Azat Khuzhin) Testing o Add regress test ipv6_for_domain. (9ec88bd Azat Khuzhin) o Add an environment variable (EVENT_DEBUG_MODE) to run unit tests in debug mode (2fad0f3) o Add a test with an active_later event at event_base_free time. (1c3147f) o Make all tests pass under EVENT_DEBUG_MODE=1 (b1b054f) o Add some verbose notes to bufferevent unit tests (9d893c9) o New test for active_later->active transition on event_active (a153874) o New tests for event_base_foreach_event() (0b096ef) o Unit tests for event_base_gettimeofday_cached() and event_base_update_cache_time() (30ea291) o A test for event_get_assignment() (f09629e) o More unit tests for initializing common timeouts. (d596739) o Fix a bug in the new main/event_foreach test (702c9aa) Windows: o use FormatMessage for winsock errors (0c6ec5d, 2078e9b, 4ccdd53, c9ad3af Patrick Pelletier) o a program to print out the error strings for winsock errors (7296512 Patrick Pelletier) o Fix a warning introduced in 0c6ec5d8 (eeb700c) o Fix another warning introduced in 0c6ec5d8 (ed26561) Examples (http) o Add sample/https-client.c, an example of stacking evhttp as a client on top of bufferevent_ssl. (be46c99 Catalin Patulea) o use ${OPENSSL_LIBS} instead of -lssl -lcrypto (bf31fa5 Patrick Pelletier) o https-client was putting newlines at 256-byte boundaries (42d7441 Patrick Pelletier) o better handling of OpenSSL errors (5754d96 Patrick Pelletier) o use Debian's default root certificate location (aacd674 Patrick Pelletier) o use iSECPartners code to validate hostname in certificate (64d9f16 Patrick Pelletier) o avoid sign mismatch warning in openssl_hostname_validation.c (6021cb5 Patrick Pelletier) o pull in wildcard matching code from cURL (4db9da6 Patrick Pelletier) o Another tweak to https-client.c (95acdaa) o Remove http_struct.h usage in sample/https-client.c (8a90a85) Changes in version 2.1.2-alpha (18 Nov 2012) Libevent 2.1.2-alpha includes more portable for monotonic timers, refactors much of Libevent's internal and external infrastructure, closes some longstanding gaps in the interface, makde other improvements. Ths log below tries to organize features by rough area of effect. It omits a few commits which were pure bugfixes on other commits listed below. For more detail, see the git changelogs. For more insight, see the "whatsnew-2.1.txt" document included in the Libevent 2.1.2-alpha distribution. Libevent 2.1.2-alpha also includes all changes made in 2.0.19-stable through 2.0.21-stable inclusive. Performance (core): o Replace pipe-based notification with EVFILT_USER where possible. This should make multithreaded programs on OSX and *BSD alert the main thread a little faster. (53a07fe) o Make th_base_lock nonrecursive. (9cd5acb) New/Changed API Functions: o New event_get_priority() function to return an event's priority (f90e255) o Add a bufferevent_get_priority() function (bd39554) o Add an event_base_loopcontinue() to tell Libevent to rescan for more events right away (7d6aa5e) o Add a new callback to get called on evbuffer_file_segment free (e9f8feb yangacer, 64051b9) o Expose event_base_foreach_event() as a public API. (84fd6d7 Roman Puls, 232055e, ffe1643) o Add an event_remove_timer() to remove timer on an event without deleting it (e3b2e08) o Make bufferevent_set_timeouts(bev, NULL, NULL) have plausible semantics (9dee36b) o Rename event_enable_lock_debuging() to ..._debugging(). (The old name should still work.) (07e132e) o Add missing implementation for event_enable_debug_logging (3b3e21d) PORTABLE MONOTONIC TIMERS: Libevent 2.1.2 includes internal support for monotonic timers on (nearly) all supported platforms, including Windows, and OSX. Libevent applications should now be more resilient to jumps forwards or backwards in the system clock. Also, on Linux systems with epoll, we now optionally support microsecond-level timeouts (whereas epoll only supports millisecond-precision timeouts). o Use mach_absolute_time() for monotonic clock support on OSX. (b8fd6f9) o Do not track use_monotonic field when is no monotonic clock (cb653a0) o EVENT_BASE_FLAG_PRECISE_TIMER indicates we want fine timer precision (ddd69d3) o On Linux, use CLOCK_MONOTONIC_COARSE by default (55780a7) o Implement a GetTickCount-based monotonic timer for Windows (d5e1d5a) o Refactor monotonic timer handling into a new type and set of functions; add a gettimeofday-based ratcheting implementation (f5e4eb0) o Add EVENT_PRECISE_TIMER environment var for selecting precise-but-slow timer (a2598ec) o Implement fast/precise monotonic clocks on Windows (2c47045) o Simple unit tests for monotonic timers (630f077) o Improve the monotonic-time unit test: make it check the step size (7428c78) o When PRECISE_TIMERS is set with epoll, use timerfd for microsecond precision (26c7582) o Split out time-related evutil functions into a new evutil_time.c (c419485) o Split out time-related prototypes into time-internal.h (71bca50) o Add evutil_time.obj to Makefile.nmake (0ba0683) o Avoid giving a spurious warning when timerfd support is unavailable (1aaf9f0 Dave Hart) o Make test_evutil_monotonic a little more tolerant (def3b83) o Avoid unused-var warning on systems with clock_gettime but without CLOCK_MONOTONIC_COARSE (9be5468) EVENT_BASE_ONCE LEAKS: If a callback added by event_base_once() is never invoked, Libevent no longer leaks internal memory. o Free dangling event_once objects on event_base_free() (c17dd59) o Add a unit test in which an event is created with event_base_once() but never fires (4343edf) TESTING SUPPORT, FIXES AND IMPROVEMENTS: Libevent now disables by default its unit tests that would touch the network, or that tend to fail on heavily-loaded systems. To re-enable them, invoke the ./test/regress program with the @all alias. o Simplify test.sh code significantly. (9b856fd Ross Lagerwall) o Make all tests that hit the network disabled by default (f2cea87) o Avoid a resource leak on error in http client benchmark (ea92fba) o Update to latest tinytest (911b4f0349377) (ef7c4f7) o Avoid (unlikely) overflow in bench_httpclient.c (5671033) o Shave 700 msec off the persistent_timeout_jump test (21205b8) o Check return value of write() in regress.c (c8009d2) o Make load-dependent monotonic timer tests off-by-default (2b6fe8b) o Add deferred_cb_skew to list of timing-dependent tests (34c8f31) o Avoid test -e; older shs don't have one. (f1bd938) o Fix renegotiation test to work around openssl 1.0.1 bug (c2f3086) o Fix a couple of compile warnings in the unit tests (5a9a014) MISC: o Change evutil_weakrand_() to avoid platform random() (e86af4b Nicholas Marriott, 3aa4415) INFRASTRUCTURE (Active-later events): As a simplification and optimization to Libevent's "deferred callback" logic (introduced in 2.0 to avoid callback recursion), Libevent now treats all of its deferrable callback types using the same logic it uses for active events. Now deferred events no longer cause priority inversion, no longer require special code to cancel them, and so on. o Refactor the callback part of an event into its own event_callback type (cba59e5) o Add "active later" event_callbacks to supersede deferred (745a63d) o event_base_assert_ok: check value of event_active_count for correctness (fec8bae) o Replace deferred_cbs with event_callback-based implementation. (ae2b84b) o Replace more deferred_cb names with event_callback (a4079aa) o Give event_base_process_active a single exit path (581b5be) o Restore our priority-inversion-prevention code with deferreds (c0e425a) o Refactor event_persist_closure: raise and extract some common logic (bec22b4) o Remove the unused bits from EVLIST_ALL (9889a3d) ||||||| merged common ancestors Changes in version 2.0.22-stable (?? Dec 2013) (As of 3b77d62829c4393bda6f9105a5d3b73b48a64b71.) BUGFIXES (evhttp) o fix #73 and fix http_connection_fail_test to catch it (crash fix) (b618204 Greg Hazel) o Avoid racy bufferevent activation (5eb1788 Nate Rosenblum) BUGFIXES (compilation and portability) o Fix compilation with WIN32_HAVE_CONDITION_VARIABLES enabled (7e45739) o Fix missing AC_PROG_SED on older Autoconfs (9ab2b3f Tay Ray Chuan) o Backport libevent to vanilla Autoconf 2.59 (as used in RHEL5) (74d4c44 Kevin Bowling) o Use AC_CONFIG_HEADERS in place of AM_CONFIG_HEADERS for autmake 1.13 compat (817ea36) o Rename configure.in to configure.ac to appease newer autoconfs (0c79787) o Avoid using top_srcdir in TESTS: new automakes do not like this (a55514e) o Use windows vsnprintf fixup logic on all windows environments (e826f19) o Fix a compiler warning when checking for arc4random_buf linker breakage. (5cb3865) o Fix another arc4random_buf-related warning (e64a2b0) BUGFIXES (resource leaks/lock errors on error) o Avoid leaking fds on evconnlistener with no callback set (69db261) o Avoid double-close on getsockname error in evutil_ersatz_socketpair (0a822a6) o Fix a locking error in bufferevent_socket_get_dns_error. (0a5eb2e) o libevent/win32_dealloc() : fix sizeof(pointer) vs sizeof(*pointer) (b8f5980 Frank Denis) BUGFIXES (miscellaneous) o Avoid other RNG initialization FS reads when urandom file is specified (9695e9c, bb52471) o Avoid redundant invocations of init_extension_functions for IOCP (3b77d62) BUFGIXES (evdns) o Checking request nameserver for NULL, before using it. (5c710c0 Belobrov Andrey) o Fix SEGFAULT after evdns_base_resume if no nameservers installed. (f8d7df8 Azat Khuzhin) BUGFIXES (evutil_secure_random) o When we seed from /proc/sys/kernel/random/uuid, count it as success (e35b540) o Document that arc4random is not a great cryptographic PRNG. (6e49696) o Add evutil_secure_rng_set_urandom_device_file (2bbb5d7) o Really remove RNG seeds from the stack (f5ced88) DOCUMENTATION FIXES o Fix a mistake in evbuffer_remove() arguments in example http server code (c322c20 Gyepi Sam) o Fix a typo in a comment in buffer.h. Spotted by Alt_F4 (773b0a5) Changes in version 2.0.21-stable (18 Nov 2012) BUGFIXES: o ssl: Don't discard SSL read event when timeout and read come close together (576b29f) o ssl: Stop looping in "consider_reading" if reading is suspended. (f719b8a Joachim Bauch) o ssl: No need to reserve space if reading is suspended. (1acf2eb Joachim Bauch) o dns: Avoid a memory-leak on OOM in evdns. (73e85dd, f2bff75 George Danchev) o build: Use python2 rather than python (0eb0109 Ross Lagerwall) o build: Compile without warnings on mingw64 (94866c2) o build: Fix compilation on mingw64 with -DUSE_DEBUG (62bd2c4) o build: Make rpcgen_wrapper.sh work on systems without a "python2" binary (f3009e4) o iocp: Close IOCP listener socket on free when LEV_OPT_CLOSE_ON_FREE is set (cb853ea Juan Pablo Fernandez) o core: Avoid crash when event_pending() called with no event_base set on event (e3cccf3) o misc: remove stray 'x' so print_err will compile when uncommented (ac35650 Patrick Pelletier) o tests: Fix renegotiation test to work around openssl 1.0.1 bug (c2f3086) o tests: Warn when openssl version in unit test mismatches compiled version. (ac009f9) Changes in version 2.0.20-stable (23 Aug 2012) BUGFIXES: o core: Make event_pending() threadsafe. (be7a95c Simon Liu) o win32: avoid crash when waiting forever on zero fds. (160e58b) o evhttp: Fix a memory leak on error in evhttp_uriencode (11c8b31) o evbuffer: Avoid possible needless call to writev. Found by coverity. (6a4ec5c) o evdns: memset sockaddr_in before using it. Found by coverity. (a1a0e67) o evhttp: Check more setsockopt return values when binding sockets. Found by coverity (a0912e3) o evdns: Avoid segfault on weird timeout during name lookup. (dc32077 Greg Hazel) o bufferevent_ssl: Correctly invoke callbacks when a SSL bufferevent reads some and then blocks. (606ac43) PORTABILITY FIXES: o check for arc4random_buf at runtime, on OS X (bff5f94 Greg Hazel) o Correctly check for arc4random_buf (fcec3e8 Sebastian Hahn) o Add explicit AC_PROG_SED to configure.in so all autoconfs will expose $(SED) (ca80ea6) BUILD FIXES: o Add GCC annotations so that the vsprintf functions get checked properly (117e327) o Fix an unused variable warning on *BSD. (c0720c1) UNIT TEST FIXES: o Fix a couple of memory leaks (found with Valgrind). (3b2529a Ross Lagerwall) o Remove deadcode in http regression tests. Found by coverity. (5553346) o Fix possible uninitialized read in dns regression tests. Found by coverity. (2259777) o Set umask before calling mkstemp in unit tests. Found by coverity (f1ce15d) o Fix various check-after-dereference issues in unit tests: found by coverity (4f3732d) o Fix resource leaks in the unit tests; found by coverity (270f279) o Add some missing null checks to unit tests; found by coverity (f021c3d) o Avoid more crashes/bad calls in unit tests; found by coverity (3cde5bf) o Remove unused variable; spotted by coverity (6355b2a) o Add checks to various return values in unit tests. Found by coverity (b9e7329) o Move assignment outside tt_assert in ssl unit tests. Appeases coverity. (a2006c0) Changes in version 2.0.19-stable (3 May 2012) BUGFIXES (CORE): o Refactor event_persist_closure: raise and extract some common logic (bec22b4) o If time has jumped so we'd reschedule a periodic event in the past, schedule it for the future instead (dfd808c) o If a higher-priority event becomes active, don't continue running events of the current priority. (2bfda40) BUGFIXES (SSL): o Fixed potential double-readcb execution with openssl bufferevents. (4e62cd1 Mark Ellzey) BUGFIXES (DNS): o Cancel a probe request when the server is freed, and ignore cancelled probe callbacks (94d2336 Greg Hazel) o Remove redundant DNS_ERR_CANCEL check, move comment (46b8060 Greg Hazel) o When retransmitting a timed-out DNS request, pick a fresh nameserver. (3d9e52a) DOCUMENTATION FIXES: o Fix a typo in the bufferevent documentation (98e9119) o Add missing ) to changelog; spotted by rransom (4c7ee6b) o Fix the website URL in the readme (f775521) COMPILATION FIXES: o Fix a compilation error with MSVC 2005 due to use of mode_t (336dcae) o Configure with gcc older than 2.95 (4a6fd43 Sebastian Hahn) o Generate event-config.h with a single sed script (30b6f88 Zack Weinberg) FORWARD-COMPATIBILITY: o Backport: provide EVENT_LOG_* names, and deprecate _EVENT_LOG_* (d1a03b2) TESTING/DEBUGGING SUPPORT: o dns-example.c can now take a resolv.conf file on the commandline (6610fa5) o Make some evdns.c debug logs more verbose (d873d67) o Work-around a stupid gcov-breaking bug in OSX 10.6 (b3887cd) Changes in version 2.0.18-stable (22 Mar 2012) BUGFIXES (core): o Make uses of open() close-on-exec safe by introducing an internal evutil_open_closeonexec. (d2b5f72 Ross Lagerwall, 03dce42) BUGFIXES (kqueue): o Properly zero the kevent in kq_setup_kevent() (c2c7b39 Sebastian Hahn) BUILD FIXES: o Added OPENSSL_LDFLAGS env variable which is appended to SSL checks. (9278196 Mark Ellzey) o Changed OPENSSL_LDFLAGS to OPENSSL_LIBADD (2d67b63 Mark Ellzey) o Don't do clang version detection when disabling some flags (083296b Sebastian Hahn) BUGFIXES (dns): o Stop crashing in evdns when nameserver probes give a weird error (bec5068) Changes in version 2.0.17-stable (10 Feb 2012) BUGFIXES (core): o Be absolutely sure to clear pncalls before leaving event_signal_closure (11f36a5) o check for sysctl before we use it (358c745 Mike Frysinger) o Remove bogus casts of socket to int before calling ev_callback (f032516) o Make evconnlistener work around bug in older Linux when getting nmapped (ecfc720) o Fix a list corruption bug when using event_reinit() with signals present (6e41cdc) o Fix a fd leak in event_reinit() (3f18ad1) o Do a memberwise comparison of threading function tables (c94a5f2 Nate R) o Use C-style comments in C source files (for compatibility with compilers such as xlc on AIX). (d84d917 Greg Hewgill) o Avoid crash when freeing event_iocp and using event_set_mem_functions (19715a6) o In the kqueue backend, do not report EBADF as an EV_READ (5d7bfa1 Nicholas Marriott) BUGFIXES (evbuffer and bufferevents): o Fix behavior of evbuffer_peek(buf,-1,NULL,NULL,0) (c986f23 Zack Weinberg) o Loop on filtering SSL reads until we are blocked or exhausted. (5b4b812) BUGFIXES (evhttp): o Force strict validation of HTTP version in response. (790f6b3 Catalin Patulea) BUGFIXES (evdns): o evdns: fix a bug in circular-queue implementation (d6094b1) BUILD FIXES: o Fix a silly compilation error with the sun compiler (1927776 Colin Watt) o Suppress a gcc warning from ignoring fwrite return in http-sample.c (7206e8c) DOCUMENTATION FIXES: o Slightly clarify evbuffer_peek documentation (7bbf6ca) o Update copyright notices to 2012 (e49e289) NEW APIS: o Backport evhttp_connection_get_bufferevent to Libevent 2.0 (da70fa7 Arno Bakker) TESTS AND TEST FIXES: o Fix a race condition in the dns/bufferevent_connect_hostname test. (cba48c7) o Add function to check referential integrity of an event_base (27737d5) o Check event_base correctness at end of each unit test (3312b02) o Workaround in the unit tests for an apparent epoll bug in Linux 3.2 (dab9187) o Better workaround for Linux 3.2 edge-triggered epoll bug (9f9e259) Changes in version 2.0.16-stable (18 Nov 2011) BUGFIXES (core): o More detailed message in case of libevent self-debugging failure. (9e6a4ef Leonid Evdokimov) o epoll: close fd on alloc fail at initialization (1aee718 Jamie Iles) o Fix compile warning from saying event2/*.h inside a comment (447b0ba) o Warn when unable to construct base because of failing make_base_notifiable (4e797f3) o Don't try to make notifiable event_base when no threading fns are configured (e787413) BUGFIXES (evbuffer): o unit test for remove_buffer bug (90bd620 Greg Hazel) o Fix an evbuffer crash in evbuffer_remove_buffer() (c37069c) BUGFIXES (bufferevent_openssl): o Refactor amount-to-read calculations in buffervent_ssl consider_reading() (a186e73 Mark Ellzey) o Move SSL rate-limit enforcement into bytes_to_read() (96c562f) o Avoid spinning on OpenSSL reads (2aa036f Mark Ellzey) BUGFIXES (dns) o Empty DNS reply with OK status is another way to say NODATA. (21a08d6 Leonid Evdokimov) TESTING: o Tests for 94fba5b and f72e8f6 (d58c15e Leonid Evdokimov) o Test for commit aff6ba1 (f7841bf Leonid Evdokimov) o Style and comment tweaks for dns/leak* tests (5e42202) o improve test to remove at least one buffer from src (7eb52eb Greg Hazel) DOCUMENTATION: o Add note about evhttp_send_reply_end to its doxygen (724bfb5) o Update copyright dates to 2011. (3c824bd) o Fix typo in whatsnew-2.0.txt (674bc6a Mansour Moufid) o Improve win32 behavior of dns-sample.c code (a3f320e Gisle Vanem) Changes in version 2.0.15-stable (12 Oct 2011) BUGFIXES (DNS): o DNS: add ttl for negative answers using RFC 2308 idea. (f72e8f6 Leonid Evdokimov) o Add DNS_ERR_NODATA error code to handle empty replies. (94fba5b Leonid Evdokimov) BUFGIXES (bufferevents and evbuffers): o Make evbuffer callbacks get the right n_added value after evbuffer_add (1ef1f68 Alex) o Prefer mmap to sendfile unless a DRAINS_TO_FD flag is set. Allows add_file to work with SSL. (0ba0af9) BUGFIXES (event loop): o When a signal callback is activated to run multiple times, allow event_base_loopbreak to work even before they all have run. (4e8eb6a) DOCUMENTATION FIXES: o Fix docstring in dns.h (2b6eae5 Leonid Evdokimov) o refer to non-deprecated evdns functions in comments (ba5c27d Greg Hazel) BUILD AND TESTING FIXES: o le-proxy and regress depend on openssl directly (9ae061a Sergey Avseyev) o Use _SOURCES, not _sources, in sample/Makefile.am (7f82382) o Fixed compiler warnings for unchecked read/write calls. (c3b62fd Mark Ellzey) o Make write-checking fixes use tt_fail_perror (2b76847) o Fix some "value never used" warnings with gcc 4.6.1 (39c0cf7) Changes in version 2.0.14-stable (31 Aug 2011) BUGFIXES (bufferevents and evbuffers): o Propagate errors on the underlying bufferevent to the user. (4a34394 Joachim Bauch) o Ignore OpenSSL deprecation warnings on OS X (5d1b255 Sebastian Hahn) o Fix handling of group rate limits under 64 bytes of burst (6d5440e) o Solaris sendfile: correctly detect amount of data sent (643922e Michael Herf) o Make rate limiting work with common_timeout logic (5b18f13) o clear read watermark on underlying bufferevent when creating filtering bev to fix potentially failing fragmented ssl handshakes (54f7e61 Joachim Bauch) BUGFIXES (IOCP): o IOCP: don't launch reads or writes on an unconnected socket (495c227) o Make IOCP rate-limiting group support stricter and less surprising. (a98da7b) o Have test-ratelim.c support IOCP (0ff2c5a) o Make overlapped reads result in evbuffer callbacks getting invoked (6acfbdd) o Correctly terminate IO on an async bufferevent on bufferevent_free (e6af35d) BUGFIXES (other): o Fix evsig_dealloc memory leak with debugging turned on. (9b724b2 Leonid Evdokimov) o Fix request_finished memory leak with debugging turned on. (aff6ba1 Leonid Evdokimov) BUILD AND TESTING FIXES: o Allow OS-neutral builds for platforms where some versions have arc4random_buf (b442302 Mitchell Livingston) o Try to fix 'make distcheck' errors when building out-of-tree (04656ea Dave Hart) o Clean up some problems identified by Coverity. (7c11e51 Harlan Stenn) Changes in version 2.0.13-stable (18 Jul 2011) BUGFIXES o Avoid race-condition when initializing global locks (b683cae) o Fix bug in SSL bufferevents backed by a bev with a write high-watermarks (e050703 Joachim Bauch) o Speed up invoke_callbacks on evbuffers when there are no callbacks (f87f568 Mark Ellzey) o Avoid a segfault when all methods are disabled or broken (27ce38b) o Fix incorrect results from evbuffer_search_eol(EOL_LF) (4461f1a) o Add some missing checks for mm_calloc failures (89d5e09) o Replace an assertion for event_base_free(NULL) with a check-and-warn (09fe97d) o Report kqueue ebadf, epipe, and eperm as EV_READ events (1fd34ab) o Check if the `evhttp_new_object' function in `http.c' returns NULL. (446cc7a Mansour Moufid) o Use the correct printf args when formatting size_t (3203f88) o Complain if the caller tries to change threading cbs after setting them (cb6ecee) DOCUMENTATION FIXES AND IMPROVEMENTS o Revise the event/evbuffer/bufferevent doxygen for clarity and accuracy (2888fac) o Update Doxyfile to produce more useful output (aea0555) TEST FIXES o Fix up test_evutil_snprintf (caf695a) o Fix tinytest invocation from windows shell (57def34 Ed Day) BUILD FIXES o Use AM_CPPFLAGS in sample/Makefile.am, not AM_CFLAGS (4a5c82d) o Fix select.c compilation on systems with no NFDBITS (49d1136) o Fix a few warnings on OpenBSD (8ee9f9c Nicholas Marriott) o Don't break when building tests from git without python installed (b031adf) o Don't install event_rpcgen.py when --disable-libevent-install is used (e23cda3 Harlan Stenn) o Fix AIX build issue with TAILQ_FOREACH definition (e934096) Changes in version 2.0.12-stable (4 Jun 2011) BUGFIXES o Fix a warn-and-fail bug in kqueue by providing kevent() room to report errors (28317a0) o Fix an assert-inducing fencepost bug in the select backend (d90149d) o Fix failing http assertion introducd in commit 0d6622e (0848814 Kevin Ko) o Fix a bug that prevented us from configuring IPv6 nameservers. (74760f1) o Prevent size_t overflow in evhttp_htmlescape. (06c51cd Mansour Moufid) o Added several checks for under/overflow conditions in evhttp_handle_chunked_read (a279272 Mark Ellzey) o Added overflow checks in evhttp_read_body and evhttp_get_body (84560fc Mark Ellzey) DOCUMENTATION: o Add missing words to EVLOOP_NONBLOCK documentation (9556a7d) BUILD FIXES o libssl depends on libcrypto, not the other way around. (274dd03 Peter Rosin) o Libtool brings in the dependencies of libevent_openssl.la automatically (7b819f2 Peter Rosin) o Use OPENSSL_LIBS in Makefile.am (292092e Sebastian Hahn) o Move the win32 detection in configure.in (ceb03b9 Sebastian Hahn) o Correctly detect openssl on windows (6619385 Sebastian Hahn) o Fix a compile warning with zlib 1.2.4 and 1.2.5 (5786b91 Sebastian Hahn) o Fix compilation with GCC 2, which had no __builtin_expect (09d39a1 Dave Hart) o Fix new warnings from GCC 4.6 (06a714f) o Link with -lshell32 and -ladvapi32 on Win32. (86090ee Peter Rosin) o Make the tests build when OpenSSL is not available. (07c41be Peter Rosin) o Bring in the compile script from automake, if needed. (f3c7a4c Peter Rosin) o MSVC does not provide S_ISDIR, so provide it manually. (70be7d1 Peter Rosin) o unistd.h and sys/time.h might not exist. (fe93022 Peter Rosin) o Make sure TINYTEST_LOCAL is defined when building tinytest.c (8fa030c Peter Rosin) o Fix winsock2.h #include issues with MSVC (3d768dc Peter Rosin) o Use evutil_gettimeofday instead of relying on the system gettimeofday. (0de87fe Peter Rosin) o Always use evutil_snprintf, even if OS provides it (d1b2d11 Sebastian Hahn) o InitializeCriticalSectionAndSpinCount requires _WIN32_WINNT >= 0x0403. (816115a Peter Rosin) o cygwin: make it possible to build DLLs (d54d3fc) Changes in version 2.0.11-stable (27 Apr 2011) [Autogenerated from the Git log, sorted and cleaned by hand.] BUGFIXES: o Fix evport handling of POLLHUP and POLLERR (b42ce4b) o Fix compilation on Windows with NDEBUG (cb8059d) o Check for POLLERR, POLLHUP and POLLNVAL for Solaris event ports (0144886 Trond Norbye) o Detect and handle more allocation failures. (666b096 Jardel Weyrich) o Use event_err() only if the failure is truly unrecoverable. (3f8d22a Jardel Weyrich) o Handle resize failures in the select backend better. (83e805a) o Correctly free selectop fields when select_resize fails in select_init (0c0ec0b) o Make --enable-gcc-warnings a no-op if not using gcc (3267703) o Fix a type error in our (unused) arc4random_stir() (f736198) o Correctly detect and stop non-chunked http requests when the body is too long (63a715e) o Have event_base_gettimeofday_cached() always return wall-clock time (a459ef7) o Workaround for http crash bug 3078187 (5dc5662 Tomash Brechko) o Fix incorrect assertions and possible use-after-free in evrpc_free() (4b8f02f Christophe Fillot) o Reset outgoing http connection when read data in idle state. (272823f Tomash Brechko) o Fix subtle recursion in evhttp_connection_cb_cleanup(). (218cf19 Tomash Brechko) o Fix the case when failed evhttp_make_request() leaved request in the queue. (0d6622e Tomash Brechko) o Fix a crash bug in evdns server circular list code (00e91b3) o Handle calloc failure in evdns. (Found by Dave Hart) (364291e) o Fix a memory leak on win32 socket->event map. (b4f89f0) o Add a forgotten NULL check to evhttp_parse_headers (12311ff Sebastian Hahn) o Fix possible NULL-deref in evdns_cancel_request (5208544 Sebastian Hahn) PORTABILITY: o Fall back to sscanf if we have no other way to implement strtoll (453317b) o Build correctly on platforms without sockaddr_storage (9184563) o Try to build correctly on platforms with no IPv6 support (713c254) o Build on systems without AI_PASSIVE (cb92113) o Fix http unit test on non-windows platforms without getaddrinfo (6092f12) o Do not check for gethostbyname_r versions if we have getaddrinfo (c1260b0) o Include arpa/inet.h as needed on HPUX (10c834c Harlan Stenn) o Include util-internal.h as needed to build on platforms with no sockaddr_storage (bbf5515 Harlan Stenn) o Check for getservbyname even if not on win32. (af08a94 Harlan Stenn) o Add -D_OSF_SOURCE to fix hpux builds (0b33479 Harlan Stenn) o Check for allocation failures in apply_socktype_protocol_hack (637d17a) o Fix the check for multicast or broadcast addresses in evutil_check_interfaces (1a21d7b) o Avoid a free(NULL) if out-of-memory in evdns_getaddrinfo. Found by Dave Hart (3417f68) DEFENSIVE PROGRAMMING: o Add compile-time check for AF_UNSPEC==PF_UNSPEC (3c8f4e7) BUGS IN TESTS: o Fix test.sh output on solaris (b4f89b6 Dave Hart) o Make test-eof fail with a timeout if we never get an eof. (05a2c22 Harlan Stenn) o Use %s with printf in test.sh (039b9bd) o Add an assert to appease clang's static analyzer (b0ff7eb Sebastian Hahn) o Add a forgotten return value check in the unit tests (3819b62 Sebastian Hahn) o Actually send NULL request in http_bad_request_test (b693c32 Sebastian Hahn) o add some (void) casts for unused variables (65707d7 Sebastian Hahn) o Refactor test_getaddrinfo_async_cancel_stress() (48c44a6 Sebastian Hahn) o Be nice and "handle" error return values in sample code (4bac793 Sebastian Hahn) o Check return value of evbuffer_add_cb in tests (93a1abb Sebastian Hahn) o Remote some dead code from dns-example.c (744c745 Sebastian Hahn) o Zero a struct sockaddr_in before using it (646f9fe Sebastian Hahn) BUILD FIXES: o Fix warnings about AC_LANG_PROGRAM usage (f663112 Sebastian Hahn) o Skip check for zlib if we have no zlib.h (a317c06 Harlan Stenn) o Fix autoconf bracket issues; make check for getaddrinfo include netdb.h (833e5e9 Harlan Stenn) o Correct an AM_CFLAGS to an AM_CPPFLAGS in test/Makefile.am (9c469db Dave Hart) o Fix make distcheck & installation of libevent 1 headers (b5a1f9f Dave Hart) o Fix compilation under LLVM/clang with --enable-gcc-warnings (ad9ff58 Sebastian Hahn) FEATURES: o Make URI parser able to tolerate nonconformant URIs. (95060b5) DOCUMENTATION: o Clarify event_set_mem_functions doc (926f816) o Correct evhttp_del_accept_socket documentation on whether socket is closed (f665924) o fix spelling mistake in whatsnew-2.0.txt (deb2f73) o Fix sample/http-server ipv6 fixes (eb692be) o Comment internal headers used in sample code. (4eb281c) o Be explicit about how long event loops run in event.h documentation (f95bafb) o Add comment to configure.in to explain gc-sections test logic (c621359) o Fix a couple of memory leaks in samples/http-server.c. Found by Dave Hart. (2e9f665) BUILD IMPROVEMENTS: Libevent 2.1.2-alpha modernizes Libevent's use of autotools, and makes numerous other build system. Parallel builds should be faster, and all builds should be quieter. o Split long lists in Makefile.am into one-item-per-line (2711cda) o Remove unnecessary code in configure.in. (e65914f Ross Lagerwall) o attempt to support OpenSSL in Makefile.nmake (eba0eb2 Patrick Pelletier) o Use newer syntax for autoconf/automake init (7d60ba8) o Enable silent build rules by default. Override with V=1 (7b18e5c) o Switch to non-recursive makefiles (7092f3b) o Rename subordinate Makefile.ams to include.am (6cdfeeb) o Make quiet build even quieter (371a123) o New --quiet option for event_rpcgen.py (aa59c1e) o Be quiet when making regress.gen.[ch] (607a8ff) o Fix handling of no-python case for nonrecursive make (1e3123d) o We now require automake 1.9 or later. Modernize! (b7f6e89) o Rename configure.in to configure.ac. (b3fea67 Ross Lagerwall) o Use correct openssl libs and includes in pkgconfig file (d70af27) o Use the same CFLAGS for openssl when building unit tests as with libevent (1d9d511) DOCUMENTATION o Note that make_base_notifiable should not be necessary (26ee5f9) o Be more clear that LEV_OPT_DEFERRED_ACCEPT has tricky prereqs (371efeb) o Add caveat to docs about bufferevent_free() with data in outbuf (6fab9ee) o Make it more clear that NOLOCK means "I promise, no multithreading" (9444524) o Fix a comment in test-fdleak after 077c7e949. (3881d8f Ross Lagerwall) o Make the Makefile.nmake warning slightly less dire (e7bf4c8) o Fix typo : events instead of evets (05f1aca Azat Khuzhin) o Additional comments about OPENSSL_DIR variable, prompted by Dave Hart (6bde2ef Patrick Pelletier) EVHTTP: o ignore LWS after field-content in headers (370a2c0 Artem Germanov) o Clean up rtrim implementation (aa59d80) o Remove trailing tabs in HTTP headers as well. (ac42519) o Remove internal ws from multiline http headers correctly (c6ff381) o Move evutil_rtrim_lws_ to evutil.c where it belongs (61b93af) o add evhttp_request_get_response_code_line (4f4d0c9 Jay R. Wren) o Use EVUTIL_SOCKET_ERROR() wrapper to save/restore errno in evhttp_connection_fail_ (7afbd60) o preserve errno in evhttp_connection_fail_ for inspection by the callback (36d0ee5 Patrick Pelletier) BUGFIXES: o Correctly handle running on a system where accept4 doesn't work. (9fbfe9b) o Avoid double-free on error in evbuffer_add_file. Found by coverity. (6a81b1f) o Fix another possible uninitialized read in dns regression tests. Found by coverity. (13525c5) o Add checks for functions in test-ratelim.c; found by Coverity (aa501e1) o Avoid memory leak in test_event_calloc unit test; found by coverity (92817a1) o Fix a shadowed variable in addfile_test_readcb; found by coverity (225344c) o Check return value when using LEV_OPT_DEFERRED_ACCEPT. Found by coverity (6487f63) o Prevent reference leak of bufferevent if getaddrinfo fails. (b757786 Joachim Bauch) o Make event_base_getnpriorities work with old "implicit base" code (c46cb9c) o Simplify and correct evutil_open_closeonexec_ (0de587f) o Fix event_dlist definition when sys/queue not included (81b6209 Derrick Pallas) Changes in version 2.1.1-alpha (4 Apr 2012) Libevent 2.1.1-alpha includes a number of new features and performance improvements. The log below tries to organize them by rough area of effect. It omits some commits which were pure bugfixes on other commits listed below. For more detail, see the git changelogs. For more insight, see the "whatsnew-2.1.txt" document included in the Libevent 2.1.1-alpha distribution. Performance: Core o Replace several TAILQ users with LIST. LIST can be a little faster than TAILQ for cases where we don't need queue-like behavior. (f9db33d, 6494772, d313c29, 974d004) o Disabled code to optimize the case where we reinsert an existing timeout (e47042f, 09cbc3d) o Remove a needless base-notify when rescheduling the first timeout (77a96fd) o Save a needless comparison when removing/adjusting timeouts (dd5189b) o Possible optimization: split event_queue_insert/remove into separate functions. needs testing (efc4dc5) o Make event_count maintenance branchless at the expense of an extra shift. Needs benchmarking (d1cee3b) o In the 2.1 branch, let's try out lazy gettimeofday/clock_gettime comparison (2a83ecc) o Optimization in event_process_active(): ignore maxcb & endtime for highest priority events. (a9866aa Alexander Drozdov) o Bypass event_add when using event_base_once() for a 0-sec timeout (35c5c95) o Remove the eventqueue list and the ev_next pointers. (604569b 066775e) Performance: Evbuffers o Roughly 20% speed increase when line-draining a buffer using EVBUFFER_EOL_CRLF (5dde0f0 Mina Naguib) o Try to squeeze a little more speed out of EVBUFFER_EOL_CRLF (7b9d139) o Fix a bug in the improved EOL_CRLF code (d927965) o Remove a needless branch in evbuffer_drain() (d19a326) Performance: Linux o Infrastructure for using faster/fewer syscalls when creating sockets (a1c042b) o Minimize syscalls during socket creation in listener.c (7e9e289) o Use a wrapper function to create the notification pipe/socketpair/eventfd (ca76cd9) o Use pipes for telling signals to main thread when possible (a35f396) o Save syscalls when constructing listener sockets for evhttp (af6c9d8) o Save some syscalls when creating evdns sockets (713e570) o Save some syscalls when constructing a socket for a bufferevent (33fca62) o Prefer epoll_create1 on Linuxen that have it (bac906c) Performance: Epoll backend o Use current event set rather than current pending change when deciding whether to no-op a del (04ba27e Mike Smellie) o Replace big chain of if/thens in epoll.c with a table lookup (8c83eb6) o Clean up error handling in epoll_apply_one_change() a little (2d55a19) Performance: Evport backend o evport: use evmap_io to track fdinfo status. Should save time and RAM. (4687ce4) o evport: Remove a linear search over recent events when reactivating them (0f77efe) o evport: Use portev_user to remember fdinfo struct (276ec0e) o evport: don't scan more events in ed_pending than needed (849a5cf) o evport: Remove artificial low limit on max events per getn call (c04d927) o Reenable main/many_events_slow_add for evport in 2.1 (e903db3) Performance: Windows o Use GetSystemTimeAsFileTime to implement gettimeofday on win32. It's faster and more accurate than our old approach. (b8b8aa5) New functions and features: debugging o Add event_enable_debug_logging() to control use of debug logs (e30a82f) New functions and features: core o Add event_config function to limit time/callbacks between calls to dispatch (fd4de1e, 9fa56bd, a37a0c0, 3c63edd) o New EVLOOP_NO_EXIT_ON_EMPTY option to keep looping even when no events are pending (084e68f) o Add event_base_get_npriorities() function. (ee3a4ee Alexander Drozdov) o Make evbase_priority_init() and evbase_get_npriorities() threadsafe (3c55b5e) o New event_base_update_cache_time() to set cached_tv to current time (212533e Abel Mathew) o Add event_self_cbarg() to be used in conjunction with event_new(). (ed36e6a Ross Lagerwall, fa931bb, 09a1906, 1338e6c, 33e43ef) o Add a new libevent_global_shutdown() to free all globals before exiting. (041ca00 Mark Ellzey, f98c158, 15296d0, 55e991b) o Use getifaddrs to detect our interfaces if possible (7085a45) o Add event_base_get_running_event() to get the event* whose cb we are in (c5732fd, 13dad99) New functions and features: building o Implement --enable-gcc-hardening configure option (7550267 Sebastian Hahn) New functions and features: evbuffers o Add evbuffer_add_file_segment() so one fd can be used efficiently in more than one evbuffer_add_file at a time (e72afae, c2d9884, 3f405d2, 0aad014) o Fix windows file segment mappings (8254de7) o Allow evbuffer_ptr_set to yield a point just after the end of the buffer. (e6fe1da) o Allow evbuffer_ptr to point to position 0 in an empty evbuffer (7aeb2fd Nir Soffer) o Set the special "not found" evbuffer_ptr consistently. (e3e97ae Nir Soffer) o support adding buffers to other buffers non-destructively (9d7368a Joachim Bauch) o prevent nested multicast references, reworked locking (26041a8 Joachim Bauch) o New EVBUFFER_EOL_NUL to read NUL-terminated strings from an evbuffer (d7a8b36 Andrea Montefusco, 54142c9) o Make evbuffer_file_segment_types adaptable (c6bbbf1) o Added evbuffer_add_iovec and unit tests. (aaec5ac Mark Ellzey, 27b5398) o Add evbuffer_copyout_from to copy data from the middle of a buffer (27e2225) New functions and features: bufferevents o Allow users to set allow_dirty_shutdown (099d27d Catalin Patulea) o Tweak allow_dirty_shutdown documentation (a44cd2b) o Fix two issues in the allow_dirty_shutdown code. (f3b89de) o Add a bufferevent_getcb() to find a bufferevent's current callbacks (a650394) o bufferevent: Add functions to set/get max_single_read/write values. (998c813 Alexander Drozdov) o bev_ssl: Be more specific in event callbacks. evhttp in particular gets confused without at least one of BEV_EVENT_{READING|WRITING}. (f7eb69a Catalin Patulea) New functions and features: evconnlisteners o Support TCP_DEFER_ACCEPT sockopts for listeners (5880e4a Mark Ellzey, a270728) o Add another caveat to the TCP_DEFER_ACCEPT documentation (a270728) o Allow evconnlistener to be created in disabled state. (9593a33 Alexander Drozdov) o The LEV_OPT_CLOSE_ON_EXEC flag now applies to accepted listener sockets too (4970329) Evhttp: o Add new evhttp_{connection_}set_timeout_tv() functions to set finger-grained http timeouts (6350e6c Constantine Verutin) o Performance tweak to evhttp_parse_request_line. (aee1a97 Mark Ellzey) o Add missing break to evhttp_parse_request_line (0fcc536) o Add evhttp callback for bufferevent creation; this lets evhttp support SSL. (8d3a850) o Remove calls to deprecated bufferevent functions from evhttp.c (4d63758) o evhttp: Add evhttp_foreach_bound_socket. (a2c48e3 Samy Al Bahra) Build improvements: o Add AC_USE_SYSTEM_EXTENSIONS to configure.in. Requires follow on patches for correctness and robustness. (1fa7dbe Kevin Bowling) o Filter '# define' statements from autoconf and generate event-private.h (321b558 Kevin Bowling) o Remove internal usage of _GNU_SOURCE (3b26541 Kevin Bowling) o Eliminate a couple more manual internal _GNU_SOURCE defines (c51ef93 Kevin Bowling) o Add AC_GNU_SOURCE to the fallback case. (ea8fa4c Kevin Bowling) o Use a Configuration Header Template for evconfig-private.h (868f888 Kevin Bowling) o Fix a comment warning and add evconfig-private.h to .gitignore (f6d66bc Kevin Bowling) o Include evconfig-private.h in internal files for great good. (0915ca0 Kevin Bowling) o Backport libevent to vanilla Autoconf 2.59 (as used in RHEL5) (ad03952 Kevin Bowling) o Prefer the ./configure evconfig-private.h in MinGW, just in case. (f964b72 Kevin Bowling) o Shell hack for weird mkdir -p commands (fd7b5a8 Kevin Bowling) o Add evconfig-private to remaining files (ded0a09 Kevin Bowling) o Allow use of --enable-silent-rules for quieter compilation with automake 1.11 (f1f8514 Dave Hart) o Use "_WIN32", not WIN32: it's standard and we don't need to fake it (9f560b) o In configure, test for _WIN32 not WIN32. (85078b1 Peter Rosin) o Do not define WIN32 in Makefile.nmake (d41f3ea Peter Rosin) o Provide the autoconf m4 macros for the new OpenSSL via pkg-config stuff. (674dc3d Harlan Stenn) o Use pkg-config (if available) to handle OpenSSL. (1c63860 Harlan Stenn) o We need AM_CPPFLAGS when compiling bufferevent_openssl.c (6d2613b Harlan Stenn) o Fix OSX build: $(OPENSSL_INCS) needs to be after $(AM_CPPFLAGS). (46f1769 Zack Weinberg) o Make gcc warnings on by default, and --enable-gcc-warnings only add -Werror (d46517e Sebastian Hahn) o Split up extra-long AC_CHECK_FUNCS/HEADERS lines in configure.in (88a30ad) o Move libevent 1.x headers to include/, to put all public headers in one place. (bbea8d6) o Put #ifdef around some files to support alternate build systems. (76d4c92 Ross Lagerwall) o Also make win32select.c conditional for IDE users (bf2c5a7) Debugging: o Add a magic number to debug_locks to better catch lock-coding errors. (b4a29c0 Dave Hart) o munge the debug_lock signature before freeing it: it might help us catch use-after-free (f28084d) o Added --enable-event-debugging in configure (bc7b4e4, a9c2c9a Mark Ellzey) o Debug addition for printing usec on TIMEOUT debugging. (ac43ce0 Mark Ellzey) o Added usec debug in another area for debug (3baab0d Mark Ellzey) o added timeout debug logs to include event ptr. (4b7d298 Mark Ellzey) o more event dbg updates (6727543 Mark Ellzey) o Clarify event_enable_debug_logging a little (6207826) o Make --enable-verbose-debug option match its help text (10c3450) o Add argument checks to some memory functions in `event.c'. (c8953d1 Mansour Moufid) Testing: o More abstraction in test.sh (cd74c4e) o Add failing test for evbuffer_search_range. (8e26154 Nir Soffer) o Tweaks to return types with end-of-buf ptrs (9ab8ab8) o Add an (internal) usleep function for use by unit tests (f25d9d3) o Synchronize with upstream tinytest (6c81be7) o Make test-changelist faster (7622d26) o Reduce the timeout in the main/fork test. (ab14f7c) o New evhttp function to adjust initial retry timeout (350a3c4) o Make regression tests run over 3x faster. (67a1763) o Use test_timeval_diff_eq more consistently (b77b43f) o Allow more slop in deferred_cb_skew test; freebsd needs it (b9f7e5f) o When including an -internal.h header outside the main tree, do so early (95e2455) o Add a new test: test-fdleak which tests for fd leaks by creating many sockets. (2ef9278 Ross Lagerwall, f7af194, 1c4288f, etc) o Add a unit test for event_base_dump_events() (7afe48a, 8d08cce) o Test more bufferevent_ratelim features (c24f91a) Documentation: o Improve evbuffer_ptr documentation (261ba63) o added comments to describe refcounting of multicast chains (ba24f61 Joachim Bauch) o Add doxygen for event_base_dump_events (cad5753) OSX: o Use "unlimited select" on OSX so that we can have more than FD_SETSIZE fds (1fb5cc6) KQueue: o Use SIG_IGN instead of a do-nothing handler for signal events with kqueue (148458e Zack Weinberg) evprc: o event_rpcgen.py now prints status information to stdout and errors to stderr. (ffb0ba0 Ross Lagerwall) Code improvement and refactoring: o Make event_reinit() more robust and maintainable (272033e) o Restore fast-path event_reinit() for slower backends (2c4b5de) o Check changelist as part of checking representational integrity (39b3f38) o Fix a compile warning in event_reinit (e4a56ed Sebastian Hahn) o Refactor the functions that run over every event. (c89b4e6) o Remove the last vestiges of _EVENT_USE_EVENTLIST (a3cec90) o Make event-config.h depend on Makefile.am (2958a5c) Build fixes: o Don't do clang version detection when disabling some flags (083296b Sebastian Hahn) C standards conformance: o Check for NULL return on win32 mm_calloc, and set ENOMEM. (af7ba69) o Convert event-config.h macros to avoid reserved identifiers (68120d9) o Generate event-config.h using the correct macros. (f82c57e) o Convert include-guard macro convention to avoid reserved identifiers (3f8c7cd) o Make event_rpcgen.py output conform to identifier conventions (372bff1) o Stop referring to an obsolete include guard in bench_http.h (5c0f7e0) o Make the generated event-config.h use correct include guards (639383a) o Fix all identifiers with names beginning with underscore. (cb9da0b) o Make event_rpcgen.py output conform to identifier conventions, more (bcefd24) o Fix some problems introduced by automated identifier cleanup script (c963534) o Have all visible internal function names end with an underscore. (8ac3c4c) o Apply the naming convention to our EVUTIL_IS* functions (c7848fa) o Clean up lingering _identifiers. (946b584) o Fix doxygen to use new macro conventions (da455e9) Bugfixes: o Do not use system EAI/AI values if we are not using the system getaddrinfo. (7bcac07) Sample Code: o Fix up sample/event-test.c to use newer interfaces and make it actually work. (19bab4f Ross Lagerwall) o On Unix, remove event.fifo left by sample/event-test.c. (c0dacd2 Ross Lagerwall) o Rename event-test.c to event-read-fifo.c. (a5b370a Ross Lagerwall) o event-read-fifo: Use EV_PERSIST appropriately (24dab0b) ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/ChangeLog-1.4������������������������������������������������0000664�0000000�0000000�00000041731�15270523712�0022056�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Changes in 1.4.14b-stable o Set the VERSION_INFO correctly for 1.4.14 Changes in 1.4.14-stable o Add a .gitignore file for the 1.4 branch. (d014edb) o Backport evbuffer_readln(). (b04cc60 Nicholas Marriott) o Make the evbuffer_readln backport follow the current API (c545485) o Valgrind fix: Clear struct kevent before checking for OSX bug. (5713d5d William Ahern) o Fix a crash when reading badly formatted resolve.conf (5b10d00 Yasuoka Masahiko) o Fix memory-leak of signal handler array with kqueue. [backport] (01f3775) o Update sample/signal-test.c to use newer APIs and not leak. (891765c Evan Jones) o Correct all versions in 1.4 branch (ac0d213) o Make evutil_make_socket_nonblocking() leave any other flags alone. (81c26ba Jardel Weyrich) o Adjusted fcntl() retval comparison on evutil_make_socket_nonblocking(). (5f2e250 Jardel Weyrich) o Correct a debug message in evhttp_parse_request_line (35df59e) o Merge branch 'readln-backport' into patches-1.4 (8771d5b) o Do not send an HTTP error when we've already closed or responded. (4fd2dd9 Pavel Plesov) o Re-add event_siglcb; some old code _was_ still using it. :( (bd03d06) o Make Libevent 1.4 build on win32 with Unicode enabled. (bce58d6 Brodie Thiesfield) o Distribute nmake makefile for 1.4 (20d706d) o do not fail while sending on http connections the client closed. (5c8b446) o make evhttp_send() safe against terminated connections, too (01ea0c5) o Fix a free(NULL) in min_heap.h (2458934) o Fix memory leak when setting up priorities; reported by Alexander Drozdov (cb1a722) o Clean up properly when adding a signal handler fails. (ae6ece0 Gilad Benjamini) o Do not abort HTTP requests missing a reason string. (29d7b32 Pierre Phaneuf) o Fix compile warning in http.c (906d573) o Define _REENTRANT as needed on Solaris, elsewhere (6cbea13) Changes in 1.4.13-stable: o If the kernel tells us that there are a negative number of bytes to read from a socket, do not believe it. Fixes bug 2841177; found by Alexander Pronchenkov. o Do not allocate the maximum event queue and fd array for the epoll backend at startup. Instead, start out accepting 32 events at a time, and double the queue's size when it seems that the OS is generating events faster than we're requesting them. Saves up to 512K per epoll-based event_base. Resolves bug 2839240. o Fix compilation on Android, which forgot to define fd_mask in its sys/select.h o Do not drop data from evbuffer when out of memory; reported by Jacek Masiulaniec o Rename our replacement compat/sys/_time.h header to avoid build a conflict on HPUX; reported by Kathryn Hogg. o Build kqueue.c correctly on GNU/kFreeBSD platforms. Patch pulled upstream from Debian. o Fix a problem with excessive memory allocation when using multiple event priorities. o When running set[ug]id, don't check the environment. Based on a patch from OpenBSD. Changes in 1.4.12-stable: o Try to contain degree of failure when running on a win32 version so heavily firewalled that we can't fake a socketpair. o Fix an obscure timing-dependent, allocator-dependent crash in the evdns code. o Use __VA_ARGS__ syntax for varargs macros in event_rpcgen when compiler is not GCC. o Activate fd events in a pseudorandom order with O(N) backends, so that we don't systematically favor low fds (select) or earlier-added fds (poll, win32). o Fix another pair of fencepost bugs in epoll.c. [Patch from Adam Langley.] o Do not break evdns connections to nameservers when our IP changes. o Set truncated flag correctly in evdns server replies. o Disable strict aliasing with GCC: our code is not compliant with it. Changes in 1.4.11-stable: o Fix a bug when removing a timeout from the heap. [Patch from Marko Kreen] o Remove the limit on size of HTTP headers by removing static buffers. o Fix a nasty dangling pointer bug in epoll.c that could occur after epoll_recalc(). [Patch from Kevin Springborn] o Distribute Win32-Code/event-config.h, not ./event-config.h Changes in 1.4.10-stable: o clean up buffered http connection data on reset; reported by Brian O'Kelley o bug fix and potential race condition in signal handling; from Alexander Drozdov o rename the Solaris event ports backend to evport o support compilation on Haiku o fix signal processing when a signal callback delivers a signal; from Alexander Drozdov o const-ify some arguments to evdns functions. o off-by-one error in epoll_recalc; reported by Victor Goya o include Doxyfile in tar ball; from Jeff Garzik o correctly parse queries with encoded \r, \n or + characters Changes in 1.4.9-stable: o event_add would not return error for some backends; from Dean McNamee o Clear the timer cache on entering the event loop; reported by Victor Chang o Only bind the socket on connect when a local address has been provided; reported by Alejo Sanchez o Allow setting of local port for evhttp connections to support millions of connections from a single system; from Richard Jones. o Clear the timer cache when leaving the event loop; reported by Robin Haberkorn o Fix a typo in setting the global event base; reported by lance. o Fix a memory leak when reading multi-line headers o Fix a memory leak by not running explicit close detection for server connections Changes in 1.4.8-stable: o Match the query in DNS replies to the query in the request; from Vsevolod Stakhov. o Fix a merge problem in which name_from_addr returned pointers to the stack; found by Jiang Hong. o Do not remove Accept-Encoding header Changes in 1.4.7-stable: o Fix a bug where headers arriving in multiple packets were not parsed; fix from Jiang Hong; test by me. Changes in 1.4.6-stable: o evutil.h now includes <stdarg.h> directly o switch all uses of [v]snprintf over to evutil o Correct handling of trailing headers in chunked replies; from Scott Lamb. o Support multi-line HTTP headers; based on a patch from Moshe Litvin o Reject negative Content-Length headers; anonymous bug report o Detect CLOCK_MONOTONIC at runtime for evdns; anonymous bug report o Fix a bug where deleting signals with the kqueue backend would cause subsequent adds to fail o Support multiple events listening on the same signal; make signals regular events that go on the same event queue; problem report by Alexander Drozdov. o Deal with evbuffer_read() returning -1 on EINTR|EAGAIN; from Adam Langley. o Fix a bug in which the DNS server would incorrectly set the type of a cname reply to a. o Fix a bug where setting the timeout on a bufferevent would take not effect if the event was already pending. o Fix a memory leak when using signals for some event bases; reported by Alexander Drozdov. o Add libevent.vcproj file to distribution to help with Windows build. o Fix a problem with epoll() and reinit; problem report by Alexander Drozdov. o Fix off-by-one errors in devpoll; from Ian Bell o Make event_add not change any state if it fails; reported by Ian Bell. o Do not warn on accept when errno is either EAGAIN or EINTR Changes in 1.4.5-stable: o Fix connection keep-alive behavior for HTTP/1.0 o Fix use of freed memory in event_reinit; pointed out by Peter Postma o Constify struct timeval * where possible; pointed out by Forest Wilkinson o allow min_heap_erase to be called on removed members; from liusifan. o Rename INPUT and OUTPUT to EVRPC_INPUT and EVRPC_OUTPUT. Retain INPUT/OUTPUT aliases on on-win32 platforms for backwards compatibility. o Do not use SO_REUSEADDR when connecting o Fix Windows build o Fix a bug in event_rpcgen when generated fixed-sized entries Changes in 1.4.4-stable: o Correct the documentation on buffer printf functions. o Don't warn on unimplemented epoll_create(): this isn't a problem, just a reason to fall back to poll or select. o Correctly handle timeouts larger than 35 minutes on Linux with epoll.c. This is probably a kernel defect, but we'll have to support old kernels anyway even if it gets fixed. o Fix a potential stack corruption bug in tagging on 64-bit CPUs. o expose bufferevent_setwatermark via header files and fix high watermark on read o fix a bug in bufferevent read water marks and add a test for them o introduce bufferevent_setcb and bufferevent_setfd to allow better manipulation of bufferevents o use libevent's internal timercmp on all platforms, to avoid bugs on old platforms where timercmp(a,b,<=) is buggy. o reduce system calls for getting current time by caching it. o fix evhttp_bind_socket() so that multiple sockets can be bound by the same http server. o Build test directory correctly with CPPFLAGS set. o Fix build under Visual C++ 2005. o Expose evhttp_accept_socket() API. o Merge windows gettimeofday() replacement into a new evutil_gettimeofday() function. o Fix autoconf script behavior on IRIX. o Make sure winsock2.h include always comes before windows.h include. Changes in 1.4.3-stable: o include Content-Length in reply for HTTP/1.0 requests with keep-alive o Patch from Tani Hosokawa: make some functions in http.c threadsafe. o Do not free the kqop file descriptor in other processes, also allow it to be 0; from Andrei Nigmatulin o make event_rpcgen.py generate code include event-config.h; reported by Sam Banks. o make event methods static so that they are not exported; from Andrei Nigmatulin o make RPC replies use application/octet-stream as mime type o do not delete uninitialized timeout event in evdns Changes in 1.4.2-rc: o remove pending timeouts on event_base_free() o also check EAGAIN for Solaris' event ports; from W.C.A. Wijngaards o devpoll and evport need reinit; tested by W.C.A Wijngaards o event_base_get_method; from Springande Ulv o Send CRLF after each chunk in HTTP output, for compliance with RFC2626. Patch from "propanbutan". Fixes bug 1894184. o Add a int64_t parsing function, with unit tests, so we can apply Scott Lamb's fix to allow large HTTP values. o Use a 64-bit field to hold HTTP content-lengths. Patch from Scott Lamb. o Allow regression code to build even without Python installed o remove NDEBUG ifdefs from evdns.c o update documentation of event_loop and event_base_loop; from Tani Hosokawa. o detect integer types properly on platforms without stdint.h o Remove "AM_MAINTAINER_MODE" declaration in configure.in: now makefiles and configure should get re-generated automatically when Makefile.am or configure.in chanes. o do not insert event into list when evsel->add fails Changes in 1.4.1-beta: o free minheap on event_base_free(); from Christopher Layne o debug cleanups in signal.c; from Christopher Layne o provide event_base_new() that does not set the current_base global o bufferevent_write now uses a const source argument; report from Charles Kerr o better documentation for event_base_loopexit; from Scott Lamb. o Make kqueue have the same behavior as other backends when a signal is caught between event_add() and event_loop(). Previously, it would catch and ignore such signals. o Make kqueue restore signal handlers correctly when event_del() is called. o provide event_reinit() to reintialize an event_base after fork o small improvements to evhttp documentation o always generate Date and Content-Length headers for HTTP/1.1 replies o set the correct event base for HTTP close events o New function, event_{base_}loopbreak. Like event_loopexit, it makes an event loop stop executing and return. Unlike event_loopexit, it keeps subsequent pending events from getting executed. Patch from Scott Lamb o Removed obsoleted recalc code o pull setters/getters out of RPC structures into a base class to which we just need to store a pointer; this reduces the memory footprint of these structures. o fix a bug with event_rpcgen for integers o move EV_PERSIST handling out of the event backends o support for 32-bit tag numbers in rpc structures; this is wire compatible, but changes the API slightly. o prefix {encode,decode}_tag functions with evtag to avoid collisions o Correctly handle DNS replies with no answers set (Fixes bug 1846282) o The configure script now takes an --enable-gcc-warnigns option that turns on many optional gcc warnings. (Nick has been building with these for a while, but they might be useful to other developers.) o When building with GCC, use the "format" attribute to verify type correctness of calls to printf-like functions. o removed linger from http server socket; reported by Ilya Martynov o allow \r or \n individually to separate HTTP headers instead of the standard "\r\n"; from Charles Kerr. o demote most http warnings to debug messages o Fix Solaris compilation; from Magne Mahre o Add a "Date" header to HTTP responses, as required by HTTP 1.1. o Support specifying the local address of an evhttp_connection using set_local_address o Fix a memory leak in which failed HTTP connections would not free the request object o Make adding of array members in event_rpcgen more efficient, but doubling memory allocation o Fix a memory leak in the DNS server o Fix compilation when DNS_USE_OPENSSL_FOR_ID is enabled o Fix buffer size and string generation in evdns_resolve_reverse_ipv6(). o Respond to nonstandard DNS queries with "NOTIMPL" rather than by ignoring them. o In DNS responses, the CD flag should be preserved, not the TC flag. o Fix http.c to compile properly with USE_DEBUG; from Christopher Layne o Handle NULL timeouts correctly on Solaris; from Trond Norbye o Recalculate pending events properly when reallocating event array on Solaris; from Trond Norbye o Add Doxygen documentation to header files; from Mark Heily o Add a evdns_set_transaction_id_fn() function to override the default transaction ID generation code. o Add an evutil module (with header evutil.h) to implement our standard cross-platform hacks, on the theory that somebody else would like to use them too. o Fix signals implementation on windows. o Fix http module on windows to close sockets properly. o Make autogen.sh script run correctly on systems where /bin/sh isn't bash. (Patch from Trond Norbye, rewritten by Hagne Mahre and then Hannah Schroeter.) o Skip calling gettime() in timeout_process if we are not in fact waiting for any events. (Patch from Trond Norbye) o Make test subdirectory compile under mingw. o Fix win32 buffer.c behavior so that it is correct for sockets (which do not like ReadFile and WriteFile). o Make the test.sh script run unit tests for the evpoll method. o Make the entire evdns.h header enclosed in "extern C" as appropriate. o Fix implementation of strsep on platforms that lack it o Fix implementation of getaddrinfo on platforms that lack it; mainly, this will make Windows http.c work better. Original patch by Lubomir Marinov. o Fix evport implementation: port_disassociate called on unassociated events resulting in bogus errors; more efficient memory management; from Trond Norbye and Prakash Sangappa o support for hooks on rpc input and output; can be used to implement rpc independent processing such as compression or authentication. o use a min heap instead of a red-black tree for timeouts; as a result finding the min is a O(1) operation now; from Maxim Yegorushkin o associate an event base with an rpc pool o added two additional libraries: libevent_core and libevent_extra in addition to the regular libevent. libevent_core contains only the event core whereas libevent_extra contains dns, http and rpc support o Begin using libtool's library versioning support correctly. If we don't mess up, this will more or less guarantee binaries linked against old versions of libevent continue working when we make changes to libevent that do not break backward compatibility. o Fix evhttp.h compilation when TAILQ_ENTRY is not defined. o Small code cleanups in epoll_dispatch(). o Increase the maximum number of addresses read from a packet in evdns to 32. o Remove support for the rtsig method: it hasn't compiled for a while, and nobody seems to miss it very much. Let us know if there's a good reason to put it back in. o Rename the "class" field in evdns_server_request to dns_question_class, so that it won't break compilation under C++. Use a macro so that old code won't break. Mark the macro as deprecated. o Fix DNS unit tests so that having a DNS server with broken IPv6 support is no longer cause for aborting the unit tests. o Make event_base_free() succeed even if there are pending non-internal events on a base. This may still leak memory and fds, but at least it no longer crashes. o Post-process the config.h file into a new, installed event-config.h file that we can install, and whose macros will be safe to include in header files. o Remove the long-deprecated acconfig.h file. o Do not require #include <sys/types.h> before #include <event.h>. o Add new evutil_timer* functions to wrap (or replace) the regular timeval manipulation functions. o Fix many build issues when using the Microsoft C compiler. o Remove a bash-ism in autogen.sh o When calling event_del on a signal, restore the signal handler's previous value rather than setting it to SIG_DFL. Patch from Christopher Layne. o Make the logic for active events work better with internal events; patch from Christopher Layne. o We do not need to specially remove a timeout before calling event_del; patch from Christopher Layne. ���������������������������������������kytensor-2.49.0.7/third_party/libevent/ChangeLog-2.0������������������������������������������������0000664�0000000�0000000�00000242623�15270523712�0022056�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Changes in version 2.0.21-stable (18 Nov 2012) BUGFIXES: o ssl: Don't discard SSL read event when timeout and read come close together (576b29f) o ssl: Stop looping in "consider_reading" if reading is suspended. (f719b8a Joachim Bauch) o ssl: No need to reserve space if reading is suspended. (1acf2eb Joachim Bauch) o dns: Avoid a memory-leak on OOM in evdns. (73e85dd, f2bff75 George Danchev) o build: Use python2 rather than python (0eb0109 Ross Lagerwall) o build: Compile without warnings on mingw64 (94866c2) o build: Fix compilation on mingw64 with -DUSE_DEBUG (62bd2c4) o build: Make rpcgen_wrapper.sh work on systems without a "python2" binary (f3009e4) o iocp: Close IOCP listener socket on free when LEV_OPT_CLOSE_ON_FREE is set (cb853ea Juan Pablo Fernandez) o core: Avoid crash when event_pending() called with no event_base set on event (e3cccf3) o misc: remove stray 'x' so print_err will compile when uncommented (ac35650 Patrick Pelletier) o tests: Fix renegotiation test to work around openssl 1.0.1 bug (c2f3086) o tests: Warn when openssl version in unit test mismatches compiled version. (ac009f9) Changes in version 2.0.20-stable (23 Aug 2012) BUGFIXES: o core: Make event_pending() threadsafe. (be7a95c Simon Liu) o win32: avoid crash when waiting forever on zero fds. (160e58b) o evhttp: Fix a memory leak on error in evhttp_uriencode (11c8b31) o evbuffer: Avoid possible needless call to writev. Found by coverity. (6a4ec5c) o evdns: memset sockaddr_in before using it. Found by coverity. (a1a0e67) o evhttp: Check more setsockopt return values when binding sockets. Found by coverity (a0912e3) o evdns: Avoid segfault on weird timeout during name lookup. (dc32077 Greg Hazel) o bufferevent_ssl: Correctly invoke callbacks when a SSL bufferevent reads some and then blocks. (606ac43) PORTABILITY FIXES: o check for arc4random_buf at runtime, on OS X (bff5f94 Greg Hazel) o Correctly check for arc4random_buf (fcec3e8 Sebastian Hahn) o Add explicit AC_PROG_SED to configure.in so all autoconfs will expose $(SED) (ca80ea6) BUILD FIXES: o Add GCC annotations so that the vsprintf functions get checked properly (117e327) o Fix an unused variable warning on *BSD. (c0720c1) UNIT TEST FIXES: o Fix a couple of memory leaks (found with Valgrind). (3b2529a Ross Lagerwall) o Remove deadcode in http regression tests. Found by coverity. (5553346) o Fix possible uninitialized read in dns regression tests. Found by coverity. (2259777) o Set umask before calling mkstemp in unit tests. Found by coverity (f1ce15d) o Fix various check-after-dereference issues in unit tests: found by coverity (4f3732d) o Fix resource leaks in the unit tests; found by coverity (270f279) o Add some missing null checks to unit tests; found by coverity (f021c3d) o Avoid more crashes/bad calls in unit tests; found by coverity (3cde5bf) o Remove unused variable; spotted by coverity (6355b2a) o Add checks to various return values in unit tests. Found by coverity (b9e7329) o Move assignment outside tt_assert in ssl unit tests. Appeases coverity. (a2006c0) Changes in version 2.0.19-stable (3 May 2012) BUGFIXES (CORE): o Refactor event_persist_closure: raise and extract some common logic (bec22b4) o If time has jumped so we'd reschedule a periodic event in the past, schedule it for the future instead (dfd808c) o If a higher-priority event becomes active, don't continue running events of the current priority. (2bfda40) BUGFIXES (SSL): o Fixed potential double-readcb execution with openssl bufferevents. (4e62cd1 Mark Ellzey) BUGFIXES (DNS): o Cancel a probe request when the server is freed, and ignore cancelled probe callbacks (94d2336 Greg Hazel) o Remove redundant DNS_ERR_CANCEL check, move comment (46b8060 Greg Hazel) o When retransmitting a timed-out DNS request, pick a fresh nameserver. (3d9e52a) DOCUMENTATION FIXES: o Fix a typo in the bufferevent documentation (98e9119) o Add missing ) to changelog; spotted by rransom (4c7ee6b) o Fix the website URL in the readme (f775521) COMPILATION FIXES: o Fix a compilation error with MSVC 2005 due to use of mode_t (336dcae) o Configure with gcc older than 2.95 (4a6fd43 Sebastian Hahn) o Generate event-config.h with a single sed script (30b6f88 Zack Weinberg) FORWARD-COMPATIBILITY: o Backport: provide EVENT_LOG_* names, and deprecate _EVENT_LOG_* (d1a03b2) TESTING/DEBUGGING SUPPORT: o dns-example.c can now take a resolv.conf file on the commandline (6610fa5) o Make some evdns.c debug logs more verbose (d873d67) o Work-around a stupid gcov-breaking bug in OSX 10.6 (b3887cd) Changes in version 2.0.18-stable (22 Mar 2012) BUGFIXES (core): o Make uses of open() close-on-exec safe by introducing an internal evutil_open_closeonexec. (d2b5f72 Ross Lagerwall, 03dce42) BUGFIXES (kqueue): o Properly zero the kevent in kq_setup_kevent() (c2c7b39 Sebastian Hahn) BUILD FIXES: o Added OPENSSL_LDFLAGS env variable which is appended to SSL checks. (9278196 Mark Ellzey) o Changed OPENSSL_LDFLAGS to OPENSSL_LIBADD (2d67b63 Mark Ellzey) o Don't do clang version detection when disabling some flags (083296b Sebastian Hahn) BUGFIXES (dns): o Stop crashing in evdns when nameserver probes give a weird error (bec5068) Changes in version 2.0.17-stable (10 Feb 2012) BUGFIXES (core): o Be absolutely sure to clear pncalls before leaving event_signal_closure (11f36a5) o check for sysctl before we use it (358c745 Mike Frysinger) o Remove bogus casts of socket to int before calling ev_callback (f032516) o Make evconnlistener work around bug in older Linux when getting nmapped (ecfc720) o Fix a list corruption bug when using event_reinit() with signals present (6e41cdc) o Fix a fd leak in event_reinit() (3f18ad1) o Do a memberwise comparison of threading function tables (c94a5f2 Nate R) o Use C-style comments in C source files (for compatibility with compilers such as xlc on AIX). (d84d917 Greg Hewgill) o Avoid crash when freeing event_iocp and using event_set_mem_functions (19715a6) o In the kqueue backend, do not report EBADF as an EV_READ (5d7bfa1 Nicholas Marriott) BUGFIXES (evbuffer and bufferevents): o Fix behavior of evbuffer_peek(buf,-1,NULL,NULL,0) (c986f23 Zack Weinberg) o Loop on filtering SSL reads until we are blocked or exhausted. (5b4b812) BUGFIXES (evhttp): o Force strict validation of HTTP version in response. (790f6b3 Catalin Patulea) BUGFIXES (evdns): o evdns: fix a bug in circular-queue implementation (d6094b1) BUILD FIXES: o Fix a silly compilation error with the sun compiler (1927776 Colin Watt) o Suppress a gcc warning from ignoring fwrite return in http-sample.c (7206e8c) DOCUMENTATION FIXES: o Slightly clarify evbuffer_peek documentation (7bbf6ca) o Update copyright notices to 2012 (e49e289) NEW APIS: o Backport evhttp_connection_get_bufferevent to Libevent 2.0 (da70fa7 Arno Bakker) TESTS AND TEST FIXES: o Fix a race condition in the dns/bufferevent_connect_hostname test. (cba48c7) o Add function to check referential integrity of an event_base (27737d5) o Check event_base correctness at end of each unit test (3312b02) o Workaround in the unit tests for an apparent epoll bug in Linux 3.2 (dab9187) o Better workaround for Linux 3.2 edge-triggered epoll bug (9f9e259) Changes in version 2.0.16-stable (18 Nov 2011) BUGFIXES (core): o More detailed message in case of libevent self-debugging failure. (9e6a4ef Leonid Evdokimov) o epoll: close fd on alloc fail at initialization (1aee718 Jamie Iles) o Fix compile warning from saying event2/*.h inside a comment (447b0ba) o Warn when unable to construct base because of failing make_base_notifiable (4e797f3) o Don't try to make notifiable event_base when no threading fns are configured (e787413) BUGFIXES (evbuffer): o unit test for remove_buffer bug (90bd620 Greg Hazel) o Fix an evbuffer crash in evbuffer_remove_buffer() (c37069c) BUGFIXES (bufferevent_openssl): o Refactor amount-to-read calculations in buffervent_ssl consider_reading() (a186e73 Mark Ellzey) o Move SSL rate-limit enforcement into bytes_to_read() (96c562f) o Avoid spinning on OpenSSL reads (2aa036f Mark Ellzey) BUGFIXES (dns) o Empty DNS reply with OK status is another way to say NODATA. (21a08d6 Leonid Evdokimov) TESTING: o Tests for 94fba5b and f72e8f6 (d58c15e Leonid Evdokimov) o Test for commit aff6ba1 (f7841bf Leonid Evdokimov) o Style and comment tweaks for dns/leak* tests (5e42202) o improve test to remove at least one buffer from src (7eb52eb Greg Hazel) DOCUMENTATION: o Add note about evhttp_send_reply_end to its doxygen (724bfb5) o Update copyright dates to 2011. (3c824bd) o Fix typo in whatsnew-2.0.txt (674bc6a Mansour Moufid) o Improve win32 behavior of dns-sample.c code (a3f320e Gisle Vanem) Changes in version 2.0.15-stable (12 Oct 2011) BUGFIXES (DNS): o DNS: add ttl for negative answers using RFC 2308 idea. (f72e8f6 Leonid Evdokimov) o Add DNS_ERR_NODATA error code to handle empty replies. (94fba5b Leonid Evdokimov) BUFGIXES (bufferevents and evbuffers): o Make evbuffer callbacks get the right n_added value after evbuffer_add (1ef1f68 Alex) o Prefer mmap to sendfile unless a DRAINS_TO_FD flag is set. Allows add_file to work with SSL. (0ba0af9) BUGFIXES (event loop): o When a signal callback is activated to run multiple times, allow event_base_loopbreak to work even before they all have run. (4e8eb6a) DOCUMENTATION FIXES: o Fix docstring in dns.h (2b6eae5 Leonid Evdokimov) o refer to non-deprecated evdns functions in comments (ba5c27d Greg Hazel) BUILD AND TESTING FIXES: o le-proxy and regress depend on openssl directly (9ae061a Sergey Avseyev) o Use _SOURCES, not _sources, in sample/Makefile.am (7f82382) o Fixed compiler warnings for unchecked read/write calls. (c3b62fd Mark Ellzey) o Make write-checking fixes use tt_fail_perror (2b76847) o Fix some "value never used" warnings with gcc 4.6.1 (39c0cf7) Changes in version 2.0.14-stable (31 Aug 2011) BUGFIXES (bufferevents and evbuffers): o Propagate errors on the underlying bufferevent to the user. (4a34394 Joachim Bauch) o Ignore OpenSSL deprecation warnings on OS X (5d1b255 Sebastian Hahn) o Fix handling of group rate limits under 64 bytes of burst (6d5440e) o Solaris sendfile: correctly detect amount of data sent (643922e Michael Herf) o Make rate limiting work with common_timeout logic (5b18f13) o clear read watermark on underlying bufferevent when creating filtering bev to fix potentially failing fragmented ssl handshakes (54f7e61 Joachim Bauch) BUGFIXES (IOCP): o IOCP: don't launch reads or writes on an unconnected socket (495c227) o Make IOCP rate-limiting group support stricter and less surprising. (a98da7b) o Have test-ratelim.c support IOCP (0ff2c5a) o Make overlapped reads result in evbuffer callbacks getting invoked (6acfbdd) o Correctly terminate IO on an async bufferevent on bufferevent_free (e6af35d) BUGFIXES (other): o Fix evsig_dealloc memory leak with debugging turned on. (9b724b2 Leonid Evdokimov) o Fix request_finished memory leak with debugging turned on. (aff6ba1 Leonid Evdokimov) BUILD AND TESTING FIXES: o Allow OS-neutral builds for platforms where some versions have arc4random_buf (b442302 Mitchell Livingston) o Try to fix 'make distcheck' errors when building out-of-tree (04656ea Dave Hart) o Clean up some problems identified by Coverity. (7c11e51 Harlan Stenn) Changes in version 2.0.13-stable (18 Jul 2011) BUGFIXES o Avoid race-condition when initializing global locks (b683cae) o Fix bug in SSL bufferevents backed by a bev with a write high-watermarks (e050703 Joachim Bauch) o Speed up invoke_callbacks on evbuffers when there are no callbacks (f87f568 Mark Ellzey) o Avoid a segfault when all methods are disabled or broken (27ce38b) o Fix incorrect results from evbuffer_search_eol(EOL_LF) (4461f1a) o Add some missing checks for mm_calloc failures (89d5e09) o Replace an assertion for event_base_free(NULL) with a check-and-warn (09fe97d) o Report kqueue ebadf, epipe, and eperm as EV_READ events (1fd34ab) o Check if the `evhttp_new_object' function in `http.c' returns NULL. (446cc7a Mansour Moufid) o Use the correct printf args when formatting size_t (3203f88) o Complain if the caller tries to change threading cbs after setting them (cb6ecee) DOCUMENTATION FIXES AND IMPROVEMENTS o Revise the event/evbuffer/bufferevent doxygen for clarity and accuracy (2888fac) o Update Doxyfile to produce more useful output (aea0555) TEST FIXES o Fix up test_evutil_snprintf (caf695a) o Fix tinytest invocation from windows shell (57def34 Ed Day) BUILD FIXES o Use AM_CPPFLAGS in sample/Makefile.am, not AM_CFLAGS (4a5c82d) o Fix select.c compilation on systems with no NFDBITS (49d1136) o Fix a few warnings on OpenBSD (8ee9f9c Nicholas Marriott) o Don't break when building tests from git without python installed (b031adf) o Don't install event_rpcgen.py when --disable-libevent-install is used (e23cda3 Harlan Stenn) o Fix AIX build issue with TAILQ_FOREACH definition (e934096) Changes in version 2.0.12-stable (4 Jun 2011) BUGFIXES o Fix a warn-and-fail bug in kqueue by providing kevent() room to report errors (28317a0) o Fix an assert-inducing fencepost bug in the select backend (d90149d) o Fix failing http assertion introducd in commit 0d6622e (0848814 Kevin Ko) o Fix a bug that prevented us from configuring IPv6 nameservers. (74760f1) o Prevent size_t overflow in evhttp_htmlescape. (06c51cd Mansour Moufid) o Added several checks for under/overflow conditions in evhttp_handle_chunked_read (a279272 Mark Ellzey) o Added overflow checks in evhttp_read_body and evhttp_get_body (84560fc Mark Ellzey) DOCUMENTATION: o Add missing words to EVLOOP_NONBLOCK documentation (9556a7d) BUILD FIXES o libssl depends on libcrypto, not the other way around. (274dd03 Peter Rosin) o Libtool brings in the dependencies of libevent_openssl.la automatically (7b819f2 Peter Rosin) o Use OPENSSL_LIBS in Makefile.am (292092e Sebastian Hahn) o Move the win32 detection in configure.in (ceb03b9 Sebastian Hahn) o Correctly detect openssl on windows (6619385 Sebastian Hahn) o Fix a compile warning with zlib 1.2.4 and 1.2.5 (5786b91 Sebastian Hahn) o Fix compilation with GCC 2, which had no __builtin_expect (09d39a1 Dave Hart) o Fix new warnings from GCC 4.6 (06a714f) o Link with -lshell32 and -ladvapi32 on Win32. (86090ee Peter Rosin) o Make the tests build when OpenSSL is not available. (07c41be Peter Rosin) o Bring in the compile script from automake, if needed. (f3c7a4c Peter Rosin) o MSVC does not provide S_ISDIR, so provide it manually. (70be7d1 Peter Rosin) o unistd.h and sys/time.h might not exist. (fe93022 Peter Rosin) o Make sure TINYTEST_LOCAL is defined when building tinytest.c (8fa030c Peter Rosin) o Fix winsock2.h #include issues with MSVC (3d768dc Peter Rosin) o Use evutil_gettimeofday instead of relying on the system gettimeofday. (0de87fe Peter Rosin) o Always use evutil_snprintf, even if OS provides it (d1b2d11 Sebastian Hahn) o InitializeCriticalSectionAndSpinCount requires _WIN32_WINNT >= 0x0403. (816115a Peter Rosin) o cygwin: make it possible to build DLLs (d54d3fc) Changes in version 2.0.11-stable (27 Apr 2011) [Autogenerated from the Git log, sorted and cleaned by hand.] BUGFIXES: o Fix evport handling of POLLHUP and POLLERR (b42ce4b) o Fix compilation on Windows with NDEBUG (cb8059d) o Check for POLLERR, POLLHUP and POLLNVAL for Solaris event ports (0144886 Trond Norbye) o Detect and handle more allocation failures. (666b096 Jardel Weyrich) o Use event_err() only if the failure is truly unrecoverable. (3f8d22a Jardel Weyrich) o Handle resize failures in the select backend better. (83e805a) o Correctly free selectop fields when select_resize fails in select_init (0c0ec0b) o Make --enable-gcc-warnings a no-op if not using gcc (3267703) o Fix a type error in our (unused) arc4random_stir() (f736198) o Correctly detect and stop non-chunked http requests when the body is too long (63a715e) o Have event_base_gettimeofday_cached() always return wall-clock time (a459ef7) o Workaround for http crash bug 3078187 (5dc5662 Tomash Brechko) o Fix incorrect assertions and possible use-after-free in evrpc_free() (4b8f02f Christophe Fillot) o Reset outgoing http connection when read data in idle state. (272823f Tomash Brechko) o Fix subtle recursion in evhttp_connection_cb_cleanup(). (218cf19 Tomash Brechko) o Fix the case when failed evhttp_make_request() leaved request in the queue. (0d6622e Tomash Brechko) o Fix a crash bug in evdns server circular list code (00e91b3) o Handle calloc failure in evdns. (Found by Dave Hart) (364291e) o Fix a memory leak on win32 socket->event map. (b4f89f0) o Add a forgotten NULL check to evhttp_parse_headers (12311ff Sebastian Hahn) o Fix possible NULL-deref in evdns_cancel_request (5208544 Sebastian Hahn) PORTABILITY: o Fall back to sscanf if we have no other way to implement strtoll (453317b) o Build correctly on platforms without sockaddr_storage (9184563) o Try to build correctly on platforms with no IPv6 support (713c254) o Build on systems without AI_PASSIVE (cb92113) o Fix http unit test on non-windows platforms without getaddrinfo (6092f12) o Do not check for gethostbyname_r versions if we have getaddrinfo (c1260b0) o Include arpa/inet.h as needed on HPUX (10c834c Harlan Stenn) o Include util-internal.h as needed to build on platforms with no sockaddr_storage (bbf5515 Harlan Stenn) o Check for getservbyname even if not on win32. (af08a94 Harlan Stenn) o Add -D_OSF_SOURCE to fix hpux builds (0b33479 Harlan Stenn) o Check for allocation failures in apply_socktype_protocol_hack (637d17a) o Fix the check for multicast or broadcast addresses in evutil_check_interfaces (1a21d7b) o Avoid a free(NULL) if out-of-memory in evdns_getaddrinfo. Found by Dave Hart (3417f68) DEFENSIVE PROGRAMMING: o Add compile-time check for AF_UNSPEC==PF_UNSPEC (3c8f4e7) BUGS IN TESTS: o Fix test.sh output on solaris (b4f89b6 Dave Hart) o Make test-eof fail with a timeout if we never get an eof. (05a2c22 Harlan Stenn) o Use %s with printf in test.sh (039b9bd) o Add an assert to appease clang's static analyzer (b0ff7eb Sebastian Hahn) o Add a forgotten return value check in the unit tests (3819b62 Sebastian Hahn) o Actually send NULL request in http_bad_request_test (b693c32 Sebastian Hahn) o add some (void) casts for unused variables (65707d7 Sebastian Hahn) o Refactor test_getaddrinfo_async_cancel_stress() (48c44a6 Sebastian Hahn) o Be nice and "handle" error return values in sample code (4bac793 Sebastian Hahn) o Check return value of evbuffer_add_cb in tests (93a1abb Sebastian Hahn) o Remote some dead code from dns-example.c (744c745 Sebastian Hahn) o Zero a struct sockaddr_in before using it (646f9fe Sebastian Hahn) BUILD FIXES: o Fix warnings about AC_LANG_PROGRAM usage (f663112 Sebastian Hahn) o Skip check for zlib if we have no zlib.h (a317c06 Harlan Stenn) o Fix autoconf bracket issues; make check for getaddrinfo include netdb.h (833e5e9 Harlan Stenn) o Correct an AM_CFLAGS to an AM_CPPFLAGS in test/Makefile.am (9c469db Dave Hart) o Fix make distcheck & installation of libevent 1 headers (b5a1f9f Dave Hart) o Fix compilation under LLVM/clang with --enable-gcc-warnings (ad9ff58 Sebastian Hahn) FEATURES: o Make URI parser able to tolerate nonconformant URIs. (95060b5) DOCUMENTATION: o Clarify event_set_mem_functions doc (926f816) o Correct evhttp_del_accept_socket documentation on whether socket is closed (f665924) o fix spelling mistake in whatsnew-2.0.txt (deb2f73) o Fix sample/http-server ipv6 fixes (eb692be) o Comment internal headers used in sample code. (4eb281c) o Be explicit about how long event loops run in event.h documentation (f95bafb) o Add comment to configure.in to explain gc-sections test logic (c621359) o Fix a couple of memory leaks in samples/http-server.c. Found by Dave Hart. (2e9f665) BUILD IMPROVEMENTS: o Use the gcc -ffunction-segments feature to allow gc when linking with static libevent (0965c56 Dave Hart) o Add configure options to disable installation, regression tests (49e9bb7 Dave Hart) Changes in version 2.0.10-stable (16 Dec 2010) [Autogenerated from the Git log, sorted and cleaned by hand.] BUGFIXES o Minor fix for IOCP shutdown handling fix (2599b2d Kelly Brock) o Correctly notify the main thread when activating an event from a subthread (5beeec9) o Reject overlong http requests early when Expect:100-continue is set (d23839f Constantine Verutin) o EVUTIL_ASSERT: Use sizeof() to avoid "unused variable" warnings with -DNDEBUG. (b63ab17 Evan Jones) CODE CLEANUPS o bufferevent-internal.h: Use the new event2/util.h header, not evutil.h (ef5e65a Evan Jones) o Use relative includes instead of system includes consistently. (fbe64f2 Evan Jones) o Make whitespace more consistent TESTING o tests: Use new event2 headers instead of old compatibility headers. (4f33209 Evan Jones) DOCUMENTATION o Document that the cpu_hint is only used on Windows with IOCP for now (57689c4) o Add stuff to "whats new in 2.0" based on reading include changes since August. (18adc3f) Changes in 2.0.9-rc (30 Nov 2010): [Autogenerated from the Git log, sorted and cleaned by hand.] NEW AND MODIFIED APIs o Add a function to change a listener's callback. (46ee061) o Make evbuffer_add_file take ev_off_t, not off_t (ac7e52d) o Make rate-limits go up to SIZE_MAX/EV_SSIZE_MAX, not just INT32_MAX (2cbb1a1) o Add a bufferevent_get_base function (aab49b6) MAJOR BUGFIXES o Disable changelist for epoll by default because of Linux dup() bug; add an option and/or an envvar to reenable it for speed. (9531763) o Fix a 100%-CPU bug where an SSL connection would sometimes never stop trying to write (1213d3d) o Fix a nasty bug related to use of dup() with epoll on Linux (c281aba) o Fix bugs in posix thread-id calculation when sizeof(pthread_t) != sizeof(long) (fbaf077) o Fix some ints to evutil_socket_t; make tests pass on win64. (f817bfa Dimitre Piskyulev) o Set _EVENT_SIZEOF_VOID_P correctly on win32 and win64 (1ae82cd Dimitre Piskyulev) o Avoid double-invocation of user callback with EVUTIL_EAI_CANCEL (abf01ed) o Set SO_UPDATE_ACCEPT_CONTEXT on sockets from AcceptEx so that shutdown() can work (52aa419) o When closing a filtering bufferevent, clear callbacks on the underlying bufferevent (fc7b1b0) NEW AND MODIFIED HTTP APIs o Add evhttp_parse_query_str to be used with evhttp_uri_parse. (2075fbc) o Add evhttp_response_code to remove one more reason to include http_struct.h (22e0a9b) o Define enumerators for all HTTP methods, including PATCH from RFC5789 (75a7341 Felix Nawothnig) o Functions to actually use evhttp_bound_socket with/as evconnlistener. (006efa7) o Add evhttp_request_get_command so code can tell GET from POST without peeking at the struct. (49f4bf7) o Introduce absolute URI parsing helpers. (86dd720 Pavel Plesov) o Revise evhttp_uri_parse implementation to handle more of RFC3986 (eaa5f1d) o Add evhttp_connection_get_base() to get the event_base from an http connection (cd00079) o Let evhttp_parse_query return -1 on failure (b1756d0) o New evhttp_uri(encode|decode) functions to handle + and NUL characters right (a8148ce) o Add evhttp_response_code to remove one more reason to include http_struct.h (22e0a9b) o Tweak interface for allowed methods (f5b391e) o Add evhttp server alias interface, correct flagging of proxy requests. (aab8c38 Christopher Davis) HTTP BUGFIXES o Add some comments to http.c and make a few functions static. (90b3ed5) o Fix Content-Length when trying send more than 100GB of data (!) on an evhttp. (525da3e) o Fix a bug where we would read too much data in HTTP bodies or requests. (58a1cc6) o Correctly count req->body_size on http usage without Content-Length (8e342e5) o Avoid missed-request bug when entire http request arrives before data is flushed (74c0e86) o reset "chunked" flag when sending non-chunked reply (aa5f55f Joachim Bauch) o evhttp_encode_uri encodes all reserved characters, including !$'()*+,/:=@ (2e63a60) o Replace exact-version checks for HTTP/1.1 with >= or < checks (647e094) o evhttp: Return 501 when we get an unrecognized method, not 400. (536311a) o Don't disable reading from the HTTP connection after sending the request to be notified of connection-close in time (c76640b Felix Nawothnig) o Never call evhttp_readcb while writing. (0512487) o Try to fix an assertion failure related to close detection (0faaa39) o Correctly detect timeouts during http connects (04861d5) o Preliminary support for Continue expectation in evhttp. (fa9305f Christopher Davis) OTHER BUGFIXES o Correct logic for realigning a chain in evbuffer_add (e4f34e8) o Fix a minor syntax error that most compilers didn't care about (e56ff65) o Fix some uses of int for socket in regress (5d389dc) o Check return value for ioctlsocket on win32 (f5ad31c Trond Norbye) o Fix som event_warns that should have been event_warnx (19c71e7) o Fix signal handler types for win64. (b81217f) o Try to clear up more size_t vs int/long issues. (598d133) o Make sure IOCP evconnlistener uses virtual events. (7b40a00 Christopher Davis) o Don't free evdns_request handles until after the callback is invoked (9ed30de) o Fix some more cancel-related bugs in getaddrinfo_async (c7cfbcf) o Make evdns_getaddrinfo_cancel threadsafe (d51b2fc) o Only clear underlying callbacks when the user hasn't reset them. (1ac5b23) o Fix bug in bufferevent_connect on an openssl bufferevent that already had an fd (4f228a1) o Resolve an evport bug in the thread/forking test (3a67d0b) o Make sure the CLOEXEC flag is set on fds we open for base notification (3ab578f) o Fix IRIX build. sa_family collides with a #define in sys/socket.h on IRIX. (e874982 Kevin Bowling) o If not WIN32, include <sys/socket.h> in event2/util.h. (1cd45e5 Kevin Bowling) o Fix some C99-style comments to work with the xlC compiler. (c2e5e22 Kevin Bowling) o Add some checks since lack of TAILQ_FOREACH doesn't imply lack of FIRST, END, NEXT, or INSERT_BEFORE. Quiet some warnings in XL C. (c4dc335 Kevin Bowling) o Reworked AIX __ss_family workaround to use AC_STRUCT_MEMBER. (2e2a3d7 Kevin Bowling) o Take select from <sys/select.h> when testing in autoconf. AIX build fix. (a3a9f6b Kevin Bowling) o Fix snprintf related failures on IRIX. (3239073 Kevin Bowling) o Remove _event_initialized(); make event_initialized() a function(); make it consistent on windows and non-windows (652024b) o Do not let EVLOOP_ONCE exit the loop until all deferred callbacks have run (2d5e1bd) o Make EVLOOP_ONCE ignore internal events (0617a81) o Possible crash fix when freeing an underlying bufferevent of an openssl bufferevent (29f7623) HTTP CLEANUPS o Stop using Libevent-1 headers in regress_http (1f507d7) o Modernize header usage in bench_http.c (e587069) o fix signed/unsigned warnings in http.c (74a91e5) o Update the HTTP regression tests to use Libevent2 apis for non-http stuff (d9ffa89) o Start porting http tests to not use legacy interfaces (8505a74) o Convert the rest of the http tests to be non-legacy unit tests. (9bb8239) o Rename the confusing "base" static variable in regress_http.c (353402a) o Stop accessing http request struct directly from in the unit tests. (0b137f4) o Refactor http version parsing into a single function (a38140b) TESTING o Improvements to tinytest_macros.h (ad923a1) o Add a huge pile of tests for the new URI functions, and make them pass. (a5a76e6) o Unit tests for evhttp_uri_set* (bc98f5e) o Increase the skew tolerance to 2 seconds in thread/deferred_cb_skew (f806476 Christopher Davis) o Reorder backends in test.sh to match preference order in event.c (ece974f) o Add a stress test for getaddrinfo_cancel (da1bf52) o Units test for unexpected evhttp methods. (75e3320) DOCUMENTATION o Document behavior of URI parsing more thoroughly. (3a33462) o Document that two bufferevent functions only work on socket bufferevents (70e1b60) o add a requested docstring for event_rpcgen.CommandLine.__init__ (f1250eb) o Fix a mistake in http documentation found by Julien Blache (229714d) o Add a basic example of how to write a static HTTP server. (4e794d5) o Document event_get_assignment (88be27d) o Note that reentrant calls to libevent from logging cbs may fail badly (e431bcd) o Clarify EVLOOP_* documentation to be more precise. (057a514) CLEANUPS o Simplify the logic for choosing EPOLL_CTL_ADD vs EPOLL_CTL_MOD (2c66983) o Rename "size" variables in win32select that were really fd counts. (b6a158c) o Fix even more win64 warnings (7484df6) o Fix even more win64 warnings: buffer, event_tagging, http, evdns, evrpc (545a611) o Fix more wn64 warnings. (34b84b9 Christopher Davis) o Use the label_len local variable in evdns instead of recalculating it over and over (ba01456) o Fix some irix compilation warnings spotted by Kevin Bowling (7bcace2) Changes in 2.0.8-rc (14 Oct 2010): [Autogenerated from the Git log, sorted and cleaned by hand.] NEW APIS o Add error callback to evconnlistener (c4be8d8 Simon Perreault) o Add a LEV_OPT_THREADSAFE option for threadsafe evconnlisteners (127d4f2) CHANGED BEHAVIOR o Correct logic on disabling underlying bufferevents when disabling a filter (ac27eb8) BUGFIXES o Obey enabled status when unsuspending (040a019 Simon Perreault) o Warn when using the error-prone EV_SIGNAL interface in an error-prone way. Also, fix a couple of race conditions in signal.c (720bd93) O Make default signal backend fully threadsafe (95a7d41) o Put internal events at highest priority (90651b3) o Fix warnings in the main codebase flagged by -Wsigned-compare (9c8db0, 5e4bafb, 5c214a, 6be589a, e06f514) o Fix compile in kqueue.c (b395392 Sebastian Hahn) o Do not search outside of the system directory for windows DLLs (d49b5e3) o Fix a spurious-call bug on epoll.c (0faaee0) o Send a shutdown(SHUT_WR) before closing an http connection (e0fd870 Christopher Davis) o Fix warnings on mingw with gcc 4.5 (5b7a370) o Fix an EINVAL on evbuffer_write_iovec on OpenSolaris. (fdc640b) o Fix allocation error for IOCP listeners. Probably harmless, since struct event is big (481ef92) o Make iocp/listener/error work; don't accept again if lev is disabled. (62b429a Christopher Davis) o Handle rate-limiting for reading on OpenSSL bufferevents correctly. (819b171) o Fix serious bugs in per-bufferevent rate-limiting code (34d64f8) o Avoid spurious reads from just-created open openssl bufferevents (223ee40) o Fix a case where an ssl bufferevent with CLOSE_ON_FREE didn't close its fd (93bb7d8) o The corrected bufferevent filter semantics let us fix our openssl tests (34331e4) TESTING o Make SSL tests cover enabling/disabling EV_READ. (a5ce9ad) o Bump to the latest version of tinytest (f0bd83e) o Unit tests for listener error callbacks (045eef4) o New unit test for ssl bufferevents starting with connected SSLs. (02f6259) DEBUGGABILITY o Make debugging output for epoll backend more comprehensive (ec2b05e) o Make event.c debugging messages report fds (e119899) o Make the --enable-gcc-warnings option include signed comparison warnings (d3b096c) DEADCODE REMOVAL o Remove the now-useless evsig_caught and evsig_process (4858b79) o Remove event_base.evsigbase; nothing used it. (38d0960) Changes in 2.0.7-rc (9 Sep 2010): [Autogenerated from the Git log, sorted and cleaned by hand.] NEW APIS o Expose a evdns_base_nameserver_sockaddr_add() function to add a nameserver by sockaddr (1952143) o Add event_config_set_num_cpus_hint() for tuning win32 IOCP thread pools, etc. (2447fe8 Christopher Davis) BUGFIXES o Fix a nasty dangling-event bug when using rate-limiting groups (0bffe43) o Clean up syntax on TAILQ_ENTRY() usage to build correctly with recent MSVC (60433a0 Gilad Benjamini) o Make definition of WIN32_LEAN_AND_MEAN in event.h conditional (3920172 Gilad Benjamini) o Correctly detect failure to delete bufferevent read-timeout event (da6e7cd) o Set close-on-exec bit for filedescriptors created by dns subsystem (d0b8843) o Fix kqueue correctness test on x84_64 (6123d12) o Detect events with no ev_base; warn instead of crashing (f1074b7) o Fix an issue with forking and signal socketpairs in select/poll backends (d61b2f3) o Stop using global arrays to implement the EVUTIL_ctype functions (1fdec20) o On windows, make lock/thread function tables static (5de2bcb) o Close th_notify_fds and open a new pair on reinit (495ed66) o Declare signal handler function as "__cdecl" on Windows (f0056d0) o Use the _func() replacements for open, fstat, etc in evutil.c on win32 (e50c0fc) o Only process up to MAX_DEFERRED deferred_cbs at a time (17a14f1 Christopher Davis) THREADING BUGFIXES o Avoid deadlock when activating signals (970e6ad) o Add a condition variable backend, with implementations for pthreads and win32 (d4977b5) o Use conditions instead of current_event_lock to fix a deadlock (e0972c2) o Fix logic error in win32 TRY_LOCK that caused problems with rate-limiting (4c32b9d) o Avoid needlessly calling evthread_notify_base() when the loop is not running (c7a06bf) o Minimize calls to base_notify implementation functions, thereby avoiding needless syscalls (4632b78) IOCP BUGFIXES o IOCP-related evbuffer fixes (03afa20 Christopher Davis) o Stop IOCP when freeing the event_base (d844242 Christopher Davis) o Some IOCP bufferevent tweaks (76f7e7a Christopher Davis) TESTS o Make the regress_pthread.c tests work on windows with current test APIs (d74ae38) o Add a unit test for conditions (5fb1095) o Allow more than one copy of regression tests to run at once (a97320a) o Fix event_del(0) instance in bench.c (b0f284c Shuo Chen) o Fix a few memory leaks in the tests (1115366) o IOCP-related unit test tweaks (499452f Christopher Davis) o Improve testing of when thread-notification occurs (ce85280) BUILD AND DISTRIBUTION o Add pkgconfig files for libevent_{openssl,pthreads} (ebcb1f0) o Change include order in Makefile.nmake (4022b28) o Make include/event2/event-config.h not included in source dist (a4af9be) o Honor NDEBUG; build without warnings with NDEBUG; make NDEBUG always-off in unit test code (743f866) o Declare evkeyvalq and event_list even if event_struct.h comes before sys/queue.h (d3ceca8) o Move evkeyvalq into a separate header for evhttp_parse_query users (ca9048f) o Prefer autoreconf -ivf to manual autogen.sh (7ea8e89) CLEANUP o Completely remove the (mostly-removed) obsolete thread functions (3808168) o Rename regress_pthread.c to regress_thread.c (041989f) o Make defer-internal.h use lock macros, not direct calls to lock fns (5218d2a) DOCUMENTATION o Document that DNS_NO_SEARCH is an obsolete alias for DNS_QUERY_NO_SEARCH (33200e7) o Update the whatsnew-2.0.txt document (4991669) Changes in 2.0.6-rc (6 Aug 2010): [Autogenerated from the Git log, sorted by hand.] DOCUMENTATION o Document a change in the semantics of event_get_struct_event_size() (e21f5d1) o Add a comment to describe our plan for library versioning (9659ece) o Fix sentence fragment in docs for event_get_struct_event_size() (7b259b6) NEW FEATURES AND INTERFACE CHANGES o Remove the obsolete evthread interfaces (c5bab56) o Let evhttp_send_error infer the right error reasons (3990669) o Add a function to retrieve the other side of a bufferevent pair (17a8e2d) o Add bufferevent_lock()/bufferevent_unlock() (215e629) o Stop asserting when asked for a (unsupported) TCP dns port. Just return NULL. (7e87a59) o Replace (unused,always 0) is_tcp argument to evdns_add_server_port*() with flags (e1c1167) o Constify a couple of arguments to evdns_server_request_add_*_reply (cc2379d) o Add an interface to expose min_share in ratelimiting groups (6ae53d6) BUGFIXES o Avoid event_del on uninitialized event in event_base_free (6d19510) o Add some missing includes to fix Linux build again (75701e8) o Avoid close of uninitialized socket in evbuffer unit test (bda21e7) o Correctly recognize .255 addresses as link-local when looking for interfaces (8c3452b) o If no evdns request can be launched, return NULL, not a handle (b14f151) o Use generic win32 interfaces, not ASCII-only ones, where possible. (899b0a3) o Fix the default HTTP error template (06bd056 Felix Nawothnig) o Close the file in evutil_read_file whether there's an error or not. (0798dd1 Pierre Phaneuf) o Fix possible nullptr dereference in evhttp_send_reply_end() (29b2e23 Felix Nawothnig) o never let bufferevent_rlim functions return negative (0859870) o Make sample/hello_world work on windows (d89fdba) o Fix a deadlock related to event-base notification. Diagnosed by Zhou Li, Avi Bab, and Scott Lamb. (17522d2) o Possible fix to 100% cpu usage with epoll and openssl (cf249e7 Mike Smellie) o Don't race when calling event_active/event_add on a running signal event (fc5e0a2) o Suppress a spurious EPERM warning in epoll.c (e73cbde) o Fix wrong size calculation of iovec buffers when exact=1 (65abdc2 niks) o Change bufferevent_openssl::do_write so it doesn't call SSL_write with a 0 length buffer (c991317 Mike Smellie) o Fixed compilation of sample/le-proxy.c on win32 (13b912e Trond Norbye) o Fix rate-limit calculation on openssl bufferevents. (009f300) o Remember to initialize timeout events for bufferevent_async (de1f5d6 Christopher Davis) BUILD AND DISTRIBUTION CHANGES o Test the unlocked-deferred callback case of bufferevents (dfb75ab) o Remove the now-unusable EVTHREAD_LOCK/UNLOCK constants (fdfc3fc) o Use -Wlogical-op on gcc 4.5 or higher (d14bb92) o Add the libtool-generated /m4/* stuff to .gitignore (c21c663) o Remove some automake-generated files from version control. (9b14911) o Have autogen.sh pass --force-missing to automake (8a44062) o Set library version for libevent_pthreads correctly (b2d7440) o Really only add libevent_core.la to LIBADD on mingw (1425003 Sebastian Hahn) o Build more cleanly with NetBSDs that dislike toupper(char) (42a8c71) o Fix unit tests with -DUSE_DEBUG enabled (28f31a4) o Fix evdns build with -DUNICODE (5fa30d2) o Move event-config.h to include/event2 (ec347b9) TESTING o Add options to test-ratelim.c to check its results (2b44dcc) o Make test-ratelim clean up after itself better. (b5bfc44) o Remove the now-obsolete setup_test() and cleanup_test() functions (e73f1d7) o Remove all non-error prints from test/regress.c (8bc1e3d) o Make test.sh exit with nonzero status if tests fail (faf2a04) o Have the unit tests report errors from test.sh (3689bd2) o Fix logic in correcting high values from FIONREAD (3467f2f) o Add test for behavior on remote socket close (44d57ee) o Unit test for event_get_struct_event_size() (7510aac) o Make test/test.sh call test-changelist (7c92691) o Fix badly-behaved subtest of dns/bufferevent_connect_hostname (840a72f Joachim Bauch) o Add option to test-ratelim to test min_share (42f6b62) o Fix an assertion bug in test-ratelim (b2c6202) o Make tests quieter on local dns resolver failure (e996b3d) o Increase the tolerance in our unit tests for sloppy clocks. (170ffd2) o Use AF_INET socketpair to test sendfile on Solaris (9b60209) o Make test-changelist count cpu usage right on win32 (ea1ea3d) INTERNALS, PERFORMANCE, AND CODE CLEANUPS o Mark the event_err() functions as __attribute__((noreturn)) (33bbbed) o Do not check that event_base is set in EVBASE_ACQUIRE_LOCK (218a3c3) o Replace (safe) use of strcpy with memcpy to appease OpenBSD (caca2f4) o Remove some dead assignments (47c5dfb) o Fix a pedantic gcc 4.4 warning in event2/event.h (276e7ee) o Drain th_notify_fd[0] more bytes at a time. (a5bc15b) o Tidy up the code in evthread_make_base_notifiable a little (61e1eee) o Pass flags to fcntl(F_SETFL) and fcntl(F_SETFD) as int, not long (7c2dea1) o Remove unused variables in test/test-changelist.c (b00d4c0) o Fix whitespace. (cb927a5) o Improve error message for failed epoll to make debugging easier. (9e725f7) o Turn our socketpair() replacement into its own function (57b30cd) Changes in 2.0.5-beta (10 May 2010): [Autogenerated from the Git log, sorted by hand.] DOCUMENTATION o Update all our copyright notices to say "2010" (17efc1c) o Add Christopher Clark and Maxim Yegorushkin to the LICENSE file (38b7b57) o Clarify Christopher Clark's status as writer of original ht code. (78772c3) o Try to comment some of the event code more (cdd4c49) o Add a few more evmap/changelist comments (c247adc) o Add a comment to explain why evdns_request is now separte from request (ceefbe8) o Document evutil_secure_rng_init() and evutil_secure_rng_add_bytes() (a5bf43a) o Stop distributing and installing manpages: they were too inaccurate (7731ec8) NEW FEATURES AND INTERFACE CHANGES o Remove signal_assign() and signal_new() macros. (2fac0f7) o Make evdns use the regular logging system by default (b2f2be6) o Allow evbuffer_read() to split across more than 2 iovecs (e470ad3) o Functions to manipulate existing rate limiting groups. (ee41aca) o Functions to track the total bytes sent over a rate limit group. (fb366c1) o Detect and refuse reentrant event_base_loop() calls (b557b17) o Limit the maximum number of events on each socket to 65535 (819f949) o Add evbuffer_copyout to copy data from an evbuffer without draining (eb86c8c) o Expose the request and reply members of rpc_req_generic() (07edf78 Shuo Chen) o Add void* arguments to request_new and reply_new evrpc hooks (755fbf1 Shuo Chen) o Seed the RNG using sysctl() as well as /dev/urandom (71fc3eb) o Make evutil_secure_rng_init() work even with builtin arc4random (f980716) o Report DNS error when lookup fails during bufferevent_socket_connect_hostname. (0ef4070 Christopher Davis) o Release locks on bufferevents while executing callbacks (a5208fe Joachim Bauch) o Make debug mode catch mixed ET and non-ET events on an fd (cb67074) o Catch attempts to enable debug_mode too late (9ecf0d4) o Refuse null keys in evhttp_parse_query() (953e229 Frank Denis) BUGFIXES o Avoid a spurious close(-1) on Linux (70a44b6) o Do not close(-1) when freeing an uninitialized socket bufferevent (b34abf3) o Free evdns_base->req_heads on evdns_base_free (859af67) o Avoid an (untriggerable so far) crash bug in bufferevent_free() (0cf1431) o Set mem_offset for every bufferevent type (657d1b6) o Fix infrequent memory leak in bufferevent_init_common(). (8398641 Jardel Weyrich) o Make evutil_signal_active() match declaration. (e1e703d Patrick Galbraith) o Fix minheap code to use replacement malloc functions (a527618) o Fix a free(NULL) in minheap-internal.h (6f20492) o Fix critical bug in evbuffer_write when writev is not available (cda56ab) o Make the no_iovecs case of write_atmost compile (8e227b0) o Fix a memory leak when appending/prepending to a buffer with unused space. (45068a3) o Clean up a mistake in pointer manipulation in evbuffer_remove (28bfed4 Christopher Davis) o Always round up when there's a fractional number of msecs. (8f9e60c Christopher Davis) o Fix compiler warnings under WIN32 (d469c50 Giuseppe Scrivano) o Clean up properly when adding a signal handler fails. (b84b598 Gilad Benjamini) o Ensure that evdns_request is a persistent handle. (15bb82d Christopher Davis) o Free search state when finished searching to avoid an infinite loop. (a625840 Christopher Davis) o Assert for valid requests as necessary. (67072f3 Christopher Davis) o do not leak the request object on persistent connections (9d8edf2) o Make evdns logging threadsafe (b1c7950) o Fix a couple of bugs in the BSD sysctl arc4seed logic (a47a4b7) o Remove one last bug in last_with_datap logic. Found with valgrind (d49b92a) o fix a leak when unpausing evrpc requests (94ee125) o Fix a memory leak when unmarshalling RPC object arrays (f6ab2a2) o Fix compilation when openssl support is disabled (40c301b) o Allow empty reason line in HTTP status (739e688 Pierre Phaneuf) o Fix a compile warning introduced in 739e688 (bd1ed5f Sebastian Hahn) o Fix nonstandard TAILQ_FOREACH_REVERSE() definition (71afc52 Frank Denis) o Try /proc on Linux as entropy fallback; use sysctl as last resort (20fda29) o Fix symbol conflict between mm_*() macros and libmm (99e50e9) o Fix some crazy macro mistakes in arc4random.c (90d4225) o Make evbuffer_add_file() work on windows (dcdae6b) o Fix unused-variable warning when building with threads disabled (ad811cd) o Numerous opensolaris compilation fixes (c44de06) o Fix getaddrinfo with protocol unset on Solaris 9. Found by Dagobert Michelsen (2cf2a28) o Fix another nasty solaris getaddrinfo() behavior (3557071) o Define _REENTRANT as needed on Solaris, elsewhere (c1cd32a) o Fix some autoconf issues on OpenBSD (7c519df) BUILD AND DISTRIBUTION CHANGES o Distribute libevent.pc.in, not libevent.pc (22aff04) o Avoid errors in evutil.c when building with _UNICODE defined (b677032 Brodie Thiesfield) o Avoid errors in http.c when building with VC 2003 .NET (13e4f3b Brodie Thiesfield) o Support the standard 'make check' target in place of 'make verify' (426c8fb) o Remove redundant stuff from EXTRA_DIST (b660edf) o Switch to using AM conditionals in place of AC_LIBOBJ (2e898f5) o Remove an orphaned RELEASE flag in Makefile.am (0794b0d) o Give a better warning for bad automake versions. (77c917d) o Use dist_bin_SCRIPTS, not EXTRA_DIST, to distribute scripts (9eb2fd7) o Never test for select() on windows (3eb044d Trond Norbye) o Do not inhibit automake dependencies generation (10c4c90 Giuseppe Scrivano) o Create shared libraries under Windows (3cbca86 Giuseppe Scrivano) o Add ctags/etags files to .gitignore (0861d17) o Only specify -no-undefined on mingw (25433b9) o Only add libevent_core.la to LIBADD on mingw (fdc6297) TESTING o Get bench_http to work on Windows; add a switch to enable IOCP. (4ac38a5 Christopher Davis) o VC has no getopt(), so do without in bench_http. (1273d2f Christopher Davis) o Fix an obnoxious typo in the bufferevent_timeout_filter test (0d047c3) o Fix a write of uninitialized RAM in regression tests (68dc742) o Fix some memory leaks in the unit tests (274a7bd) o Make 'main/many_events' test 70 fds, not 64. (33874b0) o Unit-test every evbuffer_add_file() implementation. (06a4443) o Add more unit tests for evbuffer_expand (8c83e99) o Test another case of evbuffer_prepend (1234b95) o Fix a possible double-free bug in SSL bufferevents with CLOSE_ON_FREE (7501895) o Add dns/search_cancel unit test. (39b870b Christopher Davis) o Make http_base_test stop leaking an event_base. (96730d3) o Detect broken unsetenv at unit-test runtime (f37cd4c) o Implement regress_make_tempfile on win32 to test evbuffer_add_file (b4f12a1) o add more (currently skipped) add_file tests on win32 (05de45d) o Fix bench_http build on win32. (384d124) o Make unit test for add_file able to tell "error" from "done" (88a543f) o Make test for bufferevent_connect_hostname system-neutral (f89168e) o Make test.sh support mingw/msys on win32 (0ee6f6c) o Fix test.sh on freebsd (3d9e05b) INTERNALS, PERFORMANCE, AND AND CODE CLEANUPS o Improve the speed of evbuffer_readln() (cc1600a) o more whitespace normalization (2c2618d) o Revise evbuffer to add last_with_data (2a6d2a1) o Use last_with_data in place of previous_to_last (c8ac57f) o Remove previous_to_last from evbuffer (6f47bd1) o Fix last_with_data compilation on windows (1e7b986) o Add some glass-box tests for the last_with_data code. (17da042) o Improve robustness for refcounting (f1bc125) o Remove a needless min_heap_shift_up_() call (7204b91) o Increase MIN_BUFFER_SIZE to 512 (1024 on 64-bit) (2014ae4) o Do not use evbuffer_expand() to add the first chain to a buffer (5c0ebb3) o Make evbuffer_prepend handle empty buffers better (c87272b) o Replace last_with_data with a slightly smarter version (b7442f8) o Turn the increasingly complex *_CHAIN() macros into functions (96865c4) o Rewrite evbuffer_expand and its users (d5ebcf3) o Add evutil_tv_to_msec for safe conversion of timevals to milliseconds. (850c3ff Christopher Davis) o Initialize last_with_datap correctly in evbuffer_overlapped (a0983b6) o Replace EVUTIL_CLOSESOCKET macro with a function (899c1dc Sebastian Sjöberg) o Move domain search state to evdns_request. (beaa14a Christopher Davis) o Remove redundant checks for lock!=NULL before calling EVLOCK_LOCK (50ec59f) o Rename current_base symbol to event_global_current_base_ (c16e684) o Fix whitespace in evutil.c (935e150) o Replace users of "int fd" with "evutil_socket_t fd" in portable code (c7cf6f0) Changes in 2.0.4-alpha (28 Feb 2010): [Autogenerated from the Git log, sorted by hand.] DOCUMENTATION o Add stub header for 2.0.4-alpha changelog. (94d0065) o Improve the README with more information and links. (0b42726) o Add more people who wrote patches to the acknowledgments (0af10d5) o Add a warning about the use of event_initialized. (f32b575) o Add a LICENSE file so people can find our license easily (7067006) o Add a new "hello world" sample program (becb9f9) o Clarify status of example programs (d60a1bd) o Update time-test.c to use event2 (f4190bf) o Add the arc4random.c license to the LICENSE file. (e15e1e9) NEW FEATURES AND INTERFACE CHANGES o Improved optional lock debugging. (0cd3bb9) o Rate-limiting for bufferevents; group and individual limits are supported. (737c9cd) o Testing code for bufferevent rate-limiting. (f0c0124) o Make the initial nameserver probe timeout configurable. (1e56a32) o Revise the locking API: deprecate the old locking callbacks and add trylock. (347952f) o Do not make bufferevent_setfd implicitly disable EV_READ and EV_WRITE. (8274379) o Do not ignore bufferevent_enable(EV_READ) before bufferevent_connect(). (4a5b534) o Introduced evutil_make_socket_closeonexec() to preserve fd flags for F_SETFD. (d0939d2 Jardel Weyrich) o evdns_getaddrinfo() now supports the /etc/hosts file. (72dd666) o Look at the proper /etc/hosts file on windows. (66c02c7) o Allow http connections to use evdns for hostname looksups. (c698b77) o Changelist code to defer event changes until just before dispatch (27308aa) o do not use a function to assign the evdns base; instead assign it via evhttp_connection_base_new() which is a new function introduced in 2.0 (5032e52) o Functions to access more fields of struct event. (0683950) o Make kqueue use changelists. (45e5ae3) o Remove kqueue->pend_changes. (3225dfb) o Minimize epoll_ctl calls by using changelist (c8c6a89) o Add support for a "debug mode" to try to catch common errors. (cd17c3a) o Note a missing ratelim function (361da8f) o Add ev_[u]intptr_t to include/event2/util.h (1fa4c81) o const-ify a few more functions in event.h (d38a7a1) o Deprecate EVENT_FD and EVENT_SIGNAL. (f6b2694) o Remove EVUTIL_CHECK_FMT. (6c21c89) o Add EV_*_MAX macros to event2/util.h to expose limits for ev_* types. (aba1fff) o Functions to view and manipulate rate-limiting buckets. (85047a6) o Add the rest of the integer limits, and add a test for them. (60742d5) o Remove the 'flags' argument from evdns_base_set_option() (1dd7e6d) o Add an arc4random implementation for use by evdns (d4de062) o Use off_t for the length parameter of evbuffer_add_file (3fe60fd) o Construct Windows locks using InitializeCriticalSectionAndSpinCount (32c6f1b) o Expose view of current rate limit as constrained by group limit (162ce8a) o Provide consistent, tested semantics for bufferevent timeouts (d328829) BUGFIXES AND TESTS o Tolerate code that returns from a fatal_cb. (91fe23f) o Parenthesize macro arguments more aggressively (07e9e9b) o Fix memory-leak of signal handler array with kqueue. (e1ffbb8) o Stop passing EVTHREAD_READ and EVTHREAD_WRITE to non-rw locks. (76cd2b7) o Fix two use-after-free bugs in unit tests spoted by lock debugging (d84d838) o Fix a locking bug in event_base_loop() (da1718b) o Fix an evdns lock violation. (2df1f82 Zhuang Yuyao) o Valgrind fix: Clear struct kevent before checking for OSX bug. (56771a3 William Ahern) o Fix up evthread compilation on windows (bd6f1ba Roman Puls) o Fix regress_iocp.c usage of old lock allocation macros. (31687b4 unknown) o Update nmake makefile to build evthread.c (b62d979 unknown) o Fix a crash when reading badly formatted resolve.conf; from Yasuoka Masahiko (6c7c579 Yasuoka Masahiko) o Fix a snow leopard compile warning in the unit tests. (7ae9445) o Fix compile on Snow Leopard with gcc warnings enabled (70cdfe4 Sebastian Hahn) o Only define _GNU_SOURCE if it is not already defined. (ea6b1df Joachim Bauch) o Update sample/signal-test.c to use newer APIs and not leak. (f6430ac Evan Jones) o Fix a segfault when writing a very fragmented evbuffer onto an SSL (a6adeca Joachim Bauch) o Fix a segfault when freeing SSL bufferevents in an unusual order (a773df5 Joachim Bauch) o Drop install-sh from our git repo: a mismatched version could break "make dist" (6799527) o Set all instances of the version number correctly. (5a112d3) o Fix a few locking issues on windows. (c51bb3c unknown) o Use evutil_socket_t, not int, when logging socket errors. (292467c) o Fix up behavior of never-defered callbacks a little (390e056) o Replace some cases of uint32_t with ev_uint32_t. (a47d88d) o Fix compilation of devpoll.c by adding missing thread includes. (fee2c77 Dagobert Michelsen) o Make evutil_make_socket_nonblocking() leave any other flags alone. (4c8b7cd Jardel Weyrich) o Fix an fd leak in evconnlistener_new_bind(). (24fb502 Jardel Weyrich) o Fix a bogus free in evutil_new_addrinfo() (0d64051 Jardel Weyrich) o Adjusted fcntl() retval comparison on evutil_make_socket_nonblocking(). (4df7dbc Jardel Weyrich) o Fix the code that allowed DNS options to not end with : (ee4953f) o Fix crash bugs when a bufferevent's eventcb is not set. (2e8eeea) o Fix test-ratelim compilation on Linux. (885b427) o Fix compilation of rate-limiting code on win32. (165d30e) o Eradicated the last free() call. Let mm_free() take care of deallocation. (0546ce1 Jardel Weyrich) o Fix byte counts when mixing deferred and non-deferred evbuffer callbacks. (29151e6) o Fixed a memory leak on windows threads implementation. The CRITICAL_SECTION was not being free'd in evthread_win32_lock_free(). (2f33e00 Jardel Weyrich) o Fixed a fd leak in start_accepting(), plus cosmetic changes (4367a33 Jardel Weyrich) o Improved error handling in evconnlistener_new_async(). Also keeping the fd open because it is not opened by this function, so the caller is responsible for closing it. Additionally, since evconnlistener_new_bind() creates a socket and passes it to the function above, it required error checking to close the same socket. (fec66f9 Jardel Weyrich) o Don't use a bind address for nameservers on loopback (8d4aaf9) o Fix compilation of rate-limit code when threading support is disabled (97a8c79) o Detect setenv/unsetenv; skip main/base_environ test if we can't fake them. (7296971) o Check more internal event_add() calls for failure (ff3f6cd) o Fix windows and msvc build (5c7a7bc) o Call event_debug_unassign on internal events (a19b4a0) o Try to fix a warning in hash_debug_entry (137f2c6) o Fix a dumb typo in ev_intptr_t definitions. (27c9a40) o do not fail while sending on http connections the client closed. (93d7369) o make evhttp_send() safe against terminated connections, too (3978180) o Make Libevent 1.4.12 build on win32 with Unicode enabled. (000a33e Brodie Thiesfield) o Fix some additional -DUNICODE issues on win32. (a7a9431) o Add a check to make soure our EVUTIL_AI flags do not conflict with the native ones (c18490e) o Always use our own gai_strerror() replacement. (6810bdb) o Make RNG work when we have arc4random() but not arc4random_buf() (4ec8fea) o validate close cb on server when client connection closes (2f782af) o Fix two unlocked reads in evbuffer. (7116bf2) o When working without a current event base, don't try to use IOCP listeners (cb52838) o Fix getpid() usage on Windows (ff2a134) o Add a unit test for secure rng. (48a29b6) o Add some headers to fix freebsd compilation (b72be50) o When connect() succeeds immediately, don't invoke the callback immediately. (7515de9) o Suspend read/write on bufferevents during hostname lookup (db08f64) o Make bufferevent_free() clear all callbacks immediately. (b2fbeb3) o Fix some race conditions in persistent events and event_reinit (e2642f0) o Fix a bug in resetting timeouts on persistent events when IO triggers. (38ec0a7) o Add a test for timeouts on filtering bufferevents. (c02bfe1) o Add test for periodic timers that get activated for other reasons (8fcb7a1) o Use new timeval diff comparison function in bufferevent test (f3dfe46) o Delete stack-alloced event in new unit test before returning. (7ffd387) o Fix mingw compilation (23170a6) o Try to define a sane _EVENT_SIZEOF_SIZE_T for msvc compilation (1e14f82) o Fix arc4random compilation on MSVC. (98edb89) o deal with connect() failing immediately (7bc48bf) o Small cleanups on freebsd-connect-refused patch. (57b7248) BUILD AND DISTRIBUTION CHANGES o Remove the contents of WIN32-Prj as unmaintained. (c69d5a5) o Allow the user to redirect the verbose output of test/test.sh to a file (c382de6) o Allow test.sh to be run as ./test/test.sh (7dfbe94) o Never believe that we have pthreads on win32, even if gcc thinks we do. (78ed097) o Make it compile under gcc --std=c89. (e2ca403) o Fix a number of warnings from gcc -pedantic (918e9c5) o Add the msvc-generated .lib files to .gitignore. (e244a2e) o Add the "compile" script to gitignore. (1ba6bed) INTERNALS AND CODE CLEANUPS o Add a .gitignore file. (ba34071) o New EVTHREAD_TRY_LOCK function to try to grab a lock. (689fc09) o Add the abilitity to mark some buffer callbacks as never-deferred. (438f9ed) o Refactor our 'suspend operation' logic on bufferevents. (0d744aa) o Simplify the read high-watermark checking. (5846bf6) o Improve readability of evutil_unparse_protoname() (5a43df8 Jardel Weyrich) o Expose our cached gettimeofday value with a new interface (47854a8) o Whitespace fixes in test.sh (0b151a9) o Enable branch-prediction hints with EVUTIL_UNLIKELY. (eaaf27f) o Refactor code from evdns into a new internal "read a file" function. (0f7144f) o Comestic changes in evconnlistener_new(), new_accepting_socket(), accepted_socket_invoke_user_cb() and iocp_listener_enable(). (510ab6b Jardel Weyrich) o Add unit-test for bad_request bug fixed in 1.4 recently. (6cc79c6 Pavel Plesov) o Add a comment on evthread_enable_lock_debuging. (b9f43b2) o Fix test.sh on shells without echo -n (94131e9) o More unit tests for getaddrinfo_async: v4timeout and cancel. (a334b31) o Make http use evconnlistener. (ec34533) o move dns utility functions into a separate file so that we can use them for http testing (b822639) o add a test for evhttp_connection_base_new with a dns_base (26714ca) o forgot to add void to test function (78a50fe) o Add a forgotten header (changelist-internal.h) (4b9f307) o Remove some commented-out code in evutil (26e1b6f) o Remove a needless include of rpc_compat.h (70a4a3e) o Use less memory for each entry in a hashtable (a66e947) o Try to untangle the logic in server_port_flush(). (439aea0) o Use ev_[u]intptr_t types in place of [u]intptr_t (cef61a2) o Reduce windows header includes in our own headers. (da6135e) o clean up terminate_chunked test (e8a9782) o Increment the submicro version number. (63e868e) o Update event-config.h version number to match configure.in (aae7db5) o Clean up formatting: Disallow space-before-tab. (8fdf09c) o Clean up formatting: use tabs, not 8-spaces, to indent. (e5bbd40) o Clean up formatting: remove trailing spaces (e5cf987) o Clean up formatting: function/keyword spacing consistency. (4faeaea) Changes in 2.0.3-alpha (20 Nov 2009): o Add a new code to support SSL/TLS on bufferevents, using the OpenSSL library (where available). o Fix a bug where we didn't allocate enough memory in event_get_supported_methods(). o Avoid segfault during failed allocation of locked evdns_base. (Found by Rocco Carbone.) o Export new evutil_ascii_* functions to perform locale-independent character type operations. o Try to compile better with MSVC: patches from Brodie Thiesfield o New evconnlistener_get_fd function to expose a listener's associated socket. o Expose an ev_socklen_t type for consistent use across platforms. o Make bufferevent_socket_connect() work when the original fd was -1. o Fix a bug in bufferevent_socket_connect() when the connection succeeds too quickly. o Export an evutil_sockaddr_cmp() to compare to sockaddr objects for equality. o Add a bufferevent_get_enabled() to tell what a bufferevent has been configured to do. o Add an evbuffer_search_eol() function to locate the end of a line nondestructively. o Add an evbuffer_search_range() function to search a bounded range of a buffer. o Fix a rare crash bug in evdns. o Have bufferevent_socket_connect() with no arguments put a bufferevent into connecting mode. o Support sendfile on Solaris: patch from Caitlin Mercer. o New functions to explicitly reference a socket used by an evhttp object. Patches from David Reiss. o When we send a BEV_EVENT_CONNECTED to indicate connected status, we no longer invoke the write callback as well unless we actually wrote data too. o If the kernel tells us that there are a negative number of bytes to read from a socket, do not believe it. Fixes bug 2841177; found by Alexander Pronchenkov. o Do not detect whether we have monotonic clock support every time a new event base is created: instead do it only once. Patch taken from Chromium. o Do not allocate the maximum event queue for the epoll backend at startup. Instead, start out accepting 32 events at a time, and double the queue's size when it seems that the OS is generating events faster than we're requesting them. Saves up to 374K per epoll-based event_base. Resolves bug 2839240. o Treat an event with a negative fd as valid but untriggerable by Libevent. This is useful for applications that want to manually activate events. o Fix compilation on Android, which forgot to define fd_mask in its sys/select.h o Do not drop data from evbuffer when out of memory; reported by Jacek Masiulaniec o New event_base_got_exit() and event_base_got_break() functions to tell whether an event loop exited because of an event_base_loopexit() or an event_base_loopbreak(). Patch from Ka-Hing Cheung. o When adding or deleting an event from a non-main thread, only wake up the main thread when its behavior actually needs to change. o Fix some bugs when using the old evdns interfaces to initialize the evdns module. o Detect errors during bufferevent_connect(). Patch from Christopher Davis. o Fix compilation for listener.h for C++ - missing extern "C". Patch from Ferenc Szalai. o Make the event_base_loop() family of functions respect thread-safety better. This should clear up a few hard-to-debug race conditions. o Fix a bug when using a specialized memory allocator on win32. o Have the win32 select() backend label TCP-socket-connected events as EV_WRITE, not EV_READ. This should bring it in line with the other backends, and improve portability. Patch from Christopher Davis. o Stop using enums as arguments or return values when what we mean is a bitfield of enum values. C++ doesn't believe that you can OR two enum values together and get another enum, and C++ takes its typing seriously. Patch from Christopher Davis. o Add an API to replace all fatal calls to exit() with a user-provided panic function. o Replace all assert() calls with a variant that is aware of the user-provided logging and panic functions. o Add a return value to event_assign so that it can fail rather than asserting when the user gives it bad input. event_set still dies on bad input. o The event_base_new() and event_base_new_with_config() functions now never call exit() on failure. For backward "compatibility", event_init() still does, but more consistently. o Remove compat/sys/_time.h. It interfered with system headers on HPUX, and its functionality has been subsumed by event2/util.h and util-internal.h. o Add a new bufferevent_socket_connect_hostname() to encapsulate the resolve-then-connect operation. o Build kqueue.c correctly on GNU/kFreeBSD platforms. Patch pulled upstream from Debian. o Alternative queue-based timeout algorithm for programs that use a large number of timeouts with the same value. o New event_base_config option to disable the timeval cache entirely. o Make EV_PERSIST timeouts more accurate: schedule the next event based on the scheduled time of the previous event, not based on the current time. o Allow http.c to handle cases where getaddrinfo returns an IPv6 address. Patch from Ryan Phillips. o Fix a problem with excessive memory allocation when using multiple event priorities. o Default to using arc4random for DNS transaction IDs on systems that have it; from OpenBSD. o Never check the environment when we're running setuid or setgid; from OpenBSD. o Options passed to evdns_set_option() no longer need to end with a colon. o Add an evutil_getaddrinfo() function to clone getaddrinfo on platforms that don't have it. o Add an evdns_getaddrinfo() function to provide a nonblocking getaddrinfo using evdns, so programs can perform useful hostname lookup. o Finally expose the IOCP-based bufferevent backend. It passes its unit tests, but probably still has some bugs remaining. Code by Nick Mathewson and Christopher Davis. o Numerous other bugfixes. o On FreeBSD and other OSes, connect can return ECONREFUSED immediately; instead of failing the function call, pretend with faileld in the callback. o Fix a race condition in the pthreads test case; found by Nick Mathewson o Remove most calls to event_err() in http and deal with memory errors instead Changes in 2.0.2-alpha (25 Jul 2009): o Add a new flag to bufferevents to make all callbacks automatically deferred. o Make evdns functionality locked, and automatically defer dns callbacks. o Fix a possible free(NULL) when freeing an event_base with no signals. o Add a flag to disable checking environment varibles when making an event_base o Disallow setting less than 1 priority. o Fix a bug when removing a timeout from the heap. [Patch from Marko Kreen] o Use signal.h, not sys/signal.h. [Patch from mmadia] o Try harder to build with certain older c99 compilers. o Make sure that an event_config's flags field is always initialized to 0. [Bug report from Victor Goya] o Avoid data corruption when reading data entirely into the second-to-last chain of an evbuffer. [Bug report from Victor Goya] o Make sendfile work on FreeBSD o Do not use vararg macros for accessing evrpc structures; this is not backwards compatible, but we did not promise any backwards compatibility for the rpc code. o Actually define the event_config_set_flag() function. o Try harder to compile with Visual C++. o Move event_set() and its allies to event2/event_compat.h where they belong. o Remove the event_gotsig code, which has long been deprecated and unused. o Add an event_get_base() function to return the base assigned to an event. o New function to automate connecting on a socket-based bufferevent. o New functions to automate listening for incoming TCP connections. o Do case-insensitive checks with a locale-independent comparison function. o Rename the evbuffercb and everrorcb callbacks to bufferevent_data_cb and bufferevent_event_cb respectively. The old names are available in bufferevent_compat.h. o Rename the EVBUFFER_* codes used by bufferevent event callbacks to BEV_EVENT_*, to avoid namespace collision with evbuffer flags. The old names are available in bufferevent_compat.h. o Move the EVBUFFER_INPUT and EVBUFFER_OUTPUT macros to bufferevent_compat.h o Add a bufferevent_getfd() function to mirror bufferevent_setfd() o Make bufferevent_setfd() return an error code if the operation is not successful. o Shave 22 bytes off struct event on 32-bit platforms by shrinking and re-ordering fields. The savings on 64-bit platforms is likely higher. o Cap the maximum number of priorities at 256. o Change the semantics of evbuffer_cb_set_flags() to be set-flag only; add a new evbuffer_cb_clear_flags() to remove set flags. o Change the interface of evbuffer_add_reference so that the cleanup callback gets more information o Revise the new evbuffer_reserve_space/evbuffer_commit_space() interfaces so that you can use them without causing extraneous copies or leaving gaps in the evbuffer. o Add a new evbuffer_peek() interface to inspect data in an evbuffer without removing it. o Fix a deadlock when suspending reads in a bufferevent due to a full buffer. (Spotted by Joachim Bauch.) o Fix a memory error when freeing a thread-enabled event base with registered events. (Spotted by Joachim Bauch.) o Try to contain degree of failure when running on a win32 version so heavily firewalled that we can't fake a socketpair. o Activate fd events in a pseudorandom order with O(N) backends, so that we don't systematically favor low fds (select) or earlier-added fds (poll, win32). o Replace some read()/write() instances with send()/recv() to work properly on win32. o Set truncated flag correctly in evdns server replies. o Raise RpcGenError in event_rpcgen.py; from jmanison and Zack Weinberg o Fix preamble of rpcgen-generated files to rely on event2 includes; based on work by jmansion; patch from Zack Weinberg. o Allow specifying the output filename for rpcgen; based on work by jmansion; patch from Zack Weinberg. o Allow C identifiers as struct names; allow multiple comments in .rpc files; from Zack Weinberg o Mitigate a race condition when using socket bufferevents in multiple threads. o Use AC_SEARCH_LIBS, not AC_CHECK_LIB to avoid needless library use. o Do not allow event_del(ev) to return while that event's callback is executing in another thread. This fixes a nasty race condition. o event_get_supported_methods() now lists methods that have been disabled with the EVENT_NO* environment options. o Rename encode_int[64] to evtag_encode_int[64] to avoid polluting the global namespace. The old method names are still available as macros in event2/tag_compat.h. Changes in 2.0.1-alpha (17 Apr 2009): o free minheap on event_base_free(); from Christopher Layne o debug cleanups in signal.c; from Christopher Layne o provide event_base_new() that does not set the current_base global o bufferevent_write now uses a const source argument; report from Charles Kerr o improve documentation on event_base_loopexit; patch from Scott Lamb o New function, event_{base_}loopbreak. Like event_loopexit, it makes an event loop stop executing and return. Unlike event_loopexit, it keeps subsequent pending events from getting executed. Patch from Scott Lamb o Check return value of event_add in signal.c o provide event_reinit() to reintialize an event_base after fork o New function event_set_mem_functinons. It allows the user to give libevent replacement functions to use for memory management in place of malloc(), free(), etc. This should be generally useful for memory instrumentation, specialized allocators, and so on. o The kqueue implementation now catches signals that are raised after event_add() is called but before the event_loop() call. This makes it match the other implementations. o The kqueue implementation now restores original signal handlers correctly when its signal events are removed. o Check return value of event_add in signal.c o Add a more powerful evbuffer_readln as a replacement for evbuffer_readline. The new function handles more newline styles, and is more useful with buffers that may contain a nul characters. o Do not mangle socket handles on 64-bit windows. o The configure script now takes an --enable-gcc-warnigns option that turns on many optional gcc warnings. (Nick has been building with these for a while, but they might be useful to other developers.) o move EV_PERSIST handling out of the event backends o small improvements to evhttp documentation o always generate Date and Content-Length headers for HTTP/1.1 replies o set the correct event base for HTTP close events o When building with GCC, use the "format" attribute to verify type correctness of calls to printf-like functions. o Rewrite win32.c backend to be O(n lg n) rather than O(n^2). o Removed obsoleted recalc code o support for 32-bit tag numbers in rpc structures; this is wire compatible, but changes the API slightly. o pull setters/getters out of RPC structures into a base class to which we just need to store a pointer; this reduces the memory footprint of these structures. o prefix {encode,decode}_tag functions with evtag to avoid collisions o fix a bug with event_rpcgen for integers o Correctly handle DNS replies with no answers set (Fixes bug 1846282) o add -Wstrict-aliasing to warnings and more cleanup o removed linger from http server socket; reported by Ilya Martynov o event_rpcgen now allows creating integer arrays o support string arrays in event_rpcgen o change evrpc hooking to allow pausing of RPCs; this will make it possible for the hook to do some meaning ful work; this is not backwards compatible. o allow an http request callback to take ownership of a request structure o allow association of meta data with RPC requests for hook processing o associate more context for hooks to query such as the connection object o remove pending timeouts on event_base_free() o also check EAGAIN for Solaris' event ports; from W.C.A. Wijngaards o devpoll and evport need reinit; tested by W.C.A Wijngaards o event_base_get_method; from Springande Ulv o Send CRLF after each chunk in HTTP output, for compliance with RFC2626. Patch from "propanbutan". Fixes bug 1894184. o Add a int64_t parsing function, with unit tests, so we can apply Scott Lamb's fix to allow large HTTP values. o Use a 64-bit field to hold HTTP content-lengths. Patch from Scott Lamb. o Allow regression code to build even without Python installed o remove NDEBUG ifdefs from evdns.c o detect integer types properly on platforms without stdint.h o udpate documentation of event_loop and event_base_loop; from Tani Hosokawa. o simplify evbuffer by removing orig_buffer o do not insert event into list when evsel->add fails o add support for PUT/DELETE requests; from Josh Rotenberg o introduce evhttp_accept_socket() to accept from an already created socket o include Content-Length in reply for HTTP/1.0 requests with keep-alive o increase listen queue for http sockets to 128; if that is not enough the evhttp_accpet_socket() api can be used with a prepared socket. o Patch from Tani Hosokawa: make some functions in http.c threadsafe. o test support for PUT/DELETE requests; from Josh Rotenberg o rewrite of the evbuffer code to reduce memory copies o Some older Solaris versions demand that _REENTRANT be defined to get strtok_r(); do so. o Do not free the kqop file descriptor in other processes, also allow it to be 0; from Andrei Nigmatulin o Provide OpenSSL style support for multiple threads accessing the same event_base o make event_rpcgen.py generate code include event-config.h; reported by Sam Banks. o switch thread support so that locks get allocated as they are needed. o make event methods static so that they are not exported; from Andrei Nigmatulin o make RPC replies use application/octet-stream as mime type o do not delete uninitialized timeout event in evdns o Correct the documentation on buffer printf functions. o Don't warn on unimplemented epoll_create(): this isn't a problem, just a reason to fall back to poll or select. o Correctly handle timeouts larger than 35 minutes on Linux with epoll.c. This is probably a kernel defect, but we'll have to support old kernels anyway even if it gets fixed. o Make name_from_addr() threadsafe in http.c o Add new thread-safe interfaces to evdns functions. o Make all event_tagging interfaces threadsafe. o Rename internal memory management functions. o New functions (event_assign, event_new, event_free) for use by apps that want to be safely threadsafe, or want to remain ignorant of the contents of struct event. o introduce bufferevent_read_buffer; allows reading without memory copy. o expose bufferevent_setwatermark via header files and fix high watermark on read o fix a bug in buffrevent read water marks and add a test for them o fix a bug in which bufferevent_write_buffer would not schedule a write event o provide bufferevent_input and bufferevent_output without requiring knowledge of the structure o introduce bufferevent_setcb and bufferevent_setfd to allow better manipulation of bufferevents o convert evhttp_connection to use bufferevents. o use libevent's internal timercmp on all platforms, to avoid bugs on old platforms where timercmp(a,b,<=) is buggy. o Remove the never-exported, never-used evhttp_hostportfile function. o Support input/output filters for bufferevents; somewhat similar to libio's model. This will allow us to implement SSL, compression, etc, transparently to users of bufferevents such as the http layer. o allow connections to be removed from an rpc pool o add new evtimer_assign, signal_assign, evtimer_new, and signal_new functions to manipulate timer and signal events, analagous to the now-recommended event_assign and event_new o switch internal uses of event_set over to use event_assign. o introduce evbuffer_contiguous_space() api that tells a user how much data is available in the first buffer chain o introduce evbuffer_reserve_space() and evbuffer_commit_space() to make processing in filters more efficient. o reduce system calls for getting current time by caching it. o separate signal events from io events; making the code less complex. o support for periodic timeouts o support for virtual HTTP hosts. o turn event_initialized() into a function, and add function equivalents to EVENT_SIGNAL and EVENT_FD so that people don't need to include event_struct.h o Build test directory correctly with CPPFLAGS set. o Provide an API for retrieving the supported event mechanisms. o event_base_new_with_config() and corresponding config APIs. o migrate the evhttp header to event2/ but accessors are still missing. o deprecate timeout_* event functions by moving them to event_compat.h o Move windows gettimeofday replacement into a new evutil_gettimeofday(). o Make configure script work on IRIX. o provide a method for canceling ongoing http requests. o Make vsnprintf() returns consistent on win32. o Fix connection keep-alive behavior for HTTP/1.0 o Fix use of freed memory in event_reinit; pointed out by Peter Postma o constify struct timeval * where possible o make event_get_supported_methods obey environment variables o support for edge-triggered events on epoll and kqueue backends: patch from Valery Kholodkov o support for selecting event backends by their features, and for querying the features of a backend. o change failing behavior of event_base_new_with_config: if a config is provided and no backend is selected, return NULL instead of aborting. o deliver partial data to request callbacks when chunked callback is set even if there is no chunking on the http level; allows cancelation of requests from within the chunked callback; from Scott Lamb. o allow min_heap_erase to be called on removed members; from liusifan. o Rename INPUT and OUTPUT to EVRPC_INPUT and EVRPC_OUTPUT. Retain INPUT/OUTPUT aliases on on-win32 platforms for backwards compatibility. o Do not use SO_REUSEADDR when connecting o Support 64-bit integers in RPC structs o Correct handling of trailing headers in chunked replies; from Scott Lamb. o Support multi-line HTTP headers; based on a patch from Moshe Litvin o Reject negative Content-Length headers; anonymous bug report o Detect CLOCK_MONOTONIC at runtime for evdns; anonymous bug report o Various HTTP correctness fixes from Scott Lamb o Fix a bug where deleting signals with the kqueue backend would cause subsequent adds to fail o Support multiple events listening on the same signal; make signals regular events that go on the same event queue; problem report by Alexander Drozdov. o Fix a problem with epoll() and reinit; problem report by Alexander Drozdov. o Fix off-by-one errors in devpoll; from Ian Bell o Make event_add not change any state if it fails; reported by Ian Bell. o Fix a bug where headers arriving in multiple packets were not parsed; fix from Jiang Hong; test by me. o Match the query in DNS replies to the query in the request; from Vsevolod Stakhov. o Add new utility functions to correctly observe and log winsock errors. o Do not remove Accept-Encoding header o Clear the timer cache on entering the event loop; reported by Victor Chang o Only bind the socket on connect when a local address has been provided; reported by Alejo Sanchez o Allow setting of local port for evhttp connections to support millions of connections from a single system; from Richard Jones. o Clear the timer cache when leaving the event loop; reported by Robin Haberkorn o Fix a typo in setting the global event base; reported by lance. o Set the 0x20 bit on outgoing alphabetic characters in DNS requests randomly, and insist on a match in replies. This helps resist DNS poisoning attacks. o Make the http connection close detection work properly with bufferevents and fix a potential memory leak associated with it. o Restructure the event backends so that they do not need to keep track of events themselves, as a side effect multiple events can use the same fd or signal. o Add generic implementations for parsing and emiting IPv6 addresses on platforms that do not have inet_ntop and/or inet_pton. o Allow DNS servers that have IPv6 addresses. o Add an evbuffer_write_atmost() function to write a limited number of bytes to an fd. o Refactor internal notify-main-thread logic to prefer eventfd to pipe, then pipe to socketpair, and only use socketpairs as a last resort. o Try harder to pack all evbuffer reads into as few chains as possible, using readv/WSARecv as appropriate. o New evthread_use_windows_threads() and evthread_use_pthreads() functions to set up the evthread callbacks with reasonable defaults. o Change the semantics of timeouts in conjunction with EV_PERSIST; timeouts in that case will now repeat until deleted. o sendfile, mmap and memory reference support for evbuffers. o New evutil_make_listen_socket_reuseable() to abstract SO_REUSEADDR. o New bind-to option to allow DNS clients to bind to an arbitrary port for outgoing requests. o evbuffers can now be "frozen" to prevent operations at one or both ends. o Bufferevents now notice external attempts to add data to an inbuf or remove it from an outbuf, and stop them. o Fix parsing of queries where the encoded queries contained \r, \n or + o Do not allow internal events to starve lower-priority events. �������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/Doxyfile�����������������������������������������������������0000664�0000000�0000000�00000023736�15270523712�0021517�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Doxyfile 1.5.1 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project # # All text after a hash (#) is considered a comment and will be ignored # The format is: # TAG = value [value, ...] # For lists items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (" ") #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # The PROJECT_NAME tag is a single word (or a sequence of words surrounded # by quotes) that should identify the project. PROJECT_NAME = libevent # Place all output under 'doxygen/' OUTPUT_DIRECTORY = doxygen/ # If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen # will interpret the first line (until the first dot) of a JavaDoc-style # comment as the brief description. If set to NO, the JavaDoc # comments will behave just like the Qt-style comments (thus requiring an # explicit @brief command for a brief description. JAVADOC_AUTOBRIEF = YES # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C # sources only. Doxygen will then generate output that is more tailored for C. # For instance, some of the names that are used will be different. The list # of all members will be omitted, etc. OPTIMIZE_OUTPUT_FOR_C = YES # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the # brief documentation of file, namespace and class members alphabetically # by member name. If set to NO (the default) the members will appear in # declaration order. SORT_BRIEF_DOCS = YES # If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag # can be used to strip a user-defined part of the path. Stripping is # only done if one of the specified strings matches the left-hand part of # the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the # path to strip. STRIP_FROM_PATH = include/ #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag can be used to specify the files and/or directories that contain # documented source files. You may enter file names like "myfile.cpp" or # directories like "/usr/src/myproject". Separate the files or directories # with spaces. INPUT = \ include/event2/buffer.h \ include/event2/buffer_compat.h \ include/event2/bufferevent.h \ include/event2/bufferevent_compat.h \ include/event2/bufferevent_ssl.h \ include/event2/dns.h \ include/event2/dns_compat.h \ include/event2/event.h \ include/event2/event_compat.h \ include/event2/http.h \ include/event2/http_compat.h \ include/event2/listener.h \ include/event2/rpc.h \ include/event2/rpc_compat.h \ include/event2/tag.h \ include/event2/tag_compat.h \ include/event2/thread.h \ include/event2/util.h #--------------------------------------------------------------------------- # configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES (the default) Doxygen will # generate HTML output. GENERATE_HTML = YES #--------------------------------------------------------------------------- # configuration options related to the LaTeX output #--------------------------------------------------------------------------- # If the GENERATE_LATEX tag is set to YES (the default) Doxygen will # generate Latex output. GENERATE_LATEX = YES # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `latex' will be used as the default path. LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. If left blank `latex' will be used as the default command name. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to # generate index for LaTeX. If left blank `makeindex' will be used as the # default command name. MAKEINDEX_CMD_NAME = makeindex # If the COMPACT_LATEX tag is set to YES Doxygen generates more compact # LaTeX documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_LATEX = NO # The PAPER_TYPE tag can be used to set the paper type that is used # by the printer. Possible values are: a4, a4wide, letter, legal and # executive. If left blank a4wide will be used. PAPER_TYPE = a4wide # The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX # packages that should be included in the LaTeX output. EXTRA_PACKAGES = # The LATEX_HEADER tag can be used to specify a personal LaTeX header for # the generated latex document. The header should contain everything until # the first chapter. If it is left blank doxygen will generate a # standard header. Notice: only use this tag if you know what you are doing! LATEX_HEADER = # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated # is prepared for conversion to pdf (using ps2pdf). The pdf file will # contain links (just like the HTML output) instead of page references # This makes the output suitable for online browsing using a pdf viewer. PDF_HYPERLINKS = NO # If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of # plain latex in the generated Makefile. Set this option to YES to get a # higher quality PDF documentation. USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. # command to the generated LaTeX files. This will instruct LaTeX to keep # running if errors occur, instead of asking the user for help. # This option is also used when generating formulas in HTML. LATEX_BATCHMODE = NO # If LATEX_HIDE_INDICES is set to YES then doxygen will not # include the index chapters (such as File Index, Compound Index, etc.) # in the output. LATEX_HIDE_INDICES = NO #--------------------------------------------------------------------------- # configuration options related to the man page output #--------------------------------------------------------------------------- # If the GENERATE_MAN tag is set to YES (the default) Doxygen will # generate man pages GENERATE_MAN = NO # The MAN_EXTENSION tag determines the extension that is added to # the generated man pages (default is the subroutine's section .3) MAN_EXTENSION = .3 # If the MAN_LINKS tag is set to YES and Doxygen generates man output, # then it will generate one additional man file for each entity # documented in the real man page(s). These additional files # only source the real man page, but without them the man command # would be unable to find the correct page. The default is NO. MAN_LINKS = YES #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- # If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will # evaluate all C-preprocessor directives found in the sources and include # files. ENABLE_PREPROCESSING = YES # If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro # names in the source code. If set to NO (the default) only conditional # compilation will be performed. Macro expansion can be done in a controlled # way by setting EXPAND_ONLY_PREDEF to YES. MACRO_EXPANSION = NO # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES # then the macro expansion is limited to the macros specified with the # PREDEFINED and EXPAND_AS_DEFINED tags. EXPAND_ONLY_PREDEF = NO # If the SEARCH_INCLUDES tag is set to YES (the default) the includes files # in the INCLUDE_PATH (see below) will be search if a #include is found. SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by # the preprocessor. INCLUDE_PATH = # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard # patterns (like *.h and *.hpp) to filter out the header-files in the # directories. If left blank, the patterns specified with FILE_PATTERNS will # be used. INCLUDE_FILE_PATTERNS = # The PREDEFINED tag can be used to specify one or more macro names that # are defined before the preprocessor is started (similar to the -D option of # gcc). The argument of the tag is a list of macros of the form: name # or name=definition (no spaces). If the definition and the = are # omitted =1 is assumed. To prevent a macro definition from being # undefined via #undef or recursively expanded use the := operator # instead of the = operator. PREDEFINED = TAILQ_ENTRY RB_ENTRY EVENT_DEFINED_TQENTRY_ EVENT_IN_DOXYGEN_ # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then # this tag can be used to specify a list of macro names that should be expanded. # The macro definition that is found in the sources will be used. # Use the PREDEFINED tag if you want to use a different macro definition. EXPAND_AS_DEFINED = # If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then # doxygen's preprocessor will remove all function-like macros that are alone # on a line, have an all uppercase name, and do not end with a semicolon. Such # function macros are typically used for boiler-plate code, and will confuse # the parser if not removed. SKIP_FUNCTION_MACROS = YES ����������������������������������kytensor-2.49.0.7/third_party/libevent/LICENSE������������������������������������������������������0000664�0000000�0000000�00000010660�15270523712�0021006�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Libevent is available for use under the following license, commonly known as the 3-clause (or "modified") BSD license: ============================== Copyright (c) 2000-2007 Niels Provos <provos@citi.umich.edu> Copyright (c) 2007-2012 Niels Provos and Nick Mathewson Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ============================== Portions of Libevent are based on works by others, also made available by them under the three-clause BSD license above. The copyright notices are available in the corresponding source files; the license is as above. Here's a list: log.c: Copyright (c) 2000 Dug Song <dugsong@monkey.org> Copyright (c) 1993 The Regents of the University of California. strlcpy.c: Copyright (c) 1998 Todd C. Miller <Todd.Miller@courtesan.com> win32select.c: Copyright (c) 2003 Michael A. Davis <mike@datanerds.net> evport.c: Copyright (c) 2007 Sun Microsystems ht-internal.h: Copyright (c) 2002 Christopher Clark minheap-internal.h: Copyright (c) 2006 Maxim Yegorushkin <maxim.yegorushkin@gmail.com> ============================== The arc4module is available under the following, sometimes called the "OpenBSD" license: Copyright (c) 1996, David Mazieres <dm@uun.org> Copyright (c) 2008, Damien Miller <djm@openbsd.org> Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ============================== The Windows timer code is based on code from libutp, which is distributed under this license, sometimes called the "MIT" license. Copyright (c) 2010 BitTorrent, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ��������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/Makefile.am��������������������������������������������������0000664�0000000�0000000�00000021176�15270523712�0022041�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# Makefile.am for libevent # Copyright 2000-2007 Niels Provos # Copyright 2007-2012 Niels Provos and Nick Mathewson # # See LICENSE for copying information. # 'foreign' means that we're not enforcing GNU package rules strictly. # '1.9' means that we need automake 1.9 or later (and we do). AUTOMAKE_OPTIONS = foreign 1.9 subdir-objects ACLOCAL_AMFLAGS = -I m4 # This is the "Release" of the Libevent ABI. It takes precedence over # the VERSION_INFO, so that two versions of Libevent with the same # "Release" are never binary-compatible. # # This number incremented once for the 2.0 release candidate, and # will increment for each series until we revise our interfaces enough # that we can seriously expect ABI compatibility between series. # RELEASE = -release 2.1 # This is the version info for the libevent binary API. It has three # numbers: # Current -- the number of the binary API that we're implementing # Revision -- which iteration of the implementation of the binary # API are we supplying? # Age -- How many previous binary API versions do we also # support? # # To increment a VERSION_INFO (current:revision:age): # If the ABI didn't change: # Return (current:revision+1:age) # If the ABI changed, but it's backward-compatible: # Return (current+1:0:age+1) # If the ABI changed and it isn't backward-compatible: # Return (current+1:0:0) # # Once an RC is out, DO NOT MAKE ANY ABI-BREAKING CHANGES IN THAT SERIES # UNLESS YOU REALLY REALLY HAVE TO. VERSION_INFO = 6:2:0 # History: RELEASE VERSION_INFO # 2.0.1-alpha -- 2.0 1:0:0 # 2.0.2-alpha -- 2:0:0 # 2.0.3-alpha -- 2:0:0 (should have incremented; didn't.) # 2.0.4-alpha -- 3:0:0 # 2.0.5-beta -- 4:0:0 # 2.0.6-rc -- 2.0 2:0:0 # 2.0.7-rc -- 2.0 3:0:1 # 2.0.8-rc -- 2.0 4:0:2 # 2.0.9-rc -- 2.0 5:0:0 (ABI changed slightly) # 2.0.10-stable-- 2.0 5:1:0 (No ABI change) # 2.0.11-stable-- 2.0 6:0:1 (ABI changed, backward-compatible) # 2.0.12-stable-- 2.0 6:1:1 (No ABI change) # 2.0.13-stable-- 2.0 6:2:1 (No ABI change) # 2.0.14-stable-- 2.0 6:3:1 (No ABI change) # 2.0.15-stable-- 2.0 6:3:1 (Forgot to update :( ) # 2.0.16-stable-- 2.0 6:4:1 (No ABI change) # 2.0.17-stable-- 2.0 6:5:1 (No ABI change) # 2.0.18-stable-- 2.0 6:6:1 (No ABI change) # 2.0.19-stable-- 2.0 6:7:1 (No ABI change) # 2.0.20-stable-- 2.0 6:8:1 (No ABI change) # 2.0.21-stable-- 2.0 6:9:1 (No ABI change) # # For Libevent 2.1: # 2.1.1-alpha -- 2.1 1:0:0 # 2.1.2-alpha -- 2.1 1:0:0 (should have been 2:0:1) # 2.1.3-alpha -- 2.1 3:0:0 (ABI changed slightly) # 2.1.4-alpha -- 2.1 4:0:0 (ABI changed slightly) # 2.1.5-beta -- 2.1 5:0:0 (ABI changed slightly) # 2.1.6-beta -- 2.1 6:0:0 (ABI changed slightly) # 2.1.7-beta -- 2.1 6:1:0 (ABI changed slightly) # 2.1.8-stable-- 2.1 6:2:0 (ABI changed slightly) # ABI version history for this package effectively restarts every time # we change RELEASE. Version 1.4.x had RELEASE of 1.4. # # Ideally, we would not be using RELEASE at all; instead we could just # use the VERSION_INFO field to label our backward-incompatible ABI # changes, and those would be few and far between. Unfortunately, # Libevent still exposes far too many volatile structures in its # headers, so we pretty much have to assume that most development # series will break ABI compatibility. For now, it's simplest just to # keep incrementing the RELEASE between series and resetting VERSION_INFO. # # Eventually, when we get to the point where the structures in the # headers are all non-changing (or not there at all!), we can shift to # a more normal worldview where backward-incompatible ABI changes are # nice and rare. For the next couple of years, though, 'struct event' # is user-visible, and so we can pretty much guarantee that release # series won't be binary-compatible. if INSTALL_LIBEVENT dist_bin_SCRIPTS = event_rpcgen.py endif pkgconfigdir=$(libdir)/pkgconfig LIBEVENT_PKGCONFIG=libevent.pc libevent_core.pc libevent_extra.pc # These sources are conditionally added by configure.ac or conditionally # included from other files. PLATFORM_DEPENDENT_SRC = \ arc4random.c \ epoll_sub.c EXTRA_DIST = \ ChangeLog-1.4 \ ChangeLog-2.0 \ Doxyfile \ LICENSE \ Makefile.nmake test/Makefile.nmake \ autogen.sh \ event_rpcgen.py \ libevent.pc.in \ make-event-config.sed \ whatsnew-2.0.txt \ whatsnew-2.1.txt \ $(PLATFORM_DEPENDENT_SRC) LIBEVENT_LIBS_LA = libevent.la libevent_core.la libevent_extra.la if PTHREADS LIBEVENT_LIBS_LA += libevent_pthreads.la LIBEVENT_PKGCONFIG += libevent_pthreads.pc endif if OPENSSL LIBEVENT_LIBS_LA += libevent_openssl.la LIBEVENT_PKGCONFIG += libevent_openssl.pc endif if INSTALL_LIBEVENT lib_LTLIBRARIES = $(LIBEVENT_LIBS_LA) pkgconfig_DATA = $(LIBEVENT_PKGCONFIG) else noinst_LTLIBRARIES = $(LIBEVENT_LIBS_LA) endif EXTRA_SOURCE= noinst_HEADERS= noinst_PROGRAMS= EXTRA_PROGRAMS= CLEANFILES= DISTCLEANFILES= BUILT_SOURCES = include include/include.am include sample/include.am include test/include.am if BUILD_WIN32 SYS_LIBS = -lws2_32 -lshell32 -ladvapi32 SYS_SRC = win32select.c buffer_iocp.c event_iocp.c \ bufferevent_async.c SYS_INCLUDES = -IWIN32-Code -IWIN32-Code/nmake if THREADS SYS_SRC += evthread_win32.c endif else SYS_LIBS = SYS_SRC = SYS_INCLUDES = endif if STRLCPY_IMPL SYS_SRC += strlcpy.c endif if SELECT_BACKEND SYS_SRC += select.c endif if POLL_BACKEND SYS_SRC += poll.c endif if DEVPOLL_BACKEND SYS_SRC += devpoll.c endif if KQUEUE_BACKEND SYS_SRC += kqueue.c endif if EPOLL_BACKEND SYS_SRC += epoll.c endif if EVPORT_BACKEND SYS_SRC += evport.c endif if SIGNAL_SUPPORT SYS_SRC += signal.c endif BUILT_SOURCES += include/event2/event-config.h include/event2/event-config.h: config.h make-event-config.sed $(AM_V_GEN)test -d include/event2 || $(MKDIR_P) include/event2 $(AM_V_at)$(SED) -f $(srcdir)/make-event-config.sed < config.h > $@T $(AM_V_at)mv -f $@T $@ CORE_SRC = \ buffer.c \ bufferevent.c \ bufferevent_filter.c \ bufferevent_pair.c \ bufferevent_ratelim.c \ bufferevent_sock.c \ event.c \ evmap.c \ evthread.c \ evutil.c \ evutil_rand.c \ evutil_time.c \ listener.c \ log.c \ $(SYS_SRC) EXTRAS_SRC = \ evdns.c \ event_tagging.c \ evrpc.c \ http.c if BUILD_WITH_NO_UNDEFINED NO_UNDEFINED = -no-undefined MAYBE_CORE = libevent_core.la else NO_UNDEFINED = MAYBE_CORE = endif GENERIC_LDFLAGS = -version-info $(VERSION_INFO) $(RELEASE) $(NO_UNDEFINED) libevent_la_SOURCES = $(CORE_SRC) $(EXTRAS_SRC) libevent_la_LIBADD = @LTLIBOBJS@ $(SYS_LIBS) libevent_la_LDFLAGS = $(GENERIC_LDFLAGS) libevent_core_la_SOURCES = $(CORE_SRC) libevent_core_la_LIBADD = @LTLIBOBJS@ $(SYS_LIBS) libevent_core_la_LDFLAGS = $(GENERIC_LDFLAGS) if PTHREADS libevent_pthreads_la_SOURCES = evthread_pthread.c libevent_pthreads_la_LIBADD = $(MAYBE_CORE) libevent_pthreads_la_LDFLAGS = $(GENERIC_LDFLAGS) endif libevent_extra_la_SOURCES = $(EXTRAS_SRC) libevent_extra_la_LIBADD = $(MAYBE_CORE) $(SYS_LIBS) libevent_extra_la_LDFLAGS = $(GENERIC_LDFLAGS) if OPENSSL libevent_openssl_la_SOURCES = bufferevent_openssl.c libevent_openssl_la_LIBADD = $(MAYBE_CORE) $(OPENSSL_LIBS) libevent_openssl_la_LDFLAGS = $(GENERIC_LDFLAGS) libevent_openssl_la_CPPFLAGS = $(AM_CPPFLAGS) $(OPENSSL_INCS) endif noinst_HEADERS += \ WIN32-Code/nmake/evconfig-private.h \ WIN32-Code/nmake/event2/event-config.h \ WIN32-Code/tree.h \ bufferevent-internal.h \ changelist-internal.h \ compat/sys/queue.h \ defer-internal.h \ epolltable-internal.h \ evbuffer-internal.h \ evconfig-private.h \ event-internal.h \ evmap-internal.h \ evrpc-internal.h \ evsignal-internal.h \ evthread-internal.h \ ht-internal.h \ http-internal.h \ iocp-internal.h \ ipv6-internal.h \ kqueue-internal.h \ log-internal.h \ minheap-internal.h \ mm-internal.h \ ratelim-internal.h \ ratelim-internal.h \ strlcpy-internal.h \ time-internal.h \ util-internal.h EVENT1_HDRS = \ include/evdns.h \ include/event.h \ include/evhttp.h \ include/evrpc.h \ include/evutil.h if INSTALL_LIBEVENT include_HEADERS = $(EVENT1_HDRS) else noinst_HEADERS += $(EVENT1_HDRS) endif AM_CPPFLAGS = -I$(srcdir)/compat -I$(srcdir)/include -I./include $(SYS_INCLUDES) verify: check doxygen: FORCE doxygen $(srcdir)/Doxyfile FORCE: DISTCLEANFILES += *~ libevent.pc libevent_core.pc libevent_extra.pc ./include/event2/event-config.h ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/Makefile.nmake�����������������������������������������������0000664�0000000�0000000�00000004447�15270523712�0022541�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# WATCH OUT! This makefile is a work in progress. -*- makefile -*- # # I'm not very knowledgeable about MSVC and nmake beyond their most basic # aspects. If anything here looks wrong to you, please let me know. # If OPENSSL_DIR is not set, builds without OpenSSL support. If you want # OpenSSL support, you can set the OPENSSL_DIR variable to where you # installed OpenSSL. This can be done in the environment: # set OPENSSL_DIR=c:\openssl # Or on the nmake command line: # nmake OPENSSL_DIR=C:\openssl -f Makefile.nmake # Or by uncommenting the following line here in the makefile... # OPENSSL_DIR=c:\openssl !IFDEF OPENSSL_DIR SSL_CFLAGS=/I$(OPENSSL_DIR)\include /DEVENT__HAVE_OPENSSL !ELSE SSL_CFLAGS= !ENDIF # Needed for correctness CFLAGS=/IWIN32-Code /IWIN32-Code/nmake /Iinclude /Icompat /DHAVE_CONFIG_H /I. $(SSL_CFLAGS) # For optimization and warnings CFLAGS=$(CFLAGS) /Ox /W3 /wd4996 /nologo # XXXX have a debug mode LIBFLAGS=/nologo CORE_OBJS=event.obj buffer.obj bufferevent.obj bufferevent_sock.obj \ bufferevent_pair.obj listener.obj evmap.obj log.obj evutil.obj \ strlcpy.obj signal.obj bufferevent_filter.obj evthread.obj \ bufferevent_ratelim.obj evutil_rand.obj evutil_time.obj WIN_OBJS=win32select.obj evthread_win32.obj buffer_iocp.obj \ event_iocp.obj bufferevent_async.obj EXTRA_OBJS=event_tagging.obj http.obj evdns.obj evrpc.obj !IFDEF OPENSSL_DIR SSL_OBJS=bufferevent_openssl.obj SSL_LIBS=libevent_openssl.lib !ELSE SSL_OBJS= SSL_LIBS= !ENDIF ALL_OBJS=$(CORE_OBJS) $(WIN_OBJS) $(EXTRA_OBJS) $(SSL_OBJS) STATIC_LIBS=libevent_core.lib libevent_extras.lib libevent.lib $(SSL_LIBS) all: static_libs tests static_libs: $(STATIC_LIBS) libevent_core.lib: $(CORE_OBJS) $(WIN_OBJS) lib $(LIBFLAGS) $(CORE_OBJS) $(WIN_OBJS) /out:libevent_core.lib libevent_extras.lib: $(EXTRA_OBJS) lib $(LIBFLAGS) $(EXTRA_OBJS) /out:libevent_extras.lib libevent.lib: $(CORE_OBJS) $(WIN_OBJS) $(EXTRA_OBJS) lib $(LIBFLAGS) $(CORE_OBJS) $(EXTRA_OBJS) $(WIN_OBJS) /out:libevent.lib libevent_openssl.lib: $(SSL_OBJS) lib $(LIBFLAGS) $(SSL_OBJS) /out:libevent_openssl.lib clean: del $(ALL_OBJS) del $(STATIC_LIBS) cd test $(MAKE) /F Makefile.nmake clean cd .. tests: cd test !IFDEF OPENSSL_DIR $(MAKE) OPENSSL_DIR=$(OPENSSL_DIR) /F Makefile.nmake !ELSE $(MAKE) /F Makefile.nmake !ENDIF cd .. �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/README.md����������������������������������������������������0000664�0000000�0000000�00000023665�15270523712�0021271�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������<p align="center"> <img src="https://strcpy.net/libevent3.png" alt="libevent logo"/> </p> [![Appveyor Win32 Build Status](https://ci.appveyor.com/api/projects/status/github/libevent/libevent?branch=master&svg=true)](https://ci.appveyor.com/project/nmathewson/libevent) [![Travis Build Status](https://travis-ci.org/libevent/libevent.svg?branch=master)](https://travis-ci.org/libevent/libevent) [![Coverage Status](https://coveralls.io/repos/github/libevent/libevent/badge.svg)](https://coveralls.io/github/libevent/libevent) # 0. BUILDING AND INSTALLATION (Briefly) ## Autoconf $ ./configure $ make $ make verify # (optional) $ sudo make install ## Cmake (General) The following Libevent specific Cmake variables are as follows (the values being the default). ``` # Installation directory for executables EVENT_INSTALL_BIN_DIR:PATH=bin # Installation directory for CMake files EVENT_INSTALL_CMAKE_DIR:PATH=lib/cmake/libevent ## Installation directory for header files EVENT_INSTALL_INCLUDE_DIR:PATH=include ## Installation directory for libraries EVENT_INSTALL_LIB_DIR:PATH=lib ## Define if libevent should be built with shared libraries instead of archives EVENT__BUILD_SHARED_LIBRARIES:BOOL=OFF # Enable running gcov to get a test coverage report (only works with # GCC/CLang). Make sure to enable -DCMAKE_BUILD_TYPE=Debug as well. EVENT__COVERAGE:BOOL=OFF # Defines if libevent should build without the benchmark exectuables EVENT__DISABLE_BENCHMARK:BOOL=OFF # Define if libevent should build without support for a debug mode EVENT__DISABLE_DEBUG_MODE:BOOL=OFF # Define if libevent should not allow replacing the mm functions EVENT__DISABLE_MM_REPLACEMENT:BOOL=OFF # Define if libevent should build without support for OpenSSL encrpytion EVENT__DISABLE_OPENSSL:BOOL=OFF # Disable the regress tests EVENT__DISABLE_REGRESS:BOOL=OFF # Disable sample files EVENT__DISABLE_SAMPLES:BOOL=OFF # If tests should be compiled or not EVENT__DISABLE_TESTS:BOOL=OFF # Define if libevent should not be compiled with thread support EVENT__DISABLE_THREAD_SUPPORT:BOOL=OFF # Enables verbose debugging EVENT__ENABLE_VERBOSE_DEBUG:BOOL=OFF # When crosscompiling forces running a test program that verifies that Kqueue # works with pipes. Note that this requires you to manually run the test program # on the the cross compilation target to verify that it works. See cmake # documentation for try_run for more details EVENT__FORCE_KQUEUE_CHECK:BOOL=OFF # set EVENT_STAGE_VERSION EVENT__STAGE_VERSION:STRING=beta ``` __More variables can be found by running `cmake -LAH <sourcedir_path>`__ ## CMake (Windows) Install CMake: <http://www.cmake.org> $ md build && cd build $ cmake -G "Visual Studio 10" .. # Or whatever generator you want to use cmake --help for a list. $ start libevent.sln ## CMake (Unix) $ mkdir build && cd build $ cmake .. # Default to Unix Makefiles. $ make $ make verify # (optional) # 1. BUILDING AND INSTALLATION (In Depth) ## Autoconf To build libevent, type $ ./configure && make (If you got libevent from the git repository, you will first need to run the included "autogen.sh" script in order to generate the configure script.) You can run the regression tests by running $ make verify Install as root via $ make install Before reporting any problems, please run the regression tests. To enable the low-level tracing build the library as: $ CFLAGS=-DUSE_DEBUG ./configure [...] Standard configure flags should work. In particular, see: --disable-shared Only build static libraries --prefix Install all files relative to this directory. The configure script also supports the following flags: --enable-gcc-warnings Enable extra compiler checking with GCC. --disable-malloc-replacement Don't let applications replace our memory management functions --disable-openssl Disable support for OpenSSL encryption. --disable-thread-support Don't support multithreaded environments. ## CMake (Windows) (Note that autoconf is currently the most mature and supported build enviroment for libevent; the cmake instructions here are new and experimental, though they _should_ be solid. We hope that cmake will still be supported in future versions of Libevent, and will try to make sure that happens.) First of all install <http://www.cmake.org>. To build libevent using Microsoft Visual studio open the "Visual Studio Command prompt" and type: ``` $ cd <libevent source dir> $ mkdir build && cd build $ cmake -G "Visual Studio 10" .. # Or whatever generator you want to use cmake --help for a list. $ start libevent.sln ``` In the above, the ".." refers to the dir containing the Libevent source code. You can build multiple versions (with different compile time settings) from the same source tree by creating other build directories. It is highly recommended to build "out of source" when using CMake instead of "in source" like the normal behaviour of autoconf for this reason. The "NMake Makefiles" CMake generator can be used to build entirely via the command line. To get a list of settings available for the project you can type: ``` $ cmake -LH .. ``` ### GUI CMake also provides a GUI that lets you specify the source directory and output (binary) directory that the build should be placed in. ### OpenSSL support To build Libevent with OpenSSL support you will need to have OpenSSL binaries available when building, these can be found here: <http://www.openssl.org/related/binaries.html> # 2. USEFUL LINKS: For the latest released version of Libevent, see the official website at <http://libevent.org/> . There's a pretty good work-in-progress manual up at <http://www.wangafu.net/~nickm/libevent-book/> . For the latest development versions of Libevent, access our Git repository via ``` $ git clone https://github.com/libevent/libevent.git ``` You can browse the git repository online at: <https://github.com/libevent/Libevent> To report bugs, issues, or ask for new features: __Patches__: https://github.com/libevent/libevent/pulls > OK, those are not really _patches_ You fork, modify, and hit the "Create Pull Request" button. > You can still submit normal git patchs via the mailing list. __Bugs, Features [RFC], and Issus__: https://github.com/libevent/libevent/issues > Or you can do it via the mailing list. There's also a libevent-users mailing list for talking about Libevent use and development: <http://archives.seul.org/libevent/users/> # 3. ACKNOWLEDGMENTS The following people have helped with suggestions, ideas, code or fixing bugs: * Samy Al Bahra * Antony Antony * Jacob Appelbaum * Arno Bakker * Weston Andros Adamson * William Ahern * Ivan Andropov * Sergey Avseyev * Avi Bab * Joachim Bauch * Andrey Belobrov * Gilad Benjamini * Stas Bekman * Denis Bilenko * Julien Blache * Kevin Bowling * Tomash Brechko * Kelly Brock * Ralph Castain * Adrian Chadd * Lawnstein Chan * Shuo Chen * Ka-Hing Cheung * Andrew Cox * Paul Croome * George Danchev * Andrew Danforth * Ed Day * Christopher Davis * Mike Davis * Frank Denis * Antony Dovgal * Mihai Draghicioiu * Alexander Drozdov * Mark Ellzey * Shie Erlich * Leonid Evdokimov * Juan Pablo Fernandez * Christophe Fillot * Mike Frysinger * Remi Gacogne * Artem Germanov * Alexander von Gernler * Diego Giagio * Artur Grabowski * Diwaker Gupta * Kuldeep Gupta * Sebastian Hahn * Dave Hart * Greg Hazel * Nicholas Heath * Michael Herf * Savg He * Mark Heily * Maxime Henrion * Michael Herf * Greg Hewgill * Andrew Hochhaus * Aaron Hopkins * Tani Hosokawa * Jamie Iles * Xiuqiang Jiang * Claudio Jeker * Evan Jones * Marcin Juszkiewicz * George Kadianakis * Makoto Kato * Phua Keat * Azat Khuzhin * Alexander Klauer * Kevin Ko * Brian Koehmstedt * Marko Kreen * Ondřej Kuzník * Valery Kyholodov * Ross Lagerwall * Scott Lamb * Christopher Layne * Adam Langley * Graham Leggett * Volker Lendecke * Philip Lewis * Zhou Li * David Libenzi * Yan Lin * Moshe Litvin * Simon Liu * Mitchell Livingston * Hagne Mahre * Lubomir Marinov * Abilio Marques * Nicolas Martyanoff * Abel Mathew * Nick Mathewson * James Mansion * Nicholas Marriott * Andrey Matveev * Caitlin Mercer * Dagobert Michelsen * Andrea Montefusco * Mansour Moufid * Mina Naguib * Felix Nawothnig * Trond Norbye * Linus Nordberg * Richard Nyberg * Jon Oberheide * John Ohl * Phil Oleson * Alexey Ozeritsky * Dave Pacheco * Derrick Pallas * Tassilo von Parseval * Catalin Patulea * Patrick Pelletier * Simon Perreault * Dan Petro * Pierre Phaneuf * Amarin Phaosawasdi * Ryan Phillips * Dimitre Piskyulev * Pavel Plesov * Jon Poland * Roman Puls * Nate R * Robert Ransom * Balint Reczey * Bert JW Regeer * Nate Rosenblum * Peter Rosin * Maseeb Abdul Qadir * Wang Qin * Alex S * Gyepi Sam * Hanna Schroeter * Ralf Schmitt * Mike Smellie * Steve Snyder * Nir Soffer * Dug Song * Dongsheng Song * Hannes Sowa * Joakim Soderberg * Joseph Spadavecchia * Kevin Springborn * Harlan Stenn * Andrew Sweeney * Ferenc Szalai * Brodie Thiesfield * Jason Toffaletti * Brian Utterback * Gisle Vanem * Bas Verhoeven * Constantine Verutin * Colin Watt * Zack Weinberg * Jardel Weyrich * Jay R. Wren * Zack Weinberg * Mobai Zhang * Alejo * Alex * Taral * propanbutan * masksqwe * mmadia * yangacer * Andrey Skriabin * basavesh.as * billsegall * Bill Vaughan * Christopher Wiley * David Paschich * Ed Schouten * Eduardo Panisset * Jan Heylen * jer-gentoo * Joakim Söderberg * kirillDanshin * lzmths * Marcus Sundberg * Mark Mentovai * Mattes D * Matyas Dolak * Neeraj Badlani * Nick Mathewson * Rainer Keller * Seungmo Koo * Thomas Bernard * Xiao Bao Clark * zeliard * Zonr Chang * Kurt Roeckx * Seven * Simone Basso * Vlad Shcherban * Tim Hentenaar * Breaker * johnsonlee * Philip Prindeville * Vis Virial If we have forgotten your name, please contact us. ���������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/Vagrantfile��������������������������������������������������0000664�0000000�0000000�00000033150�15270523712�0022165�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# -*- mode: ruby -*- # vi: set ft=ruby : # DESCRIPTION: # ============ # Vagrant for running libevent tests with: # - timeout 30min, to avoid hungs # - run tests in parallel under ctest (10 concurency) # - if you have uncommited changes, you should commit them first to check # - unix only, because of some tar'ing to avoid one vm affect another # # ENVIRONMENT: # ============ # - NO_PKG -- do not install packages # - NO_CMAKE -- do not run with cmake # - NO_AUTOTOOLS -- do not run with autoconf/automake Vagrant.configure("2") do |config| # to allow running boxes provisions in parallel, we can't share the same dirs # via virtualbox, however sometimes it is the only way, so instead let's # create an archive of HEAD (this way we will not have any trash there) and # extract it for every box to the separate folder. # # P.S. we will change this --prefix with tar(1) --trasnform system('git archive --prefix=libevent/ --output=.vagrant/libevent.tar HEAD') config.vm.provider "virtualbox" do |vb| vb.memory = "512" # otherwise osx fails, anyway we do not need this vb.customize ["modifyvm", :id, "--usb", "off"] vb.customize ["modifyvm", :id, "--usbehci", "off"] end # disable /vagrant share, in case we will not use default mount config.vm.synced_folder ".", "/vagrant", disabled: true config.vm.define "ubuntu" do |ubuntu| system('tar --overwrite --transform=s/libevent/libevent-linux/ -xf .vagrant/libevent.tar -C .vagrant/') ubuntu.vm.box = "ubuntu/xenial64" ubuntu.vm.synced_folder ".vagrant/libevent-linux", "/vagrant", type: "rsync" if ENV['NO_PKG'] != "true" ubuntu.vm.provision "shell", inline: <<-SHELL apt-get update apt-get install -y zlib1g-dev libssl-dev python2.7 apt-get install -y build-essential cmake ninja-build apt-get install -y autoconf automake libtool SHELL end if ENV['NO_CMAKE'] != "true" ubuntu.vm.provision "shell", privileged: false, inline: <<-SHELL cd /vagrant rm -fr .cmake-vagrant mkdir -p .cmake-vagrant cd .cmake-vagrant cmake -G Ninja .. export CTEST_TEST_TIMEOUT=1800 export CTEST_OUTPUT_ON_FAILURE=1 export CTEST_PARALLEL_LEVEL=20 cmake --build . --target verify SHELL end if ENV['NO_AUTOTOOLS'] != "true" ubuntu.vm.provision "shell", privileged: false, inline: <<-SHELL cd /vagrant ./autogen.sh ./configure make -j20 verify SHELL end end config.vm.define "freebsd" do |freebsd| system('tar --overwrite --transform=s/libevent/libevent-freebsd/ -xf .vagrant/libevent.tar -C .vagrant/') freebsd.vm.box = "freebsd/FreeBSD-11.0-STABLE" freebsd.vm.synced_folder ".vagrant/libevent-freebsd", "/vagrant", type: "rsync", group: "wheel" # otherwise reports error freebsd.ssh.shell = "sh" if ENV['NO_PKG'] != "true" freebsd.vm.provision "shell", inline: <<-SHELL pkg install --yes openssl cmake ninja automake autotools SHELL end if ENV['NO_CMAKE'] != "true" freebsd.vm.provision "shell", privileged: false, inline: <<-SHELL cd /vagrant rm -fr .cmake-vagrant mkdir -p .cmake-vagrant cd .cmake-vagrant cmake -G Ninja .. export CTEST_TEST_TIMEOUT=1800 export CTEST_OUTPUT_ON_FAILURE=1 export CTEST_PARALLEL_LEVEL=20 cmake --build . --target verify SHELL end if ENV['NO_AUTOTOOLS'] != "true" freebsd.vm.provision "shell", privileged: false, inline: <<-SHELL cd /vagrant ./autogen.sh ./configure make -j20 verify SHELL end end config.vm.define "netbsd" do |netbsd| system('tar --overwrite --transform=s/libevent/libevent-netbsd/ -xf .vagrant/libevent.tar -C .vagrant/') netbsd.vm.box = "kja/netbsd-7-amd64" netbsd.vm.synced_folder ".vagrant/libevent-netbsd", "/vagrant", type: "rsync", group: "wheel" if ENV['NO_PKG'] != "true" netbsd.vm.provision "shell", inline: <<-SHELL export PKG_PATH="ftp://ftp.netbsd.org/pub/pkgsrc/packages/NetBSD/x86_64/7.0_2016Q2/All/" pkg_add ncurses ninja-build automake cmake libtool SHELL end if ENV['NO_CMAKE'] != "true" netbsd.vm.provision "shell", privileged: false, inline: <<-SHELL cd /vagrant rm -fr .cmake-vagrant mkdir -p .cmake-vagrant cd .cmake-vagrant cmake -G Ninja .. export CTEST_TEST_TIMEOUT=1800 export CTEST_OUTPUT_ON_FAILURE=1 export CTEST_PARALLEL_LEVEL=20 cmake --build . --target verify SHELL end if ENV['NO_AUTOTOOLS'] != "true" netbsd.vm.provision "shell", privileged: false, inline: <<-SHELL cd /vagrant ./autogen.sh ./configure make -j20 verify SHELL end end config.vm.define "solaris" do |solaris| system('tar --overwrite --transform=s/libevent/libevent-solaris/ -xf .vagrant/libevent.tar -C .vagrant/') # XXX: # - solaris do not have '-or' it only has '-o' for find(1), so we can't use # rsync # - and autoconf(1) doesn't work on virtualbox share, ugh solaris.vm.synced_folder ".vagrant/libevent-solaris", "/vagrant-vbox", type: "virtualbox" solaris.vm.box = "tnarik/solaris10-minimal" if ENV['NO_PKG'] != "true" # TODO: opencsw does not include ninja(1) solaris.vm.provision "shell", inline: <<-SHELL pkgadd -d http://get.opencsw.org/now pkgutil -U pkgutil -y -i libssl_dev cmake rsync python gmake gcc5core automake autoconf libtool SHELL end # copy from virtualbox mount to newly created folder solaris.vm.provision "shell", privileged: false, inline: <<-SHELL rm -fr ~/vagrant cp -r /vagrant-vbox ~/vagrant SHELL if ENV['NO_CMAKE'] != "true" # builtin compiler cc(1) is a wrapper, so we should use gcc5 manually, # otherwise it will not work. # Plus we should set some paths so that cmake/compiler can find tham. solaris.vm.provision "shell", privileged: false, inline: <<-SHELL export CMAKE_INCLUDE_PATH=/opt/csw/include export CMAKE_LIBRARY_PATH=/opt/csw/lib export CFLAGS=-I$CMAKE_INCLUDE_PATH export LDFLAGS=-L$CMAKE_LIBRARY_PATH cd ~/vagrant rm -rf .cmake-vagrant mkdir -p .cmake-vagrant cd .cmake-vagrant cmake -DCMAKE_C_COMPILER=gcc .. export CTEST_TEST_TIMEOUT=1800 export CTEST_OUTPUT_ON_FAILURE=1 export CTEST_PARALLEL_LEVEL=20 cmake --build . --target verify SHELL end if ENV['NO_AUTOTOOLS'] != "true" # and we should set MAKE for `configure` otherwise it will try to use # `make` solaris.vm.provision "shell", privileged: false, inline: <<-SHELL cd ~/vagrant ./autogen.sh MAKE=gmake ./configure gmake -j20 verify SHELL end end # known failures: # - sometimes vm hangs config.vm.define "osx" do |osx| system('tar --overwrite --transform=s/libevent/libevent-osx/ -xf .vagrant/libevent.tar -C .vagrant/') osx.vm.synced_folder ".vagrant/libevent-osx", "/vagrant", type: "rsync", group: "wheel" osx.vm.box = "jhcook/osx-elcapitan-10.11" if ENV['NO_PKG'] != "true" osx.vm.provision "shell", privileged: false, inline: <<-SHELL ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew uninstall libtool brew install libtool openssl ninja cmake autoconf automake SHELL end if ENV['NO_CMAKE'] != "true" # we should set some paths so that cmake/compiler can find tham osx.vm.provision "shell", privileged: false, inline: <<-SHELL export OPENSSL_ROOT=$(echo /usr/local/Cellar/openssl/*) export CMAKE_INCLUDE_PATH=$OPENSSL_ROOT/include export CMAKE_LIBRARY_PATH=$OPENSSL_ROOT/lib cd /vagrant mkdir -p .cmake-vagrant cd .cmake-vagrant cmake -G Ninja .. export CTEST_TEST_TIMEOUT=1800 export CTEST_OUTPUT_ON_FAILURE=1 export CTEST_PARALLEL_LEVEL=20 cmake --build . --target verify SHELL end if ENV['NO_AUTOTOOLS'] != "true" osx.vm.provision "shell", privileged: false, inline: <<-SHELL export OPENSSL_ROOT=$(echo /usr/local/Cellar/openssl/*) export CFLAGS=-I$OPENSSL_ROOT/include export LDFLAGS=-L$OPENSSL_ROOT/lib cd /vagrant ./autogen.sh ./configure make -j20 verify SHELL end end config.vm.define "centos" do |centos| system('tar --overwrite --transform=s/libevent/libevent-centos/ -xf .vagrant/libevent.tar -C .vagrant/') centos.vm.synced_folder ".vagrant/libevent-centos", "/vagrant", type: "rsync", group: "wheel" centos.vm.box = "centos/7" if ENV['NO_PKG'] != "true" centos.vm.provision "shell", inline: <<-SHELL echo "[russianfedora]" > /etc/yum.repos.d/russianfedora.repo echo name=russianfedora >> /etc/yum.repos.d/russianfedora.repo echo baseurl=http://mirror.yandex.ru/fedora/russianfedora/russianfedora/free/el/releases/7/Everything/x86_64/os/ >> /etc/yum.repos.d/russianfedora.repo echo enabled=1 >> /etc/yum.repos.d/russianfedora.repo echo gpgcheck=0 >> /etc/yum.repos.d/russianfedora.repo SHELL centos.vm.provision "shell", inline: <<-SHELL yum -y install zlib-devel openssl-devel python yum -y install gcc cmake ninja-build yum -y install autoconf automake libtool SHELL end if ENV['NO_CMAKE'] != "true" centos.vm.provision "shell", privileged: false, inline: <<-SHELL cd /vagrant rm -fr .cmake-vagrant mkdir -p .cmake-vagrant cd .cmake-vagrant cmake -G Ninja .. export CTEST_TEST_TIMEOUT=1800 export CTEST_OUTPUT_ON_FAILURE=1 export CTEST_PARALLEL_LEVEL=20 cmake --build . --target verify SHELL end if ENV['NO_AUTOTOOLS'] != "true" centos.vm.provision "shell", privileged: false, inline: <<-SHELL cd /vagrant ./autogen.sh ./configure make -j20 verify SHELL end end # known failures: # - issues with timers (not enough allowed error) config.vm.define "win" do |win| system('tar --overwrite --transform=s/libevent/libevent-win/ -xf .vagrant/libevent.tar -C .vagrant/') # 512MB not enough after libtool install, huh win.vm.provider "virtualbox" do |vb| vb.memory = "1024" end # windows does not have rsync builtin, let's use virtualbox for now win.vm.synced_folder ".vagrant/libevent-win", "/vagrant", type: "virtualbox" win.vm.box = "senglin/win-10-enterprise-vs2015community" if ENV['NO_PKG'] != "true" # box with vs2015 does not have C++ support, so let's install it manually # plus chocolatey that includes in this box, can't handle sha1 checksum for # cmake.install, so let's update it< win.vm.provision "shell", inline: <<-SHELL choco upgrade -y chocolatey -pre -f choco install -y VisualStudioCommunity2013 choco install -y openssl.light choco install -y cygwin cyg-get choco install -y cmake choco install -y cmake.install choco install -y python2 SHELL # chocolatey openssl.light package does not contains headers win.vm.provision "shell", inline: <<-SHELL (new-object System.Net.WebClient).DownloadFile('http://strcpy.net/packages/Win32OpenSSL-1_0_2a.exe', '/openssl.exe') /openssl.exe /silent /verysilent /sp- /suppressmsgboxes SHELL # XXX: # - cyg-get depends from cygwinsetup.exe # https://github.com/chocolatey/chocolatey-coreteampackages/issues/200 # - cyg-get only downloads, do not installs them, ugh. so let's do not use # it win.vm.provision "shell", privileged: false, inline: <<-SHELL (new-object System.Net.WebClient).DownloadFile('https://cygwin.com/setup-x86_64.exe', '/tools/cygwin/cygwinsetup.exe') $env:PATH="/tools/cygwin/bin;$($env:PATH);/tools/cygwin" cygwinsetup --root c:/tools/cygwin/ --local-package-dir c:/tools/cygwin/packages/ --no-desktop --no-startmenu --verbose --quiet-mode --download --packages automake,autoconf,gcc-core,libtool,make,python,openssl-devel cygwinsetup --root c:/tools/cygwin/ --local-package-dir c:/tools/cygwin/packages/ --no-desktop --no-startmenu --verbose --quiet-mode --local-install --packages automake,autoconf,gcc-core,libtool,make,python,openssl-devel SHELL end if ENV['NO_CMAKE'] != "true" win.vm.provision "shell", privileged: false, inline: <<-SHELL $env:PATH="/Program Files/CMake/bin;/tools/python2;$($env:PATH)" cd /vagrant Remove-Item -Recurse -Force .cmake-vagrant mkdir -p .cmake-vagrant cd .cmake-vagrant cmake -G "Visual Studio 12" .. $env:CTEST_TEST_TIMEOUT = "1800" $env:CTEST_OUTPUT_ON_FAILURE = "1" $env:CTEST_PARALLEL_LEVEL = "10" cmake --build . --target verify SHELL end if ENV['NO_AUTOTOOLS'] != "true" win.vm.provision "shell", privileged: false, inline: <<-SHELL $env:PATH="/tools/cygwin/bin;$($env:PATH)" bash -lc "echo 'C:/tools/mingw64 /mingw ntfs binary 0 0' > /etc/fstab" bash -lc "echo 'C:/OpenSSL-Win32 /ssl ntfs binary 0 0' >> /etc/fstab" bash -lc "echo 'C:/vagrant /vagrant ntfs binary 0 0' >> /etc/fstab" bash -lc "exec 0</dev/null; exec 2>&1; cd /vagrant; bash -x ./autogen.sh && ./configure LDFLAGS='-L/ssl -L/ssl/lib -L/ssl/lib/MinGW' CFLAGS=-I/ssl/include && make -j20 verify" SHELL end end end ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/��������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021450�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/getopt.c������������������������������������������0000664�0000000�0000000�00000010206�15270523712�0023115�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* $NetBSD: getopt.c,v 1.16 1999/12/02 13:15:56 kleink Exp $ */ /* * Copyright (c) 1987, 1993, 1994, 1995 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the names of the copyright holders nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS * IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #if 0 static char sccsid[] = "@(#)getopt.c 8.3 (Berkeley) 4/27/95"; #endif #include <assert.h> #include <errno.h> #include <stdio.h> #include <string.h> #define __P(x) x #define _DIAGASSERT(x) assert(x) #ifdef __weak_alias __weak_alias(getopt,_getopt); #endif int opterr = 1, /* if error message should be printed */ optind = 1, /* index into parent argv vector */ optopt, /* character checked for validity */ optreset; /* reset getopt */ char *optarg; /* argument associated with option */ static char * _progname __P((char *)); int getopt_internal __P((int, char * const *, const char *)); static char * _progname(nargv0) char * nargv0; { char * tmp; _DIAGASSERT(nargv0 != NULL); tmp = strrchr(nargv0, '/'); if (tmp) tmp++; else tmp = nargv0; return(tmp); } #define BADCH (int)'?' #define BADARG (int)':' #define EMSG "" /* * getopt -- * Parse argc/argv argument vector. */ int getopt(nargc, nargv, ostr) int nargc; char * const nargv[]; const char *ostr; { static char *__progname = 0; static char *place = EMSG; /* option letter processing */ char *oli; /* option letter list index */ __progname = __progname?__progname:_progname(*nargv); _DIAGASSERT(nargv != NULL); _DIAGASSERT(ostr != NULL); if (optreset || !*place) { /* update scanning pointer */ optreset = 0; if (optind >= nargc || *(place = nargv[optind]) != '-') { place = EMSG; return (-1); } if (place[1] && *++place == '-' /* found "--" */ && place[1] == '\0') { ++optind; place = EMSG; return (-1); } } /* option letter okay? */ if ((optopt = (int)*place++) == (int)':' || !(oli = strchr(ostr, optopt))) { /* * if the user didn't specify '-' as an option, * assume it means -1. */ if (optopt == (int)'-') return (-1); if (!*place) ++optind; if (opterr && *ostr != ':') (void)fprintf(stderr, "%s: illegal option -- %c\n", __progname, optopt); return (BADCH); } if (*++oli != ':') { /* don't need argument */ optarg = NULL; if (!*place) ++optind; } else { /* need an argument */ if (*place) /* no white space */ optarg = place; else if (nargc <= ++optind) { /* no arg */ place = EMSG; if (*ostr == ':') return (BADARG); if (opterr) (void)fprintf(stderr, "%s: option requires an argument -- %c\n", __progname, optopt); return (BADCH); } else /* white space */ optarg = nargv[optind]; place = EMSG; ++optind; } return (optopt); /* dump back option letter */ } ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/getopt.h������������������������������������������0000664�0000000�0000000�00000001260�15270523712�0023122�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#ifndef __GETOPT_H__ #define __GETOPT_H__ #ifdef __cplusplus extern "C" { #endif extern int opterr; /* if error message should be printed */ extern int optind; /* index into parent argv vector */ extern int optopt; /* character checked for validity */ extern int optreset; /* reset getopt */ extern char *optarg; /* argument associated with option */ struct option { const char *name; int has_arg; int *flag; int val; }; #define no_argument 0 #define required_argument 1 #define optional_argument 2 int getopt(int, char**, const char*); int getopt_long(int, char**, const char*, const struct option*, int*); #ifdef __cplusplus } #endif #endif /* __GETOPT_H__ */ ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/getopt_long.c�������������������������������������0000664�0000000�0000000�00000014243�15270523712�0024141�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������ /* * Copyright (c) 1987, 1993, 1994, 1996 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the names of the copyright holders nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS * IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include <assert.h> #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include "getopt.h" extern int opterr; /* if error message should be printed */ extern int optind; /* index into parent argv vector */ extern int optopt; /* character checked for validity */ extern int optreset; /* reset getopt */ extern char *optarg; /* argument associated with option */ #define __P(x) x #define _DIAGASSERT(x) assert(x) static char * __progname __P((char *)); int getopt_internal __P((int, char * const *, const char *)); static char * __progname(nargv0) char * nargv0; { char * tmp; _DIAGASSERT(nargv0 != NULL); tmp = strrchr(nargv0, '/'); if (tmp) tmp++; else tmp = nargv0; return(tmp); } #define BADCH (int)'?' #define BADARG (int)':' #define EMSG "" /* * getopt -- * Parse argc/argv argument vector. */ int getopt_internal(nargc, nargv, ostr) int nargc; char * const *nargv; const char *ostr; { static char *place = EMSG; /* option letter processing */ char *oli; /* option letter list index */ _DIAGASSERT(nargv != NULL); _DIAGASSERT(ostr != NULL); if (optreset || !*place) { /* update scanning pointer */ optreset = 0; if (optind >= nargc || *(place = nargv[optind]) != '-') { place = EMSG; return (-1); } if (place[1] && *++place == '-') { /* found "--" */ /* ++optind; */ place = EMSG; return (-2); } } /* option letter okay? */ if ((optopt = (int)*place++) == (int)':' || !(oli = strchr(ostr, optopt))) { /* * if the user didn't specify '-' as an option, * assume it means -1. */ if (optopt == (int)'-') return (-1); if (!*place) ++optind; if (opterr && *ostr != ':') (void)fprintf(stderr, "%s: illegal option -- %c\n", __progname(nargv[0]), optopt); return (BADCH); } if (*++oli != ':') { /* don't need argument */ optarg = NULL; if (!*place) ++optind; } else { /* need an argument */ if (*place) /* no white space */ optarg = place; else if (nargc <= ++optind) { /* no arg */ place = EMSG; if ((opterr) && (*ostr != ':')) (void)fprintf(stderr, "%s: option requires an argument -- %c\n", __progname(nargv[0]), optopt); return (BADARG); } else /* white space */ optarg = nargv[optind]; place = EMSG; ++optind; } return (optopt); /* dump back option letter */ } #if 0 /* * getopt -- * Parse argc/argv argument vector. */ int getopt2(nargc, nargv, ostr) int nargc; char * const *nargv; const char *ostr; { int retval; if ((retval = getopt_internal(nargc, nargv, ostr)) == -2) { retval = -1; ++optind; } return(retval); } #endif /* * getopt_long -- * Parse argc/argv argument vector. */ int getopt_long(nargc, nargv, options, long_options, index) int nargc; char ** nargv; const char * options; const struct option * long_options; int * index; { int retval; _DIAGASSERT(nargv != NULL); _DIAGASSERT(options != NULL); _DIAGASSERT(long_options != NULL); /* index may be NULL */ if ((retval = getopt_internal(nargc, nargv, options)) == -2) { char *current_argv = nargv[optind++] + 2, *has_equal; int i, current_argv_len, match = -1; if (*current_argv == '\0') { return(-1); } if ((has_equal = strchr(current_argv, '=')) != NULL) { current_argv_len = has_equal - current_argv; has_equal++; } else current_argv_len = strlen(current_argv); for (i = 0; long_options[i].name; i++) { if (strncmp(current_argv, long_options[i].name, current_argv_len)) continue; if (strlen(long_options[i].name) == (unsigned)current_argv_len) { match = i; break; } if (match == -1) match = i; } if (match != -1) { if (long_options[match].has_arg == required_argument || long_options[match].has_arg == optional_argument) { if (has_equal) optarg = has_equal; else optarg = nargv[optind++]; } if ((long_options[match].has_arg == required_argument) && (optarg == NULL)) { /* * Missing argument, leading : * indicates no error should be generated */ if ((opterr) && (*options != ':')) (void)fprintf(stderr, "%s: option requires an argument -- %s\n", __progname(nargv[0]), current_argv); return (BADARG); } } else { /* No matching argument */ if ((opterr) && (*options != ':')) (void)fprintf(stderr, "%s: illegal option -- %s\n", __progname(nargv[0]), current_argv); return (BADCH); } if (long_options[match].flag) { *long_options[match].flag = long_options[match].val; retval = 0; } else retval = long_options[match].val; if (index) *index = match; } return(retval); } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/nmake/��������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022543�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/nmake/evconfig-private.h��������������������������0000664�0000000�0000000�00000000226�15270523712�0026164�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#if !defined(EVENT_EVCONFIG__PRIVATE_H_) && !defined(__MINGW32__) #define EVENT_EVCONFIG__PRIVATE_H_ /* Nothing to see here. Move along. */ #endif ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/nmake/event2/�������������������������������������0000775�0000000�0000000�00000000000�15270523712�0023746�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/nmake/event2/event-config.h�����������������������0000664�0000000�0000000�00000025465�15270523712�0026517�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* event2/event-config.h * * This file was generated by autoconf when libevent was built, and post- * processed by Libevent so that its macros would have a uniform prefix. * * DO NOT EDIT THIS FILE. * * Do not rely on macros in this file existing in later versions. */ #ifndef EVENT_CONFIG_H__ #define EVENT_CONFIG_H__ /* config.h. Generated by configure. */ /* config.h.in. Generated from configure.in by autoheader. */ /* Define if libevent should not allow replacing the mm functions */ /* #undef EVENT__DISABLE_MM_REPLACEMENT */ /* Define if libevent should not be compiled with thread support */ /* #undef EVENT__DISABLE_THREAD_SUPPORT */ /* Define if clock_gettime is available in libc */ /* #undef _EVENT_DNS_USE_CPU_CLOCK_FOR_ID */ /* Define is no secure id variant is available */ /* #define _EVENT_DNS_USE_GETTIMEOFDAY_FOR_ID 1 */ #define EVENT_DNS_USE_FTIME_FOR_ID_ 1 /* Define to 1 if you have the <arpa/inet.h> header file. */ /* #undef EVENT__HAVE_ARPA_INET_H */ /* Define to 1 if you have the `clock_gettime' function. */ /* #undef EVENT__HAVE_CLOCK_GETTIME */ /* Define if /dev/poll is available */ /* #undef EVENT__HAVE_DEVPOLL */ /* Define to 1 if you have the <dlfcn.h> header file. */ /* #undef EVENT__HAVE_DLFCN_H */ /* Define if your system supports the epoll system calls */ /* #undef EVENT__HAVE_EPOLL */ /* Define to 1 if you have the `epoll_ctl' function. */ /* #undef EVENT__HAVE_EPOLL_CTL */ /* Define to 1 if you have the `eventfd' function. */ /* #undef EVENT__HAVE_EVENTFD */ /* Define if your system supports event ports */ /* #undef EVENT__HAVE_EVENT_PORTS */ /* Define to 1 if you have the `fcntl' function. */ /* #undef EVENT__HAVE_FCNTL */ /* Define to 1 if you have the <fcntl.h> header file. */ #define EVENT__HAVE_FCNTL_H 1 /* Define to 1 if you have the `getaddrinfo' function. */ #define EVENT__HAVE_GETADDRINFO 1 /* Define to 1 if you have the `getnameinfo' function. */ #define EVENT__HAVE_GETNAMEINFO 1 /* Define to 1 if you have the `getprotobynumber' function. */ #define EVENT__HAVE_GETPROTOBYNUMBER 1 /* Define to 1 if you have the `getservbyname' function. */ #define EVENT__HAVE_GETSERVBYNAME 1 /* Define to 1 if you have the `gettimeofday' function. */ /* #define EVENT__HAVE_GETTIMEOFDAY 1 */ /* Define to 1 if you have the `inet_ntop' function. */ /* #undef EVENT__HAVE_INET_NTOP */ /* Define to 1 if you have the `inet_pton' function. */ /* #undef EVENT__HAVE_INET_PTON */ /* Define to 1 if you have the <inttypes.h> header file. */ /* #define EVENT__HAVE_INTTYPES_H 1 */ /* Define to 1 if you have the `kqueue' function. */ /* #undef EVENT__HAVE_KQUEUE */ /* Define if the system has zlib */ /* #undef EVENT__HAVE_LIBZ */ /* Define to 1 if you have the <memory.h> header file. */ #define EVENT__HAVE_MEMORY_H 1 /* Define to 1 if you have the `mmap' function. */ /* #undef EVENT__HAVE_MMAP */ /* Define to 1 if you have the <netinet/in6.h> header file. */ /* #undef EVENT__HAVE_NETINET_IN6_H */ /* Define to 1 if you have the <netinet/in.h> header file. */ /* #undef EVENT__HAVE_NETINET_IN_H */ /* Define to 1 if you have the `pipe' function. */ /* #undef EVENT__HAVE_PIPE */ /* Define to 1 if you have the `poll' function. */ /* #undef EVENT__HAVE_POLL */ /* Define to 1 if you have the <poll.h> header file. */ /* #undef EVENT__HAVE_POLL_H */ /* Define to 1 if you have the `port_create' function. */ /* #undef EVENT__HAVE_PORT_CREATE */ /* Define to 1 if you have the <port.h> header file. */ /* #undef EVENT__HAVE_PORT_H */ /* Define if you have POSIX threads libraries and header files. */ /* #undef EVENT__HAVE_PTHREAD */ /* Define if we have pthreads on this system */ /* #undef EVENT__HAVE_PTHREADS */ /* Define to 1 if the system has the type `sa_family_t'. */ /* #undef EVENT__HAVE_SA_FAMILY_T */ /* Define to 1 if you have the `select' function. */ /* #undef EVENT__HAVE_SELECT */ /* Define to 1 if you have the `sendfile' function. */ /* #undef EVENT__HAVE_SENDFILE */ /* Define if F_SETFD is defined in <fcntl.h> */ /* #undef EVENT__HAVE_SETFD */ /* Define to 1 if you have the `sigaction' function. */ /* #undef EVENT__HAVE_SIGACTION */ /* Define to 1 if you have the `signal' function. */ #define EVENT__HAVE_SIGNAL 1 /* Define to 1 if you have the `splice' function. */ /* #undef EVENT__HAVE_SPLICE */ /* Define to 1 if you have the <stdarg.h> header file. */ #define EVENT__HAVE_STDARG_H 1 /* Define to 1 if you have the <stddef.h> header file. */ #define EVENT__HAVE_STDDEF_H 1 /* Define to 1 if you have the <stdint.h> header file. */ /* #define EVENT__HAVE_STDINT_H 1 */ /* Define to 1 if you have the <stdlib.h> header file. */ #define EVENT__HAVE_STDLIB_H 1 /* Define to 1 if you have the <strings.h> header file. */ #define EVENT__HAVE_STRINGS_H 1 /* Define to 1 if you have the <string.h> header file. */ #define EVENT__HAVE_STRING_H 1 /* Define to 1 if you have the `strlcpy' function. */ /* #undef EVENT__HAVE_STRLCPY */ /* Define to 1 if you have the `strsep' function. */ /* #undef EVENT__HAVE_STRSEP */ /* Define to 1 if you have the `strtok_r' function. */ /* #undef EVENT__HAVE_STRTOK_R */ /* Define to 1 if you have the `strtoll' function. */ /* #define EVENT__HAVE_STRTOLL 1 */ #define EVENT__HAVE_STRUCT_ADDRINFO 1 /* Define to 1 if the system has the type `struct in6_addr'. */ #define EVENT__HAVE_STRUCT_IN6_ADDR 1 /* Define to 1 if `s6_addr16' is member of `struct in6_addr'. */ #define EVENT__HAVE_STRUCT_IN6_ADDR_S6_ADDR16 1 /* Define to 1 if `s6_addr32' is member of `struct in6_addr'. */ #define EVENT__HAVE_STRUCT_IN6_ADDR_S6_ADDR32 1 /* Define to 1 if the system has the type `struct sockaddr_in6'. */ #define EVENT__HAVE_STRUCT_SOCKADDR_IN6 1 /* Define to 1 if `sin6_len' is member of `struct sockaddr_in6'. */ /* #undef EVENT__HAVE_STRUCT_SOCKADDR_IN6_SIN6_LEN */ /* Define to 1 if `sin_len' is member of `struct sockaddr_in'. */ /* #undef EVENT__HAVE_STRUCT_SOCKADDR_IN_SIN_LEN */ /* Define to 1 if the system has the type `struct sockaddr_storage'. */ #define EVENT__HAVE_STRUCT_SOCKADDR_STORAGE 1 /* Define to 1 if you have the <sys/devpoll.h> header file. */ /* #undef EVENT__HAVE_SYS_DEVPOLL_H */ /* Define to 1 if you have the <sys/epoll.h> header file. */ /* #undef EVENT__HAVE_SYS_EPOLL_H */ /* Define to 1 if you have the <sys/eventfd.h> header file. */ /* #undef EVENT__HAVE_SYS_EVENTFD_H */ /* Define to 1 if you have the <sys/event.h> header file. */ /* #undef EVENT__HAVE_SYS_EVENT_H */ /* Define to 1 if you have the <sys/ioctl.h> header file. */ /* #undef EVENT__HAVE_SYS_IOCTL_H */ /* Define to 1 if you have the <sys/mman.h> header file. */ /* #undef EVENT__HAVE_SYS_MMAN_H */ /* Define to 1 if you have the <sys/param.h> header file. */ /* #define EVENT__HAVE_SYS_PARAM_H 1 */ /* Define to 1 if you have the <sys/queue.h> header file. */ /* #undef EVENT__HAVE_SYS_QUEUE_H */ /* Define to 1 if you have the <sys/select.h> header file. */ /* #undef EVENT__HAVE_SYS_SELECT_H */ /* Define to 1 if you have the <sys/sendfile.h> header file. */ /* #undef EVENT__HAVE_SYS_SENDFILE_H */ /* Define to 1 if you have the <sys/socket.h> header file. */ /* #undef EVENT__HAVE_SYS_SOCKET_H */ /* Define to 1 if you have the <sys/stat.h> header file. */ #define EVENT__HAVE_SYS_STAT_H 1 /* Define to 1 if you have the <sys/time.h> header file. */ /* #define EVENT__HAVE_SYS_TIME_H 1 */ /* Define to 1 if you have the <sys/types.h> header file. */ #define EVENT__HAVE_SYS_TYPES_H 1 /* Define to 1 if you have the <sys/uio.h> header file. */ /* #undef EVENT__HAVE_SYS_UIO_H */ /* Define if TAILQ_FOREACH is defined in <sys/queue.h> */ /* #undef EVENT__HAVE_TAILQFOREACH */ /* Define if timeradd is defined in <sys/time.h> */ /* #undef EVENT__HAVE_TIMERADD */ /* Define if timerclear is defined in <sys/time.h> */ #define EVENT__HAVE_TIMERCLEAR 1 /* Define if timercmp is defined in <sys/time.h> */ #define EVENT__HAVE_TIMERCMP 1 /* Define if timerisset is defined in <sys/time.h> */ #define EVENT__HAVE_TIMERISSET 1 /* Define to 1 if the system has the type `uint16_t'. */ /* #define EVENT__HAVE_UINT16_T 1 */ /* Define to 1 if the system has the type `uint32_t'. */ /* #define EVENT__HAVE_UINT32_T 1 */ /* Define to 1 if the system has the type `uint64_t'. */ /* #define EVENT__HAVE_UINT64_T 1 */ /* Define to 1 if the system has the type `uint8_t'. */ /* #define EVENT__HAVE_UINT8_T 1 */ /* Define to 1 if you have the <unistd.h> header file. */ /* #define EVENT__HAVE_UNISTD_H 1 */ /* Define to 1 if you have the `vasprintf' function. */ /* #undef EVENT__HAVE_VASPRINTF */ /* Define if kqueue works correctly with pipes */ /* #undef EVENT__HAVE_WORKING_KQUEUE */ /* Numeric representation of the version */ #define EVENT__NUMERIC_VERSION 0x02010800 /* Name of package */ #define EVENT__PACKAGE "libevent" /* Define to the address where bug reports for this package should be sent. */ #define EVENT__PACKAGE_BUGREPORT "" /* Define to the full name of this package. */ #define EVENT__PACKAGE_NAME "" /* Define to the full name and version of this package. */ #define EVENT__PACKAGE_STRING "" /* Define to the one symbol short name of this package. */ #define EVENT__PACKAGE_TARNAME "" /* Define to the version of this package. */ #define EVENT__PACKAGE_VERSION "" /* Define to necessary symbol if this constant uses a non-standard name on your system. */ /* #undef EVENT__PTHREAD_CREATE_JOINABLE */ /* The size of a `int', as computed by sizeof. */ #define EVENT__SIZEOF_INT 4 /* The size of a `long', as computed by sizeof. */ #define EVENT__SIZEOF_LONG 4 /* The size of a `long long', as computed by sizeof. */ #define EVENT__SIZEOF_LONG_LONG 8 /* The size of a `short', as computed by sizeof. */ #define EVENT__SIZEOF_SHORT 2 /* The size of `size_t', as computed by sizeof. */ #ifdef _WIN64 #define EVENT__SIZEOF_SIZE_T 8 #else #define EVENT__SIZEOF_SIZE_T 4 #endif /* The size of `void *', as computed by sizeof. */ #ifdef _WIN64 #define EVENT__SIZEOF_VOID_P 8 #else #define EVENT__SIZEOF_VOID_P 4 #endif /* Define to 1 if you have the ANSI C header files. */ #define EVENT__STDC_HEADERS 1 /* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */ #define EVENT__TIME_WITH_SYS_TIME 1 /* Version number of package */ #define EVENT__VERSION "2.1.8-stable" /* Define to appropriate substitue if compiler doesnt have __func__ */ #define EVENT____func__ __FUNCTION__ /* Define to empty if `const' does not conform to ANSI C. */ /* #undef EVENT__const */ /* Define to `__inline__' or `__inline' if that's what the C compiler calls it, or to nothing if 'inline' is not supported under any name. */ #ifndef _EVENT___cplusplus #define EVENT__inline __inline #endif /* Define to `int' if <sys/types.h> does not define. */ /* #undef EVENT__pid_t */ /* Define to `unsigned' if <sys/types.h> does not define. */ /* #undef EVENT__size_t */ /* Define to unsigned int if you dont have it */ #define EVENT__socklen_t unsigned int /* Define to `int' if <sys/types.h> does not define. */ #define EVENT__ssize_t SSIZE_T #endif �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/WIN32-Code/tree.h��������������������������������������������0000664�0000000�0000000�00000054166�15270523712�0022574�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* $OpenBSD: tree.h,v 1.7 2002/10/17 21:51:54 art Exp $ */ /* * Copyright 2002 Niels Provos <provos@citi.umich.edu> * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _SYS_TREE_H_ #define _SYS_TREE_H_ /* * This file defines data structures for different types of trees: * splay trees and red-black trees. * * A splay tree is a self-organizing data structure. Every operation * on the tree causes a splay to happen. The splay moves the requested * node to the root of the tree and partly rebalances it. * * This has the benefit that request locality causes faster lookups as * the requested nodes move to the top of the tree. On the other hand, * every lookup causes memory writes. * * The Balance Theorem bounds the total access time for m operations * and n inserts on an initially empty tree as O((m + n)lg n). The * amortized cost for a sequence of m accesses to a splay tree is O(lg n); * * A red-black tree is a binary search tree with the node color as an * extra attribute. It fulfills a set of conditions: * - every search path from the root to a leaf consists of the * same number of black nodes, * - each red node (except for the root) has a black parent, * - each leaf node is black. * * Every operation on a red-black tree is bounded as O(lg n). * The maximum height of a red-black tree is 2lg (n+1). */ #define SPLAY_HEAD(name, type) \ struct name { \ struct type *sph_root; /* root of the tree */ \ } #define SPLAY_INITIALIZER(root) \ { NULL } #define SPLAY_INIT(root) do { \ (root)->sph_root = NULL; \ } while (0) #define SPLAY_ENTRY(type) \ struct { \ struct type *spe_left; /* left element */ \ struct type *spe_right; /* right element */ \ } #define SPLAY_LEFT(elm, field) (elm)->field.spe_left #define SPLAY_RIGHT(elm, field) (elm)->field.spe_right #define SPLAY_ROOT(head) (head)->sph_root #define SPLAY_EMPTY(head) (SPLAY_ROOT(head) == NULL) /* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */ #define SPLAY_ROTATE_RIGHT(head, tmp, field) do { \ SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field); \ SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ (head)->sph_root = tmp; \ } while (0) #define SPLAY_ROTATE_LEFT(head, tmp, field) do { \ SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field); \ SPLAY_LEFT(tmp, field) = (head)->sph_root; \ (head)->sph_root = tmp; \ } while (0) #define SPLAY_LINKLEFT(head, tmp, field) do { \ SPLAY_LEFT(tmp, field) = (head)->sph_root; \ tmp = (head)->sph_root; \ (head)->sph_root = SPLAY_LEFT((head)->sph_root, field); \ } while (0) #define SPLAY_LINKRIGHT(head, tmp, field) do { \ SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ tmp = (head)->sph_root; \ (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field); \ } while (0) #define SPLAY_ASSEMBLE(head, node, left, right, field) do { \ SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field); \ SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\ SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field); \ SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field); \ } while (0) /* Generates prototypes and inline functions */ #define SPLAY_PROTOTYPE(name, type, field, cmp) \ void name##_SPLAY(struct name *, struct type *); \ void name##_SPLAY_MINMAX(struct name *, int); \ struct type *name##_SPLAY_INSERT(struct name *, struct type *); \ struct type *name##_SPLAY_REMOVE(struct name *, struct type *); \ \ /* Finds the node with the same key as elm */ \ static __inline struct type * \ name##_SPLAY_FIND(struct name *head, struct type *elm) \ { \ if (SPLAY_EMPTY(head)) \ return(NULL); \ name##_SPLAY(head, elm); \ if ((cmp)(elm, (head)->sph_root) == 0) \ return (head->sph_root); \ return (NULL); \ } \ \ static __inline struct type * \ name##_SPLAY_NEXT(struct name *head, struct type *elm) \ { \ name##_SPLAY(head, elm); \ if (SPLAY_RIGHT(elm, field) != NULL) { \ elm = SPLAY_RIGHT(elm, field); \ while (SPLAY_LEFT(elm, field) != NULL) { \ elm = SPLAY_LEFT(elm, field); \ } \ } else \ elm = NULL; \ return (elm); \ } \ \ static __inline struct type * \ name##_SPLAY_MIN_MAX(struct name *head, int val) \ { \ name##_SPLAY_MINMAX(head, val); \ return (SPLAY_ROOT(head)); \ } /* Main splay operation. * Moves node close to the key of elm to top */ #define SPLAY_GENERATE(name, type, field, cmp) \ struct type * \ name##_SPLAY_INSERT(struct name *head, struct type *elm) \ { \ if (SPLAY_EMPTY(head)) { \ SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL; \ } else { \ int __comp; \ name##_SPLAY(head, elm); \ __comp = (cmp)(elm, (head)->sph_root); \ if(__comp < 0) { \ SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\ SPLAY_RIGHT(elm, field) = (head)->sph_root; \ SPLAY_LEFT((head)->sph_root, field) = NULL; \ } else if (__comp > 0) { \ SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\ SPLAY_LEFT(elm, field) = (head)->sph_root; \ SPLAY_RIGHT((head)->sph_root, field) = NULL; \ } else \ return ((head)->sph_root); \ } \ (head)->sph_root = (elm); \ return (NULL); \ } \ \ struct type * \ name##_SPLAY_REMOVE(struct name *head, struct type *elm) \ { \ struct type *__tmp; \ if (SPLAY_EMPTY(head)) \ return (NULL); \ name##_SPLAY(head, elm); \ if ((cmp)(elm, (head)->sph_root) == 0) { \ if (SPLAY_LEFT((head)->sph_root, field) == NULL) { \ (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\ } else { \ __tmp = SPLAY_RIGHT((head)->sph_root, field); \ (head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\ name##_SPLAY(head, elm); \ SPLAY_RIGHT((head)->sph_root, field) = __tmp; \ } \ return (elm); \ } \ return (NULL); \ } \ \ void \ name##_SPLAY(struct name *head, struct type *elm) \ { \ struct type __node, *__left, *__right, *__tmp; \ int __comp; \ \ SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ __left = __right = &__node; \ \ while ((__comp = (cmp)(elm, (head)->sph_root))) { \ if (__comp < 0) { \ __tmp = SPLAY_LEFT((head)->sph_root, field); \ if (__tmp == NULL) \ break; \ if ((cmp)(elm, __tmp) < 0){ \ SPLAY_ROTATE_RIGHT(head, __tmp, field); \ if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ break; \ } \ SPLAY_LINKLEFT(head, __right, field); \ } else if (__comp > 0) { \ __tmp = SPLAY_RIGHT((head)->sph_root, field); \ if (__tmp == NULL) \ break; \ if ((cmp)(elm, __tmp) > 0){ \ SPLAY_ROTATE_LEFT(head, __tmp, field); \ if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ break; \ } \ SPLAY_LINKRIGHT(head, __left, field); \ } \ } \ SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ } \ \ /* Splay with either the minimum or the maximum element \ * Used to find minimum or maximum element in tree. \ */ \ void name##_SPLAY_MINMAX(struct name *head, int __comp) \ { \ struct type __node, *__left, *__right, *__tmp; \ \ SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ __left = __right = &__node; \ \ while (1) { \ if (__comp < 0) { \ __tmp = SPLAY_LEFT((head)->sph_root, field); \ if (__tmp == NULL) \ break; \ if (__comp < 0){ \ SPLAY_ROTATE_RIGHT(head, __tmp, field); \ if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ break; \ } \ SPLAY_LINKLEFT(head, __right, field); \ } else if (__comp > 0) { \ __tmp = SPLAY_RIGHT((head)->sph_root, field); \ if (__tmp == NULL) \ break; \ if (__comp > 0) { \ SPLAY_ROTATE_LEFT(head, __tmp, field); \ if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ break; \ } \ SPLAY_LINKRIGHT(head, __left, field); \ } \ } \ SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ } #define SPLAY_NEGINF -1 #define SPLAY_INF 1 #define SPLAY_INSERT(name, x, y) name##_SPLAY_INSERT(x, y) #define SPLAY_REMOVE(name, x, y) name##_SPLAY_REMOVE(x, y) #define SPLAY_FIND(name, x, y) name##_SPLAY_FIND(x, y) #define SPLAY_NEXT(name, x, y) name##_SPLAY_NEXT(x, y) #define SPLAY_MIN(name, x) (SPLAY_EMPTY(x) ? NULL \ : name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF)) #define SPLAY_MAX(name, x) (SPLAY_EMPTY(x) ? NULL \ : name##_SPLAY_MIN_MAX(x, SPLAY_INF)) #define SPLAY_FOREACH(x, name, head) \ for ((x) = SPLAY_MIN(name, head); \ (x) != NULL; \ (x) = SPLAY_NEXT(name, head, x)) /* Macros that define a red-back tree */ #define RB_HEAD(name, type) \ struct name { \ struct type *rbh_root; /* root of the tree */ \ } #define RB_INITIALIZER(root) \ { NULL } #define RB_INIT(root) do { \ (root)->rbh_root = NULL; \ } while (0) #define RB_BLACK 0 #define RB_RED 1 #define RB_ENTRY(type) \ struct { \ struct type *rbe_left; /* left element */ \ struct type *rbe_right; /* right element */ \ struct type *rbe_parent; /* parent element */ \ int rbe_color; /* node color */ \ } #define RB_LEFT(elm, field) (elm)->field.rbe_left #define RB_RIGHT(elm, field) (elm)->field.rbe_right #define RB_PARENT(elm, field) (elm)->field.rbe_parent #define RB_COLOR(elm, field) (elm)->field.rbe_color #define RB_ROOT(head) (head)->rbh_root #define RB_EMPTY(head) (RB_ROOT(head) == NULL) #define RB_SET(elm, parent, field) do { \ RB_PARENT(elm, field) = parent; \ RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \ RB_COLOR(elm, field) = RB_RED; \ } while (0) #define RB_SET_BLACKRED(black, red, field) do { \ RB_COLOR(black, field) = RB_BLACK; \ RB_COLOR(red, field) = RB_RED; \ } while (0) #ifndef RB_AUGMENT #define RB_AUGMENT(x) #endif #define RB_ROTATE_LEFT(head, elm, tmp, field) do { \ (tmp) = RB_RIGHT(elm, field); \ if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field))) { \ RB_PARENT(RB_LEFT(tmp, field), field) = (elm); \ } \ RB_AUGMENT(elm); \ if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ else \ RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ } else \ (head)->rbh_root = (tmp); \ RB_LEFT(tmp, field) = (elm); \ RB_PARENT(elm, field) = (tmp); \ RB_AUGMENT(tmp); \ if ((RB_PARENT(tmp, field))) \ RB_AUGMENT(RB_PARENT(tmp, field)); \ } while (0) #define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \ (tmp) = RB_LEFT(elm, field); \ if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field))) { \ RB_PARENT(RB_RIGHT(tmp, field), field) = (elm); \ } \ RB_AUGMENT(elm); \ if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ else \ RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ } else \ (head)->rbh_root = (tmp); \ RB_RIGHT(tmp, field) = (elm); \ RB_PARENT(elm, field) = (tmp); \ RB_AUGMENT(tmp); \ if ((RB_PARENT(tmp, field))) \ RB_AUGMENT(RB_PARENT(tmp, field)); \ } while (0) /* Generates prototypes and inline functions */ #define RB_PROTOTYPE(name, type, field, cmp) \ void name##_RB_INSERT_COLOR(struct name *, struct type *); \ void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\ struct type *name##_RB_REMOVE(struct name *, struct type *); \ struct type *name##_RB_INSERT(struct name *, struct type *); \ struct type *name##_RB_FIND(struct name *, struct type *); \ struct type *name##_RB_NEXT(struct type *); \ struct type *name##_RB_MINMAX(struct name *, int); \ \ /* Main rb operation. * Moves node close to the key of elm to top */ #define RB_GENERATE(name, type, field, cmp) \ void \ name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \ { \ struct type *parent, *gparent, *tmp; \ while ((parent = RB_PARENT(elm, field)) && \ RB_COLOR(parent, field) == RB_RED) { \ gparent = RB_PARENT(parent, field); \ if (parent == RB_LEFT(gparent, field)) { \ tmp = RB_RIGHT(gparent, field); \ if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ RB_COLOR(tmp, field) = RB_BLACK; \ RB_SET_BLACKRED(parent, gparent, field);\ elm = gparent; \ continue; \ } \ if (RB_RIGHT(parent, field) == elm) { \ RB_ROTATE_LEFT(head, parent, tmp, field);\ tmp = parent; \ parent = elm; \ elm = tmp; \ } \ RB_SET_BLACKRED(parent, gparent, field); \ RB_ROTATE_RIGHT(head, gparent, tmp, field); \ } else { \ tmp = RB_LEFT(gparent, field); \ if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ RB_COLOR(tmp, field) = RB_BLACK; \ RB_SET_BLACKRED(parent, gparent, field);\ elm = gparent; \ continue; \ } \ if (RB_LEFT(parent, field) == elm) { \ RB_ROTATE_RIGHT(head, parent, tmp, field);\ tmp = parent; \ parent = elm; \ elm = tmp; \ } \ RB_SET_BLACKRED(parent, gparent, field); \ RB_ROTATE_LEFT(head, gparent, tmp, field); \ } \ } \ RB_COLOR(head->rbh_root, field) = RB_BLACK; \ } \ \ void \ name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \ { \ struct type *tmp; \ while ((elm == NULL || RB_COLOR(elm, field) == RB_BLACK) && \ elm != RB_ROOT(head)) { \ if (RB_LEFT(parent, field) == elm) { \ tmp = RB_RIGHT(parent, field); \ if (RB_COLOR(tmp, field) == RB_RED) { \ RB_SET_BLACKRED(tmp, parent, field); \ RB_ROTATE_LEFT(head, parent, tmp, field);\ tmp = RB_RIGHT(parent, field); \ } \ if ((RB_LEFT(tmp, field) == NULL || \ RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ (RB_RIGHT(tmp, field) == NULL || \ RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ RB_COLOR(tmp, field) = RB_RED; \ elm = parent; \ parent = RB_PARENT(elm, field); \ } else { \ if (RB_RIGHT(tmp, field) == NULL || \ RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK) {\ struct type *oleft; \ if ((oleft = RB_LEFT(tmp, field)))\ RB_COLOR(oleft, field) = RB_BLACK;\ RB_COLOR(tmp, field) = RB_RED; \ RB_ROTATE_RIGHT(head, tmp, oleft, field);\ tmp = RB_RIGHT(parent, field); \ } \ RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ RB_COLOR(parent, field) = RB_BLACK; \ if (RB_RIGHT(tmp, field)) \ RB_COLOR(RB_RIGHT(tmp, field), field) = RB_BLACK;\ RB_ROTATE_LEFT(head, parent, tmp, field);\ elm = RB_ROOT(head); \ break; \ } \ } else { \ tmp = RB_LEFT(parent, field); \ if (RB_COLOR(tmp, field) == RB_RED) { \ RB_SET_BLACKRED(tmp, parent, field); \ RB_ROTATE_RIGHT(head, parent, tmp, field);\ tmp = RB_LEFT(parent, field); \ } \ if ((RB_LEFT(tmp, field) == NULL || \ RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ (RB_RIGHT(tmp, field) == NULL || \ RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ RB_COLOR(tmp, field) = RB_RED; \ elm = parent; \ parent = RB_PARENT(elm, field); \ } else { \ if (RB_LEFT(tmp, field) == NULL || \ RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) {\ struct type *oright; \ if ((oright = RB_RIGHT(tmp, field)))\ RB_COLOR(oright, field) = RB_BLACK;\ RB_COLOR(tmp, field) = RB_RED; \ RB_ROTATE_LEFT(head, tmp, oright, field);\ tmp = RB_LEFT(parent, field); \ } \ RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ RB_COLOR(parent, field) = RB_BLACK; \ if (RB_LEFT(tmp, field)) \ RB_COLOR(RB_LEFT(tmp, field), field) = RB_BLACK;\ RB_ROTATE_RIGHT(head, parent, tmp, field);\ elm = RB_ROOT(head); \ break; \ } \ } \ } \ if (elm) \ RB_COLOR(elm, field) = RB_BLACK; \ } \ \ struct type * \ name##_RB_REMOVE(struct name *head, struct type *elm) \ { \ struct type *child, *parent, *old = elm; \ int color; \ if (RB_LEFT(elm, field) == NULL) \ child = RB_RIGHT(elm, field); \ else if (RB_RIGHT(elm, field) == NULL) \ child = RB_LEFT(elm, field); \ else { \ struct type *left; \ elm = RB_RIGHT(elm, field); \ while ((left = RB_LEFT(elm, field))) \ elm = left; \ child = RB_RIGHT(elm, field); \ parent = RB_PARENT(elm, field); \ color = RB_COLOR(elm, field); \ if (child) \ RB_PARENT(child, field) = parent; \ if (parent) { \ if (RB_LEFT(parent, field) == elm) \ RB_LEFT(parent, field) = child; \ else \ RB_RIGHT(parent, field) = child; \ RB_AUGMENT(parent); \ } else \ RB_ROOT(head) = child; \ if (RB_PARENT(elm, field) == old) \ parent = elm; \ (elm)->field = (old)->field; \ if (RB_PARENT(old, field)) { \ if (RB_LEFT(RB_PARENT(old, field), field) == old)\ RB_LEFT(RB_PARENT(old, field), field) = elm;\ else \ RB_RIGHT(RB_PARENT(old, field), field) = elm;\ RB_AUGMENT(RB_PARENT(old, field)); \ } else \ RB_ROOT(head) = elm; \ RB_PARENT(RB_LEFT(old, field), field) = elm; \ if (RB_RIGHT(old, field)) \ RB_PARENT(RB_RIGHT(old, field), field) = elm; \ if (parent) { \ left = parent; \ do { \ RB_AUGMENT(left); \ } while ((left = RB_PARENT(left, field))); \ } \ goto color; \ } \ parent = RB_PARENT(elm, field); \ color = RB_COLOR(elm, field); \ if (child) \ RB_PARENT(child, field) = parent; \ if (parent) { \ if (RB_LEFT(parent, field) == elm) \ RB_LEFT(parent, field) = child; \ else \ RB_RIGHT(parent, field) = child; \ RB_AUGMENT(parent); \ } else \ RB_ROOT(head) = child; \ color: \ if (color == RB_BLACK) \ name##_RB_REMOVE_COLOR(head, parent, child); \ return (old); \ } \ \ /* Inserts a node into the RB tree */ \ struct type * \ name##_RB_INSERT(struct name *head, struct type *elm) \ { \ struct type *tmp; \ struct type *parent = NULL; \ int comp = 0; \ tmp = RB_ROOT(head); \ while (tmp) { \ parent = tmp; \ comp = (cmp)(elm, parent); \ if (comp < 0) \ tmp = RB_LEFT(tmp, field); \ else if (comp > 0) \ tmp = RB_RIGHT(tmp, field); \ else \ return (tmp); \ } \ RB_SET(elm, parent, field); \ if (parent != NULL) { \ if (comp < 0) \ RB_LEFT(parent, field) = elm; \ else \ RB_RIGHT(parent, field) = elm; \ RB_AUGMENT(parent); \ } else \ RB_ROOT(head) = elm; \ name##_RB_INSERT_COLOR(head, elm); \ return (NULL); \ } \ \ /* Finds the node with the same key as elm */ \ struct type * \ name##_RB_FIND(struct name *head, struct type *elm) \ { \ struct type *tmp = RB_ROOT(head); \ int comp; \ while (tmp) { \ comp = cmp(elm, tmp); \ if (comp < 0) \ tmp = RB_LEFT(tmp, field); \ else if (comp > 0) \ tmp = RB_RIGHT(tmp, field); \ else \ return (tmp); \ } \ return (NULL); \ } \ \ struct type * \ name##_RB_NEXT(struct type *elm) \ { \ if (RB_RIGHT(elm, field)) { \ elm = RB_RIGHT(elm, field); \ while (RB_LEFT(elm, field)) \ elm = RB_LEFT(elm, field); \ } else { \ if (RB_PARENT(elm, field) && \ (elm == RB_LEFT(RB_PARENT(elm, field), field))) \ elm = RB_PARENT(elm, field); \ else { \ while (RB_PARENT(elm, field) && \ (elm == RB_RIGHT(RB_PARENT(elm, field), field)))\ elm = RB_PARENT(elm, field); \ elm = RB_PARENT(elm, field); \ } \ } \ return (elm); \ } \ \ struct type * \ name##_RB_MINMAX(struct name *head, int val) \ { \ struct type *tmp = RB_ROOT(head); \ struct type *parent = NULL; \ while (tmp) { \ parent = tmp; \ if (val < 0) \ tmp = RB_LEFT(tmp, field); \ else \ tmp = RB_RIGHT(tmp, field); \ } \ return (parent); \ } #define RB_NEGINF -1 #define RB_INF 1 #define RB_INSERT(name, x, y) name##_RB_INSERT(x, y) #define RB_REMOVE(name, x, y) name##_RB_REMOVE(x, y) #define RB_FIND(name, x, y) name##_RB_FIND(x, y) #define RB_NEXT(name, x, y) name##_RB_NEXT(y) #define RB_MIN(name, x) name##_RB_MINMAX(x, RB_NEGINF) #define RB_MAX(name, x) name##_RB_MINMAX(x, RB_INF) #define RB_FOREACH(x, name, head) \ for ((x) = RB_MIN(name, head); \ (x) != NULL; \ (x) = name##_RB_NEXT(x)) #endif /* _SYS_TREE_H_ */ ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/appveyor.yml�������������������������������������������������0000664�0000000�0000000�00000004521�15270523712�0022370�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������version: 2.1.8.{build} os: Visual Studio 2015 build: verbosity: detailed environment: global: CYG_ROOT: C:/MinGW/msys/1.0 matrix: - EVENT_BUILD_METHOD: "autotools" EVENT_CONFIGURE_OPTIONS: "" - EVENT_BUILD_METHOD: "autotools" EVENT_CONFIGURE_OPTIONS: "--disable-openssl" - EVENT_BUILD_METHOD: "autotools" EVENT_CONFIGURE_OPTIONS: "--disable-thread-support" - EVENT_BUILD_METHOD: "autotools" EVENT_CONFIGURE_OPTIONS: "--disable-debug-mode" - EVENT_BUILD_METHOD: "autotools" EVENT_CONFIGURE_OPTIONS: "--disable-malloc-replacement" - EVENT_BUILD_METHOD: "cmake" EVENT_CMAKE_OPTIONS: "" - EVENT_BUILD_METHOD: "cmake" EVENT_CMAKE_OPTIONS: "-DEVENT__DISABLE_OPENSSL=ON" - EVENT_BUILD_METHOD: "cmake" EVENT_CMAKE_OPTIONS: "-DEVENT__DISABLE_THREAD_SUPPORT=ON" - EVENT_BUILD_METHOD: "cmake" EVENT_CMAKE_OPTIONS: "-DEVENT__DISABLE_DEBUG_MODE=ON" - EVENT_BUILD_METHOD: "cmake" EVENT_CMAKE_OPTIONS: "-DEVENT__DISABLE_MM_REPLACEMENT=ON" - EVENT_BUILD_METHOD: "cmake" EVENT_CMAKE_OPTIONS: "-DEVENT__ENABLE_VERBOSE_DEBUG=ON" - EVENT_BUILD_METHOD: "cmake" EVENT_CMAKE_OPTIONS: "-DCMAKE_C_FLAGS='-DUNICODE -D_UNICODE'" init: - 'echo Building libevent %version% for Windows' - 'echo System architecture: %PLATFORM%' - 'echo Repo build branch is: %APPVEYOR_REPO_BRANCH%' - 'echo Build folder is: %APPVEYOR_BUILD_FOLDER%' - 'echo Repo build commit is: %APPVEYOR_REPO_COMMIT%' - 'echo Cygwin root is: %CYG_ROOT%' install: - C:\MinGW\bin\mingw-get install autotools autoconf automake build_script: - ps: | if ($env:EVENT_BUILD_METHOD -eq 'autotools') { $env:PATH="$env:CYG_ROOT\bin;C:\MinGW\bin;$($env:PATH)" bash -lc "echo 'C:\MinGW /mingw' > /etc/fstab" bash -lc "echo 'C:\OpenSSL-Win32 /ssl' >> /etc/fstab" $env:APPVEYOR_BUILD_FOLDER = $env:APPVEYOR_BUILD_FOLDER -replace "\\", "/" bash -lc "exec 0</dev/null; exec 2>&1; cd $env:APPVEYOR_BUILD_FOLDER; bash -x ./autogen.sh && ./configure LDFLAGS='-L/ssl -L/ssl/lib -L/ssl/lib/MinGW' CFLAGS=-I/ssl/include $env:EVENT_CONFIGURE_OPTIONS && make && make verify -j20" } else { md build cd build cmake .. $env:EVENT_CMAKE_OPTIONS cmake --build . $env:CTEST_PARALLEL_LEVEL="20" ctest --output-on-failure } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/arc4random.c�������������������������������������������������0000664�0000000�0000000�00000031520�15270523712�0022175�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* Portable arc4random.c based on arc4random.c from OpenBSD. * Portable version by Chris Davis, adapted for Libevent by Nick Mathewson * Copyright (c) 2010 Chris Davis, Niels Provos, and Nick Mathewson * Copyright (c) 2010-2012 Niels Provos and Nick Mathewson * * Note that in Libevent, this file isn't compiled directly. Instead, * it's included from evutil_rand.c */ /* * Copyright (c) 1996, David Mazieres <dm@uun.org> * Copyright (c) 2008, Damien Miller <djm@openbsd.org> * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* * Arc4 random number generator for OpenBSD. * * This code is derived from section 17.1 of Applied Cryptography, * second edition, which describes a stream cipher allegedly * compatible with RSA Labs "RC4" cipher (the actual description of * which is a trade secret). The same algorithm is used as a stream * cipher called "arcfour" in Tatu Ylonen's ssh package. * * Here the stream cipher has been modified always to include the time * when initializing the state. That makes it impossible to * regenerate the same random sequence twice, so this can't be used * for encryption, but will generate good random numbers. * * RC4 is a registered trademark of RSA Laboratories. */ #ifndef ARC4RANDOM_EXPORT #define ARC4RANDOM_EXPORT #endif #ifndef ARC4RANDOM_UINT32 #define ARC4RANDOM_UINT32 uint32_t #endif #ifndef ARC4RANDOM_NO_INCLUDES #include "evconfig-private.h" #ifdef _WIN32 #include <wincrypt.h> #include <process.h> #else #include <fcntl.h> #include <unistd.h> #include <sys/param.h> #include <sys/time.h> #ifdef EVENT__HAVE_SYS_SYSCTL_H #include <sys/sysctl.h> #endif #endif #include <limits.h> #include <stdlib.h> #include <string.h> #endif /* Add platform entropy 32 bytes (256 bits) at a time. */ #define ADD_ENTROPY 32 /* Re-seed from the platform RNG after generating this many bytes. */ #define BYTES_BEFORE_RESEED 1600000 struct arc4_stream { unsigned char i; unsigned char j; unsigned char s[256]; }; #ifdef _WIN32 #define getpid _getpid #define pid_t int #endif static int rs_initialized; static struct arc4_stream rs; static pid_t arc4_stir_pid; static int arc4_count; static int arc4_seeded_ok; static inline unsigned char arc4_getbyte(void); static inline void arc4_init(void) { int n; for (n = 0; n < 256; n++) rs.s[n] = n; rs.i = 0; rs.j = 0; } static inline void arc4_addrandom(const unsigned char *dat, int datlen) { int n; unsigned char si; rs.i--; for (n = 0; n < 256; n++) { rs.i = (rs.i + 1); si = rs.s[rs.i]; rs.j = (rs.j + si + dat[n % datlen]); rs.s[rs.i] = rs.s[rs.j]; rs.s[rs.j] = si; } rs.j = rs.i; } #ifndef _WIN32 static ssize_t read_all(int fd, unsigned char *buf, size_t count) { size_t numread = 0; ssize_t result; while (numread < count) { result = read(fd, buf+numread, count-numread); if (result<0) return -1; else if (result == 0) break; numread += result; } return (ssize_t)numread; } #endif #ifdef _WIN32 #define TRY_SEED_WIN32 static int arc4_seed_win32(void) { /* This is adapted from Tor's crypto_seed_rng() */ static int provider_set = 0; static HCRYPTPROV provider; unsigned char buf[ADD_ENTROPY]; if (!provider_set) { if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) { if (GetLastError() != (DWORD)NTE_BAD_KEYSET) return -1; } provider_set = 1; } if (!CryptGenRandom(provider, sizeof(buf), buf)) return -1; arc4_addrandom(buf, sizeof(buf)); evutil_memclear_(buf, sizeof(buf)); arc4_seeded_ok = 1; return 0; } #endif #if defined(EVENT__HAVE_SYS_SYSCTL_H) && defined(EVENT__HAVE_SYSCTL) #if EVENT__HAVE_DECL_CTL_KERN && EVENT__HAVE_DECL_KERN_RANDOM && EVENT__HAVE_DECL_RANDOM_UUID #define TRY_SEED_SYSCTL_LINUX static int arc4_seed_sysctl_linux(void) { /* Based on code by William Ahern, this function tries to use the * RANDOM_UUID sysctl to get entropy from the kernel. This can work * even if /dev/urandom is inaccessible for some reason (e.g., we're * running in a chroot). */ int mib[] = { CTL_KERN, KERN_RANDOM, RANDOM_UUID }; unsigned char buf[ADD_ENTROPY]; size_t len, n; unsigned i; int any_set; memset(buf, 0, sizeof(buf)); for (len = 0; len < sizeof(buf); len += n) { n = sizeof(buf) - len; if (0 != sysctl(mib, 3, &buf[len], &n, NULL, 0)) return -1; } /* make sure that the buffer actually got set. */ for (i=0,any_set=0; i<sizeof(buf); ++i) { any_set |= buf[i]; } if (!any_set) return -1; arc4_addrandom(buf, sizeof(buf)); evutil_memclear_(buf, sizeof(buf)); arc4_seeded_ok = 1; return 0; } #endif #if EVENT__HAVE_DECL_CTL_KERN && EVENT__HAVE_DECL_KERN_ARND #define TRY_SEED_SYSCTL_BSD static int arc4_seed_sysctl_bsd(void) { /* Based on code from William Ahern and from OpenBSD, this function * tries to use the KERN_ARND syscall to get entropy from the kernel. * This can work even if /dev/urandom is inaccessible for some reason * (e.g., we're running in a chroot). */ int mib[] = { CTL_KERN, KERN_ARND }; unsigned char buf[ADD_ENTROPY]; size_t len, n; int i, any_set; memset(buf, 0, sizeof(buf)); len = sizeof(buf); if (sysctl(mib, 2, buf, &len, NULL, 0) == -1) { for (len = 0; len < sizeof(buf); len += sizeof(unsigned)) { n = sizeof(unsigned); if (n + len > sizeof(buf)) n = len - sizeof(buf); if (sysctl(mib, 2, &buf[len], &n, NULL, 0) == -1) return -1; } } /* make sure that the buffer actually got set. */ for (i=any_set=0; i<sizeof(buf); ++i) { any_set |= buf[i]; } if (!any_set) return -1; arc4_addrandom(buf, sizeof(buf)); evutil_memclear_(buf, sizeof(buf)); arc4_seeded_ok = 1; return 0; } #endif #endif /* defined(EVENT__HAVE_SYS_SYSCTL_H) */ #ifdef __linux__ #define TRY_SEED_PROC_SYS_KERNEL_RANDOM_UUID static int arc4_seed_proc_sys_kernel_random_uuid(void) { /* Occasionally, somebody will make /proc/sys accessible in a chroot, * but not /dev/urandom. Let's try /proc/sys/kernel/random/uuid. * Its format is stupid, so we need to decode it from hex. */ int fd; char buf[128]; unsigned char entropy[64]; int bytes, n, i, nybbles; for (bytes = 0; bytes<ADD_ENTROPY; ) { fd = evutil_open_closeonexec_("/proc/sys/kernel/random/uuid", O_RDONLY, 0); if (fd < 0) return -1; n = read(fd, buf, sizeof(buf)); close(fd); if (n<=0) return -1; memset(entropy, 0, sizeof(entropy)); for (i=nybbles=0; i<n; ++i) { if (EVUTIL_ISXDIGIT_(buf[i])) { int nyb = evutil_hex_char_to_int_(buf[i]); if (nybbles & 1) { entropy[nybbles/2] |= nyb; } else { entropy[nybbles/2] |= nyb<<4; } ++nybbles; } } if (nybbles < 2) return -1; arc4_addrandom(entropy, nybbles/2); bytes += nybbles/2; } evutil_memclear_(entropy, sizeof(entropy)); evutil_memclear_(buf, sizeof(buf)); arc4_seeded_ok = 1; return 0; } #endif #ifndef _WIN32 #define TRY_SEED_URANDOM static char *arc4random_urandom_filename = NULL; static int arc4_seed_urandom_helper_(const char *fname) { unsigned char buf[ADD_ENTROPY]; int fd; size_t n; fd = evutil_open_closeonexec_(fname, O_RDONLY, 0); if (fd<0) return -1; n = read_all(fd, buf, sizeof(buf)); close(fd); if (n != sizeof(buf)) return -1; arc4_addrandom(buf, sizeof(buf)); evutil_memclear_(buf, sizeof(buf)); arc4_seeded_ok = 1; return 0; } static int arc4_seed_urandom(void) { /* This is adapted from Tor's crypto_seed_rng() */ static const char *filenames[] = { "/dev/srandom", "/dev/urandom", "/dev/random", NULL }; int i; if (arc4random_urandom_filename) return arc4_seed_urandom_helper_(arc4random_urandom_filename); for (i = 0; filenames[i]; ++i) { if (arc4_seed_urandom_helper_(filenames[i]) == 0) { return 0; } } return -1; } #endif static int arc4_seed(void) { int ok = 0; /* We try every method that might work, and don't give up even if one * does seem to work. There's no real harm in over-seeding, and if * one of these sources turns out to be broken, that would be bad. */ #ifdef TRY_SEED_WIN32 if (0 == arc4_seed_win32()) ok = 1; #endif #ifdef TRY_SEED_URANDOM if (0 == arc4_seed_urandom()) ok = 1; #endif #ifdef TRY_SEED_PROC_SYS_KERNEL_RANDOM_UUID if (arc4random_urandom_filename == NULL && 0 == arc4_seed_proc_sys_kernel_random_uuid()) ok = 1; #endif #ifdef TRY_SEED_SYSCTL_LINUX /* Apparently Linux is deprecating sysctl, and spewing warning * messages when you try to use it. */ if (!ok && 0 == arc4_seed_sysctl_linux()) ok = 1; #endif #ifdef TRY_SEED_SYSCTL_BSD if (0 == arc4_seed_sysctl_bsd()) ok = 1; #endif return ok ? 0 : -1; } static int arc4_stir(void) { int i; if (!rs_initialized) { arc4_init(); rs_initialized = 1; } arc4_seed(); if (!arc4_seeded_ok) return -1; /* * Discard early keystream, as per recommendations in * "Weaknesses in the Key Scheduling Algorithm of RC4" by * Scott Fluhrer, Itsik Mantin, and Adi Shamir. * http://www.wisdom.weizmann.ac.il/~itsik/RC4/Papers/Rc4_ksa.ps * * Ilya Mironov's "(Not So) Random Shuffles of RC4" suggests that * we drop at least 2*256 bytes, with 12*256 as a conservative * value. * * RFC4345 says to drop 6*256. * * At least some versions of this code drop 4*256, in a mistaken * belief that "words" in the Fluhrer/Mantin/Shamir paper refers * to processor words. * * We add another sect to the cargo cult, and choose 12*256. */ for (i = 0; i < 12*256; i++) (void)arc4_getbyte(); arc4_count = BYTES_BEFORE_RESEED; return 0; } static void arc4_stir_if_needed(void) { pid_t pid = getpid(); if (arc4_count <= 0 || !rs_initialized || arc4_stir_pid != pid) { arc4_stir_pid = pid; arc4_stir(); } } static inline unsigned char arc4_getbyte(void) { unsigned char si, sj; rs.i = (rs.i + 1); si = rs.s[rs.i]; rs.j = (rs.j + si); sj = rs.s[rs.j]; rs.s[rs.i] = sj; rs.s[rs.j] = si; return (rs.s[(si + sj) & 0xff]); } static inline unsigned int arc4_getword(void) { unsigned int val; val = arc4_getbyte() << 24; val |= arc4_getbyte() << 16; val |= arc4_getbyte() << 8; val |= arc4_getbyte(); return val; } #ifndef ARC4RANDOM_NOSTIR ARC4RANDOM_EXPORT int arc4random_stir(void) { int val; ARC4_LOCK_(); val = arc4_stir(); ARC4_UNLOCK_(); return val; } #endif #ifndef ARC4RANDOM_NOADDRANDOM ARC4RANDOM_EXPORT void arc4random_addrandom(const unsigned char *dat, int datlen) { int j; ARC4_LOCK_(); if (!rs_initialized) arc4_stir(); for (j = 0; j < datlen; j += 256) { /* arc4_addrandom() ignores all but the first 256 bytes of * its input. We want to make sure to look at ALL the * data in 'dat', just in case the user is doing something * crazy like passing us all the files in /var/log. */ arc4_addrandom(dat + j, datlen - j); } ARC4_UNLOCK_(); } #endif #ifndef ARC4RANDOM_NORANDOM ARC4RANDOM_EXPORT ARC4RANDOM_UINT32 arc4random(void) { ARC4RANDOM_UINT32 val; ARC4_LOCK_(); arc4_count -= 4; arc4_stir_if_needed(); val = arc4_getword(); ARC4_UNLOCK_(); return val; } #endif ARC4RANDOM_EXPORT void arc4random_buf(void *buf_, size_t n) { unsigned char *buf = buf_; ARC4_LOCK_(); arc4_stir_if_needed(); while (n--) { if (--arc4_count <= 0) arc4_stir(); buf[n] = arc4_getbyte(); } ARC4_UNLOCK_(); } #ifndef ARC4RANDOM_NOUNIFORM /* * Calculate a uniformly distributed random number less than upper_bound * avoiding "modulo bias". * * Uniformity is achieved by generating new random numbers until the one * returned is outside the range [0, 2**32 % upper_bound). This * guarantees the selected random number will be inside * [2**32 % upper_bound, 2**32) which maps back to [0, upper_bound) * after reduction modulo upper_bound. */ ARC4RANDOM_EXPORT unsigned int arc4random_uniform(unsigned int upper_bound) { ARC4RANDOM_UINT32 r, min; if (upper_bound < 2) return 0; #if (UINT_MAX > 0xffffffffUL) min = 0x100000000UL % upper_bound; #else /* Calculate (2**32 % upper_bound) avoiding 64-bit math */ if (upper_bound > 0x80000000) min = 1 + ~upper_bound; /* 2**32 - upper_bound */ else { /* (2**32 - (x * 2)) % x == 2**32 % x when x <= 2**31 */ min = ((0xffffffff - (upper_bound * 2)) + 1) % upper_bound; } #endif /* * This could theoretically loop forever but each retry has * p > 0.5 (worst case, usually far better) of selecting a * number inside the range we need, so it should rarely need * to re-roll. */ for (;;) { r = arc4random(); if (r >= min) break; } return r % upper_bound; } #endif ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/autogen.sh���������������������������������������������������0000775�0000000�0000000�00000000657�15270523712�0022007�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/bin/sh MAKE=make if command -v gmake >/dev/null 2>/dev/null; then MAKE=gmake fi $MAKE maintainer-clean >/dev/null 2>/dev/null if [ -x "`which autoreconf 2>/dev/null`" ] ; then exec autoreconf -ivf fi LIBTOOLIZE=libtoolize SYSNAME=`uname` if [ "x$SYSNAME" = "xDarwin" ] ; then LIBTOOLIZE=glibtoolize fi aclocal -I m4 && \ autoheader && \ $LIBTOOLIZE && \ autoconf && \ automake --add-missing --force-missing --copy ���������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/buffer.c�����������������������������������������������������0000664�0000000�0000000�00000240471�15270523712�0021423�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #include <winsock2.h> #include <windows.h> #include <io.h> #endif #ifdef EVENT__HAVE_VASPRINTF /* If we have vasprintf, we need to define _GNU_SOURCE before we include * stdio.h. This comes from evconfig-private.h. */ #endif #include <sys/types.h> #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #ifdef EVENT__HAVE_SYS_SOCKET_H #include <sys/socket.h> #endif #ifdef EVENT__HAVE_SYS_UIO_H #include <sys/uio.h> #endif #ifdef EVENT__HAVE_SYS_IOCTL_H #include <sys/ioctl.h> #endif #ifdef EVENT__HAVE_SYS_MMAN_H #include <sys/mman.h> #endif #ifdef EVENT__HAVE_SYS_SENDFILE_H #include <sys/sendfile.h> #endif #ifdef EVENT__HAVE_SYS_STAT_H #include <sys/stat.h> #endif #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #ifdef EVENT__HAVE_STDARG_H #include <stdarg.h> #endif #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #include <limits.h> #include "event2/event.h" #include "event2/buffer.h" #include "event2/buffer_compat.h" #include "event2/bufferevent.h" #include "event2/bufferevent_compat.h" #include "event2/bufferevent_struct.h" #include "event2/thread.h" #include "log-internal.h" #include "mm-internal.h" #include "util-internal.h" #include "evthread-internal.h" #include "evbuffer-internal.h" #include "bufferevent-internal.h" /* some systems do not have MAP_FAILED */ #ifndef MAP_FAILED #define MAP_FAILED ((void *)-1) #endif /* send file support */ #if defined(EVENT__HAVE_SYS_SENDFILE_H) && defined(EVENT__HAVE_SENDFILE) && defined(__linux__) #define USE_SENDFILE 1 #define SENDFILE_IS_LINUX 1 #elif defined(EVENT__HAVE_SENDFILE) && defined(__FreeBSD__) #define USE_SENDFILE 1 #define SENDFILE_IS_FREEBSD 1 #elif defined(EVENT__HAVE_SENDFILE) && defined(__APPLE__) #define USE_SENDFILE 1 #define SENDFILE_IS_MACOSX 1 #elif defined(EVENT__HAVE_SENDFILE) && defined(__sun__) && defined(__svr4__) #define USE_SENDFILE 1 #define SENDFILE_IS_SOLARIS 1 #endif /* Mask of user-selectable callback flags. */ #define EVBUFFER_CB_USER_FLAGS 0xffff /* Mask of all internal-use-only flags. */ #define EVBUFFER_CB_INTERNAL_FLAGS 0xffff0000 /* Flag set if the callback is using the cb_obsolete function pointer */ #define EVBUFFER_CB_OBSOLETE 0x00040000 /* evbuffer_chain support */ #define CHAIN_SPACE_PTR(ch) ((ch)->buffer + (ch)->misalign + (ch)->off) #define CHAIN_SPACE_LEN(ch) ((ch)->flags & EVBUFFER_IMMUTABLE ? \ 0 : (ch)->buffer_len - ((ch)->misalign + (ch)->off)) #define CHAIN_PINNED(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_ANY) != 0) #define CHAIN_PINNED_R(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_R) != 0) /* evbuffer_ptr support */ #define PTR_NOT_FOUND(ptr) do { \ (ptr)->pos = -1; \ (ptr)->internal_.chain = NULL; \ (ptr)->internal_.pos_in_chain = 0; \ } while (0) static void evbuffer_chain_align(struct evbuffer_chain *chain); static int evbuffer_chain_should_realign(struct evbuffer_chain *chain, size_t datalen); static void evbuffer_deferred_callback(struct event_callback *cb, void *arg); static int evbuffer_ptr_memcmp(const struct evbuffer *buf, const struct evbuffer_ptr *pos, const char *mem, size_t len); static struct evbuffer_chain *evbuffer_expand_singlechain(struct evbuffer *buf, size_t datlen); static int evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos, size_t howfar); static int evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg); static inline void evbuffer_chain_incref(struct evbuffer_chain *chain); static struct evbuffer_chain * evbuffer_chain_new(size_t size) { struct evbuffer_chain *chain; size_t to_alloc; if (size > EVBUFFER_CHAIN_MAX - EVBUFFER_CHAIN_SIZE) return (NULL); size += EVBUFFER_CHAIN_SIZE; /* get the next largest memory that can hold the buffer */ if (size < EVBUFFER_CHAIN_MAX / 2) { to_alloc = MIN_BUFFER_SIZE; while (to_alloc < size) { to_alloc <<= 1; } } else { to_alloc = size; } /* we get everything in one chunk */ if ((chain = mm_malloc(to_alloc)) == NULL) return (NULL); memset(chain, 0, EVBUFFER_CHAIN_SIZE); chain->buffer_len = to_alloc - EVBUFFER_CHAIN_SIZE; /* this way we can manipulate the buffer to different addresses, * which is required for mmap for example. */ chain->buffer = EVBUFFER_CHAIN_EXTRA(unsigned char, chain); chain->refcnt = 1; return (chain); } static inline void evbuffer_chain_free(struct evbuffer_chain *chain) { EVUTIL_ASSERT(chain->refcnt > 0); if (--chain->refcnt > 0) { /* chain is still referenced by other chains */ return; } if (CHAIN_PINNED(chain)) { /* will get freed once no longer dangling */ chain->refcnt++; chain->flags |= EVBUFFER_DANGLING; return; } /* safe to release chain, it's either a referencing * chain or all references to it have been freed */ if (chain->flags & EVBUFFER_REFERENCE) { struct evbuffer_chain_reference *info = EVBUFFER_CHAIN_EXTRA( struct evbuffer_chain_reference, chain); if (info->cleanupfn) (*info->cleanupfn)(chain->buffer, chain->buffer_len, info->extra); } if (chain->flags & EVBUFFER_FILESEGMENT) { struct evbuffer_chain_file_segment *info = EVBUFFER_CHAIN_EXTRA( struct evbuffer_chain_file_segment, chain); if (info->segment) { #ifdef _WIN32 if (info->segment->is_mapping) UnmapViewOfFile(chain->buffer); #endif evbuffer_file_segment_free(info->segment); } } if (chain->flags & EVBUFFER_MULTICAST) { struct evbuffer_multicast_parent *info = EVBUFFER_CHAIN_EXTRA( struct evbuffer_multicast_parent, chain); /* referencing chain is being freed, decrease * refcounts of source chain and associated * evbuffer (which get freed once both reach * zero) */ EVUTIL_ASSERT(info->source != NULL); EVUTIL_ASSERT(info->parent != NULL); EVBUFFER_LOCK(info->source); evbuffer_chain_free(info->parent); evbuffer_decref_and_unlock_(info->source); } mm_free(chain); } static void evbuffer_free_all_chains(struct evbuffer_chain *chain) { struct evbuffer_chain *next; for (; chain; chain = next) { next = chain->next; evbuffer_chain_free(chain); } } #ifndef NDEBUG static int evbuffer_chains_all_empty(struct evbuffer_chain *chain) { for (; chain; chain = chain->next) { if (chain->off) return 0; } return 1; } #else /* The definition is needed for EVUTIL_ASSERT, which uses sizeof to avoid "unused variable" warnings. */ static inline int evbuffer_chains_all_empty(struct evbuffer_chain *chain) { return 1; } #endif /* Free all trailing chains in 'buf' that are neither pinned nor empty, prior * to replacing them all with a new chain. Return a pointer to the place * where the new chain will go. * * Internal; requires lock. The caller must fix up buf->last and buf->first * as needed; they might have been freed. */ static struct evbuffer_chain ** evbuffer_free_trailing_empty_chains(struct evbuffer *buf) { struct evbuffer_chain **ch = buf->last_with_datap; /* Find the first victim chain. It might be *last_with_datap */ while ((*ch) && ((*ch)->off != 0 || CHAIN_PINNED(*ch))) ch = &(*ch)->next; if (*ch) { EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch)); evbuffer_free_all_chains(*ch); *ch = NULL; } return ch; } /* Add a single chain 'chain' to the end of 'buf', freeing trailing empty * chains as necessary. Requires lock. Does not schedule callbacks. */ static void evbuffer_chain_insert(struct evbuffer *buf, struct evbuffer_chain *chain) { ASSERT_EVBUFFER_LOCKED(buf); if (*buf->last_with_datap == NULL) { /* There are no chains data on the buffer at all. */ EVUTIL_ASSERT(buf->last_with_datap == &buf->first); EVUTIL_ASSERT(buf->first == NULL); buf->first = buf->last = chain; } else { struct evbuffer_chain **chp; chp = evbuffer_free_trailing_empty_chains(buf); *chp = chain; if (chain->off) buf->last_with_datap = chp; buf->last = chain; } buf->total_len += chain->off; } static inline struct evbuffer_chain * evbuffer_chain_insert_new(struct evbuffer *buf, size_t datlen) { struct evbuffer_chain *chain; if ((chain = evbuffer_chain_new(datlen)) == NULL) return NULL; evbuffer_chain_insert(buf, chain); return chain; } void evbuffer_chain_pin_(struct evbuffer_chain *chain, unsigned flag) { EVUTIL_ASSERT((chain->flags & flag) == 0); chain->flags |= flag; } void evbuffer_chain_unpin_(struct evbuffer_chain *chain, unsigned flag) { EVUTIL_ASSERT((chain->flags & flag) != 0); chain->flags &= ~flag; if (chain->flags & EVBUFFER_DANGLING) evbuffer_chain_free(chain); } static inline void evbuffer_chain_incref(struct evbuffer_chain *chain) { ++chain->refcnt; } struct evbuffer * evbuffer_new(void) { struct evbuffer *buffer; buffer = mm_calloc(1, sizeof(struct evbuffer)); if (buffer == NULL) return (NULL); LIST_INIT(&buffer->callbacks); buffer->refcnt = 1; buffer->last_with_datap = &buffer->first; return (buffer); } int evbuffer_set_flags(struct evbuffer *buf, ev_uint64_t flags) { EVBUFFER_LOCK(buf); buf->flags |= (ev_uint32_t)flags; EVBUFFER_UNLOCK(buf); return 0; } int evbuffer_clear_flags(struct evbuffer *buf, ev_uint64_t flags) { EVBUFFER_LOCK(buf); buf->flags &= ~(ev_uint32_t)flags; EVBUFFER_UNLOCK(buf); return 0; } void evbuffer_incref_(struct evbuffer *buf) { EVBUFFER_LOCK(buf); ++buf->refcnt; EVBUFFER_UNLOCK(buf); } void evbuffer_incref_and_lock_(struct evbuffer *buf) { EVBUFFER_LOCK(buf); ++buf->refcnt; } int evbuffer_defer_callbacks(struct evbuffer *buffer, struct event_base *base) { EVBUFFER_LOCK(buffer); buffer->cb_queue = base; buffer->deferred_cbs = 1; event_deferred_cb_init_(&buffer->deferred, event_base_get_npriorities(base) / 2, evbuffer_deferred_callback, buffer); EVBUFFER_UNLOCK(buffer); return 0; } int evbuffer_enable_locking(struct evbuffer *buf, void *lock) { #ifdef EVENT__DISABLE_THREAD_SUPPORT return -1; #else if (buf->lock) return -1; if (!lock) { EVTHREAD_ALLOC_LOCK(lock, EVTHREAD_LOCKTYPE_RECURSIVE); if (!lock) return -1; buf->lock = lock; buf->own_lock = 1; } else { buf->lock = lock; buf->own_lock = 0; } return 0; #endif } void evbuffer_set_parent_(struct evbuffer *buf, struct bufferevent *bev) { EVBUFFER_LOCK(buf); buf->parent = bev; EVBUFFER_UNLOCK(buf); } static void evbuffer_run_callbacks(struct evbuffer *buffer, int running_deferred) { struct evbuffer_cb_entry *cbent, *next; struct evbuffer_cb_info info; size_t new_size; ev_uint32_t mask, masked_val; int clear = 1; if (running_deferred) { mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED; masked_val = EVBUFFER_CB_ENABLED; } else if (buffer->deferred_cbs) { mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED; masked_val = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED; /* Don't zero-out n_add/n_del, since the deferred callbacks will want to see them. */ clear = 0; } else { mask = EVBUFFER_CB_ENABLED; masked_val = EVBUFFER_CB_ENABLED; } ASSERT_EVBUFFER_LOCKED(buffer); if (LIST_EMPTY(&buffer->callbacks)) { buffer->n_add_for_cb = buffer->n_del_for_cb = 0; return; } if (buffer->n_add_for_cb == 0 && buffer->n_del_for_cb == 0) return; new_size = buffer->total_len; info.orig_size = new_size + buffer->n_del_for_cb - buffer->n_add_for_cb; info.n_added = buffer->n_add_for_cb; info.n_deleted = buffer->n_del_for_cb; if (clear) { buffer->n_add_for_cb = 0; buffer->n_del_for_cb = 0; } for (cbent = LIST_FIRST(&buffer->callbacks); cbent != LIST_END(&buffer->callbacks); cbent = next) { /* Get the 'next' pointer now in case this callback decides * to remove itself or something. */ next = LIST_NEXT(cbent, next); if ((cbent->flags & mask) != masked_val) continue; if ((cbent->flags & EVBUFFER_CB_OBSOLETE)) cbent->cb.cb_obsolete(buffer, info.orig_size, new_size, cbent->cbarg); else cbent->cb.cb_func(buffer, &info, cbent->cbarg); } } void evbuffer_invoke_callbacks_(struct evbuffer *buffer) { if (LIST_EMPTY(&buffer->callbacks)) { buffer->n_add_for_cb = buffer->n_del_for_cb = 0; return; } if (buffer->deferred_cbs) { if (event_deferred_cb_schedule_(buffer->cb_queue, &buffer->deferred)) { evbuffer_incref_and_lock_(buffer); if (buffer->parent) bufferevent_incref_(buffer->parent); } EVBUFFER_UNLOCK(buffer); } evbuffer_run_callbacks(buffer, 0); } static void evbuffer_deferred_callback(struct event_callback *cb, void *arg) { struct bufferevent *parent = NULL; struct evbuffer *buffer = arg; /* XXXX It would be better to run these callbacks without holding the * lock */ EVBUFFER_LOCK(buffer); parent = buffer->parent; evbuffer_run_callbacks(buffer, 1); evbuffer_decref_and_unlock_(buffer); if (parent) bufferevent_decref_(parent); } static void evbuffer_remove_all_callbacks(struct evbuffer *buffer) { struct evbuffer_cb_entry *cbent; while ((cbent = LIST_FIRST(&buffer->callbacks))) { LIST_REMOVE(cbent, next); mm_free(cbent); } } void evbuffer_decref_and_unlock_(struct evbuffer *buffer) { struct evbuffer_chain *chain, *next; ASSERT_EVBUFFER_LOCKED(buffer); EVUTIL_ASSERT(buffer->refcnt > 0); if (--buffer->refcnt > 0) { EVBUFFER_UNLOCK(buffer); return; } for (chain = buffer->first; chain != NULL; chain = next) { next = chain->next; evbuffer_chain_free(chain); } evbuffer_remove_all_callbacks(buffer); if (buffer->deferred_cbs) event_deferred_cb_cancel_(buffer->cb_queue, &buffer->deferred); EVBUFFER_UNLOCK(buffer); if (buffer->own_lock) EVTHREAD_FREE_LOCK(buffer->lock, EVTHREAD_LOCKTYPE_RECURSIVE); mm_free(buffer); } void evbuffer_free(struct evbuffer *buffer) { EVBUFFER_LOCK(buffer); evbuffer_decref_and_unlock_(buffer); } void evbuffer_lock(struct evbuffer *buf) { EVBUFFER_LOCK(buf); } void evbuffer_unlock(struct evbuffer *buf) { EVBUFFER_UNLOCK(buf); } size_t evbuffer_get_length(const struct evbuffer *buffer) { size_t result; EVBUFFER_LOCK(buffer); result = (buffer->total_len); EVBUFFER_UNLOCK(buffer); return result; } size_t evbuffer_get_contiguous_space(const struct evbuffer *buf) { struct evbuffer_chain *chain; size_t result; EVBUFFER_LOCK(buf); chain = buf->first; result = (chain != NULL ? chain->off : 0); EVBUFFER_UNLOCK(buf); return result; } size_t evbuffer_add_iovec(struct evbuffer * buf, struct evbuffer_iovec * vec, int n_vec) { int n; size_t res; size_t to_alloc; EVBUFFER_LOCK(buf); res = to_alloc = 0; for (n = 0; n < n_vec; n++) { to_alloc += vec[n].iov_len; } if (evbuffer_expand_fast_(buf, to_alloc, 2) < 0) { goto done; } for (n = 0; n < n_vec; n++) { /* XXX each 'add' call here does a bunch of setup that's * obviated by evbuffer_expand_fast_, and some cleanup that we * would like to do only once. Instead we should just extract * the part of the code that's needed. */ if (evbuffer_add(buf, vec[n].iov_base, vec[n].iov_len) < 0) { goto done; } res += vec[n].iov_len; } done: EVBUFFER_UNLOCK(buf); return res; } int evbuffer_reserve_space(struct evbuffer *buf, ev_ssize_t size, struct evbuffer_iovec *vec, int n_vecs) { struct evbuffer_chain *chain, **chainp; int n = -1; EVBUFFER_LOCK(buf); if (buf->freeze_end) goto done; if (n_vecs < 1) goto done; if (n_vecs == 1) { if ((chain = evbuffer_expand_singlechain(buf, size)) == NULL) goto done; vec[0].iov_base = (void *)CHAIN_SPACE_PTR(chain); vec[0].iov_len = (size_t)CHAIN_SPACE_LEN(chain); EVUTIL_ASSERT(size<0 || (size_t)vec[0].iov_len >= (size_t)size); n = 1; } else { if (evbuffer_expand_fast_(buf, size, n_vecs)<0) goto done; n = evbuffer_read_setup_vecs_(buf, size, vec, n_vecs, &chainp, 0); } done: EVBUFFER_UNLOCK(buf); return n; } static int advance_last_with_data(struct evbuffer *buf) { int n = 0; ASSERT_EVBUFFER_LOCKED(buf); if (!*buf->last_with_datap) return 0; while ((*buf->last_with_datap)->next && (*buf->last_with_datap)->next->off) { buf->last_with_datap = &(*buf->last_with_datap)->next; ++n; } return n; } int evbuffer_commit_space(struct evbuffer *buf, struct evbuffer_iovec *vec, int n_vecs) { struct evbuffer_chain *chain, **firstchainp, **chainp; int result = -1; size_t added = 0; int i; EVBUFFER_LOCK(buf); if (buf->freeze_end) goto done; if (n_vecs == 0) { result = 0; goto done; } else if (n_vecs == 1 && (buf->last && vec[0].iov_base == (void *)CHAIN_SPACE_PTR(buf->last))) { /* The user only got or used one chain; it might not * be the first one with space in it. */ if ((size_t)vec[0].iov_len > (size_t)CHAIN_SPACE_LEN(buf->last)) goto done; buf->last->off += vec[0].iov_len; added = vec[0].iov_len; if (added) advance_last_with_data(buf); goto okay; } /* Advance 'firstchain' to the first chain with space in it. */ firstchainp = buf->last_with_datap; if (!*firstchainp) goto done; if (CHAIN_SPACE_LEN(*firstchainp) == 0) { firstchainp = &(*firstchainp)->next; } chain = *firstchainp; /* pass 1: make sure that the pointers and lengths of vecs[] are in * bounds before we try to commit anything. */ for (i=0; i<n_vecs; ++i) { if (!chain) goto done; if (vec[i].iov_base != (void *)CHAIN_SPACE_PTR(chain) || (size_t)vec[i].iov_len > CHAIN_SPACE_LEN(chain)) goto done; chain = chain->next; } /* pass 2: actually adjust all the chains. */ chainp = firstchainp; for (i=0; i<n_vecs; ++i) { (*chainp)->off += vec[i].iov_len; added += vec[i].iov_len; if (vec[i].iov_len) { buf->last_with_datap = chainp; } chainp = &(*chainp)->next; } okay: buf->total_len += added; buf->n_add_for_cb += added; result = 0; evbuffer_invoke_callbacks_(buf); done: EVBUFFER_UNLOCK(buf); return result; } static inline int HAS_PINNED_R(struct evbuffer *buf) { return (buf->last && CHAIN_PINNED_R(buf->last)); } static inline void ZERO_CHAIN(struct evbuffer *dst) { ASSERT_EVBUFFER_LOCKED(dst); dst->first = NULL; dst->last = NULL; dst->last_with_datap = &(dst)->first; dst->total_len = 0; } /* Prepares the contents of src to be moved to another buffer by removing * read-pinned chains. The first pinned chain is saved in first, and the * last in last. If src has no read-pinned chains, first and last are set * to NULL. */ static int PRESERVE_PINNED(struct evbuffer *src, struct evbuffer_chain **first, struct evbuffer_chain **last) { struct evbuffer_chain *chain, **pinned; ASSERT_EVBUFFER_LOCKED(src); if (!HAS_PINNED_R(src)) { *first = *last = NULL; return 0; } pinned = src->last_with_datap; if (!CHAIN_PINNED_R(*pinned)) pinned = &(*pinned)->next; EVUTIL_ASSERT(CHAIN_PINNED_R(*pinned)); chain = *first = *pinned; *last = src->last; /* If there's data in the first pinned chain, we need to allocate * a new chain and copy the data over. */ if (chain->off) { struct evbuffer_chain *tmp; EVUTIL_ASSERT(pinned == src->last_with_datap); tmp = evbuffer_chain_new(chain->off); if (!tmp) return -1; memcpy(tmp->buffer, chain->buffer + chain->misalign, chain->off); tmp->off = chain->off; *src->last_with_datap = tmp; src->last = tmp; chain->misalign += chain->off; chain->off = 0; } else { src->last = *src->last_with_datap; *pinned = NULL; } return 0; } static inline void RESTORE_PINNED(struct evbuffer *src, struct evbuffer_chain *pinned, struct evbuffer_chain *last) { ASSERT_EVBUFFER_LOCKED(src); if (!pinned) { ZERO_CHAIN(src); return; } src->first = pinned; src->last = last; src->last_with_datap = &src->first; src->total_len = 0; } static inline void COPY_CHAIN(struct evbuffer *dst, struct evbuffer *src) { ASSERT_EVBUFFER_LOCKED(dst); ASSERT_EVBUFFER_LOCKED(src); dst->first = src->first; if (src->last_with_datap == &src->first) dst->last_with_datap = &dst->first; else dst->last_with_datap = src->last_with_datap; dst->last = src->last; dst->total_len = src->total_len; } static void APPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src) { struct evbuffer_chain **chp; ASSERT_EVBUFFER_LOCKED(dst); ASSERT_EVBUFFER_LOCKED(src); chp = evbuffer_free_trailing_empty_chains(dst); *chp = src->first; if (src->last_with_datap == &src->first) dst->last_with_datap = chp; else dst->last_with_datap = src->last_with_datap; dst->last = src->last; dst->total_len += src->total_len; } static inline void APPEND_CHAIN_MULTICAST(struct evbuffer *dst, struct evbuffer *src) { struct evbuffer_chain *tmp; struct evbuffer_chain *chain = src->first; struct evbuffer_multicast_parent *extra; ASSERT_EVBUFFER_LOCKED(dst); ASSERT_EVBUFFER_LOCKED(src); for (; chain; chain = chain->next) { if (!chain->off || chain->flags & EVBUFFER_DANGLING) { /* skip empty chains */ continue; } tmp = evbuffer_chain_new(sizeof(struct evbuffer_multicast_parent)); if (!tmp) { event_warn("%s: out of memory", __func__); return; } extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_multicast_parent, tmp); /* reference evbuffer containing source chain so it * doesn't get released while the chain is still * being referenced to */ evbuffer_incref_(src); extra->source = src; /* reference source chain which now becomes immutable */ evbuffer_chain_incref(chain); extra->parent = chain; chain->flags |= EVBUFFER_IMMUTABLE; tmp->buffer_len = chain->buffer_len; tmp->misalign = chain->misalign; tmp->off = chain->off; tmp->flags |= EVBUFFER_MULTICAST|EVBUFFER_IMMUTABLE; tmp->buffer = chain->buffer; evbuffer_chain_insert(dst, tmp); } } static void PREPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src) { ASSERT_EVBUFFER_LOCKED(dst); ASSERT_EVBUFFER_LOCKED(src); src->last->next = dst->first; dst->first = src->first; dst->total_len += src->total_len; if (*dst->last_with_datap == NULL) { if (src->last_with_datap == &(src)->first) dst->last_with_datap = &dst->first; else dst->last_with_datap = src->last_with_datap; } else if (dst->last_with_datap == &dst->first) { dst->last_with_datap = &src->last->next; } } int evbuffer_add_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf) { struct evbuffer_chain *pinned, *last; size_t in_total_len, out_total_len; int result = 0; EVBUFFER_LOCK2(inbuf, outbuf); in_total_len = inbuf->total_len; out_total_len = outbuf->total_len; if (in_total_len == 0 || outbuf == inbuf) goto done; if (outbuf->freeze_end || inbuf->freeze_start) { result = -1; goto done; } if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) { result = -1; goto done; } if (out_total_len == 0) { /* There might be an empty chain at the start of outbuf; free * it. */ evbuffer_free_all_chains(outbuf->first); COPY_CHAIN(outbuf, inbuf); } else { APPEND_CHAIN(outbuf, inbuf); } RESTORE_PINNED(inbuf, pinned, last); inbuf->n_del_for_cb += in_total_len; outbuf->n_add_for_cb += in_total_len; evbuffer_invoke_callbacks_(inbuf); evbuffer_invoke_callbacks_(outbuf); done: EVBUFFER_UNLOCK2(inbuf, outbuf); return result; } int evbuffer_add_buffer_reference(struct evbuffer *outbuf, struct evbuffer *inbuf) { size_t in_total_len, out_total_len; struct evbuffer_chain *chain; int result = 0; EVBUFFER_LOCK2(inbuf, outbuf); in_total_len = inbuf->total_len; out_total_len = outbuf->total_len; chain = inbuf->first; if (in_total_len == 0) goto done; if (outbuf->freeze_end || outbuf == inbuf) { result = -1; goto done; } for (; chain; chain = chain->next) { if ((chain->flags & (EVBUFFER_FILESEGMENT|EVBUFFER_SENDFILE|EVBUFFER_MULTICAST)) != 0) { /* chain type can not be referenced */ result = -1; goto done; } } if (out_total_len == 0) { /* There might be an empty chain at the start of outbuf; free * it. */ evbuffer_free_all_chains(outbuf->first); } APPEND_CHAIN_MULTICAST(outbuf, inbuf); outbuf->n_add_for_cb += in_total_len; evbuffer_invoke_callbacks_(outbuf); done: EVBUFFER_UNLOCK2(inbuf, outbuf); return result; } int evbuffer_prepend_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf) { struct evbuffer_chain *pinned, *last; size_t in_total_len, out_total_len; int result = 0; EVBUFFER_LOCK2(inbuf, outbuf); in_total_len = inbuf->total_len; out_total_len = outbuf->total_len; if (!in_total_len || inbuf == outbuf) goto done; if (outbuf->freeze_start || inbuf->freeze_start) { result = -1; goto done; } if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) { result = -1; goto done; } if (out_total_len == 0) { /* There might be an empty chain at the start of outbuf; free * it. */ evbuffer_free_all_chains(outbuf->first); COPY_CHAIN(outbuf, inbuf); } else { PREPEND_CHAIN(outbuf, inbuf); } RESTORE_PINNED(inbuf, pinned, last); inbuf->n_del_for_cb += in_total_len; outbuf->n_add_for_cb += in_total_len; evbuffer_invoke_callbacks_(inbuf); evbuffer_invoke_callbacks_(outbuf); done: EVBUFFER_UNLOCK2(inbuf, outbuf); return result; } int evbuffer_drain(struct evbuffer *buf, size_t len) { struct evbuffer_chain *chain, *next; size_t remaining, old_len; int result = 0; EVBUFFER_LOCK(buf); old_len = buf->total_len; if (old_len == 0) goto done; if (buf->freeze_start) { result = -1; goto done; } if (len >= old_len && !HAS_PINNED_R(buf)) { len = old_len; for (chain = buf->first; chain != NULL; chain = next) { next = chain->next; evbuffer_chain_free(chain); } ZERO_CHAIN(buf); } else { if (len >= old_len) len = old_len; buf->total_len -= len; remaining = len; for (chain = buf->first; remaining >= chain->off; chain = next) { next = chain->next; remaining -= chain->off; if (chain == *buf->last_with_datap) { buf->last_with_datap = &buf->first; } if (&chain->next == buf->last_with_datap) buf->last_with_datap = &buf->first; if (CHAIN_PINNED_R(chain)) { EVUTIL_ASSERT(remaining == 0); chain->misalign += chain->off; chain->off = 0; break; } else evbuffer_chain_free(chain); } buf->first = chain; EVUTIL_ASSERT(chain && remaining <= chain->off); chain->misalign += remaining; chain->off -= remaining; } buf->n_del_for_cb += len; /* Tell someone about changes in this buffer */ evbuffer_invoke_callbacks_(buf); done: EVBUFFER_UNLOCK(buf); return result; } /* Reads data from an event buffer and drains the bytes read */ int evbuffer_remove(struct evbuffer *buf, void *data_out, size_t datlen) { ev_ssize_t n; EVBUFFER_LOCK(buf); n = evbuffer_copyout_from(buf, NULL, data_out, datlen); if (n > 0) { if (evbuffer_drain(buf, n)<0) n = -1; } EVBUFFER_UNLOCK(buf); return (int)n; } ev_ssize_t evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen) { return evbuffer_copyout_from(buf, NULL, data_out, datlen); } ev_ssize_t evbuffer_copyout_from(struct evbuffer *buf, const struct evbuffer_ptr *pos, void *data_out, size_t datlen) { /*XXX fails badly on sendfile case. */ struct evbuffer_chain *chain; char *data = data_out; size_t nread; ev_ssize_t result = 0; size_t pos_in_chain; EVBUFFER_LOCK(buf); if (pos) { if (datlen > (size_t)(EV_SSIZE_MAX - pos->pos)) { result = -1; goto done; } chain = pos->internal_.chain; pos_in_chain = pos->internal_.pos_in_chain; if (datlen + pos->pos > buf->total_len) datlen = buf->total_len - pos->pos; } else { chain = buf->first; pos_in_chain = 0; if (datlen > buf->total_len) datlen = buf->total_len; } if (datlen == 0) goto done; if (buf->freeze_start) { result = -1; goto done; } nread = datlen; while (datlen && datlen >= chain->off - pos_in_chain) { size_t copylen = chain->off - pos_in_chain; memcpy(data, chain->buffer + chain->misalign + pos_in_chain, copylen); data += copylen; datlen -= copylen; chain = chain->next; pos_in_chain = 0; EVUTIL_ASSERT(chain || datlen==0); } if (datlen) { EVUTIL_ASSERT(chain); EVUTIL_ASSERT(datlen+pos_in_chain <= chain->off); memcpy(data, chain->buffer + chain->misalign + pos_in_chain, datlen); } result = nread; done: EVBUFFER_UNLOCK(buf); return result; } /* reads data from the src buffer to the dst buffer, avoids memcpy as * possible. */ /* XXXX should return ev_ssize_t */ int evbuffer_remove_buffer(struct evbuffer *src, struct evbuffer *dst, size_t datlen) { /*XXX We should have an option to force this to be zero-copy.*/ /*XXX can fail badly on sendfile case. */ struct evbuffer_chain *chain, *previous; size_t nread = 0; int result; EVBUFFER_LOCK2(src, dst); chain = previous = src->first; if (datlen == 0 || dst == src) { result = 0; goto done; } if (dst->freeze_end || src->freeze_start) { result = -1; goto done; } /* short-cut if there is no more data buffered */ if (datlen >= src->total_len) { datlen = src->total_len; evbuffer_add_buffer(dst, src); result = (int)datlen; /*XXXX should return ev_ssize_t*/ goto done; } /* removes chains if possible */ while (chain->off <= datlen) { /* We can't remove the last with data from src unless we * remove all chains, in which case we would have done the if * block above */ EVUTIL_ASSERT(chain != *src->last_with_datap); nread += chain->off; datlen -= chain->off; previous = chain; if (src->last_with_datap == &chain->next) src->last_with_datap = &src->first; chain = chain->next; } if (nread) { /* we can remove the chain */ struct evbuffer_chain **chp; chp = evbuffer_free_trailing_empty_chains(dst); if (dst->first == NULL) { dst->first = src->first; } else { *chp = src->first; } dst->last = previous; previous->next = NULL; src->first = chain; advance_last_with_data(dst); dst->total_len += nread; dst->n_add_for_cb += nread; } /* we know that there is more data in the src buffer than * we want to read, so we manually drain the chain */ evbuffer_add(dst, chain->buffer + chain->misalign, datlen); chain->misalign += datlen; chain->off -= datlen; nread += datlen; /* You might think we would want to increment dst->n_add_for_cb * here too. But evbuffer_add above already took care of that. */ src->total_len -= nread; src->n_del_for_cb += nread; if (nread) { evbuffer_invoke_callbacks_(dst); evbuffer_invoke_callbacks_(src); } result = (int)nread;/*XXXX should change return type */ done: EVBUFFER_UNLOCK2(src, dst); return result; } unsigned char * evbuffer_pullup(struct evbuffer *buf, ev_ssize_t size) { struct evbuffer_chain *chain, *next, *tmp, *last_with_data; unsigned char *buffer, *result = NULL; ev_ssize_t remaining; int removed_last_with_data = 0; int removed_last_with_datap = 0; EVBUFFER_LOCK(buf); chain = buf->first; if (size < 0) size = buf->total_len; /* if size > buf->total_len, we cannot guarantee to the user that she * is going to have a long enough buffer afterwards; so we return * NULL */ if (size == 0 || (size_t)size > buf->total_len) goto done; /* No need to pull up anything; the first size bytes are * already here. */ if (chain->off >= (size_t)size) { result = chain->buffer + chain->misalign; goto done; } /* Make sure that none of the chains we need to copy from is pinned. */ remaining = size - chain->off; EVUTIL_ASSERT(remaining >= 0); for (tmp=chain->next; tmp; tmp=tmp->next) { if (CHAIN_PINNED(tmp)) goto done; if (tmp->off >= (size_t)remaining) break; remaining -= tmp->off; } if (CHAIN_PINNED(chain)) { size_t old_off = chain->off; if (CHAIN_SPACE_LEN(chain) < size - chain->off) { /* not enough room at end of chunk. */ goto done; } buffer = CHAIN_SPACE_PTR(chain); tmp = chain; tmp->off = size; size -= old_off; chain = chain->next; } else if (chain->buffer_len - chain->misalign >= (size_t)size) { /* already have enough space in the first chain */ size_t old_off = chain->off; buffer = chain->buffer + chain->misalign + chain->off; tmp = chain; tmp->off = size; size -= old_off; chain = chain->next; } else { if ((tmp = evbuffer_chain_new(size)) == NULL) { event_warn("%s: out of memory", __func__); goto done; } buffer = tmp->buffer; tmp->off = size; buf->first = tmp; } /* TODO(niels): deal with buffers that point to NULL like sendfile */ /* Copy and free every chunk that will be entirely pulled into tmp */ last_with_data = *buf->last_with_datap; for (; chain != NULL && (size_t)size >= chain->off; chain = next) { next = chain->next; memcpy(buffer, chain->buffer + chain->misalign, chain->off); size -= chain->off; buffer += chain->off; if (chain == last_with_data) removed_last_with_data = 1; if (&chain->next == buf->last_with_datap) removed_last_with_datap = 1; evbuffer_chain_free(chain); } if (chain != NULL) { memcpy(buffer, chain->buffer + chain->misalign, size); chain->misalign += size; chain->off -= size; } else { buf->last = tmp; } tmp->next = chain; if (removed_last_with_data) { buf->last_with_datap = &buf->first; } else if (removed_last_with_datap) { if (buf->first->next && buf->first->next->off) buf->last_with_datap = &buf->first->next; else buf->last_with_datap = &buf->first; } result = (tmp->buffer + tmp->misalign); done: EVBUFFER_UNLOCK(buf); return result; } /* * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'. * The returned buffer needs to be freed by the called. */ char * evbuffer_readline(struct evbuffer *buffer) { return evbuffer_readln(buffer, NULL, EVBUFFER_EOL_ANY); } static inline ev_ssize_t evbuffer_strchr(struct evbuffer_ptr *it, const char chr) { struct evbuffer_chain *chain = it->internal_.chain; size_t i = it->internal_.pos_in_chain; while (chain != NULL) { char *buffer = (char *)chain->buffer + chain->misalign; char *cp = memchr(buffer+i, chr, chain->off-i); if (cp) { it->internal_.chain = chain; it->internal_.pos_in_chain = cp - buffer; it->pos += (cp - buffer - i); return it->pos; } it->pos += chain->off - i; i = 0; chain = chain->next; } return (-1); } static inline char * find_eol_char(char *s, size_t len) { #define CHUNK_SZ 128 /* Lots of benchmarking found this approach to be faster in practice * than doing two memchrs over the whole buffer, doin a memchr on each * char of the buffer, or trying to emulate memchr by hand. */ char *s_end, *cr, *lf; s_end = s+len; while (s < s_end) { size_t chunk = (s + CHUNK_SZ < s_end) ? CHUNK_SZ : (s_end - s); cr = memchr(s, '\r', chunk); lf = memchr(s, '\n', chunk); if (cr) { if (lf && lf < cr) return lf; return cr; } else if (lf) { return lf; } s += CHUNK_SZ; } return NULL; #undef CHUNK_SZ } static ev_ssize_t evbuffer_find_eol_char(struct evbuffer_ptr *it) { struct evbuffer_chain *chain = it->internal_.chain; size_t i = it->internal_.pos_in_chain; while (chain != NULL) { char *buffer = (char *)chain->buffer + chain->misalign; char *cp = find_eol_char(buffer+i, chain->off-i); if (cp) { it->internal_.chain = chain; it->internal_.pos_in_chain = cp - buffer; it->pos += (cp - buffer) - i; return it->pos; } it->pos += chain->off - i; i = 0; chain = chain->next; } return (-1); } static inline int evbuffer_strspn( struct evbuffer_ptr *ptr, const char *chrset) { int count = 0; struct evbuffer_chain *chain = ptr->internal_.chain; size_t i = ptr->internal_.pos_in_chain; if (!chain) return 0; while (1) { char *buffer = (char *)chain->buffer + chain->misalign; for (; i < chain->off; ++i) { const char *p = chrset; while (*p) { if (buffer[i] == *p++) goto next; } ptr->internal_.chain = chain; ptr->internal_.pos_in_chain = i; ptr->pos += count; return count; next: ++count; } i = 0; if (! chain->next) { ptr->internal_.chain = chain; ptr->internal_.pos_in_chain = i; ptr->pos += count; return count; } chain = chain->next; } } static inline int evbuffer_getchr(struct evbuffer_ptr *it) { struct evbuffer_chain *chain = it->internal_.chain; size_t off = it->internal_.pos_in_chain; if (chain == NULL) return -1; return (unsigned char)chain->buffer[chain->misalign + off]; } struct evbuffer_ptr evbuffer_search_eol(struct evbuffer *buffer, struct evbuffer_ptr *start, size_t *eol_len_out, enum evbuffer_eol_style eol_style) { struct evbuffer_ptr it, it2; size_t extra_drain = 0; int ok = 0; /* Avoid locking in trivial edge cases */ if (start && start->internal_.chain == NULL) { PTR_NOT_FOUND(&it); if (eol_len_out) *eol_len_out = extra_drain; return it; } EVBUFFER_LOCK(buffer); if (start) { memcpy(&it, start, sizeof(it)); } else { it.pos = 0; it.internal_.chain = buffer->first; it.internal_.pos_in_chain = 0; } /* the eol_style determines our first stop character and how many * characters we are going to drain afterwards. */ switch (eol_style) { case EVBUFFER_EOL_ANY: if (evbuffer_find_eol_char(&it) < 0) goto done; memcpy(&it2, &it, sizeof(it)); extra_drain = evbuffer_strspn(&it2, "\r\n"); break; case EVBUFFER_EOL_CRLF_STRICT: { it = evbuffer_search(buffer, "\r\n", 2, &it); if (it.pos < 0) goto done; extra_drain = 2; break; } case EVBUFFER_EOL_CRLF: { ev_ssize_t start_pos = it.pos; /* Look for a LF ... */ if (evbuffer_strchr(&it, '\n') < 0) goto done; extra_drain = 1; /* ... optionally preceeded by a CR. */ if (it.pos == start_pos) break; /* If the first character is \n, don't back up */ /* This potentially does an extra linear walk over the first * few chains. Probably, that's not too expensive unless you * have a really pathological setup. */ memcpy(&it2, &it, sizeof(it)); if (evbuffer_ptr_subtract(buffer, &it2, 1)<0) break; if (evbuffer_getchr(&it2) == '\r') { memcpy(&it, &it2, sizeof(it)); extra_drain = 2; } break; } case EVBUFFER_EOL_LF: if (evbuffer_strchr(&it, '\n') < 0) goto done; extra_drain = 1; break; case EVBUFFER_EOL_NUL: if (evbuffer_strchr(&it, '\0') < 0) goto done; extra_drain = 1; break; default: goto done; } ok = 1; done: EVBUFFER_UNLOCK(buffer); if (!ok) PTR_NOT_FOUND(&it); if (eol_len_out) *eol_len_out = extra_drain; return it; } char * evbuffer_readln(struct evbuffer *buffer, size_t *n_read_out, enum evbuffer_eol_style eol_style) { struct evbuffer_ptr it; char *line; size_t n_to_copy=0, extra_drain=0; char *result = NULL; EVBUFFER_LOCK(buffer); if (buffer->freeze_start) { goto done; } it = evbuffer_search_eol(buffer, NULL, &extra_drain, eol_style); if (it.pos < 0) goto done; n_to_copy = it.pos; if ((line = mm_malloc(n_to_copy+1)) == NULL) { event_warn("%s: out of memory", __func__); goto done; } evbuffer_remove(buffer, line, n_to_copy); line[n_to_copy] = '\0'; evbuffer_drain(buffer, extra_drain); result = line; done: EVBUFFER_UNLOCK(buffer); if (n_read_out) *n_read_out = result ? n_to_copy : 0; return result; } #define EVBUFFER_CHAIN_MAX_AUTO_SIZE 4096 /* Adds data to an event buffer */ int evbuffer_add(struct evbuffer *buf, const void *data_in, size_t datlen) { struct evbuffer_chain *chain, *tmp; const unsigned char *data = data_in; size_t remain, to_alloc; int result = -1; EVBUFFER_LOCK(buf); if (buf->freeze_end) { goto done; } /* Prevent buf->total_len overflow */ if (datlen > EV_SIZE_MAX - buf->total_len) { goto done; } if (*buf->last_with_datap == NULL) { chain = buf->last; } else { chain = *buf->last_with_datap; } /* If there are no chains allocated for this buffer, allocate one * big enough to hold all the data. */ if (chain == NULL) { chain = evbuffer_chain_new(datlen); if (!chain) goto done; evbuffer_chain_insert(buf, chain); } if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) { /* Always true for mutable buffers */ EVUTIL_ASSERT(chain->misalign >= 0 && (ev_uint64_t)chain->misalign <= EVBUFFER_CHAIN_MAX); remain = chain->buffer_len - (size_t)chain->misalign - chain->off; if (remain >= datlen) { /* there's enough space to hold all the data in the * current last chain */ memcpy(chain->buffer + chain->misalign + chain->off, data, datlen); chain->off += datlen; buf->total_len += datlen; buf->n_add_for_cb += datlen; goto out; } else if (!CHAIN_PINNED(chain) && evbuffer_chain_should_realign(chain, datlen)) { /* we can fit the data into the misalignment */ evbuffer_chain_align(chain); memcpy(chain->buffer + chain->off, data, datlen); chain->off += datlen; buf->total_len += datlen; buf->n_add_for_cb += datlen; goto out; } } else { /* we cannot write any data to the last chain */ remain = 0; } /* we need to add another chain */ to_alloc = chain->buffer_len; if (to_alloc <= EVBUFFER_CHAIN_MAX_AUTO_SIZE/2) to_alloc <<= 1; if (datlen > to_alloc) to_alloc = datlen; tmp = evbuffer_chain_new(to_alloc); if (tmp == NULL) goto done; if (remain) { memcpy(chain->buffer + chain->misalign + chain->off, data, remain); chain->off += remain; buf->total_len += remain; buf->n_add_for_cb += remain; } data += remain; datlen -= remain; memcpy(tmp->buffer, data, datlen); tmp->off = datlen; evbuffer_chain_insert(buf, tmp); buf->n_add_for_cb += datlen; out: evbuffer_invoke_callbacks_(buf); result = 0; done: EVBUFFER_UNLOCK(buf); return result; } int evbuffer_prepend(struct evbuffer *buf, const void *data, size_t datlen) { struct evbuffer_chain *chain, *tmp; int result = -1; EVBUFFER_LOCK(buf); if (buf->freeze_start) { goto done; } if (datlen > EV_SIZE_MAX - buf->total_len) { goto done; } chain = buf->first; if (chain == NULL) { chain = evbuffer_chain_new(datlen); if (!chain) goto done; evbuffer_chain_insert(buf, chain); } /* we cannot touch immutable buffers */ if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) { /* Always true for mutable buffers */ EVUTIL_ASSERT(chain->misalign >= 0 && (ev_uint64_t)chain->misalign <= EVBUFFER_CHAIN_MAX); /* If this chain is empty, we can treat it as * 'empty at the beginning' rather than 'empty at the end' */ if (chain->off == 0) chain->misalign = chain->buffer_len; if ((size_t)chain->misalign >= datlen) { /* we have enough space to fit everything */ memcpy(chain->buffer + chain->misalign - datlen, data, datlen); chain->off += datlen; chain->misalign -= datlen; buf->total_len += datlen; buf->n_add_for_cb += datlen; goto out; } else if (chain->misalign) { /* we can only fit some of the data. */ memcpy(chain->buffer, (char*)data + datlen - chain->misalign, (size_t)chain->misalign); chain->off += (size_t)chain->misalign; buf->total_len += (size_t)chain->misalign; buf->n_add_for_cb += (size_t)chain->misalign; datlen -= (size_t)chain->misalign; chain->misalign = 0; } } /* we need to add another chain */ if ((tmp = evbuffer_chain_new(datlen)) == NULL) goto done; buf->first = tmp; if (buf->last_with_datap == &buf->first) buf->last_with_datap = &tmp->next; tmp->next = chain; tmp->off = datlen; EVUTIL_ASSERT(datlen <= tmp->buffer_len); tmp->misalign = tmp->buffer_len - datlen; memcpy(tmp->buffer + tmp->misalign, data, datlen); buf->total_len += datlen; buf->n_add_for_cb += datlen; out: evbuffer_invoke_callbacks_(buf); result = 0; done: EVBUFFER_UNLOCK(buf); return result; } /** Helper: realigns the memory in chain->buffer so that misalign is 0. */ static void evbuffer_chain_align(struct evbuffer_chain *chain) { EVUTIL_ASSERT(!(chain->flags & EVBUFFER_IMMUTABLE)); EVUTIL_ASSERT(!(chain->flags & EVBUFFER_MEM_PINNED_ANY)); memmove(chain->buffer, chain->buffer + chain->misalign, chain->off); chain->misalign = 0; } #define MAX_TO_COPY_IN_EXPAND 4096 #define MAX_TO_REALIGN_IN_EXPAND 2048 /** Helper: return true iff we should realign chain to fit datalen bytes of data in it. */ static int evbuffer_chain_should_realign(struct evbuffer_chain *chain, size_t datlen) { return chain->buffer_len - chain->off >= datlen && (chain->off < chain->buffer_len / 2) && (chain->off <= MAX_TO_REALIGN_IN_EXPAND); } /* Expands the available space in the event buffer to at least datlen, all in * a single chunk. Return that chunk. */ static struct evbuffer_chain * evbuffer_expand_singlechain(struct evbuffer *buf, size_t datlen) { struct evbuffer_chain *chain, **chainp; struct evbuffer_chain *result = NULL; ASSERT_EVBUFFER_LOCKED(buf); chainp = buf->last_with_datap; /* XXX If *chainp is no longer writeable, but has enough space in its * misalign, this might be a bad idea: we could still use *chainp, not * (*chainp)->next. */ if (*chainp && CHAIN_SPACE_LEN(*chainp) == 0) chainp = &(*chainp)->next; /* 'chain' now points to the first chain with writable space (if any) * We will either use it, realign it, replace it, or resize it. */ chain = *chainp; if (chain == NULL || (chain->flags & (EVBUFFER_IMMUTABLE|EVBUFFER_MEM_PINNED_ANY))) { /* We can't use the last_with_data chain at all. Just add a * new one that's big enough. */ goto insert_new; } /* If we can fit all the data, then we don't have to do anything */ if (CHAIN_SPACE_LEN(chain) >= datlen) { result = chain; goto ok; } /* If the chain is completely empty, just replace it by adding a new * empty chain. */ if (chain->off == 0) { goto insert_new; } /* If the misalignment plus the remaining space fulfills our data * needs, we could just force an alignment to happen. Afterwards, we * have enough space. But only do this if we're saving a lot of space * and not moving too much data. Otherwise the space savings are * probably offset by the time lost in copying. */ if (evbuffer_chain_should_realign(chain, datlen)) { evbuffer_chain_align(chain); result = chain; goto ok; } /* At this point, we can either resize the last chunk with space in * it, use the next chunk after it, or If we add a new chunk, we waste * CHAIN_SPACE_LEN(chain) bytes in the former last chunk. If we * resize, we have to copy chain->off bytes. */ /* Would expanding this chunk be affordable and worthwhile? */ if (CHAIN_SPACE_LEN(chain) < chain->buffer_len / 8 || chain->off > MAX_TO_COPY_IN_EXPAND || datlen >= (EVBUFFER_CHAIN_MAX - chain->off)) { /* It's not worth resizing this chain. Can the next one be * used? */ if (chain->next && CHAIN_SPACE_LEN(chain->next) >= datlen) { /* Yes, we can just use the next chain (which should * be empty. */ result = chain->next; goto ok; } else { /* No; append a new chain (which will free all * terminal empty chains.) */ goto insert_new; } } else { /* Okay, we're going to try to resize this chain: Not doing so * would waste at least 1/8 of its current allocation, and we * can do so without having to copy more than * MAX_TO_COPY_IN_EXPAND bytes. */ /* figure out how much space we need */ size_t length = chain->off + datlen; struct evbuffer_chain *tmp = evbuffer_chain_new(length); if (tmp == NULL) goto err; /* copy the data over that we had so far */ tmp->off = chain->off; memcpy(tmp->buffer, chain->buffer + chain->misalign, chain->off); /* fix up the list */ EVUTIL_ASSERT(*chainp == chain); result = *chainp = tmp; if (buf->last == chain) buf->last = tmp; tmp->next = chain->next; evbuffer_chain_free(chain); goto ok; } insert_new: result = evbuffer_chain_insert_new(buf, datlen); if (!result) goto err; ok: EVUTIL_ASSERT(result); EVUTIL_ASSERT(CHAIN_SPACE_LEN(result) >= datlen); err: return result; } /* Make sure that datlen bytes are available for writing in the last n * chains. Never copies or moves data. */ int evbuffer_expand_fast_(struct evbuffer *buf, size_t datlen, int n) { struct evbuffer_chain *chain = buf->last, *tmp, *next; size_t avail; int used; ASSERT_EVBUFFER_LOCKED(buf); EVUTIL_ASSERT(n >= 2); if (chain == NULL || (chain->flags & EVBUFFER_IMMUTABLE)) { /* There is no last chunk, or we can't touch the last chunk. * Just add a new chunk. */ chain = evbuffer_chain_new(datlen); if (chain == NULL) return (-1); evbuffer_chain_insert(buf, chain); return (0); } used = 0; /* number of chains we're using space in. */ avail = 0; /* how much space they have. */ /* How many bytes can we stick at the end of buffer as it is? Iterate * over the chains at the end of the buffer, tring to see how much * space we have in the first n. */ for (chain = *buf->last_with_datap; chain; chain = chain->next) { if (chain->off) { size_t space = (size_t) CHAIN_SPACE_LEN(chain); EVUTIL_ASSERT(chain == *buf->last_with_datap); if (space) { avail += space; ++used; } } else { /* No data in chain; realign it. */ chain->misalign = 0; avail += chain->buffer_len; ++used; } if (avail >= datlen) { /* There is already enough space. Just return */ return (0); } if (used == n) break; } /* There wasn't enough space in the first n chains with space in * them. Either add a new chain with enough space, or replace all * empty chains with one that has enough space, depending on n. */ if (used < n) { /* The loop ran off the end of the chains before it hit n * chains; we can add another. */ EVUTIL_ASSERT(chain == NULL); tmp = evbuffer_chain_new(datlen - avail); if (tmp == NULL) return (-1); buf->last->next = tmp; buf->last = tmp; /* (we would only set last_with_data if we added the first * chain. But if the buffer had no chains, we would have * just allocated a new chain earlier) */ return (0); } else { /* Nuke _all_ the empty chains. */ int rmv_all = 0; /* True iff we removed last_with_data. */ chain = *buf->last_with_datap; if (!chain->off) { EVUTIL_ASSERT(chain == buf->first); rmv_all = 1; avail = 0; } else { /* can't overflow, since only mutable chains have * huge misaligns. */ avail = (size_t) CHAIN_SPACE_LEN(chain); chain = chain->next; } for (; chain; chain = next) { next = chain->next; EVUTIL_ASSERT(chain->off == 0); evbuffer_chain_free(chain); } EVUTIL_ASSERT(datlen >= avail); tmp = evbuffer_chain_new(datlen - avail); if (tmp == NULL) { if (rmv_all) { ZERO_CHAIN(buf); } else { buf->last = *buf->last_with_datap; (*buf->last_with_datap)->next = NULL; } return (-1); } if (rmv_all) { buf->first = buf->last = tmp; buf->last_with_datap = &buf->first; } else { (*buf->last_with_datap)->next = tmp; buf->last = tmp; } return (0); } } int evbuffer_expand(struct evbuffer *buf, size_t datlen) { struct evbuffer_chain *chain; EVBUFFER_LOCK(buf); chain = evbuffer_expand_singlechain(buf, datlen); EVBUFFER_UNLOCK(buf); return chain ? 0 : -1; } /* * Reads data from a file descriptor into a buffer. */ #if defined(EVENT__HAVE_SYS_UIO_H) || defined(_WIN32) #define USE_IOVEC_IMPL #endif #ifdef USE_IOVEC_IMPL #ifdef EVENT__HAVE_SYS_UIO_H /* number of iovec we use for writev, fragmentation is going to determine * how much we end up writing */ #define DEFAULT_WRITE_IOVEC 128 #if defined(UIO_MAXIOV) && UIO_MAXIOV < DEFAULT_WRITE_IOVEC #define NUM_WRITE_IOVEC UIO_MAXIOV #elif defined(IOV_MAX) && IOV_MAX < DEFAULT_WRITE_IOVEC #define NUM_WRITE_IOVEC IOV_MAX #else #define NUM_WRITE_IOVEC DEFAULT_WRITE_IOVEC #endif #define IOV_TYPE struct iovec #define IOV_PTR_FIELD iov_base #define IOV_LEN_FIELD iov_len #define IOV_LEN_TYPE size_t #else #define NUM_WRITE_IOVEC 16 #define IOV_TYPE WSABUF #define IOV_PTR_FIELD buf #define IOV_LEN_FIELD len #define IOV_LEN_TYPE unsigned long #endif #endif #define NUM_READ_IOVEC 4 #define EVBUFFER_MAX_READ 4096 /** Helper function to figure out which space to use for reading data into an evbuffer. Internal use only. @param buf The buffer to read into @param howmuch How much we want to read. @param vecs An array of two or more iovecs or WSABUFs. @param n_vecs_avail The length of vecs @param chainp A pointer to a variable to hold the first chain we're reading into. @param exact Boolean: if true, we do not provide more than 'howmuch' space in the vectors, even if more space is available. @return The number of buffers we're using. */ int evbuffer_read_setup_vecs_(struct evbuffer *buf, ev_ssize_t howmuch, struct evbuffer_iovec *vecs, int n_vecs_avail, struct evbuffer_chain ***chainp, int exact) { struct evbuffer_chain *chain; struct evbuffer_chain **firstchainp; size_t so_far; int i; ASSERT_EVBUFFER_LOCKED(buf); if (howmuch < 0) return -1; so_far = 0; /* Let firstchain be the first chain with any space on it */ firstchainp = buf->last_with_datap; if (CHAIN_SPACE_LEN(*firstchainp) == 0) { firstchainp = &(*firstchainp)->next; } chain = *firstchainp; for (i = 0; i < n_vecs_avail && so_far < (size_t)howmuch; ++i) { size_t avail = (size_t) CHAIN_SPACE_LEN(chain); if (avail > (howmuch - so_far) && exact) avail = howmuch - so_far; vecs[i].iov_base = (void *)CHAIN_SPACE_PTR(chain); vecs[i].iov_len = avail; so_far += avail; chain = chain->next; } *chainp = firstchainp; return i; } static int get_n_bytes_readable_on_socket(evutil_socket_t fd) { #if defined(FIONREAD) && defined(_WIN32) unsigned long lng = EVBUFFER_MAX_READ; if (ioctlsocket(fd, FIONREAD, &lng) < 0) return -1; /* Can overflow, but mostly harmlessly. XXXX */ return (int)lng; #elif defined(FIONREAD) int n = EVBUFFER_MAX_READ; if (ioctl(fd, FIONREAD, &n) < 0) return -1; return n; #else return EVBUFFER_MAX_READ; #endif } /* TODO(niels): should this function return ev_ssize_t and take ev_ssize_t * as howmuch? */ int evbuffer_read(struct evbuffer *buf, evutil_socket_t fd, int howmuch) { struct evbuffer_chain **chainp; int n; int result; #ifdef USE_IOVEC_IMPL int nvecs, i, remaining; #else struct evbuffer_chain *chain; unsigned char *p; #endif EVBUFFER_LOCK(buf); if (buf->freeze_end) { result = -1; goto done; } n = get_n_bytes_readable_on_socket(fd); if (n <= 0 || n > EVBUFFER_MAX_READ) n = EVBUFFER_MAX_READ; if (howmuch < 0 || howmuch > n) howmuch = n; #ifdef USE_IOVEC_IMPL /* Since we can use iovecs, we're willing to use the last * NUM_READ_IOVEC chains. */ if (evbuffer_expand_fast_(buf, howmuch, NUM_READ_IOVEC) == -1) { result = -1; goto done; } else { IOV_TYPE vecs[NUM_READ_IOVEC]; #ifdef EVBUFFER_IOVEC_IS_NATIVE_ nvecs = evbuffer_read_setup_vecs_(buf, howmuch, vecs, NUM_READ_IOVEC, &chainp, 1); #else /* We aren't using the native struct iovec. Therefore, we are on win32. */ struct evbuffer_iovec ev_vecs[NUM_READ_IOVEC]; nvecs = evbuffer_read_setup_vecs_(buf, howmuch, ev_vecs, 2, &chainp, 1); for (i=0; i < nvecs; ++i) WSABUF_FROM_EVBUFFER_IOV(&vecs[i], &ev_vecs[i]); #endif #ifdef _WIN32 { DWORD bytesRead; DWORD flags=0; if (WSARecv(fd, vecs, nvecs, &bytesRead, &flags, NULL, NULL)) { /* The read failed. It might be a close, * or it might be an error. */ if (WSAGetLastError() == WSAECONNABORTED) n = 0; else n = -1; } else n = bytesRead; } #else n = readv(fd, vecs, nvecs); #endif } #else /*!USE_IOVEC_IMPL*/ /* If we don't have FIONREAD, we might waste some space here */ /* XXX we _will_ waste some space here if there is any space left * over on buf->last. */ if ((chain = evbuffer_expand_singlechain(buf, howmuch)) == NULL) { result = -1; goto done; } /* We can append new data at this point */ p = chain->buffer + chain->misalign + chain->off; #ifndef _WIN32 n = read(fd, p, howmuch); #else n = recv(fd, p, howmuch, 0); #endif #endif /* USE_IOVEC_IMPL */ if (n == -1) { result = -1; goto done; } if (n == 0) { result = 0; goto done; } #ifdef USE_IOVEC_IMPL remaining = n; for (i=0; i < nvecs; ++i) { /* can't overflow, since only mutable chains have * huge misaligns. */ size_t space = (size_t) CHAIN_SPACE_LEN(*chainp); /* XXXX This is a kludge that can waste space in perverse * situations. */ if (space > EVBUFFER_CHAIN_MAX) space = EVBUFFER_CHAIN_MAX; if ((ev_ssize_t)space < remaining) { (*chainp)->off += space; remaining -= (int)space; } else { (*chainp)->off += remaining; buf->last_with_datap = chainp; break; } chainp = &(*chainp)->next; } #else chain->off += n; advance_last_with_data(buf); #endif buf->total_len += n; buf->n_add_for_cb += n; /* Tell someone about changes in this buffer */ evbuffer_invoke_callbacks_(buf); result = n; done: EVBUFFER_UNLOCK(buf); return result; } #ifdef USE_IOVEC_IMPL static inline int evbuffer_write_iovec(struct evbuffer *buffer, evutil_socket_t fd, ev_ssize_t howmuch) { IOV_TYPE iov[NUM_WRITE_IOVEC]; struct evbuffer_chain *chain = buffer->first; int n, i = 0; if (howmuch < 0) return -1; ASSERT_EVBUFFER_LOCKED(buffer); /* XXX make this top out at some maximal data length? if the * buffer has (say) 1MB in it, split over 128 chains, there's * no way it all gets written in one go. */ while (chain != NULL && i < NUM_WRITE_IOVEC && howmuch) { #ifdef USE_SENDFILE /* we cannot write the file info via writev */ if (chain->flags & EVBUFFER_SENDFILE) break; #endif iov[i].IOV_PTR_FIELD = (void *) (chain->buffer + chain->misalign); if ((size_t)howmuch >= chain->off) { /* XXXcould be problematic when windows supports mmap*/ iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)chain->off; howmuch -= chain->off; } else { /* XXXcould be problematic when windows supports mmap*/ iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)howmuch; break; } chain = chain->next; } if (! i) return 0; #ifdef _WIN32 { DWORD bytesSent; if (WSASend(fd, iov, i, &bytesSent, 0, NULL, NULL)) n = -1; else n = bytesSent; } #else n = writev(fd, iov, i); #endif return (n); } #endif #ifdef USE_SENDFILE static inline int evbuffer_write_sendfile(struct evbuffer *buffer, evutil_socket_t dest_fd, ev_ssize_t howmuch) { struct evbuffer_chain *chain = buffer->first; struct evbuffer_chain_file_segment *info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment, chain); const int source_fd = info->segment->fd; #if defined(SENDFILE_IS_MACOSX) || defined(SENDFILE_IS_FREEBSD) int res; ev_off_t len = chain->off; #elif defined(SENDFILE_IS_LINUX) || defined(SENDFILE_IS_SOLARIS) ev_ssize_t res; off_t offset = chain->misalign; #endif ASSERT_EVBUFFER_LOCKED(buffer); #if defined(SENDFILE_IS_MACOSX) res = sendfile(source_fd, dest_fd, chain->misalign, &len, NULL, 0); if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno)) return (-1); return (len); #elif defined(SENDFILE_IS_FREEBSD) res = sendfile(source_fd, dest_fd, chain->misalign, chain->off, NULL, &len, 0); if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno)) return (-1); return (len); #elif defined(SENDFILE_IS_LINUX) /* TODO(niels): implement splice */ res = sendfile(dest_fd, source_fd, &offset, chain->off); if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) { /* if this is EAGAIN or EINTR return 0; otherwise, -1 */ return (0); } return (res); #elif defined(SENDFILE_IS_SOLARIS) { const off_t offset_orig = offset; res = sendfile(dest_fd, source_fd, &offset, chain->off); if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) { if (offset - offset_orig) return offset - offset_orig; /* if this is EAGAIN or EINTR and no bytes were * written, return 0 */ return (0); } return (res); } #endif } #endif int evbuffer_write_atmost(struct evbuffer *buffer, evutil_socket_t fd, ev_ssize_t howmuch) { int n = -1; EVBUFFER_LOCK(buffer); if (buffer->freeze_start) { goto done; } if (howmuch < 0 || (size_t)howmuch > buffer->total_len) howmuch = buffer->total_len; if (howmuch > 0) { #ifdef USE_SENDFILE struct evbuffer_chain *chain = buffer->first; if (chain != NULL && (chain->flags & EVBUFFER_SENDFILE)) n = evbuffer_write_sendfile(buffer, fd, howmuch); else { #endif #ifdef USE_IOVEC_IMPL n = evbuffer_write_iovec(buffer, fd, howmuch); #elif defined(_WIN32) /* XXX(nickm) Don't disable this code until we know if * the WSARecv code above works. */ void *p = evbuffer_pullup(buffer, howmuch); EVUTIL_ASSERT(p || !howmuch); n = send(fd, p, howmuch, 0); #else void *p = evbuffer_pullup(buffer, howmuch); EVUTIL_ASSERT(p || !howmuch); n = write(fd, p, howmuch); #endif #ifdef USE_SENDFILE } #endif } if (n > 0) evbuffer_drain(buffer, n); done: EVBUFFER_UNLOCK(buffer); return (n); } int evbuffer_write(struct evbuffer *buffer, evutil_socket_t fd) { return evbuffer_write_atmost(buffer, fd, -1); } unsigned char * evbuffer_find(struct evbuffer *buffer, const unsigned char *what, size_t len) { unsigned char *search; struct evbuffer_ptr ptr; EVBUFFER_LOCK(buffer); ptr = evbuffer_search(buffer, (const char *)what, len, NULL); if (ptr.pos < 0) { search = NULL; } else { search = evbuffer_pullup(buffer, ptr.pos + len); if (search) search += ptr.pos; } EVBUFFER_UNLOCK(buffer); return search; } /* Subract <b>howfar</b> from the position of <b>pos</b> within * <b>buf</b>. Returns 0 on success, -1 on failure. * * This isn't exposed yet, because of potential inefficiency issues. * Maybe it should be. */ static int evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos, size_t howfar) { if (pos->pos < 0) return -1; if (howfar > (size_t)pos->pos) return -1; if (pos->internal_.chain && howfar <= pos->internal_.pos_in_chain) { pos->internal_.pos_in_chain -= howfar; pos->pos -= howfar; return 0; } else { const size_t newpos = pos->pos - howfar; /* Here's the inefficient part: it walks over the * chains until we hit newpos. */ return evbuffer_ptr_set(buf, pos, newpos, EVBUFFER_PTR_SET); } } int evbuffer_ptr_set(struct evbuffer *buf, struct evbuffer_ptr *pos, size_t position, enum evbuffer_ptr_how how) { size_t left = position; struct evbuffer_chain *chain = NULL; int result = 0; EVBUFFER_LOCK(buf); switch (how) { case EVBUFFER_PTR_SET: chain = buf->first; pos->pos = position; position = 0; break; case EVBUFFER_PTR_ADD: /* this avoids iterating over all previous chains if we just want to advance the position */ if (pos->pos < 0 || EV_SIZE_MAX - position < (size_t)pos->pos) { EVBUFFER_UNLOCK(buf); return -1; } chain = pos->internal_.chain; pos->pos += position; position = pos->internal_.pos_in_chain; break; } EVUTIL_ASSERT(EV_SIZE_MAX - left >= position); while (chain && position + left >= chain->off) { left -= chain->off - position; chain = chain->next; position = 0; } if (chain) { pos->internal_.chain = chain; pos->internal_.pos_in_chain = position + left; } else if (left == 0) { /* The first byte in the (nonexistent) chain after the last chain */ pos->internal_.chain = NULL; pos->internal_.pos_in_chain = 0; } else { PTR_NOT_FOUND(pos); result = -1; } EVBUFFER_UNLOCK(buf); return result; } /** Compare the bytes in buf at position pos to the len bytes in mem. Return less than 0, 0, or greater than 0 as memcmp. */ static int evbuffer_ptr_memcmp(const struct evbuffer *buf, const struct evbuffer_ptr *pos, const char *mem, size_t len) { struct evbuffer_chain *chain; size_t position; int r; ASSERT_EVBUFFER_LOCKED(buf); if (pos->pos < 0 || EV_SIZE_MAX - len < (size_t)pos->pos || pos->pos + len > buf->total_len) return -1; chain = pos->internal_.chain; position = pos->internal_.pos_in_chain; while (len && chain) { size_t n_comparable; if (len + position > chain->off) n_comparable = chain->off - position; else n_comparable = len; r = memcmp(chain->buffer + chain->misalign + position, mem, n_comparable); if (r) return r; mem += n_comparable; len -= n_comparable; position = 0; chain = chain->next; } return 0; } struct evbuffer_ptr evbuffer_search(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start) { return evbuffer_search_range(buffer, what, len, start, NULL); } struct evbuffer_ptr evbuffer_search_range(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start, const struct evbuffer_ptr *end) { struct evbuffer_ptr pos; struct evbuffer_chain *chain, *last_chain = NULL; const unsigned char *p; char first; EVBUFFER_LOCK(buffer); if (start) { memcpy(&pos, start, sizeof(pos)); chain = pos.internal_.chain; } else { pos.pos = 0; chain = pos.internal_.chain = buffer->first; pos.internal_.pos_in_chain = 0; } if (end) last_chain = end->internal_.chain; if (!len || len > EV_SSIZE_MAX) goto done; first = what[0]; while (chain) { const unsigned char *start_at = chain->buffer + chain->misalign + pos.internal_.pos_in_chain; p = memchr(start_at, first, chain->off - pos.internal_.pos_in_chain); if (p) { pos.pos += p - start_at; pos.internal_.pos_in_chain += p - start_at; if (!evbuffer_ptr_memcmp(buffer, &pos, what, len)) { if (end && pos.pos + (ev_ssize_t)len > end->pos) goto not_found; else goto done; } ++pos.pos; ++pos.internal_.pos_in_chain; if (pos.internal_.pos_in_chain == chain->off) { chain = pos.internal_.chain = chain->next; pos.internal_.pos_in_chain = 0; } } else { if (chain == last_chain) goto not_found; pos.pos += chain->off - pos.internal_.pos_in_chain; chain = pos.internal_.chain = chain->next; pos.internal_.pos_in_chain = 0; } } not_found: PTR_NOT_FOUND(&pos); done: EVBUFFER_UNLOCK(buffer); return pos; } int evbuffer_peek(struct evbuffer *buffer, ev_ssize_t len, struct evbuffer_ptr *start_at, struct evbuffer_iovec *vec, int n_vec) { struct evbuffer_chain *chain; int idx = 0; ev_ssize_t len_so_far = 0; /* Avoid locking in trivial edge cases */ if (start_at && start_at->internal_.chain == NULL) return 0; EVBUFFER_LOCK(buffer); if (start_at) { chain = start_at->internal_.chain; len_so_far = chain->off - start_at->internal_.pos_in_chain; idx = 1; if (n_vec > 0) { vec[0].iov_base = (void *)(chain->buffer + chain->misalign + start_at->internal_.pos_in_chain); vec[0].iov_len = len_so_far; } chain = chain->next; } else { chain = buffer->first; } if (n_vec == 0 && len < 0) { /* If no vectors are provided and they asked for "everything", * pretend they asked for the actual available amount. */ len = buffer->total_len; if (start_at) { len -= start_at->pos; } } while (chain) { if (len >= 0 && len_so_far >= len) break; if (idx<n_vec) { vec[idx].iov_base = (void *)(chain->buffer + chain->misalign); vec[idx].iov_len = chain->off; } else if (len<0) { break; } ++idx; len_so_far += chain->off; chain = chain->next; } EVBUFFER_UNLOCK(buffer); return idx; } int evbuffer_add_vprintf(struct evbuffer *buf, const char *fmt, va_list ap) { char *buffer; size_t space; int sz, result = -1; va_list aq; struct evbuffer_chain *chain; EVBUFFER_LOCK(buf); if (buf->freeze_end) { goto done; } /* make sure that at least some space is available */ if ((chain = evbuffer_expand_singlechain(buf, 64)) == NULL) goto done; for (;;) { #if 0 size_t used = chain->misalign + chain->off; buffer = (char *)chain->buffer + chain->misalign + chain->off; EVUTIL_ASSERT(chain->buffer_len >= used); space = chain->buffer_len - used; #endif buffer = (char*) CHAIN_SPACE_PTR(chain); space = (size_t) CHAIN_SPACE_LEN(chain); #ifndef va_copy #define va_copy(dst, src) memcpy(&(dst), &(src), sizeof(va_list)) #endif va_copy(aq, ap); sz = evutil_vsnprintf(buffer, space, fmt, aq); va_end(aq); if (sz < 0) goto done; if (INT_MAX >= EVBUFFER_CHAIN_MAX && (size_t)sz >= EVBUFFER_CHAIN_MAX) goto done; if ((size_t)sz < space) { chain->off += sz; buf->total_len += sz; buf->n_add_for_cb += sz; advance_last_with_data(buf); evbuffer_invoke_callbacks_(buf); result = sz; goto done; } if ((chain = evbuffer_expand_singlechain(buf, sz + 1)) == NULL) goto done; } /* NOTREACHED */ done: EVBUFFER_UNLOCK(buf); return result; } int evbuffer_add_printf(struct evbuffer *buf, const char *fmt, ...) { int res = -1; va_list ap; va_start(ap, fmt); res = evbuffer_add_vprintf(buf, fmt, ap); va_end(ap); return (res); } int evbuffer_add_reference(struct evbuffer *outbuf, const void *data, size_t datlen, evbuffer_ref_cleanup_cb cleanupfn, void *extra) { struct evbuffer_chain *chain; struct evbuffer_chain_reference *info; int result = -1; chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_reference)); if (!chain) return (-1); chain->flags |= EVBUFFER_REFERENCE | EVBUFFER_IMMUTABLE; chain->buffer = (unsigned char *)data; chain->buffer_len = datlen; chain->off = datlen; info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_reference, chain); info->cleanupfn = cleanupfn; info->extra = extra; EVBUFFER_LOCK(outbuf); if (outbuf->freeze_end) { /* don't call chain_free; we do not want to actually invoke * the cleanup function */ mm_free(chain); goto done; } evbuffer_chain_insert(outbuf, chain); outbuf->n_add_for_cb += datlen; evbuffer_invoke_callbacks_(outbuf); result = 0; done: EVBUFFER_UNLOCK(outbuf); return result; } /* TODO(niels): we may want to add to automagically convert to mmap, in * case evbuffer_remove() or evbuffer_pullup() are being used. */ struct evbuffer_file_segment * evbuffer_file_segment_new( int fd, ev_off_t offset, ev_off_t length, unsigned flags) { struct evbuffer_file_segment *seg = mm_calloc(sizeof(struct evbuffer_file_segment), 1); if (!seg) return NULL; seg->refcnt = 1; seg->fd = fd; seg->flags = flags; seg->file_offset = offset; seg->cleanup_cb = NULL; seg->cleanup_cb_arg = NULL; #ifdef _WIN32 #ifndef lseek #define lseek _lseeki64 #endif #ifndef fstat #define fstat _fstat #endif #ifndef stat #define stat _stat #endif #endif if (length == -1) { struct stat st; if (fstat(fd, &st) < 0) goto err; length = st.st_size; } seg->length = length; if (offset < 0 || length < 0 || ((ev_uint64_t)length > EVBUFFER_CHAIN_MAX) || (ev_uint64_t)offset > (ev_uint64_t)(EVBUFFER_CHAIN_MAX - length)) goto err; #if defined(USE_SENDFILE) if (!(flags & EVBUF_FS_DISABLE_SENDFILE)) { seg->can_sendfile = 1; goto done; } #endif if (evbuffer_file_segment_materialize(seg)<0) goto err; #if defined(USE_SENDFILE) done: #endif if (!(flags & EVBUF_FS_DISABLE_LOCKING)) { EVTHREAD_ALLOC_LOCK(seg->lock, 0); } return seg; err: mm_free(seg); return NULL; } #ifdef EVENT__HAVE_MMAP static long get_page_size(void) { #ifdef SC_PAGE_SIZE return sysconf(SC_PAGE_SIZE); #elif defined(_SC_PAGE_SIZE) return sysconf(_SC_PAGE_SIZE); #else return 1; #endif } #endif /* DOCDOC */ /* Requires lock */ static int evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg) { const unsigned flags = seg->flags; const int fd = seg->fd; const ev_off_t length = seg->length; const ev_off_t offset = seg->file_offset; if (seg->contents) return 0; /* already materialized */ #if defined(EVENT__HAVE_MMAP) if (!(flags & EVBUF_FS_DISABLE_MMAP)) { off_t offset_rounded = 0, offset_leftover = 0; void *mapped; if (offset) { /* mmap implementations don't generally like us * to have an offset that isn't a round */ long page_size = get_page_size(); if (page_size == -1) goto err; offset_leftover = offset % page_size; offset_rounded = offset - offset_leftover; } mapped = mmap(NULL, length + offset_leftover, PROT_READ, #ifdef MAP_NOCACHE MAP_NOCACHE | /* ??? */ #endif #ifdef MAP_FILE MAP_FILE | #endif MAP_PRIVATE, fd, offset_rounded); if (mapped == MAP_FAILED) { event_warn("%s: mmap(%d, %d, %zu) failed", __func__, fd, 0, (size_t)(offset + length)); } else { seg->mapping = mapped; seg->contents = (char*)mapped+offset_leftover; seg->mmap_offset = 0; seg->is_mapping = 1; goto done; } } #endif #ifdef _WIN32 if (!(flags & EVBUF_FS_DISABLE_MMAP)) { intptr_t h = _get_osfhandle(fd); HANDLE m; ev_uint64_t total_size = length+offset; if ((HANDLE)h == INVALID_HANDLE_VALUE) goto err; m = CreateFileMapping((HANDLE)h, NULL, PAGE_READONLY, (total_size >> 32), total_size & 0xfffffffful, NULL); if (m != INVALID_HANDLE_VALUE) { /* Does h leak? */ seg->mapping_handle = m; seg->mmap_offset = offset; seg->is_mapping = 1; goto done; } } #endif { ev_off_t start_pos = lseek(fd, 0, SEEK_CUR), pos; ev_off_t read_so_far = 0; char *mem; int e; ev_ssize_t n = 0; if (!(mem = mm_malloc(length))) goto err; if (start_pos < 0) { mm_free(mem); goto err; } if (lseek(fd, offset, SEEK_SET) < 0) { mm_free(mem); goto err; } while (read_so_far < length) { n = read(fd, mem+read_so_far, length-read_so_far); if (n <= 0) break; read_so_far += n; } e = errno; pos = lseek(fd, start_pos, SEEK_SET); if (n < 0 || (n == 0 && length > read_so_far)) { mm_free(mem); errno = e; goto err; } else if (pos < 0) { mm_free(mem); goto err; } seg->contents = mem; } done: return 0; err: return -1; } void evbuffer_file_segment_add_cleanup_cb(struct evbuffer_file_segment *seg, evbuffer_file_segment_cleanup_cb cb, void* arg) { EVUTIL_ASSERT(seg->refcnt > 0); seg->cleanup_cb = cb; seg->cleanup_cb_arg = arg; } void evbuffer_file_segment_free(struct evbuffer_file_segment *seg) { int refcnt; EVLOCK_LOCK(seg->lock, 0); refcnt = --seg->refcnt; EVLOCK_UNLOCK(seg->lock, 0); if (refcnt > 0) return; EVUTIL_ASSERT(refcnt == 0); if (seg->is_mapping) { #ifdef _WIN32 CloseHandle(seg->mapping_handle); #elif defined (EVENT__HAVE_MMAP) off_t offset_leftover; offset_leftover = seg->file_offset % get_page_size(); if (munmap(seg->mapping, seg->length + offset_leftover) == -1) event_warn("%s: munmap failed", __func__); #endif } else if (seg->contents) { mm_free(seg->contents); } if ((seg->flags & EVBUF_FS_CLOSE_ON_FREE) && seg->fd >= 0) { close(seg->fd); } if (seg->cleanup_cb) { (*seg->cleanup_cb)((struct evbuffer_file_segment const*)seg, seg->flags, seg->cleanup_cb_arg); seg->cleanup_cb = NULL; seg->cleanup_cb_arg = NULL; } EVTHREAD_FREE_LOCK(seg->lock, 0); mm_free(seg); } int evbuffer_add_file_segment(struct evbuffer *buf, struct evbuffer_file_segment *seg, ev_off_t offset, ev_off_t length) { struct evbuffer_chain *chain; struct evbuffer_chain_file_segment *extra; int can_use_sendfile = 0; EVBUFFER_LOCK(buf); EVLOCK_LOCK(seg->lock, 0); if (buf->flags & EVBUFFER_FLAG_DRAINS_TO_FD) { can_use_sendfile = 1; } else { if (!seg->contents) { if (evbuffer_file_segment_materialize(seg)<0) { EVLOCK_UNLOCK(seg->lock, 0); EVBUFFER_UNLOCK(buf); return -1; } } } ++seg->refcnt; EVLOCK_UNLOCK(seg->lock, 0); if (buf->freeze_end) goto err; if (length < 0) { if (offset > seg->length) goto err; length = seg->length - offset; } /* Can we actually add this? */ if (offset+length > seg->length) goto err; chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_file_segment)); if (!chain) goto err; extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment, chain); chain->flags |= EVBUFFER_IMMUTABLE|EVBUFFER_FILESEGMENT; if (can_use_sendfile && seg->can_sendfile) { chain->flags |= EVBUFFER_SENDFILE; chain->misalign = seg->file_offset + offset; chain->off = length; chain->buffer_len = chain->misalign + length; } else if (seg->is_mapping) { #ifdef _WIN32 ev_uint64_t total_offset = seg->mmap_offset+offset; ev_uint64_t offset_rounded=0, offset_remaining=0; LPVOID data; if (total_offset) { SYSTEM_INFO si; memset(&si, 0, sizeof(si)); /* cargo cult */ GetSystemInfo(&si); offset_remaining = total_offset % si.dwAllocationGranularity; offset_rounded = total_offset - offset_remaining; } data = MapViewOfFile( seg->mapping_handle, FILE_MAP_READ, offset_rounded >> 32, offset_rounded & 0xfffffffful, length + offset_remaining); if (data == NULL) { mm_free(chain); goto err; } chain->buffer = (unsigned char*) data; chain->buffer_len = length+offset_remaining; chain->misalign = offset_remaining; chain->off = length; #else chain->buffer = (unsigned char*)(seg->contents + offset); chain->buffer_len = length; chain->off = length; #endif } else { chain->buffer = (unsigned char*)(seg->contents + offset); chain->buffer_len = length; chain->off = length; } extra->segment = seg; buf->n_add_for_cb += length; evbuffer_chain_insert(buf, chain); evbuffer_invoke_callbacks_(buf); EVBUFFER_UNLOCK(buf); return 0; err: EVBUFFER_UNLOCK(buf); evbuffer_file_segment_free(seg); /* Lowers the refcount */ return -1; } int evbuffer_add_file(struct evbuffer *buf, int fd, ev_off_t offset, ev_off_t length) { struct evbuffer_file_segment *seg; unsigned flags = EVBUF_FS_CLOSE_ON_FREE; int r; seg = evbuffer_file_segment_new(fd, offset, length, flags); if (!seg) return -1; r = evbuffer_add_file_segment(buf, seg, 0, length); if (r == 0) evbuffer_file_segment_free(seg); return r; } void evbuffer_setcb(struct evbuffer *buffer, evbuffer_cb cb, void *cbarg) { EVBUFFER_LOCK(buffer); if (!LIST_EMPTY(&buffer->callbacks)) evbuffer_remove_all_callbacks(buffer); if (cb) { struct evbuffer_cb_entry *ent = evbuffer_add_cb(buffer, NULL, cbarg); ent->cb.cb_obsolete = cb; ent->flags |= EVBUFFER_CB_OBSOLETE; } EVBUFFER_UNLOCK(buffer); } struct evbuffer_cb_entry * evbuffer_add_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg) { struct evbuffer_cb_entry *e; if (! (e = mm_calloc(1, sizeof(struct evbuffer_cb_entry)))) return NULL; EVBUFFER_LOCK(buffer); e->cb.cb_func = cb; e->cbarg = cbarg; e->flags = EVBUFFER_CB_ENABLED; LIST_INSERT_HEAD(&buffer->callbacks, e, next); EVBUFFER_UNLOCK(buffer); return e; } int evbuffer_remove_cb_entry(struct evbuffer *buffer, struct evbuffer_cb_entry *ent) { EVBUFFER_LOCK(buffer); LIST_REMOVE(ent, next); EVBUFFER_UNLOCK(buffer); mm_free(ent); return 0; } int evbuffer_remove_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg) { struct evbuffer_cb_entry *cbent; int result = -1; EVBUFFER_LOCK(buffer); LIST_FOREACH(cbent, &buffer->callbacks, next) { if (cb == cbent->cb.cb_func && cbarg == cbent->cbarg) { result = evbuffer_remove_cb_entry(buffer, cbent); goto done; } } done: EVBUFFER_UNLOCK(buffer); return result; } int evbuffer_cb_set_flags(struct evbuffer *buffer, struct evbuffer_cb_entry *cb, ev_uint32_t flags) { /* the user isn't allowed to mess with these. */ flags &= ~EVBUFFER_CB_INTERNAL_FLAGS; EVBUFFER_LOCK(buffer); cb->flags |= flags; EVBUFFER_UNLOCK(buffer); return 0; } int evbuffer_cb_clear_flags(struct evbuffer *buffer, struct evbuffer_cb_entry *cb, ev_uint32_t flags) { /* the user isn't allowed to mess with these. */ flags &= ~EVBUFFER_CB_INTERNAL_FLAGS; EVBUFFER_LOCK(buffer); cb->flags &= ~flags; EVBUFFER_UNLOCK(buffer); return 0; } int evbuffer_freeze(struct evbuffer *buffer, int start) { EVBUFFER_LOCK(buffer); if (start) buffer->freeze_start = 1; else buffer->freeze_end = 1; EVBUFFER_UNLOCK(buffer); return 0; } int evbuffer_unfreeze(struct evbuffer *buffer, int start) { EVBUFFER_LOCK(buffer); if (start) buffer->freeze_start = 0; else buffer->freeze_end = 0; EVBUFFER_UNLOCK(buffer); return 0; } #if 0 void evbuffer_cb_suspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb) { if (!(cb->flags & EVBUFFER_CB_SUSPENDED)) { cb->size_before_suspend = evbuffer_get_length(buffer); cb->flags |= EVBUFFER_CB_SUSPENDED; } } void evbuffer_cb_unsuspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb) { if ((cb->flags & EVBUFFER_CB_SUSPENDED)) { unsigned call = (cb->flags & EVBUFFER_CB_CALL_ON_UNSUSPEND); size_t sz = cb->size_before_suspend; cb->flags &= ~(EVBUFFER_CB_SUSPENDED| EVBUFFER_CB_CALL_ON_UNSUSPEND); cb->size_before_suspend = 0; if (call && (cb->flags & EVBUFFER_CB_ENABLED)) { cb->cb(buffer, sz, evbuffer_get_length(buffer), cb->cbarg); } } } #endif int evbuffer_get_callbacks_(struct evbuffer *buffer, struct event_callback **cbs, int max_cbs) { int r = 0; EVBUFFER_LOCK(buffer); if (buffer->deferred_cbs) { if (max_cbs < 1) { r = -1; goto done; } cbs[0] = &buffer->deferred; r = 1; } done: EVBUFFER_UNLOCK(buffer); return r; } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/buffer_iocp.c������������������������������������������������0000664�0000000�0000000�00000020677�15270523712�0022441�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** @file buffer_iocp.c This module implements overlapped read and write functions for evbuffer objects on Windows. */ #include "event2/event-config.h" #include "evconfig-private.h" #include "event2/buffer.h" #include "event2/buffer_compat.h" #include "event2/util.h" #include "event2/thread.h" #include "util-internal.h" #include "evthread-internal.h" #include "evbuffer-internal.h" #include "iocp-internal.h" #include "mm-internal.h" #include <winsock2.h> #include <windows.h> #include <stdio.h> #define MAX_WSABUFS 16 /** An evbuffer that can handle overlapped IO. */ struct evbuffer_overlapped { struct evbuffer buffer; /** The socket that we're doing overlapped IO on. */ evutil_socket_t fd; /** pending I/O type */ unsigned read_in_progress : 1; unsigned write_in_progress : 1; /** The first pinned chain in the buffer. */ struct evbuffer_chain *first_pinned; /** How many chains are pinned; how many of the fields in buffers * are we using. */ int n_buffers; WSABUF buffers[MAX_WSABUFS]; }; /** Given an evbuffer, return the correponding evbuffer structure, or NULL if * the evbuffer isn't overlapped. */ static inline struct evbuffer_overlapped * upcast_evbuffer(struct evbuffer *buf) { if (!buf || !buf->is_overlapped) return NULL; return EVUTIL_UPCAST(buf, struct evbuffer_overlapped, buffer); } /** Unpin all the chains noted as pinned in 'eo'. */ static void pin_release(struct evbuffer_overlapped *eo, unsigned flag) { int i; struct evbuffer_chain *next, *chain = eo->first_pinned; for (i = 0; i < eo->n_buffers; ++i) { EVUTIL_ASSERT(chain); next = chain->next; evbuffer_chain_unpin_(chain, flag); chain = next; } } void evbuffer_commit_read_(struct evbuffer *evbuf, ev_ssize_t nBytes) { struct evbuffer_overlapped *buf = upcast_evbuffer(evbuf); struct evbuffer_chain **chainp; size_t remaining, len; unsigned i; EVBUFFER_LOCK(evbuf); EVUTIL_ASSERT(buf->read_in_progress && !buf->write_in_progress); EVUTIL_ASSERT(nBytes >= 0); /* XXXX Can this be false? */ evbuffer_unfreeze(evbuf, 0); chainp = evbuf->last_with_datap; if (!((*chainp)->flags & EVBUFFER_MEM_PINNED_R)) chainp = &(*chainp)->next; remaining = nBytes; for (i = 0; remaining > 0 && i < (unsigned)buf->n_buffers; ++i) { EVUTIL_ASSERT(*chainp); len = buf->buffers[i].len; if (remaining < len) len = remaining; (*chainp)->off += len; evbuf->last_with_datap = chainp; remaining -= len; chainp = &(*chainp)->next; } pin_release(buf, EVBUFFER_MEM_PINNED_R); buf->read_in_progress = 0; evbuf->total_len += nBytes; evbuf->n_add_for_cb += nBytes; evbuffer_invoke_callbacks_(evbuf); evbuffer_decref_and_unlock_(evbuf); } void evbuffer_commit_write_(struct evbuffer *evbuf, ev_ssize_t nBytes) { struct evbuffer_overlapped *buf = upcast_evbuffer(evbuf); EVBUFFER_LOCK(evbuf); EVUTIL_ASSERT(buf->write_in_progress && !buf->read_in_progress); evbuffer_unfreeze(evbuf, 1); evbuffer_drain(evbuf, nBytes); pin_release(buf,EVBUFFER_MEM_PINNED_W); buf->write_in_progress = 0; evbuffer_decref_and_unlock_(evbuf); } struct evbuffer * evbuffer_overlapped_new_(evutil_socket_t fd) { struct evbuffer_overlapped *evo; evo = mm_calloc(1, sizeof(struct evbuffer_overlapped)); if (!evo) return NULL; LIST_INIT(&evo->buffer.callbacks); evo->buffer.refcnt = 1; evo->buffer.last_with_datap = &evo->buffer.first; evo->buffer.is_overlapped = 1; evo->fd = fd; return &evo->buffer; } int evbuffer_launch_write_(struct evbuffer *buf, ev_ssize_t at_most, struct event_overlapped *ol) { struct evbuffer_overlapped *buf_o = upcast_evbuffer(buf); int r = -1; int i; struct evbuffer_chain *chain; DWORD bytesSent; if (!buf) { /* No buffer, or it isn't overlapped */ return -1; } EVBUFFER_LOCK(buf); EVUTIL_ASSERT(!buf_o->read_in_progress); if (buf->freeze_start || buf_o->write_in_progress) goto done; if (!buf->total_len) { /* Nothing to write */ r = 0; goto done; } else if (at_most < 0 || (size_t)at_most > buf->total_len) { at_most = buf->total_len; } evbuffer_freeze(buf, 1); buf_o->first_pinned = NULL; buf_o->n_buffers = 0; memset(buf_o->buffers, 0, sizeof(buf_o->buffers)); chain = buf_o->first_pinned = buf->first; for (i=0; i < MAX_WSABUFS && chain; ++i, chain=chain->next) { WSABUF *b = &buf_o->buffers[i]; b->buf = (char*)( chain->buffer + chain->misalign ); evbuffer_chain_pin_(chain, EVBUFFER_MEM_PINNED_W); if ((size_t)at_most > chain->off) { /* XXXX Cast is safe for now, since win32 has no mmaped chains. But later, we need to have this add more WSAbufs if chain->off is greater than ULONG_MAX */ b->len = (unsigned long)chain->off; at_most -= chain->off; } else { b->len = (unsigned long)at_most; ++i; break; } } buf_o->n_buffers = i; evbuffer_incref_(buf); if (WSASend(buf_o->fd, buf_o->buffers, i, &bytesSent, 0, &ol->overlapped, NULL)) { int error = WSAGetLastError(); if (error != WSA_IO_PENDING) { /* An actual error. */ pin_release(buf_o, EVBUFFER_MEM_PINNED_W); evbuffer_unfreeze(buf, 1); evbuffer_free(buf); /* decref */ goto done; } } buf_o->write_in_progress = 1; r = 0; done: EVBUFFER_UNLOCK(buf); return r; } int evbuffer_launch_read_(struct evbuffer *buf, size_t at_most, struct event_overlapped *ol) { struct evbuffer_overlapped *buf_o = upcast_evbuffer(buf); int r = -1, i; int nvecs; int npin=0; struct evbuffer_chain *chain=NULL, **chainp; DWORD bytesRead; DWORD flags = 0; struct evbuffer_iovec vecs[MAX_WSABUFS]; if (!buf_o) return -1; EVBUFFER_LOCK(buf); EVUTIL_ASSERT(!buf_o->write_in_progress); if (buf->freeze_end || buf_o->read_in_progress) goto done; buf_o->first_pinned = NULL; buf_o->n_buffers = 0; memset(buf_o->buffers, 0, sizeof(buf_o->buffers)); if (evbuffer_expand_fast_(buf, at_most, MAX_WSABUFS) == -1) goto done; evbuffer_freeze(buf, 0); nvecs = evbuffer_read_setup_vecs_(buf, at_most, vecs, MAX_WSABUFS, &chainp, 1); for (i=0;i<nvecs;++i) { WSABUF_FROM_EVBUFFER_IOV( &buf_o->buffers[i], &vecs[i]); } buf_o->n_buffers = nvecs; buf_o->first_pinned = chain = *chainp; npin=0; for ( ; chain; chain = chain->next) { evbuffer_chain_pin_(chain, EVBUFFER_MEM_PINNED_R); ++npin; } EVUTIL_ASSERT(npin == nvecs); evbuffer_incref_(buf); if (WSARecv(buf_o->fd, buf_o->buffers, nvecs, &bytesRead, &flags, &ol->overlapped, NULL)) { int error = WSAGetLastError(); if (error != WSA_IO_PENDING) { /* An actual error. */ pin_release(buf_o, EVBUFFER_MEM_PINNED_R); evbuffer_unfreeze(buf, 0); evbuffer_free(buf); /* decref */ goto done; } } buf_o->read_in_progress = 1; r = 0; done: EVBUFFER_UNLOCK(buf); return r; } evutil_socket_t evbuffer_overlapped_get_fd_(struct evbuffer *buf) { struct evbuffer_overlapped *buf_o = upcast_evbuffer(buf); return buf_o ? buf_o->fd : -1; } void evbuffer_overlapped_set_fd_(struct evbuffer *buf, evutil_socket_t fd) { struct evbuffer_overlapped *buf_o = upcast_evbuffer(buf); EVBUFFER_LOCK(buf); /* XXX is this right?, should it cancel current I/O operations? */ if (buf_o) buf_o->fd = fd; EVBUFFER_UNLOCK(buf); } �����������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/bufferevent-internal.h���������������������������������������0000664�0000000�0000000�00000044066�15270523712�0024306�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2008-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef BUFFEREVENT_INTERNAL_H_INCLUDED_ #define BUFFEREVENT_INTERNAL_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif #include "event2/event-config.h" #include "event2/event_struct.h" #include "evconfig-private.h" #include "event2/util.h" #include "defer-internal.h" #include "evthread-internal.h" #include "event2/thread.h" #include "ratelim-internal.h" #include "event2/bufferevent_struct.h" #include "ipv6-internal.h" #ifdef _WIN32 #include <ws2tcpip.h> #endif #ifdef EVENT__HAVE_NETINET_IN_H #include <netinet/in.h> #endif #ifdef EVENT__HAVE_NETINET_IN6_H #include <netinet/in6.h> #endif /* These flags are reasons that we might be declining to actually enable reading or writing on a bufferevent. */ /* On a all bufferevents, for reading: used when we have read up to the watermark value. On a filtering bufferevent, for writing: used when the underlying bufferevent's write buffer has been filled up to its watermark value. */ #define BEV_SUSPEND_WM 0x01 /* On a base bufferevent: when we have emptied a bandwidth buckets */ #define BEV_SUSPEND_BW 0x02 /* On a base bufferevent: when we have emptied the group's bandwidth bucket. */ #define BEV_SUSPEND_BW_GROUP 0x04 /* On a socket bufferevent: can't do any operations while we're waiting for * name lookup to finish. */ #define BEV_SUSPEND_LOOKUP 0x08 /* On a base bufferevent, for reading: used when a filter has choked this * (underlying) bufferevent because it has stopped reading from it. */ #define BEV_SUSPEND_FILT_READ 0x10 typedef ev_uint16_t bufferevent_suspend_flags; struct bufferevent_rate_limit_group { /** List of all members in the group */ LIST_HEAD(rlim_group_member_list, bufferevent_private) members; /** Current limits for the group. */ struct ev_token_bucket rate_limit; struct ev_token_bucket_cfg rate_limit_cfg; /** True iff we don't want to read from any member of the group.until * the token bucket refills. */ unsigned read_suspended : 1; /** True iff we don't want to write from any member of the group.until * the token bucket refills. */ unsigned write_suspended : 1; /** True iff we were unable to suspend one of the bufferevents in the * group for reading the last time we tried, and we should try * again. */ unsigned pending_unsuspend_read : 1; /** True iff we were unable to suspend one of the bufferevents in the * group for writing the last time we tried, and we should try * again. */ unsigned pending_unsuspend_write : 1; /*@{*/ /** Total number of bytes read or written in this group since last * reset. */ ev_uint64_t total_read; ev_uint64_t total_written; /*@}*/ /** The number of bufferevents in the group. */ int n_members; /** The smallest number of bytes that any member of the group should * be limited to read or write at a time. */ ev_ssize_t min_share; ev_ssize_t configured_min_share; /** Timeout event that goes off once a tick, when the bucket is ready * to refill. */ struct event master_refill_event; /** Seed for weak random number generator. Protected by 'lock' */ struct evutil_weakrand_state weakrand_seed; /** Lock to protect the members of this group. This lock should nest * within every bufferevent lock: if you are holding this lock, do * not assume you can lock another bufferevent. */ void *lock; }; /** Fields for rate-limiting a single bufferevent. */ struct bufferevent_rate_limit { /* Linked-list elements for storing this bufferevent_private in a * group. * * Note that this field is supposed to be protected by the group * lock */ LIST_ENTRY(bufferevent_private) next_in_group; /** The rate-limiting group for this bufferevent, or NULL if it is * only rate-limited on its own. */ struct bufferevent_rate_limit_group *group; /* This bufferevent's current limits. */ struct ev_token_bucket limit; /* Pointer to the rate-limit configuration for this bufferevent. * Can be shared. XXX reference-count this? */ struct ev_token_bucket_cfg *cfg; /* Timeout event used when one this bufferevent's buckets are * empty. */ struct event refill_bucket_event; }; /** Parts of the bufferevent structure that are shared among all bufferevent * types, but not exposed in bufferevent_struct.h. */ struct bufferevent_private { /** The underlying bufferevent structure. */ struct bufferevent bev; /** Evbuffer callback to enforce watermarks on input. */ struct evbuffer_cb_entry *read_watermarks_cb; /** If set, we should free the lock when we free the bufferevent. */ unsigned own_lock : 1; /** Flag: set if we have deferred callbacks and a read callback is * pending. */ unsigned readcb_pending : 1; /** Flag: set if we have deferred callbacks and a write callback is * pending. */ unsigned writecb_pending : 1; /** Flag: set if we are currently busy connecting. */ unsigned connecting : 1; /** Flag: set if a connect failed prematurely; this is a hack for * getting around the bufferevent abstraction. */ unsigned connection_refused : 1; /** Set to the events pending if we have deferred callbacks and * an events callback is pending. */ short eventcb_pending; /** If set, read is suspended until one or more conditions are over. * The actual value here is a bitfield of those conditions; see the * BEV_SUSPEND_* flags above. */ bufferevent_suspend_flags read_suspended; /** If set, writing is suspended until one or more conditions are over. * The actual value here is a bitfield of those conditions; see the * BEV_SUSPEND_* flags above. */ bufferevent_suspend_flags write_suspended; /** Set to the current socket errno if we have deferred callbacks and * an events callback is pending. */ int errno_pending; /** The DNS error code for bufferevent_socket_connect_hostname */ int dns_error; /** Used to implement deferred callbacks */ struct event_callback deferred; /** The options this bufferevent was constructed with */ enum bufferevent_options options; /** Current reference count for this bufferevent. */ int refcnt; /** Lock for this bufferevent. Shared by the inbuf and the outbuf. * If NULL, locking is disabled. */ void *lock; /** No matter how big our bucket gets, don't try to read more than this * much in a single read operation. */ ev_ssize_t max_single_read; /** No matter how big our bucket gets, don't try to write more than this * much in a single write operation. */ ev_ssize_t max_single_write; /** Rate-limiting information for this bufferevent */ struct bufferevent_rate_limit *rate_limiting; /* Saved conn_addr, to extract IP address from it. * * Because some servers may reset/close connection without waiting clients, * in that case we can't extract IP address even in close_cb. * So we need to save it, just after we connected to remote server, or * after resolving (to avoid extra dns requests during retrying, since UDP * is slow) */ union { struct sockaddr_in6 in6; struct sockaddr_in in; } conn_address; struct evdns_getaddrinfo_request *dns_request; }; /** Possible operations for a control callback. */ enum bufferevent_ctrl_op { BEV_CTRL_SET_FD, BEV_CTRL_GET_FD, BEV_CTRL_GET_UNDERLYING, BEV_CTRL_CANCEL_ALL }; /** Possible data types for a control callback */ union bufferevent_ctrl_data { void *ptr; evutil_socket_t fd; }; /** Implementation table for a bufferevent: holds function pointers and other information to make the various bufferevent types work. */ struct bufferevent_ops { /** The name of the bufferevent's type. */ const char *type; /** At what offset into the implementation type will we find a bufferevent structure? Example: if the type is implemented as struct bufferevent_x { int extra_data; struct bufferevent bev; } then mem_offset should be offsetof(struct bufferevent_x, bev) */ off_t mem_offset; /** Enables one or more of EV_READ|EV_WRITE on a bufferevent. Does not need to adjust the 'enabled' field. Returns 0 on success, -1 on failure. */ int (*enable)(struct bufferevent *, short); /** Disables one or more of EV_READ|EV_WRITE on a bufferevent. Does not need to adjust the 'enabled' field. Returns 0 on success, -1 on failure. */ int (*disable)(struct bufferevent *, short); /** Detatches the bufferevent from related data structures. Called as * soon as its reference count reaches 0. */ void (*unlink)(struct bufferevent *); /** Free any storage and deallocate any extra data or structures used in this implementation. Called when the bufferevent is finalized. */ void (*destruct)(struct bufferevent *); /** Called when the timeouts on the bufferevent have changed.*/ int (*adj_timeouts)(struct bufferevent *); /** Called to flush data. */ int (*flush)(struct bufferevent *, short, enum bufferevent_flush_mode); /** Called to access miscellaneous fields. */ int (*ctrl)(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *); }; extern const struct bufferevent_ops bufferevent_ops_socket; extern const struct bufferevent_ops bufferevent_ops_filter; extern const struct bufferevent_ops bufferevent_ops_pair; #define BEV_IS_SOCKET(bevp) ((bevp)->be_ops == &bufferevent_ops_socket) #define BEV_IS_FILTER(bevp) ((bevp)->be_ops == &bufferevent_ops_filter) #define BEV_IS_PAIR(bevp) ((bevp)->be_ops == &bufferevent_ops_pair) #ifdef _WIN32 extern const struct bufferevent_ops bufferevent_ops_async; #define BEV_IS_ASYNC(bevp) ((bevp)->be_ops == &bufferevent_ops_async) #else #define BEV_IS_ASYNC(bevp) 0 #endif /** Initialize the shared parts of a bufferevent. */ int bufferevent_init_common_(struct bufferevent_private *, struct event_base *, const struct bufferevent_ops *, enum bufferevent_options options); /** For internal use: temporarily stop all reads on bufev, until the conditions * in 'what' are over. */ void bufferevent_suspend_read_(struct bufferevent *bufev, bufferevent_suspend_flags what); /** For internal use: clear the conditions 'what' on bufev, and re-enable * reading if there are no conditions left. */ void bufferevent_unsuspend_read_(struct bufferevent *bufev, bufferevent_suspend_flags what); /** For internal use: temporarily stop all writes on bufev, until the conditions * in 'what' are over. */ void bufferevent_suspend_write_(struct bufferevent *bufev, bufferevent_suspend_flags what); /** For internal use: clear the conditions 'what' on bufev, and re-enable * writing if there are no conditions left. */ void bufferevent_unsuspend_write_(struct bufferevent *bufev, bufferevent_suspend_flags what); #define bufferevent_wm_suspend_read(b) \ bufferevent_suspend_read_((b), BEV_SUSPEND_WM) #define bufferevent_wm_unsuspend_read(b) \ bufferevent_unsuspend_read_((b), BEV_SUSPEND_WM) /* Disable a bufferevent. Equivalent to bufferevent_disable(), but first resets 'connecting' flag to force EV_WRITE down for sure. XXXX this method will go away in the future; try not to add new users. See comment in evhttp_connection_reset_() for discussion. @param bufev the bufferevent to be disabled @param event any combination of EV_READ | EV_WRITE. @return 0 if successful, or -1 if an error occurred @see bufferevent_disable() */ int bufferevent_disable_hard_(struct bufferevent *bufev, short event); /** Internal: Set up locking on a bufferevent. If lock is set, use it. * Otherwise, use a new lock. */ int bufferevent_enable_locking_(struct bufferevent *bufev, void *lock); /** Internal: backwards compat macro for the now public function * Increment the reference count on bufev. */ #define bufferevent_incref_(bufev) bufferevent_incref(bufev) /** Internal: Lock bufev and increase its reference count. * unlocking it otherwise. */ void bufferevent_incref_and_lock_(struct bufferevent *bufev); /** Internal: backwards compat macro for the now public function * Decrement the reference count on bufev. Returns 1 if it freed * the bufferevent.*/ #define bufferevent_decref_(bufev) bufferevent_decref(bufev) /** Internal: Drop the reference count on bufev, freeing as necessary, and * unlocking it otherwise. Returns 1 if it freed the bufferevent. */ int bufferevent_decref_and_unlock_(struct bufferevent *bufev); /** Internal: If callbacks are deferred and we have a read callback, schedule * a readcb. Otherwise just run the readcb. Ignores watermarks. */ void bufferevent_run_readcb_(struct bufferevent *bufev, int options); /** Internal: If callbacks are deferred and we have a write callback, schedule * a writecb. Otherwise just run the writecb. Ignores watermarks. */ void bufferevent_run_writecb_(struct bufferevent *bufev, int options); /** Internal: If callbacks are deferred and we have an eventcb, schedule * it to run with events "what". Otherwise just run the eventcb. * See bufferevent_trigger_event for meaning of "options". */ void bufferevent_run_eventcb_(struct bufferevent *bufev, short what, int options); /** Internal: Run or schedule (if deferred or options contain * BEV_TRIG_DEFER_CALLBACKS) I/O callbacks specified in iotype. * Must already hold the bufev lock. Honors watermarks unless * BEV_TRIG_IGNORE_WATERMARKS is in options. */ static inline void bufferevent_trigger_nolock_(struct bufferevent *bufev, short iotype, int options); /* Making this inline since all of the common-case calls to this function in * libevent use constant arguments. */ static inline void bufferevent_trigger_nolock_(struct bufferevent *bufev, short iotype, int options) { if ((iotype & EV_READ) && ((options & BEV_TRIG_IGNORE_WATERMARKS) || evbuffer_get_length(bufev->input) >= bufev->wm_read.low)) bufferevent_run_readcb_(bufev, options); if ((iotype & EV_WRITE) && ((options & BEV_TRIG_IGNORE_WATERMARKS) || evbuffer_get_length(bufev->output) <= bufev->wm_write.low)) bufferevent_run_writecb_(bufev, options); } /** Internal: Add the event 'ev' with timeout tv, unless tv is set to 0, in * which case add ev with no timeout. */ int bufferevent_add_event_(struct event *ev, const struct timeval *tv); /* ========= * These next functions implement timeouts for bufferevents that aren't doing * anything else with ev_read and ev_write, to handle timeouts. * ========= */ /** Internal use: Set up the ev_read and ev_write callbacks so that * the other "generic_timeout" functions will work on it. Call this from * the constructor function. */ void bufferevent_init_generic_timeout_cbs_(struct bufferevent *bev); /** Internal use: Add or delete the generic timeout events as appropriate. * (If an event is enabled and a timeout is set, we add the event. Otherwise * we delete it.) Call this from anything that changes the timeout values, * that enabled EV_READ or EV_WRITE, or that disables EV_READ or EV_WRITE. */ int bufferevent_generic_adj_timeouts_(struct bufferevent *bev); int bufferevent_generic_adj_existing_timeouts_(struct bufferevent *bev); enum bufferevent_options bufferevent_get_options_(struct bufferevent *bev); const struct sockaddr* bufferevent_socket_get_conn_address_(struct bufferevent *bev); /** Internal use: We have just successfully read data into an inbuf, so * reset the read timeout (if any). */ #define BEV_RESET_GENERIC_READ_TIMEOUT(bev) \ do { \ if (evutil_timerisset(&(bev)->timeout_read)) \ event_add(&(bev)->ev_read, &(bev)->timeout_read); \ } while (0) /** Internal use: We have just successfully written data from an inbuf, so * reset the read timeout (if any). */ #define BEV_RESET_GENERIC_WRITE_TIMEOUT(bev) \ do { \ if (evutil_timerisset(&(bev)->timeout_write)) \ event_add(&(bev)->ev_write, &(bev)->timeout_write); \ } while (0) #define BEV_DEL_GENERIC_READ_TIMEOUT(bev) \ event_del(&(bev)->ev_read) #define BEV_DEL_GENERIC_WRITE_TIMEOUT(bev) \ event_del(&(bev)->ev_write) /** Internal: Given a bufferevent, return its corresponding * bufferevent_private. */ #define BEV_UPCAST(b) EVUTIL_UPCAST((b), struct bufferevent_private, bev) #ifdef EVENT__DISABLE_THREAD_SUPPORT #define BEV_LOCK(b) EVUTIL_NIL_STMT_ #define BEV_UNLOCK(b) EVUTIL_NIL_STMT_ #else /** Internal: Grab the lock (if any) on a bufferevent */ #define BEV_LOCK(b) do { \ struct bufferevent_private *locking = BEV_UPCAST(b); \ EVLOCK_LOCK(locking->lock, 0); \ } while (0) /** Internal: Release the lock (if any) on a bufferevent */ #define BEV_UNLOCK(b) do { \ struct bufferevent_private *locking = BEV_UPCAST(b); \ EVLOCK_UNLOCK(locking->lock, 0); \ } while (0) #endif /* ==== For rate-limiting. */ int bufferevent_decrement_write_buckets_(struct bufferevent_private *bev, ev_ssize_t bytes); int bufferevent_decrement_read_buckets_(struct bufferevent_private *bev, ev_ssize_t bytes); ev_ssize_t bufferevent_get_read_max_(struct bufferevent_private *bev); ev_ssize_t bufferevent_get_write_max_(struct bufferevent_private *bev); int bufferevent_ratelim_init_(struct bufferevent_private *bev); #ifdef __cplusplus } #endif #endif /* BUFFEREVENT_INTERNAL_H_INCLUDED_ */ ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/bufferevent.c������������������������������������������������0000664�0000000�0000000�00000062771�15270523712�0022472�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos, Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #include <sys/types.h> #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #ifdef EVENT__HAVE_STDARG_H #include <stdarg.h> #endif #ifdef _WIN32 #include <winsock2.h> #endif #include "event2/util.h" #include "event2/buffer.h" #include "event2/buffer_compat.h" #include "event2/bufferevent.h" #include "event2/bufferevent_struct.h" #include "event2/bufferevent_compat.h" #include "event2/event.h" #include "event-internal.h" #include "log-internal.h" #include "mm-internal.h" #include "bufferevent-internal.h" #include "evbuffer-internal.h" #include "util-internal.h" static void bufferevent_cancel_all_(struct bufferevent *bev); static void bufferevent_finalize_cb_(struct event_callback *evcb, void *arg_); void bufferevent_suspend_read_(struct bufferevent *bufev, bufferevent_suspend_flags what) { struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); if (!bufev_private->read_suspended) bufev->be_ops->disable(bufev, EV_READ); bufev_private->read_suspended |= what; BEV_UNLOCK(bufev); } void bufferevent_unsuspend_read_(struct bufferevent *bufev, bufferevent_suspend_flags what) { struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); bufev_private->read_suspended &= ~what; if (!bufev_private->read_suspended && (bufev->enabled & EV_READ)) bufev->be_ops->enable(bufev, EV_READ); BEV_UNLOCK(bufev); } void bufferevent_suspend_write_(struct bufferevent *bufev, bufferevent_suspend_flags what) { struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); if (!bufev_private->write_suspended) bufev->be_ops->disable(bufev, EV_WRITE); bufev_private->write_suspended |= what; BEV_UNLOCK(bufev); } void bufferevent_unsuspend_write_(struct bufferevent *bufev, bufferevent_suspend_flags what) { struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); bufev_private->write_suspended &= ~what; if (!bufev_private->write_suspended && (bufev->enabled & EV_WRITE)) bufev->be_ops->enable(bufev, EV_WRITE); BEV_UNLOCK(bufev); } /* Callback to implement watermarks on the input buffer. Only enabled * if the watermark is set. */ static void bufferevent_inbuf_wm_cb(struct evbuffer *buf, const struct evbuffer_cb_info *cbinfo, void *arg) { struct bufferevent *bufev = arg; size_t size; size = evbuffer_get_length(buf); if (size >= bufev->wm_read.high) bufferevent_wm_suspend_read(bufev); else bufferevent_wm_unsuspend_read(bufev); } static void bufferevent_run_deferred_callbacks_locked(struct event_callback *cb, void *arg) { struct bufferevent_private *bufev_private = arg; struct bufferevent *bufev = &bufev_private->bev; BEV_LOCK(bufev); if ((bufev_private->eventcb_pending & BEV_EVENT_CONNECTED) && bufev->errorcb) { /* The "connected" happened before any reads or writes, so send it first. */ bufev_private->eventcb_pending &= ~BEV_EVENT_CONNECTED; bufev->errorcb(bufev, BEV_EVENT_CONNECTED, bufev->cbarg); } if (bufev_private->readcb_pending && bufev->readcb) { bufev_private->readcb_pending = 0; bufev->readcb(bufev, bufev->cbarg); } if (bufev_private->writecb_pending && bufev->writecb) { bufev_private->writecb_pending = 0; bufev->writecb(bufev, bufev->cbarg); } if (bufev_private->eventcb_pending && bufev->errorcb) { short what = bufev_private->eventcb_pending; int err = bufev_private->errno_pending; bufev_private->eventcb_pending = 0; bufev_private->errno_pending = 0; EVUTIL_SET_SOCKET_ERROR(err); bufev->errorcb(bufev, what, bufev->cbarg); } bufferevent_decref_and_unlock_(bufev); } static void bufferevent_run_deferred_callbacks_unlocked(struct event_callback *cb, void *arg) { struct bufferevent_private *bufev_private = arg; struct bufferevent *bufev = &bufev_private->bev; BEV_LOCK(bufev); #define UNLOCKED(stmt) \ do { BEV_UNLOCK(bufev); stmt; BEV_LOCK(bufev); } while(0) if ((bufev_private->eventcb_pending & BEV_EVENT_CONNECTED) && bufev->errorcb) { /* The "connected" happened before any reads or writes, so send it first. */ bufferevent_event_cb errorcb = bufev->errorcb; void *cbarg = bufev->cbarg; bufev_private->eventcb_pending &= ~BEV_EVENT_CONNECTED; UNLOCKED(errorcb(bufev, BEV_EVENT_CONNECTED, cbarg)); } if (bufev_private->readcb_pending && bufev->readcb) { bufferevent_data_cb readcb = bufev->readcb; void *cbarg = bufev->cbarg; bufev_private->readcb_pending = 0; UNLOCKED(readcb(bufev, cbarg)); } if (bufev_private->writecb_pending && bufev->writecb) { bufferevent_data_cb writecb = bufev->writecb; void *cbarg = bufev->cbarg; bufev_private->writecb_pending = 0; UNLOCKED(writecb(bufev, cbarg)); } if (bufev_private->eventcb_pending && bufev->errorcb) { bufferevent_event_cb errorcb = bufev->errorcb; void *cbarg = bufev->cbarg; short what = bufev_private->eventcb_pending; int err = bufev_private->errno_pending; bufev_private->eventcb_pending = 0; bufev_private->errno_pending = 0; EVUTIL_SET_SOCKET_ERROR(err); UNLOCKED(errorcb(bufev,what,cbarg)); } bufferevent_decref_and_unlock_(bufev); #undef UNLOCKED } #define SCHEDULE_DEFERRED(bevp) \ do { \ if (event_deferred_cb_schedule_( \ (bevp)->bev.ev_base, \ &(bevp)->deferred)) \ bufferevent_incref_(&(bevp)->bev); \ } while (0) void bufferevent_run_readcb_(struct bufferevent *bufev, int options) { /* Requires that we hold the lock and a reference */ struct bufferevent_private *p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); if (bufev->readcb == NULL) return; if ((p->options|options) & BEV_OPT_DEFER_CALLBACKS) { p->readcb_pending = 1; SCHEDULE_DEFERRED(p); } else { bufev->readcb(bufev, bufev->cbarg); } } void bufferevent_run_writecb_(struct bufferevent *bufev, int options) { /* Requires that we hold the lock and a reference */ struct bufferevent_private *p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); if (bufev->writecb == NULL) return; if ((p->options|options) & BEV_OPT_DEFER_CALLBACKS) { p->writecb_pending = 1; SCHEDULE_DEFERRED(p); } else { bufev->writecb(bufev, bufev->cbarg); } } #define BEV_TRIG_ALL_OPTS ( \ BEV_TRIG_IGNORE_WATERMARKS| \ BEV_TRIG_DEFER_CALLBACKS \ ) void bufferevent_trigger(struct bufferevent *bufev, short iotype, int options) { bufferevent_incref_and_lock_(bufev); bufferevent_trigger_nolock_(bufev, iotype, options&BEV_TRIG_ALL_OPTS); bufferevent_decref_and_unlock_(bufev); } void bufferevent_run_eventcb_(struct bufferevent *bufev, short what, int options) { /* Requires that we hold the lock and a reference */ struct bufferevent_private *p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); if (bufev->errorcb == NULL) return; if ((p->options|options) & BEV_OPT_DEFER_CALLBACKS) { p->eventcb_pending |= what; p->errno_pending = EVUTIL_SOCKET_ERROR(); SCHEDULE_DEFERRED(p); } else { bufev->errorcb(bufev, what, bufev->cbarg); } } void bufferevent_trigger_event(struct bufferevent *bufev, short what, int options) { bufferevent_incref_and_lock_(bufev); bufferevent_run_eventcb_(bufev, what, options&BEV_TRIG_ALL_OPTS); bufferevent_decref_and_unlock_(bufev); } int bufferevent_init_common_(struct bufferevent_private *bufev_private, struct event_base *base, const struct bufferevent_ops *ops, enum bufferevent_options options) { struct bufferevent *bufev = &bufev_private->bev; if (!bufev->input) { if ((bufev->input = evbuffer_new()) == NULL) return -1; } if (!bufev->output) { if ((bufev->output = evbuffer_new()) == NULL) { evbuffer_free(bufev->input); return -1; } } bufev_private->refcnt = 1; bufev->ev_base = base; /* Disable timeouts. */ evutil_timerclear(&bufev->timeout_read); evutil_timerclear(&bufev->timeout_write); bufev->be_ops = ops; bufferevent_ratelim_init_(bufev_private); /* * Set to EV_WRITE so that using bufferevent_write is going to * trigger a callback. Reading needs to be explicitly enabled * because otherwise no data will be available. */ bufev->enabled = EV_WRITE; #ifndef EVENT__DISABLE_THREAD_SUPPORT if (options & BEV_OPT_THREADSAFE) { if (bufferevent_enable_locking_(bufev, NULL) < 0) { /* cleanup */ evbuffer_free(bufev->input); evbuffer_free(bufev->output); bufev->input = NULL; bufev->output = NULL; return -1; } } #endif if ((options & (BEV_OPT_DEFER_CALLBACKS|BEV_OPT_UNLOCK_CALLBACKS)) == BEV_OPT_UNLOCK_CALLBACKS) { event_warnx("UNLOCK_CALLBACKS requires DEFER_CALLBACKS"); return -1; } if (options & BEV_OPT_UNLOCK_CALLBACKS) event_deferred_cb_init_( &bufev_private->deferred, event_base_get_npriorities(base) / 2, bufferevent_run_deferred_callbacks_unlocked, bufev_private); else event_deferred_cb_init_( &bufev_private->deferred, event_base_get_npriorities(base) / 2, bufferevent_run_deferred_callbacks_locked, bufev_private); bufev_private->options = options; evbuffer_set_parent_(bufev->input, bufev); evbuffer_set_parent_(bufev->output, bufev); return 0; } void bufferevent_setcb(struct bufferevent *bufev, bufferevent_data_cb readcb, bufferevent_data_cb writecb, bufferevent_event_cb eventcb, void *cbarg) { BEV_LOCK(bufev); bufev->readcb = readcb; bufev->writecb = writecb; bufev->errorcb = eventcb; bufev->cbarg = cbarg; BEV_UNLOCK(bufev); } void bufferevent_getcb(struct bufferevent *bufev, bufferevent_data_cb *readcb_ptr, bufferevent_data_cb *writecb_ptr, bufferevent_event_cb *eventcb_ptr, void **cbarg_ptr) { BEV_LOCK(bufev); if (readcb_ptr) *readcb_ptr = bufev->readcb; if (writecb_ptr) *writecb_ptr = bufev->writecb; if (eventcb_ptr) *eventcb_ptr = bufev->errorcb; if (cbarg_ptr) *cbarg_ptr = bufev->cbarg; BEV_UNLOCK(bufev); } struct evbuffer * bufferevent_get_input(struct bufferevent *bufev) { return bufev->input; } struct evbuffer * bufferevent_get_output(struct bufferevent *bufev) { return bufev->output; } struct event_base * bufferevent_get_base(struct bufferevent *bufev) { return bufev->ev_base; } int bufferevent_get_priority(const struct bufferevent *bufev) { if (event_initialized(&bufev->ev_read)) { return event_get_priority(&bufev->ev_read); } else { return event_base_get_npriorities(bufev->ev_base) / 2; } } int bufferevent_write(struct bufferevent *bufev, const void *data, size_t size) { if (evbuffer_add(bufev->output, data, size) == -1) return (-1); return 0; } int bufferevent_write_buffer(struct bufferevent *bufev, struct evbuffer *buf) { if (evbuffer_add_buffer(bufev->output, buf) == -1) return (-1); return 0; } size_t bufferevent_read(struct bufferevent *bufev, void *data, size_t size) { return (evbuffer_remove(bufev->input, data, size)); } int bufferevent_read_buffer(struct bufferevent *bufev, struct evbuffer *buf) { return (evbuffer_add_buffer(buf, bufev->input)); } int bufferevent_enable(struct bufferevent *bufev, short event) { struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); short impl_events = event; int r = 0; bufferevent_incref_and_lock_(bufev); if (bufev_private->read_suspended) impl_events &= ~EV_READ; if (bufev_private->write_suspended) impl_events &= ~EV_WRITE; bufev->enabled |= event; if (impl_events && bufev->be_ops->enable(bufev, impl_events) < 0) r = -1; bufferevent_decref_and_unlock_(bufev); return r; } int bufferevent_set_timeouts(struct bufferevent *bufev, const struct timeval *tv_read, const struct timeval *tv_write) { int r = 0; BEV_LOCK(bufev); if (tv_read) { bufev->timeout_read = *tv_read; } else { evutil_timerclear(&bufev->timeout_read); } if (tv_write) { bufev->timeout_write = *tv_write; } else { evutil_timerclear(&bufev->timeout_write); } if (bufev->be_ops->adj_timeouts) r = bufev->be_ops->adj_timeouts(bufev); BEV_UNLOCK(bufev); return r; } /* Obsolete; use bufferevent_set_timeouts */ void bufferevent_settimeout(struct bufferevent *bufev, int timeout_read, int timeout_write) { struct timeval tv_read, tv_write; struct timeval *ptv_read = NULL, *ptv_write = NULL; memset(&tv_read, 0, sizeof(tv_read)); memset(&tv_write, 0, sizeof(tv_write)); if (timeout_read) { tv_read.tv_sec = timeout_read; ptv_read = &tv_read; } if (timeout_write) { tv_write.tv_sec = timeout_write; ptv_write = &tv_write; } bufferevent_set_timeouts(bufev, ptv_read, ptv_write); } int bufferevent_disable_hard_(struct bufferevent *bufev, short event) { int r = 0; struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); bufev->enabled &= ~event; bufev_private->connecting = 0; if (bufev->be_ops->disable(bufev, event) < 0) r = -1; BEV_UNLOCK(bufev); return r; } int bufferevent_disable(struct bufferevent *bufev, short event) { int r = 0; BEV_LOCK(bufev); bufev->enabled &= ~event; if (bufev->be_ops->disable(bufev, event) < 0) r = -1; BEV_UNLOCK(bufev); return r; } /* * Sets the water marks */ void bufferevent_setwatermark(struct bufferevent *bufev, short events, size_t lowmark, size_t highmark) { struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); if (events & EV_WRITE) { bufev->wm_write.low = lowmark; bufev->wm_write.high = highmark; } if (events & EV_READ) { bufev->wm_read.low = lowmark; bufev->wm_read.high = highmark; if (highmark) { /* There is now a new high-water mark for read. enable the callback if needed, and see if we should suspend/bufferevent_wm_unsuspend. */ if (bufev_private->read_watermarks_cb == NULL) { bufev_private->read_watermarks_cb = evbuffer_add_cb(bufev->input, bufferevent_inbuf_wm_cb, bufev); } evbuffer_cb_set_flags(bufev->input, bufev_private->read_watermarks_cb, EVBUFFER_CB_ENABLED|EVBUFFER_CB_NODEFER); if (evbuffer_get_length(bufev->input) >= highmark) bufferevent_wm_suspend_read(bufev); else if (evbuffer_get_length(bufev->input) < highmark) bufferevent_wm_unsuspend_read(bufev); } else { /* There is now no high-water mark for read. */ if (bufev_private->read_watermarks_cb) evbuffer_cb_clear_flags(bufev->input, bufev_private->read_watermarks_cb, EVBUFFER_CB_ENABLED); bufferevent_wm_unsuspend_read(bufev); } } BEV_UNLOCK(bufev); } int bufferevent_getwatermark(struct bufferevent *bufev, short events, size_t *lowmark, size_t *highmark) { if (events == EV_WRITE) { BEV_LOCK(bufev); if (lowmark) *lowmark = bufev->wm_write.low; if (highmark) *highmark = bufev->wm_write.high; BEV_UNLOCK(bufev); return 0; } if (events == EV_READ) { BEV_LOCK(bufev); if (lowmark) *lowmark = bufev->wm_read.low; if (highmark) *highmark = bufev->wm_read.high; BEV_UNLOCK(bufev); return 0; } return -1; } int bufferevent_flush(struct bufferevent *bufev, short iotype, enum bufferevent_flush_mode mode) { int r = -1; BEV_LOCK(bufev); if (bufev->be_ops->flush) r = bufev->be_ops->flush(bufev, iotype, mode); BEV_UNLOCK(bufev); return r; } void bufferevent_incref_and_lock_(struct bufferevent *bufev) { struct bufferevent_private *bufev_private = BEV_UPCAST(bufev); BEV_LOCK(bufev); ++bufev_private->refcnt; } #if 0 static void bufferevent_transfer_lock_ownership_(struct bufferevent *donor, struct bufferevent *recipient) { struct bufferevent_private *d = BEV_UPCAST(donor); struct bufferevent_private *r = BEV_UPCAST(recipient); if (d->lock != r->lock) return; if (r->own_lock) return; if (d->own_lock) { d->own_lock = 0; r->own_lock = 1; } } #endif int bufferevent_decref_and_unlock_(struct bufferevent *bufev) { struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); int n_cbs = 0; #define MAX_CBS 16 struct event_callback *cbs[MAX_CBS]; EVUTIL_ASSERT(bufev_private->refcnt > 0); if (--bufev_private->refcnt) { BEV_UNLOCK(bufev); return 0; } if (bufev->be_ops->unlink) bufev->be_ops->unlink(bufev); /* Okay, we're out of references. Let's finalize this once all the * callbacks are done running. */ cbs[0] = &bufev->ev_read.ev_evcallback; cbs[1] = &bufev->ev_write.ev_evcallback; cbs[2] = &bufev_private->deferred; n_cbs = 3; if (bufev_private->rate_limiting) { struct event *e = &bufev_private->rate_limiting->refill_bucket_event; if (event_initialized(e)) cbs[n_cbs++] = &e->ev_evcallback; } n_cbs += evbuffer_get_callbacks_(bufev->input, cbs+n_cbs, MAX_CBS-n_cbs); n_cbs += evbuffer_get_callbacks_(bufev->output, cbs+n_cbs, MAX_CBS-n_cbs); event_callback_finalize_many_(bufev->ev_base, n_cbs, cbs, bufferevent_finalize_cb_); #undef MAX_CBS BEV_UNLOCK(bufev); return 1; } static void bufferevent_finalize_cb_(struct event_callback *evcb, void *arg_) { struct bufferevent *bufev = arg_; struct bufferevent *underlying; struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); underlying = bufferevent_get_underlying(bufev); /* Clean up the shared info */ if (bufev->be_ops->destruct) bufev->be_ops->destruct(bufev); /* XXX what happens if refcnt for these buffers is > 1? * The buffers can share a lock with this bufferevent object, * but the lock might be destroyed below. */ /* evbuffer will free the callbacks */ evbuffer_free(bufev->input); evbuffer_free(bufev->output); if (bufev_private->rate_limiting) { if (bufev_private->rate_limiting->group) bufferevent_remove_from_rate_limit_group_internal_(bufev,0); mm_free(bufev_private->rate_limiting); bufev_private->rate_limiting = NULL; } BEV_UNLOCK(bufev); if (bufev_private->own_lock) EVTHREAD_FREE_LOCK(bufev_private->lock, EVTHREAD_LOCKTYPE_RECURSIVE); /* Free the actual allocated memory. */ mm_free(((char*)bufev) - bufev->be_ops->mem_offset); /* Release the reference to underlying now that we no longer need the * reference to it. We wait this long mainly in case our lock is * shared with underlying. * * The 'destruct' function will also drop a reference to underlying * if BEV_OPT_CLOSE_ON_FREE is set. * * XXX Should we/can we just refcount evbuffer/bufferevent locks? * It would probably save us some headaches. */ if (underlying) bufferevent_decref_(underlying); } int bufferevent_decref(struct bufferevent *bufev) { BEV_LOCK(bufev); return bufferevent_decref_and_unlock_(bufev); } void bufferevent_free(struct bufferevent *bufev) { BEV_LOCK(bufev); bufferevent_setcb(bufev, NULL, NULL, NULL, NULL); bufferevent_cancel_all_(bufev); bufferevent_decref_and_unlock_(bufev); } void bufferevent_incref(struct bufferevent *bufev) { struct bufferevent_private *bufev_private = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); /* XXX: now that this function is public, we might want to * - return the count from this function * - create a new function to atomically grab the current refcount */ BEV_LOCK(bufev); ++bufev_private->refcnt; BEV_UNLOCK(bufev); } int bufferevent_enable_locking_(struct bufferevent *bufev, void *lock) { #ifdef EVENT__DISABLE_THREAD_SUPPORT return -1; #else struct bufferevent *underlying; if (BEV_UPCAST(bufev)->lock) return -1; underlying = bufferevent_get_underlying(bufev); if (!lock && underlying && BEV_UPCAST(underlying)->lock) { lock = BEV_UPCAST(underlying)->lock; BEV_UPCAST(bufev)->lock = lock; BEV_UPCAST(bufev)->own_lock = 0; } else if (!lock) { EVTHREAD_ALLOC_LOCK(lock, EVTHREAD_LOCKTYPE_RECURSIVE); if (!lock) return -1; BEV_UPCAST(bufev)->lock = lock; BEV_UPCAST(bufev)->own_lock = 1; } else { BEV_UPCAST(bufev)->lock = lock; BEV_UPCAST(bufev)->own_lock = 0; } evbuffer_enable_locking(bufev->input, lock); evbuffer_enable_locking(bufev->output, lock); if (underlying && !BEV_UPCAST(underlying)->lock) bufferevent_enable_locking_(underlying, lock); return 0; #endif } int bufferevent_setfd(struct bufferevent *bev, evutil_socket_t fd) { union bufferevent_ctrl_data d; int res = -1; d.fd = fd; BEV_LOCK(bev); if (bev->be_ops->ctrl) res = bev->be_ops->ctrl(bev, BEV_CTRL_SET_FD, &d); BEV_UNLOCK(bev); return res; } evutil_socket_t bufferevent_getfd(struct bufferevent *bev) { union bufferevent_ctrl_data d; int res = -1; d.fd = -1; BEV_LOCK(bev); if (bev->be_ops->ctrl) res = bev->be_ops->ctrl(bev, BEV_CTRL_GET_FD, &d); BEV_UNLOCK(bev); return (res<0) ? -1 : d.fd; } enum bufferevent_options bufferevent_get_options_(struct bufferevent *bev) { struct bufferevent_private *bev_p = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); enum bufferevent_options options; BEV_LOCK(bev); options = bev_p->options; BEV_UNLOCK(bev); return options; } static void bufferevent_cancel_all_(struct bufferevent *bev) { union bufferevent_ctrl_data d; memset(&d, 0, sizeof(d)); BEV_LOCK(bev); if (bev->be_ops->ctrl) bev->be_ops->ctrl(bev, BEV_CTRL_CANCEL_ALL, &d); BEV_UNLOCK(bev); } short bufferevent_get_enabled(struct bufferevent *bufev) { short r; BEV_LOCK(bufev); r = bufev->enabled; BEV_UNLOCK(bufev); return r; } struct bufferevent * bufferevent_get_underlying(struct bufferevent *bev) { union bufferevent_ctrl_data d; int res = -1; d.ptr = NULL; BEV_LOCK(bev); if (bev->be_ops->ctrl) res = bev->be_ops->ctrl(bev, BEV_CTRL_GET_UNDERLYING, &d); BEV_UNLOCK(bev); return (res<0) ? NULL : d.ptr; } static void bufferevent_generic_read_timeout_cb(evutil_socket_t fd, short event, void *ctx) { struct bufferevent *bev = ctx; bufferevent_incref_and_lock_(bev); bufferevent_disable(bev, EV_READ); bufferevent_run_eventcb_(bev, BEV_EVENT_TIMEOUT|BEV_EVENT_READING, 0); bufferevent_decref_and_unlock_(bev); } static void bufferevent_generic_write_timeout_cb(evutil_socket_t fd, short event, void *ctx) { struct bufferevent *bev = ctx; bufferevent_incref_and_lock_(bev); bufferevent_disable(bev, EV_WRITE); bufferevent_run_eventcb_(bev, BEV_EVENT_TIMEOUT|BEV_EVENT_WRITING, 0); bufferevent_decref_and_unlock_(bev); } void bufferevent_init_generic_timeout_cbs_(struct bufferevent *bev) { event_assign(&bev->ev_read, bev->ev_base, -1, EV_FINALIZE, bufferevent_generic_read_timeout_cb, bev); event_assign(&bev->ev_write, bev->ev_base, -1, EV_FINALIZE, bufferevent_generic_write_timeout_cb, bev); } int bufferevent_generic_adj_timeouts_(struct bufferevent *bev) { const short enabled = bev->enabled; struct bufferevent_private *bev_p = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); int r1=0, r2=0; if ((enabled & EV_READ) && !bev_p->read_suspended && evutil_timerisset(&bev->timeout_read)) r1 = event_add(&bev->ev_read, &bev->timeout_read); else r1 = event_del(&bev->ev_read); if ((enabled & EV_WRITE) && !bev_p->write_suspended && evutil_timerisset(&bev->timeout_write) && evbuffer_get_length(bev->output)) r2 = event_add(&bev->ev_write, &bev->timeout_write); else r2 = event_del(&bev->ev_write); if (r1 < 0 || r2 < 0) return -1; return 0; } int bufferevent_generic_adj_existing_timeouts_(struct bufferevent *bev) { int r = 0; if (event_pending(&bev->ev_read, EV_READ, NULL)) { if (evutil_timerisset(&bev->timeout_read)) { if (bufferevent_add_event_(&bev->ev_read, &bev->timeout_read) < 0) r = -1; } else { event_remove_timer(&bev->ev_read); } } if (event_pending(&bev->ev_write, EV_WRITE, NULL)) { if (evutil_timerisset(&bev->timeout_write)) { if (bufferevent_add_event_(&bev->ev_write, &bev->timeout_write) < 0) r = -1; } else { event_remove_timer(&bev->ev_write); } } return r; } int bufferevent_add_event_(struct event *ev, const struct timeval *tv) { if (!evutil_timerisset(tv)) return event_add(ev, NULL); else return event_add(ev, tv); } /* For use by user programs only; internally, we should be calling either bufferevent_incref_and_lock_(), or BEV_LOCK. */ void bufferevent_lock(struct bufferevent *bev) { bufferevent_incref_and_lock_(bev); } void bufferevent_unlock(struct bufferevent *bev) { bufferevent_decref_and_unlock_(bev); } �������kytensor-2.49.0.7/third_party/libevent/bufferevent_async.c������������������������������������������0000664�0000000�0000000�00000043403�15270523712�0023656�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #ifdef EVENT__HAVE_STDARG_H #include <stdarg.h> #endif #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #ifdef _WIN32 #include <winsock2.h> #include <ws2tcpip.h> #endif #include <sys/queue.h> #include "event2/util.h" #include "event2/bufferevent.h" #include "event2/buffer.h" #include "event2/bufferevent_struct.h" #include "event2/event.h" #include "event2/util.h" #include "event-internal.h" #include "log-internal.h" #include "mm-internal.h" #include "bufferevent-internal.h" #include "util-internal.h" #include "iocp-internal.h" #ifndef SO_UPDATE_CONNECT_CONTEXT /* Mingw is sometimes missing this */ #define SO_UPDATE_CONNECT_CONTEXT 0x7010 #endif /* prototypes */ static int be_async_enable(struct bufferevent *, short); static int be_async_disable(struct bufferevent *, short); static void be_async_destruct(struct bufferevent *); static int be_async_flush(struct bufferevent *, short, enum bufferevent_flush_mode); static int be_async_ctrl(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *); struct bufferevent_async { struct bufferevent_private bev; struct event_overlapped connect_overlapped; struct event_overlapped read_overlapped; struct event_overlapped write_overlapped; size_t read_in_progress; size_t write_in_progress; unsigned ok : 1; unsigned read_added : 1; unsigned write_added : 1; }; const struct bufferevent_ops bufferevent_ops_async = { "socket_async", evutil_offsetof(struct bufferevent_async, bev.bev), be_async_enable, be_async_disable, NULL, /* Unlink */ be_async_destruct, bufferevent_generic_adj_timeouts_, be_async_flush, be_async_ctrl, }; static inline struct bufferevent_async * upcast(struct bufferevent *bev) { struct bufferevent_async *bev_a; if (bev->be_ops != &bufferevent_ops_async) return NULL; bev_a = EVUTIL_UPCAST(bev, struct bufferevent_async, bev.bev); return bev_a; } static inline struct bufferevent_async * upcast_connect(struct event_overlapped *eo) { struct bufferevent_async *bev_a; bev_a = EVUTIL_UPCAST(eo, struct bufferevent_async, connect_overlapped); EVUTIL_ASSERT(BEV_IS_ASYNC(&bev_a->bev.bev)); return bev_a; } static inline struct bufferevent_async * upcast_read(struct event_overlapped *eo) { struct bufferevent_async *bev_a; bev_a = EVUTIL_UPCAST(eo, struct bufferevent_async, read_overlapped); EVUTIL_ASSERT(BEV_IS_ASYNC(&bev_a->bev.bev)); return bev_a; } static inline struct bufferevent_async * upcast_write(struct event_overlapped *eo) { struct bufferevent_async *bev_a; bev_a = EVUTIL_UPCAST(eo, struct bufferevent_async, write_overlapped); EVUTIL_ASSERT(BEV_IS_ASYNC(&bev_a->bev.bev)); return bev_a; } static void bev_async_del_write(struct bufferevent_async *beva) { struct bufferevent *bev = &beva->bev.bev; if (beva->write_added) { beva->write_added = 0; event_base_del_virtual_(bev->ev_base); } } static void bev_async_del_read(struct bufferevent_async *beva) { struct bufferevent *bev = &beva->bev.bev; if (beva->read_added) { beva->read_added = 0; event_base_del_virtual_(bev->ev_base); } } static void bev_async_add_write(struct bufferevent_async *beva) { struct bufferevent *bev = &beva->bev.bev; if (!beva->write_added) { beva->write_added = 1; event_base_add_virtual_(bev->ev_base); } } static void bev_async_add_read(struct bufferevent_async *beva) { struct bufferevent *bev = &beva->bev.bev; if (!beva->read_added) { beva->read_added = 1; event_base_add_virtual_(bev->ev_base); } } static void bev_async_consider_writing(struct bufferevent_async *beva) { size_t at_most; int limit; struct bufferevent *bev = &beva->bev.bev; /* Don't write if there's a write in progress, or we do not * want to write, or when there's nothing left to write. */ if (beva->write_in_progress || beva->bev.connecting) return; if (!beva->ok || !(bev->enabled&EV_WRITE) || !evbuffer_get_length(bev->output)) { bev_async_del_write(beva); return; } at_most = evbuffer_get_length(bev->output); /* This is safe so long as bufferevent_get_write_max never returns * more than INT_MAX. That's true for now. XXXX */ limit = (int)bufferevent_get_write_max_(&beva->bev); if (at_most >= (size_t)limit && limit >= 0) at_most = limit; if (beva->bev.write_suspended) { bev_async_del_write(beva); return; } /* XXXX doesn't respect low-water mark very well. */ bufferevent_incref_(bev); if (evbuffer_launch_write_(bev->output, at_most, &beva->write_overlapped)) { bufferevent_decref_(bev); beva->ok = 0; bufferevent_run_eventcb_(bev, BEV_EVENT_ERROR, 0); } else { beva->write_in_progress = at_most; bufferevent_decrement_write_buckets_(&beva->bev, at_most); bev_async_add_write(beva); } } static void bev_async_consider_reading(struct bufferevent_async *beva) { size_t cur_size; size_t read_high; size_t at_most; int limit; struct bufferevent *bev = &beva->bev.bev; /* Don't read if there is a read in progress, or we do not * want to read. */ if (beva->read_in_progress || beva->bev.connecting) return; if (!beva->ok || !(bev->enabled&EV_READ)) { bev_async_del_read(beva); return; } /* Don't read if we're full */ cur_size = evbuffer_get_length(bev->input); read_high = bev->wm_read.high; if (read_high) { if (cur_size >= read_high) { bev_async_del_read(beva); return; } at_most = read_high - cur_size; } else { at_most = 16384; /* FIXME totally magic. */ } /* XXXX This over-commits. */ /* XXXX see also not above on cast on bufferevent_get_write_max_() */ limit = (int)bufferevent_get_read_max_(&beva->bev); if (at_most >= (size_t)limit && limit >= 0) at_most = limit; if (beva->bev.read_suspended) { bev_async_del_read(beva); return; } bufferevent_incref_(bev); if (evbuffer_launch_read_(bev->input, at_most, &beva->read_overlapped)) { beva->ok = 0; bufferevent_run_eventcb_(bev, BEV_EVENT_ERROR, 0); bufferevent_decref_(bev); } else { beva->read_in_progress = at_most; bufferevent_decrement_read_buckets_(&beva->bev, at_most); bev_async_add_read(beva); } return; } static void be_async_outbuf_callback(struct evbuffer *buf, const struct evbuffer_cb_info *cbinfo, void *arg) { struct bufferevent *bev = arg; struct bufferevent_async *bev_async = upcast(bev); /* If we added data to the outbuf and were not writing before, * we may want to write now. */ bufferevent_incref_and_lock_(bev); if (cbinfo->n_added) bev_async_consider_writing(bev_async); bufferevent_decref_and_unlock_(bev); } static void be_async_inbuf_callback(struct evbuffer *buf, const struct evbuffer_cb_info *cbinfo, void *arg) { struct bufferevent *bev = arg; struct bufferevent_async *bev_async = upcast(bev); /* If we drained data from the inbuf and were not reading before, * we may want to read now */ bufferevent_incref_and_lock_(bev); if (cbinfo->n_deleted) bev_async_consider_reading(bev_async); bufferevent_decref_and_unlock_(bev); } static int be_async_enable(struct bufferevent *buf, short what) { struct bufferevent_async *bev_async = upcast(buf); if (!bev_async->ok) return -1; if (bev_async->bev.connecting) { /* Don't launch anything during connection attempts. */ return 0; } if (what & EV_READ) BEV_RESET_GENERIC_READ_TIMEOUT(buf); if (what & EV_WRITE) BEV_RESET_GENERIC_WRITE_TIMEOUT(buf); /* If we newly enable reading or writing, and we aren't reading or writing already, consider launching a new read or write. */ if (what & EV_READ) bev_async_consider_reading(bev_async); if (what & EV_WRITE) bev_async_consider_writing(bev_async); return 0; } static int be_async_disable(struct bufferevent *bev, short what) { struct bufferevent_async *bev_async = upcast(bev); /* XXXX If we disable reading or writing, we may want to consider * canceling any in-progress read or write operation, though it might * not work. */ if (what & EV_READ) { BEV_DEL_GENERIC_READ_TIMEOUT(bev); bev_async_del_read(bev_async); } if (what & EV_WRITE) { BEV_DEL_GENERIC_WRITE_TIMEOUT(bev); bev_async_del_write(bev_async); } return 0; } static void be_async_destruct(struct bufferevent *bev) { struct bufferevent_async *bev_async = upcast(bev); struct bufferevent_private *bev_p = BEV_UPCAST(bev); evutil_socket_t fd; EVUTIL_ASSERT(!upcast(bev)->write_in_progress && !upcast(bev)->read_in_progress); bev_async_del_read(bev_async); bev_async_del_write(bev_async); fd = evbuffer_overlapped_get_fd_(bev->input); if (fd != (evutil_socket_t)INVALID_SOCKET && (bev_p->options & BEV_OPT_CLOSE_ON_FREE)) { evutil_closesocket(fd); evbuffer_overlapped_set_fd_(bev->input, INVALID_SOCKET); } } /* GetQueuedCompletionStatus doesn't reliably yield WSA error codes, so * we use WSAGetOverlappedResult to translate. */ static void bev_async_set_wsa_error(struct bufferevent *bev, struct event_overlapped *eo) { DWORD bytes, flags; evutil_socket_t fd; fd = evbuffer_overlapped_get_fd_(bev->input); WSAGetOverlappedResult(fd, &eo->overlapped, &bytes, FALSE, &flags); } static int be_async_flush(struct bufferevent *bev, short what, enum bufferevent_flush_mode mode) { return 0; } static void connect_complete(struct event_overlapped *eo, ev_uintptr_t key, ev_ssize_t nbytes, int ok) { struct bufferevent_async *bev_a = upcast_connect(eo); struct bufferevent *bev = &bev_a->bev.bev; evutil_socket_t sock; BEV_LOCK(bev); EVUTIL_ASSERT(bev_a->bev.connecting); bev_a->bev.connecting = 0; sock = evbuffer_overlapped_get_fd_(bev_a->bev.bev.input); /* XXXX Handle error? */ setsockopt(sock, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, NULL, 0); if (ok) bufferevent_async_set_connected_(bev); else bev_async_set_wsa_error(bev, eo); bufferevent_run_eventcb_(bev, ok? BEV_EVENT_CONNECTED : BEV_EVENT_ERROR, 0); event_base_del_virtual_(bev->ev_base); bufferevent_decref_and_unlock_(bev); } static void read_complete(struct event_overlapped *eo, ev_uintptr_t key, ev_ssize_t nbytes, int ok) { struct bufferevent_async *bev_a = upcast_read(eo); struct bufferevent *bev = &bev_a->bev.bev; short what = BEV_EVENT_READING; ev_ssize_t amount_unread; BEV_LOCK(bev); EVUTIL_ASSERT(bev_a->read_in_progress); amount_unread = bev_a->read_in_progress - nbytes; evbuffer_commit_read_(bev->input, nbytes); bev_a->read_in_progress = 0; if (amount_unread) bufferevent_decrement_read_buckets_(&bev_a->bev, -amount_unread); if (!ok) bev_async_set_wsa_error(bev, eo); if (bev_a->ok) { if (ok && nbytes) { BEV_RESET_GENERIC_READ_TIMEOUT(bev); bufferevent_trigger_nolock_(bev, EV_READ, 0); bev_async_consider_reading(bev_a); } else if (!ok) { what |= BEV_EVENT_ERROR; bev_a->ok = 0; bufferevent_run_eventcb_(bev, what, 0); } else if (!nbytes) { what |= BEV_EVENT_EOF; bev_a->ok = 0; bufferevent_run_eventcb_(bev, what, 0); } } bufferevent_decref_and_unlock_(bev); } static void write_complete(struct event_overlapped *eo, ev_uintptr_t key, ev_ssize_t nbytes, int ok) { struct bufferevent_async *bev_a = upcast_write(eo); struct bufferevent *bev = &bev_a->bev.bev; short what = BEV_EVENT_WRITING; ev_ssize_t amount_unwritten; BEV_LOCK(bev); EVUTIL_ASSERT(bev_a->write_in_progress); amount_unwritten = bev_a->write_in_progress - nbytes; evbuffer_commit_write_(bev->output, nbytes); bev_a->write_in_progress = 0; if (amount_unwritten) bufferevent_decrement_write_buckets_(&bev_a->bev, -amount_unwritten); if (!ok) bev_async_set_wsa_error(bev, eo); if (bev_a->ok) { if (ok && nbytes) { BEV_RESET_GENERIC_WRITE_TIMEOUT(bev); bufferevent_trigger_nolock_(bev, EV_WRITE, 0); bev_async_consider_writing(bev_a); } else if (!ok) { what |= BEV_EVENT_ERROR; bev_a->ok = 0; bufferevent_run_eventcb_(bev, what, 0); } else if (!nbytes) { what |= BEV_EVENT_EOF; bev_a->ok = 0; bufferevent_run_eventcb_(bev, what, 0); } } bufferevent_decref_and_unlock_(bev); } struct bufferevent * bufferevent_async_new_(struct event_base *base, evutil_socket_t fd, int options) { struct bufferevent_async *bev_a; struct bufferevent *bev; struct event_iocp_port *iocp; options |= BEV_OPT_THREADSAFE; if (!(iocp = event_base_get_iocp_(base))) return NULL; if (fd >= 0 && event_iocp_port_associate_(iocp, fd, 1)<0) { int err = GetLastError(); /* We may have alrady associated this fd with a port. * Let's hope it's this port, and that the error code * for doing this neer changes. */ if (err != ERROR_INVALID_PARAMETER) return NULL; } if (!(bev_a = mm_calloc(1, sizeof(struct bufferevent_async)))) return NULL; bev = &bev_a->bev.bev; if (!(bev->input = evbuffer_overlapped_new_(fd))) { mm_free(bev_a); return NULL; } if (!(bev->output = evbuffer_overlapped_new_(fd))) { evbuffer_free(bev->input); mm_free(bev_a); return NULL; } if (bufferevent_init_common_(&bev_a->bev, base, &bufferevent_ops_async, options)<0) goto err; evbuffer_add_cb(bev->input, be_async_inbuf_callback, bev); evbuffer_add_cb(bev->output, be_async_outbuf_callback, bev); event_overlapped_init_(&bev_a->connect_overlapped, connect_complete); event_overlapped_init_(&bev_a->read_overlapped, read_complete); event_overlapped_init_(&bev_a->write_overlapped, write_complete); bufferevent_init_generic_timeout_cbs_(bev); bev_a->ok = fd >= 0; return bev; err: bufferevent_free(&bev_a->bev.bev); return NULL; } void bufferevent_async_set_connected_(struct bufferevent *bev) { struct bufferevent_async *bev_async = upcast(bev); bev_async->ok = 1; bufferevent_init_generic_timeout_cbs_(bev); /* Now's a good time to consider reading/writing */ be_async_enable(bev, bev->enabled); } int bufferevent_async_can_connect_(struct bufferevent *bev) { const struct win32_extension_fns *ext = event_get_win32_extension_fns_(); if (BEV_IS_ASYNC(bev) && event_base_get_iocp_(bev->ev_base) && ext && ext->ConnectEx) return 1; return 0; } int bufferevent_async_connect_(struct bufferevent *bev, evutil_socket_t fd, const struct sockaddr *sa, int socklen) { BOOL rc; struct bufferevent_async *bev_async = upcast(bev); struct sockaddr_storage ss; const struct win32_extension_fns *ext = event_get_win32_extension_fns_(); EVUTIL_ASSERT(ext && ext->ConnectEx && fd >= 0 && sa != NULL); /* ConnectEx() requires that the socket be bound to an address * with bind() before using, otherwise it will fail. We attempt * to issue a bind() here, taking into account that the error * code is set to WSAEINVAL when the socket is already bound. */ memset(&ss, 0, sizeof(ss)); if (sa->sa_family == AF_INET) { struct sockaddr_in *sin = (struct sockaddr_in *)&ss; sin->sin_family = AF_INET; sin->sin_addr.s_addr = INADDR_ANY; } else if (sa->sa_family == AF_INET6) { struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; sin6->sin6_family = AF_INET6; sin6->sin6_addr = in6addr_any; } else { /* Well, the user will have to bind() */ return -1; } if (bind(fd, (struct sockaddr *)&ss, sizeof(ss)) < 0 && WSAGetLastError() != WSAEINVAL) return -1; event_base_add_virtual_(bev->ev_base); bufferevent_incref_(bev); rc = ext->ConnectEx(fd, sa, socklen, NULL, 0, NULL, &bev_async->connect_overlapped.overlapped); if (rc || WSAGetLastError() == ERROR_IO_PENDING) return 0; event_base_del_virtual_(bev->ev_base); bufferevent_decref_(bev); return -1; } static int be_async_ctrl(struct bufferevent *bev, enum bufferevent_ctrl_op op, union bufferevent_ctrl_data *data) { switch (op) { case BEV_CTRL_GET_FD: data->fd = evbuffer_overlapped_get_fd_(bev->input); return 0; case BEV_CTRL_SET_FD: { struct event_iocp_port *iocp; if (data->fd == evbuffer_overlapped_get_fd_(bev->input)) return 0; if (!(iocp = event_base_get_iocp_(bev->ev_base))) return -1; if (event_iocp_port_associate_(iocp, data->fd, 1) < 0) return -1; evbuffer_overlapped_set_fd_(bev->input, data->fd); evbuffer_overlapped_set_fd_(bev->output, data->fd); return 0; } case BEV_CTRL_CANCEL_ALL: { struct bufferevent_async *bev_a = upcast(bev); evutil_socket_t fd = evbuffer_overlapped_get_fd_(bev->input); if (fd != (evutil_socket_t)INVALID_SOCKET && (bev_a->bev.options & BEV_OPT_CLOSE_ON_FREE)) { closesocket(fd); evbuffer_overlapped_set_fd_(bev->input, INVALID_SOCKET); } bev_a->ok = 0; return 0; } case BEV_CTRL_GET_UNDERLYING: default: return -1; } } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/bufferevent_filter.c�����������������������������������������0000664�0000000�0000000�00000043127�15270523712�0024031�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * Copyright (c) 2002-2006 Niels Provos <provos@citi.umich.edu> * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "evconfig-private.h" #include <sys/types.h> #include "event2/event-config.h" #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #ifdef EVENT__HAVE_STDARG_H #include <stdarg.h> #endif #ifdef _WIN32 #include <winsock2.h> #endif #include "event2/util.h" #include "event2/bufferevent.h" #include "event2/buffer.h" #include "event2/bufferevent_struct.h" #include "event2/event.h" #include "log-internal.h" #include "mm-internal.h" #include "bufferevent-internal.h" #include "util-internal.h" /* prototypes */ static int be_filter_enable(struct bufferevent *, short); static int be_filter_disable(struct bufferevent *, short); static void be_filter_unlink(struct bufferevent *); static void be_filter_destruct(struct bufferevent *); static void be_filter_readcb(struct bufferevent *, void *); static void be_filter_writecb(struct bufferevent *, void *); static void be_filter_eventcb(struct bufferevent *, short, void *); static int be_filter_flush(struct bufferevent *bufev, short iotype, enum bufferevent_flush_mode mode); static int be_filter_ctrl(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *); static void bufferevent_filtered_inbuf_cb(struct evbuffer *buf, const struct evbuffer_cb_info *cbinfo, void *arg); static void bufferevent_filtered_outbuf_cb(struct evbuffer *buf, const struct evbuffer_cb_info *info, void *arg); struct bufferevent_filtered { struct bufferevent_private bev; /** The bufferevent that we read/write filtered data from/to. */ struct bufferevent *underlying; /** A callback on our inbuf to notice somebory removes data */ struct evbuffer_cb_entry *inbuf_cb; /** A callback on our outbuf to notice when somebody adds data */ struct evbuffer_cb_entry *outbuf_cb; /** True iff we have received an EOF callback from the underlying * bufferevent. */ unsigned got_eof; /** Function to free context when we're done. */ void (*free_context)(void *); /** Input filter */ bufferevent_filter_cb process_in; /** Output filter */ bufferevent_filter_cb process_out; /** User-supplied argument to the filters. */ void *context; }; const struct bufferevent_ops bufferevent_ops_filter = { "filter", evutil_offsetof(struct bufferevent_filtered, bev.bev), be_filter_enable, be_filter_disable, be_filter_unlink, be_filter_destruct, bufferevent_generic_adj_timeouts_, be_filter_flush, be_filter_ctrl, }; /* Given a bufferevent that's really the bev filter of a bufferevent_filtered, * return that bufferevent_filtered. Returns NULL otherwise.*/ static inline struct bufferevent_filtered * upcast(struct bufferevent *bev) { struct bufferevent_filtered *bev_f; if (bev->be_ops != &bufferevent_ops_filter) return NULL; bev_f = (void*)( ((char*)bev) - evutil_offsetof(struct bufferevent_filtered, bev.bev)); EVUTIL_ASSERT(bev_f->bev.bev.be_ops == &bufferevent_ops_filter); return bev_f; } #define downcast(bev_f) (&(bev_f)->bev.bev) /** Return 1 iff bevf's underlying bufferevent's output buffer is at or * over its high watermark such that we should not write to it in a given * flush mode. */ static int be_underlying_writebuf_full(struct bufferevent_filtered *bevf, enum bufferevent_flush_mode state) { struct bufferevent *u = bevf->underlying; return state == BEV_NORMAL && u->wm_write.high && evbuffer_get_length(u->output) >= u->wm_write.high; } /** Return 1 if our input buffer is at or over its high watermark such that we * should not write to it in a given flush mode. */ static int be_readbuf_full(struct bufferevent_filtered *bevf, enum bufferevent_flush_mode state) { struct bufferevent *bufev = downcast(bevf); return state == BEV_NORMAL && bufev->wm_read.high && evbuffer_get_length(bufev->input) >= bufev->wm_read.high; } /* Filter to use when we're created with a NULL filter. */ static enum bufferevent_filter_result be_null_filter(struct evbuffer *src, struct evbuffer *dst, ev_ssize_t lim, enum bufferevent_flush_mode state, void *ctx) { (void)state; if (evbuffer_remove_buffer(src, dst, lim) == 0) return BEV_OK; else return BEV_ERROR; } struct bufferevent * bufferevent_filter_new(struct bufferevent *underlying, bufferevent_filter_cb input_filter, bufferevent_filter_cb output_filter, int options, void (*free_context)(void *), void *ctx) { struct bufferevent_filtered *bufev_f; int tmp_options = options & ~BEV_OPT_THREADSAFE; if (!underlying) return NULL; if (!input_filter) input_filter = be_null_filter; if (!output_filter) output_filter = be_null_filter; bufev_f = mm_calloc(1, sizeof(struct bufferevent_filtered)); if (!bufev_f) return NULL; if (bufferevent_init_common_(&bufev_f->bev, underlying->ev_base, &bufferevent_ops_filter, tmp_options) < 0) { mm_free(bufev_f); return NULL; } if (options & BEV_OPT_THREADSAFE) { bufferevent_enable_locking_(downcast(bufev_f), NULL); } bufev_f->underlying = underlying; bufev_f->process_in = input_filter; bufev_f->process_out = output_filter; bufev_f->free_context = free_context; bufev_f->context = ctx; bufferevent_setcb(bufev_f->underlying, be_filter_readcb, be_filter_writecb, be_filter_eventcb, bufev_f); bufev_f->inbuf_cb = evbuffer_add_cb(downcast(bufev_f)->input, bufferevent_filtered_inbuf_cb, bufev_f); evbuffer_cb_clear_flags(downcast(bufev_f)->input, bufev_f->inbuf_cb, EVBUFFER_CB_ENABLED); bufev_f->outbuf_cb = evbuffer_add_cb(downcast(bufev_f)->output, bufferevent_filtered_outbuf_cb, bufev_f); bufferevent_init_generic_timeout_cbs_(downcast(bufev_f)); bufferevent_incref_(underlying); bufferevent_enable(underlying, EV_READ|EV_WRITE); bufferevent_suspend_read_(underlying, BEV_SUSPEND_FILT_READ); return downcast(bufev_f); } static void be_filter_unlink(struct bufferevent *bev) { struct bufferevent_filtered *bevf = upcast(bev); EVUTIL_ASSERT(bevf); if (bevf->bev.options & BEV_OPT_CLOSE_ON_FREE) { /* Yes, there is also a decref in bufferevent_decref_. * That decref corresponds to the incref when we set * underlying for the first time. This decref is an * extra one to remove the last reference. */ if (BEV_UPCAST(bevf->underlying)->refcnt < 2) { event_warnx("BEV_OPT_CLOSE_ON_FREE set on an " "bufferevent with too few references"); } else { bufferevent_free(bevf->underlying); } } else { if (bevf->underlying) { if (bevf->underlying->errorcb == be_filter_eventcb) bufferevent_setcb(bevf->underlying, NULL, NULL, NULL, NULL); bufferevent_unsuspend_read_(bevf->underlying, BEV_SUSPEND_FILT_READ); } } } static void be_filter_destruct(struct bufferevent *bev) { struct bufferevent_filtered *bevf = upcast(bev); EVUTIL_ASSERT(bevf); if (bevf->free_context) bevf->free_context(bevf->context); if (bevf->inbuf_cb) evbuffer_remove_cb_entry(bev->input, bevf->inbuf_cb); if (bevf->outbuf_cb) evbuffer_remove_cb_entry(bev->output, bevf->outbuf_cb); } static int be_filter_enable(struct bufferevent *bev, short event) { struct bufferevent_filtered *bevf = upcast(bev); if (event & EV_WRITE) BEV_RESET_GENERIC_WRITE_TIMEOUT(bev); if (event & EV_READ) { BEV_RESET_GENERIC_READ_TIMEOUT(bev); bufferevent_unsuspend_read_(bevf->underlying, BEV_SUSPEND_FILT_READ); } return 0; } static int be_filter_disable(struct bufferevent *bev, short event) { struct bufferevent_filtered *bevf = upcast(bev); if (event & EV_WRITE) BEV_DEL_GENERIC_WRITE_TIMEOUT(bev); if (event & EV_READ) { BEV_DEL_GENERIC_READ_TIMEOUT(bev); bufferevent_suspend_read_(bevf->underlying, BEV_SUSPEND_FILT_READ); } return 0; } static enum bufferevent_filter_result be_filter_process_input(struct bufferevent_filtered *bevf, enum bufferevent_flush_mode state, int *processed_out) { enum bufferevent_filter_result res; struct bufferevent *bev = downcast(bevf); if (state == BEV_NORMAL) { /* If we're in 'normal' mode, don't urge data on the filter * unless we're reading data and under our high-water mark.*/ if (!(bev->enabled & EV_READ) || be_readbuf_full(bevf, state)) return BEV_OK; } do { ev_ssize_t limit = -1; if (state == BEV_NORMAL && bev->wm_read.high) limit = bev->wm_read.high - evbuffer_get_length(bev->input); res = bevf->process_in(bevf->underlying->input, bev->input, limit, state, bevf->context); if (res == BEV_OK) *processed_out = 1; } while (res == BEV_OK && (bev->enabled & EV_READ) && evbuffer_get_length(bevf->underlying->input) && !be_readbuf_full(bevf, state)); if (*processed_out) BEV_RESET_GENERIC_READ_TIMEOUT(bev); return res; } static enum bufferevent_filter_result be_filter_process_output(struct bufferevent_filtered *bevf, enum bufferevent_flush_mode state, int *processed_out) { /* Requires references and lock: might call writecb */ enum bufferevent_filter_result res = BEV_OK; struct bufferevent *bufev = downcast(bevf); int again = 0; if (state == BEV_NORMAL) { /* If we're in 'normal' mode, don't urge data on the * filter unless we're writing data, and the underlying * bufferevent is accepting data, and we have data to * give the filter. If we're in 'flush' or 'finish', * call the filter no matter what. */ if (!(bufev->enabled & EV_WRITE) || be_underlying_writebuf_full(bevf, state) || !evbuffer_get_length(bufev->output)) return BEV_OK; } /* disable the callback that calls this function when the user adds to the output buffer. */ evbuffer_cb_clear_flags(bufev->output, bevf->outbuf_cb, EVBUFFER_CB_ENABLED); do { int processed = 0; again = 0; do { ev_ssize_t limit = -1; if (state == BEV_NORMAL && bevf->underlying->wm_write.high) limit = bevf->underlying->wm_write.high - evbuffer_get_length(bevf->underlying->output); res = bevf->process_out(downcast(bevf)->output, bevf->underlying->output, limit, state, bevf->context); if (res == BEV_OK) processed = *processed_out = 1; } while (/* Stop if the filter wasn't successful...*/ res == BEV_OK && /* Or if we aren't writing any more. */ (bufev->enabled & EV_WRITE) && /* Of if we have nothing more to write and we are * not flushing. */ evbuffer_get_length(bufev->output) && /* Or if we have filled the underlying output buffer. */ !be_underlying_writebuf_full(bevf,state)); if (processed) { /* call the write callback.*/ bufferevent_trigger_nolock_(bufev, EV_WRITE, 0); if (res == BEV_OK && (bufev->enabled & EV_WRITE) && evbuffer_get_length(bufev->output) && !be_underlying_writebuf_full(bevf, state)) { again = 1; } } } while (again); /* reenable the outbuf_cb */ evbuffer_cb_set_flags(bufev->output,bevf->outbuf_cb, EVBUFFER_CB_ENABLED); if (*processed_out) BEV_RESET_GENERIC_WRITE_TIMEOUT(bufev); return res; } /* Called when the size of our outbuf changes. */ static void bufferevent_filtered_outbuf_cb(struct evbuffer *buf, const struct evbuffer_cb_info *cbinfo, void *arg) { struct bufferevent_filtered *bevf = arg; struct bufferevent *bev = downcast(bevf); if (cbinfo->n_added) { int processed_any = 0; /* Somebody added more data to the output buffer. Try to * process it, if we should. */ bufferevent_incref_and_lock_(bev); be_filter_process_output(bevf, BEV_NORMAL, &processed_any); bufferevent_decref_and_unlock_(bev); } } static void be_filter_read_nolock_(struct bufferevent *underlying, void *me_) { struct bufferevent_filtered *bevf = me_; enum bufferevent_filter_result res; enum bufferevent_flush_mode state; struct bufferevent *bufev = downcast(bevf); struct bufferevent_private *bufev_private = BEV_UPCAST(bufev); int processed_any = 0; // It's possible our refcount is 0 at this point if another thread free'd our filterevent EVUTIL_ASSERT(bufev_private->refcnt >= 0); // If our refcount is > 0 if (bufev_private->refcnt > 0) { if (bevf->got_eof) state = BEV_FINISHED; else state = BEV_NORMAL; /* XXXX use return value */ res = be_filter_process_input(bevf, state, &processed_any); (void)res; /* XXX This should be in process_input, not here. There are * other places that can call process-input, and they should * force readcb calls as needed. */ if (processed_any) { bufferevent_trigger_nolock_(bufev, EV_READ, 0); if (evbuffer_get_length(underlying->input) > 0 && be_readbuf_full(bevf, state)) { /* data left in underlying buffer and filter input buffer * hit its read high watermark. * Schedule callback to avoid data gets stuck in underlying * input buffer. */ evbuffer_cb_set_flags(bufev->input, bevf->inbuf_cb, EVBUFFER_CB_ENABLED); } } } } /* Called when the size of our inbuf changes. */ static void bufferevent_filtered_inbuf_cb(struct evbuffer *buf, const struct evbuffer_cb_info *cbinfo, void *arg) { struct bufferevent_filtered *bevf = arg; enum bufferevent_flush_mode state; struct bufferevent *bev = downcast(bevf); BEV_LOCK(bev); if (bevf->got_eof) state = BEV_FINISHED; else state = BEV_NORMAL; if (!be_readbuf_full(bevf, state)) { /* opportunity to read data which was left in underlying * input buffer because filter input buffer hit read * high watermark. */ evbuffer_cb_clear_flags(bev->input, bevf->inbuf_cb, EVBUFFER_CB_ENABLED); if (evbuffer_get_length(bevf->underlying->input) > 0) be_filter_read_nolock_(bevf->underlying, bevf); } BEV_UNLOCK(bev); } /* Called when the underlying socket has read. */ static void be_filter_readcb(struct bufferevent *underlying, void *me_) { struct bufferevent_filtered *bevf = me_; struct bufferevent *bev = downcast(bevf); BEV_LOCK(bev); be_filter_read_nolock_(underlying, me_); BEV_UNLOCK(bev); } /* Called when the underlying socket has drained enough that we can write to it. */ static void be_filter_writecb(struct bufferevent *underlying, void *me_) { struct bufferevent_filtered *bevf = me_; struct bufferevent *bev = downcast(bevf); struct bufferevent_private *bufev_private = BEV_UPCAST(bev); int processed_any = 0; BEV_LOCK(bev); // It's possible our refcount is 0 at this point if another thread free'd our filterevent EVUTIL_ASSERT(bufev_private->refcnt >= 0); // If our refcount is > 0 if (bufev_private->refcnt > 0) { be_filter_process_output(bevf, BEV_NORMAL, &processed_any); } BEV_UNLOCK(bev); } /* Called when the underlying socket has given us an error */ static void be_filter_eventcb(struct bufferevent *underlying, short what, void *me_) { struct bufferevent_filtered *bevf = me_; struct bufferevent *bev = downcast(bevf); struct bufferevent_private *bufev_private = BEV_UPCAST(bev); BEV_LOCK(bev); // It's possible our refcount is 0 at this point if another thread free'd our filterevent EVUTIL_ASSERT(bufev_private->refcnt >= 0); // If our refcount is > 0 if (bufev_private->refcnt > 0) { /* All we can really to is tell our own eventcb. */ bufferevent_run_eventcb_(bev, what, 0); } BEV_UNLOCK(bev); } static int be_filter_flush(struct bufferevent *bufev, short iotype, enum bufferevent_flush_mode mode) { struct bufferevent_filtered *bevf = upcast(bufev); int processed_any = 0; EVUTIL_ASSERT(bevf); bufferevent_incref_and_lock_(bufev); if (iotype & EV_READ) { be_filter_process_input(bevf, mode, &processed_any); } if (iotype & EV_WRITE) { be_filter_process_output(bevf, mode, &processed_any); } /* XXX check the return value? */ /* XXX does this want to recursively call lower-level flushes? */ bufferevent_flush(bevf->underlying, iotype, mode); bufferevent_decref_and_unlock_(bufev); return processed_any; } static int be_filter_ctrl(struct bufferevent *bev, enum bufferevent_ctrl_op op, union bufferevent_ctrl_data *data) { struct bufferevent_filtered *bevf; switch (op) { case BEV_CTRL_GET_UNDERLYING: bevf = upcast(bev); data->ptr = bevf->underlying; return 0; case BEV_CTRL_SET_FD: bevf = upcast(bev); if (bevf->underlying && bevf->underlying->be_ops && bevf->underlying->be_ops->ctrl) { return (bevf->underlying->be_ops->ctrl)(bevf->underlying, op, data); } case BEV_CTRL_GET_FD: case BEV_CTRL_CANCEL_ALL: default: return -1; } return -1; } �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/bufferevent_openssl.c����������������������������������������0000664�0000000�0000000�00000115114�15270523712�0024223�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ // Get rid of OSX 10.7 and greater deprecation warnings. #if defined(__APPLE__) && defined(__clang__) #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif #include "event2/event-config.h" #include "evconfig-private.h" #include <sys/types.h> #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #ifdef EVENT__HAVE_STDARG_H #include <stdarg.h> #endif #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #ifdef _WIN32 #include <winsock2.h> #endif #include "event2/bufferevent.h" #include "event2/bufferevent_struct.h" #include "event2/bufferevent_ssl.h" #include "event2/buffer.h" #include "event2/event.h" #include "mm-internal.h" #include "bufferevent-internal.h" #include "log-internal.h" #include <openssl/bio.h> #include <openssl/ssl.h> #include <openssl/err.h> #include "openssl-compat.h" /* * Define an OpenSSL bio that targets a bufferevent. */ /* -------------------- A BIO is an OpenSSL abstraction that handles reading and writing data. The library will happily speak SSL over anything that implements a BIO interface. Here we define a BIO implementation that directs its output to a bufferevent. We'll want to use this only when none of OpenSSL's built-in IO mechanisms work for us. -------------------- */ /* every BIO type needs its own integer type value. */ #define BIO_TYPE_LIBEVENT 57 /* ???? Arguably, we should set BIO_TYPE_FILTER or BIO_TYPE_SOURCE_SINK on * this. */ #if 0 static void print_err(int val) { int err; printf("Error was %d\n", val); while ((err = ERR_get_error())) { const char *msg = (const char*)ERR_reason_error_string(err); const char *lib = (const char*)ERR_lib_error_string(err); const char *func = (const char*)ERR_func_error_string(err); printf("%s in %s %s\n", msg, lib, func); } } #else #define print_err(v) ((void)0) #endif /* Called to initialize a new BIO */ static int bio_bufferevent_new(BIO *b) { BIO_set_init(b, 0); BIO_set_data(b, NULL); /* We'll be putting the bufferevent in this field.*/ return 1; } /* Called to uninitialize the BIO. */ static int bio_bufferevent_free(BIO *b) { if (!b) return 0; if (BIO_get_shutdown(b)) { if (BIO_get_init(b) && BIO_get_data(b)) bufferevent_free(BIO_get_data(b)); BIO_free(b); } return 1; } /* Called to extract data from the BIO. */ static int bio_bufferevent_read(BIO *b, char *out, int outlen) { int r = 0; struct evbuffer *input; BIO_clear_retry_flags(b); if (!out) return 0; if (!BIO_get_data(b)) return -1; input = bufferevent_get_input(BIO_get_data(b)); if (evbuffer_get_length(input) == 0) { /* If there's no data to read, say so. */ BIO_set_retry_read(b); return -1; } else { r = evbuffer_remove(input, out, outlen); } return r; } /* Called to write data info the BIO */ static int bio_bufferevent_write(BIO *b, const char *in, int inlen) { struct bufferevent *bufev = BIO_get_data(b); struct evbuffer *output; size_t outlen; BIO_clear_retry_flags(b); if (!BIO_get_data(b)) return -1; output = bufferevent_get_output(bufev); outlen = evbuffer_get_length(output); /* Copy only as much data onto the output buffer as can fit under the * high-water mark. */ if (bufev->wm_write.high && bufev->wm_write.high <= (outlen+inlen)) { if (bufev->wm_write.high <= outlen) { /* If no data can fit, we'll need to retry later. */ BIO_set_retry_write(b); return -1; } inlen = bufev->wm_write.high - outlen; } EVUTIL_ASSERT(inlen > 0); evbuffer_add(output, in, inlen); return inlen; } /* Called to handle various requests */ static long bio_bufferevent_ctrl(BIO *b, int cmd, long num, void *ptr) { struct bufferevent *bufev = BIO_get_data(b); long ret = 1; switch (cmd) { case BIO_CTRL_GET_CLOSE: ret = BIO_get_shutdown(b); break; case BIO_CTRL_SET_CLOSE: BIO_set_shutdown(b, (int)num); break; case BIO_CTRL_PENDING: ret = evbuffer_get_length(bufferevent_get_input(bufev)) != 0; break; case BIO_CTRL_WPENDING: ret = evbuffer_get_length(bufferevent_get_output(bufev)) != 0; break; /* XXXX These two are given a special-case treatment because * of cargo-cultism. I should come up with a better reason. */ case BIO_CTRL_DUP: case BIO_CTRL_FLUSH: ret = 1; break; default: ret = 0; break; } return ret; } /* Called to write a string to the BIO */ static int bio_bufferevent_puts(BIO *b, const char *s) { return bio_bufferevent_write(b, s, strlen(s)); } /* Method table for the bufferevent BIO */ static BIO_METHOD *methods_bufferevent; /* Return the method table for the bufferevents BIO */ static BIO_METHOD * BIO_s_bufferevent(void) { if (methods_bufferevent == NULL) { methods_bufferevent = BIO_meth_new(BIO_TYPE_LIBEVENT, "bufferevent"); if (methods_bufferevent == NULL) return NULL; BIO_meth_set_write(methods_bufferevent, bio_bufferevent_write); BIO_meth_set_read(methods_bufferevent, bio_bufferevent_read); BIO_meth_set_puts(methods_bufferevent, bio_bufferevent_puts); BIO_meth_set_ctrl(methods_bufferevent, bio_bufferevent_ctrl); BIO_meth_set_create(methods_bufferevent, bio_bufferevent_new); BIO_meth_set_destroy(methods_bufferevent, bio_bufferevent_free); } return methods_bufferevent; } /* Create a new BIO to wrap communication around a bufferevent. If close_flag * is true, the bufferevent will be freed when the BIO is closed. */ static BIO * BIO_new_bufferevent(struct bufferevent *bufferevent, int close_flag) { BIO *result; if (!bufferevent) return NULL; if (!(result = BIO_new(BIO_s_bufferevent()))) return NULL; BIO_set_init(result, 1); BIO_set_data(result, bufferevent); BIO_set_shutdown(result, close_flag ? 1 : 0); return result; } /* -------------------- Now, here's the OpenSSL-based implementation of bufferevent. The implementation comes in two flavors: one that connects its SSL object to an underlying bufferevent using a BIO_bufferevent, and one that has the SSL object connect to a socket directly. The latter should generally be faster, except on Windows, where your best bet is using a bufferevent_async. (OpenSSL supports many other BIO types, too. But we can't use any unless we have a good way to get notified when they become readable/writable.) -------------------- */ struct bio_data_counts { unsigned long n_written; unsigned long n_read; }; struct bufferevent_openssl { /* Shared fields with common bufferevent implementation code. If we were set up with an underlying bufferevent, we use the events here as timers only. If we have an SSL, then we use the events as socket events. */ struct bufferevent_private bev; /* An underlying bufferevent that we're directing our output to. If it's NULL, then we're connected to an fd, not an evbuffer. */ struct bufferevent *underlying; /* The SSL object doing our encryption. */ SSL *ssl; /* A callback that's invoked when data arrives on our outbuf so we know to write data to the SSL. */ struct evbuffer_cb_entry *outbuf_cb; /* A count of how much data the bios have read/written total. Used for rate-limiting. */ struct bio_data_counts counts; /* If this value is greater than 0, then the last SSL_write blocked, * and we need to try it again with this many bytes. */ ev_ssize_t last_write; #define NUM_ERRORS 3 ev_uint32_t errors[NUM_ERRORS]; /* When we next get available space, we should say "read" instead of "write". This can happen if there's a renegotiation during a read operation. */ unsigned read_blocked_on_write : 1; /* When we next get data, we should say "write" instead of "read". */ unsigned write_blocked_on_read : 1; /* Treat TCP close before SSL close on SSL >= v3 as clean EOF. */ unsigned allow_dirty_shutdown : 1; /* XXX */ unsigned n_errors : 2; /* Are we currently connecting, accepting, or doing IO? */ unsigned state : 2; /* If we reset fd, we sould reset state too */ unsigned old_state : 2; }; static int be_openssl_enable(struct bufferevent *, short); static int be_openssl_disable(struct bufferevent *, short); static void be_openssl_unlink(struct bufferevent *); static void be_openssl_destruct(struct bufferevent *); static int be_openssl_adj_timeouts(struct bufferevent *); static int be_openssl_flush(struct bufferevent *bufev, short iotype, enum bufferevent_flush_mode mode); static int be_openssl_ctrl(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *); const struct bufferevent_ops bufferevent_ops_openssl = { "ssl", evutil_offsetof(struct bufferevent_openssl, bev.bev), be_openssl_enable, be_openssl_disable, be_openssl_unlink, be_openssl_destruct, be_openssl_adj_timeouts, be_openssl_flush, be_openssl_ctrl, }; /* Given a bufferevent, return a pointer to the bufferevent_openssl that * contains it, if any. */ static inline struct bufferevent_openssl * upcast(struct bufferevent *bev) { struct bufferevent_openssl *bev_o; if (bev->be_ops != &bufferevent_ops_openssl) return NULL; bev_o = (void*)( ((char*)bev) - evutil_offsetof(struct bufferevent_openssl, bev.bev)); EVUTIL_ASSERT(bev_o->bev.bev.be_ops == &bufferevent_ops_openssl); return bev_o; } static inline void put_error(struct bufferevent_openssl *bev_ssl, unsigned long err) { if (bev_ssl->n_errors == NUM_ERRORS) return; /* The error type according to openssl is "unsigned long", but openssl never uses more than 32 bits of it. It _can't_ use more than 32 bits of it, since it needs to report errors on systems where long is only 32 bits. */ bev_ssl->errors[bev_ssl->n_errors++] = (ev_uint32_t) err; } /* Have the base communications channel (either the underlying bufferevent or * ev_read and ev_write) start reading. Take the read-blocked-on-write flag * into account. */ static int start_reading(struct bufferevent_openssl *bev_ssl) { if (bev_ssl->underlying) { bufferevent_unsuspend_read_(bev_ssl->underlying, BEV_SUSPEND_FILT_READ); return 0; } else { struct bufferevent *bev = &bev_ssl->bev.bev; int r; r = bufferevent_add_event_(&bev->ev_read, &bev->timeout_read); if (r == 0 && bev_ssl->read_blocked_on_write) r = bufferevent_add_event_(&bev->ev_write, &bev->timeout_write); return r; } } /* Have the base communications channel (either the underlying bufferevent or * ev_read and ev_write) start writing. Take the write-blocked-on-read flag * into account. */ static int start_writing(struct bufferevent_openssl *bev_ssl) { int r = 0; if (bev_ssl->underlying) { if (bev_ssl->write_blocked_on_read) { bufferevent_unsuspend_read_(bev_ssl->underlying, BEV_SUSPEND_FILT_READ); } } else { struct bufferevent *bev = &bev_ssl->bev.bev; r = bufferevent_add_event_(&bev->ev_write, &bev->timeout_write); if (!r && bev_ssl->write_blocked_on_read) r = bufferevent_add_event_(&bev->ev_read, &bev->timeout_read); } return r; } static void stop_reading(struct bufferevent_openssl *bev_ssl) { if (bev_ssl->write_blocked_on_read) return; if (bev_ssl->underlying) { bufferevent_suspend_read_(bev_ssl->underlying, BEV_SUSPEND_FILT_READ); } else { struct bufferevent *bev = &bev_ssl->bev.bev; event_del(&bev->ev_read); } } static void stop_writing(struct bufferevent_openssl *bev_ssl) { if (bev_ssl->read_blocked_on_write) return; if (bev_ssl->underlying) { bufferevent_unsuspend_read_(bev_ssl->underlying, BEV_SUSPEND_FILT_READ); } else { struct bufferevent *bev = &bev_ssl->bev.bev; event_del(&bev->ev_write); } } static int set_rbow(struct bufferevent_openssl *bev_ssl) { if (!bev_ssl->underlying) stop_reading(bev_ssl); bev_ssl->read_blocked_on_write = 1; return start_writing(bev_ssl); } static int set_wbor(struct bufferevent_openssl *bev_ssl) { if (!bev_ssl->underlying) stop_writing(bev_ssl); bev_ssl->write_blocked_on_read = 1; return start_reading(bev_ssl); } static int clear_rbow(struct bufferevent_openssl *bev_ssl) { struct bufferevent *bev = &bev_ssl->bev.bev; int r = 0; bev_ssl->read_blocked_on_write = 0; if (!(bev->enabled & EV_WRITE)) stop_writing(bev_ssl); if (bev->enabled & EV_READ) r = start_reading(bev_ssl); return r; } static int clear_wbor(struct bufferevent_openssl *bev_ssl) { struct bufferevent *bev = &bev_ssl->bev.bev; int r = 0; bev_ssl->write_blocked_on_read = 0; if (!(bev->enabled & EV_READ)) stop_reading(bev_ssl); if (bev->enabled & EV_WRITE) r = start_writing(bev_ssl); return r; } static void conn_closed(struct bufferevent_openssl *bev_ssl, int when, int errcode, int ret) { int event = BEV_EVENT_ERROR; int dirty_shutdown = 0; unsigned long err; switch (errcode) { case SSL_ERROR_ZERO_RETURN: /* Possibly a clean shutdown. */ if (SSL_get_shutdown(bev_ssl->ssl) & SSL_RECEIVED_SHUTDOWN) event = BEV_EVENT_EOF; else dirty_shutdown = 1; break; case SSL_ERROR_SYSCALL: /* IO error; possibly a dirty shutdown. */ if ((ret == 0 || ret == -1) && ERR_peek_error() == 0) dirty_shutdown = 1; break; case SSL_ERROR_SSL: /* Protocol error. */ break; case SSL_ERROR_WANT_X509_LOOKUP: /* XXXX handle this. */ break; case SSL_ERROR_NONE: case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: case SSL_ERROR_WANT_CONNECT: case SSL_ERROR_WANT_ACCEPT: default: /* should be impossible; treat as normal error. */ event_warnx("BUG: Unexpected OpenSSL error code %d", errcode); break; } while ((err = ERR_get_error())) { put_error(bev_ssl, err); } if (dirty_shutdown && bev_ssl->allow_dirty_shutdown) event = BEV_EVENT_EOF; stop_reading(bev_ssl); stop_writing(bev_ssl); /* when is BEV_EVENT_{READING|WRITING} */ event = when | event; bufferevent_run_eventcb_(&bev_ssl->bev.bev, event, 0); } static void init_bio_counts(struct bufferevent_openssl *bev_ssl) { BIO *rbio, *wbio; wbio = SSL_get_wbio(bev_ssl->ssl); bev_ssl->counts.n_written = wbio ? BIO_number_written(wbio) : 0; rbio = SSL_get_rbio(bev_ssl->ssl); bev_ssl->counts.n_read = rbio ? BIO_number_read(rbio) : 0; } static inline void decrement_buckets(struct bufferevent_openssl *bev_ssl) { unsigned long num_w = BIO_number_written(SSL_get_wbio(bev_ssl->ssl)); unsigned long num_r = BIO_number_read(SSL_get_rbio(bev_ssl->ssl)); /* These next two subtractions can wrap around. That's okay. */ unsigned long w = num_w - bev_ssl->counts.n_written; unsigned long r = num_r - bev_ssl->counts.n_read; if (w) bufferevent_decrement_write_buckets_(&bev_ssl->bev, w); if (r) bufferevent_decrement_read_buckets_(&bev_ssl->bev, r); bev_ssl->counts.n_written = num_w; bev_ssl->counts.n_read = num_r; } #define OP_MADE_PROGRESS 1 #define OP_BLOCKED 2 #define OP_ERR 4 /* Return a bitmask of OP_MADE_PROGRESS (if we read anything); OP_BLOCKED (if we're now blocked); and OP_ERR (if an error occurred). */ static int do_read(struct bufferevent_openssl *bev_ssl, int n_to_read) { /* Requires lock */ struct bufferevent *bev = &bev_ssl->bev.bev; struct evbuffer *input = bev->input; int r, n, i, n_used = 0, atmost; struct evbuffer_iovec space[2]; int result = 0; if (bev_ssl->bev.read_suspended) return 0; atmost = bufferevent_get_read_max_(&bev_ssl->bev); if (n_to_read > atmost) n_to_read = atmost; n = evbuffer_reserve_space(input, n_to_read, space, 2); if (n < 0) return OP_ERR; for (i=0; i<n; ++i) { if (bev_ssl->bev.read_suspended) break; ERR_clear_error(); r = SSL_read(bev_ssl->ssl, space[i].iov_base, space[i].iov_len); if (r>0) { result |= OP_MADE_PROGRESS; if (bev_ssl->read_blocked_on_write) if (clear_rbow(bev_ssl) < 0) return OP_ERR | result; ++n_used; space[i].iov_len = r; decrement_buckets(bev_ssl); } else { int err = SSL_get_error(bev_ssl->ssl, r); print_err(err); switch (err) { case SSL_ERROR_WANT_READ: /* Can't read until underlying has more data. */ if (bev_ssl->read_blocked_on_write) if (clear_rbow(bev_ssl) < 0) return OP_ERR | result; break; case SSL_ERROR_WANT_WRITE: /* This read operation requires a write, and the * underlying is full */ if (!bev_ssl->read_blocked_on_write) if (set_rbow(bev_ssl) < 0) return OP_ERR | result; break; default: conn_closed(bev_ssl, BEV_EVENT_READING, err, r); break; } result |= OP_BLOCKED; break; /* out of the loop */ } } if (n_used) { evbuffer_commit_space(input, space, n_used); if (bev_ssl->underlying) BEV_RESET_GENERIC_READ_TIMEOUT(bev); } return result; } /* Return a bitmask of OP_MADE_PROGRESS (if we wrote anything); OP_BLOCKED (if we're now blocked); and OP_ERR (if an error occurred). */ static int do_write(struct bufferevent_openssl *bev_ssl, int atmost) { int i, r, n, n_written = 0; struct bufferevent *bev = &bev_ssl->bev.bev; struct evbuffer *output = bev->output; struct evbuffer_iovec space[8]; int result = 0; if (bev_ssl->last_write > 0) atmost = bev_ssl->last_write; else atmost = bufferevent_get_write_max_(&bev_ssl->bev); n = evbuffer_peek(output, atmost, NULL, space, 8); if (n < 0) return OP_ERR | result; if (n > 8) n = 8; for (i=0; i < n; ++i) { if (bev_ssl->bev.write_suspended) break; /* SSL_write will (reasonably) return 0 if we tell it to send 0 data. Skip this case so we don't interpret the result as an error */ if (space[i].iov_len == 0) continue; ERR_clear_error(); r = SSL_write(bev_ssl->ssl, space[i].iov_base, space[i].iov_len); if (r > 0) { result |= OP_MADE_PROGRESS; if (bev_ssl->write_blocked_on_read) if (clear_wbor(bev_ssl) < 0) return OP_ERR | result; n_written += r; bev_ssl->last_write = -1; decrement_buckets(bev_ssl); } else { int err = SSL_get_error(bev_ssl->ssl, r); print_err(err); switch (err) { case SSL_ERROR_WANT_WRITE: /* Can't read until underlying has more data. */ if (bev_ssl->write_blocked_on_read) if (clear_wbor(bev_ssl) < 0) return OP_ERR | result; bev_ssl->last_write = space[i].iov_len; break; case SSL_ERROR_WANT_READ: /* This read operation requires a write, and the * underlying is full */ if (!bev_ssl->write_blocked_on_read) if (set_wbor(bev_ssl) < 0) return OP_ERR | result; bev_ssl->last_write = space[i].iov_len; break; default: conn_closed(bev_ssl, BEV_EVENT_WRITING, err, r); bev_ssl->last_write = -1; break; } result |= OP_BLOCKED; break; } } if (n_written) { evbuffer_drain(output, n_written); if (bev_ssl->underlying) BEV_RESET_GENERIC_WRITE_TIMEOUT(bev); bufferevent_trigger_nolock_(bev, EV_WRITE, BEV_OPT_DEFER_CALLBACKS); } return result; } #define WRITE_FRAME 15000 #define READ_DEFAULT 4096 /* Try to figure out how many bytes to read; return 0 if we shouldn't be * reading. */ static int bytes_to_read(struct bufferevent_openssl *bev) { struct evbuffer *input = bev->bev.bev.input; struct event_watermark *wm = &bev->bev.bev.wm_read; int result = READ_DEFAULT; ev_ssize_t limit; /* XXX 99% of this is generic code that nearly all bufferevents will * want. */ if (bev->write_blocked_on_read) { return 0; } if (! (bev->bev.bev.enabled & EV_READ)) { return 0; } if (bev->bev.read_suspended) { return 0; } if (wm->high) { if (evbuffer_get_length(input) >= wm->high) { return 0; } result = wm->high - evbuffer_get_length(input); } else { result = READ_DEFAULT; } /* Respect the rate limit */ limit = bufferevent_get_read_max_(&bev->bev); if (result > limit) { result = limit; } return result; } /* Things look readable. If write is blocked on read, write till it isn't. * Read from the underlying buffer until we block or we hit our high-water * mark. */ static void consider_reading(struct bufferevent_openssl *bev_ssl) { int r; int n_to_read; int all_result_flags = 0; while (bev_ssl->write_blocked_on_read) { r = do_write(bev_ssl, WRITE_FRAME); if (r & (OP_BLOCKED|OP_ERR)) break; } if (bev_ssl->write_blocked_on_read) return; n_to_read = bytes_to_read(bev_ssl); while (n_to_read) { r = do_read(bev_ssl, n_to_read); all_result_flags |= r; if (r & (OP_BLOCKED|OP_ERR)) break; if (bev_ssl->bev.read_suspended) break; /* Read all pending data. This won't hit the network * again, and will (most importantly) put us in a state * where we don't need to read anything else until the * socket is readable again. It'll potentially make us * overrun our read high-watermark (somewhat * regrettable). The damage to the rate-limit has * already been done, since OpenSSL went and read a * whole SSL record anyway. */ n_to_read = SSL_pending(bev_ssl->ssl); /* XXX This if statement is actually a bad bug, added to avoid * XXX a worse bug. * * The bad bug: It can potentially cause resource unfairness * by reading too much data from the underlying bufferevent; * it can potentially cause read looping if the underlying * bufferevent is a bufferevent_pair and deferred callbacks * aren't used. * * The worse bug: If we didn't do this, then we would * potentially not read any more from bev_ssl->underlying * until more data arrived there, which could lead to us * waiting forever. */ if (!n_to_read && bev_ssl->underlying) n_to_read = bytes_to_read(bev_ssl); } if (all_result_flags & OP_MADE_PROGRESS) { struct bufferevent *bev = &bev_ssl->bev.bev; bufferevent_trigger_nolock_(bev, EV_READ, 0); } if (!bev_ssl->underlying) { /* Should be redundant, but let's avoid busy-looping */ if (bev_ssl->bev.read_suspended || !(bev_ssl->bev.bev.enabled & EV_READ)) { event_del(&bev_ssl->bev.bev.ev_read); } } } static void consider_writing(struct bufferevent_openssl *bev_ssl) { int r; struct evbuffer *output = bev_ssl->bev.bev.output; struct evbuffer *target = NULL; struct event_watermark *wm = NULL; while (bev_ssl->read_blocked_on_write) { r = do_read(bev_ssl, 1024); /* XXXX 1024 is a hack */ if (r & OP_MADE_PROGRESS) { struct bufferevent *bev = &bev_ssl->bev.bev; bufferevent_trigger_nolock_(bev, EV_READ, 0); } if (r & (OP_ERR|OP_BLOCKED)) break; } if (bev_ssl->read_blocked_on_write) return; if (bev_ssl->underlying) { target = bev_ssl->underlying->output; wm = &bev_ssl->underlying->wm_write; } while ((bev_ssl->bev.bev.enabled & EV_WRITE) && (! bev_ssl->bev.write_suspended) && evbuffer_get_length(output) && (!target || (! wm->high || evbuffer_get_length(target) < wm->high))) { int n_to_write; if (wm && wm->high) n_to_write = wm->high - evbuffer_get_length(target); else n_to_write = WRITE_FRAME; r = do_write(bev_ssl, n_to_write); if (r & (OP_BLOCKED|OP_ERR)) break; } if (!bev_ssl->underlying) { if (evbuffer_get_length(output) == 0) { event_del(&bev_ssl->bev.bev.ev_write); } else if (bev_ssl->bev.write_suspended || !(bev_ssl->bev.bev.enabled & EV_WRITE)) { /* Should be redundant, but let's avoid busy-looping */ event_del(&bev_ssl->bev.bev.ev_write); } } } static void be_openssl_readcb(struct bufferevent *bev_base, void *ctx) { struct bufferevent_openssl *bev_ssl = ctx; consider_reading(bev_ssl); } static void be_openssl_writecb(struct bufferevent *bev_base, void *ctx) { struct bufferevent_openssl *bev_ssl = ctx; consider_writing(bev_ssl); } static void be_openssl_eventcb(struct bufferevent *bev_base, short what, void *ctx) { struct bufferevent_openssl *bev_ssl = ctx; int event = 0; if (what & BEV_EVENT_EOF) { if (bev_ssl->allow_dirty_shutdown) event = BEV_EVENT_EOF; else event = BEV_EVENT_ERROR; } else if (what & BEV_EVENT_TIMEOUT) { /* We sure didn't set this. Propagate it to the user. */ event = what; } else if (what & BEV_EVENT_ERROR) { /* An error occurred on the connection. Propagate it to the user. */ event = what; } else if (what & BEV_EVENT_CONNECTED) { /* Ignore it. We're saying SSL_connect() already, which will eat it. */ } if (event) bufferevent_run_eventcb_(&bev_ssl->bev.bev, event, 0); } static void be_openssl_readeventcb(evutil_socket_t fd, short what, void *ptr) { struct bufferevent_openssl *bev_ssl = ptr; bufferevent_incref_and_lock_(&bev_ssl->bev.bev); if (what == EV_TIMEOUT) { bufferevent_run_eventcb_(&bev_ssl->bev.bev, BEV_EVENT_TIMEOUT|BEV_EVENT_READING, 0); } else { consider_reading(bev_ssl); } bufferevent_decref_and_unlock_(&bev_ssl->bev.bev); } static void be_openssl_writeeventcb(evutil_socket_t fd, short what, void *ptr) { struct bufferevent_openssl *bev_ssl = ptr; bufferevent_incref_and_lock_(&bev_ssl->bev.bev); if (what == EV_TIMEOUT) { bufferevent_run_eventcb_(&bev_ssl->bev.bev, BEV_EVENT_TIMEOUT|BEV_EVENT_WRITING, 0); } else { consider_writing(bev_ssl); } bufferevent_decref_and_unlock_(&bev_ssl->bev.bev); } static int be_openssl_auto_fd(struct bufferevent_openssl *bev_ssl, int fd) { if (!bev_ssl->underlying) { struct bufferevent *bev = &bev_ssl->bev.bev; if (event_initialized(&bev->ev_read) && fd < 0) { fd = event_get_fd(&bev->ev_read); } } return fd; } static int set_open_callbacks(struct bufferevent_openssl *bev_ssl, evutil_socket_t fd) { if (bev_ssl->underlying) { bufferevent_setcb(bev_ssl->underlying, be_openssl_readcb, be_openssl_writecb, be_openssl_eventcb, bev_ssl); return 0; } else { struct bufferevent *bev = &bev_ssl->bev.bev; int rpending=0, wpending=0, r1=0, r2=0; if (event_initialized(&bev->ev_read)) { rpending = event_pending(&bev->ev_read, EV_READ, NULL); wpending = event_pending(&bev->ev_write, EV_WRITE, NULL); event_del(&bev->ev_read); event_del(&bev->ev_write); } event_assign(&bev->ev_read, bev->ev_base, fd, EV_READ|EV_PERSIST|EV_FINALIZE, be_openssl_readeventcb, bev_ssl); event_assign(&bev->ev_write, bev->ev_base, fd, EV_WRITE|EV_PERSIST|EV_FINALIZE, be_openssl_writeeventcb, bev_ssl); if (rpending) r1 = bufferevent_add_event_(&bev->ev_read, &bev->timeout_read); if (wpending) r2 = bufferevent_add_event_(&bev->ev_write, &bev->timeout_write); return (r1<0 || r2<0) ? -1 : 0; } } static int do_handshake(struct bufferevent_openssl *bev_ssl) { int r; switch (bev_ssl->state) { default: case BUFFEREVENT_SSL_OPEN: EVUTIL_ASSERT(0); return -1; case BUFFEREVENT_SSL_CONNECTING: case BUFFEREVENT_SSL_ACCEPTING: ERR_clear_error(); r = SSL_do_handshake(bev_ssl->ssl); break; } decrement_buckets(bev_ssl); if (r==1) { int fd = event_get_fd(&bev_ssl->bev.bev.ev_read); /* We're done! */ bev_ssl->state = BUFFEREVENT_SSL_OPEN; set_open_callbacks(bev_ssl, fd); /* XXXX handle failure */ /* Call do_read and do_write as needed */ bufferevent_enable(&bev_ssl->bev.bev, bev_ssl->bev.bev.enabled); bufferevent_run_eventcb_(&bev_ssl->bev.bev, BEV_EVENT_CONNECTED, 0); return 1; } else { int err = SSL_get_error(bev_ssl->ssl, r); print_err(err); switch (err) { case SSL_ERROR_WANT_WRITE: stop_reading(bev_ssl); return start_writing(bev_ssl); case SSL_ERROR_WANT_READ: stop_writing(bev_ssl); return start_reading(bev_ssl); default: conn_closed(bev_ssl, BEV_EVENT_READING, err, r); return -1; } } } static void be_openssl_handshakecb(struct bufferevent *bev_base, void *ctx) { struct bufferevent_openssl *bev_ssl = ctx; do_handshake(bev_ssl);/* XXX handle failure */ } static void be_openssl_handshakeeventcb(evutil_socket_t fd, short what, void *ptr) { struct bufferevent_openssl *bev_ssl = ptr; bufferevent_incref_and_lock_(&bev_ssl->bev.bev); if (what & EV_TIMEOUT) { bufferevent_run_eventcb_(&bev_ssl->bev.bev, BEV_EVENT_TIMEOUT, 0); } else do_handshake(bev_ssl);/* XXX handle failure */ bufferevent_decref_and_unlock_(&bev_ssl->bev.bev); } static int set_handshake_callbacks(struct bufferevent_openssl *bev_ssl, evutil_socket_t fd) { if (bev_ssl->underlying) { bufferevent_setcb(bev_ssl->underlying, be_openssl_handshakecb, be_openssl_handshakecb, be_openssl_eventcb, bev_ssl); if (fd < 0) return 0; if (bufferevent_setfd(bev_ssl->underlying, fd)) return 1; return do_handshake(bev_ssl); } else { struct bufferevent *bev = &bev_ssl->bev.bev; if (event_initialized(&bev->ev_read)) { event_del(&bev->ev_read); event_del(&bev->ev_write); } event_assign(&bev->ev_read, bev->ev_base, fd, EV_READ|EV_PERSIST|EV_FINALIZE, be_openssl_handshakeeventcb, bev_ssl); event_assign(&bev->ev_write, bev->ev_base, fd, EV_WRITE|EV_PERSIST|EV_FINALIZE, be_openssl_handshakeeventcb, bev_ssl); if (fd >= 0) bufferevent_enable(bev, bev->enabled); return 0; } } int bufferevent_ssl_renegotiate(struct bufferevent *bev) { struct bufferevent_openssl *bev_ssl = upcast(bev); if (!bev_ssl) return -1; if (SSL_renegotiate(bev_ssl->ssl) < 0) return -1; bev_ssl->state = BUFFEREVENT_SSL_CONNECTING; if (set_handshake_callbacks(bev_ssl, be_openssl_auto_fd(bev_ssl, -1)) < 0) return -1; if (!bev_ssl->underlying) return do_handshake(bev_ssl); return 0; } static void be_openssl_outbuf_cb(struct evbuffer *buf, const struct evbuffer_cb_info *cbinfo, void *arg) { struct bufferevent_openssl *bev_ssl = arg; int r = 0; /* XXX need to hold a reference here. */ if (cbinfo->n_added && bev_ssl->state == BUFFEREVENT_SSL_OPEN) { if (cbinfo->orig_size == 0) r = bufferevent_add_event_(&bev_ssl->bev.bev.ev_write, &bev_ssl->bev.bev.timeout_write); if (bev_ssl->underlying) consider_writing(bev_ssl); } /* XXX Handle r < 0 */ (void)r; } static int be_openssl_enable(struct bufferevent *bev, short events) { struct bufferevent_openssl *bev_ssl = upcast(bev); int r1 = 0, r2 = 0; if (events & EV_READ) r1 = start_reading(bev_ssl); if (events & EV_WRITE) r2 = start_writing(bev_ssl); if (bev_ssl->underlying) { if (events & EV_READ) BEV_RESET_GENERIC_READ_TIMEOUT(bev); if (events & EV_WRITE) BEV_RESET_GENERIC_WRITE_TIMEOUT(bev); if (events & EV_READ) consider_reading(bev_ssl); if (events & EV_WRITE) consider_writing(bev_ssl); } return (r1 < 0 || r2 < 0) ? -1 : 0; } static int be_openssl_disable(struct bufferevent *bev, short events) { struct bufferevent_openssl *bev_ssl = upcast(bev); if (events & EV_READ) stop_reading(bev_ssl); if (events & EV_WRITE) stop_writing(bev_ssl); if (bev_ssl->underlying) { if (events & EV_READ) BEV_DEL_GENERIC_READ_TIMEOUT(bev); if (events & EV_WRITE) BEV_DEL_GENERIC_WRITE_TIMEOUT(bev); } return 0; } static void be_openssl_unlink(struct bufferevent *bev) { struct bufferevent_openssl *bev_ssl = upcast(bev); if (bev_ssl->bev.options & BEV_OPT_CLOSE_ON_FREE) { if (bev_ssl->underlying) { if (BEV_UPCAST(bev_ssl->underlying)->refcnt < 2) { event_warnx("BEV_OPT_CLOSE_ON_FREE set on an " "bufferevent with too few references"); } else { bufferevent_free(bev_ssl->underlying); /* We still have a reference to it, via our * BIO. So we don't drop this. */ // bev_ssl->underlying = NULL; } } } else { if (bev_ssl->underlying) { if (bev_ssl->underlying->errorcb == be_openssl_eventcb) bufferevent_setcb(bev_ssl->underlying, NULL,NULL,NULL,NULL); bufferevent_unsuspend_read_(bev_ssl->underlying, BEV_SUSPEND_FILT_READ); } } } static void be_openssl_destruct(struct bufferevent *bev) { struct bufferevent_openssl *bev_ssl = upcast(bev); if (bev_ssl->bev.options & BEV_OPT_CLOSE_ON_FREE) { if (! bev_ssl->underlying) { evutil_socket_t fd = -1; BIO *bio = SSL_get_wbio(bev_ssl->ssl); if (bio) fd = BIO_get_fd(bio, NULL); if (fd >= 0) evutil_closesocket(fd); } SSL_free(bev_ssl->ssl); } } static int be_openssl_adj_timeouts(struct bufferevent *bev) { struct bufferevent_openssl *bev_ssl = upcast(bev); if (bev_ssl->underlying) { return bufferevent_generic_adj_timeouts_(bev); } else { return bufferevent_generic_adj_existing_timeouts_(bev); } } static int be_openssl_flush(struct bufferevent *bufev, short iotype, enum bufferevent_flush_mode mode) { /* XXXX Implement this. */ return 0; } static int be_openssl_set_fd(struct bufferevent_openssl *bev_ssl, enum bufferevent_ssl_state state, int fd) { bev_ssl->state = state; switch (state) { case BUFFEREVENT_SSL_ACCEPTING: SSL_set_accept_state(bev_ssl->ssl); if (set_handshake_callbacks(bev_ssl, fd) < 0) return -1; break; case BUFFEREVENT_SSL_CONNECTING: SSL_set_connect_state(bev_ssl->ssl); if (set_handshake_callbacks(bev_ssl, fd) < 0) return -1; break; case BUFFEREVENT_SSL_OPEN: if (set_open_callbacks(bev_ssl, fd) < 0) return -1; break; default: return -1; } return 0; } static int be_openssl_ctrl(struct bufferevent *bev, enum bufferevent_ctrl_op op, union bufferevent_ctrl_data *data) { struct bufferevent_openssl *bev_ssl = upcast(bev); switch (op) { case BEV_CTRL_SET_FD: if (!bev_ssl->underlying) { BIO *bio; bio = BIO_new_socket(data->fd, 0); SSL_set_bio(bev_ssl->ssl, bio, bio); } else { BIO *bio; if (!(bio = BIO_new_bufferevent(bev_ssl->underlying, 0))) return -1; SSL_set_bio(bev_ssl->ssl, bio, bio); } return be_openssl_set_fd(bev_ssl, bev_ssl->old_state, data->fd); case BEV_CTRL_GET_FD: if (bev_ssl->underlying) { data->fd = event_get_fd(&bev_ssl->underlying->ev_read); } else { data->fd = event_get_fd(&bev->ev_read); } return 0; case BEV_CTRL_GET_UNDERLYING: data->ptr = bev_ssl->underlying; return 0; case BEV_CTRL_CANCEL_ALL: default: return -1; } } SSL * bufferevent_openssl_get_ssl(struct bufferevent *bufev) { struct bufferevent_openssl *bev_ssl = upcast(bufev); if (!bev_ssl) return NULL; return bev_ssl->ssl; } static struct bufferevent * bufferevent_openssl_new_impl(struct event_base *base, struct bufferevent *underlying, evutil_socket_t fd, SSL *ssl, enum bufferevent_ssl_state state, int options) { struct bufferevent_openssl *bev_ssl = NULL; struct bufferevent_private *bev_p = NULL; int tmp_options = options & ~BEV_OPT_THREADSAFE; if (underlying != NULL && fd >= 0) return NULL; /* Only one can be set. */ if (!(bev_ssl = mm_calloc(1, sizeof(struct bufferevent_openssl)))) goto err; bev_p = &bev_ssl->bev; if (bufferevent_init_common_(bev_p, base, &bufferevent_ops_openssl, tmp_options) < 0) goto err; /* Don't explode if we decide to realloc a chunk we're writing from in * the output buffer. */ SSL_set_mode(ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); bev_ssl->underlying = underlying; bev_ssl->ssl = ssl; bev_ssl->outbuf_cb = evbuffer_add_cb(bev_p->bev.output, be_openssl_outbuf_cb, bev_ssl); if (options & BEV_OPT_THREADSAFE) bufferevent_enable_locking_(&bev_ssl->bev.bev, NULL); if (underlying) { bufferevent_init_generic_timeout_cbs_(&bev_ssl->bev.bev); bufferevent_incref_(underlying); } bev_ssl->old_state = state; bev_ssl->last_write = -1; init_bio_counts(bev_ssl); fd = be_openssl_auto_fd(bev_ssl, fd); if (be_openssl_set_fd(bev_ssl, state, fd)) goto err; if (underlying) { bufferevent_setwatermark(underlying, EV_READ, 0, 0); bufferevent_enable(underlying, EV_READ|EV_WRITE); if (state == BUFFEREVENT_SSL_OPEN) bufferevent_suspend_read_(underlying, BEV_SUSPEND_FILT_READ); } return &bev_ssl->bev.bev; err: if (bev_ssl) bufferevent_free(&bev_ssl->bev.bev); return NULL; } struct bufferevent * bufferevent_openssl_filter_new(struct event_base *base, struct bufferevent *underlying, SSL *ssl, enum bufferevent_ssl_state state, int options) { /* We don't tell the BIO to close the bufferevent; we do it ourselves * on be_openssl_destruct */ int close_flag = 0; /* options & BEV_OPT_CLOSE_ON_FREE; */ BIO *bio; if (!underlying) return NULL; if (!(bio = BIO_new_bufferevent(underlying, close_flag))) return NULL; SSL_set_bio(ssl, bio, bio); return bufferevent_openssl_new_impl( base, underlying, -1, ssl, state, options); } struct bufferevent * bufferevent_openssl_socket_new(struct event_base *base, evutil_socket_t fd, SSL *ssl, enum bufferevent_ssl_state state, int options) { /* Does the SSL already have an fd? */ BIO *bio = SSL_get_wbio(ssl); long have_fd = -1; if (bio) have_fd = BIO_get_fd(bio, NULL); if (have_fd >= 0) { /* The SSL is already configured with an fd. */ if (fd < 0) { /* We should learn the fd from the SSL. */ fd = (evutil_socket_t) have_fd; } else if (have_fd == (long)fd) { /* We already know the fd from the SSL; do nothing */ } else { /* We specified an fd different from that of the SSL. This is probably an error on our part. Fail. */ return NULL; } (void) BIO_set_close(bio, 0); } else { /* The SSL isn't configured with a BIO with an fd. */ if (fd >= 0) { /* ... and we have an fd we want to use. */ bio = BIO_new_socket(fd, 0); SSL_set_bio(ssl, bio, bio); } else { /* Leave the fd unset. */ } } return bufferevent_openssl_new_impl( base, NULL, fd, ssl, state, options); } int bufferevent_openssl_get_allow_dirty_shutdown(struct bufferevent *bev) { int allow_dirty_shutdown = -1; struct bufferevent_openssl *bev_ssl; BEV_LOCK(bev); bev_ssl = upcast(bev); if (bev_ssl) allow_dirty_shutdown = bev_ssl->allow_dirty_shutdown; BEV_UNLOCK(bev); return allow_dirty_shutdown; } void bufferevent_openssl_set_allow_dirty_shutdown(struct bufferevent *bev, int allow_dirty_shutdown) { struct bufferevent_openssl *bev_ssl; BEV_LOCK(bev); bev_ssl = upcast(bev); if (bev_ssl) bev_ssl->allow_dirty_shutdown = !!allow_dirty_shutdown; BEV_UNLOCK(bev); } unsigned long bufferevent_get_openssl_error(struct bufferevent *bev) { unsigned long err = 0; struct bufferevent_openssl *bev_ssl; BEV_LOCK(bev); bev_ssl = upcast(bev); if (bev_ssl && bev_ssl->n_errors) { err = bev_ssl->errors[--bev_ssl->n_errors]; } BEV_UNLOCK(bev); return err; } ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/bufferevent_pair.c�������������������������������������������0000664�0000000�0000000�00000023152�15270523712�0023473�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2009-2012 Niels Provos, Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #include <sys/types.h> #ifdef _WIN32 #include <winsock2.h> #endif #include "event2/util.h" #include "event2/buffer.h" #include "event2/bufferevent.h" #include "event2/bufferevent_struct.h" #include "event2/event.h" #include "defer-internal.h" #include "bufferevent-internal.h" #include "mm-internal.h" #include "util-internal.h" struct bufferevent_pair { struct bufferevent_private bev; struct bufferevent_pair *partner; /* For ->destruct() lock checking */ struct bufferevent_pair *unlinked_partner; }; /* Given a bufferevent that's really a bev part of a bufferevent_pair, * return that bufferevent_filtered. Returns NULL otherwise.*/ static inline struct bufferevent_pair * upcast(struct bufferevent *bev) { struct bufferevent_pair *bev_p; if (bev->be_ops != &bufferevent_ops_pair) return NULL; bev_p = EVUTIL_UPCAST(bev, struct bufferevent_pair, bev.bev); EVUTIL_ASSERT(bev_p->bev.bev.be_ops == &bufferevent_ops_pair); return bev_p; } #define downcast(bev_pair) (&(bev_pair)->bev.bev) static inline void incref_and_lock(struct bufferevent *b) { struct bufferevent_pair *bevp; bufferevent_incref_and_lock_(b); bevp = upcast(b); if (bevp->partner) bufferevent_incref_and_lock_(downcast(bevp->partner)); } static inline void decref_and_unlock(struct bufferevent *b) { struct bufferevent_pair *bevp = upcast(b); if (bevp->partner) bufferevent_decref_and_unlock_(downcast(bevp->partner)); bufferevent_decref_and_unlock_(b); } /* XXX Handle close */ static void be_pair_outbuf_cb(struct evbuffer *, const struct evbuffer_cb_info *, void *); static struct bufferevent_pair * bufferevent_pair_elt_new(struct event_base *base, int options) { struct bufferevent_pair *bufev; if (! (bufev = mm_calloc(1, sizeof(struct bufferevent_pair)))) return NULL; if (bufferevent_init_common_(&bufev->bev, base, &bufferevent_ops_pair, options)) { mm_free(bufev); return NULL; } if (!evbuffer_add_cb(bufev->bev.bev.output, be_pair_outbuf_cb, bufev)) { bufferevent_free(downcast(bufev)); return NULL; } bufferevent_init_generic_timeout_cbs_(&bufev->bev.bev); return bufev; } int bufferevent_pair_new(struct event_base *base, int options, struct bufferevent *pair[2]) { struct bufferevent_pair *bufev1 = NULL, *bufev2 = NULL; int tmp_options; options |= BEV_OPT_DEFER_CALLBACKS; tmp_options = options & ~BEV_OPT_THREADSAFE; bufev1 = bufferevent_pair_elt_new(base, options); if (!bufev1) return -1; bufev2 = bufferevent_pair_elt_new(base, tmp_options); if (!bufev2) { bufferevent_free(downcast(bufev1)); return -1; } if (options & BEV_OPT_THREADSAFE) { /*XXXX check return */ bufferevent_enable_locking_(downcast(bufev2), bufev1->bev.lock); } bufev1->partner = bufev2; bufev2->partner = bufev1; evbuffer_freeze(downcast(bufev1)->input, 0); evbuffer_freeze(downcast(bufev1)->output, 1); evbuffer_freeze(downcast(bufev2)->input, 0); evbuffer_freeze(downcast(bufev2)->output, 1); pair[0] = downcast(bufev1); pair[1] = downcast(bufev2); return 0; } static void be_pair_transfer(struct bufferevent *src, struct bufferevent *dst, int ignore_wm) { size_t dst_size; size_t n; evbuffer_unfreeze(src->output, 1); evbuffer_unfreeze(dst->input, 0); if (dst->wm_read.high) { dst_size = evbuffer_get_length(dst->input); if (dst_size < dst->wm_read.high) { n = dst->wm_read.high - dst_size; evbuffer_remove_buffer(src->output, dst->input, n); } else { if (!ignore_wm) goto done; n = evbuffer_get_length(src->output); evbuffer_add_buffer(dst->input, src->output); } } else { n = evbuffer_get_length(src->output); evbuffer_add_buffer(dst->input, src->output); } if (n) { BEV_RESET_GENERIC_READ_TIMEOUT(dst); if (evbuffer_get_length(dst->output)) BEV_RESET_GENERIC_WRITE_TIMEOUT(dst); else BEV_DEL_GENERIC_WRITE_TIMEOUT(dst); } bufferevent_trigger_nolock_(dst, EV_READ, 0); bufferevent_trigger_nolock_(src, EV_WRITE, 0); done: evbuffer_freeze(src->output, 1); evbuffer_freeze(dst->input, 0); } static inline int be_pair_wants_to_talk(struct bufferevent_pair *src, struct bufferevent_pair *dst) { return (downcast(src)->enabled & EV_WRITE) && (downcast(dst)->enabled & EV_READ) && !dst->bev.read_suspended && evbuffer_get_length(downcast(src)->output); } static void be_pair_outbuf_cb(struct evbuffer *outbuf, const struct evbuffer_cb_info *info, void *arg) { struct bufferevent_pair *bev_pair = arg; struct bufferevent_pair *partner = bev_pair->partner; incref_and_lock(downcast(bev_pair)); if (info->n_added > info->n_deleted && partner) { /* We got more data. If the other side's reading, then hand it over. */ if (be_pair_wants_to_talk(bev_pair, partner)) { be_pair_transfer(downcast(bev_pair), downcast(partner), 0); } } decref_and_unlock(downcast(bev_pair)); } static int be_pair_enable(struct bufferevent *bufev, short events) { struct bufferevent_pair *bev_p = upcast(bufev); struct bufferevent_pair *partner = bev_p->partner; incref_and_lock(bufev); if (events & EV_READ) { BEV_RESET_GENERIC_READ_TIMEOUT(bufev); } if ((events & EV_WRITE) && evbuffer_get_length(bufev->output)) BEV_RESET_GENERIC_WRITE_TIMEOUT(bufev); /* We're starting to read! Does the other side have anything to write?*/ if ((events & EV_READ) && partner && be_pair_wants_to_talk(partner, bev_p)) { be_pair_transfer(downcast(partner), bufev, 0); } /* We're starting to write! Does the other side want to read? */ if ((events & EV_WRITE) && partner && be_pair_wants_to_talk(bev_p, partner)) { be_pair_transfer(bufev, downcast(partner), 0); } decref_and_unlock(bufev); return 0; } static int be_pair_disable(struct bufferevent *bev, short events) { if (events & EV_READ) { BEV_DEL_GENERIC_READ_TIMEOUT(bev); } if (events & EV_WRITE) { BEV_DEL_GENERIC_WRITE_TIMEOUT(bev); } return 0; } static void be_pair_unlink(struct bufferevent *bev) { struct bufferevent_pair *bev_p = upcast(bev); if (bev_p->partner) { bev_p->unlinked_partner = bev_p->partner; bev_p->partner->partner = NULL; bev_p->partner = NULL; } } /* Free *shared* lock in the latest be (since we share it between two of them). */ static void be_pair_destruct(struct bufferevent *bev) { struct bufferevent_pair *bev_p = upcast(bev); /* Transfer ownership of the lock into partner, otherwise we will use * already free'd lock during freeing second bev, see next example: * * bev1->own_lock = 1 * bev2->own_lock = 0 * bev2->lock = bev1->lock * * bufferevent_free(bev1) # refcnt == 0 -> unlink * bufferevent_free(bev2) # refcnt == 0 -> unlink * * event_base_free() -> finilizers -> EVTHREAD_FREE_LOCK(bev1->lock) * -> BEV_LOCK(bev2->lock) <-- already freed * * Where bev1 == pair[0], bev2 == pair[1]. */ if (bev_p->unlinked_partner && bev_p->bev.own_lock) { bev_p->unlinked_partner->bev.own_lock = 1; bev_p->bev.own_lock = 0; } bev_p->unlinked_partner = NULL; } static int be_pair_flush(struct bufferevent *bev, short iotype, enum bufferevent_flush_mode mode) { struct bufferevent_pair *bev_p = upcast(bev); struct bufferevent *partner; if (!bev_p->partner) return -1; if (mode == BEV_NORMAL) return 0; incref_and_lock(bev); partner = downcast(bev_p->partner); if ((iotype & EV_READ) != 0) be_pair_transfer(partner, bev, 1); if ((iotype & EV_WRITE) != 0) be_pair_transfer(bev, partner, 1); if (mode == BEV_FINISHED) { short what = BEV_EVENT_EOF; if (iotype & EV_READ) what |= BEV_EVENT_WRITING; if (iotype & EV_WRITE) what |= BEV_EVENT_READING; bufferevent_run_eventcb_(partner, what, 0); } decref_and_unlock(bev); return 0; } struct bufferevent * bufferevent_pair_get_partner(struct bufferevent *bev) { struct bufferevent_pair *bev_p; struct bufferevent *partner = NULL; bev_p = upcast(bev); if (! bev_p) return NULL; incref_and_lock(bev); if (bev_p->partner) partner = downcast(bev_p->partner); decref_and_unlock(bev); return partner; } const struct bufferevent_ops bufferevent_ops_pair = { "pair_elt", evutil_offsetof(struct bufferevent_pair, bev.bev), be_pair_enable, be_pair_disable, be_pair_unlink, be_pair_destruct, bufferevent_generic_adj_timeouts_, be_pair_flush, NULL, /* ctrl */ }; ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/bufferevent_ratelim.c����������������������������������������0000664�0000000�0000000�00000073007�15270523712�0024201�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * Copyright (c) 2002-2006 Niels Provos <provos@citi.umich.edu> * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "evconfig-private.h" #include <sys/types.h> #include <limits.h> #include <string.h> #include <stdlib.h> #include "event2/event.h" #include "event2/event_struct.h" #include "event2/util.h" #include "event2/bufferevent.h" #include "event2/bufferevent_struct.h" #include "event2/buffer.h" #include "ratelim-internal.h" #include "bufferevent-internal.h" #include "mm-internal.h" #include "util-internal.h" #include "event-internal.h" int ev_token_bucket_init_(struct ev_token_bucket *bucket, const struct ev_token_bucket_cfg *cfg, ev_uint32_t current_tick, int reinitialize) { if (reinitialize) { /* on reinitialization, we only clip downwards, since we've already used who-knows-how-much bandwidth this tick. We leave "last_updated" as it is; the next update will add the appropriate amount of bandwidth to the bucket. */ if (bucket->read_limit > (ev_int64_t) cfg->read_maximum) bucket->read_limit = cfg->read_maximum; if (bucket->write_limit > (ev_int64_t) cfg->write_maximum) bucket->write_limit = cfg->write_maximum; } else { bucket->read_limit = cfg->read_rate; bucket->write_limit = cfg->write_rate; bucket->last_updated = current_tick; } return 0; } int ev_token_bucket_update_(struct ev_token_bucket *bucket, const struct ev_token_bucket_cfg *cfg, ev_uint32_t current_tick) { /* It's okay if the tick number overflows, since we'll just * wrap around when we do the unsigned substraction. */ unsigned n_ticks = current_tick - bucket->last_updated; /* Make sure some ticks actually happened, and that time didn't * roll back. */ if (n_ticks == 0 || n_ticks > INT_MAX) return 0; /* Naively, we would say bucket->limit += n_ticks * cfg->rate; if (bucket->limit > cfg->maximum) bucket->limit = cfg->maximum; But we're worried about overflow, so we do it like this: */ if ((cfg->read_maximum - bucket->read_limit) / n_ticks < cfg->read_rate) bucket->read_limit = cfg->read_maximum; else bucket->read_limit += n_ticks * cfg->read_rate; if ((cfg->write_maximum - bucket->write_limit) / n_ticks < cfg->write_rate) bucket->write_limit = cfg->write_maximum; else bucket->write_limit += n_ticks * cfg->write_rate; bucket->last_updated = current_tick; return 1; } static inline void bufferevent_update_buckets(struct bufferevent_private *bev) { /* Must hold lock on bev. */ struct timeval now; unsigned tick; event_base_gettimeofday_cached(bev->bev.ev_base, &now); tick = ev_token_bucket_get_tick_(&now, bev->rate_limiting->cfg); if (tick != bev->rate_limiting->limit.last_updated) ev_token_bucket_update_(&bev->rate_limiting->limit, bev->rate_limiting->cfg, tick); } ev_uint32_t ev_token_bucket_get_tick_(const struct timeval *tv, const struct ev_token_bucket_cfg *cfg) { /* This computation uses two multiplies and a divide. We could do * fewer if we knew that the tick length was an integer number of * seconds, or if we knew it divided evenly into a second. We should * investigate that more. */ /* We cast to an ev_uint64_t first, since we don't want to overflow * before we do the final divide. */ ev_uint64_t msec = (ev_uint64_t)tv->tv_sec * 1000 + tv->tv_usec / 1000; return (unsigned)(msec / cfg->msec_per_tick); } struct ev_token_bucket_cfg * ev_token_bucket_cfg_new(size_t read_rate, size_t read_burst, size_t write_rate, size_t write_burst, const struct timeval *tick_len) { struct ev_token_bucket_cfg *r; struct timeval g; if (! tick_len) { g.tv_sec = 1; g.tv_usec = 0; tick_len = &g; } if (read_rate > read_burst || write_rate > write_burst || read_rate < 1 || write_rate < 1) return NULL; if (read_rate > EV_RATE_LIMIT_MAX || write_rate > EV_RATE_LIMIT_MAX || read_burst > EV_RATE_LIMIT_MAX || write_burst > EV_RATE_LIMIT_MAX) return NULL; r = mm_calloc(1, sizeof(struct ev_token_bucket_cfg)); if (!r) return NULL; r->read_rate = read_rate; r->write_rate = write_rate; r->read_maximum = read_burst; r->write_maximum = write_burst; memcpy(&r->tick_timeout, tick_len, sizeof(struct timeval)); r->msec_per_tick = (tick_len->tv_sec * 1000) + (tick_len->tv_usec & COMMON_TIMEOUT_MICROSECONDS_MASK)/1000; return r; } void ev_token_bucket_cfg_free(struct ev_token_bucket_cfg *cfg) { mm_free(cfg); } /* Default values for max_single_read & max_single_write variables. */ #define MAX_SINGLE_READ_DEFAULT 16384 #define MAX_SINGLE_WRITE_DEFAULT 16384 #define LOCK_GROUP(g) EVLOCK_LOCK((g)->lock, 0) #define UNLOCK_GROUP(g) EVLOCK_UNLOCK((g)->lock, 0) static int bev_group_suspend_reading_(struct bufferevent_rate_limit_group *g); static int bev_group_suspend_writing_(struct bufferevent_rate_limit_group *g); static void bev_group_unsuspend_reading_(struct bufferevent_rate_limit_group *g); static void bev_group_unsuspend_writing_(struct bufferevent_rate_limit_group *g); /** Helper: figure out the maximum amount we should write if is_write, or the maximum amount we should read if is_read. Return that maximum, or 0 if our bucket is wholly exhausted. */ static inline ev_ssize_t bufferevent_get_rlim_max_(struct bufferevent_private *bev, int is_write) { /* needs lock on bev. */ ev_ssize_t max_so_far = is_write?bev->max_single_write:bev->max_single_read; #define LIM(x) \ (is_write ? (x).write_limit : (x).read_limit) #define GROUP_SUSPENDED(g) \ (is_write ? (g)->write_suspended : (g)->read_suspended) /* Sets max_so_far to MIN(x, max_so_far) */ #define CLAMPTO(x) \ do { \ if (max_so_far > (x)) \ max_so_far = (x); \ } while (0); if (!bev->rate_limiting) return max_so_far; /* If rate-limiting is enabled at all, update the appropriate bucket, and take the smaller of our rate limit and the group rate limit. */ if (bev->rate_limiting->cfg) { bufferevent_update_buckets(bev); max_so_far = LIM(bev->rate_limiting->limit); } if (bev->rate_limiting->group) { struct bufferevent_rate_limit_group *g = bev->rate_limiting->group; ev_ssize_t share; LOCK_GROUP(g); if (GROUP_SUSPENDED(g)) { /* We can get here if we failed to lock this * particular bufferevent while suspending the whole * group. */ if (is_write) bufferevent_suspend_write_(&bev->bev, BEV_SUSPEND_BW_GROUP); else bufferevent_suspend_read_(&bev->bev, BEV_SUSPEND_BW_GROUP); share = 0; } else { /* XXXX probably we should divide among the active * members, not the total members. */ share = LIM(g->rate_limit) / g->n_members; if (share < g->min_share) share = g->min_share; } UNLOCK_GROUP(g); CLAMPTO(share); } if (max_so_far < 0) max_so_far = 0; return max_so_far; } ev_ssize_t bufferevent_get_read_max_(struct bufferevent_private *bev) { return bufferevent_get_rlim_max_(bev, 0); } ev_ssize_t bufferevent_get_write_max_(struct bufferevent_private *bev) { return bufferevent_get_rlim_max_(bev, 1); } int bufferevent_decrement_read_buckets_(struct bufferevent_private *bev, ev_ssize_t bytes) { /* XXXXX Make sure all users of this function check its return value */ int r = 0; /* need to hold lock on bev */ if (!bev->rate_limiting) return 0; if (bev->rate_limiting->cfg) { bev->rate_limiting->limit.read_limit -= bytes; if (bev->rate_limiting->limit.read_limit <= 0) { bufferevent_suspend_read_(&bev->bev, BEV_SUSPEND_BW); if (event_add(&bev->rate_limiting->refill_bucket_event, &bev->rate_limiting->cfg->tick_timeout) < 0) r = -1; } else if (bev->read_suspended & BEV_SUSPEND_BW) { if (!(bev->write_suspended & BEV_SUSPEND_BW)) event_del(&bev->rate_limiting->refill_bucket_event); bufferevent_unsuspend_read_(&bev->bev, BEV_SUSPEND_BW); } } if (bev->rate_limiting->group) { LOCK_GROUP(bev->rate_limiting->group); bev->rate_limiting->group->rate_limit.read_limit -= bytes; bev->rate_limiting->group->total_read += bytes; if (bev->rate_limiting->group->rate_limit.read_limit <= 0) { bev_group_suspend_reading_(bev->rate_limiting->group); } else if (bev->rate_limiting->group->read_suspended) { bev_group_unsuspend_reading_(bev->rate_limiting->group); } UNLOCK_GROUP(bev->rate_limiting->group); } return r; } int bufferevent_decrement_write_buckets_(struct bufferevent_private *bev, ev_ssize_t bytes) { /* XXXXX Make sure all users of this function check its return value */ int r = 0; /* need to hold lock */ if (!bev->rate_limiting) return 0; if (bev->rate_limiting->cfg) { bev->rate_limiting->limit.write_limit -= bytes; if (bev->rate_limiting->limit.write_limit <= 0) { bufferevent_suspend_write_(&bev->bev, BEV_SUSPEND_BW); if (event_add(&bev->rate_limiting->refill_bucket_event, &bev->rate_limiting->cfg->tick_timeout) < 0) r = -1; } else if (bev->write_suspended & BEV_SUSPEND_BW) { if (!(bev->read_suspended & BEV_SUSPEND_BW)) event_del(&bev->rate_limiting->refill_bucket_event); bufferevent_unsuspend_write_(&bev->bev, BEV_SUSPEND_BW); } } if (bev->rate_limiting->group) { LOCK_GROUP(bev->rate_limiting->group); bev->rate_limiting->group->rate_limit.write_limit -= bytes; bev->rate_limiting->group->total_written += bytes; if (bev->rate_limiting->group->rate_limit.write_limit <= 0) { bev_group_suspend_writing_(bev->rate_limiting->group); } else if (bev->rate_limiting->group->write_suspended) { bev_group_unsuspend_writing_(bev->rate_limiting->group); } UNLOCK_GROUP(bev->rate_limiting->group); } return r; } /** Stop reading on every bufferevent in <b>g</b> */ static int bev_group_suspend_reading_(struct bufferevent_rate_limit_group *g) { /* Needs group lock */ struct bufferevent_private *bev; g->read_suspended = 1; g->pending_unsuspend_read = 0; /* Note that in this loop we call EVLOCK_TRY_LOCK_ instead of BEV_LOCK, to prevent a deadlock. (Ordinarily, the group lock nests inside the bufferevent locks. If we are unable to lock any individual bufferevent, it will find out later when it looks at its limit and sees that its group is suspended.) */ LIST_FOREACH(bev, &g->members, rate_limiting->next_in_group) { if (EVLOCK_TRY_LOCK_(bev->lock)) { bufferevent_suspend_read_(&bev->bev, BEV_SUSPEND_BW_GROUP); EVLOCK_UNLOCK(bev->lock, 0); } } return 0; } /** Stop writing on every bufferevent in <b>g</b> */ static int bev_group_suspend_writing_(struct bufferevent_rate_limit_group *g) { /* Needs group lock */ struct bufferevent_private *bev; g->write_suspended = 1; g->pending_unsuspend_write = 0; LIST_FOREACH(bev, &g->members, rate_limiting->next_in_group) { if (EVLOCK_TRY_LOCK_(bev->lock)) { bufferevent_suspend_write_(&bev->bev, BEV_SUSPEND_BW_GROUP); EVLOCK_UNLOCK(bev->lock, 0); } } return 0; } /** Timer callback invoked on a single bufferevent with one or more exhausted buckets when they are ready to refill. */ static void bev_refill_callback_(evutil_socket_t fd, short what, void *arg) { unsigned tick; struct timeval now; struct bufferevent_private *bev = arg; int again = 0; BEV_LOCK(&bev->bev); if (!bev->rate_limiting || !bev->rate_limiting->cfg) { BEV_UNLOCK(&bev->bev); return; } /* First, update the bucket */ event_base_gettimeofday_cached(bev->bev.ev_base, &now); tick = ev_token_bucket_get_tick_(&now, bev->rate_limiting->cfg); ev_token_bucket_update_(&bev->rate_limiting->limit, bev->rate_limiting->cfg, tick); /* Now unsuspend any read/write operations as appropriate. */ if ((bev->read_suspended & BEV_SUSPEND_BW)) { if (bev->rate_limiting->limit.read_limit > 0) bufferevent_unsuspend_read_(&bev->bev, BEV_SUSPEND_BW); else again = 1; } if ((bev->write_suspended & BEV_SUSPEND_BW)) { if (bev->rate_limiting->limit.write_limit > 0) bufferevent_unsuspend_write_(&bev->bev, BEV_SUSPEND_BW); else again = 1; } if (again) { /* One or more of the buckets may need another refill if they started negative. XXXX if we need to be quiet for more ticks, we should maybe figure out what timeout we really want. */ /* XXXX Handle event_add failure somehow */ event_add(&bev->rate_limiting->refill_bucket_event, &bev->rate_limiting->cfg->tick_timeout); } BEV_UNLOCK(&bev->bev); } /** Helper: grab a random element from a bufferevent group. * * Requires that we hold the lock on the group. */ static struct bufferevent_private * bev_group_random_element_(struct bufferevent_rate_limit_group *group) { int which; struct bufferevent_private *bev; /* requires group lock */ if (!group->n_members) return NULL; EVUTIL_ASSERT(! LIST_EMPTY(&group->members)); which = evutil_weakrand_range_(&group->weakrand_seed, group->n_members); bev = LIST_FIRST(&group->members); while (which--) bev = LIST_NEXT(bev, rate_limiting->next_in_group); return bev; } /** Iterate over the elements of a rate-limiting group 'g' with a random starting point, assigning each to the variable 'bev', and executing the block 'block'. We do this in a half-baked effort to get fairness among group members. XXX Round-robin or some kind of priority queue would be even more fair. */ #define FOREACH_RANDOM_ORDER(block) \ do { \ first = bev_group_random_element_(g); \ for (bev = first; bev != LIST_END(&g->members); \ bev = LIST_NEXT(bev, rate_limiting->next_in_group)) { \ block ; \ } \ for (bev = LIST_FIRST(&g->members); bev && bev != first; \ bev = LIST_NEXT(bev, rate_limiting->next_in_group)) { \ block ; \ } \ } while (0) static void bev_group_unsuspend_reading_(struct bufferevent_rate_limit_group *g) { int again = 0; struct bufferevent_private *bev, *first; g->read_suspended = 0; FOREACH_RANDOM_ORDER({ if (EVLOCK_TRY_LOCK_(bev->lock)) { bufferevent_unsuspend_read_(&bev->bev, BEV_SUSPEND_BW_GROUP); EVLOCK_UNLOCK(bev->lock, 0); } else { again = 1; } }); g->pending_unsuspend_read = again; } static void bev_group_unsuspend_writing_(struct bufferevent_rate_limit_group *g) { int again = 0; struct bufferevent_private *bev, *first; g->write_suspended = 0; FOREACH_RANDOM_ORDER({ if (EVLOCK_TRY_LOCK_(bev->lock)) { bufferevent_unsuspend_write_(&bev->bev, BEV_SUSPEND_BW_GROUP); EVLOCK_UNLOCK(bev->lock, 0); } else { again = 1; } }); g->pending_unsuspend_write = again; } /** Callback invoked every tick to add more elements to the group bucket and unsuspend group members as needed. */ static void bev_group_refill_callback_(evutil_socket_t fd, short what, void *arg) { struct bufferevent_rate_limit_group *g = arg; unsigned tick; struct timeval now; event_base_gettimeofday_cached(event_get_base(&g->master_refill_event), &now); LOCK_GROUP(g); tick = ev_token_bucket_get_tick_(&now, &g->rate_limit_cfg); ev_token_bucket_update_(&g->rate_limit, &g->rate_limit_cfg, tick); if (g->pending_unsuspend_read || (g->read_suspended && (g->rate_limit.read_limit >= g->min_share))) { bev_group_unsuspend_reading_(g); } if (g->pending_unsuspend_write || (g->write_suspended && (g->rate_limit.write_limit >= g->min_share))){ bev_group_unsuspend_writing_(g); } /* XXXX Rather than waiting to the next tick to unsuspend stuff * with pending_unsuspend_write/read, we should do it on the * next iteration of the mainloop. */ UNLOCK_GROUP(g); } int bufferevent_set_rate_limit(struct bufferevent *bev, struct ev_token_bucket_cfg *cfg) { struct bufferevent_private *bevp = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); int r = -1; struct bufferevent_rate_limit *rlim; struct timeval now; ev_uint32_t tick; int reinit = 0, suspended = 0; /* XXX reference-count cfg */ BEV_LOCK(bev); if (cfg == NULL) { if (bevp->rate_limiting) { rlim = bevp->rate_limiting; rlim->cfg = NULL; bufferevent_unsuspend_read_(bev, BEV_SUSPEND_BW); bufferevent_unsuspend_write_(bev, BEV_SUSPEND_BW); if (event_initialized(&rlim->refill_bucket_event)) event_del(&rlim->refill_bucket_event); } r = 0; goto done; } event_base_gettimeofday_cached(bev->ev_base, &now); tick = ev_token_bucket_get_tick_(&now, cfg); if (bevp->rate_limiting && bevp->rate_limiting->cfg == cfg) { /* no-op */ r = 0; goto done; } if (bevp->rate_limiting == NULL) { rlim = mm_calloc(1, sizeof(struct bufferevent_rate_limit)); if (!rlim) goto done; bevp->rate_limiting = rlim; } else { rlim = bevp->rate_limiting; } reinit = rlim->cfg != NULL; rlim->cfg = cfg; ev_token_bucket_init_(&rlim->limit, cfg, tick, reinit); if (reinit) { EVUTIL_ASSERT(event_initialized(&rlim->refill_bucket_event)); event_del(&rlim->refill_bucket_event); } event_assign(&rlim->refill_bucket_event, bev->ev_base, -1, EV_FINALIZE, bev_refill_callback_, bevp); if (rlim->limit.read_limit > 0) { bufferevent_unsuspend_read_(bev, BEV_SUSPEND_BW); } else { bufferevent_suspend_read_(bev, BEV_SUSPEND_BW); suspended=1; } if (rlim->limit.write_limit > 0) { bufferevent_unsuspend_write_(bev, BEV_SUSPEND_BW); } else { bufferevent_suspend_write_(bev, BEV_SUSPEND_BW); suspended = 1; } if (suspended) event_add(&rlim->refill_bucket_event, &cfg->tick_timeout); r = 0; done: BEV_UNLOCK(bev); return r; } struct bufferevent_rate_limit_group * bufferevent_rate_limit_group_new(struct event_base *base, const struct ev_token_bucket_cfg *cfg) { struct bufferevent_rate_limit_group *g; struct timeval now; ev_uint32_t tick; event_base_gettimeofday_cached(base, &now); tick = ev_token_bucket_get_tick_(&now, cfg); g = mm_calloc(1, sizeof(struct bufferevent_rate_limit_group)); if (!g) return NULL; memcpy(&g->rate_limit_cfg, cfg, sizeof(g->rate_limit_cfg)); LIST_INIT(&g->members); ev_token_bucket_init_(&g->rate_limit, cfg, tick, 0); event_assign(&g->master_refill_event, base, -1, EV_PERSIST|EV_FINALIZE, bev_group_refill_callback_, g); /*XXXX handle event_add failure */ event_add(&g->master_refill_event, &cfg->tick_timeout); EVTHREAD_ALLOC_LOCK(g->lock, EVTHREAD_LOCKTYPE_RECURSIVE); bufferevent_rate_limit_group_set_min_share(g, 64); evutil_weakrand_seed_(&g->weakrand_seed, (ev_uint32_t) ((now.tv_sec + now.tv_usec) + (ev_intptr_t)g)); return g; } int bufferevent_rate_limit_group_set_cfg( struct bufferevent_rate_limit_group *g, const struct ev_token_bucket_cfg *cfg) { int same_tick; if (!g || !cfg) return -1; LOCK_GROUP(g); same_tick = evutil_timercmp( &g->rate_limit_cfg.tick_timeout, &cfg->tick_timeout, ==); memcpy(&g->rate_limit_cfg, cfg, sizeof(g->rate_limit_cfg)); if (g->rate_limit.read_limit > (ev_ssize_t)cfg->read_maximum) g->rate_limit.read_limit = cfg->read_maximum; if (g->rate_limit.write_limit > (ev_ssize_t)cfg->write_maximum) g->rate_limit.write_limit = cfg->write_maximum; if (!same_tick) { /* This can cause a hiccup in the schedule */ event_add(&g->master_refill_event, &cfg->tick_timeout); } /* The new limits might force us to adjust min_share differently. */ bufferevent_rate_limit_group_set_min_share(g, g->configured_min_share); UNLOCK_GROUP(g); return 0; } int bufferevent_rate_limit_group_set_min_share( struct bufferevent_rate_limit_group *g, size_t share) { if (share > EV_SSIZE_MAX) return -1; g->configured_min_share = share; /* Can't set share to less than the one-tick maximum. IOW, at steady * state, at least one connection can go per tick. */ if (share > g->rate_limit_cfg.read_rate) share = g->rate_limit_cfg.read_rate; if (share > g->rate_limit_cfg.write_rate) share = g->rate_limit_cfg.write_rate; g->min_share = share; return 0; } void bufferevent_rate_limit_group_free(struct bufferevent_rate_limit_group *g) { LOCK_GROUP(g); EVUTIL_ASSERT(0 == g->n_members); event_del(&g->master_refill_event); UNLOCK_GROUP(g); EVTHREAD_FREE_LOCK(g->lock, EVTHREAD_LOCKTYPE_RECURSIVE); mm_free(g); } int bufferevent_add_to_rate_limit_group(struct bufferevent *bev, struct bufferevent_rate_limit_group *g) { int wsuspend, rsuspend; struct bufferevent_private *bevp = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); BEV_LOCK(bev); if (!bevp->rate_limiting) { struct bufferevent_rate_limit *rlim; rlim = mm_calloc(1, sizeof(struct bufferevent_rate_limit)); if (!rlim) { BEV_UNLOCK(bev); return -1; } event_assign(&rlim->refill_bucket_event, bev->ev_base, -1, EV_FINALIZE, bev_refill_callback_, bevp); bevp->rate_limiting = rlim; } if (bevp->rate_limiting->group == g) { BEV_UNLOCK(bev); return 0; } if (bevp->rate_limiting->group) bufferevent_remove_from_rate_limit_group(bev); LOCK_GROUP(g); bevp->rate_limiting->group = g; ++g->n_members; LIST_INSERT_HEAD(&g->members, bevp, rate_limiting->next_in_group); rsuspend = g->read_suspended; wsuspend = g->write_suspended; UNLOCK_GROUP(g); if (rsuspend) bufferevent_suspend_read_(bev, BEV_SUSPEND_BW_GROUP); if (wsuspend) bufferevent_suspend_write_(bev, BEV_SUSPEND_BW_GROUP); BEV_UNLOCK(bev); return 0; } int bufferevent_remove_from_rate_limit_group(struct bufferevent *bev) { return bufferevent_remove_from_rate_limit_group_internal_(bev, 1); } int bufferevent_remove_from_rate_limit_group_internal_(struct bufferevent *bev, int unsuspend) { struct bufferevent_private *bevp = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); BEV_LOCK(bev); if (bevp->rate_limiting && bevp->rate_limiting->group) { struct bufferevent_rate_limit_group *g = bevp->rate_limiting->group; LOCK_GROUP(g); bevp->rate_limiting->group = NULL; --g->n_members; LIST_REMOVE(bevp, rate_limiting->next_in_group); UNLOCK_GROUP(g); } if (unsuspend) { bufferevent_unsuspend_read_(bev, BEV_SUSPEND_BW_GROUP); bufferevent_unsuspend_write_(bev, BEV_SUSPEND_BW_GROUP); } BEV_UNLOCK(bev); return 0; } /* === * API functions to expose rate limits. * * Don't use these from inside Libevent; they're meant to be for use by * the program. * === */ /* Mostly you don't want to use this function from inside libevent; * bufferevent_get_read_max_() is more likely what you want*/ ev_ssize_t bufferevent_get_read_limit(struct bufferevent *bev) { ev_ssize_t r; struct bufferevent_private *bevp; BEV_LOCK(bev); bevp = BEV_UPCAST(bev); if (bevp->rate_limiting && bevp->rate_limiting->cfg) { bufferevent_update_buckets(bevp); r = bevp->rate_limiting->limit.read_limit; } else { r = EV_SSIZE_MAX; } BEV_UNLOCK(bev); return r; } /* Mostly you don't want to use this function from inside libevent; * bufferevent_get_write_max_() is more likely what you want*/ ev_ssize_t bufferevent_get_write_limit(struct bufferevent *bev) { ev_ssize_t r; struct bufferevent_private *bevp; BEV_LOCK(bev); bevp = BEV_UPCAST(bev); if (bevp->rate_limiting && bevp->rate_limiting->cfg) { bufferevent_update_buckets(bevp); r = bevp->rate_limiting->limit.write_limit; } else { r = EV_SSIZE_MAX; } BEV_UNLOCK(bev); return r; } int bufferevent_set_max_single_read(struct bufferevent *bev, size_t size) { struct bufferevent_private *bevp; BEV_LOCK(bev); bevp = BEV_UPCAST(bev); if (size == 0 || size > EV_SSIZE_MAX) bevp->max_single_read = MAX_SINGLE_READ_DEFAULT; else bevp->max_single_read = size; BEV_UNLOCK(bev); return 0; } int bufferevent_set_max_single_write(struct bufferevent *bev, size_t size) { struct bufferevent_private *bevp; BEV_LOCK(bev); bevp = BEV_UPCAST(bev); if (size == 0 || size > EV_SSIZE_MAX) bevp->max_single_write = MAX_SINGLE_WRITE_DEFAULT; else bevp->max_single_write = size; BEV_UNLOCK(bev); return 0; } ev_ssize_t bufferevent_get_max_single_read(struct bufferevent *bev) { ev_ssize_t r; BEV_LOCK(bev); r = BEV_UPCAST(bev)->max_single_read; BEV_UNLOCK(bev); return r; } ev_ssize_t bufferevent_get_max_single_write(struct bufferevent *bev) { ev_ssize_t r; BEV_LOCK(bev); r = BEV_UPCAST(bev)->max_single_write; BEV_UNLOCK(bev); return r; } ev_ssize_t bufferevent_get_max_to_read(struct bufferevent *bev) { ev_ssize_t r; BEV_LOCK(bev); r = bufferevent_get_read_max_(BEV_UPCAST(bev)); BEV_UNLOCK(bev); return r; } ev_ssize_t bufferevent_get_max_to_write(struct bufferevent *bev) { ev_ssize_t r; BEV_LOCK(bev); r = bufferevent_get_write_max_(BEV_UPCAST(bev)); BEV_UNLOCK(bev); return r; } const struct ev_token_bucket_cfg * bufferevent_get_token_bucket_cfg(const struct bufferevent *bev) { struct bufferevent_private *bufev_private = BEV_UPCAST(bev); struct ev_token_bucket_cfg *cfg; BEV_LOCK(bev); if (bufev_private->rate_limiting) { cfg = bufev_private->rate_limiting->cfg; } else { cfg = NULL; } BEV_UNLOCK(bev); return cfg; } /* Mostly you don't want to use this function from inside libevent; * bufferevent_get_read_max_() is more likely what you want*/ ev_ssize_t bufferevent_rate_limit_group_get_read_limit( struct bufferevent_rate_limit_group *grp) { ev_ssize_t r; LOCK_GROUP(grp); r = grp->rate_limit.read_limit; UNLOCK_GROUP(grp); return r; } /* Mostly you don't want to use this function from inside libevent; * bufferevent_get_write_max_() is more likely what you want. */ ev_ssize_t bufferevent_rate_limit_group_get_write_limit( struct bufferevent_rate_limit_group *grp) { ev_ssize_t r; LOCK_GROUP(grp); r = grp->rate_limit.write_limit; UNLOCK_GROUP(grp); return r; } int bufferevent_decrement_read_limit(struct bufferevent *bev, ev_ssize_t decr) { int r = 0; ev_ssize_t old_limit, new_limit; struct bufferevent_private *bevp; BEV_LOCK(bev); bevp = BEV_UPCAST(bev); EVUTIL_ASSERT(bevp->rate_limiting && bevp->rate_limiting->cfg); old_limit = bevp->rate_limiting->limit.read_limit; new_limit = (bevp->rate_limiting->limit.read_limit -= decr); if (old_limit > 0 && new_limit <= 0) { bufferevent_suspend_read_(bev, BEV_SUSPEND_BW); if (event_add(&bevp->rate_limiting->refill_bucket_event, &bevp->rate_limiting->cfg->tick_timeout) < 0) r = -1; } else if (old_limit <= 0 && new_limit > 0) { if (!(bevp->write_suspended & BEV_SUSPEND_BW)) event_del(&bevp->rate_limiting->refill_bucket_event); bufferevent_unsuspend_read_(bev, BEV_SUSPEND_BW); } BEV_UNLOCK(bev); return r; } int bufferevent_decrement_write_limit(struct bufferevent *bev, ev_ssize_t decr) { /* XXXX this is mostly copy-and-paste from * bufferevent_decrement_read_limit */ int r = 0; ev_ssize_t old_limit, new_limit; struct bufferevent_private *bevp; BEV_LOCK(bev); bevp = BEV_UPCAST(bev); EVUTIL_ASSERT(bevp->rate_limiting && bevp->rate_limiting->cfg); old_limit = bevp->rate_limiting->limit.write_limit; new_limit = (bevp->rate_limiting->limit.write_limit -= decr); if (old_limit > 0 && new_limit <= 0) { bufferevent_suspend_write_(bev, BEV_SUSPEND_BW); if (event_add(&bevp->rate_limiting->refill_bucket_event, &bevp->rate_limiting->cfg->tick_timeout) < 0) r = -1; } else if (old_limit <= 0 && new_limit > 0) { if (!(bevp->read_suspended & BEV_SUSPEND_BW)) event_del(&bevp->rate_limiting->refill_bucket_event); bufferevent_unsuspend_write_(bev, BEV_SUSPEND_BW); } BEV_UNLOCK(bev); return r; } int bufferevent_rate_limit_group_decrement_read( struct bufferevent_rate_limit_group *grp, ev_ssize_t decr) { int r = 0; ev_ssize_t old_limit, new_limit; LOCK_GROUP(grp); old_limit = grp->rate_limit.read_limit; new_limit = (grp->rate_limit.read_limit -= decr); if (old_limit > 0 && new_limit <= 0) { bev_group_suspend_reading_(grp); } else if (old_limit <= 0 && new_limit > 0) { bev_group_unsuspend_reading_(grp); } UNLOCK_GROUP(grp); return r; } int bufferevent_rate_limit_group_decrement_write( struct bufferevent_rate_limit_group *grp, ev_ssize_t decr) { int r = 0; ev_ssize_t old_limit, new_limit; LOCK_GROUP(grp); old_limit = grp->rate_limit.write_limit; new_limit = (grp->rate_limit.write_limit -= decr); if (old_limit > 0 && new_limit <= 0) { bev_group_suspend_writing_(grp); } else if (old_limit <= 0 && new_limit > 0) { bev_group_unsuspend_writing_(grp); } UNLOCK_GROUP(grp); return r; } void bufferevent_rate_limit_group_get_totals(struct bufferevent_rate_limit_group *grp, ev_uint64_t *total_read_out, ev_uint64_t *total_written_out) { EVUTIL_ASSERT(grp != NULL); if (total_read_out) *total_read_out = grp->total_read; if (total_written_out) *total_written_out = grp->total_written; } void bufferevent_rate_limit_group_reset_totals(struct bufferevent_rate_limit_group *grp) { grp->total_read = grp->total_written = 0; } int bufferevent_ratelim_init_(struct bufferevent_private *bev) { bev->rate_limiting = NULL; bev->max_single_read = MAX_SINGLE_READ_DEFAULT; bev->max_single_write = MAX_SINGLE_WRITE_DEFAULT; return 0; } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/bufferevent_sock.c�������������������������������������������0000664�0000000�0000000�00000044576�15270523712�0023514�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * Copyright (c) 2002-2006 Niels Provos <provos@citi.umich.edu> * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #include <sys/types.h> #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #ifdef EVENT__HAVE_STDARG_H #include <stdarg.h> #endif #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #ifdef _WIN32 #include <winsock2.h> #include <ws2tcpip.h> #endif #ifdef EVENT__HAVE_SYS_SOCKET_H #include <sys/socket.h> #endif #ifdef EVENT__HAVE_NETINET_IN_H #include <netinet/in.h> #endif #ifdef EVENT__HAVE_NETINET_IN6_H #include <netinet/in6.h> #endif #include "event2/util.h" #include "event2/bufferevent.h" #include "event2/buffer.h" #include "event2/bufferevent_struct.h" #include "event2/bufferevent_compat.h" #include "event2/event.h" #include "log-internal.h" #include "mm-internal.h" #include "bufferevent-internal.h" #include "util-internal.h" #ifdef _WIN32 #include "iocp-internal.h" #endif /* prototypes */ static int be_socket_enable(struct bufferevent *, short); static int be_socket_disable(struct bufferevent *, short); static void be_socket_destruct(struct bufferevent *); static int be_socket_flush(struct bufferevent *, short, enum bufferevent_flush_mode); static int be_socket_ctrl(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *); static void be_socket_setfd(struct bufferevent *, evutil_socket_t); const struct bufferevent_ops bufferevent_ops_socket = { "socket", evutil_offsetof(struct bufferevent_private, bev), be_socket_enable, be_socket_disable, NULL, /* unlink */ be_socket_destruct, bufferevent_generic_adj_existing_timeouts_, be_socket_flush, be_socket_ctrl, }; const struct sockaddr* bufferevent_socket_get_conn_address_(struct bufferevent *bev) { struct bufferevent_private *bev_p = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); return (struct sockaddr *)&bev_p->conn_address; } static void bufferevent_socket_set_conn_address_fd(struct bufferevent_private *bev_p, int fd) { socklen_t len = sizeof(bev_p->conn_address); struct sockaddr *addr = (struct sockaddr *)&bev_p->conn_address; if (addr->sa_family != AF_UNSPEC) getpeername(fd, addr, &len); } static void bufferevent_socket_set_conn_address(struct bufferevent_private *bev_p, struct sockaddr *addr, size_t addrlen) { EVUTIL_ASSERT(addrlen <= sizeof(bev_p->conn_address)); memcpy(&bev_p->conn_address, addr, addrlen); } static void bufferevent_socket_outbuf_cb(struct evbuffer *buf, const struct evbuffer_cb_info *cbinfo, void *arg) { struct bufferevent *bufev = arg; struct bufferevent_private *bufev_p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); if (cbinfo->n_added && (bufev->enabled & EV_WRITE) && !event_pending(&bufev->ev_write, EV_WRITE, NULL) && !bufev_p->write_suspended) { /* Somebody added data to the buffer, and we would like to * write, and we were not writing. So, start writing. */ if (bufferevent_add_event_(&bufev->ev_write, &bufev->timeout_write) == -1) { /* Should we log this? */ } } } static void bufferevent_readcb(evutil_socket_t fd, short event, void *arg) { struct bufferevent *bufev = arg; struct bufferevent_private *bufev_p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); struct evbuffer *input; int res = 0; short what = BEV_EVENT_READING; ev_ssize_t howmuch = -1, readmax=-1; bufferevent_incref_and_lock_(bufev); if (event == EV_TIMEOUT) { /* Note that we only check for event==EV_TIMEOUT. If * event==EV_TIMEOUT|EV_READ, we can safely ignore the * timeout, since a read has occurred */ what |= BEV_EVENT_TIMEOUT; goto error; } input = bufev->input; /* * If we have a high watermark configured then we don't want to * read more data than would make us reach the watermark. */ if (bufev->wm_read.high != 0) { howmuch = bufev->wm_read.high - evbuffer_get_length(input); /* we somehow lowered the watermark, stop reading */ if (howmuch <= 0) { bufferevent_wm_suspend_read(bufev); goto done; } } readmax = bufferevent_get_read_max_(bufev_p); if (howmuch < 0 || howmuch > readmax) /* The use of -1 for "unlimited" * uglifies this code. XXXX */ howmuch = readmax; if (bufev_p->read_suspended) goto done; evbuffer_unfreeze(input, 0); res = evbuffer_read(input, fd, (int)howmuch); /* XXXX evbuffer_read would do better to take and return ev_ssize_t */ evbuffer_freeze(input, 0); if (res == -1) { int err = evutil_socket_geterror(fd); if (EVUTIL_ERR_RW_RETRIABLE(err)) goto reschedule; if (EVUTIL_ERR_CONNECT_REFUSED(err)) { bufev_p->connection_refused = 1; goto done; } /* error case */ what |= BEV_EVENT_ERROR; } else if (res == 0) { /* eof case */ what |= BEV_EVENT_EOF; } if (res <= 0) goto error; bufferevent_decrement_read_buckets_(bufev_p, res); /* Invoke the user callback - must always be called last */ bufferevent_trigger_nolock_(bufev, EV_READ, 0); goto done; reschedule: goto done; error: bufferevent_disable(bufev, EV_READ); bufferevent_run_eventcb_(bufev, what, 0); done: bufferevent_decref_and_unlock_(bufev); } static void bufferevent_writecb(evutil_socket_t fd, short event, void *arg) { struct bufferevent *bufev = arg; struct bufferevent_private *bufev_p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); int res = 0; short what = BEV_EVENT_WRITING; int connected = 0; ev_ssize_t atmost = -1; bufferevent_incref_and_lock_(bufev); if (event == EV_TIMEOUT) { /* Note that we only check for event==EV_TIMEOUT. If * event==EV_TIMEOUT|EV_WRITE, we can safely ignore the * timeout, since a read has occurred */ what |= BEV_EVENT_TIMEOUT; goto error; } if (bufev_p->connecting) { int c = evutil_socket_finished_connecting_(fd); /* we need to fake the error if the connection was refused * immediately - usually connection to localhost on BSD */ if (bufev_p->connection_refused) { bufev_p->connection_refused = 0; c = -1; } if (c == 0) goto done; bufev_p->connecting = 0; if (c < 0) { event_del(&bufev->ev_write); event_del(&bufev->ev_read); bufferevent_run_eventcb_(bufev, BEV_EVENT_ERROR, 0); goto done; } else { connected = 1; bufferevent_socket_set_conn_address_fd(bufev_p, fd); #ifdef _WIN32 if (BEV_IS_ASYNC(bufev)) { event_del(&bufev->ev_write); bufferevent_async_set_connected_(bufev); bufferevent_run_eventcb_(bufev, BEV_EVENT_CONNECTED, 0); goto done; } #endif bufferevent_run_eventcb_(bufev, BEV_EVENT_CONNECTED, 0); if (!(bufev->enabled & EV_WRITE) || bufev_p->write_suspended) { event_del(&bufev->ev_write); goto done; } } } atmost = bufferevent_get_write_max_(bufev_p); if (bufev_p->write_suspended) goto done; if (evbuffer_get_length(bufev->output)) { evbuffer_unfreeze(bufev->output, 1); res = evbuffer_write_atmost(bufev->output, fd, atmost); evbuffer_freeze(bufev->output, 1); if (res == -1) { int err = evutil_socket_geterror(fd); if (EVUTIL_ERR_RW_RETRIABLE(err)) goto reschedule; what |= BEV_EVENT_ERROR; } else if (res == 0) { /* eof case XXXX Actually, a 0 on write doesn't indicate an EOF. An ECONNRESET might be more typical. */ what |= BEV_EVENT_EOF; } if (res <= 0) goto error; bufferevent_decrement_write_buckets_(bufev_p, res); } if (evbuffer_get_length(bufev->output) == 0) { event_del(&bufev->ev_write); } /* * Invoke the user callback if our buffer is drained or below the * low watermark. */ if (res || !connected) { bufferevent_trigger_nolock_(bufev, EV_WRITE, 0); } goto done; reschedule: if (evbuffer_get_length(bufev->output) == 0) { event_del(&bufev->ev_write); } goto done; error: bufferevent_disable(bufev, EV_WRITE); bufferevent_run_eventcb_(bufev, what, 0); done: bufferevent_decref_and_unlock_(bufev); } struct bufferevent * bufferevent_socket_new(struct event_base *base, evutil_socket_t fd, int options) { struct bufferevent_private *bufev_p; struct bufferevent *bufev; #ifdef _WIN32 if (base && event_base_get_iocp_(base)) return bufferevent_async_new_(base, fd, options); #endif if ((bufev_p = mm_calloc(1, sizeof(struct bufferevent_private)))== NULL) return NULL; if (bufferevent_init_common_(bufev_p, base, &bufferevent_ops_socket, options) < 0) { mm_free(bufev_p); return NULL; } bufev = &bufev_p->bev; evbuffer_set_flags(bufev->output, EVBUFFER_FLAG_DRAINS_TO_FD); event_assign(&bufev->ev_read, bufev->ev_base, fd, EV_READ|EV_PERSIST|EV_FINALIZE, bufferevent_readcb, bufev); event_assign(&bufev->ev_write, bufev->ev_base, fd, EV_WRITE|EV_PERSIST|EV_FINALIZE, bufferevent_writecb, bufev); evbuffer_add_cb(bufev->output, bufferevent_socket_outbuf_cb, bufev); evbuffer_freeze(bufev->input, 0); evbuffer_freeze(bufev->output, 1); return bufev; } int bufferevent_socket_connect(struct bufferevent *bev, const struct sockaddr *sa, int socklen) { struct bufferevent_private *bufev_p = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); evutil_socket_t fd; int r = 0; int result=-1; int ownfd = 0; bufferevent_incref_and_lock_(bev); if (!bufev_p) goto done; fd = bufferevent_getfd(bev); if (fd < 0) { if (!sa) goto done; fd = evutil_socket_(sa->sa_family, SOCK_STREAM|EVUTIL_SOCK_NONBLOCK, 0); if (fd < 0) goto done; ownfd = 1; } if (sa) { #ifdef _WIN32 if (bufferevent_async_can_connect_(bev)) { bufferevent_setfd(bev, fd); r = bufferevent_async_connect_(bev, fd, sa, socklen); if (r < 0) goto freesock; bufev_p->connecting = 1; result = 0; goto done; } else #endif r = evutil_socket_connect_(&fd, sa, socklen); if (r < 0) goto freesock; } #ifdef _WIN32 /* ConnectEx() isn't always around, even when IOCP is enabled. * Here, we borrow the socket object's write handler to fall back * on a non-blocking connect() when ConnectEx() is unavailable. */ if (BEV_IS_ASYNC(bev)) { event_assign(&bev->ev_write, bev->ev_base, fd, EV_WRITE|EV_PERSIST|EV_FINALIZE, bufferevent_writecb, bev); } #endif bufferevent_setfd(bev, fd); if (r == 0) { if (! be_socket_enable(bev, EV_WRITE)) { bufev_p->connecting = 1; result = 0; goto done; } } else if (r == 1) { /* The connect succeeded already. How very BSD of it. */ result = 0; bufev_p->connecting = 1; bufferevent_trigger_nolock_(bev, EV_WRITE, BEV_OPT_DEFER_CALLBACKS); } else { /* The connect failed already. How very BSD of it. */ result = 0; bufferevent_run_eventcb_(bev, BEV_EVENT_ERROR, BEV_OPT_DEFER_CALLBACKS); bufferevent_disable(bev, EV_WRITE|EV_READ); } goto done; freesock: bufferevent_run_eventcb_(bev, BEV_EVENT_ERROR, 0); if (ownfd) evutil_closesocket(fd); /* do something about the error? */ done: bufferevent_decref_and_unlock_(bev); return result; } static void bufferevent_connect_getaddrinfo_cb(int result, struct evutil_addrinfo *ai, void *arg) { struct bufferevent *bev = arg; struct bufferevent_private *bev_p = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); int r; BEV_LOCK(bev); bufferevent_unsuspend_write_(bev, BEV_SUSPEND_LOOKUP); bufferevent_unsuspend_read_(bev, BEV_SUSPEND_LOOKUP); bev_p->dns_request = NULL; if (result == EVUTIL_EAI_CANCEL) { bev_p->dns_error = result; bufferevent_decref_and_unlock_(bev); return; } if (result != 0) { bev_p->dns_error = result; bufferevent_run_eventcb_(bev, BEV_EVENT_ERROR, 0); bufferevent_decref_and_unlock_(bev); if (ai) evutil_freeaddrinfo(ai); return; } /* XXX use the other addrinfos? */ /* XXX use this return value */ bufferevent_socket_set_conn_address(bev_p, ai->ai_addr, (int)ai->ai_addrlen); r = bufferevent_socket_connect(bev, ai->ai_addr, (int)ai->ai_addrlen); (void)r; bufferevent_decref_and_unlock_(bev); evutil_freeaddrinfo(ai); } int bufferevent_socket_connect_hostname(struct bufferevent *bev, struct evdns_base *evdns_base, int family, const char *hostname, int port) { char portbuf[10]; struct evutil_addrinfo hint; struct bufferevent_private *bev_p = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); if (family != AF_INET && family != AF_INET6 && family != AF_UNSPEC) return -1; if (port < 1 || port > 65535) return -1; memset(&hint, 0, sizeof(hint)); hint.ai_family = family; hint.ai_protocol = IPPROTO_TCP; hint.ai_socktype = SOCK_STREAM; evutil_snprintf(portbuf, sizeof(portbuf), "%d", port); BEV_LOCK(bev); bev_p->dns_error = 0; bufferevent_suspend_write_(bev, BEV_SUSPEND_LOOKUP); bufferevent_suspend_read_(bev, BEV_SUSPEND_LOOKUP); bufferevent_incref_(bev); bev_p->dns_request = evutil_getaddrinfo_async_(evdns_base, hostname, portbuf, &hint, bufferevent_connect_getaddrinfo_cb, bev); BEV_UNLOCK(bev); return 0; } int bufferevent_socket_get_dns_error(struct bufferevent *bev) { int rv; struct bufferevent_private *bev_p = EVUTIL_UPCAST(bev, struct bufferevent_private, bev); BEV_LOCK(bev); rv = bev_p->dns_error; BEV_UNLOCK(bev); return rv; } /* * Create a new buffered event object. * * The read callback is invoked whenever we read new data. * The write callback is invoked whenever the output buffer is drained. * The error callback is invoked on a write/read error or on EOF. * * Both read and write callbacks maybe NULL. The error callback is not * allowed to be NULL and have to be provided always. */ struct bufferevent * bufferevent_new(evutil_socket_t fd, bufferevent_data_cb readcb, bufferevent_data_cb writecb, bufferevent_event_cb eventcb, void *cbarg) { struct bufferevent *bufev; if (!(bufev = bufferevent_socket_new(NULL, fd, 0))) return NULL; bufferevent_setcb(bufev, readcb, writecb, eventcb, cbarg); return bufev; } static int be_socket_enable(struct bufferevent *bufev, short event) { if (event & EV_READ && bufferevent_add_event_(&bufev->ev_read, &bufev->timeout_read) == -1) return -1; if (event & EV_WRITE && bufferevent_add_event_(&bufev->ev_write, &bufev->timeout_write) == -1) return -1; return 0; } static int be_socket_disable(struct bufferevent *bufev, short event) { struct bufferevent_private *bufev_p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); if (event & EV_READ) { if (event_del(&bufev->ev_read) == -1) return -1; } /* Don't actually disable the write if we are trying to connect. */ if ((event & EV_WRITE) && ! bufev_p->connecting) { if (event_del(&bufev->ev_write) == -1) return -1; } return 0; } static void be_socket_destruct(struct bufferevent *bufev) { struct bufferevent_private *bufev_p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); evutil_socket_t fd; EVUTIL_ASSERT(bufev->be_ops == &bufferevent_ops_socket); fd = event_get_fd(&bufev->ev_read); if ((bufev_p->options & BEV_OPT_CLOSE_ON_FREE) && fd >= 0) EVUTIL_CLOSESOCKET(fd); evutil_getaddrinfo_cancel_async_(bufev_p->dns_request); } static int be_socket_flush(struct bufferevent *bev, short iotype, enum bufferevent_flush_mode mode) { return 0; } static void be_socket_setfd(struct bufferevent *bufev, evutil_socket_t fd) { struct bufferevent_private *bufev_p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); EVUTIL_ASSERT(bufev->be_ops == &bufferevent_ops_socket); event_del(&bufev->ev_read); event_del(&bufev->ev_write); evbuffer_unfreeze(bufev->input, 0); evbuffer_unfreeze(bufev->output, 1); event_assign(&bufev->ev_read, bufev->ev_base, fd, EV_READ|EV_PERSIST|EV_FINALIZE, bufferevent_readcb, bufev); event_assign(&bufev->ev_write, bufev->ev_base, fd, EV_WRITE|EV_PERSIST|EV_FINALIZE, bufferevent_writecb, bufev); if (fd >= 0) bufferevent_enable(bufev, bufev->enabled); evutil_getaddrinfo_cancel_async_(bufev_p->dns_request); BEV_UNLOCK(bufev); } /* XXXX Should non-socket bufferevents support this? */ int bufferevent_priority_set(struct bufferevent *bufev, int priority) { int r = -1; struct bufferevent_private *bufev_p = EVUTIL_UPCAST(bufev, struct bufferevent_private, bev); BEV_LOCK(bufev); if (bufev->be_ops != &bufferevent_ops_socket) goto done; if (event_priority_set(&bufev->ev_read, priority) == -1) goto done; if (event_priority_set(&bufev->ev_write, priority) == -1) goto done; event_deferred_cb_set_priority_(&bufev_p->deferred, priority); r = 0; done: BEV_UNLOCK(bufev); return r; } /* XXXX Should non-socket bufferevents support this? */ int bufferevent_base_set(struct event_base *base, struct bufferevent *bufev) { int res = -1; BEV_LOCK(bufev); if (bufev->be_ops != &bufferevent_ops_socket) goto done; bufev->ev_base = base; res = event_base_set(base, &bufev->ev_read); if (res == -1) goto done; res = event_base_set(base, &bufev->ev_write); done: BEV_UNLOCK(bufev); return res; } static int be_socket_ctrl(struct bufferevent *bev, enum bufferevent_ctrl_op op, union bufferevent_ctrl_data *data) { switch (op) { case BEV_CTRL_SET_FD: be_socket_setfd(bev, data->fd); return 0; case BEV_CTRL_GET_FD: data->fd = event_get_fd(&bev->ev_read); return 0; case BEV_CTRL_GET_UNDERLYING: case BEV_CTRL_CANCEL_ALL: default: return -1; } } ����������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/changelist-internal.h����������������������������������������0000664�0000000�0000000�00000011023�15270523712�0024077�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef CHANGELIST_INTERNAL_H_INCLUDED_ #define CHANGELIST_INTERNAL_H_INCLUDED_ /* A "changelist" is a list of all the fd status changes that should be made between calls to the backend's dispatch function. There are a few reasons that a backend would want to queue changes like this rather than processing them immediately. 1) Sometimes applications will add and delete the same event more than once between calls to dispatch. Processing these changes immediately is needless, and potentially expensive (especially if we're on a system that makes one syscall per changed event). 2) Sometimes we can coalesce multiple changes on the same fd into a single syscall if we know about them in advance. For example, epoll can do an add and a delete at the same time, but only if we have found out about both of them before we tell epoll. 3) Sometimes adding an event that we immediately delete can cause unintended consequences: in kqueue, this makes pending events get reported spuriously. */ #include "event2/util.h" /** Represents a */ struct event_change { /** The fd or signal whose events are to be changed */ evutil_socket_t fd; /* The events that were enabled on the fd before any of these changes were made. May include EV_READ or EV_WRITE. */ short old_events; /* The changes that we want to make in reading and writing on this fd. * If this is a signal, then read_change has EV_CHANGE_SIGNAL set, * and write_change is unused. */ ev_uint8_t read_change; ev_uint8_t write_change; ev_uint8_t close_change; }; /* Flags for read_change and write_change. */ /* If set, add the event. */ #define EV_CHANGE_ADD 0x01 /* If set, delete the event. Exclusive with EV_CHANGE_ADD */ #define EV_CHANGE_DEL 0x02 /* If set, this event refers a signal, not an fd. */ #define EV_CHANGE_SIGNAL EV_SIGNAL /* Set for persistent events. Currently not used. */ #define EV_CHANGE_PERSIST EV_PERSIST /* Set for adding edge-triggered events. */ #define EV_CHANGE_ET EV_ET /* The value of fdinfo_size that a backend should use if it is letting * changelist handle its add and delete functions. */ #define EVENT_CHANGELIST_FDINFO_SIZE sizeof(int) /** Set up the data fields in a changelist. */ void event_changelist_init_(struct event_changelist *changelist); /** Remove every change in the changelist, and make corresponding changes * in the event maps in the base. This function is generally used right * after making all the changes in the changelist. */ void event_changelist_remove_all_(struct event_changelist *changelist, struct event_base *base); /** Free all memory held in a changelist. */ void event_changelist_freemem_(struct event_changelist *changelist); /** Implementation of eventop_add that queues the event in a changelist. */ int event_changelist_add_(struct event_base *base, evutil_socket_t fd, short old, short events, void *p); /** Implementation of eventop_del that queues the event in a changelist. */ int event_changelist_del_(struct event_base *base, evutil_socket_t fd, short old, short events, void *p); #endif �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/checkpatch.sh������������������������������������������������0000775�0000000�0000000�00000015577�15270523712�0022451�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/usr/bin/env bash # TODO: # - inline replace # - clang-format-diff replacement # - uncrustify for patches (not git refs) # - maybe integrate into travis-ci? function usage() { cat <<EOL $0 [ OPTS ] [ file-or-gitref [ ... ] ] Example: # Chech HEAD git ref $ $0 -r $ $0 -r HEAD # Check patch $ git format-patch --stdout -1 | $0 -p $ git show -1 | $0 -p # Or via regular files $ git format-patch --stdout -2 $ $0 *.patch # Over a file $ $0 -d event.c $ $0 -d < event.c # And print the whole file not only summary $ $0 -f event.c $ $0 -f < event.c OPTS: -p - treat as patch -f - treat as regular file -f - treat as regular file and print diff -r - treat as git revision (default) -C - check using clang-format (default) -U - check with uncrustify -c - config for clang-format/uncrustify -h - print this message EOL } function cfg() { [ -z "${options[cfg]}" ] || { echo "${options[cfg]}" return } local dir="$(dirname "${BASH_SOURCE[0]}")" [ "${options[clang]}" -eq 0 ] || { echo "$dir/.clang-format" return } [ "${options[uncrustify]}" -eq 0 ] || { echo "$dir/.uncrustify" return } } function abort() { local msg="$1" shift printf "$msg\n" "$@" >&2 exit 1 } function default_arg() { if [ "${options[ref]}" -eq 1 ]; then echo "HEAD" else [ ! -t 0 ] || abort "<stdin> is a tty" echo "/dev/stdin" fi } function parse_options() { options[patch]=0 options[file]=0 options[file_diff]=0 options[ref]=1 options[clang]=1 options[uncrustify]=0 options[cfg]= local OPTARG OPTIND c while getopts "pfrdCUc:h?" c; do case "$c" in p) options[patch]=1 options[ref]=0 options[file]=0 options[file_diff]=0 ;; f) options[file]=1 options[ref]=0 options[patch]=0 options[file_diff]=0 ;; r) options[ref]=1 options[file]=0 options[patch]=0 options[file_diff]=0 ;; d) options[file_diff]=1 options[file]=0 options[patch]=0 options[ref]=0 ;; C) options[clang]=1 options[uncrustify]=0 ;; U) options[uncrustify]=1 options[clang]=0 ;; c) options[cfg]="$OPTIND" ;; ?|h) usage exit 0 ;; *) usage exit 1 ;; esac done options[cfg]="$(cfg)" [ -f "${options[cfg]}" ] || \ abort "Config '%s' does not exist" "${options[cfg]}" shift $((OPTIND - 1)) args=( "$@" ) if [ ${#args[@]} -eq 0 ]; then # exit on error globally, not only in subshell default_arg > /dev/null args=( "$(default_arg)" ) fi if [ "${args[0]}" = "/dev/stdin" ]; then TMP_FILE="/tmp/libevent.checkpatch.$RANDOM" cat > "$TMP_FILE" trap "rm '$TMP_FILE'" EXIT args[0]="$TMP_FILE" fi } function diff() { command diff --color=always "$@"; } function clang_style() { local c="${options[cfg]}" echo "{ $(sed -e 's/#.*//' -e '/---/d' -e '/\.\.\./d' "$c" | tr $'\n' ,) }" } function clang_format() { clang-format --style="$(clang_style)" "$@"; } function clang_format_diff() { clang-format-diff --style="$(clang_style)" "$@"; } # for non-bare repo will work function clang_format_git() { git format-patch --stdout "$@" -1 | clang_format_diff; } function uncrustify() { command uncrustify -c "${options[cfg]}" "$@"; } function uncrustify_frag() { uncrustify -l C --frag "$@"; } function uncrustify_indent_off() { echo '/* *INDENT-OFF* */'; } function uncrustify_indent_on() { echo '/* *INDENT-ON* */'; } function git_hunk() { local ref=$1 f=$2 shift 2 git cat-file -p $ref:$f } function uncrustify_git_indent_hunk() { local start=$1 end=$2 shift 2 # Will be beatier with tee(1), but doh bash async substitution { uncrustify_indent_off; git_hunk "$@" | head -n$((start - 1)); } { uncrustify_indent_on; git_hunk "$@" | head -n$((end - 1)) | tail -n+$start; } { uncrustify_indent_off; git_hunk "$@" | tail -n+$((end + 1)); } } function strip() { local start=$1 end=$2 shift 2 # seek indent_{on,off}() let start+=2 head -n$end | tail -n+$start } function patch_ranges() { egrep -o '^@@ -[0-9]+(,[0-9]+|) \+[0-9]+(,[0-9]+|) @@' | \ cut -d' ' -f3 } function git_ranges() { local ref=$1 f=$2 shift 2 git diff -W $ref^..$ref -- $f | patch_ranges } function diff_substitute() { local f="$1" shift sed \ -e "s#^--- /dev/fd.*\$#--- a/$f#" \ -e "s#^+++ /dev/fd.*\$#+++ b/$f#" } function uncrustify_git() { local ref=$1 r f start end length shift local files=( $(git diff --name-only $ref^..$ref | egrep "\.(c|h)$") ) for f in "${files[@]}"; do local ranges=( $(git_ranges $ref "$f") ) for r in "${ranges[@]}"; do [[ ! "$r" =~ ^\+([0-9]+)(,([0-9]+)|)$ ]] && continue start=${BASH_REMATCH[1]} [ -n "${BASH_REMATCH[3]}" ] && \ length=${BASH_REMATCH[3]} || \ length=1 end=$((start + length)) echo "Range: $start:$end ($length)" >&2 diff -u \ <(uncrustify_git_indent_hunk $start $end $ref "$f" | strip $start $end) \ <(uncrustify_git_indent_hunk $start $end $ref "$f" | uncrustify_frag | strip $start $end) \ | diff_substitute "$f" done done } function uncrustify_diff() { abort "Not implemented"; } function uncrustify_file() { uncrustify -f "$@"; } function checker() { local c=$1 u=$2 shift 2 [ "${options[clang]}" -eq 0 ] || { $c "$@" return } [ "${options[uncrustify]}" -eq 0 ] || { $u "$@" return } } function check_patch() { checker clang_format_diff uncrustify_diff "$@"; } function check_file() { checker clang_format uncrustify_file "$@"; } function check_ref() { checker clang_format_git uncrustify_git "$@"; } function check_arg() { [ "${options[patch]}" -eq 0 ] || { check_patch "$@" return } [ "${options[file]}" -eq 0 ] || { check_file "$@" return } [ "${options[file_diff]}" -eq 0 ] || { diff -u "$@" <(check_file "$@") | diff_substitute "$@" return } [ "${options[ref]}" -eq 0 ] || { check_ref "$@" return } } function main() { local a for a in "${args}"; do check_arg "$a" done } declare -A options parse_options "$@" main "$@" | less -FRSX ���������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/�������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021056�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/AddCompilerFlags.cmake���������������������������������0000664�0000000�0000000�00000000520�15270523712�0025215�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������include(CheckCCompilerFlag) macro(add_compiler_flags) foreach(flag ${ARGN}) string(REGEX REPLACE "[-.+/:= ]" "_" _flag_esc "${flag}") check_c_compiler_flag("${flag}" check_c_compiler_flag_${_flag_esc}) if (check_c_compiler_flag_${_flag_esc}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${flag}") endif() endforeach() endmacro() ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/COPYING-CMAKE-SCRIPTS����������������������������������0000664�0000000�0000000�00000002456�15270523712�0024063�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckFileOffsetBits.c����������������������������������0000664�0000000�0000000�00000000447�15270523712�0025035�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#include <sys/types.h> #define KB ((off_t)1024) #define MB ((off_t)1024 * KB) #define GB ((off_t)1024 * MB) #define TB ((off_t)1024 * GB) int t2[(((64 * GB -1) % 671088649) == 268434537) && (((TB - (64 * GB -1) + 255) % 1792151290) == 305159546)? 1: -1]; int main() { ; return 0; } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckFileOffsetBits.cmake������������������������������0000664�0000000�0000000�00000003667�15270523712�0025702�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# - Check if _FILE_OFFSET_BITS macro needed for large files # CHECK_FILE_OFFSET_BITS () # # The following variables may be set before calling this macro to # modify the way the check is run: # # CMAKE_REQUIRED_FLAGS = string of compile command line flags # CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar) # CMAKE_REQUIRED_INCLUDES = list of include directories # Copyright (c) 2009, Michihiro NAKAJIMA # # Redistribution and use is allowed according to the terms of the BSD license. # For details see the accompanying COPYING-CMAKE-SCRIPTS file. #INCLUDE(CheckCSourceCompiles) GET_FILENAME_COMPONENT(_selfdir_CheckFileOffsetBits "${CMAKE_CURRENT_LIST_FILE}" PATH) MACRO (CHECK_FILE_OFFSET_BITS) IF(NOT DEFINED _FILE_OFFSET_BITS) MESSAGE(STATUS "Cheking _FILE_OFFSET_BITS for large files") TRY_COMPILE(__WITHOUT_FILE_OFFSET_BITS_64 ${CMAKE_CURRENT_BINARY_DIR} ${_selfdir_CheckFileOffsetBits}/CheckFileOffsetBits.c COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}) IF(NOT __WITHOUT_FILE_OFFSET_BITS_64) TRY_COMPILE(__WITH_FILE_OFFSET_BITS_64 ${CMAKE_CURRENT_BINARY_DIR} ${_selfdir_CheckFileOffsetBits}/CheckFileOffsetBits.c COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} -D_FILE_OFFSET_BITS=64) ENDIF(NOT __WITHOUT_FILE_OFFSET_BITS_64) IF(NOT __WITHOUT_FILE_OFFSET_BITS_64 AND __WITH_FILE_OFFSET_BITS_64) SET(_FILE_OFFSET_BITS 64 CACHE INTERNAL "_FILE_OFFSET_BITS macro needed for large files") MESSAGE(STATUS "Cheking _FILE_OFFSET_BITS for large files - needed") ELSE(NOT __WITHOUT_FILE_OFFSET_BITS_64 AND __WITH_FILE_OFFSET_BITS_64) SET(_FILE_OFFSET_BITS "" CACHE INTERNAL "_FILE_OFFSET_BITS macro needed for large files") MESSAGE(STATUS "Cheking _FILE_OFFSET_BITS for large files - not needed") ENDIF(NOT __WITHOUT_FILE_OFFSET_BITS_64 AND __WITH_FILE_OFFSET_BITS_64) ENDIF(NOT DEFINED _FILE_OFFSET_BITS) ENDMACRO (CHECK_FILE_OFFSET_BITS) �������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckFunctionExistsEx.c��������������������������������0000664�0000000�0000000�00000000723�15270523712�0025444�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#ifdef CHECK_FUNCTION_EXISTS #ifndef _WIN32 char CHECK_FUNCTION_EXISTS(); #endif #ifdef __CLASSIC_C__ int main(){ int ac; char*av[]; #else int main(int ac, char*av[]){ #endif #ifdef _WIN32 void * p = &CHECK_FUNCTION_EXISTS; #else CHECK_FUNCTION_EXISTS(); #endif if(ac > 1000) { return *av[0]; } return 0; } #else /* CHECK_FUNCTION_EXISTS */ # error "CHECK_FUNCTION_EXISTS has to specify the function" #endif /* CHECK_FUNCTION_EXISTS */ ���������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckFunctionExistsEx.cmake����������������������������0000664�0000000�0000000�00000006203�15270523712�0026301�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# - Check if a C function can be linked # CHECK_FUNCTION_EXISTS(<function> <variable>) # # Check that the <function> is provided by libraries on the system and # store the result in a <variable>. This does not verify that any # system header file declares the function, only that it can be found # at link time (considure using CheckSymbolExists). # # The following variables may be set before calling this macro to # modify the way the check is run: # # CMAKE_REQUIRED_FLAGS = string of compile command line flags # CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar) # CMAKE_REQUIRED_INCLUDES = list of include directories # CMAKE_REQUIRED_LIBRARIES = list of libraries to link #============================================================================= # Copyright 2002-2011 Kitware, Inc. # # Distributed under the OSI-approved BSD License (the "License"); # see accompanying file Copyright.txt for details. # # This software is distributed WITHOUT ANY WARRANTY; without even the # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the License for more information. #============================================================================= # (To distribute this file outside of CMake, substitute the full # License text for the above reference.) MACRO(CHECK_FUNCTION_EXISTS_EX FUNCTION VARIABLE) IF(${VARIABLE} MATCHES "^${VARIABLE}$") SET(MACRO_CHECK_FUNCTION_DEFINITIONS "-DCHECK_FUNCTION_EXISTS=${FUNCTION} ${CMAKE_REQUIRED_FLAGS}") MESSAGE(STATUS "Looking for ${FUNCTION}") IF(CMAKE_REQUIRED_LIBRARIES) SET(CHECK_FUNCTION_EXISTS_ADD_LIBRARIES "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}") ELSE(CMAKE_REQUIRED_LIBRARIES) SET(CHECK_FUNCTION_EXISTS_ADD_LIBRARIES) ENDIF(CMAKE_REQUIRED_LIBRARIES) IF(CMAKE_REQUIRED_INCLUDES) SET(CHECK_FUNCTION_EXISTS_ADD_INCLUDES "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}") ELSE(CMAKE_REQUIRED_INCLUDES) SET(CHECK_FUNCTION_EXISTS_ADD_INCLUDES) ENDIF(CMAKE_REQUIRED_INCLUDES) TRY_COMPILE(${VARIABLE} ${CMAKE_BINARY_DIR} ${PROJECT_SOURCE_DIR}/cmake/CheckFunctionExistsEx.c COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS} "${CHECK_FUNCTION_EXISTS_ADD_LIBRARIES}" "${CHECK_FUNCTION_EXISTS_ADD_INCLUDES}" OUTPUT_VARIABLE OUTPUT) IF(${VARIABLE}) SET(${VARIABLE} 1 CACHE INTERNAL "Have function ${FUNCTION}") MESSAGE(STATUS "Looking for ${FUNCTION} - found") FILE(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log "Determining if the function ${FUNCTION} exists passed with the following output:\n" "${OUTPUT}\n\n") ELSE(${VARIABLE}) MESSAGE(STATUS "Looking for ${FUNCTION} - not found") SET(${VARIABLE} "" CACHE INTERNAL "Have function ${FUNCTION}") FILE(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log "Determining if the function ${FUNCTION} exists failed with the following output:\n" "${OUTPUT}\n\n") ENDIF(${VARIABLE}) ENDIF() ENDMACRO(CHECK_FUNCTION_EXISTS_EX) ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckFunctionKeywords.cmake����������������������������0000664�0000000�0000000�00000001041�15270523712�0026327�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������include(CheckCSourceCompiles) macro(check_function_keywords _wordlist) set(${_result} "") foreach(flag ${_wordlist}) string(REGEX REPLACE "[-+/ ()]" "_" flagname "${flag}") string(TOUPPER "${flagname}" flagname) set(have_flag "HAVE_${flagname}") check_c_source_compiles("${flag} void func(); void func() { } int main() { func(); return 0; }" ${have_flag}) if(${have_flag} AND NOT ${_result}) set(${_result} "${flag}") endif(${have_flag} AND NOT ${_result}) endforeach(flag) endmacro(check_function_keywords) �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckPrototypeDefinition.c.in��������������������������0000664�0000000�0000000�00000001002�15270523712�0026574�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������@CHECK_PROTOTYPE_DEFINITION_HEADER@ static void cmakeRequireSymbol(int dummy, ...) { (void) dummy; } static void checkSymbol(void) { #ifndef @CHECK_PROTOTYPE_DEFINITION_SYMBOL@ cmakeRequireSymbol(0, &@CHECK_PROTOTYPE_DEFINITION_SYMBOL@); #endif } @CHECK_PROTOTYPE_DEFINITION_PROTO@ { return @CHECK_PROTOTYPE_DEFINITION_RETURN@; } #ifdef __CLASSIC_C__ int main() { int ac; char*av[]; #else int main(int ac, char *av[]) { #endif checkSymbol(); if (ac > 1000) { return *av[0]; } return 0; } ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckPrototypeDefinition.cmake�������������������������0000664�0000000�0000000�00000007315�15270523712�0027042�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# - Check if the protoype we expect is correct. # check_prototype_definition(FUNCTION PROTOTYPE RETURN HEADER VARIABLE) # # FUNCTION - The name of the function (used to check if prototype exists) # PROTOTYPE- The prototype to check. # RETURN - The return value of the function. # HEADER - The header files required. # VARIABLE - The variable to store the result. # # Example: # # check_prototype_definition(getpwent_r # "struct passwd *getpwent_r(struct passwd *src, char *buf, int buflen)" # "NULL" # "unistd.h;pwd.h" # SOLARIS_GETPWENT_R) # # The following variables may be set before calling this macro to # modify the way the check is run: # # CMAKE_REQUIRED_FLAGS = string of compile command line flags # CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar) # CMAKE_REQUIRED_INCLUDES = list of include directories # CMAKE_REQUIRED_LIBRARIES = list of libraries to link function(CHECK_PROTOTYPE_DEFINITION _FUNCTION _PROTOTYPE _RETURN _HEADER _VARIABLE) if (${_VARIABLE} MATCHES "^${_VARIABLE}$") set(CHECK_PROTOTYPE_DEFINITION_CONTENT "/* */\n") set(CHECK_PROTOTYPE_DEFINITION_FLAGS ${CMAKE_REQUIRED_FLAGS}) if (CMAKE_REQUIRED_LIBRARIES) set(CHECK_PROTOTYPE_DEFINITION_LIBS "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}") else(CMAKE_REQUIRED_LIBRARIES) set(CHECK_PROTOTYPE_DEFINITION_LIBS) endif(CMAKE_REQUIRED_LIBRARIES) if (CMAKE_REQUIRED_INCLUDES) set(CMAKE_SYMBOL_EXISTS_INCLUDES "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}") else(CMAKE_REQUIRED_INCLUDES) set(CMAKE_SYMBOL_EXISTS_INCLUDES) endif(CMAKE_REQUIRED_INCLUDES) foreach(_FILE ${_HEADER}) set(CHECK_PROTOTYPE_DEFINITION_HEADER "${CHECK_PROTOTYPE_DEFINITION_HEADER}#include <${_FILE}>\n") endforeach(_FILE) set(CHECK_PROTOTYPE_DEFINITION_SYMBOL ${_FUNCTION}) set(CHECK_PROTOTYPE_DEFINITION_PROTO ${_PROTOTYPE}) set(CHECK_PROTOTYPE_DEFINITION_RETURN ${_RETURN}) configure_file("${PROJECT_SOURCE_DIR}/cmake/CheckPrototypeDefinition.c.in" "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckPrototypeDefinition.c" @ONLY) file(READ ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckPrototypeDefinition.c _SOURCE) try_compile(${_VARIABLE} ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckPrototypeDefinition.c COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${CHECK_PROTOTYPE_DEFINITION_FLAGS} "${CHECK_PROTOTYPE_DEFINITION_LIBS}" "${CMAKE_SYMBOL_EXISTS_INCLUDES}" OUTPUT_VARIABLE OUTPUT) if (${_VARIABLE}) set(${_VARIABLE} 1 CACHE INTERNAL "Have correct prototype for ${_FUNCTION}") message(STATUS "Checking prototype ${_FUNCTION} for ${_VARIABLE} - True") file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log "Determining if the prototype ${_FUNCTION} exists for ${_VARIABLE} passed with the following output:\n" "${OUTPUT}\n\n") else (${_VARIABLE}) message(STATUS "Checking prototype ${_FUNCTION} for ${_VARIABLE} - False") set(${_VARIABLE} 0 CACHE INTERNAL "Have correct prototype for ${_FUNCTION}") file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log "Determining if the prototype ${_FUNCTION} exists for ${_VARIABLE} failed with the following output:\n" "${OUTPUT}\n\n${_SOURCE}\n\n") endif (${_VARIABLE}) endif() endfunction(CHECK_PROTOTYPE_DEFINITION) �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckWaitpidSupportWNOWAIT.cmake�����������������������0000664�0000000�0000000�00000000506�15270523712�0027066�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������include(CheckCSourceRuns) check_c_source_runs( " #include <unistd.h> #include <sys/types.h> #include <sys/wait.h> #include <stdlib.h> int main(int argc, char** argv) { pid_t pid; int status; if ((pid = fork()) == 0) _exit(0); _exit(waitpid(pid, &status, WNOWAIT) == -1); }" EVENT__HAVE_WAITPID_WITH_WNOWAIT) ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CheckWorkingKqueue.cmake�������������������������������0000664�0000000�0000000�00000001674�15270523712�0025626�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������include(CheckCSourceRuns) check_c_source_runs( " #include <sys/types.h> #include <sys/time.h> #include <sys/event.h> #include <stdio.h> #include <unistd.h> #include <fcntl.h> int main(int argc, char **argv) { int kq; int n; int fd[2]; struct kevent ev; struct timespec ts; char buf[8000]; if (pipe(fd) == -1) exit(1); if (fcntl(fd[1], F_SETFL, O_NONBLOCK) == -1) exit(1); while ((n = write(fd[1], buf, sizeof(buf))) == sizeof(buf)) ; if ((kq = kqueue()) == -1) exit(1); memset(&ev, 0, sizeof(ev)); ev.ident = fd[1]; ev.filter = EVFILT_WRITE; ev.flags = EV_ADD | EV_ENABLE; n = kevent(kq, &ev, 1, NULL, 0, NULL); if (n == -1) exit(1); read(fd[0], buf, sizeof(buf)); ts.tv_sec = 0; ts.tv_nsec = 0; n = kevent(kq, NULL, 0, &ev, 1, &ts); if (n == -1 || n == 0) exit(1); exit(0); } " EVENT__HAVE_WORKING_KQUEUE)��������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/CodeCoverage.cmake�������������������������������������0000664�0000000�0000000�00000014305�15270523712�0024411�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# # Boost Software License - Version 1.0 - August 17th, 2003 # # Permission is hereby granted, free of charge, to any person or organization # obtaining a copy of the software and accompanying documentation covered by # this license (the "Software") to use, reproduce, display, distribute, # execute, and transmit the Software, and to prepare derivative works of the # Software, and to permit third-parties to whom the Software is furnished to # do so, all subject to the following: # # The copyright notices in the Software and this entire statement, including # the above license grant, this restriction and the following disclaimer, # must be included in all copies of the Software, in whole or in part, and # all derivative works of the Software, unless such copies or derivative # works are solely in the form of machine-executable object code generated by # a source language processor. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT # SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE # FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. # # 2012-01-31, Lars Bilke # - Enable Code Coverage # # 2013-09-17, Joakim Söderberg # - Added support for Clang. # - Some additional usage instructions. # # 2016-11-02, Azat Khuzhin # - Adopt for C compiler only (libevent) # # USAGE: # 1. Copy this file into your cmake modules path. # # 2. Add the following line to your CMakeLists.txt: # INCLUDE(CodeCoverage) # # 3. Set compiler flags to turn off optimization and enable coverage: # SET(CMAKE_CXX_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") # SET(CMAKE_C_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") # # 3. Use the function SETUP_TARGET_FOR_COVERAGE to create a custom make target # which runs your test executable and produces a lcov code coverage report: # Example: # SETUP_TARGET_FOR_COVERAGE( # my_coverage_target # Name for custom target. # test_driver # Name of the test driver executable that runs the tests. # # NOTE! This should always have a ZERO as exit code # # otherwise the coverage generation will not complete. # coverage # Name of output directory. # ) # # 4. Build a Debug build: # cmake -DCMAKE_BUILD_TYPE=Debug .. # make # make my_coverage_target # # # Check prereqs FIND_PROGRAM( GCOV_PATH gcov ) FIND_PROGRAM( LCOV_PATH lcov ) FIND_PROGRAM( GENHTML_PATH genhtml ) FIND_PROGRAM( GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/tests) IF(NOT GCOV_PATH) MESSAGE(FATAL_ERROR "gcov not found! Aborting...") ENDIF() # NOT GCOV_PATH IF(NOT CMAKE_COMPILER_IS_GNUCC) # Clang version 3.0.0 and greater now supports gcov as well. MESSAGE(WARNING "Compiler is not GNU gcc! Clang Version 3.0.0 and greater supports gcov as well, but older versions don't.") IF(NOT "${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") MESSAGE(FATAL_ERROR "Compiler is not GNU gcc! Aborting...") ENDIF() ENDIF() # NOT CMAKE_COMPILER_IS_GNUCC IF ( NOT CMAKE_BUILD_TYPE STREQUAL "Debug" ) MESSAGE( WARNING "Code coverage results with an optimized (non-Debug) build may be misleading" ) ENDIF() # NOT CMAKE_BUILD_TYPE STREQUAL "Debug" # Param _targetname The name of new the custom make target # Param _testrunner The name of the target which runs the tests. # MUST return ZERO always, even on errors. # If not, no coverage report will be created! # Param _outputname lcov output is generated as _outputname.info # HTML report is generated in _outputname/index.html # Optional fourth parameter is passed as arguments to _testrunner # Pass them in list form, e.g.: "-j;2" for -j 2 FUNCTION(SETUP_TARGET_FOR_COVERAGE _targetname _testrunner _outputname) IF(NOT LCOV_PATH) MESSAGE(FATAL_ERROR "lcov not found! Aborting...") ENDIF() # NOT LCOV_PATH IF(NOT GENHTML_PATH) MESSAGE(FATAL_ERROR "genhtml not found! Aborting...") ENDIF() # NOT GENHTML_PATH # Setup target ADD_CUSTOM_TARGET(${_targetname} # Cleanup lcov ${LCOV_PATH} --directory . --zerocounters # Run tests COMMAND ${_testrunner} ${ARGV3} # Capturing lcov counters and generating report COMMAND ${LCOV_PATH} --directory . --capture --output-file ${_outputname}.info COMMAND ${LCOV_PATH} --remove ${_outputname}.info 'tests/*' '/usr/*' --output-file ${_outputname}.info.cleaned COMMAND ${GENHTML_PATH} -o ${_outputname} ${_outputname}.info.cleaned COMMAND ${CMAKE_COMMAND} -E remove ${_outputname}.info ${_outputname}.info.cleaned WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Resetting code coverage counters to zero.\nProcessing code coverage counters and generating report." ) # Show info where to find the report ADD_CUSTOM_COMMAND(TARGET ${_targetname} POST_BUILD COMMAND ; COMMENT "Open ./${_outputname}/index.html in your browser to view the coverage report." ) ENDFUNCTION() # SETUP_TARGET_FOR_COVERAGE # Param _targetname The name of new the custom make target # Param _testrunner The name of the target which runs the tests # Param _outputname cobertura output is generated as _outputname.xml # Optional fourth parameter is passed as arguments to _testrunner # Pass them in list form, e.g.: "-j;2" for -j 2 FUNCTION(SETUP_TARGET_FOR_COVERAGE_COBERTURA _targetname _testrunner _outputname) IF(NOT PYTHON_EXECUTABLE) MESSAGE(FATAL_ERROR "Python not found! Aborting...") ENDIF() # NOT PYTHON_EXECUTABLE IF(NOT GCOVR_PATH) MESSAGE(FATAL_ERROR "gcovr not found! Aborting...") ENDIF() # NOT GCOVR_PATH ADD_CUSTOM_TARGET(${_targetname} # Run tests ${_testrunner} ${ARGV3} # Running gcovr COMMAND ${GCOVR_PATH} -x -r ${CMAKE_SOURCE_DIR} -e '${CMAKE_SOURCE_DIR}/tests/' -o ${_outputname}.xml WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Running gcovr to produce Cobertura code coverage report." ) # Show info where to find the report ADD_CUSTOM_COMMAND(TARGET ${_targetname} POST_BUILD COMMAND ; COMMENT "Cobertura code coverage report saved in ${_outputname}.xml." ) ENDFUNCTION() # SETUP_TARGET_FOR_COVERAGE_COBERTURA ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/Copyright.txt������������������������������������������0000664�0000000�0000000�00000005137�15270523712�0023575�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������CMake - Cross Platform Makefile Generator Copyright 2000-2013 Kitware, Inc. Copyright 2000-2011 Insight Software Consortium All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the names of Kitware, Inc., the Insight Software Consortium, nor the names of their contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ The above copyright and license notice applies to distributions of CMake in source and binary form. Some source files contain additional notices of original copyright by their contributors; see each source for details. Third-party software packages supplied with CMake under compatible licenses provide their own copyright notices documented in corresponding subdirectories. ------------------------------------------------------------------------------ CMake was initially developed by Kitware with the following sponsorship: * National Library of Medicine at the National Institutes of Health as part of the Insight Segmentation and Registration Toolkit (ITK). * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel Visualization Initiative. * National Alliance for Medical Image Computing (NAMIC) is funded by the National Institutes of Health through the NIH Roadmap for Medical Research, Grant U54 EB005149. * Kitware, Inc.���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/FindGit.cmake������������������������������������������0000664�0000000�0000000�00000002664�15270523712�0023414�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# The module defines the following variables: # GIT_EXECUTABLE - path to git command line client # GIT_FOUND - true if the command line client was found # Example usage: # find_package(Git) # if(GIT_FOUND) # message("git found: ${GIT_EXECUTABLE}") # endif() #============================================================================= # Copyright 2010 Kitware, Inc. # # Distributed under the OSI-approved BSD License (the "License"); # see accompanying file Copyright.txt for details. # # This software is distributed WITHOUT ANY WARRANTY; without even the # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the License for more information. #============================================================================= # (To distributed this file outside of CMake, substitute the full # License text for the above reference.) # Look for 'git' or 'eg' (easy git) set(git_names git eg) # Prefer .cmd variants on Windows unless running in a Makefile # in the MSYS shell. if(WIN32) if(NOT CMAKE_GENERATOR MATCHES "MSYS") set(git_names git.cmd git eg.cmd eg) endif() endif() find_program(GIT_EXECUTABLE NAMES ${git_names} DOC "git command line client") mark_as_advanced(GIT_EXECUTABLE) # Handle the QUIETLY and REQUIRED arguments and set GIT_FOUND to TRUE if # all listed variables are TRUE include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Git DEFAULT_MSG GIT_EXECUTABLE) ����������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/LibeventConfig.cmake.in��������������������������������0000664�0000000�0000000�00000001167�15270523712�0025370�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# - Config file for the Libevent package # It defines the following variables # LIBEVENT_INCLUDE_DIRS - include directories for FooBar # LIBEVENT_LIBRARIES - libraries to link against # Get the path of the current file. get_filename_component(LIBEVENT_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) # Set the include directories. set(LIBEVENT_INCLUDE_DIRS "@EVENT_INSTALL_INCLUDE_DIR@") # Include the project Targets file, this contains definitions for IMPORTED targets. include(${LIBEVENT_CMAKE_DIR}/LibeventTargets.cmake) # IMPORTED targets from LibeventTargets.cmake set(LIBEVENT_LIBRARIES event event_core event_extra) ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/LibeventConfigBuildTree.cmake.in�����������������������0000664�0000000�0000000�00000001161�15270523712�0027162�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# - Config file for the Libevent package # It defines the following variables # LIBEVENT_INCLUDE_DIRS - include directories for FooBar # LIBEVENT_LIBRARIES - libraries to link against # Get the path of the current file. get_filename_component(LIBEVENT_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) # Set the include directories. set(LIBEVENT_INCLUDE_DIRS "@EVENT__INCLUDE_DIRS@") # Include the project Targets file, this contains definitions for IMPORTED targets. include(${LIBEVENT_CMAKE_DIR}/LibeventTargets.cmake) # IMPORTED targets from LibeventTargets.cmake set(LIBEVENT_LIBRARIES event event_core event_extra) ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/LibeventConfigVersion.cmake.in�������������������������0000664�0000000�0000000�00000000602�15270523712�0026727�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������set(PACKAGE_VERSION "@EVENT_PACKAGE_VERSION@") # Check whether the requested PACKAGE_FIND_VERSION is compatible if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") set(PACKAGE_VERSION_COMPATIBLE FALSE) else() set(PACKAGE_VERSION_COMPATIBLE TRUE) if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") set(PACKAGE_VERSION_EXACT TRUE) endif() endif() ������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/cmake/VersionViaGit.cmake������������������������������������0000664�0000000�0000000�00000003353�15270523712�0024615�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������# This module defines the following variables utilizing # git to determine the parent tag. And if found the macro # will attempt to parse them in the github tag fomat # # Usful for auto-versionin in ou CMakeLists # # EVENT_GIT___VERSION_FOUND - Version variables foud # EVENT_GIT___VERSION_MAJOR - Major version. # EVENT_GIT___VERSION_MINOR - Minor version # EVENT_GIT___VERSION_STAGE - Stage version # # Example usage: # # event_fuzzy_version_from_git() # if (EVENT_GIT___VERSION_FOUND) # message("Libvent major=${EVENT_GIT___VERSION_MAJOR}") # message(" minor=${EVENT_GIT___VERSION_MINOR}") # message(" patch=${EVENT_GIT___VERSION_PATCH}") # message(" stage=${EVENT_GIT___VERSION_STAGE}") # endif() include(FindGit) macro(event_fuzzy_version_from_git) set(EVENT_GIT___VERSION_FOUND FALSE) # set our defaults. set(EVENT_GIT___VERSION_MAJOR 2) set(EVENT_GIT___VERSION_MINOR 1) set(EVENT_GIT___VERSION_PATCH 8) set(EVENT_GIT___VERSION_STAGE "beta") find_package(Git) if (GIT_FOUND) execute_process( COMMAND ${GIT_EXECUTABLE} describe --abbrev=0 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE GITRET OUTPUT_VARIABLE GITVERSION) if (GITRET EQUAL 0) string(REGEX REPLACE "^release-([0-9]+)\\.([0-9]+)\\.([0-9]+)-(.*)" "\\1" EVENT_GIT___VERSION_MAJOR ${GITVERSION}) string(REGEX REPLACE "^release-([0-9]+)\\.([0-9]+)\\.([0-9]+)-(.*)" "\\2" EVENT_GIT___VERSION_MINOR ${GITVERSION}) string(REGEX REPLACE "^release-([0-9]+)\\.([0-9]+)\\.([0-9]+)-(.*)" "\\3" EVENT_GIT___VERSION_PATCH ${GITVERSION}) string(REGEX REPLACE "^release-([0-9]+)\\.([0-9]+)\\.([0-9]+)-([aA-zZ]+)" "\\4" EVENT_GIT___VERSION_STAGE ${GITVERSION}) endif() endif() endmacro() �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/compat/������������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0021261�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/compat/sys/��������������������������������������������������0000775�0000000�0000000�00000000000�15270523712�0022077�5����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/compat/sys/queue.h�������������������������������������������0000664�0000000�0000000�00000040442�15270523712�0023400�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* $OpenBSD: queue.h,v 1.16 2000/09/07 19:47:59 art Exp $ */ /* $NetBSD: queue.h,v 1.11 1996/05/16 05:17:14 mycroft Exp $ */ /* * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)queue.h 8.5 (Berkeley) 8/20/94 */ #ifndef SYS_QUEUE_H__ #define SYS_QUEUE_H__ /* * This file defines five types of data structures: singly-linked lists, * lists, simple queues, tail queues, and circular queues. * * * A singly-linked list is headed by a single forward pointer. The elements * are singly linked for minimum space and pointer manipulation overhead at * the expense of O(n) removal for arbitrary elements. New elements can be * added to the list after an existing element or at the head of the list. * Elements being removed from the head of the list should use the explicit * macro for this purpose for optimum efficiency. A singly-linked list may * only be traversed in the forward direction. Singly-linked lists are ideal * for applications with large datasets and few or no removals or for * implementing a LIFO queue. * * A list is headed by a single forward pointer (or an array of forward * pointers for a hash table header). The elements are doubly linked * so that an arbitrary element can be removed without a need to * traverse the list. New elements can be added to the list before * or after an existing element or at the head of the list. A list * may only be traversed in the forward direction. * * A simple queue is headed by a pair of pointers, one the head of the * list and the other to the tail of the list. The elements are singly * linked to save space, so elements can only be removed from the * head of the list. New elements can be added to the list before or after * an existing element, at the head of the list, or at the end of the * list. A simple queue may only be traversed in the forward direction. * * A tail queue is headed by a pair of pointers, one to the head of the * list and the other to the tail of the list. The elements are doubly * linked so that an arbitrary element can be removed without a need to * traverse the list. New elements can be added to the list before or * after an existing element, at the head of the list, or at the end of * the list. A tail queue may be traversed in either direction. * * A circle queue is headed by a pair of pointers, one to the head of the * list and the other to the tail of the list. The elements are doubly * linked so that an arbitrary element can be removed without a need to * traverse the list. New elements can be added to the list before or after * an existing element, at the head of the list, or at the end of the list. * A circle queue may be traversed in either direction, but has a more * complex end of list detection. * * For details on the use of these macros, see the queue(3) manual page. */ /* * Singly-linked List definitions. */ #define SLIST_HEAD(name, type) \ struct name { \ struct type *slh_first; /* first element */ \ } #define SLIST_HEAD_INITIALIZER(head) \ { NULL } #ifndef _WIN32 #define SLIST_ENTRY(type) \ struct { \ struct type *sle_next; /* next element */ \ } #endif /* * Singly-linked List access methods. */ #define SLIST_FIRST(head) ((head)->slh_first) #define SLIST_END(head) NULL #define SLIST_EMPTY(head) (SLIST_FIRST(head) == SLIST_END(head)) #define SLIST_NEXT(elm, field) ((elm)->field.sle_next) #define SLIST_FOREACH(var, head, field) \ for((var) = SLIST_FIRST(head); \ (var) != SLIST_END(head); \ (var) = SLIST_NEXT(var, field)) /* * Singly-linked List functions. */ #define SLIST_INIT(head) { \ SLIST_FIRST(head) = SLIST_END(head); \ } #define SLIST_INSERT_AFTER(slistelm, elm, field) do { \ (elm)->field.sle_next = (slistelm)->field.sle_next; \ (slistelm)->field.sle_next = (elm); \ } while (0) #define SLIST_INSERT_HEAD(head, elm, field) do { \ (elm)->field.sle_next = (head)->slh_first; \ (head)->slh_first = (elm); \ } while (0) #define SLIST_REMOVE_HEAD(head, field) do { \ (head)->slh_first = (head)->slh_first->field.sle_next; \ } while (0) /* * List definitions. */ #define LIST_HEAD(name, type) \ struct name { \ struct type *lh_first; /* first element */ \ } #define LIST_HEAD_INITIALIZER(head) \ { NULL } #define LIST_ENTRY(type) \ struct { \ struct type *le_next; /* next element */ \ struct type **le_prev; /* address of previous next element */ \ } /* * List access methods */ #define LIST_FIRST(head) ((head)->lh_first) #define LIST_END(head) NULL #define LIST_EMPTY(head) (LIST_FIRST(head) == LIST_END(head)) #define LIST_NEXT(elm, field) ((elm)->field.le_next) #define LIST_FOREACH(var, head, field) \ for((var) = LIST_FIRST(head); \ (var)!= LIST_END(head); \ (var) = LIST_NEXT(var, field)) /* * List functions. */ #define LIST_INIT(head) do { \ LIST_FIRST(head) = LIST_END(head); \ } while (0) #define LIST_INSERT_AFTER(listelm, elm, field) do { \ if (((elm)->field.le_next = (listelm)->field.le_next) != NULL) \ (listelm)->field.le_next->field.le_prev = \ &(elm)->field.le_next; \ (listelm)->field.le_next = (elm); \ (elm)->field.le_prev = &(listelm)->field.le_next; \ } while (0) #define LIST_INSERT_BEFORE(listelm, elm, field) do { \ (elm)->field.le_prev = (listelm)->field.le_prev; \ (elm)->field.le_next = (listelm); \ *(listelm)->field.le_prev = (elm); \ (listelm)->field.le_prev = &(elm)->field.le_next; \ } while (0) #define LIST_INSERT_HEAD(head, elm, field) do { \ if (((elm)->field.le_next = (head)->lh_first) != NULL) \ (head)->lh_first->field.le_prev = &(elm)->field.le_next;\ (head)->lh_first = (elm); \ (elm)->field.le_prev = &(head)->lh_first; \ } while (0) #define LIST_REMOVE(elm, field) do { \ if ((elm)->field.le_next != NULL) \ (elm)->field.le_next->field.le_prev = \ (elm)->field.le_prev; \ *(elm)->field.le_prev = (elm)->field.le_next; \ } while (0) #define LIST_REPLACE(elm, elm2, field) do { \ if (((elm2)->field.le_next = (elm)->field.le_next) != NULL) \ (elm2)->field.le_next->field.le_prev = \ &(elm2)->field.le_next; \ (elm2)->field.le_prev = (elm)->field.le_prev; \ *(elm2)->field.le_prev = (elm2); \ } while (0) /* * Simple queue definitions. */ #define SIMPLEQ_HEAD(name, type) \ struct name { \ struct type *sqh_first; /* first element */ \ struct type **sqh_last; /* addr of last next element */ \ } #define SIMPLEQ_HEAD_INITIALIZER(head) \ { NULL, &(head).sqh_first } #define SIMPLEQ_ENTRY(type) \ struct { \ struct type *sqe_next; /* next element */ \ } /* * Simple queue access methods. */ #define SIMPLEQ_FIRST(head) ((head)->sqh_first) #define SIMPLEQ_END(head) NULL #define SIMPLEQ_EMPTY(head) (SIMPLEQ_FIRST(head) == SIMPLEQ_END(head)) #define SIMPLEQ_NEXT(elm, field) ((elm)->field.sqe_next) #define SIMPLEQ_FOREACH(var, head, field) \ for((var) = SIMPLEQ_FIRST(head); \ (var) != SIMPLEQ_END(head); \ (var) = SIMPLEQ_NEXT(var, field)) /* * Simple queue functions. */ #define SIMPLEQ_INIT(head) do { \ (head)->sqh_first = NULL; \ (head)->sqh_last = &(head)->sqh_first; \ } while (0) #define SIMPLEQ_INSERT_HEAD(head, elm, field) do { \ if (((elm)->field.sqe_next = (head)->sqh_first) == NULL) \ (head)->sqh_last = &(elm)->field.sqe_next; \ (head)->sqh_first = (elm); \ } while (0) #define SIMPLEQ_INSERT_TAIL(head, elm, field) do { \ (elm)->field.sqe_next = NULL; \ *(head)->sqh_last = (elm); \ (head)->sqh_last = &(elm)->field.sqe_next; \ } while (0) #define SIMPLEQ_INSERT_AFTER(head, listelm, elm, field) do { \ if (((elm)->field.sqe_next = (listelm)->field.sqe_next) == NULL)\ (head)->sqh_last = &(elm)->field.sqe_next; \ (listelm)->field.sqe_next = (elm); \ } while (0) #define SIMPLEQ_REMOVE_HEAD(head, elm, field) do { \ if (((head)->sqh_first = (elm)->field.sqe_next) == NULL) \ (head)->sqh_last = &(head)->sqh_first; \ } while (0) /* * Tail queue definitions. */ #define TAILQ_HEAD(name, type) \ struct name { \ struct type *tqh_first; /* first element */ \ struct type **tqh_last; /* addr of last next element */ \ } #define TAILQ_HEAD_INITIALIZER(head) \ { NULL, &(head).tqh_first } #define TAILQ_ENTRY(type) \ struct { \ struct type *tqe_next; /* next element */ \ struct type **tqe_prev; /* address of previous next element */ \ } /* * tail queue access methods */ #define TAILQ_FIRST(head) ((head)->tqh_first) #define TAILQ_END(head) NULL #define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) #define TAILQ_LAST(head, headname) \ (*(((struct headname *)((head)->tqh_last))->tqh_last)) /* XXX */ #define TAILQ_PREV(elm, headname, field) \ (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) #define TAILQ_EMPTY(head) \ (TAILQ_FIRST(head) == TAILQ_END(head)) #define TAILQ_FOREACH(var, head, field) \ for((var) = TAILQ_FIRST(head); \ (var) != TAILQ_END(head); \ (var) = TAILQ_NEXT(var, field)) #define TAILQ_FOREACH_REVERSE(var, head, headname, field) \ for((var) = TAILQ_LAST(head, headname); \ (var) != TAILQ_END(head); \ (var) = TAILQ_PREV(var, headname, field)) /* * Tail queue functions. */ #define TAILQ_INIT(head) do { \ (head)->tqh_first = NULL; \ (head)->tqh_last = &(head)->tqh_first; \ } while (0) #define TAILQ_INSERT_HEAD(head, elm, field) do { \ if (((elm)->field.tqe_next = (head)->tqh_first) != NULL) \ (head)->tqh_first->field.tqe_prev = \ &(elm)->field.tqe_next; \ else \ (head)->tqh_last = &(elm)->field.tqe_next; \ (head)->tqh_first = (elm); \ (elm)->field.tqe_prev = &(head)->tqh_first; \ } while (0) #define TAILQ_INSERT_TAIL(head, elm, field) do { \ (elm)->field.tqe_next = NULL; \ (elm)->field.tqe_prev = (head)->tqh_last; \ *(head)->tqh_last = (elm); \ (head)->tqh_last = &(elm)->field.tqe_next; \ } while (0) #define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ if (((elm)->field.tqe_next = (listelm)->field.tqe_next) != NULL)\ (elm)->field.tqe_next->field.tqe_prev = \ &(elm)->field.tqe_next; \ else \ (head)->tqh_last = &(elm)->field.tqe_next; \ (listelm)->field.tqe_next = (elm); \ (elm)->field.tqe_prev = &(listelm)->field.tqe_next; \ } while (0) #define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ (elm)->field.tqe_next = (listelm); \ *(listelm)->field.tqe_prev = (elm); \ (listelm)->field.tqe_prev = &(elm)->field.tqe_next; \ } while (0) #define TAILQ_REMOVE(head, elm, field) do { \ if (((elm)->field.tqe_next) != NULL) \ (elm)->field.tqe_next->field.tqe_prev = \ (elm)->field.tqe_prev; \ else \ (head)->tqh_last = (elm)->field.tqe_prev; \ *(elm)->field.tqe_prev = (elm)->field.tqe_next; \ } while (0) #define TAILQ_REPLACE(head, elm, elm2, field) do { \ if (((elm2)->field.tqe_next = (elm)->field.tqe_next) != NULL) \ (elm2)->field.tqe_next->field.tqe_prev = \ &(elm2)->field.tqe_next; \ else \ (head)->tqh_last = &(elm2)->field.tqe_next; \ (elm2)->field.tqe_prev = (elm)->field.tqe_prev; \ *(elm2)->field.tqe_prev = (elm2); \ } while (0) /* * Circular queue definitions. */ #define CIRCLEQ_HEAD(name, type) \ struct name { \ struct type *cqh_first; /* first element */ \ struct type *cqh_last; /* last element */ \ } #define CIRCLEQ_HEAD_INITIALIZER(head) \ { CIRCLEQ_END(&head), CIRCLEQ_END(&head) } #define CIRCLEQ_ENTRY(type) \ struct { \ struct type *cqe_next; /* next element */ \ struct type *cqe_prev; /* previous element */ \ } /* * Circular queue access methods */ #define CIRCLEQ_FIRST(head) ((head)->cqh_first) #define CIRCLEQ_LAST(head) ((head)->cqh_last) #define CIRCLEQ_END(head) ((void *)(head)) #define CIRCLEQ_NEXT(elm, field) ((elm)->field.cqe_next) #define CIRCLEQ_PREV(elm, field) ((elm)->field.cqe_prev) #define CIRCLEQ_EMPTY(head) \ (CIRCLEQ_FIRST(head) == CIRCLEQ_END(head)) #define CIRCLEQ_FOREACH(var, head, field) \ for((var) = CIRCLEQ_FIRST(head); \ (var) != CIRCLEQ_END(head); \ (var) = CIRCLEQ_NEXT(var, field)) #define CIRCLEQ_FOREACH_REVERSE(var, head, field) \ for((var) = CIRCLEQ_LAST(head); \ (var) != CIRCLEQ_END(head); \ (var) = CIRCLEQ_PREV(var, field)) /* * Circular queue functions. */ #define CIRCLEQ_INIT(head) do { \ (head)->cqh_first = CIRCLEQ_END(head); \ (head)->cqh_last = CIRCLEQ_END(head); \ } while (0) #define CIRCLEQ_INSERT_AFTER(head, listelm, elm, field) do { \ (elm)->field.cqe_next = (listelm)->field.cqe_next; \ (elm)->field.cqe_prev = (listelm); \ if ((listelm)->field.cqe_next == CIRCLEQ_END(head)) \ (head)->cqh_last = (elm); \ else \ (listelm)->field.cqe_next->field.cqe_prev = (elm); \ (listelm)->field.cqe_next = (elm); \ } while (0) #define CIRCLEQ_INSERT_BEFORE(head, listelm, elm, field) do { \ (elm)->field.cqe_next = (listelm); \ (elm)->field.cqe_prev = (listelm)->field.cqe_prev; \ if ((listelm)->field.cqe_prev == CIRCLEQ_END(head)) \ (head)->cqh_first = (elm); \ else \ (listelm)->field.cqe_prev->field.cqe_next = (elm); \ (listelm)->field.cqe_prev = (elm); \ } while (0) #define CIRCLEQ_INSERT_HEAD(head, elm, field) do { \ (elm)->field.cqe_next = (head)->cqh_first; \ (elm)->field.cqe_prev = CIRCLEQ_END(head); \ if ((head)->cqh_last == CIRCLEQ_END(head)) \ (head)->cqh_last = (elm); \ else \ (head)->cqh_first->field.cqe_prev = (elm); \ (head)->cqh_first = (elm); \ } while (0) #define CIRCLEQ_INSERT_TAIL(head, elm, field) do { \ (elm)->field.cqe_next = CIRCLEQ_END(head); \ (elm)->field.cqe_prev = (head)->cqh_last; \ if ((head)->cqh_first == CIRCLEQ_END(head)) \ (head)->cqh_first = (elm); \ else \ (head)->cqh_last->field.cqe_next = (elm); \ (head)->cqh_last = (elm); \ } while (0) #define CIRCLEQ_REMOVE(head, elm, field) do { \ if ((elm)->field.cqe_next == CIRCLEQ_END(head)) \ (head)->cqh_last = (elm)->field.cqe_prev; \ else \ (elm)->field.cqe_next->field.cqe_prev = \ (elm)->field.cqe_prev; \ if ((elm)->field.cqe_prev == CIRCLEQ_END(head)) \ (head)->cqh_first = (elm)->field.cqe_next; \ else \ (elm)->field.cqe_prev->field.cqe_next = \ (elm)->field.cqe_next; \ } while (0) #define CIRCLEQ_REPLACE(head, elm, elm2, field) do { \ if (((elm2)->field.cqe_next = (elm)->field.cqe_next) == \ CIRCLEQ_END(head)) \ (head).cqh_last = (elm2); \ else \ (elm2)->field.cqe_next->field.cqe_prev = (elm2); \ if (((elm2)->field.cqe_prev = (elm)->field.cqe_prev) == \ CIRCLEQ_END(head)) \ (head).cqh_first = (elm2); \ else \ (elm2)->field.cqe_prev->field.cqe_next = (elm2); \ } while (0) #endif /* !SYS_QUEUE_H__ */ ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/configure.ac�������������������������������������������������0000664�0000000�0000000�00000057745�15270523712�0022306�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������dnl Copyright 2000-2007 Niels Provos dnl Copyright 2007-2012 Niels Provos and Nick Mathewson dnl dnl See LICENSE for copying information. dnl dnl Original version Dug Song <dugsong@monkey.org> AC_INIT(libevent,2.1.8-stable) AC_PREREQ(2.59) AC_CONFIG_SRCDIR(event.c) AC_CONFIG_MACRO_DIR([m4]) AM_INIT_AUTOMAKE dnl AM_SILENT_RULES req. automake 1.11. [no] defaults V=1 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AC_CONFIG_HEADERS(config.h evconfig-private.h:evconfig-private.h.in) AC_DEFINE(NUMERIC_VERSION, 0x02010800, [Numeric representation of the version]) dnl Initialize prefix. if test "$prefix" = "NONE"; then prefix="/usr/local" fi dnl Try and get a full POSIX environment on obscure systems ifdef([AC_USE_SYSTEM_EXTENSIONS], [ AC_USE_SYSTEM_EXTENSIONS ], [ AC_AIX AC_GNU_SOURCE AC_MINIX ]) AC_CANONICAL_BUILD AC_CANONICAL_HOST dnl the 'build' machine is where we run configure and compile dnl the 'host' machine is where the resulting stuff runs. #case "$host_os" in # # osf5*) # CFLAGS="$CFLAGS -D_OSF_SOURCE" # ;; #esac dnl Checks for programs. AM_PROG_CC_C_O AC_PROG_INSTALL AC_PROG_LN_S # AC_PROG_MKDIR_P - $(MKDIR_P) should be defined by AM_INIT_AUTOMAKE # AC_PROG_SED is only available in Autoconf >= 2.59b; workaround for older # versions ifdef([AC_PROG_SED], [AC_PROG_SED], [ AC_CHECK_PROGS(SED, [gsed sed]) ]) AC_PROG_GCC_TRADITIONAL # We need to test for at least gcc 2.95 here, because older versions don't # have -fno-strict-aliasing AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 95) #error #endif])], have_gcc295=yes, have_gcc295=no) if test "$GCC" = "yes" ; then # Enable many gcc warnings by default... CFLAGS="$CFLAGS -Wall" # And disable the strict-aliasing optimization, since it breaks # our sockaddr-handling code in strange ways. if test x$have_gcc295 = xyes; then CFLAGS="$CFLAGS -fno-strict-aliasing" fi fi # OS X Lion started deprecating the system openssl. Let's just disable # all deprecation warnings on OS X; but do so only for gcc... if test "$GCC" = "yes" ; then case "$host_os" in darwin*) CFLAGS="$CFLAGS -Wno-deprecated-declarations" ;; esac fi AC_ARG_ENABLE(gcc-warnings, AS_HELP_STRING(--disable-gcc-warnings, disable verbose warnings with GCC)) AC_ARG_ENABLE(gcc-hardening, AS_HELP_STRING(--enable-gcc-hardening, enable compiler security checks), [if test x$enableval = xyes; then CFLAGS="$CFLAGS -D_FORTIFY_SOURCE=2 -fstack-protector-all" CFLAGS="$CFLAGS -fwrapv -fPIE -Wstack-protector" CFLAGS="$CFLAGS --param ssp-buffer-size=1" fi]) AC_ARG_ENABLE(thread-support, AS_HELP_STRING(--disable-thread-support, disable support for threading), [], [enable_thread_support=yes]) AC_ARG_ENABLE(malloc-replacement, AS_HELP_STRING(--disable-malloc-replacement, disable support for replacing the memory mgt functions), [], [enable_malloc_replacement=yes]) AC_ARG_ENABLE(openssl, AS_HELP_STRING(--disable-openssl, disable support for openssl encryption), [], [enable_openssl=yes]) AC_ARG_ENABLE(debug-mode, AS_HELP_STRING(--disable-debug-mode, disable support for running in debug mode), [], [enable_debug_mode=yes]) AC_ARG_ENABLE([libevent-install], AS_HELP_STRING([--disable-libevent-install, disable installation of libevent]), [], [enable_libevent_install=yes]) AC_ARG_ENABLE([libevent-regress], AS_HELP_STRING([--disable-libevent-regress, skip regress in make check]), [], [enable_libevent_regress=yes]) AC_ARG_ENABLE([samples], AS_HELP_STRING([--disable-samples, skip building of sample programs]), [], [enable_samples=yes]) AC_ARG_ENABLE([function-sections], AS_HELP_STRING([--enable-function-sections, make static library allow smaller binaries with --gc-sections]), [], [enable_function_sections=no]) AC_ARG_ENABLE([verbose-debug], AS_HELP_STRING([--enable-verbose-debug, verbose debug logging]), [], [enable_verbose_debug=no]) AC_ARG_ENABLE([clock-gettime], AS_HELP_STRING(--disable-clock-gettime, do not use clock_gettime even if it is available), [], [enable_clock_gettime=yes]) AC_PROG_LIBTOOL dnl Uncomment "AC_DISABLE_SHARED" to make shared libraries not get dnl built by default. You can also turn shared libs on and off from dnl the command line with --enable-shared and --disable-shared. dnl AC_DISABLE_SHARED AC_SUBST(LIBTOOL_DEPS) AM_CONDITIONAL([BUILD_SAMPLES], [test "$enable_samples" = "yes"]) AM_CONDITIONAL([BUILD_REGRESS], [test "$enable_libevent_regress" = "yes"]) dnl Checks for libraries. AC_SEARCH_LIBS([inet_ntoa], [nsl]) AC_SEARCH_LIBS([socket], [socket]) AC_SEARCH_LIBS([inet_aton], [resolv]) if test "x$enable_clock_gettime" = "xyes"; then AC_SEARCH_LIBS([clock_gettime], [rt]) AC_CHECK_FUNCS([clock_gettime]) fi AC_SEARCH_LIBS([sendfile], [sendfile]) dnl - check if the macro _WIN32 is defined on this compiler. dnl - (this is how we check for a windows compiler) AC_MSG_CHECKING(for WIN32) AC_TRY_COMPILE(, [ #ifndef _WIN32 die horribly #endif ], bwin32=true; AC_MSG_RESULT(yes), bwin32=false; AC_MSG_RESULT(no), ) dnl - check if the macro __CYGWIN__ is defined on this compiler. dnl - (this is how we check for a cygwin version of GCC) AC_MSG_CHECKING(for CYGWIN) AC_TRY_COMPILE(, [ #ifndef __CYGWIN__ die horribly #endif ], cygwin=true; AC_MSG_RESULT(yes), cygwin=false; AC_MSG_RESULT(no), ) AC_CHECK_HEADERS([zlib.h]) if test "x$ac_cv_header_zlib_h" = "xyes"; then dnl Determine if we have zlib for regression tests dnl Don't put this one in LIBS save_LIBS="$LIBS" LIBS="" ZLIB_LIBS="" have_zlib=no AC_SEARCH_LIBS([inflateEnd], [z], [have_zlib=yes ZLIB_LIBS="$LIBS" AC_DEFINE(HAVE_LIBZ, 1, [Define if the system has zlib])]) LIBS="$save_LIBS" AC_SUBST(ZLIB_LIBS) fi AM_CONDITIONAL(ZLIB_REGRESS, [test "$have_zlib" = "yes"]) dnl See if we have openssl. This doesn't go in LIBS either. if test "$bwin32" = true; then EV_LIB_WS32=-lws2_32 EV_LIB_GDI=-lgdi32 else EV_LIB_WS32= EV_LIB_GDI= fi AC_SUBST(EV_LIB_WS32) AC_SUBST(EV_LIB_GDI) AC_SUBST(OPENSSL_LIBADD) AC_SYS_LARGEFILE LIBEVENT_OPENSSL dnl Checks for header files. AC_CHECK_HEADERS([ \ arpa/inet.h \ fcntl.h \ ifaddrs.h \ mach/mach_time.h \ netdb.h \ netinet/in.h \ netinet/in6.h \ netinet/tcp.h \ poll.h \ port.h \ stdarg.h \ stddef.h \ sys/devpoll.h \ sys/epoll.h \ sys/event.h \ sys/eventfd.h \ sys/ioctl.h \ sys/mman.h \ sys/param.h \ sys/queue.h \ sys/resource.h \ sys/select.h \ sys/sendfile.h \ sys/socket.h \ sys/stat.h \ sys/time.h \ sys/timerfd.h \ sys/uio.h \ sys/wait.h \ errno.h \ ]) AC_CHECK_HEADERS(sys/sysctl.h, [], [], [ #ifdef HAVE_SYS_PARAM_H #include <sys/param.h> #endif ]) if test "x$ac_cv_header_sys_queue_h" = "xyes"; then AC_MSG_CHECKING(for TAILQ_FOREACH in sys/queue.h) AC_EGREP_CPP(yes, [ #include <sys/queue.h> #ifdef TAILQ_FOREACH yes #endif ], [AC_MSG_RESULT(yes) AC_DEFINE(HAVE_TAILQFOREACH, 1, [Define if TAILQ_FOREACH is defined in <sys/queue.h>])], AC_MSG_RESULT(no) ) fi if test "x$ac_cv_header_sys_time_h" = "xyes"; then AC_MSG_CHECKING(for timeradd in sys/time.h) AC_EGREP_CPP(yes, [ #include <sys/time.h> #ifdef timeradd yes #endif ], [ AC_DEFINE(HAVE_TIMERADD, 1, [Define if timeradd is defined in <sys/time.h>]) AC_MSG_RESULT(yes)] ,AC_MSG_RESULT(no) ) fi if test "x$ac_cv_header_sys_time_h" = "xyes"; then AC_MSG_CHECKING(for timercmp in sys/time.h) AC_EGREP_CPP(yes, [ #include <sys/time.h> #ifdef timercmp yes #endif ], [ AC_DEFINE(HAVE_TIMERCMP, 1, [Define if timercmp is defined in <sys/time.h>]) AC_MSG_RESULT(yes)] ,AC_MSG_RESULT(no) ) fi if test "x$ac_cv_header_sys_time_h" = "xyes"; then AC_MSG_CHECKING(for timerclear in sys/time.h) AC_EGREP_CPP(yes, [ #include <sys/time.h> #ifdef timerclear yes #endif ], [ AC_DEFINE(HAVE_TIMERCLEAR, 1, [Define if timerclear is defined in <sys/time.h>]) AC_MSG_RESULT(yes)] ,AC_MSG_RESULT(no) ) fi if test "x$ac_cv_header_sys_time_h" = "xyes"; then AC_MSG_CHECKING(for timerisset in sys/time.h) AC_EGREP_CPP(yes, [ #include <sys/time.h> #ifdef timerisset yes #endif ], [ AC_DEFINE(HAVE_TIMERISSET, 1, [Define if timerisset is defined in <sys/time.h>]) AC_MSG_RESULT(yes)] ,AC_MSG_RESULT(no) ) fi if test "x$ac_cv_header_sys_sysctl_h" = "xyes"; then AC_CHECK_DECLS([CTL_KERN, KERN_RANDOM, RANDOM_UUID, KERN_ARND], [], [], [[#include <sys/types.h> #include <sys/sysctl.h>]] ) fi AM_CONDITIONAL(BUILD_WIN32, test x$bwin32 = xtrue) AM_CONDITIONAL(BUILD_CYGWIN, test x$cygwin = xtrue) AM_CONDITIONAL(BUILD_WITH_NO_UNDEFINED, test x$bwin32 = xtrue || test x$cygwin = xtrue) if test x$bwin32 = xtrue; then AC_SEARCH_LIBS([getservbyname],[ws2_32]) fi dnl Checks for typedefs, structures, and compiler characteristics. AC_C_CONST AC_C_INLINE AC_HEADER_TIME dnl Checks for library functions. AC_CHECK_FUNCS([ \ accept4 \ arc4random \ arc4random_buf \ eventfd \ epoll_create1 \ fcntl \ getegid \ geteuid \ getifaddrs \ getnameinfo \ getprotobynumber \ gettimeofday \ inet_ntop \ inet_pton \ issetugid \ mach_absolute_time \ mmap \ nanosleep \ pipe \ pipe2 \ putenv \ sendfile \ setenv \ setrlimit \ sigaction \ signal \ splice \ strlcpy \ strsep \ strtok_r \ strtoll \ sysctl \ timerfd_create \ umask \ unsetenv \ usleep \ vasprintf \ getservbyname \ ]) AM_CONDITIONAL(STRLCPY_IMPL, [test x"$ac_cv_func_strlcpy" = xno]) AC_CACHE_CHECK( [for getaddrinfo], [libevent_cv_getaddrinfo], [AC_LINK_IFELSE( [AC_LANG_PROGRAM( [[ #ifdef HAVE_NETDB_H #include <netdb.h> #endif ]], [[ getaddrinfo; ]] )], [libevent_cv_getaddrinfo=yes], [libevent_cv_getaddrinfo=no] )] ) if test "$libevent_cv_getaddrinfo" = "yes" ; then AC_DEFINE([HAVE_GETADDRINFO], [1], [Do we have getaddrinfo()?]) else # Check for gethostbyname_r in all its glorious incompatible versions. # (This is cut-and-pasted from Tor, which based its logic on # Python's configure.in.) AH_TEMPLATE(HAVE_GETHOSTBYNAME_R, [Define this if you have any gethostbyname_r()]) AC_CHECK_FUNC(gethostbyname_r, [ AC_MSG_CHECKING([how many arguments gethostbyname_r() wants]) OLD_CFLAGS=$CFLAGS CFLAGS="$CFLAGS $MY_CPPFLAGS $MY_THREAD_CPPFLAGS $MY_CFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([ #include <netdb.h> ], [[ char *cp1, *cp2; struct hostent *h1, *h2; int i1, i2; (void)gethostbyname_r(cp1,h1,cp2,i1,&h2,&i2); ]])],[ AC_DEFINE(HAVE_GETHOSTBYNAME_R) AC_DEFINE(HAVE_GETHOSTBYNAME_R_6_ARG, 1, [Define this if gethostbyname_r takes 6 arguments]) AC_MSG_RESULT(6) ], [ AC_TRY_COMPILE([ #include <netdb.h> ], [ char *cp1, *cp2; struct hostent *h1; int i1, i2; (void)gethostbyname_r(cp1,h1,cp2,i1,&i2); ], [ AC_DEFINE(HAVE_GETHOSTBYNAME_R) AC_DEFINE(HAVE_GETHOSTBYNAME_R_5_ARG, 1, [Define this if gethostbyname_r takes 5 arguments]) AC_MSG_RESULT(5) ], [ AC_TRY_COMPILE([ #include <netdb.h> ], [ char *cp1; struct hostent *h1; struct hostent_data hd; (void) gethostbyname_r(cp1,h1,&hd); ], [ AC_DEFINE(HAVE_GETHOSTBYNAME_R) AC_DEFINE(HAVE_GETHOSTBYNAME_R_3_ARG, 1, [Define this if gethostbyname_r takes 3 arguments]) AC_MSG_RESULT(3) ], [ AC_MSG_RESULT(0) ]) ]) ]) CFLAGS=$OLD_CFLAGS ]) fi AC_MSG_CHECKING(for F_SETFD in fcntl.h) AC_EGREP_CPP(yes, [ #define _GNU_SOURCE #include <fcntl.h> #ifdef F_SETFD yes #endif ], [ AC_DEFINE(HAVE_SETFD, 1, [Define if F_SETFD is defined in <fcntl.h>]) AC_MSG_RESULT(yes) ], AC_MSG_RESULT(no)) needsignal=no haveselect=no if test x$bwin32 != xtrue; then AC_CHECK_FUNCS(select, [haveselect=yes], ) if test "x$haveselect" = "xyes" ; then needsignal=yes fi fi AM_CONDITIONAL(SELECT_BACKEND, [test "x$haveselect" = "xyes"]) havepoll=no AC_CHECK_FUNCS(poll, [havepoll=yes], ) if test "x$havepoll" = "xyes" ; then needsignal=yes fi AM_CONDITIONAL(POLL_BACKEND, [test "x$havepoll" = "xyes"]) havedevpoll=no if test "x$ac_cv_header_sys_devpoll_h" = "xyes"; then AC_DEFINE(HAVE_DEVPOLL, 1, [Define if /dev/poll is available]) fi AM_CONDITIONAL(DEVPOLL_BACKEND, [test "x$ac_cv_header_sys_devpoll_h" = "xyes"]) havekqueue=no if test "x$ac_cv_header_sys_event_h" = "xyes"; then AC_CHECK_FUNCS(kqueue, [havekqueue=yes], ) if test "x$havekqueue" = "xyes" ; then AC_MSG_CHECKING(for working kqueue) AC_TRY_RUN( #include <sys/types.h> #include <sys/time.h> #include <sys/event.h> #include <stdio.h> #include <unistd.h> #include <fcntl.h> int main(int argc, char **argv) { int kq; int n; int fd[[2]]; struct kevent ev; struct timespec ts; char buf[[8000]]; if (pipe(fd) == -1) exit(1); if (fcntl(fd[[1]], F_SETFL, O_NONBLOCK) == -1) exit(1); while ((n = write(fd[[1]], buf, sizeof(buf))) == sizeof(buf)) ; if ((kq = kqueue()) == -1) exit(1); memset(&ev, 0, sizeof(ev)); ev.ident = fd[[1]]; ev.filter = EVFILT_WRITE; ev.flags = EV_ADD | EV_ENABLE; n = kevent(kq, &ev, 1, NULL, 0, NULL); if (n == -1) exit(1); read(fd[[0]], buf, sizeof(buf)); ts.tv_sec = 0; ts.tv_nsec = 0; n = kevent(kq, NULL, 0, &ev, 1, &ts); if (n == -1 || n == 0) exit(1); exit(0); }, [AC_MSG_RESULT(yes) AC_DEFINE(HAVE_WORKING_KQUEUE, 1, [Define if kqueue works correctly with pipes]) havekqueue=yes ], AC_MSG_RESULT(no), AC_MSG_RESULT(no)) fi fi AM_CONDITIONAL(KQUEUE_BACKEND, [test "x$havekqueue" = "xyes"]) haveepollsyscall=no haveepoll=no AC_CHECK_FUNCS(epoll_ctl, [haveepoll=yes], ) if test "x$haveepoll" = "xyes" ; then AC_DEFINE(HAVE_EPOLL, 1, [Define if your system supports the epoll system calls]) needsignal=yes fi if test "x$ac_cv_header_sys_epoll_h" = "xyes"; then if test "x$haveepoll" = "xno" ; then AC_MSG_CHECKING(for epoll system call) AC_TRY_RUN( #include <stdint.h> #include <sys/param.h> #include <sys/types.h> #include <sys/syscall.h> #include <sys/epoll.h> #include <unistd.h> int epoll_create(int size) { return (syscall(__NR_epoll_create, size)); } int main(int argc, char **argv) { int epfd; epfd = epoll_create(256); exit (epfd == -1 ? 1 : 0); }, [AC_MSG_RESULT(yes) AC_DEFINE(HAVE_EPOLL, 1, [Define if your system supports the epoll system calls]) needsignal=yes have_epoll=yes AC_LIBOBJ(epoll_sub) ], AC_MSG_RESULT(no), AC_MSG_RESULT(no)) fi fi AM_CONDITIONAL(EPOLL_BACKEND, [test "x$haveepoll" = "xyes"]) AC_MSG_CHECKING(waitpid support WNOWAIT) AC_TRY_RUN( #include <unistd.h> #include <sys/types.h> #include <sys/wait.h> #include <stdlib.h> int main(int argc, char** argv) { pid_t pid; int status; if ((pid = fork()) == 0) _exit(0); _exit(waitpid(pid, &status, WNOWAIT) == -1); }, [AC_MSG_RESULT(yes) AC_DEFINE(HAVE_WAITPID_WITH_WNOWAIT, 1, [Define if waitpid() supports WNOWAIT]) ], AC_MSG_RESULT(no), AC_MSG_RESULT(no)) haveeventports=no AC_CHECK_FUNCS(port_create, [haveeventports=yes], ) if test "x$haveeventports" = "xyes" ; then AC_DEFINE(HAVE_EVENT_PORTS, 1, [Define if your system supports event ports]) needsignal=yes fi AM_CONDITIONAL(EVPORT_BACKEND, [test "x$haveeventports" = "xyes"]) if test "x$bwin32" = "xtrue"; then needsignal=yes fi AM_CONDITIONAL(SIGNAL_SUPPORT, [test "x$needsignal" = "xyes"]) AC_TYPE_PID_T AC_TYPE_SIZE_T AC_TYPE_SSIZE_T AC_CHECK_TYPES([uint64_t, uint32_t, uint16_t, uint8_t, uintptr_t], , , [#ifdef HAVE_STDINT_H #include <stdint.h> #elif defined(HAVE_INTTYPES_H) #include <inttypes.h> #endif #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif]) AC_CHECK_TYPES([fd_mask], , , [#ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif #ifdef HAVE_SYS_SELECT_H #include <sys/select.h> #endif]) AC_CHECK_SIZEOF(long long) AC_CHECK_SIZEOF(long) AC_CHECK_SIZEOF(int) AC_CHECK_SIZEOF(short) AC_CHECK_SIZEOF(size_t) AC_CHECK_SIZEOF(void *) AC_CHECK_SIZEOF(off_t) AC_CHECK_TYPES([struct in6_addr, struct sockaddr_in6, sa_family_t, struct addrinfo, struct sockaddr_storage], , , [#define _GNU_SOURCE #include <sys/types.h> #ifdef HAVE_NETINET_IN_H #include <netinet/in.h> #endif #ifdef HAVE_NETINET_IN6_H #include <netinet/in6.h> #endif #ifdef HAVE_SYS_SOCKET_H #include <sys/socket.h> #endif #ifdef HAVE_NETDB_H #include <netdb.h> #endif #ifdef _WIN32 #define WIN32_WINNT 0x400 #define _WIN32_WINNT 0x400 #define WIN32_LEAN_AND_MEAN #if defined(_MSC_VER) && (_MSC_VER < 1300) #include <winsock.h> #else #include <winsock2.h> #include <ws2tcpip.h> #endif #endif ]) AC_CHECK_MEMBERS([struct in6_addr.s6_addr32, struct in6_addr.s6_addr16, struct sockaddr_in.sin_len, struct sockaddr_in6.sin6_len, struct sockaddr_storage.ss_family, struct sockaddr_storage.__ss_family], , , [#include <sys/types.h> #ifdef HAVE_NETINET_IN_H #include <netinet/in.h> #endif #ifdef HAVE_NETINET_IN6_H #include <netinet/in6.h> #endif #ifdef HAVE_SYS_SOCKET_H #include <sys/socket.h> #endif #ifdef _WIN32 #define WIN32_WINNT 0x400 #define _WIN32_WINNT 0x400 #define WIN32_LEAN_AND_MEAN #if defined(_MSC_VER) && (_MSC_VER < 1300) #include <winsock.h> #else #include <winsock2.h> #include <ws2tcpip.h> #endif #endif ]) AC_CHECK_TYPES([struct so_linger], [#define HAVE_SO_LINGER], , [ #ifdef HAVE_SYS_SOCKET_H #include <sys/socket.h> #endif ]) AC_MSG_CHECKING([for socklen_t]) AC_TRY_COMPILE([ #include <sys/types.h> #include <sys/socket.h>], [socklen_t x;], AC_MSG_RESULT([yes]), [AC_MSG_RESULT([no]) AC_DEFINE(socklen_t, unsigned int, [Define to unsigned int if you dont have it])] ) AC_MSG_CHECKING([whether our compiler supports __func__]) AC_TRY_COMPILE([], [ const char *cp = __func__; ], AC_MSG_RESULT([yes]), AC_MSG_RESULT([no]) AC_MSG_CHECKING([whether our compiler supports __FUNCTION__]) AC_TRY_COMPILE([], [ const char *cp = __FUNCTION__; ], AC_MSG_RESULT([yes]) AC_DEFINE(__func__, __FUNCTION__, [Define to appropriate substitue if compiler doesnt have __func__]), AC_MSG_RESULT([no]) AC_DEFINE(__func__, __FILE__, [Define to appropriate substitue if compiler doesnt have __func__]))) # check if we can compile with pthreads have_pthreads=no if test x$bwin32 != xtrue && test "$enable_thread_support" != "no"; then ACX_PTHREAD([ AC_DEFINE(HAVE_PTHREADS, 1, [Define if we have pthreads on this system]) have_pthreads=yes]) CFLAGS="$CFLAGS $PTHREAD_CFLAGS" AC_CHECK_SIZEOF(pthread_t, , [AC_INCLUDES_DEFAULT() #include <pthread.h> ] ) fi AM_CONDITIONAL(THREADS, [test "$enable_thread_support" != "no"]) AM_CONDITIONAL(PTHREADS, [test "$have_pthreads" != "no" && test "$enable_thread_support" != "no"]) # check if we should compile locking into the library if test x$enable_thread_support = xno; then AC_DEFINE(DISABLE_THREAD_SUPPORT, 1, [Define if libevent should not be compiled with thread support]) fi # check if we should hard-code the mm functions. if test x$enable_malloc_replacement = xno; then AC_DEFINE(DISABLE_MM_REPLACEMENT, 1, [Define if libevent should not allow replacing the mm functions]) fi # check if we should hard-code debugging out if test x$enable_debug_mode = xno; then AC_DEFINE(DISABLE_DEBUG_MODE, 1, [Define if libevent should build without support for a debug mode]) fi # check if we should enable verbose debugging if test x$enable_verbose_debug = xyes; then CFLAGS="$CFLAGS -DUSE_DEBUG" fi # check if we have and should use openssl AM_CONDITIONAL(OPENSSL, [test "$enable_openssl" != "no" && test "$have_openssl" = "yes"]) if test "x$enable_openssl" = "xyes"; then AC_SEARCH_LIBS([ERR_remove_thread_state], [crypto eay32], [AC_DEFINE(HAVE_ERR_REMOVE_THREAD_STATE, 1, [Define to 1 if you have ERR_remove_thread_stat().])]) fi # Add some more warnings which we use in development but not in the # released versions. (Some relevant gcc versions can't handle these.) if test x$enable_gcc_warnings != xno && test "$GCC" = "yes"; then AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ #if !defined(__GNUC__) || (__GNUC__ < 4) #error #endif])], have_gcc4=yes, have_gcc4=no) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ #if !defined(__GNUC__) || (__GNUC__ < 4) || (__GNUC__ == 4 && __GNUC_MINOR__ < 2) #error #endif])], have_gcc42=yes, have_gcc42=no) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ #if !defined(__GNUC__) || (__GNUC__ < 4) || (__GNUC__ == 4 && __GNUC_MINOR__ < 5) #error #endif])], have_gcc45=yes, have_gcc45=no) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ #if !defined(__clang__) #error #endif])], have_clang=yes, have_clang=no) CFLAGS="$CFLAGS -W -Wfloat-equal -Wundef -Wpointer-arith -Wstrict-prototypes -Wmissing-prototypes -Wwrite-strings -Wredundant-decls -Wchar-subscripts -Wcomment -Wformat -Wwrite-strings -Wmissing-declarations -Wredundant-decls -Wnested-externs -Wbad-function-cast -Wswitch" if test x$enable_gcc_warnings = xyes; then CFLAGS="$CFLAGS -Werror" fi CFLAGS="$CFLAGS -Wno-unused-parameter -Wstrict-aliasing" if test x$have_gcc4 = xyes ; then # These warnings break gcc 3.3.5 and work on gcc 4.0.2 CFLAGS="$CFLAGS -Winit-self -Wmissing-field-initializers -Wdeclaration-after-statement" #CFLAGS="$CFLAGS -Wold-style-definition" fi if test x$have_gcc42 = xyes ; then # These warnings break gcc 4.0.2 and work on gcc 4.2 CFLAGS="$CFLAGS -Waddress" fi if test x$have_gcc42 = xyes && test x$have_clang = xno; then # These warnings break gcc 4.0.2 and clang, but work on gcc 4.2 CFLAGS="$CFLAGS -Wnormalized=id -Woverride-init" fi if test x$have_gcc45 = xyes ; then # These warnings work on gcc 4.5 CFLAGS="$CFLAGS -Wlogical-op" fi if test x$have_clang = xyes; then # Disable the unused-function warnings, because these trigger # for minheap-internal.h related code. CFLAGS="$CFLAGS -Wno-unused-function" # clang on macosx emits warnigns for each directory specified which # isn't "used" generating a lot of build noise (typically 3 warnings # per file case "$host_os" in darwin*) CFLAGS="$CFLAGS -Qunused-arguments" ;; esac fi ##This will break the world on some 64-bit architectures # CFLAGS="$CFLAGS -Winline" fi LIBEVENT_GC_SECTIONS= if test "$GCC" = yes && test "$enable_function_sections" = yes ; then AC_CACHE_CHECK( [if linker supports omitting unused code and data], [libevent_cv_gc_sections_runs], [ dnl NetBSD will link but likely not run with --gc-sections dnl http://bugs.ntp.org/1844 dnl http://gnats.netbsd.org/40401 dnl --gc-sections causes attempt to load as linux elf, with dnl wrong syscalls in place. Test a little gauntlet of dnl simple stdio read code checking for errors, expecting dnl enough syscall differences that the NetBSD code will dnl fail even with Linux emulation working as designed. dnl A shorter test could be refined by someone with access dnl to a NetBSD host with Linux emulation working. origCFLAGS="$CFLAGS" CFLAGS="$CFLAGS -Wl,--gc-sections" AC_LINK_IFELSE( [AC_LANG_PROGRAM( [[ #include <stdlib.h> #include <stdio.h> ]], [[ FILE * fpC; char buf[32]; size_t cch; int read_success_once; fpC = fopen("conftest.c", "r"); if (NULL == fpC) exit(1); do { cch = fread(buf, sizeof(buf), 1, fpC); read_success_once |= (0 != cch); } while (0 != cch); if (!read_success_once) exit(2); if (!feof(fpC)) exit(3); if (0 != fclose(fpC)) exit(4); exit(EXIT_SUCCESS); ]] )], [ dnl We have to do this invocation manually so that we can dnl get the output of conftest.err to make sure it doesn't dnl mention gc-sections. if test "X$cross_compiling" = "Xyes" || grep gc-sections conftest.err ; then libevent_cv_gc_sections_runs=no else libevent_cv_gc_sections_runs=no ./conftest >/dev/null 2>&1 && libevent_cv_gc_sections_runs=yes fi ], [libevent_cv_gc_sections_runs=no] ) CFLAGS="$origCFLAGS" AS_UNSET([origCFLAGS]) ] ) case "$libevent_cv_gc_sections_runs" in yes) CFLAGS="-ffunction-sections -fdata-sections $CFLAGS" LIBEVENT_GC_SECTIONS="-Wl,--gc-sections" ;; esac fi AC_SUBST([LIBEVENT_GC_SECTIONS]) AM_CONDITIONAL([INSTALL_LIBEVENT], [test "$enable_libevent_install" = "yes"]) AC_CONFIG_FILES( [libevent.pc libevent_openssl.pc libevent_pthreads.pc libevent_core.pc libevent_extra.pc] ) AC_OUTPUT(Makefile) ���������������������������kytensor-2.49.0.7/third_party/libevent/defer-internal.h���������������������������������������������0000664�0000000�0000000�00000005316�15270523712�0023053�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef DEFER_INTERNAL_H_INCLUDED_ #define DEFER_INTERNAL_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif #include "event2/event-config.h" #include "evconfig-private.h" #include <sys/queue.h> struct event_callback; typedef void (*deferred_cb_fn)(struct event_callback *, void *); /** Initialize an empty, non-pending event_callback. @param deferred The struct event_callback structure to initialize. @param priority The priority that the callback should run at. @param cb The function to run when the struct event_callback executes. @param arg The function's second argument. */ void event_deferred_cb_init_(struct event_callback *, ev_uint8_t, deferred_cb_fn, void *); /** Change the priority of a non-pending event_callback. */ void event_deferred_cb_set_priority_(struct event_callback *, ev_uint8_t); /** Cancel a struct event_callback if it is currently scheduled in an event_base. */ void event_deferred_cb_cancel_(struct event_base *, struct event_callback *); /** Activate a struct event_callback if it is not currently scheduled in an event_base. Return true if it was not previously scheduled. */ int event_deferred_cb_schedule_(struct event_base *, struct event_callback *); #ifdef __cplusplus } #endif #endif /* EVENT_INTERNAL_H_INCLUDED_ */ ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/devpoll.c����������������������������������������������������0000664�0000000�0000000�00000017321�15270523712�0021613�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright 2000-2009 Niels Provos <provos@citi.umich.edu> * Copyright 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_DEVPOLL #include <sys/types.h> #include <sys/resource.h> #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <sys/queue.h> #include <sys/devpoll.h> #include <signal.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <unistd.h> #include <fcntl.h> #include <errno.h> #include "event2/event.h" #include "event2/event_struct.h" #include "event2/thread.h" #include "event-internal.h" #include "evsignal-internal.h" #include "log-internal.h" #include "evmap-internal.h" #include "evthread-internal.h" struct devpollop { struct pollfd *events; int nevents; int dpfd; struct pollfd *changes; int nchanges; }; static void *devpoll_init(struct event_base *); static int devpoll_add(struct event_base *, int fd, short old, short events, void *); static int devpoll_del(struct event_base *, int fd, short old, short events, void *); static int devpoll_dispatch(struct event_base *, struct timeval *); static void devpoll_dealloc(struct event_base *); const struct eventop devpollops = { "devpoll", devpoll_init, devpoll_add, devpoll_del, devpoll_dispatch, devpoll_dealloc, 1, /* need reinit */ EV_FEATURE_FDS|EV_FEATURE_O1, 0 }; #define NEVENT 32000 static int devpoll_commit(struct devpollop *devpollop) { /* * Due to a bug in Solaris, we have to use pwrite with an offset of 0. * Write is limited to 2GB of data, until it will fail. */ if (pwrite(devpollop->dpfd, devpollop->changes, sizeof(struct pollfd) * devpollop->nchanges, 0) == -1) return (-1); devpollop->nchanges = 0; return (0); } static int devpoll_queue(struct devpollop *devpollop, int fd, int events) { struct pollfd *pfd; if (devpollop->nchanges >= devpollop->nevents) { /* * Change buffer is full, must commit it to /dev/poll before * adding more */ if (devpoll_commit(devpollop) != 0) return (-1); } pfd = &devpollop->changes[devpollop->nchanges++]; pfd->fd = fd; pfd->events = events; pfd->revents = 0; return (0); } static void * devpoll_init(struct event_base *base) { int dpfd, nfiles = NEVENT; struct rlimit rl; struct devpollop *devpollop; if (!(devpollop = mm_calloc(1, sizeof(struct devpollop)))) return (NULL); if (getrlimit(RLIMIT_NOFILE, &rl) == 0 && rl.rlim_cur != RLIM_INFINITY) nfiles = rl.rlim_cur; /* Initialize the kernel queue */ if ((dpfd = evutil_open_closeonexec_("/dev/poll", O_RDWR, 0)) == -1) { event_warn("open: /dev/poll"); mm_free(devpollop); return (NULL); } devpollop->dpfd = dpfd; /* Initialize fields */ /* FIXME: allocating 'nfiles' worth of space here can be * expensive and unnecessary. See how epoll.c does it instead. */ devpollop->events = mm_calloc(nfiles, sizeof(struct pollfd)); if (devpollop->events == NULL) { mm_free(devpollop); close(dpfd); return (NULL); } devpollop->nevents = nfiles; devpollop->changes = mm_calloc(nfiles, sizeof(struct pollfd)); if (devpollop->changes == NULL) { mm_free(devpollop->events); mm_free(devpollop); close(dpfd); return (NULL); } evsig_init_(base); return (devpollop); } static int devpoll_dispatch(struct event_base *base, struct timeval *tv) { struct devpollop *devpollop = base->evbase; struct pollfd *events = devpollop->events; struct dvpoll dvp; int i, res, timeout = -1; if (devpollop->nchanges) devpoll_commit(devpollop); if (tv != NULL) timeout = tv->tv_sec * 1000 + (tv->tv_usec + 999) / 1000; dvp.dp_fds = devpollop->events; dvp.dp_nfds = devpollop->nevents; dvp.dp_timeout = timeout; EVBASE_RELEASE_LOCK(base, th_base_lock); res = ioctl(devpollop->dpfd, DP_POLL, &dvp); EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (res == -1) { if (errno != EINTR) { event_warn("ioctl: DP_POLL"); return (-1); } return (0); } event_debug(("%s: devpoll_wait reports %d", __func__, res)); for (i = 0; i < res; i++) { int which = 0; int what = events[i].revents; if (what & POLLHUP) what |= POLLIN | POLLOUT; else if (what & POLLERR) what |= POLLIN | POLLOUT; if (what & POLLIN) which |= EV_READ; if (what & POLLOUT) which |= EV_WRITE; if (!which) continue; /* XXX(niels): not sure if this works for devpoll */ evmap_io_active_(base, events[i].fd, which); } return (0); } static int devpoll_add(struct event_base *base, int fd, short old, short events, void *p) { struct devpollop *devpollop = base->evbase; int res; (void)p; /* * It's not necessary to OR the existing read/write events that we * are currently interested in with the new event we are adding. * The /dev/poll driver ORs any new events with the existing events * that it has cached for the fd. */ res = 0; if (events & EV_READ) res |= POLLIN; if (events & EV_WRITE) res |= POLLOUT; if (devpoll_queue(devpollop, fd, res) != 0) return (-1); return (0); } static int devpoll_del(struct event_base *base, int fd, short old, short events, void *p) { struct devpollop *devpollop = base->evbase; int res; (void)p; res = 0; if (events & EV_READ) res |= POLLIN; if (events & EV_WRITE) res |= POLLOUT; /* * The only way to remove an fd from the /dev/poll monitored set is * to use POLLREMOVE by itself. This removes ALL events for the fd * provided so if we care about two events and are only removing one * we must re-add the other event after POLLREMOVE. */ if (devpoll_queue(devpollop, fd, POLLREMOVE) != 0) return (-1); if ((res & (POLLIN|POLLOUT)) != (POLLIN|POLLOUT)) { /* * We're not deleting all events, so we must resubmit the * event that we are still interested in if one exists. */ if ((res & POLLIN) && (old & EV_WRITE)) { /* Deleting read, still care about write */ devpoll_queue(devpollop, fd, POLLOUT); } else if ((res & POLLOUT) && (old & EV_READ)) { /* Deleting write, still care about read */ devpoll_queue(devpollop, fd, POLLIN); } } return (0); } static void devpoll_dealloc(struct event_base *base) { struct devpollop *devpollop = base->evbase; evsig_dealloc_(base); if (devpollop->events) mm_free(devpollop->events); if (devpollop->changes) mm_free(devpollop->changes); if (devpollop->dpfd >= 0) close(devpollop->dpfd); memset(devpollop, 0, sizeof(struct devpollop)); mm_free(devpollop); } #endif /* EVENT__HAVE_DEVPOLL */ ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/epoll.c������������������������������������������������������0000664�0000000�0000000�00000034572�15270523712�0021270�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright 2000-2007 Niels Provos <provos@citi.umich.edu> * Copyright 2007-2012 Niels Provos, Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_EPOLL #include <stdint.h> #include <sys/types.h> #include <sys/resource.h> #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <sys/queue.h> #include <sys/epoll.h> #include <signal.h> #include <limits.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <unistd.h> #include <errno.h> #ifdef EVENT__HAVE_FCNTL_H #include <fcntl.h> #endif #ifdef EVENT__HAVE_SYS_TIMERFD_H #include <sys/timerfd.h> #endif #include "event-internal.h" #include "evsignal-internal.h" #include "event2/thread.h" #include "evthread-internal.h" #include "log-internal.h" #include "evmap-internal.h" #include "changelist-internal.h" #include "time-internal.h" /* Since Linux 2.6.17, epoll is able to report about peer half-closed connection using special EPOLLRDHUP flag on a read event. */ #if !defined(EPOLLRDHUP) #define EPOLLRDHUP 0 #define EARLY_CLOSE_IF_HAVE_RDHUP 0 #else #define EARLY_CLOSE_IF_HAVE_RDHUP EV_FEATURE_EARLY_CLOSE #endif #include "epolltable-internal.h" #if defined(EVENT__HAVE_SYS_TIMERFD_H) && \ defined(EVENT__HAVE_TIMERFD_CREATE) && \ defined(HAVE_POSIX_MONOTONIC) && defined(TFD_NONBLOCK) && \ defined(TFD_CLOEXEC) /* Note that we only use timerfd if TFD_NONBLOCK and TFD_CLOEXEC are available and working. This means that we can't support it on 2.6.25 (where timerfd was introduced) or 2.6.26, since 2.6.27 introduced those flags. */ #define USING_TIMERFD #endif struct epollop { struct epoll_event *events; int nevents; int epfd; #ifdef USING_TIMERFD int timerfd; #endif }; static void *epoll_init(struct event_base *); static int epoll_dispatch(struct event_base *, struct timeval *); static void epoll_dealloc(struct event_base *); static const struct eventop epollops_changelist = { "epoll (with changelist)", epoll_init, event_changelist_add_, event_changelist_del_, epoll_dispatch, epoll_dealloc, 1, /* need reinit */ EV_FEATURE_ET|EV_FEATURE_O1| EARLY_CLOSE_IF_HAVE_RDHUP, EVENT_CHANGELIST_FDINFO_SIZE }; static int epoll_nochangelist_add(struct event_base *base, evutil_socket_t fd, short old, short events, void *p); static int epoll_nochangelist_del(struct event_base *base, evutil_socket_t fd, short old, short events, void *p); const struct eventop epollops = { "epoll", epoll_init, epoll_nochangelist_add, epoll_nochangelist_del, epoll_dispatch, epoll_dealloc, 1, /* need reinit */ EV_FEATURE_ET|EV_FEATURE_O1|EV_FEATURE_EARLY_CLOSE, 0 }; #define INITIAL_NEVENT 32 #define MAX_NEVENT 4096 /* On Linux kernels at least up to 2.6.24.4, epoll can't handle timeout * values bigger than (LONG_MAX - 999ULL)/HZ. HZ in the wild can be * as big as 1000, and LONG_MAX can be as small as (1<<31)-1, so the * largest number of msec we can support here is 2147482. Let's * round that down by 47 seconds. */ #define MAX_EPOLL_TIMEOUT_MSEC (35*60*1000) static void * epoll_init(struct event_base *base) { int epfd = -1; struct epollop *epollop; #ifdef EVENT__HAVE_EPOLL_CREATE1 /* First, try the shiny new epoll_create1 interface, if we have it. */ epfd = epoll_create1(EPOLL_CLOEXEC); #endif if (epfd == -1) { /* Initialize the kernel queue using the old interface. (The size field is ignored since 2.6.8.) */ if ((epfd = epoll_create(32000)) == -1) { if (errno != ENOSYS) event_warn("epoll_create"); return (NULL); } evutil_make_socket_closeonexec(epfd); } if (!(epollop = mm_calloc(1, sizeof(struct epollop)))) { close(epfd); return (NULL); } epollop->epfd = epfd; /* Initialize fields */ epollop->events = mm_calloc(INITIAL_NEVENT, sizeof(struct epoll_event)); if (epollop->events == NULL) { mm_free(epollop); close(epfd); return (NULL); } epollop->nevents = INITIAL_NEVENT; if ((base->flags & EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST) != 0 || ((base->flags & EVENT_BASE_FLAG_IGNORE_ENV) == 0 && evutil_getenv_("EVENT_EPOLL_USE_CHANGELIST") != NULL)) { base->evsel = &epollops_changelist; } #ifdef USING_TIMERFD /* The epoll interface ordinarily gives us one-millisecond precision, so on Linux it makes perfect sense to use the CLOCK_MONOTONIC_COARSE timer. But when the user has set the new PRECISE_TIMER flag for an event_base, we can try to use timerfd to give them finer granularity. */ if ((base->flags & EVENT_BASE_FLAG_PRECISE_TIMER) && base->monotonic_timer.monotonic_clock == CLOCK_MONOTONIC) { int fd; fd = epollop->timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK|TFD_CLOEXEC); if (epollop->timerfd >= 0) { struct epoll_event epev; memset(&epev, 0, sizeof(epev)); epev.data.fd = epollop->timerfd; epev.events = EPOLLIN; if (epoll_ctl(epollop->epfd, EPOLL_CTL_ADD, fd, &epev) < 0) { event_warn("epoll_ctl(timerfd)"); close(fd); epollop->timerfd = -1; } } else { if (errno != EINVAL && errno != ENOSYS) { /* These errors probably mean that we were * compiled with timerfd/TFD_* support, but * we're running on a kernel that lacks those. */ event_warn("timerfd_create"); } epollop->timerfd = -1; } } else { epollop->timerfd = -1; } #endif evsig_init_(base); return (epollop); } static const char * change_to_string(int change) { change &= (EV_CHANGE_ADD|EV_CHANGE_DEL); if (change == EV_CHANGE_ADD) { return "add"; } else if (change == EV_CHANGE_DEL) { return "del"; } else if (change == 0) { return "none"; } else { return "???"; } } static const char * epoll_op_to_string(int op) { return op == EPOLL_CTL_ADD?"ADD": op == EPOLL_CTL_DEL?"DEL": op == EPOLL_CTL_MOD?"MOD": "???"; } #define PRINT_CHANGES(op, events, ch, status) \ "Epoll %s(%d) on fd %d " status ". " \ "Old events were %d; " \ "read change was %d (%s); " \ "write change was %d (%s); " \ "close change was %d (%s)", \ epoll_op_to_string(op), \ events, \ ch->fd, \ ch->old_events, \ ch->read_change, \ change_to_string(ch->read_change), \ ch->write_change, \ change_to_string(ch->write_change), \ ch->close_change, \ change_to_string(ch->close_change) static int epoll_apply_one_change(struct event_base *base, struct epollop *epollop, const struct event_change *ch) { struct epoll_event epev; int op, events = 0; int idx; idx = EPOLL_OP_TABLE_INDEX(ch); op = epoll_op_table[idx].op; events = epoll_op_table[idx].events; if (!events) { EVUTIL_ASSERT(op == 0); return 0; } if ((ch->read_change|ch->write_change) & EV_CHANGE_ET) events |= EPOLLET; memset(&epev, 0, sizeof(epev)); epev.data.fd = ch->fd; epev.events = events; if (epoll_ctl(epollop->epfd, op, ch->fd, &epev) == 0) { event_debug((PRINT_CHANGES(op, epev.events, ch, "okay"))); return 0; } switch (op) { case EPOLL_CTL_MOD: if (errno == ENOENT) { /* If a MOD operation fails with ENOENT, the * fd was probably closed and re-opened. We * should retry the operation as an ADD. */ if (epoll_ctl(epollop->epfd, EPOLL_CTL_ADD, ch->fd, &epev) == -1) { event_warn("Epoll MOD(%d) on %d retried as ADD; that failed too", (int)epev.events, ch->fd); return -1; } else { event_debug(("Epoll MOD(%d) on %d retried as ADD; succeeded.", (int)epev.events, ch->fd)); return 0; } } break; case EPOLL_CTL_ADD: if (errno == EEXIST) { /* If an ADD operation fails with EEXIST, * either the operation was redundant (as with a * precautionary add), or we ran into a fun * kernel bug where using dup*() to duplicate the * same file into the same fd gives you the same epitem * rather than a fresh one. For the second case, * we must retry with MOD. */ if (epoll_ctl(epollop->epfd, EPOLL_CTL_MOD, ch->fd, &epev) == -1) { event_warn("Epoll ADD(%d) on %d retried as MOD; that failed too", (int)epev.events, ch->fd); return -1; } else { event_debug(("Epoll ADD(%d) on %d retried as MOD; succeeded.", (int)epev.events, ch->fd)); return 0; } } break; case EPOLL_CTL_DEL: if (errno == ENOENT || errno == EBADF || errno == EPERM) { /* If a delete fails with one of these errors, * that's fine too: we closed the fd before we * got around to calling epoll_dispatch. */ event_debug(("Epoll DEL(%d) on fd %d gave %s: DEL was unnecessary.", (int)epev.events, ch->fd, strerror(errno))); return 0; } break; default: break; } event_warn(PRINT_CHANGES(op, epev.events, ch, "failed")); return -1; } static int epoll_apply_changes(struct event_base *base) { struct event_changelist *changelist = &base->changelist; struct epollop *epollop = base->evbase; struct event_change *ch; int r = 0; int i; for (i = 0; i < changelist->n_changes; ++i) { ch = &changelist->changes[i]; if (epoll_apply_one_change(base, epollop, ch) < 0) r = -1; } return (r); } static int epoll_nochangelist_add(struct event_base *base, evutil_socket_t fd, short old, short events, void *p) { struct event_change ch; ch.fd = fd; ch.old_events = old; ch.read_change = ch.write_change = ch.close_change = 0; if (events & EV_WRITE) ch.write_change = EV_CHANGE_ADD | (events & EV_ET); if (events & EV_READ) ch.read_change = EV_CHANGE_ADD | (events & EV_ET); if (events & EV_CLOSED) ch.close_change = EV_CHANGE_ADD | (events & EV_ET); return epoll_apply_one_change(base, base->evbase, &ch); } static int epoll_nochangelist_del(struct event_base *base, evutil_socket_t fd, short old, short events, void *p) { struct event_change ch; ch.fd = fd; ch.old_events = old; ch.read_change = ch.write_change = ch.close_change = 0; if (events & EV_WRITE) ch.write_change = EV_CHANGE_DEL; if (events & EV_READ) ch.read_change = EV_CHANGE_DEL; if (events & EV_CLOSED) ch.close_change = EV_CHANGE_DEL; return epoll_apply_one_change(base, base->evbase, &ch); } static int epoll_dispatch(struct event_base *base, struct timeval *tv) { struct epollop *epollop = base->evbase; struct epoll_event *events = epollop->events; int i, res; long timeout = -1; #ifdef USING_TIMERFD if (epollop->timerfd >= 0) { struct itimerspec is; is.it_interval.tv_sec = 0; is.it_interval.tv_nsec = 0; if (tv == NULL) { /* No timeout; disarm the timer. */ is.it_value.tv_sec = 0; is.it_value.tv_nsec = 0; } else { if (tv->tv_sec == 0 && tv->tv_usec == 0) { /* we need to exit immediately; timerfd can't * do that. */ timeout = 0; } is.it_value.tv_sec = tv->tv_sec; is.it_value.tv_nsec = tv->tv_usec * 1000; } /* TODO: we could avoid unnecessary syscalls here by only calling timerfd_settime when the top timeout changes, or when we're called with a different timeval. */ if (timerfd_settime(epollop->timerfd, 0, &is, NULL) < 0) { event_warn("timerfd_settime"); } } else #endif if (tv != NULL) { timeout = evutil_tv_to_msec_(tv); if (timeout < 0 || timeout > MAX_EPOLL_TIMEOUT_MSEC) { /* Linux kernels can wait forever if the timeout is * too big; see comment on MAX_EPOLL_TIMEOUT_MSEC. */ timeout = MAX_EPOLL_TIMEOUT_MSEC; } } epoll_apply_changes(base); event_changelist_remove_all_(&base->changelist, base); EVBASE_RELEASE_LOCK(base, th_base_lock); res = epoll_wait(epollop->epfd, events, epollop->nevents, timeout); EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (res == -1) { if (errno != EINTR) { event_warn("epoll_wait"); return (-1); } return (0); } event_debug(("%s: epoll_wait reports %d", __func__, res)); EVUTIL_ASSERT(res <= epollop->nevents); for (i = 0; i < res; i++) { int what = events[i].events; short ev = 0; #ifdef USING_TIMERFD if (events[i].data.fd == epollop->timerfd) continue; #endif if (what & (EPOLLHUP|EPOLLERR)) { ev = EV_READ | EV_WRITE; } else { if (what & EPOLLIN) ev |= EV_READ; if (what & EPOLLOUT) ev |= EV_WRITE; if (what & EPOLLRDHUP) ev |= EV_CLOSED; } if (!ev) continue; evmap_io_active_(base, events[i].data.fd, ev | EV_ET); } if (res == epollop->nevents && epollop->nevents < MAX_NEVENT) { /* We used all of the event space this time. We should be ready for more events next time. */ int new_nevents = epollop->nevents * 2; struct epoll_event *new_events; new_events = mm_realloc(epollop->events, new_nevents * sizeof(struct epoll_event)); if (new_events) { epollop->events = new_events; epollop->nevents = new_nevents; } } return (0); } static void epoll_dealloc(struct event_base *base) { struct epollop *epollop = base->evbase; evsig_dealloc_(base); if (epollop->events) mm_free(epollop->events); if (epollop->epfd >= 0) close(epollop->epfd); #ifdef USING_TIMERFD if (epollop->timerfd >= 0) close(epollop->timerfd); #endif memset(epollop, 0, sizeof(struct epollop)); mm_free(epollop); } #endif /* EVENT__HAVE_EPOLL */ ��������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/epoll_sub.c��������������������������������������������������0000664�0000000�0000000�00000004527�15270523712�0022136�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright 2003-2009 Niels Provos <provos@citi.umich.edu> * Copyright 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "evconfig-private.h" #include <stdint.h> #include <sys/param.h> #include <sys/types.h> #include <sys/syscall.h> #include <sys/epoll.h> #include <unistd.h> #include <errno.h> int epoll_create(int size) { #if !defined(__NR_epoll_create) && defined(__NR_epoll_create1) if (size <= 0) { errno = EINVAL; return -1; } return (syscall(__NR_epoll_create1, 0)); #else return (syscall(__NR_epoll_create, size)); #endif } int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { return (syscall(__NR_epoll_ctl, epfd, op, fd, event)); } int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { #if !defined(__NR_epoll_wait) && defined(__NR_epoll_pwait) return (syscall(__NR_epoll_pwait, epfd, events, maxevents, timeout, NULL, 0)); #else return (syscall(__NR_epoll_wait, epfd, events, maxevents, timeout)); #endif } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/epolltable-internal.h����������������������������������������0000664�0000000�0000000�00000120625�15270523712�0024112�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2000-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EPOLLTABLE_INTERNAL_H_INCLUDED_ #define EPOLLTABLE_INTERNAL_H_INCLUDED_ /* Here are the values we're masking off to decide what operations to do. Note that since EV_READ|EV_WRITE. Note also that this table is a little sparse, since ADD+DEL is nonsensical ("xxx" in the list below.) Note also also that we are shifting old_events by only 5 bits, since EV_READ is 2 and EV_WRITE is 4. The table was auto-generated with a python script, according to this pseudocode:[*0] If either the read or the write change is add+del: This is impossible; Set op==-1, events=0. Else, if either the read or the write change is add: Set events to 0. If the read change is add, or (the read change is not del, and ev_read is in old_events): Add EPOLLIN to events. If the write change is add, or (the write change is not del, and ev_write is in old_events): Add EPOLLOUT to events. If old_events is set: Set op to EPOLL_CTL_MOD [*1,*2] Else: Set op to EPOLL_CTL_ADD [*3] Else, if the read or the write change is del: Set op to EPOLL_CTL_DEL. If the read change is del: If the write change is del: Set events to EPOLLIN|EPOLLOUT Else if ev_write is in old_events: Set events to EPOLLOUT Set op to EPOLL_CTL_MOD Else Set events to EPOLLIN Else: {The write change is del.} If ev_read is in old_events: Set events to EPOLLIN Set op to EPOLL_CTL_MOD Else: Set the events to EPOLLOUT Else: There is no read or write change; set op to 0 and events to 0. The logic is a little tricky, since we had no events set on the fd before, we need to set op="ADD" and set events=the events we want to add. If we had any events set on the fd before, and we want any events to remain on the fd, we need to say op="MOD" and set events=the events we want to remain. But if we want to delete the last event, we say op="DEL" and set events=(any non-null pointer). [*0] Actually, the Python script has gotten a bit more complicated, to support EPOLLRDHUP. [*1] This MOD is only a guess. MOD might fail with ENOENT if the file was closed and a new file was opened with the same fd. If so, we'll retry with ADD. [*2] We can't replace this with a no-op even if old_events is the same as the new events: if the file was closed and reopened, we need to retry with an ADD. (We do a MOD in this case since "no change" is more common than "close and reopen", so we'll usually wind up doing 1 syscalls instead of 2.) [*3] This ADD is only a guess. There is a fun Linux kernel issue where if you have two fds for the same file (via dup) and you ADD one to an epfd, then close it, then re-create it with the same fd (via dup2 or an unlucky dup), then try to ADD it again, you'll get an EEXIST, since the struct epitem is not actually removed from the struct eventpoll until the file itself is closed. EV_CHANGE_ADD==1 EV_CHANGE_DEL==2 EV_READ ==2 EV_WRITE ==4 EV_CLOSED ==0x80 Bit 0: close change is add Bit 1: close change is del Bit 2: read change is add Bit 3: read change is del Bit 4: write change is add Bit 5: write change is del Bit 6: old events had EV_READ Bit 7: old events had EV_WRITE Bit 8: old events had EV_CLOSED */ #define EPOLL_OP_TABLE_INDEX(c) \ ( (((c)->close_change&(EV_CHANGE_ADD|EV_CHANGE_DEL))) | \ (((c)->read_change&(EV_CHANGE_ADD|EV_CHANGE_DEL)) << 2) | \ (((c)->write_change&(EV_CHANGE_ADD|EV_CHANGE_DEL)) << 4) | \ (((c)->old_events&(EV_READ|EV_WRITE)) << 5) | \ (((c)->old_events&(EV_CLOSED)) << 1) \ ) #if EV_READ != 2 || EV_WRITE != 4 || EV_CLOSED != 0x80 || EV_CHANGE_ADD != 1 || EV_CHANGE_DEL != 2 #error "Libevent's internals changed! Regenerate the op_table in epolltable-internal.h" #endif static const struct operation { int events; int op; } epoll_op_table[] = { /* old= 0, write: 0, read: 0, close: 0 */ { 0, 0 }, /* old= 0, write: 0, read: 0, close:add */ { EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write: 0, read: 0, close:del */ { EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= 0, write: 0, read: 0, close:xxx */ { 0, 255 }, /* old= 0, write: 0, read:add, close: 0 */ { EPOLLIN, EPOLL_CTL_ADD }, /* old= 0, write: 0, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write: 0, read:add, close:del */ { EPOLLIN, EPOLL_CTL_ADD }, /* old= 0, write: 0, read:add, close:xxx */ { 0, 255 }, /* old= 0, write: 0, read:del, close: 0 */ { EPOLLIN, EPOLL_CTL_DEL }, /* old= 0, write: 0, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write: 0, read:del, close:del */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= 0, write: 0, read:del, close:xxx */ { 0, 255 }, /* old= 0, write: 0, read:xxx, close: 0 */ { 0, 255 }, /* old= 0, write: 0, read:xxx, close:add */ { 0, 255 }, /* old= 0, write: 0, read:xxx, close:del */ { 0, 255 }, /* old= 0, write: 0, read:xxx, close:xxx */ { 0, 255 }, /* old= 0, write:add, read: 0, close: 0 */ { EPOLLOUT, EPOLL_CTL_ADD }, /* old= 0, write:add, read: 0, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write:add, read: 0, close:del */ { EPOLLOUT, EPOLL_CTL_ADD }, /* old= 0, write:add, read: 0, close:xxx */ { 0, 255 }, /* old= 0, write:add, read:add, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_ADD }, /* old= 0, write:add, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write:add, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_ADD }, /* old= 0, write:add, read:add, close:xxx */ { 0, 255 }, /* old= 0, write:add, read:del, close: 0 */ { EPOLLOUT, EPOLL_CTL_ADD }, /* old= 0, write:add, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write:add, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_ADD }, /* old= 0, write:add, read:del, close:xxx */ { 0, 255 }, /* old= 0, write:add, read:xxx, close: 0 */ { 0, 255 }, /* old= 0, write:add, read:xxx, close:add */ { 0, 255 }, /* old= 0, write:add, read:xxx, close:del */ { 0, 255 }, /* old= 0, write:add, read:xxx, close:xxx */ { 0, 255 }, /* old= 0, write:del, read: 0, close: 0 */ { EPOLLOUT, EPOLL_CTL_DEL }, /* old= 0, write:del, read: 0, close:add */ { EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write:del, read: 0, close:del */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= 0, write:del, read: 0, close:xxx */ { 0, 255 }, /* old= 0, write:del, read:add, close: 0 */ { EPOLLIN, EPOLL_CTL_ADD }, /* old= 0, write:del, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write:del, read:add, close:del */ { EPOLLIN, EPOLL_CTL_ADD }, /* old= 0, write:del, read:add, close:xxx */ { 0, 255 }, /* old= 0, write:del, read:del, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_DEL }, /* old= 0, write:del, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_ADD }, /* old= 0, write:del, read:del, close:del */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= 0, write:del, read:del, close:xxx */ { 0, 255 }, /* old= 0, write:del, read:xxx, close: 0 */ { 0, 255 }, /* old= 0, write:del, read:xxx, close:add */ { 0, 255 }, /* old= 0, write:del, read:xxx, close:del */ { 0, 255 }, /* old= 0, write:del, read:xxx, close:xxx */ { 0, 255 }, /* old= 0, write:xxx, read: 0, close: 0 */ { 0, 255 }, /* old= 0, write:xxx, read: 0, close:add */ { 0, 255 }, /* old= 0, write:xxx, read: 0, close:del */ { 0, 255 }, /* old= 0, write:xxx, read: 0, close:xxx */ { 0, 255 }, /* old= 0, write:xxx, read:add, close: 0 */ { 0, 255 }, /* old= 0, write:xxx, read:add, close:add */ { 0, 255 }, /* old= 0, write:xxx, read:add, close:del */ { 0, 255 }, /* old= 0, write:xxx, read:add, close:xxx */ { 0, 255 }, /* old= 0, write:xxx, read:del, close: 0 */ { 0, 255 }, /* old= 0, write:xxx, read:del, close:add */ { 0, 255 }, /* old= 0, write:xxx, read:del, close:del */ { 0, 255 }, /* old= 0, write:xxx, read:del, close:xxx */ { 0, 255 }, /* old= 0, write:xxx, read:xxx, close: 0 */ { 0, 255 }, /* old= 0, write:xxx, read:xxx, close:add */ { 0, 255 }, /* old= 0, write:xxx, read:xxx, close:del */ { 0, 255 }, /* old= 0, write:xxx, read:xxx, close:xxx */ { 0, 255 }, /* old= r, write: 0, read: 0, close: 0 */ { 0, 0 }, /* old= r, write: 0, read: 0, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write: 0, read: 0, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= r, write: 0, read: 0, close:xxx */ { 0, 255 }, /* old= r, write: 0, read:add, close: 0 */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= r, write: 0, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write: 0, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= r, write: 0, read:add, close:xxx */ { 0, 255 }, /* old= r, write: 0, read:del, close: 0 */ { EPOLLIN, EPOLL_CTL_DEL }, /* old= r, write: 0, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write: 0, read:del, close:del */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= r, write: 0, read:del, close:xxx */ { 0, 255 }, /* old= r, write: 0, read:xxx, close: 0 */ { 0, 255 }, /* old= r, write: 0, read:xxx, close:add */ { 0, 255 }, /* old= r, write: 0, read:xxx, close:del */ { 0, 255 }, /* old= r, write: 0, read:xxx, close:xxx */ { 0, 255 }, /* old= r, write:add, read: 0, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= r, write:add, read: 0, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write:add, read: 0, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= r, write:add, read: 0, close:xxx */ { 0, 255 }, /* old= r, write:add, read:add, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= r, write:add, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write:add, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= r, write:add, read:add, close:xxx */ { 0, 255 }, /* old= r, write:add, read:del, close: 0 */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= r, write:add, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write:add, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= r, write:add, read:del, close:xxx */ { 0, 255 }, /* old= r, write:add, read:xxx, close: 0 */ { 0, 255 }, /* old= r, write:add, read:xxx, close:add */ { 0, 255 }, /* old= r, write:add, read:xxx, close:del */ { 0, 255 }, /* old= r, write:add, read:xxx, close:xxx */ { 0, 255 }, /* old= r, write:del, read: 0, close: 0 */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= r, write:del, read: 0, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write:del, read: 0, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= r, write:del, read: 0, close:xxx */ { 0, 255 }, /* old= r, write:del, read:add, close: 0 */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= r, write:del, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write:del, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= r, write:del, read:add, close:xxx */ { 0, 255 }, /* old= r, write:del, read:del, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_DEL }, /* old= r, write:del, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= r, write:del, read:del, close:del */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= r, write:del, read:del, close:xxx */ { 0, 255 }, /* old= r, write:del, read:xxx, close: 0 */ { 0, 255 }, /* old= r, write:del, read:xxx, close:add */ { 0, 255 }, /* old= r, write:del, read:xxx, close:del */ { 0, 255 }, /* old= r, write:del, read:xxx, close:xxx */ { 0, 255 }, /* old= r, write:xxx, read: 0, close: 0 */ { 0, 255 }, /* old= r, write:xxx, read: 0, close:add */ { 0, 255 }, /* old= r, write:xxx, read: 0, close:del */ { 0, 255 }, /* old= r, write:xxx, read: 0, close:xxx */ { 0, 255 }, /* old= r, write:xxx, read:add, close: 0 */ { 0, 255 }, /* old= r, write:xxx, read:add, close:add */ { 0, 255 }, /* old= r, write:xxx, read:add, close:del */ { 0, 255 }, /* old= r, write:xxx, read:add, close:xxx */ { 0, 255 }, /* old= r, write:xxx, read:del, close: 0 */ { 0, 255 }, /* old= r, write:xxx, read:del, close:add */ { 0, 255 }, /* old= r, write:xxx, read:del, close:del */ { 0, 255 }, /* old= r, write:xxx, read:del, close:xxx */ { 0, 255 }, /* old= r, write:xxx, read:xxx, close: 0 */ { 0, 255 }, /* old= r, write:xxx, read:xxx, close:add */ { 0, 255 }, /* old= r, write:xxx, read:xxx, close:del */ { 0, 255 }, /* old= r, write:xxx, read:xxx, close:xxx */ { 0, 255 }, /* old= w, write: 0, read: 0, close: 0 */ { 0, 0 }, /* old= w, write: 0, read: 0, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write: 0, read: 0, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write: 0, read: 0, close:xxx */ { 0, 255 }, /* old= w, write: 0, read:add, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write: 0, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write: 0, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write: 0, read:add, close:xxx */ { 0, 255 }, /* old= w, write: 0, read:del, close: 0 */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write: 0, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write: 0, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write: 0, read:del, close:xxx */ { 0, 255 }, /* old= w, write: 0, read:xxx, close: 0 */ { 0, 255 }, /* old= w, write: 0, read:xxx, close:add */ { 0, 255 }, /* old= w, write: 0, read:xxx, close:del */ { 0, 255 }, /* old= w, write: 0, read:xxx, close:xxx */ { 0, 255 }, /* old= w, write:add, read: 0, close: 0 */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write:add, read: 0, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write:add, read: 0, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write:add, read: 0, close:xxx */ { 0, 255 }, /* old= w, write:add, read:add, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write:add, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write:add, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write:add, read:add, close:xxx */ { 0, 255 }, /* old= w, write:add, read:del, close: 0 */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write:add, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write:add, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= w, write:add, read:del, close:xxx */ { 0, 255 }, /* old= w, write:add, read:xxx, close: 0 */ { 0, 255 }, /* old= w, write:add, read:xxx, close:add */ { 0, 255 }, /* old= w, write:add, read:xxx, close:del */ { 0, 255 }, /* old= w, write:add, read:xxx, close:xxx */ { 0, 255 }, /* old= w, write:del, read: 0, close: 0 */ { EPOLLOUT, EPOLL_CTL_DEL }, /* old= w, write:del, read: 0, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write:del, read: 0, close:del */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= w, write:del, read: 0, close:xxx */ { 0, 255 }, /* old= w, write:del, read:add, close: 0 */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= w, write:del, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write:del, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= w, write:del, read:add, close:xxx */ { 0, 255 }, /* old= w, write:del, read:del, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_DEL }, /* old= w, write:del, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= w, write:del, read:del, close:del */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= w, write:del, read:del, close:xxx */ { 0, 255 }, /* old= w, write:del, read:xxx, close: 0 */ { 0, 255 }, /* old= w, write:del, read:xxx, close:add */ { 0, 255 }, /* old= w, write:del, read:xxx, close:del */ { 0, 255 }, /* old= w, write:del, read:xxx, close:xxx */ { 0, 255 }, /* old= w, write:xxx, read: 0, close: 0 */ { 0, 255 }, /* old= w, write:xxx, read: 0, close:add */ { 0, 255 }, /* old= w, write:xxx, read: 0, close:del */ { 0, 255 }, /* old= w, write:xxx, read: 0, close:xxx */ { 0, 255 }, /* old= w, write:xxx, read:add, close: 0 */ { 0, 255 }, /* old= w, write:xxx, read:add, close:add */ { 0, 255 }, /* old= w, write:xxx, read:add, close:del */ { 0, 255 }, /* old= w, write:xxx, read:add, close:xxx */ { 0, 255 }, /* old= w, write:xxx, read:del, close: 0 */ { 0, 255 }, /* old= w, write:xxx, read:del, close:add */ { 0, 255 }, /* old= w, write:xxx, read:del, close:del */ { 0, 255 }, /* old= w, write:xxx, read:del, close:xxx */ { 0, 255 }, /* old= w, write:xxx, read:xxx, close: 0 */ { 0, 255 }, /* old= w, write:xxx, read:xxx, close:add */ { 0, 255 }, /* old= w, write:xxx, read:xxx, close:del */ { 0, 255 }, /* old= w, write:xxx, read:xxx, close:xxx */ { 0, 255 }, /* old= rw, write: 0, read: 0, close: 0 */ { 0, 0 }, /* old= rw, write: 0, read: 0, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write: 0, read: 0, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write: 0, read: 0, close:xxx */ { 0, 255 }, /* old= rw, write: 0, read:add, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write: 0, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write: 0, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write: 0, read:add, close:xxx */ { 0, 255 }, /* old= rw, write: 0, read:del, close: 0 */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write: 0, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write: 0, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write: 0, read:del, close:xxx */ { 0, 255 }, /* old= rw, write: 0, read:xxx, close: 0 */ { 0, 255 }, /* old= rw, write: 0, read:xxx, close:add */ { 0, 255 }, /* old= rw, write: 0, read:xxx, close:del */ { 0, 255 }, /* old= rw, write: 0, read:xxx, close:xxx */ { 0, 255 }, /* old= rw, write:add, read: 0, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write:add, read: 0, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write:add, read: 0, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write:add, read: 0, close:xxx */ { 0, 255 }, /* old= rw, write:add, read:add, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write:add, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write:add, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write:add, read:add, close:xxx */ { 0, 255 }, /* old= rw, write:add, read:del, close: 0 */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write:add, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write:add, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= rw, write:add, read:del, close:xxx */ { 0, 255 }, /* old= rw, write:add, read:xxx, close: 0 */ { 0, 255 }, /* old= rw, write:add, read:xxx, close:add */ { 0, 255 }, /* old= rw, write:add, read:xxx, close:del */ { 0, 255 }, /* old= rw, write:add, read:xxx, close:xxx */ { 0, 255 }, /* old= rw, write:del, read: 0, close: 0 */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= rw, write:del, read: 0, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write:del, read: 0, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= rw, write:del, read: 0, close:xxx */ { 0, 255 }, /* old= rw, write:del, read:add, close: 0 */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= rw, write:del, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write:del, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= rw, write:del, read:add, close:xxx */ { 0, 255 }, /* old= rw, write:del, read:del, close: 0 */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_DEL }, /* old= rw, write:del, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= rw, write:del, read:del, close:del */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= rw, write:del, read:del, close:xxx */ { 0, 255 }, /* old= rw, write:del, read:xxx, close: 0 */ { 0, 255 }, /* old= rw, write:del, read:xxx, close:add */ { 0, 255 }, /* old= rw, write:del, read:xxx, close:del */ { 0, 255 }, /* old= rw, write:del, read:xxx, close:xxx */ { 0, 255 }, /* old= rw, write:xxx, read: 0, close: 0 */ { 0, 255 }, /* old= rw, write:xxx, read: 0, close:add */ { 0, 255 }, /* old= rw, write:xxx, read: 0, close:del */ { 0, 255 }, /* old= rw, write:xxx, read: 0, close:xxx */ { 0, 255 }, /* old= rw, write:xxx, read:add, close: 0 */ { 0, 255 }, /* old= rw, write:xxx, read:add, close:add */ { 0, 255 }, /* old= rw, write:xxx, read:add, close:del */ { 0, 255 }, /* old= rw, write:xxx, read:add, close:xxx */ { 0, 255 }, /* old= rw, write:xxx, read:del, close: 0 */ { 0, 255 }, /* old= rw, write:xxx, read:del, close:add */ { 0, 255 }, /* old= rw, write:xxx, read:del, close:del */ { 0, 255 }, /* old= rw, write:xxx, read:del, close:xxx */ { 0, 255 }, /* old= rw, write:xxx, read:xxx, close: 0 */ { 0, 255 }, /* old= rw, write:xxx, read:xxx, close:add */ { 0, 255 }, /* old= rw, write:xxx, read:xxx, close:del */ { 0, 255 }, /* old= rw, write:xxx, read:xxx, close:xxx */ { 0, 255 }, /* old= c, write: 0, read: 0, close: 0 */ { 0, 0 }, /* old= c, write: 0, read: 0, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write: 0, read: 0, close:del */ { EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= c, write: 0, read: 0, close:xxx */ { 0, 255 }, /* old= c, write: 0, read:add, close: 0 */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write: 0, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write: 0, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= c, write: 0, read:add, close:xxx */ { 0, 255 }, /* old= c, write: 0, read:del, close: 0 */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write: 0, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write: 0, read:del, close:del */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= c, write: 0, read:del, close:xxx */ { 0, 255 }, /* old= c, write: 0, read:xxx, close: 0 */ { 0, 255 }, /* old= c, write: 0, read:xxx, close:add */ { 0, 255 }, /* old= c, write: 0, read:xxx, close:del */ { 0, 255 }, /* old= c, write: 0, read:xxx, close:xxx */ { 0, 255 }, /* old= c, write:add, read: 0, close: 0 */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:add, read: 0, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:add, read: 0, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= c, write:add, read: 0, close:xxx */ { 0, 255 }, /* old= c, write:add, read:add, close: 0 */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:add, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:add, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= c, write:add, read:add, close:xxx */ { 0, 255 }, /* old= c, write:add, read:del, close: 0 */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:add, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:add, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= c, write:add, read:del, close:xxx */ { 0, 255 }, /* old= c, write:add, read:xxx, close: 0 */ { 0, 255 }, /* old= c, write:add, read:xxx, close:add */ { 0, 255 }, /* old= c, write:add, read:xxx, close:del */ { 0, 255 }, /* old= c, write:add, read:xxx, close:xxx */ { 0, 255 }, /* old= c, write:del, read: 0, close: 0 */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:del, read: 0, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:del, read: 0, close:del */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= c, write:del, read: 0, close:xxx */ { 0, 255 }, /* old= c, write:del, read:add, close: 0 */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:del, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:del, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= c, write:del, read:add, close:xxx */ { 0, 255 }, /* old= c, write:del, read:del, close: 0 */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:del, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= c, write:del, read:del, close:del */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= c, write:del, read:del, close:xxx */ { 0, 255 }, /* old= c, write:del, read:xxx, close: 0 */ { 0, 255 }, /* old= c, write:del, read:xxx, close:add */ { 0, 255 }, /* old= c, write:del, read:xxx, close:del */ { 0, 255 }, /* old= c, write:del, read:xxx, close:xxx */ { 0, 255 }, /* old= c, write:xxx, read: 0, close: 0 */ { 0, 255 }, /* old= c, write:xxx, read: 0, close:add */ { 0, 255 }, /* old= c, write:xxx, read: 0, close:del */ { 0, 255 }, /* old= c, write:xxx, read: 0, close:xxx */ { 0, 255 }, /* old= c, write:xxx, read:add, close: 0 */ { 0, 255 }, /* old= c, write:xxx, read:add, close:add */ { 0, 255 }, /* old= c, write:xxx, read:add, close:del */ { 0, 255 }, /* old= c, write:xxx, read:add, close:xxx */ { 0, 255 }, /* old= c, write:xxx, read:del, close: 0 */ { 0, 255 }, /* old= c, write:xxx, read:del, close:add */ { 0, 255 }, /* old= c, write:xxx, read:del, close:del */ { 0, 255 }, /* old= c, write:xxx, read:del, close:xxx */ { 0, 255 }, /* old= c, write:xxx, read:xxx, close: 0 */ { 0, 255 }, /* old= c, write:xxx, read:xxx, close:add */ { 0, 255 }, /* old= c, write:xxx, read:xxx, close:del */ { 0, 255 }, /* old= c, write:xxx, read:xxx, close:xxx */ { 0, 255 }, /* old= cr, write: 0, read: 0, close: 0 */ { 0, 0 }, /* old= cr, write: 0, read: 0, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write: 0, read: 0, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= cr, write: 0, read: 0, close:xxx */ { 0, 255 }, /* old= cr, write: 0, read:add, close: 0 */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write: 0, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write: 0, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= cr, write: 0, read:add, close:xxx */ { 0, 255 }, /* old= cr, write: 0, read:del, close: 0 */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write: 0, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write: 0, read:del, close:del */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= cr, write: 0, read:del, close:xxx */ { 0, 255 }, /* old= cr, write: 0, read:xxx, close: 0 */ { 0, 255 }, /* old= cr, write: 0, read:xxx, close:add */ { 0, 255 }, /* old= cr, write: 0, read:xxx, close:del */ { 0, 255 }, /* old= cr, write: 0, read:xxx, close:xxx */ { 0, 255 }, /* old= cr, write:add, read: 0, close: 0 */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:add, read: 0, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:add, read: 0, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= cr, write:add, read: 0, close:xxx */ { 0, 255 }, /* old= cr, write:add, read:add, close: 0 */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:add, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:add, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= cr, write:add, read:add, close:xxx */ { 0, 255 }, /* old= cr, write:add, read:del, close: 0 */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:add, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:add, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= cr, write:add, read:del, close:xxx */ { 0, 255 }, /* old= cr, write:add, read:xxx, close: 0 */ { 0, 255 }, /* old= cr, write:add, read:xxx, close:add */ { 0, 255 }, /* old= cr, write:add, read:xxx, close:del */ { 0, 255 }, /* old= cr, write:add, read:xxx, close:xxx */ { 0, 255 }, /* old= cr, write:del, read: 0, close: 0 */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:del, read: 0, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:del, read: 0, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= cr, write:del, read: 0, close:xxx */ { 0, 255 }, /* old= cr, write:del, read:add, close: 0 */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:del, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:del, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= cr, write:del, read:add, close:xxx */ { 0, 255 }, /* old= cr, write:del, read:del, close: 0 */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:del, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cr, write:del, read:del, close:del */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= cr, write:del, read:del, close:xxx */ { 0, 255 }, /* old= cr, write:del, read:xxx, close: 0 */ { 0, 255 }, /* old= cr, write:del, read:xxx, close:add */ { 0, 255 }, /* old= cr, write:del, read:xxx, close:del */ { 0, 255 }, /* old= cr, write:del, read:xxx, close:xxx */ { 0, 255 }, /* old= cr, write:xxx, read: 0, close: 0 */ { 0, 255 }, /* old= cr, write:xxx, read: 0, close:add */ { 0, 255 }, /* old= cr, write:xxx, read: 0, close:del */ { 0, 255 }, /* old= cr, write:xxx, read: 0, close:xxx */ { 0, 255 }, /* old= cr, write:xxx, read:add, close: 0 */ { 0, 255 }, /* old= cr, write:xxx, read:add, close:add */ { 0, 255 }, /* old= cr, write:xxx, read:add, close:del */ { 0, 255 }, /* old= cr, write:xxx, read:add, close:xxx */ { 0, 255 }, /* old= cr, write:xxx, read:del, close: 0 */ { 0, 255 }, /* old= cr, write:xxx, read:del, close:add */ { 0, 255 }, /* old= cr, write:xxx, read:del, close:del */ { 0, 255 }, /* old= cr, write:xxx, read:del, close:xxx */ { 0, 255 }, /* old= cr, write:xxx, read:xxx, close: 0 */ { 0, 255 }, /* old= cr, write:xxx, read:xxx, close:add */ { 0, 255 }, /* old= cr, write:xxx, read:xxx, close:del */ { 0, 255 }, /* old= cr, write:xxx, read:xxx, close:xxx */ { 0, 255 }, /* old= cw, write: 0, read: 0, close: 0 */ { 0, 0 }, /* old= cw, write: 0, read: 0, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write: 0, read: 0, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= cw, write: 0, read: 0, close:xxx */ { 0, 255 }, /* old= cw, write: 0, read:add, close: 0 */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write: 0, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write: 0, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= cw, write: 0, read:add, close:xxx */ { 0, 255 }, /* old= cw, write: 0, read:del, close: 0 */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write: 0, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write: 0, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= cw, write: 0, read:del, close:xxx */ { 0, 255 }, /* old= cw, write: 0, read:xxx, close: 0 */ { 0, 255 }, /* old= cw, write: 0, read:xxx, close:add */ { 0, 255 }, /* old= cw, write: 0, read:xxx, close:del */ { 0, 255 }, /* old= cw, write: 0, read:xxx, close:xxx */ { 0, 255 }, /* old= cw, write:add, read: 0, close: 0 */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:add, read: 0, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:add, read: 0, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= cw, write:add, read: 0, close:xxx */ { 0, 255 }, /* old= cw, write:add, read:add, close: 0 */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:add, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:add, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old= cw, write:add, read:add, close:xxx */ { 0, 255 }, /* old= cw, write:add, read:del, close: 0 */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:add, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:add, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old= cw, write:add, read:del, close:xxx */ { 0, 255 }, /* old= cw, write:add, read:xxx, close: 0 */ { 0, 255 }, /* old= cw, write:add, read:xxx, close:add */ { 0, 255 }, /* old= cw, write:add, read:xxx, close:del */ { 0, 255 }, /* old= cw, write:add, read:xxx, close:xxx */ { 0, 255 }, /* old= cw, write:del, read: 0, close: 0 */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:del, read: 0, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:del, read: 0, close:del */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= cw, write:del, read: 0, close:xxx */ { 0, 255 }, /* old= cw, write:del, read:add, close: 0 */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:del, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:del, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old= cw, write:del, read:add, close:xxx */ { 0, 255 }, /* old= cw, write:del, read:del, close: 0 */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:del, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old= cw, write:del, read:del, close:del */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old= cw, write:del, read:del, close:xxx */ { 0, 255 }, /* old= cw, write:del, read:xxx, close: 0 */ { 0, 255 }, /* old= cw, write:del, read:xxx, close:add */ { 0, 255 }, /* old= cw, write:del, read:xxx, close:del */ { 0, 255 }, /* old= cw, write:del, read:xxx, close:xxx */ { 0, 255 }, /* old= cw, write:xxx, read: 0, close: 0 */ { 0, 255 }, /* old= cw, write:xxx, read: 0, close:add */ { 0, 255 }, /* old= cw, write:xxx, read: 0, close:del */ { 0, 255 }, /* old= cw, write:xxx, read: 0, close:xxx */ { 0, 255 }, /* old= cw, write:xxx, read:add, close: 0 */ { 0, 255 }, /* old= cw, write:xxx, read:add, close:add */ { 0, 255 }, /* old= cw, write:xxx, read:add, close:del */ { 0, 255 }, /* old= cw, write:xxx, read:add, close:xxx */ { 0, 255 }, /* old= cw, write:xxx, read:del, close: 0 */ { 0, 255 }, /* old= cw, write:xxx, read:del, close:add */ { 0, 255 }, /* old= cw, write:xxx, read:del, close:del */ { 0, 255 }, /* old= cw, write:xxx, read:del, close:xxx */ { 0, 255 }, /* old= cw, write:xxx, read:xxx, close: 0 */ { 0, 255 }, /* old= cw, write:xxx, read:xxx, close:add */ { 0, 255 }, /* old= cw, write:xxx, read:xxx, close:del */ { 0, 255 }, /* old= cw, write:xxx, read:xxx, close:xxx */ { 0, 255 }, /* old=crw, write: 0, read: 0, close: 0 */ { 0, 0 }, /* old=crw, write: 0, read: 0, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write: 0, read: 0, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old=crw, write: 0, read: 0, close:xxx */ { 0, 255 }, /* old=crw, write: 0, read:add, close: 0 */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write: 0, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write: 0, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old=crw, write: 0, read:add, close:xxx */ { 0, 255 }, /* old=crw, write: 0, read:del, close: 0 */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write: 0, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write: 0, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old=crw, write: 0, read:del, close:xxx */ { 0, 255 }, /* old=crw, write: 0, read:xxx, close: 0 */ { 0, 255 }, /* old=crw, write: 0, read:xxx, close:add */ { 0, 255 }, /* old=crw, write: 0, read:xxx, close:del */ { 0, 255 }, /* old=crw, write: 0, read:xxx, close:xxx */ { 0, 255 }, /* old=crw, write:add, read: 0, close: 0 */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:add, read: 0, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:add, read: 0, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old=crw, write:add, read: 0, close:xxx */ { 0, 255 }, /* old=crw, write:add, read:add, close: 0 */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:add, read:add, close:add */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:add, read:add, close:del */ { EPOLLIN|EPOLLOUT, EPOLL_CTL_MOD }, /* old=crw, write:add, read:add, close:xxx */ { 0, 255 }, /* old=crw, write:add, read:del, close: 0 */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:add, read:del, close:add */ { EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:add, read:del, close:del */ { EPOLLOUT, EPOLL_CTL_MOD }, /* old=crw, write:add, read:del, close:xxx */ { 0, 255 }, /* old=crw, write:add, read:xxx, close: 0 */ { 0, 255 }, /* old=crw, write:add, read:xxx, close:add */ { 0, 255 }, /* old=crw, write:add, read:xxx, close:del */ { 0, 255 }, /* old=crw, write:add, read:xxx, close:xxx */ { 0, 255 }, /* old=crw, write:del, read: 0, close: 0 */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:del, read: 0, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:del, read: 0, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old=crw, write:del, read: 0, close:xxx */ { 0, 255 }, /* old=crw, write:del, read:add, close: 0 */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:del, read:add, close:add */ { EPOLLIN|EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:del, read:add, close:del */ { EPOLLIN, EPOLL_CTL_MOD }, /* old=crw, write:del, read:add, close:xxx */ { 0, 255 }, /* old=crw, write:del, read:del, close: 0 */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:del, read:del, close:add */ { EPOLLRDHUP, EPOLL_CTL_MOD }, /* old=crw, write:del, read:del, close:del */ { EPOLLIN|EPOLLOUT|EPOLLRDHUP, EPOLL_CTL_DEL }, /* old=crw, write:del, read:del, close:xxx */ { 0, 255 }, /* old=crw, write:del, read:xxx, close: 0 */ { 0, 255 }, /* old=crw, write:del, read:xxx, close:add */ { 0, 255 }, /* old=crw, write:del, read:xxx, close:del */ { 0, 255 }, /* old=crw, write:del, read:xxx, close:xxx */ { 0, 255 }, /* old=crw, write:xxx, read: 0, close: 0 */ { 0, 255 }, /* old=crw, write:xxx, read: 0, close:add */ { 0, 255 }, /* old=crw, write:xxx, read: 0, close:del */ { 0, 255 }, /* old=crw, write:xxx, read: 0, close:xxx */ { 0, 255 }, /* old=crw, write:xxx, read:add, close: 0 */ { 0, 255 }, /* old=crw, write:xxx, read:add, close:add */ { 0, 255 }, /* old=crw, write:xxx, read:add, close:del */ { 0, 255 }, /* old=crw, write:xxx, read:add, close:xxx */ { 0, 255 }, /* old=crw, write:xxx, read:del, close: 0 */ { 0, 255 }, /* old=crw, write:xxx, read:del, close:add */ { 0, 255 }, /* old=crw, write:xxx, read:del, close:del */ { 0, 255 }, /* old=crw, write:xxx, read:del, close:xxx */ { 0, 255 }, /* old=crw, write:xxx, read:xxx, close: 0 */ { 0, 255 }, /* old=crw, write:xxx, read:xxx, close:add */ { 0, 255 }, /* old=crw, write:xxx, read:xxx, close:del */ { 0, 255 }, /* old=crw, write:xxx, read:xxx, close:xxx */ { 0, 255 }, }; #endif �����������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evbuffer-internal.h������������������������������������������0000664�0000000�0000000�00000031006�15270523712�0023565�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2000-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVBUFFER_INTERNAL_H_INCLUDED_ #define EVBUFFER_INTERNAL_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif #include "event2/event-config.h" #include "evconfig-private.h" #include "event2/util.h" #include "event2/event_struct.h" #include "util-internal.h" #include "defer-internal.h" /* Experimental cb flag: "never deferred." Implementation note: * these callbacks may get an inaccurate view of n_del/n_added in their * arguments. */ #define EVBUFFER_CB_NODEFER 2 #ifdef _WIN32 #include <winsock2.h> #endif #include <sys/queue.h> /* Minimum allocation for a chain. We define this so that we're burning no * more than 5% of each allocation on overhead. It would be nice to lose even * less space, though. */ #if EVENT__SIZEOF_VOID_P < 8 #define MIN_BUFFER_SIZE 512 #else #define MIN_BUFFER_SIZE 1024 #endif /** A single evbuffer callback for an evbuffer. This function will be invoked * when bytes are added to or removed from the evbuffer. */ struct evbuffer_cb_entry { /** Structures to implement a doubly-linked queue of callbacks */ LIST_ENTRY(evbuffer_cb_entry) next; /** The callback function to invoke when this callback is called. If EVBUFFER_CB_OBSOLETE is set in flags, the cb_obsolete field is valid; otherwise, cb_func is valid. */ union { evbuffer_cb_func cb_func; evbuffer_cb cb_obsolete; } cb; /** Argument to pass to cb. */ void *cbarg; /** Currently set flags on this callback. */ ev_uint32_t flags; }; struct bufferevent; struct evbuffer_chain; struct evbuffer { /** The first chain in this buffer's linked list of chains. */ struct evbuffer_chain *first; /** The last chain in this buffer's linked list of chains. */ struct evbuffer_chain *last; /** Pointer to the next pointer pointing at the 'last_with_data' chain. * * To unpack: * * The last_with_data chain is the last chain that has any data in it. * If all chains in the buffer are empty, it is the first chain. * If the buffer has no chains, it is NULL. * * The last_with_datap pointer points at _whatever 'next' pointer_ * points at the last_with_datap chain. If the last_with_data chain * is the first chain, or it is NULL, then the last_with_datap pointer * is &buf->first. */ struct evbuffer_chain **last_with_datap; /** Total amount of bytes stored in all chains.*/ size_t total_len; /** Number of bytes we have added to the buffer since we last tried to * invoke callbacks. */ size_t n_add_for_cb; /** Number of bytes we have removed from the buffer since we last * tried to invoke callbacks. */ size_t n_del_for_cb; #ifndef EVENT__DISABLE_THREAD_SUPPORT /** A lock used to mediate access to this buffer. */ void *lock; #endif /** True iff we should free the lock field when we free this * evbuffer. */ unsigned own_lock : 1; /** True iff we should not allow changes to the front of the buffer * (drains or prepends). */ unsigned freeze_start : 1; /** True iff we should not allow changes to the end of the buffer * (appends) */ unsigned freeze_end : 1; /** True iff this evbuffer's callbacks are not invoked immediately * upon a change in the buffer, but instead are deferred to be invoked * from the event_base's loop. Useful for preventing enormous stack * overflows when we have mutually recursive callbacks, and for * serializing callbacks in a single thread. */ unsigned deferred_cbs : 1; #ifdef _WIN32 /** True iff this buffer is set up for overlapped IO. */ unsigned is_overlapped : 1; #endif /** Zero or more EVBUFFER_FLAG_* bits */ ev_uint32_t flags; /** Used to implement deferred callbacks. */ struct event_base *cb_queue; /** A reference count on this evbuffer. When the reference count * reaches 0, the buffer is destroyed. Manipulated with * evbuffer_incref and evbuffer_decref_and_unlock and * evbuffer_free. */ int refcnt; /** A struct event_callback handle to make all of this buffer's callbacks * invoked from the event loop. */ struct event_callback deferred; /** A doubly-linked-list of callback functions */ LIST_HEAD(evbuffer_cb_queue, evbuffer_cb_entry) callbacks; /** The parent bufferevent object this evbuffer belongs to. * NULL if the evbuffer stands alone. */ struct bufferevent *parent; }; #if EVENT__SIZEOF_OFF_T < EVENT__SIZEOF_SIZE_T typedef ev_ssize_t ev_misalign_t; #define EVBUFFER_CHAIN_MAX ((size_t)EV_SSIZE_MAX) #else typedef ev_off_t ev_misalign_t; #if EVENT__SIZEOF_OFF_T > EVENT__SIZEOF_SIZE_T #define EVBUFFER_CHAIN_MAX EV_SIZE_MAX #else #define EVBUFFER_CHAIN_MAX ((size_t)EV_SSIZE_MAX) #endif #endif /** A single item in an evbuffer. */ struct evbuffer_chain { /** points to next buffer in the chain */ struct evbuffer_chain *next; /** total allocation available in the buffer field. */ size_t buffer_len; /** unused space at the beginning of buffer or an offset into a * file for sendfile buffers. */ ev_misalign_t misalign; /** Offset into buffer + misalign at which to start writing. * In other words, the total number of bytes actually stored * in buffer. */ size_t off; /** Set if special handling is required for this chain */ unsigned flags; #define EVBUFFER_FILESEGMENT 0x0001 /**< A chain used for a file segment */ #define EVBUFFER_SENDFILE 0x0002 /**< a chain used with sendfile */ #define EVBUFFER_REFERENCE 0x0004 /**< a chain with a mem reference */ #define EVBUFFER_IMMUTABLE 0x0008 /**< read-only chain */ /** a chain that mustn't be reallocated or freed, or have its contents * memmoved, until the chain is un-pinned. */ #define EVBUFFER_MEM_PINNED_R 0x0010 #define EVBUFFER_MEM_PINNED_W 0x0020 #define EVBUFFER_MEM_PINNED_ANY (EVBUFFER_MEM_PINNED_R|EVBUFFER_MEM_PINNED_W) /** a chain that should be freed, but can't be freed until it is * un-pinned. */ #define EVBUFFER_DANGLING 0x0040 /** a chain that is a referenced copy of another chain */ #define EVBUFFER_MULTICAST 0x0080 /** number of references to this chain */ int refcnt; /** Usually points to the read-write memory belonging to this * buffer allocated as part of the evbuffer_chain allocation. * For mmap, this can be a read-only buffer and * EVBUFFER_IMMUTABLE will be set in flags. For sendfile, it * may point to NULL. */ unsigned char *buffer; }; /** callback for a reference chain; lets us know what to do with it when * we're done with it. Lives at the end of an evbuffer_chain with the * EVBUFFER_REFERENCE flag set */ struct evbuffer_chain_reference { evbuffer_ref_cleanup_cb cleanupfn; void *extra; }; /** File segment for a file-segment chain. Lives at the end of an * evbuffer_chain with the EVBUFFER_FILESEGMENT flag set. */ struct evbuffer_chain_file_segment { struct evbuffer_file_segment *segment; #ifdef _WIN32 /** If we're using CreateFileMapping, this is the handle to the view. */ HANDLE view_handle; #endif }; /* Declared in event2/buffer.h; defined here. */ struct evbuffer_file_segment { void *lock; /**< lock prevent concurrent access to refcnt */ int refcnt; /**< Reference count for this file segment */ unsigned flags; /**< combination of EVBUF_FS_* flags */ /** What kind of file segment is this? */ unsigned can_sendfile : 1; unsigned is_mapping : 1; /** The fd that we read the data from. */ int fd; /** If we're using mmap, this is the raw mapped memory. */ void *mapping; #ifdef _WIN32 /** If we're using CreateFileMapping, this is the mapping */ HANDLE mapping_handle; #endif /** If we're using mmap or IO, this is the content of the file * segment. */ char *contents; /** Position of this segment within the file. */ ev_off_t file_offset; /** If we're using mmap, this is the offset within 'mapping' where * this data segment begins. */ ev_off_t mmap_offset; /** The length of this segment. */ ev_off_t length; /** Cleanup callback function */ evbuffer_file_segment_cleanup_cb cleanup_cb; /** Argument to be pass to cleanup callback function */ void *cleanup_cb_arg; }; /** Information about the multicast parent of a chain. Lives at the * end of an evbuffer_chain with the EVBUFFER_MULTICAST flag set. */ struct evbuffer_multicast_parent { /** source buffer the multicast parent belongs to */ struct evbuffer *source; /** multicast parent for this chain */ struct evbuffer_chain *parent; }; #define EVBUFFER_CHAIN_SIZE sizeof(struct evbuffer_chain) /** Return a pointer to extra data allocated along with an evbuffer. */ #define EVBUFFER_CHAIN_EXTRA(t, c) (t *)((struct evbuffer_chain *)(c) + 1) /** Assert that we are holding the lock on an evbuffer */ #define ASSERT_EVBUFFER_LOCKED(buffer) \ EVLOCK_ASSERT_LOCKED((buffer)->lock) #define EVBUFFER_LOCK(buffer) \ do { \ EVLOCK_LOCK((buffer)->lock, 0); \ } while (0) #define EVBUFFER_UNLOCK(buffer) \ do { \ EVLOCK_UNLOCK((buffer)->lock, 0); \ } while (0) #define EVBUFFER_LOCK2(buffer1, buffer2) \ do { \ EVLOCK_LOCK2((buffer1)->lock, (buffer2)->lock, 0, 0); \ } while (0) #define EVBUFFER_UNLOCK2(buffer1, buffer2) \ do { \ EVLOCK_UNLOCK2((buffer1)->lock, (buffer2)->lock, 0, 0); \ } while (0) /** Increase the reference count of buf by one. */ void evbuffer_incref_(struct evbuffer *buf); /** Increase the reference count of buf by one and acquire the lock. */ void evbuffer_incref_and_lock_(struct evbuffer *buf); /** Pin a single buffer chain using a given flag. A pinned chunk may not be * moved or freed until it is unpinned. */ void evbuffer_chain_pin_(struct evbuffer_chain *chain, unsigned flag); /** Unpin a single buffer chain using a given flag. */ void evbuffer_chain_unpin_(struct evbuffer_chain *chain, unsigned flag); /** As evbuffer_free, but requires that we hold a lock on the buffer, and * releases the lock before freeing it and the buffer. */ void evbuffer_decref_and_unlock_(struct evbuffer *buffer); /** As evbuffer_expand, but does not guarantee that the newly allocated memory * is contiguous. Instead, it may be split across two or more chunks. */ int evbuffer_expand_fast_(struct evbuffer *, size_t, int); /** Helper: prepares for a readv/WSARecv call by expanding the buffer to * hold enough memory to read 'howmuch' bytes in possibly noncontiguous memory. * Sets up the one or two iovecs in 'vecs' to point to the free memory and its * extent, and *chainp to point to the first chain that we'll try to read into. * Returns the number of vecs used. */ int evbuffer_read_setup_vecs_(struct evbuffer *buf, ev_ssize_t howmuch, struct evbuffer_iovec *vecs, int n_vecs, struct evbuffer_chain ***chainp, int exact); /* Helper macro: copies an evbuffer_iovec in ei to a win32 WSABUF in i. */ #define WSABUF_FROM_EVBUFFER_IOV(i,ei) do { \ (i)->buf = (ei)->iov_base; \ (i)->len = (unsigned long)(ei)->iov_len; \ } while (0) /* XXXX the cast above is safe for now, but not if we allow mmaps on win64. * See note in buffer_iocp's launch_write function */ /** Set the parent bufferevent object for buf to bev */ void evbuffer_set_parent_(struct evbuffer *buf, struct bufferevent *bev); void evbuffer_invoke_callbacks_(struct evbuffer *buf); int evbuffer_get_callbacks_(struct evbuffer *buffer, struct event_callback **cbs, int max_cbs); #ifdef __cplusplus } #endif #endif /* EVBUFFER_INTERNAL_H_INCLUDED_ */ ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evconfig-private.h.cmake�������������������������������������0000664�0000000�0000000�00000001660�15270523712�0024501�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������ #ifndef EVCONFIG_PRIVATE_H_INCLUDED_ #define EVCONFIG_PRIVATE_H_INCLUDED_ /* Enable extensions on AIX 3, Interix. */ #cmakedefine _ALL_SOURCE /* Enable GNU extensions on systems that have them. */ #cmakedefine _GNU_SOURCE 1 /* Enable threading extensions on Solaris. */ #cmakedefine _POSIX_PTHREAD_SEMANTICS 1 /* Enable extensions on HP NonStop. */ #cmakedefine _TANDEM_SOURCE 1 /* Enable general extensions on Solaris. */ #cmakedefine __EXTENSIONS__ /* Number of bits in a file offset, on hosts where this is settable. */ #cmakedefine _FILE_OFFSET_BITS 1 /* Define for large files, on AIX-style hosts. */ #cmakedefine _LARGE_FILES 1 /* Define to 1 if on MINIX. */ #cmakedefine _MINIX 1 /* Define to 2 if the system does not provide POSIX.1 features except with this defined. */ #cmakedefine _POSIX_1_SOURCE 1 /* Define to 1 if you need to in order for `stat' and other things to work. */ #cmakedefine _POSIX_SOURCE 1 #endif ��������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evconfig-private.h.in����������������������������������������0000664�0000000�0000000�00000002326�15270523712�0024027�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* evconfig-private.h template - see "Configuration Header Templates" */ /* in AC manual. Kevin Bowling <kevin.bowling@kev009.com */ #ifndef EVCONFIG_PRIVATE_H_INCLUDED_ #define EVCONFIG_PRIVATE_H_INCLUDED_ /* Enable extensions on AIX 3, Interix. */ #ifndef _ALL_SOURCE # undef _ALL_SOURCE #endif /* Enable GNU extensions on systems that have them. */ #ifndef _GNU_SOURCE # undef _GNU_SOURCE #endif /* Enable threading extensions on Solaris. */ #ifndef _POSIX_PTHREAD_SEMANTICS # undef _POSIX_PTHREAD_SEMANTICS #endif /* Enable extensions on HP NonStop. */ #ifndef _TANDEM_SOURCE # undef _TANDEM_SOURCE #endif /* Enable general extensions on Solaris. */ #ifndef __EXTENSIONS__ # undef __EXTENSIONS__ #endif /* Number of bits in a file offset, on hosts where this is settable. */ #undef _FILE_OFFSET_BITS /* Define for large files, on AIX-style hosts. */ #undef _LARGE_FILES /* Define to 1 if on MINIX. */ #ifndef _MINIX #undef _MINIX #endif /* Define to 2 if the system does not provide POSIX.1 features except with this defined. */ #ifndef _POSIX_1_SOURCE #undef _POSIX_1_SOURCE #endif /* Define to 1 if you need to in order for `stat' and other things to work. */ #ifndef _POSIX_SOURCE #undef _POSIX_SOURCE #endif #endif ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evdns.3������������������������������������������������������0000664�0000000�0000000�00000026453�15270523712�0021213�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������.\" .\" Copyright (c) 2006 Niels Provos <provos@citi.umich.edu> .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" 3. The name of the author may not be used to endorse or promote products .\" derived from this software without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, .\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY .\" AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL .\" THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, .\" EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, .\" PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; .\" OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, .\" WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF .\" ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .Dd October 7, 2006 .Dt EVDNS 3 .Os .Sh NAME .Nm evdns_init .Nm evdns_shutdown .Nm evdns_err_to_string .Nm evdns_nameserver_add .Nm evdns_count_nameservers .Nm evdns_clear_nameservers_and_suspend .Nm evdns_resume .Nm evdns_nameserver_ip_add .Nm evdns_resolve_ipv4 .Nm evdns_resolve_reverse .Nm evdns_resolv_conf_parse .Nm evdns_config_windows_nameservers .Nm evdns_search_clear .Nm evdns_search_add .Nm evdns_search_ndots_set .Nm evdns_set_log_fn .Nd asynchronous functions for DNS resolution. .Sh SYNOPSIS .Fd #include <sys/time.h> .Fd #include <event.h> .Fd #include <evdns.h> .Ft int .Fn evdns_init .Ft void .Fn evdns_shutdown "int fail_requests" .Ft "const char *" .Fn evdns_err_to_string "int err" .Ft int .Fn evdns_nameserver_add "unsigned long int address" .Ft int .Fn evdns_count_nameservers .Ft int .Fn evdns_clear_nameservers_and_suspend .Ft int .Fn evdns_resume .Ft int .Fn evdns_nameserver_ip_add(const char *ip_as_string); .Ft int .Fn evdns_resolve_ipv4 "const char *name" "int flags" "evdns_callback_type callback" "void *ptr" .Ft int .Fn evdns_resolve_reverse "struct in_addr *in" "int flags" "evdns_callback_type callback" "void *ptr" .Ft int .Fn evdns_resolv_conf_parse "int flags" "const char *" .Ft void .Fn evdns_search_clear .Ft void .Fn evdns_search_add "const char *domain" .Ft void .Fn evdns_search_ndots_set "const int ndots" .Ft void .Fn evdns_set_log_fn "evdns_debug_log_fn_type fn" .Ft int .Fn evdns_config_windows_nameservers .Sh DESCRIPTION Welcome, gentle reader .Pp Async DNS lookups are really a whole lot harder than they should be, mostly stemming from the fact that the libc resolver has never been very good at them. Before you use this library you should see if libc can do the job for you with the modern async call getaddrinfo_a (see http://www.imperialviolet.org/page25.html#e498). Otherwise, please continue. .Pp This code is based on libevent and you must call event_init before any of the APIs in this file. You must also seed the OpenSSL random source if you are using OpenSSL for ids (see below). .Pp This library is designed to be included and shipped with your source code. You statically link with it. You should also test for the existence of strtok_r and define HAVE_STRTOK_R if you have it. .Pp The DNS protocol requires a good source of id numbers and these numbers should be unpredictable for spoofing reasons. There are three methods for generating them here and you must define exactly one of them. In increasing order of preference: .Pp .Bl -tag -width "DNS_USE_GETTIMEOFDAY_FOR_ID" -compact -offset indent .It DNS_USE_GETTIMEOFDAY_FOR_ID Using the bottom 16 bits of the usec result from gettimeofday. This is a pretty poor solution but should work anywhere. .It DNS_USE_CPU_CLOCK_FOR_ID Using the bottom 16 bits of the nsec result from the CPU's time counter. This is better, but may not work everywhere. Requires POSIX realtime support and you'll need to link against -lrt on glibc systems at least. .It DNS_USE_OPENSSL_FOR_ID Uses the OpenSSL RAND_bytes call to generate the data. You must have seeded the pool before making any calls to this library. .El .Pp The library keeps track of the state of nameservers and will avoid them when they go down. Otherwise it will round robin between them. .Pp Quick start guide: #include "evdns.h" void callback(int result, char type, int count, int ttl, void *addresses, void *arg); evdns_resolv_conf_parse(DNS_OPTIONS_ALL, "/etc/resolv.conf"); evdns_resolve("www.hostname.com", 0, callback, NULL); .Pp When the lookup is complete the callback function is called. The first argument will be one of the DNS_ERR_* defines in evdns.h. Hopefully it will be DNS_ERR_NONE, in which case type will be DNS_IPv4_A, count will be the number of IP addresses, ttl is the time which the data can be cached for (in seconds), addresses will point to an array of uint32_t's and arg will be whatever you passed to evdns_resolve. .Pp Searching: .Pp In order for this library to be a good replacement for glibc's resolver it supports searching. This involves setting a list of default domains, in which names will be queried for. The number of dots in the query name determines the order in which this list is used. .Pp Searching appears to be a single lookup from the point of view of the API, although many DNS queries may be generated from a single call to evdns_resolve. Searching can also drastically slow down the resolution of names. .Pp To disable searching: .Bl -enum -compact -offset indent .It Never set it up. If you never call .Fn evdns_resolv_conf_parse, .Fn evdns_init, or .Fn evdns_search_add then no searching will occur. .It If you do call .Fn evdns_resolv_conf_parse then don't pass .Va DNS_OPTION_SEARCH (or .Va DNS_OPTIONS_ALL, which implies it). .It When calling .Fn evdns_resolve, pass the .Va DNS_QUERY_NO_SEARCH flag. .El .Pp The order of searches depends on the number of dots in the name. If the number is greater than the ndots setting then the names is first tried globally. Otherwise each search domain is appended in turn. .Pp The ndots setting can either be set from a resolv.conf, or by calling evdns_search_ndots_set. .Pp For example, with ndots set to 1 (the default) and a search domain list of ["myhome.net"]: Query: www Order: www.myhome.net, www. .Pp Query: www.abc Order: www.abc., www.abc.myhome.net .Pp .Sh API reference .Pp .Bl -tag -width 0123456 .It Ft int Fn evdns_init Initializes support for non-blocking name resolution by calling .Fn evdns_resolv_conf_parse on UNIX and .Fn evdns_config_windows_nameservers on Windows. .It Ft int Fn evdns_nameserver_add "unsigned long int address" Add a nameserver. The address should be an IP address in network byte order. The type of address is chosen so that it matches in_addr.s_addr. Returns non-zero on error. .It Ft int Fn evdns_nameserver_ip_add "const char *ip_as_string" This wraps the above function by parsing a string as an IP address and adds it as a nameserver. Returns non-zero on error .It Ft int Fn evdns_resolve "const char *name" "int flags" "evdns_callback_type callback" "void *ptr" Resolve a name. The name parameter should be a DNS name. The flags parameter should be 0, or DNS_QUERY_NO_SEARCH which disables searching for this query. (see defn of searching above). .Pp The callback argument is a function which is called when this query completes and ptr is an argument which is passed to that callback function. .Pp Returns non-zero on error .It Ft void Fn evdns_search_clear Clears the list of search domains .It Ft void Fn evdns_search_add "const char *domain" Add a domain to the list of search domains .It Ft void Fn evdns_search_ndots_set "int ndots" Set the number of dots which, when found in a name, causes the first query to be without any search domain. .It Ft int Fn evdns_count_nameservers "void" Return the number of configured nameservers (not necessarily the number of running nameservers). This is useful for double-checking whether our calls to the various nameserver configuration functions have been successful. .It Ft int Fn evdns_clear_nameservers_and_suspend "void" Remove all currently configured nameservers, and suspend all pending resolves. Resolves will not necessarily be re-attempted until evdns_resume() is called. .It Ft int Fn evdns_resume "void" Re-attempt resolves left in limbo after an earlier call to evdns_clear_nameservers_and_suspend(). .It Ft int Fn evdns_config_windows_nameservers "void" Attempt to configure a set of nameservers based on platform settings on a win32 host. Preferentially tries to use GetNetworkParams; if that fails, looks in the registry. Returns 0 on success, nonzero on failure. .It Ft int Fn evdns_resolv_conf_parse "int flags" "const char *filename" Parse a resolv.conf like file from the given filename. .Pp See the man page for resolv.conf for the format of this file. The flags argument determines what information is parsed from this file: .Bl -tag -width "DNS_OPTION_NAMESERVERS" -offset indent -compact -nested .It DNS_OPTION_SEARCH domain, search and ndots options .It DNS_OPTION_NAMESERVERS nameserver lines .It DNS_OPTION_MISC timeout and attempts options .It DNS_OPTIONS_ALL all of the above .El .Pp The following directives are not parsed from the file: sortlist, rotate, no-check-names, inet6, debug .Pp Returns non-zero on error: .Bl -tag -width "0" -offset indent -compact -nested .It 0 no errors .It 1 failed to open file .It 2 failed to stat file .It 3 file too large .It 4 out of memory .It 5 short read from file .El .El .Sh Internals: Requests are kept in two queues. The first is the inflight queue. In this queue requests have an allocated transaction id and nameserver. They will soon be transmitted if they haven't already been. .Pp The second is the waiting queue. The size of the inflight ring is limited and all other requests wait in waiting queue for space. This bounds the number of concurrent requests so that we don't flood the nameserver. Several algorithms require a full walk of the inflight queue and so bounding its size keeps thing going nicely under huge (many thousands of requests) loads. .Pp If a nameserver loses too many requests it is considered down and we try not to use it. After a while we send a probe to that nameserver (a lookup for google.com) and, if it replies, we consider it working again. If the nameserver fails a probe we wait longer to try again with the next probe. .Sh SEE ALSO .Xr event 3 , .Xr gethostbyname 3 , .Xr resolv.conf 5 .Sh HISTORY The .Nm evdns API was developed by Adam Langley on top of the .Nm libevent API. The code was integrate into .Nm Tor by Nick Mathewson and finally put into .Nm libevent itself by Niels Provos. .Sh AUTHORS The .Nm evdns API and code was written by Adam Langley with significant contributions by Nick Mathewson. .Sh BUGS This documentation is neither complete nor authoritative. If you are in doubt about the usage of this API then check the source code to find out how it works, write up the missing piece of documentation and send it to me for inclusion in this man page. ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evdns.c������������������������������������������������������0000664�0000000�0000000�00000376632�15270523712�0021302�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* Copyright 2006-2007 Niels Provos * Copyright 2007-2012 Nick Mathewson and Niels Provos * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Based on software by Adam Langly. Adam's original message: * * Async DNS Library * Adam Langley <agl@imperialviolet.org> * http://www.imperialviolet.org/eventdns.html * Public Domain code * * This software is Public Domain. To view a copy of the public domain dedication, * visit http://creativecommons.org/licenses/publicdomain/ or send a letter to * Creative Commons, 559 Nathan Abbott Way, Stanford, California 94305, USA. * * I ask and expect, but do not require, that all derivative works contain an * attribution similar to: * Parts developed by Adam Langley <agl@imperialviolet.org> * * You may wish to replace the word "Parts" with something else depending on * the amount of original code. * * (Derivative works does not include programs which link against, run or include * the source verbatim in their source distributions) * * Version: 0.1b */ #include "event2/event-config.h" #include "evconfig-private.h" #include <sys/types.h> #ifndef _FORTIFY_SOURCE #define _FORTIFY_SOURCE 3 #endif #include <string.h> #include <fcntl.h> #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #ifdef EVENT__HAVE_STDINT_H #include <stdint.h> #endif #include <stdlib.h> #include <string.h> #include <errno.h> #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #include <limits.h> #include <sys/stat.h> #include <stdio.h> #include <stdarg.h> #ifdef _WIN32 #include <winsock2.h> #include <ws2tcpip.h> #ifndef _WIN32_IE #define _WIN32_IE 0x400 #endif #include <shlobj.h> #endif #include "event2/dns.h" #include "event2/dns_struct.h" #include "event2/dns_compat.h" #include "event2/util.h" #include "event2/event.h" #include "event2/event_struct.h" #include "event2/thread.h" #include "defer-internal.h" #include "log-internal.h" #include "mm-internal.h" #include "strlcpy-internal.h" #include "ipv6-internal.h" #include "util-internal.h" #include "evthread-internal.h" #ifdef _WIN32 #include <ctype.h> #include <winsock2.h> #include <windows.h> #include <iphlpapi.h> #include <io.h> #else #include <sys/socket.h> #include <netinet/in.h> #include <arpa/inet.h> #endif #ifdef EVENT__HAVE_NETINET_IN6_H #include <netinet/in6.h> #endif #define EVDNS_LOG_DEBUG EVENT_LOG_DEBUG #define EVDNS_LOG_WARN EVENT_LOG_WARN #define EVDNS_LOG_MSG EVENT_LOG_MSG #ifndef HOST_NAME_MAX #define HOST_NAME_MAX 255 #endif #include <stdio.h> #undef MIN #define MIN(a,b) ((a)<(b)?(a):(b)) #define ASSERT_VALID_REQUEST(req) \ EVUTIL_ASSERT((req)->handle && (req)->handle->current_req == (req)) #define u64 ev_uint64_t #define u32 ev_uint32_t #define u16 ev_uint16_t #define u8 ev_uint8_t /* maximum number of addresses from a single packet */ /* that we bother recording */ #define MAX_V4_ADDRS 32 #define MAX_V6_ADDRS 32 #define TYPE_A EVDNS_TYPE_A #define TYPE_CNAME 5 #define TYPE_PTR EVDNS_TYPE_PTR #define TYPE_SOA EVDNS_TYPE_SOA #define TYPE_AAAA EVDNS_TYPE_AAAA #define CLASS_INET EVDNS_CLASS_INET /* Persistent handle. We keep this separate from 'struct request' since we * need some object to last for as long as an evdns_request is outstanding so * that it can be canceled, whereas a search request can lead to multiple * 'struct request' instances being created over its lifetime. */ struct evdns_request { struct request *current_req; struct evdns_base *base; int pending_cb; /* Waiting for its callback to be invoked; not * owned by event base any more. */ /* elements used by the searching code */ int search_index; struct search_state *search_state; char *search_origname; /* needs to be free()ed */ int search_flags; }; struct request { u8 *request; /* the dns packet data */ u8 request_type; /* TYPE_PTR or TYPE_A or TYPE_AAAA */ unsigned int request_len; int reissue_count; int tx_count; /* the number of times that this packet has been sent */ void *user_pointer; /* the pointer given to us for this request */ evdns_callback_type user_callback; struct nameserver *ns; /* the server which we last sent it */ /* these objects are kept in a circular list */ /* XXX We could turn this into a CIRCLEQ. */ struct request *next, *prev; struct event timeout_event; u16 trans_id; /* the transaction id */ unsigned request_appended :1; /* true if the request pointer is data which follows this struct */ unsigned transmit_me :1; /* needs to be transmitted */ /* XXXX This is a horrible hack. */ char **put_cname_in_ptr; /* store the cname here if we get one. */ struct evdns_base *base; struct evdns_request *handle; }; struct reply { unsigned int type; unsigned int have_answer : 1; union { struct { u32 addrcount; u32 addresses[MAX_V4_ADDRS]; } a; struct { u32 addrcount; struct in6_addr addresses[MAX_V6_ADDRS]; } aaaa; struct { char name[HOST_NAME_MAX]; } ptr; } data; }; struct nameserver { evutil_socket_t socket; /* a connected UDP socket */ struct sockaddr_storage address; ev_socklen_t addrlen; int failed_times; /* number of times which we have given this server a chance */ int timedout; /* number of times in a row a request has timed out */ struct event event; /* these objects are kept in a circular list */ struct nameserver *next, *prev; struct event timeout_event; /* used to keep the timeout for */ /* when we next probe this server. */ /* Valid if state == 0 */ /* Outstanding probe request for this nameserver, if any */ struct evdns_request *probe_request; char state; /* zero if we think that this server is down */ char choked; /* true if we have an EAGAIN from this server's socket */ char write_waiting; /* true if we are waiting for EV_WRITE events */ struct evdns_base *base; /* Number of currently inflight requests: used * to track when we should add/del the event. */ int requests_inflight; }; /* Represents a local port where we're listening for DNS requests. Right now, */ /* only UDP is supported. */ struct evdns_server_port { evutil_socket_t socket; /* socket we use to read queries and write replies. */ int refcnt; /* reference count. */ char choked; /* Are we currently blocked from writing? */ char closing; /* Are we trying to close this port, pending writes? */ evdns_request_callback_fn_type user_callback; /* Fn to handle requests */ void *user_data; /* Opaque pointer passed to user_callback */ struct event event; /* Read/write event */ /* circular list of replies that we want to write. */ struct server_request *pending_replies; struct event_base *event_base; #ifndef EVENT__DISABLE_THREAD_SUPPORT void *lock; #endif }; /* Represents part of a reply being built. (That is, a single RR.) */ struct server_reply_item { struct server_reply_item *next; /* next item in sequence. */ char *name; /* name part of the RR */ u16 type; /* The RR type */ u16 class; /* The RR class (usually CLASS_INET) */ u32 ttl; /* The RR TTL */ char is_name; /* True iff data is a label */ u16 datalen; /* Length of data; -1 if data is a label */ void *data; /* The contents of the RR */ }; /* Represents a request that we've received as a DNS server, and holds */ /* the components of the reply as we're constructing it. */ struct server_request { /* Pointers to the next and previous entries on the list of replies */ /* that we're waiting to write. Only set if we have tried to respond */ /* and gotten EAGAIN. */ struct server_request *next_pending; struct server_request *prev_pending; u16 trans_id; /* Transaction id. */ struct evdns_server_port *port; /* Which port received this request on? */ struct sockaddr_storage addr; /* Where to send the response */ ev_socklen_t addrlen; /* length of addr */ int n_answer; /* how many answer RRs have been set? */ int n_authority; /* how many authority RRs have been set? */ int n_additional; /* how many additional RRs have been set? */ struct server_reply_item *answer; /* linked list of answer RRs */ struct server_reply_item *authority; /* linked list of authority RRs */ struct server_reply_item *additional; /* linked list of additional RRs */ /* Constructed response. Only set once we're ready to send a reply. */ /* Once this is set, the RR fields are cleared, and no more should be set. */ char *response; size_t response_len; /* Caller-visible fields: flags, questions. */ struct evdns_server_request base; }; struct evdns_base { /* An array of n_req_heads circular lists for inflight requests. * Each inflight request req is in req_heads[req->trans_id % n_req_heads]. */ struct request **req_heads; /* A circular list of requests that we're waiting to send, but haven't * sent yet because there are too many requests inflight */ struct request *req_waiting_head; /* A circular list of nameservers. */ struct nameserver *server_head; int n_req_heads; struct event_base *event_base; /* The number of good nameservers that we have */ int global_good_nameservers; /* inflight requests are contained in the req_head list */ /* and are actually going out across the network */ int global_requests_inflight; /* requests which aren't inflight are in the waiting list */ /* and are counted here */ int global_requests_waiting; int global_max_requests_inflight; struct timeval global_timeout; /* 5 seconds by default */ int global_max_reissues; /* a reissue occurs when we get some errors from the server */ int global_max_retransmits; /* number of times we'll retransmit a request which timed out */ /* number of timeouts in a row before we consider this server to be down */ int global_max_nameserver_timeout; /* true iff we will use the 0x20 hack to prevent poisoning attacks. */ int global_randomize_case; /* The first time that a nameserver fails, how long do we wait before * probing to see if it has returned? */ struct timeval global_nameserver_probe_initial_timeout; /** Port to bind to for outgoing DNS packets. */ struct sockaddr_storage global_outgoing_address; /** ev_socklen_t for global_outgoing_address. 0 if it isn't set. */ ev_socklen_t global_outgoing_addrlen; struct timeval global_getaddrinfo_allow_skew; int getaddrinfo_ipv4_timeouts; int getaddrinfo_ipv6_timeouts; int getaddrinfo_ipv4_answered; int getaddrinfo_ipv6_answered; struct search_state *global_search_state; TAILQ_HEAD(hosts_list, hosts_entry) hostsdb; #ifndef EVENT__DISABLE_THREAD_SUPPORT void *lock; #endif int disable_when_inactive; }; struct hosts_entry { TAILQ_ENTRY(hosts_entry) next; union { struct sockaddr sa; struct sockaddr_in sin; struct sockaddr_in6 sin6; } addr; int addrlen; char hostname[1]; }; static struct evdns_base *current_base = NULL; struct evdns_base * evdns_get_global_base(void) { return current_base; } /* Given a pointer to an evdns_server_request, get the corresponding */ /* server_request. */ #define TO_SERVER_REQUEST(base_ptr) \ ((struct server_request*) \ (((char*)(base_ptr) - evutil_offsetof(struct server_request, base)))) #define REQ_HEAD(base, id) ((base)->req_heads[id % (base)->n_req_heads]) static struct nameserver *nameserver_pick(struct evdns_base *base); static void evdns_request_insert(struct request *req, struct request **head); static void evdns_request_remove(struct request *req, struct request **head); static void nameserver_ready_callback(evutil_socket_t fd, short events, void *arg); static int evdns_transmit(struct evdns_base *base); static int evdns_request_transmit(struct request *req); static void nameserver_send_probe(struct nameserver *const ns); static void search_request_finished(struct evdns_request *const); static int search_try_next(struct evdns_request *const req); static struct request *search_request_new(struct evdns_base *base, struct evdns_request *handle, int type, const char *const name, int flags, evdns_callback_type user_callback, void *user_arg); static void evdns_requests_pump_waiting_queue(struct evdns_base *base); static u16 transaction_id_pick(struct evdns_base *base); static struct request *request_new(struct evdns_base *base, struct evdns_request *handle, int type, const char *name, int flags, evdns_callback_type callback, void *ptr); static void request_submit(struct request *const req); static int server_request_free(struct server_request *req); static void server_request_free_answers(struct server_request *req); static void server_port_free(struct evdns_server_port *port); static void server_port_ready_callback(evutil_socket_t fd, short events, void *arg); static int evdns_base_resolv_conf_parse_impl(struct evdns_base *base, int flags, const char *const filename); static int evdns_base_set_option_impl(struct evdns_base *base, const char *option, const char *val, int flags); static void evdns_base_free_and_unlock(struct evdns_base *base, int fail_requests); static void evdns_request_timeout_callback(evutil_socket_t fd, short events, void *arg); static int strtoint(const char *const str); #ifdef EVENT__DISABLE_THREAD_SUPPORT #define EVDNS_LOCK(base) EVUTIL_NIL_STMT_ #define EVDNS_UNLOCK(base) EVUTIL_NIL_STMT_ #define ASSERT_LOCKED(base) EVUTIL_NIL_STMT_ #else #define EVDNS_LOCK(base) \ EVLOCK_LOCK((base)->lock, 0) #define EVDNS_UNLOCK(base) \ EVLOCK_UNLOCK((base)->lock, 0) #define ASSERT_LOCKED(base) \ EVLOCK_ASSERT_LOCKED((base)->lock) #endif static evdns_debug_log_fn_type evdns_log_fn = NULL; void evdns_set_log_fn(evdns_debug_log_fn_type fn) { evdns_log_fn = fn; } #ifdef __GNUC__ #define EVDNS_LOG_CHECK __attribute__ ((format(printf, 2, 3))) #else #define EVDNS_LOG_CHECK #endif static void evdns_log_(int severity, const char *fmt, ...) EVDNS_LOG_CHECK; static void evdns_log_(int severity, const char *fmt, ...) { va_list args; va_start(args,fmt); if (evdns_log_fn) { char buf[512]; int is_warn = (severity == EVDNS_LOG_WARN); evutil_vsnprintf(buf, sizeof(buf), fmt, args); evdns_log_fn(is_warn, buf); } else { event_logv_(severity, NULL, fmt, args); } va_end(args); } #define log evdns_log_ /* This walks the list of inflight requests to find the */ /* one with a matching transaction id. Returns NULL on */ /* failure */ static struct request * request_find_from_trans_id(struct evdns_base *base, u16 trans_id) { struct request *req = REQ_HEAD(base, trans_id); struct request *const started_at = req; ASSERT_LOCKED(base); if (req) { do { if (req->trans_id == trans_id) return req; req = req->next; } while (req != started_at); } return NULL; } /* a libevent callback function which is called when a nameserver */ /* has gone down and we want to test if it has came back to life yet */ static void nameserver_prod_callback(evutil_socket_t fd, short events, void *arg) { struct nameserver *const ns = (struct nameserver *) arg; (void)fd; (void)events; EVDNS_LOCK(ns->base); nameserver_send_probe(ns); EVDNS_UNLOCK(ns->base); } /* a libevent callback which is called when a nameserver probe (to see if */ /* it has come back to life) times out. We increment the count of failed_times */ /* and wait longer to send the next probe packet. */ static void nameserver_probe_failed(struct nameserver *const ns) { struct timeval timeout; int i; ASSERT_LOCKED(ns->base); (void) evtimer_del(&ns->timeout_event); if (ns->state == 1) { /* This can happen if the nameserver acts in a way which makes us mark */ /* it as bad and then starts sending good replies. */ return; } #define MAX_PROBE_TIMEOUT 3600 #define TIMEOUT_BACKOFF_FACTOR 3 memcpy(&timeout, &ns->base->global_nameserver_probe_initial_timeout, sizeof(struct timeval)); for (i=ns->failed_times; i > 0 && timeout.tv_sec < MAX_PROBE_TIMEOUT; --i) { timeout.tv_sec *= TIMEOUT_BACKOFF_FACTOR; timeout.tv_usec *= TIMEOUT_BACKOFF_FACTOR; if (timeout.tv_usec > 1000000) { timeout.tv_sec += timeout.tv_usec / 1000000; timeout.tv_usec %= 1000000; } } if (timeout.tv_sec > MAX_PROBE_TIMEOUT) { timeout.tv_sec = MAX_PROBE_TIMEOUT; timeout.tv_usec = 0; } ns->failed_times++; if (evtimer_add(&ns->timeout_event, &timeout) < 0) { char addrbuf[128]; log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", evutil_format_sockaddr_port_( (struct sockaddr *)&ns->address, addrbuf, sizeof(addrbuf))); } } static void request_swap_ns(struct request *req, struct nameserver *ns) { if (ns && req->ns != ns) { EVUTIL_ASSERT(req->ns->requests_inflight > 0); req->ns->requests_inflight--; ns->requests_inflight++; req->ns = ns; } } /* called when a nameserver has been deemed to have failed. For example, too */ /* many packets have timed out etc */ static void nameserver_failed(struct nameserver *const ns, const char *msg) { struct request *req, *started_at; struct evdns_base *base = ns->base; int i; char addrbuf[128]; ASSERT_LOCKED(base); /* if this nameserver has already been marked as failed */ /* then don't do anything */ if (!ns->state) return; log(EVDNS_LOG_MSG, "Nameserver %s has failed: %s", evutil_format_sockaddr_port_( (struct sockaddr *)&ns->address, addrbuf, sizeof(addrbuf)), msg); base->global_good_nameservers--; EVUTIL_ASSERT(base->global_good_nameservers >= 0); if (base->global_good_nameservers == 0) { log(EVDNS_LOG_MSG, "All nameservers have failed"); } ns->state = 0; ns->failed_times = 1; if (evtimer_add(&ns->timeout_event, &base->global_nameserver_probe_initial_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", evutil_format_sockaddr_port_( (struct sockaddr *)&ns->address, addrbuf, sizeof(addrbuf))); /* ???? Do more? */ } /* walk the list of inflight requests to see if any can be reassigned to */ /* a different server. Requests in the waiting queue don't have a */ /* nameserver assigned yet */ /* if we don't have *any* good nameservers then there's no point */ /* trying to reassign requests to one */ if (!base->global_good_nameservers) return; for (i = 0; i < base->n_req_heads; ++i) { req = started_at = base->req_heads[i]; if (req) { do { if (req->tx_count == 0 && req->ns == ns) { /* still waiting to go out, can be moved */ /* to another server */ request_swap_ns(req, nameserver_pick(base)); } req = req->next; } while (req != started_at); } } } static void nameserver_up(struct nameserver *const ns) { char addrbuf[128]; ASSERT_LOCKED(ns->base); if (ns->state) return; log(EVDNS_LOG_MSG, "Nameserver %s is back up", evutil_format_sockaddr_port_( (struct sockaddr *)&ns->address, addrbuf, sizeof(addrbuf))); evtimer_del(&ns->timeout_event); if (ns->probe_request) { evdns_cancel_request(ns->base, ns->probe_request); ns->probe_request = NULL; } ns->state = 1; ns->failed_times = 0; ns->timedout = 0; ns->base->global_good_nameservers++; } static void request_trans_id_set(struct request *const req, const u16 trans_id) { req->trans_id = trans_id; *((u16 *) req->request) = htons(trans_id); } /* Called to remove a request from a list and dealloc it. */ /* head is a pointer to the head of the list it should be */ /* removed from or NULL if the request isn't in a list. */ /* when free_handle is one, free the handle as well. */ static void request_finished(struct request *const req, struct request **head, int free_handle) { struct evdns_base *base = req->base; int was_inflight = (head != &base->req_waiting_head); EVDNS_LOCK(base); ASSERT_VALID_REQUEST(req); if (head) evdns_request_remove(req, head); log(EVDNS_LOG_DEBUG, "Removing timeout for request %p", req); if (was_inflight) { evtimer_del(&req->timeout_event); base->global_requests_inflight--; req->ns->requests_inflight--; } else { base->global_requests_waiting--; } /* it was initialized during request_new / evtimer_assign */ event_debug_unassign(&req->timeout_event); if (req->ns && req->ns->requests_inflight == 0 && req->base->disable_when_inactive) { event_del(&req->ns->event); evtimer_del(&req->ns->timeout_event); } if (!req->request_appended) { /* need to free the request data on it's own */ mm_free(req->request); } else { /* the request data is appended onto the header */ /* so everything gets free()ed when we: */ } if (req->handle) { EVUTIL_ASSERT(req->handle->current_req == req); if (free_handle) { search_request_finished(req->handle); req->handle->current_req = NULL; if (! req->handle->pending_cb) { /* If we're planning to run the callback, * don't free the handle until later. */ mm_free(req->handle); } req->handle = NULL; /* If we have a bug, let's crash * early */ } else { req->handle->current_req = NULL; } } mm_free(req); evdns_requests_pump_waiting_queue(base); EVDNS_UNLOCK(base); } /* This is called when a server returns a funny error code. */ /* We try the request again with another server. */ /* */ /* return: */ /* 0 ok */ /* 1 failed/reissue is pointless */ static int request_reissue(struct request *req) { const struct nameserver *const last_ns = req->ns; ASSERT_LOCKED(req->base); ASSERT_VALID_REQUEST(req); /* the last nameserver should have been marked as failing */ /* by the caller of this function, therefore pick will try */ /* not to return it */ request_swap_ns(req, nameserver_pick(req->base)); if (req->ns == last_ns) { /* ... but pick did return it */ /* not a lot of point in trying again with the */ /* same server */ return 1; } req->reissue_count++; req->tx_count = 0; req->transmit_me = 1; return 0; } /* this function looks for space on the inflight queue and promotes */ /* requests from the waiting queue if it can. */ /* */ /* TODO: */ /* add return code, see at nameserver_pick() and other functions. */ static void evdns_requests_pump_waiting_queue(struct evdns_base *base) { ASSERT_LOCKED(base); while (base->global_requests_inflight < base->global_max_requests_inflight && base->global_requests_waiting) { struct request *req; EVUTIL_ASSERT(base->req_waiting_head); req = base->req_waiting_head; req->ns = nameserver_pick(base); if (!req->ns) return; /* move a request from the waiting queue to the inflight queue */ req->ns->requests_inflight++; evdns_request_remove(req, &base->req_waiting_head); base->global_requests_waiting--; base->global_requests_inflight++; request_trans_id_set(req, transaction_id_pick(base)); evdns_request_insert(req, &REQ_HEAD(base, req->trans_id)); evdns_request_transmit(req); evdns_transmit(base); } } /* TODO(nickm) document */ struct deferred_reply_callback { struct event_callback deferred; struct evdns_request *handle; u8 request_type; u8 have_reply; u32 ttl; u32 err; evdns_callback_type user_callback; struct reply reply; }; static void reply_run_callback(struct event_callback *d, void *user_pointer) { struct deferred_reply_callback *cb = EVUTIL_UPCAST(d, struct deferred_reply_callback, deferred); switch (cb->request_type) { case TYPE_A: if (cb->have_reply) cb->user_callback(DNS_ERR_NONE, DNS_IPv4_A, cb->reply.data.a.addrcount, cb->ttl, cb->reply.data.a.addresses, user_pointer); else cb->user_callback(cb->err, 0, 0, cb->ttl, NULL, user_pointer); break; case TYPE_PTR: if (cb->have_reply) { char *name = cb->reply.data.ptr.name; cb->user_callback(DNS_ERR_NONE, DNS_PTR, 1, cb->ttl, &name, user_pointer); } else { cb->user_callback(cb->err, 0, 0, cb->ttl, NULL, user_pointer); } break; case TYPE_AAAA: if (cb->have_reply) cb->user_callback(DNS_ERR_NONE, DNS_IPv6_AAAA, cb->reply.data.aaaa.addrcount, cb->ttl, cb->reply.data.aaaa.addresses, user_pointer); else cb->user_callback(cb->err, 0, 0, cb->ttl, NULL, user_pointer); break; default: EVUTIL_ASSERT(0); } if (cb->handle && cb->handle->pending_cb) { mm_free(cb->handle); } mm_free(cb); } static void reply_schedule_callback(struct request *const req, u32 ttl, u32 err, struct reply *reply) { struct deferred_reply_callback *d = mm_calloc(1, sizeof(*d)); if (!d) { event_warn("%s: Couldn't allocate space for deferred callback.", __func__); return; } ASSERT_LOCKED(req->base); d->request_type = req->request_type; d->user_callback = req->user_callback; d->ttl = ttl; d->err = err; if (reply) { d->have_reply = 1; memcpy(&d->reply, reply, sizeof(struct reply)); } if (req->handle) { req->handle->pending_cb = 1; d->handle = req->handle; } event_deferred_cb_init_( &d->deferred, event_get_priority(&req->timeout_event), reply_run_callback, req->user_pointer); event_deferred_cb_schedule_( req->base->event_base, &d->deferred); } /* this processes a parsed reply packet */ static void reply_handle(struct request *const req, u16 flags, u32 ttl, struct reply *reply) { int error; char addrbuf[128]; static const int error_codes[] = { DNS_ERR_FORMAT, DNS_ERR_SERVERFAILED, DNS_ERR_NOTEXIST, DNS_ERR_NOTIMPL, DNS_ERR_REFUSED }; ASSERT_LOCKED(req->base); ASSERT_VALID_REQUEST(req); if (flags & 0x020f || !reply || !reply->have_answer) { /* there was an error */ if (flags & 0x0200) { error = DNS_ERR_TRUNCATED; } else if (flags & 0x000f) { u16 error_code = (flags & 0x000f) - 1; if (error_code > 4) { error = DNS_ERR_UNKNOWN; } else { error = error_codes[error_code]; } } else if (reply && !reply->have_answer) { error = DNS_ERR_NODATA; } else { error = DNS_ERR_UNKNOWN; } switch (error) { case DNS_ERR_NOTIMPL: case DNS_ERR_REFUSED: /* we regard these errors as marking a bad nameserver */ if (req->reissue_count < req->base->global_max_reissues) { char msg[64]; evutil_snprintf(msg, sizeof(msg), "Bad response %d (%s)", error, evdns_err_to_string(error)); nameserver_failed(req->ns, msg); if (!request_reissue(req)) return; } break; case DNS_ERR_SERVERFAILED: /* rcode 2 (servfailed) sometimes means "we * are broken" and sometimes (with some binds) * means "that request was very confusing." * Treat this as a timeout, not a failure. */ log(EVDNS_LOG_DEBUG, "Got a SERVERFAILED from nameserver" "at %s; will allow the request to time out.", evutil_format_sockaddr_port_( (struct sockaddr *)&req->ns->address, addrbuf, sizeof(addrbuf))); /* Call the timeout function */ evdns_request_timeout_callback(0, 0, req); return; default: /* we got a good reply from the nameserver: it is up. */ if (req->handle == req->ns->probe_request) { /* Avoid double-free */ req->ns->probe_request = NULL; } nameserver_up(req->ns); } if (req->handle->search_state && req->request_type != TYPE_PTR) { /* if we have a list of domains to search in, * try the next one */ if (!search_try_next(req->handle)) { /* a new request was issued so this * request is finished and */ /* the user callback will be made when * that request (or a */ /* child of it) finishes. */ return; } } /* all else failed. Pass the failure up */ reply_schedule_callback(req, ttl, error, NULL); request_finished(req, &REQ_HEAD(req->base, req->trans_id), 1); } else { /* all ok, tell the user */ reply_schedule_callback(req, ttl, 0, reply); if (req->handle == req->ns->probe_request) req->ns->probe_request = NULL; /* Avoid double-free */ nameserver_up(req->ns); request_finished(req, &REQ_HEAD(req->base, req->trans_id), 1); } } static int name_parse(u8 *packet, int length, int *idx, char *name_out, int name_out_len) { int name_end = -1; int j = *idx; int ptr_count = 0; #define GET32(x) do { if (j + 4 > length) goto err; memcpy(&t32_, packet + j, 4); j += 4; x = ntohl(t32_); } while (0) #define GET16(x) do { if (j + 2 > length) goto err; memcpy(&t_, packet + j, 2); j += 2; x = ntohs(t_); } while (0) #define GET8(x) do { if (j >= length) goto err; x = packet[j++]; } while (0) char *cp = name_out; const char *const end = name_out + name_out_len; /* Normally, names are a series of length prefixed strings terminated */ /* with a length of 0 (the lengths are u8's < 63). */ /* However, the length can start with a pair of 1 bits and that */ /* means that the next 14 bits are a pointer within the current */ /* packet. */ for (;;) { u8 label_len; GET8(label_len); if (!label_len) break; if (label_len & 0xc0) { u8 ptr_low; GET8(ptr_low); if (name_end < 0) name_end = j; j = (((int)label_len & 0x3f) << 8) + ptr_low; /* Make sure that the target offset is in-bounds. */ if (j < 0 || j >= length) return -1; /* If we've jumped more times than there are characters in the * message, we must have a loop. */ if (++ptr_count > length) return -1; continue; } if (label_len > 63) return -1; if (cp != name_out) { if (cp + 1 >= end) return -1; *cp++ = '.'; } if (cp + label_len >= end) return -1; if (j + label_len > length) return -1; memcpy(cp, packet + j, label_len); cp += label_len; j += label_len; } if (cp >= end) return -1; *cp = '\0'; if (name_end < 0) *idx = j; else *idx = name_end; return 0; err: return -1; } /* parses a raw request from a nameserver */ static int reply_parse(struct evdns_base *base, u8 *packet, int length) { int j = 0, k = 0; /* index into packet */ u16 t_; /* used by the macros */ u32 t32_; /* used by the macros */ char tmp_name[256], cmp_name[256]; /* used by the macros */ int name_matches = 0; u16 trans_id, questions, answers, authority, additional, datalength; u16 flags = 0; u32 ttl, ttl_r = 0xffffffff; struct reply reply; struct request *req = NULL; unsigned int i; ASSERT_LOCKED(base); GET16(trans_id); GET16(flags); GET16(questions); GET16(answers); GET16(authority); GET16(additional); (void) authority; /* suppress "unused variable" warnings. */ (void) additional; /* suppress "unused variable" warnings. */ req = request_find_from_trans_id(base, trans_id); if (!req) return -1; EVUTIL_ASSERT(req->base == base); memset(&reply, 0, sizeof(reply)); /* If it's not an answer, it doesn't correspond to any request. */ if (!(flags & 0x8000)) return -1; /* must be an answer */ if ((flags & 0x020f) && (flags & 0x020f) != DNS_ERR_NOTEXIST) { /* there was an error and it's not NXDOMAIN */ goto err; } /* if (!answers) return; */ /* must have an answer of some form */ /* This macro skips a name in the DNS reply. */ #define SKIP_NAME \ do { tmp_name[0] = '\0'; \ if (name_parse(packet, length, &j, tmp_name, \ sizeof(tmp_name))<0) \ goto err; \ } while (0) reply.type = req->request_type; /* skip over each question in the reply */ for (i = 0; i < questions; ++i) { /* the question looks like * <label:name><u16:type><u16:class> */ tmp_name[0] = '\0'; cmp_name[0] = '\0'; k = j; if (name_parse(packet, length, &j, tmp_name, sizeof(tmp_name)) < 0) goto err; if (name_parse(req->request, req->request_len, &k, cmp_name, sizeof(cmp_name))<0) goto err; if (!base->global_randomize_case) { if (strcmp(tmp_name, cmp_name) == 0) name_matches = 1; } else { if (evutil_ascii_strcasecmp(tmp_name, cmp_name) == 0) name_matches = 1; } j += 4; if (j > length) goto err; } if (!name_matches) goto err; /* now we have the answer section which looks like * <label:name><u16:type><u16:class><u32:ttl><u16:len><data...> */ for (i = 0; i < answers; ++i) { u16 type, class; SKIP_NAME; GET16(type); GET16(class); GET32(ttl); GET16(datalength); if (type == TYPE_A && class == CLASS_INET) { int addrcount, addrtocopy; if (req->request_type != TYPE_A) { j += datalength; continue; } if ((datalength & 3) != 0) /* not an even number of As. */ goto err; addrcount = datalength >> 2; addrtocopy = MIN(MAX_V4_ADDRS - reply.data.a.addrcount, (unsigned)addrcount); ttl_r = MIN(ttl_r, ttl); /* we only bother with the first four addresses. */ if (j + 4*addrtocopy > length) goto err; memcpy(&reply.data.a.addresses[reply.data.a.addrcount], packet + j, 4*addrtocopy); j += 4*addrtocopy; reply.data.a.addrcount += addrtocopy; reply.have_answer = 1; if (reply.data.a.addrcount == MAX_V4_ADDRS) break; } else if (type == TYPE_PTR && class == CLASS_INET) { if (req->request_type != TYPE_PTR) { j += datalength; continue; } if (name_parse(packet, length, &j, reply.data.ptr.name, sizeof(reply.data.ptr.name))<0) goto err; ttl_r = MIN(ttl_r, ttl); reply.have_answer = 1; break; } else if (type == TYPE_CNAME) { char cname[HOST_NAME_MAX]; if (!req->put_cname_in_ptr || *req->put_cname_in_ptr) { j += datalength; continue; } if (name_parse(packet, length, &j, cname, sizeof(cname))<0) goto err; *req->put_cname_in_ptr = mm_strdup(cname); } else if (type == TYPE_AAAA && class == CLASS_INET) { int addrcount, addrtocopy; if (req->request_type != TYPE_AAAA) { j += datalength; continue; } if ((datalength & 15) != 0) /* not an even number of AAAAs. */ goto err; addrcount = datalength >> 4; /* each address is 16 bytes long */ addrtocopy = MIN(MAX_V6_ADDRS - reply.data.aaaa.addrcount, (unsigned)addrcount); ttl_r = MIN(ttl_r, ttl); /* we only bother with the first four addresses. */ if (j + 16*addrtocopy > length) goto err; memcpy(&reply.data.aaaa.addresses[reply.data.aaaa.addrcount], packet + j, 16*addrtocopy); reply.data.aaaa.addrcount += addrtocopy; j += 16*addrtocopy; reply.have_answer = 1; if (reply.data.aaaa.addrcount == MAX_V6_ADDRS) break; } else { /* skip over any other type of resource */ j += datalength; } } if (!reply.have_answer) { for (i = 0; i < authority; ++i) { u16 type, class; SKIP_NAME; GET16(type); GET16(class); GET32(ttl); GET16(datalength); if (type == TYPE_SOA && class == CLASS_INET) { u32 serial, refresh, retry, expire, minimum; SKIP_NAME; SKIP_NAME; GET32(serial); GET32(refresh); GET32(retry); GET32(expire); GET32(minimum); (void)expire; (void)retry; (void)refresh; (void)serial; ttl_r = MIN(ttl_r, ttl); ttl_r = MIN(ttl_r, minimum); } else { /* skip over any other type of resource */ j += datalength; } } } if (ttl_r == 0xffffffff) ttl_r = 0; reply_handle(req, flags, ttl_r, &reply); return 0; err: if (req) reply_handle(req, flags, 0, NULL); return -1; } /* Parse a raw request (packet,length) sent to a nameserver port (port) from */ /* a DNS client (addr,addrlen), and if it's well-formed, call the corresponding */ /* callback. */ static int request_parse(u8 *packet, int length, struct evdns_server_port *port, struct sockaddr *addr, ev_socklen_t addrlen) { int j = 0; /* index into packet */ u16 t_; /* used by the macros */ char tmp_name[256]; /* used by the macros */ int i; u16 trans_id, flags, questions, answers, authority, additional; struct server_request *server_req = NULL; ASSERT_LOCKED(port); /* Get the header fields */ GET16(trans_id); GET16(flags); GET16(questions); GET16(answers); GET16(authority); GET16(additional); (void)answers; (void)additional; (void)authority; if (flags & 0x8000) return -1; /* Must not be an answer. */ flags &= 0x0110; /* Only RD and CD get preserved. */ server_req = mm_malloc(sizeof(struct server_request)); if (server_req == NULL) return -1; memset(server_req, 0, sizeof(struct server_request)); server_req->trans_id = trans_id; memcpy(&server_req->addr, addr, addrlen); server_req->addrlen = addrlen; server_req->base.flags = flags; server_req->base.nquestions = 0; server_req->base.questions = mm_calloc(sizeof(struct evdns_server_question *), questions); if (server_req->base.questions == NULL) goto err; for (i = 0; i < questions; ++i) { u16 type, class; struct evdns_server_question *q; int namelen; if (name_parse(packet, length, &j, tmp_name, sizeof(tmp_name))<0) goto err; GET16(type); GET16(class); namelen = (int)strlen(tmp_name); q = mm_malloc(sizeof(struct evdns_server_question) + namelen); if (!q) goto err; q->type = type; q->dns_question_class = class; memcpy(q->name, tmp_name, namelen+1); server_req->base.questions[server_req->base.nquestions++] = q; } /* Ignore answers, authority, and additional. */ server_req->port = port; port->refcnt++; /* Only standard queries are supported. */ if (flags & 0x7800) { evdns_server_request_respond(&(server_req->base), DNS_ERR_NOTIMPL); return -1; } port->user_callback(&(server_req->base), port->user_data); return 0; err: if (server_req) { if (server_req->base.questions) { for (i = 0; i < server_req->base.nquestions; ++i) mm_free(server_req->base.questions[i]); mm_free(server_req->base.questions); } mm_free(server_req); } return -1; #undef SKIP_NAME #undef GET32 #undef GET16 #undef GET8 } void evdns_set_transaction_id_fn(ev_uint16_t (*fn)(void)) { } void evdns_set_random_bytes_fn(void (*fn)(char *, size_t)) { } /* Try to choose a strong transaction id which isn't already in flight */ static u16 transaction_id_pick(struct evdns_base *base) { ASSERT_LOCKED(base); for (;;) { u16 trans_id; evutil_secure_rng_get_bytes(&trans_id, sizeof(trans_id)); if (trans_id == 0xffff) continue; /* now check to see if that id is already inflight */ if (request_find_from_trans_id(base, trans_id) == NULL) return trans_id; } } /* choose a namesever to use. This function will try to ignore */ /* nameservers which we think are down and load balance across the rest */ /* by updating the server_head global each time. */ static struct nameserver * nameserver_pick(struct evdns_base *base) { struct nameserver *started_at = base->server_head, *picked; ASSERT_LOCKED(base); if (!base->server_head) return NULL; /* if we don't have any good nameservers then there's no */ /* point in trying to find one. */ if (!base->global_good_nameservers) { base->server_head = base->server_head->next; return base->server_head; } /* remember that nameservers are in a circular list */ for (;;) { if (base->server_head->state) { /* we think this server is currently good */ picked = base->server_head; base->server_head = base->server_head->next; return picked; } base->server_head = base->server_head->next; if (base->server_head == started_at) { /* all the nameservers seem to be down */ /* so we just return this one and hope for the */ /* best */ EVUTIL_ASSERT(base->global_good_nameservers == 0); picked = base->server_head; base->server_head = base->server_head->next; return picked; } } } /* this is called when a namesever socket is ready for reading */ static void nameserver_read(struct nameserver *ns) { struct sockaddr_storage ss; ev_socklen_t addrlen = sizeof(ss); u8 packet[1500]; char addrbuf[128]; ASSERT_LOCKED(ns->base); for (;;) { const int r = recvfrom(ns->socket, (void*)packet, sizeof(packet), 0, (struct sockaddr*)&ss, &addrlen); if (r < 0) { int err = evutil_socket_geterror(ns->socket); if (EVUTIL_ERR_RW_RETRIABLE(err)) return; nameserver_failed(ns, evutil_socket_error_to_string(err)); return; } if (evutil_sockaddr_cmp((struct sockaddr*)&ss, (struct sockaddr*)&ns->address, 0)) { log(EVDNS_LOG_WARN, "Address mismatch on received " "DNS packet. Apparent source was %s", evutil_format_sockaddr_port_( (struct sockaddr *)&ss, addrbuf, sizeof(addrbuf))); return; } ns->timedout = 0; reply_parse(ns->base, packet, r); } } /* Read a packet from a DNS client on a server port s, parse it, and */ /* act accordingly. */ static void server_port_read(struct evdns_server_port *s) { u8 packet[1500]; struct sockaddr_storage addr; ev_socklen_t addrlen; int r; ASSERT_LOCKED(s); for (;;) { addrlen = sizeof(struct sockaddr_storage); r = recvfrom(s->socket, (void*)packet, sizeof(packet), 0, (struct sockaddr*) &addr, &addrlen); if (r < 0) { int err = evutil_socket_geterror(s->socket); if (EVUTIL_ERR_RW_RETRIABLE(err)) return; log(EVDNS_LOG_WARN, "Error %s (%d) while reading request.", evutil_socket_error_to_string(err), err); return; } request_parse(packet, r, s, (struct sockaddr*) &addr, addrlen); } } /* Try to write all pending replies on a given DNS server port. */ static void server_port_flush(struct evdns_server_port *port) { struct server_request *req = port->pending_replies; ASSERT_LOCKED(port); while (req) { int r = sendto(port->socket, req->response, (int)req->response_len, 0, (struct sockaddr*) &req->addr, (ev_socklen_t)req->addrlen); if (r < 0) { int err = evutil_socket_geterror(port->socket); if (EVUTIL_ERR_RW_RETRIABLE(err)) return; log(EVDNS_LOG_WARN, "Error %s (%d) while writing response to port; dropping", evutil_socket_error_to_string(err), err); } if (server_request_free(req)) { /* we released the last reference to req->port. */ return; } else { EVUTIL_ASSERT(req != port->pending_replies); req = port->pending_replies; } } /* We have no more pending requests; stop listening for 'writeable' events. */ (void) event_del(&port->event); event_assign(&port->event, port->event_base, port->socket, EV_READ | EV_PERSIST, server_port_ready_callback, port); if (event_add(&port->event, NULL) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding event for DNS server."); /* ???? Do more? */ } } /* set if we are waiting for the ability to write to this server. */ /* if waiting is true then we ask libevent for EV_WRITE events, otherwise */ /* we stop these events. */ static void nameserver_write_waiting(struct nameserver *ns, char waiting) { ASSERT_LOCKED(ns->base); if (ns->write_waiting == waiting) return; ns->write_waiting = waiting; (void) event_del(&ns->event); event_assign(&ns->event, ns->base->event_base, ns->socket, EV_READ | (waiting ? EV_WRITE : 0) | EV_PERSIST, nameserver_ready_callback, ns); if (event_add(&ns->event, NULL) < 0) { char addrbuf[128]; log(EVDNS_LOG_WARN, "Error from libevent when adding event for %s", evutil_format_sockaddr_port_( (struct sockaddr *)&ns->address, addrbuf, sizeof(addrbuf))); /* ???? Do more? */ } } /* a callback function. Called by libevent when the kernel says that */ /* a nameserver socket is ready for writing or reading */ static void nameserver_ready_callback(evutil_socket_t fd, short events, void *arg) { struct nameserver *ns = (struct nameserver *) arg; (void)fd; EVDNS_LOCK(ns->base); if (events & EV_WRITE) { ns->choked = 0; if (!evdns_transmit(ns->base)) { nameserver_write_waiting(ns, 0); } } if (events & EV_READ) { nameserver_read(ns); } EVDNS_UNLOCK(ns->base); } /* a callback function. Called by libevent when the kernel says that */ /* a server socket is ready for writing or reading. */ static void server_port_ready_callback(evutil_socket_t fd, short events, void *arg) { struct evdns_server_port *port = (struct evdns_server_port *) arg; (void) fd; EVDNS_LOCK(port); if (events & EV_WRITE) { port->choked = 0; server_port_flush(port); } if (events & EV_READ) { server_port_read(port); } EVDNS_UNLOCK(port); } /* This is an inefficient representation; only use it via the dnslabel_table_* * functions, so that is can be safely replaced with something smarter later. */ #define MAX_LABELS 128 /* Structures used to implement name compression */ struct dnslabel_entry { char *v; off_t pos; }; struct dnslabel_table { int n_labels; /* number of current entries */ /* map from name to position in message */ struct dnslabel_entry labels[MAX_LABELS]; }; /* Initialize dnslabel_table. */ static void dnslabel_table_init(struct dnslabel_table *table) { table->n_labels = 0; } /* Free all storage held by table, but not the table itself. */ static void dnslabel_clear(struct dnslabel_table *table) { int i; for (i = 0; i < table->n_labels; ++i) mm_free(table->labels[i].v); table->n_labels = 0; } /* return the position of the label in the current message, or -1 if the label */ /* hasn't been used yet. */ static int dnslabel_table_get_pos(const struct dnslabel_table *table, const char *label) { int i; for (i = 0; i < table->n_labels; ++i) { if (!strcmp(label, table->labels[i].v)) return table->labels[i].pos; } return -1; } /* remember that we've used the label at position pos */ static int dnslabel_table_add(struct dnslabel_table *table, const char *label, off_t pos) { char *v; int p; if (table->n_labels == MAX_LABELS) return (-1); v = mm_strdup(label); if (v == NULL) return (-1); p = table->n_labels++; table->labels[p].v = v; table->labels[p].pos = pos; return (0); } /* Converts a string to a length-prefixed set of DNS labels, starting */ /* at buf[j]. name and buf must not overlap. name_len should be the length */ /* of name. table is optional, and is used for compression. */ /* */ /* Input: abc.def */ /* Output: <3>abc<3>def<0> */ /* */ /* Returns the first index after the encoded name, or negative on error. */ /* -1 label was > 63 bytes */ /* -2 name too long to fit in buffer. */ /* */ static off_t dnsname_to_labels(u8 *const buf, size_t buf_len, off_t j, const char *name, const size_t name_len, struct dnslabel_table *table) { const char *end = name + name_len; int ref = 0; u16 t_; #define APPEND16(x) do { \ if (j + 2 > (off_t)buf_len) \ goto overflow; \ t_ = htons(x); \ memcpy(buf + j, &t_, 2); \ j += 2; \ } while (0) #define APPEND32(x) do { \ if (j + 4 > (off_t)buf_len) \ goto overflow; \ t32_ = htonl(x); \ memcpy(buf + j, &t32_, 4); \ j += 4; \ } while (0) if (name_len > 255) return -2; for (;;) { const char *const start = name; if (table && (ref = dnslabel_table_get_pos(table, name)) >= 0) { APPEND16(ref | 0xc000); return j; } name = strchr(name, '.'); if (!name) { const size_t label_len = end - start; if (label_len > 63) return -1; if ((size_t)(j+label_len+1) > buf_len) return -2; if (table) dnslabel_table_add(table, start, j); buf[j++] = (ev_uint8_t)label_len; memcpy(buf + j, start, label_len); j += (int) label_len; break; } else { /* append length of the label. */ const size_t label_len = name - start; if (label_len > 63) return -1; if ((size_t)(j+label_len+1) > buf_len) return -2; if (table) dnslabel_table_add(table, start, j); buf[j++] = (ev_uint8_t)label_len; memcpy(buf + j, start, label_len); j += (int) label_len; /* hop over the '.' */ name++; } } /* the labels must be terminated by a 0. */ /* It's possible that the name ended in a . */ /* in which case the zero is already there */ if (!j || buf[j-1]) buf[j++] = 0; return j; overflow: return (-2); } /* Finds the length of a dns request for a DNS name of the given */ /* length. The actual request may be smaller than the value returned */ /* here */ static size_t evdns_request_len(const size_t name_len) { return 96 + /* length of the DNS standard header */ name_len + 2 + 4; /* space for the resource type */ } /* build a dns request packet into buf. buf should be at least as long */ /* as evdns_request_len told you it should be. */ /* */ /* Returns the amount of space used. Negative on error. */ static int evdns_request_data_build(const char *const name, const size_t name_len, const u16 trans_id, const u16 type, const u16 class, u8 *const buf, size_t buf_len) { off_t j = 0; /* current offset into buf */ u16 t_; /* used by the macros */ APPEND16(trans_id); APPEND16(0x0100); /* standard query, recusion needed */ APPEND16(1); /* one question */ APPEND16(0); /* no answers */ APPEND16(0); /* no authority */ APPEND16(0); /* no additional */ j = dnsname_to_labels(buf, buf_len, j, name, name_len, NULL); if (j < 0) { return (int)j; } APPEND16(type); APPEND16(class); return (int)j; overflow: return (-1); } /* exported function */ struct evdns_server_port * evdns_add_server_port_with_base(struct event_base *base, evutil_socket_t socket, int flags, evdns_request_callback_fn_type cb, void *user_data) { struct evdns_server_port *port; if (flags) return NULL; /* flags not yet implemented */ if (!(port = mm_malloc(sizeof(struct evdns_server_port)))) return NULL; memset(port, 0, sizeof(struct evdns_server_port)); port->socket = socket; port->refcnt = 1; port->choked = 0; port->closing = 0; port->user_callback = cb; port->user_data = user_data; port->pending_replies = NULL; port->event_base = base; event_assign(&port->event, port->event_base, port->socket, EV_READ | EV_PERSIST, server_port_ready_callback, port); if (event_add(&port->event, NULL) < 0) { mm_free(port); return NULL; } EVTHREAD_ALLOC_LOCK(port->lock, EVTHREAD_LOCKTYPE_RECURSIVE); return port; } struct evdns_server_port * evdns_add_server_port(evutil_socket_t socket, int flags, evdns_request_callback_fn_type cb, void *user_data) { return evdns_add_server_port_with_base(NULL, socket, flags, cb, user_data); } /* exported function */ void evdns_close_server_port(struct evdns_server_port *port) { EVDNS_LOCK(port); if (--port->refcnt == 0) { EVDNS_UNLOCK(port); server_port_free(port); } else { port->closing = 1; } } /* exported function */ int evdns_server_request_add_reply(struct evdns_server_request *req_, int section, const char *name, int type, int class, int ttl, int datalen, int is_name, const char *data) { struct server_request *req = TO_SERVER_REQUEST(req_); struct server_reply_item **itemp, *item; int *countp; int result = -1; EVDNS_LOCK(req->port); if (req->response) /* have we already answered? */ goto done; switch (section) { case EVDNS_ANSWER_SECTION: itemp = &req->answer; countp = &req->n_answer; break; case EVDNS_AUTHORITY_SECTION: itemp = &req->authority; countp = &req->n_authority; break; case EVDNS_ADDITIONAL_SECTION: itemp = &req->additional; countp = &req->n_additional; break; default: goto done; } while (*itemp) { itemp = &((*itemp)->next); } item = mm_malloc(sizeof(struct server_reply_item)); if (!item) goto done; item->next = NULL; if (!(item->name = mm_strdup(name))) { mm_free(item); goto done; } item->type = type; item->dns_question_class = class; item->ttl = ttl; item->is_name = is_name != 0; item->datalen = 0; item->data = NULL; if (data) { if (item->is_name) { if (!(item->data = mm_strdup(data))) { mm_free(item->name); mm_free(item); goto done; } item->datalen = (u16)-1; } else { if (!(item->data = mm_malloc(datalen))) { mm_free(item->name); mm_free(item); goto done; } item->datalen = datalen; memcpy(item->data, data, datalen); } } *itemp = item; ++(*countp); result = 0; done: EVDNS_UNLOCK(req->port); return result; } /* exported function */ int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_A, CLASS_INET, ttl, n*4, 0, addrs); } /* exported function */ int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_AAAA, CLASS_INET, ttl, n*16, 0, addrs); } /* exported function */ int evdns_server_request_add_ptr_reply(struct evdns_server_request *req, struct in_addr *in, const char *inaddr_name, const char *hostname, int ttl) { u32 a; char buf[32]; if (in && inaddr_name) return -1; else if (!in && !inaddr_name) return -1; if (in) { a = ntohl(in->s_addr); evutil_snprintf(buf, sizeof(buf), "%d.%d.%d.%d.in-addr.arpa", (int)(u8)((a )&0xff), (int)(u8)((a>>8 )&0xff), (int)(u8)((a>>16)&0xff), (int)(u8)((a>>24)&0xff)); inaddr_name = buf; } return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, inaddr_name, TYPE_PTR, CLASS_INET, ttl, -1, 1, hostname); } /* exported function */ int evdns_server_request_add_cname_reply(struct evdns_server_request *req, const char *name, const char *cname, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_CNAME, CLASS_INET, ttl, -1, 1, cname); } /* exported function */ void evdns_server_request_set_flags(struct evdns_server_request *exreq, int flags) { struct server_request *req = TO_SERVER_REQUEST(exreq); req->base.flags &= ~(EVDNS_FLAGS_AA|EVDNS_FLAGS_RD); req->base.flags |= flags; } static int evdns_server_request_format_response(struct server_request *req, int err) { unsigned char buf[1500]; size_t buf_len = sizeof(buf); off_t j = 0, r; u16 t_; u32 t32_; int i; u16 flags; struct dnslabel_table table; if (err < 0 || err > 15) return -1; /* Set response bit and error code; copy OPCODE and RD fields from * question; copy RA and AA if set by caller. */ flags = req->base.flags; flags |= (0x8000 | err); dnslabel_table_init(&table); APPEND16(req->trans_id); APPEND16(flags); APPEND16(req->base.nquestions); APPEND16(req->n_answer); APPEND16(req->n_authority); APPEND16(req->n_additional); /* Add questions. */ for (i=0; i < req->base.nquestions; ++i) { const char *s = req->base.questions[i]->name; j = dnsname_to_labels(buf, buf_len, j, s, strlen(s), &table); if (j < 0) { dnslabel_clear(&table); return (int) j; } APPEND16(req->base.questions[i]->type); APPEND16(req->base.questions[i]->dns_question_class); } /* Add answer, authority, and additional sections. */ for (i=0; i<3; ++i) { struct server_reply_item *item; if (i==0) item = req->answer; else if (i==1) item = req->authority; else item = req->additional; while (item) { r = dnsname_to_labels(buf, buf_len, j, item->name, strlen(item->name), &table); if (r < 0) goto overflow; j = r; APPEND16(item->type); APPEND16(item->dns_question_class); APPEND32(item->ttl); if (item->is_name) { off_t len_idx = j, name_start; j += 2; name_start = j; r = dnsname_to_labels(buf, buf_len, j, item->data, strlen(item->data), &table); if (r < 0) goto overflow; j = r; t_ = htons( (short) (j-name_start) ); memcpy(buf+len_idx, &t_, 2); } else { APPEND16(item->datalen); if (j+item->datalen > (off_t)buf_len) goto overflow; memcpy(buf+j, item->data, item->datalen); j += item->datalen; } item = item->next; } } if (j > 512) { overflow: j = 512; buf[2] |= 0x02; /* set the truncated bit. */ } req->response_len = j; if (!(req->response = mm_malloc(req->response_len))) { server_request_free_answers(req); dnslabel_clear(&table); return (-1); } memcpy(req->response, buf, req->response_len); server_request_free_answers(req); dnslabel_clear(&table); return (0); } /* exported function */ int evdns_server_request_respond(struct evdns_server_request *req_, int err) { struct server_request *req = TO_SERVER_REQUEST(req_); struct evdns_server_port *port = req->port; int r = -1; EVDNS_LOCK(port); if (!req->response) { if ((r = evdns_server_request_format_response(req, err))<0) goto done; } r = sendto(port->socket, req->response, (int)req->response_len, 0, (struct sockaddr*) &req->addr, (ev_socklen_t)req->addrlen); if (r<0) { int sock_err = evutil_socket_geterror(port->socket); if (EVUTIL_ERR_RW_RETRIABLE(sock_err)) goto done; if (port->pending_replies) { req->prev_pending = port->pending_replies->prev_pending; req->next_pending = port->pending_replies; req->prev_pending->next_pending = req->next_pending->prev_pending = req; } else { req->prev_pending = req->next_pending = req; port->pending_replies = req; port->choked = 1; (void) event_del(&port->event); event_assign(&port->event, port->event_base, port->socket, (port->closing?0:EV_READ) | EV_WRITE | EV_PERSIST, server_port_ready_callback, port); if (event_add(&port->event, NULL) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding event for DNS server"); } } r = 1; goto done; } if (server_request_free(req)) { r = 0; goto done; } if (port->pending_replies) server_port_flush(port); r = 0; done: EVDNS_UNLOCK(port); return r; } /* Free all storage held by RRs in req. */ static void server_request_free_answers(struct server_request *req) { struct server_reply_item *victim, *next, **list; int i; for (i = 0; i < 3; ++i) { if (i==0) list = &req->answer; else if (i==1) list = &req->authority; else list = &req->additional; victim = *list; while (victim) { next = victim->next; mm_free(victim->name); if (victim->data) mm_free(victim->data); mm_free(victim); victim = next; } *list = NULL; } } /* Free all storage held by req, and remove links to it. */ /* return true iff we just wound up freeing the server_port. */ static int server_request_free(struct server_request *req) { int i, rc=1, lock=0; if (req->base.questions) { for (i = 0; i < req->base.nquestions; ++i) mm_free(req->base.questions[i]); mm_free(req->base.questions); } if (req->port) { EVDNS_LOCK(req->port); lock=1; if (req->port->pending_replies == req) { if (req->next_pending && req->next_pending != req) req->port->pending_replies = req->next_pending; else req->port->pending_replies = NULL; } rc = --req->port->refcnt; } if (req->response) { mm_free(req->response); } server_request_free_answers(req); if (req->next_pending && req->next_pending != req) { req->next_pending->prev_pending = req->prev_pending; req->prev_pending->next_pending = req->next_pending; } if (rc == 0) { EVDNS_UNLOCK(req->port); /* ????? nickm */ server_port_free(req->port); mm_free(req); return (1); } if (lock) EVDNS_UNLOCK(req->port); mm_free(req); return (0); } /* Free all storage held by an evdns_server_port. Only called when */ static void server_port_free(struct evdns_server_port *port) { EVUTIL_ASSERT(port); EVUTIL_ASSERT(!port->refcnt); EVUTIL_ASSERT(!port->pending_replies); if (port->socket > 0) { evutil_closesocket(port->socket); port->socket = -1; } (void) event_del(&port->event); event_debug_unassign(&port->event); EVTHREAD_FREE_LOCK(port->lock, EVTHREAD_LOCKTYPE_RECURSIVE); mm_free(port); } /* exported function */ int evdns_server_request_drop(struct evdns_server_request *req_) { struct server_request *req = TO_SERVER_REQUEST(req_); server_request_free(req); return 0; } /* exported function */ int evdns_server_request_get_requesting_addr(struct evdns_server_request *req_, struct sockaddr *sa, int addr_len) { struct server_request *req = TO_SERVER_REQUEST(req_); if (addr_len < (int)req->addrlen) return -1; memcpy(sa, &(req->addr), req->addrlen); return req->addrlen; } #undef APPEND16 #undef APPEND32 /* this is a libevent callback function which is called when a request */ /* has timed out. */ static void evdns_request_timeout_callback(evutil_socket_t fd, short events, void *arg) { struct request *const req = (struct request *) arg; struct evdns_base *base = req->base; (void) fd; (void) events; log(EVDNS_LOG_DEBUG, "Request %p timed out", arg); EVDNS_LOCK(base); if (req->tx_count >= req->base->global_max_retransmits) { struct nameserver *ns = req->ns; /* this request has failed */ log(EVDNS_LOG_DEBUG, "Giving up on request %p; tx_count==%d", arg, req->tx_count); reply_schedule_callback(req, 0, DNS_ERR_TIMEOUT, NULL); request_finished(req, &REQ_HEAD(req->base, req->trans_id), 1); nameserver_failed(ns, "request timed out."); } else { /* retransmit it */ log(EVDNS_LOG_DEBUG, "Retransmitting request %p; tx_count==%d", arg, req->tx_count); (void) evtimer_del(&req->timeout_event); request_swap_ns(req, nameserver_pick(base)); evdns_request_transmit(req); req->ns->timedout++; if (req->ns->timedout > req->base->global_max_nameserver_timeout) { req->ns->timedout = 0; nameserver_failed(req->ns, "request timed out."); } } EVDNS_UNLOCK(base); } /* try to send a request to a given server. */ /* */ /* return: */ /* 0 ok */ /* 1 temporary failure */ /* 2 other failure */ static int evdns_request_transmit_to(struct request *req, struct nameserver *server) { int r; ASSERT_LOCKED(req->base); ASSERT_VALID_REQUEST(req); if (server->requests_inflight == 1 && req->base->disable_when_inactive && event_add(&server->event, NULL) < 0) { return 1; } r = sendto(server->socket, (void*)req->request, req->request_len, 0, (struct sockaddr *)&server->address, server->addrlen); if (r < 0) { int err = evutil_socket_geterror(server->socket); if (EVUTIL_ERR_RW_RETRIABLE(err)) return 1; nameserver_failed(req->ns, evutil_socket_error_to_string(err)); return 2; } else if (r != (int)req->request_len) { return 1; /* short write */ } else { return 0; } } /* try to send a request, updating the fields of the request */ /* as needed */ /* */ /* return: */ /* 0 ok */ /* 1 failed */ static int evdns_request_transmit(struct request *req) { int retcode = 0, r; ASSERT_LOCKED(req->base); ASSERT_VALID_REQUEST(req); /* if we fail to send this packet then this flag marks it */ /* for evdns_transmit */ req->transmit_me = 1; EVUTIL_ASSERT(req->trans_id != 0xffff); if (!req->ns) { /* unable to transmit request if no nameservers */ return 1; } if (req->ns->choked) { /* don't bother trying to write to a socket */ /* which we have had EAGAIN from */ return 1; } r = evdns_request_transmit_to(req, req->ns); switch (r) { case 1: /* temp failure */ req->ns->choked = 1; nameserver_write_waiting(req->ns, 1); return 1; case 2: /* failed to transmit the request entirely. */ retcode = 1; /* fall through: we'll set a timeout, which will time out, * and make us retransmit the request anyway. */ default: /* all ok */ log(EVDNS_LOG_DEBUG, "Setting timeout for request %p, sent to nameserver %p", req, req->ns); if (evtimer_add(&req->timeout_event, &req->base->global_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer for request %p", req); /* ???? Do more? */ } req->tx_count++; req->transmit_me = 0; return retcode; } } static void nameserver_probe_callback(int result, char type, int count, int ttl, void *addresses, void *arg) { struct nameserver *const ns = (struct nameserver *) arg; (void) type; (void) count; (void) ttl; (void) addresses; if (result == DNS_ERR_CANCEL) { /* We canceled this request because the nameserver came up * for some other reason. Do not change our opinion about * the nameserver. */ return; } EVDNS_LOCK(ns->base); ns->probe_request = NULL; if (result == DNS_ERR_NONE || result == DNS_ERR_NOTEXIST) { /* this is a good reply */ nameserver_up(ns); } else { nameserver_probe_failed(ns); } EVDNS_UNLOCK(ns->base); } static void nameserver_send_probe(struct nameserver *const ns) { struct evdns_request *handle; struct request *req; char addrbuf[128]; /* here we need to send a probe to a given nameserver */ /* in the hope that it is up now. */ ASSERT_LOCKED(ns->base); log(EVDNS_LOG_DEBUG, "Sending probe to %s", evutil_format_sockaddr_port_( (struct sockaddr *)&ns->address, addrbuf, sizeof(addrbuf))); handle = mm_calloc(1, sizeof(*handle)); if (!handle) return; req = request_new(ns->base, handle, TYPE_A, "google.com", DNS_QUERY_NO_SEARCH, nameserver_probe_callback, ns); if (!req) { mm_free(handle); return; } ns->probe_request = handle; /* we force this into the inflight queue no matter what */ request_trans_id_set(req, transaction_id_pick(ns->base)); req->ns = ns; request_submit(req); } /* returns: */ /* 0 didn't try to transmit anything */ /* 1 tried to transmit something */ static int evdns_transmit(struct evdns_base *base) { char did_try_to_transmit = 0; int i; ASSERT_LOCKED(base); for (i = 0; i < base->n_req_heads; ++i) { if (base->req_heads[i]) { struct request *const started_at = base->req_heads[i], *req = started_at; /* first transmit all the requests which are currently waiting */ do { if (req->transmit_me) { did_try_to_transmit = 1; evdns_request_transmit(req); } req = req->next; } while (req != started_at); } } return did_try_to_transmit; } /* exported function */ int evdns_base_count_nameservers(struct evdns_base *base) { const struct nameserver *server; int n = 0; EVDNS_LOCK(base); server = base->server_head; if (!server) goto done; do { ++n; server = server->next; } while (server != base->server_head); done: EVDNS_UNLOCK(base); return n; } int evdns_count_nameservers(void) { return evdns_base_count_nameservers(current_base); } /* exported function */ int evdns_base_clear_nameservers_and_suspend(struct evdns_base *base) { struct nameserver *server, *started_at; int i; EVDNS_LOCK(base); server = base->server_head; started_at = base->server_head; if (!server) { EVDNS_UNLOCK(base); return 0; } while (1) { struct nameserver *next = server->next; (void) event_del(&server->event); if (evtimer_initialized(&server->timeout_event)) (void) evtimer_del(&server->timeout_event); if (server->probe_request) { evdns_cancel_request(server->base, server->probe_request); server->probe_request = NULL; } if (server->socket >= 0) evutil_closesocket(server->socket); mm_free(server); if (next == started_at) break; server = next; } base->server_head = NULL; base->global_good_nameservers = 0; for (i = 0; i < base->n_req_heads; ++i) { struct request *req, *req_started_at; req = req_started_at = base->req_heads[i]; while (req) { struct request *next = req->next; req->tx_count = req->reissue_count = 0; req->ns = NULL; /* ???? What to do about searches? */ (void) evtimer_del(&req->timeout_event); req->trans_id = 0; req->transmit_me = 0; base->global_requests_waiting++; evdns_request_insert(req, &base->req_waiting_head); /* We want to insert these suspended elements at the front of * the waiting queue, since they were pending before any of * the waiting entries were added. This is a circular list, * so we can just shift the start back by one.*/ base->req_waiting_head = base->req_waiting_head->prev; if (next == req_started_at) break; req = next; } base->req_heads[i] = NULL; } base->global_requests_inflight = 0; EVDNS_UNLOCK(base); return 0; } int evdns_clear_nameservers_and_suspend(void) { return evdns_base_clear_nameservers_and_suspend(current_base); } /* exported function */ int evdns_base_resume(struct evdns_base *base) { EVDNS_LOCK(base); evdns_requests_pump_waiting_queue(base); EVDNS_UNLOCK(base); return 0; } int evdns_resume(void) { return evdns_base_resume(current_base); } static int evdns_nameserver_add_impl_(struct evdns_base *base, const struct sockaddr *address, int addrlen) { /* first check to see if we already have this nameserver */ const struct nameserver *server = base->server_head, *const started_at = base->server_head; struct nameserver *ns; int err = 0; char addrbuf[128]; ASSERT_LOCKED(base); if (server) { do { if (!evutil_sockaddr_cmp((struct sockaddr*)&server->address, address, 1)) return 3; server = server->next; } while (server != started_at); } if (addrlen > (int)sizeof(ns->address)) { log(EVDNS_LOG_DEBUG, "Addrlen %d too long.", (int)addrlen); return 2; } ns = (struct nameserver *) mm_malloc(sizeof(struct nameserver)); if (!ns) return -1; memset(ns, 0, sizeof(struct nameserver)); ns->base = base; evtimer_assign(&ns->timeout_event, ns->base->event_base, nameserver_prod_callback, ns); ns->socket = evutil_socket_(address->sa_family, SOCK_DGRAM|EVUTIL_SOCK_NONBLOCK|EVUTIL_SOCK_CLOEXEC, 0); if (ns->socket < 0) { err = 1; goto out1; } if (base->global_outgoing_addrlen && !evutil_sockaddr_is_loopback_(address)) { if (bind(ns->socket, (struct sockaddr*)&base->global_outgoing_address, base->global_outgoing_addrlen) < 0) { log(EVDNS_LOG_WARN,"Couldn't bind to outgoing address"); err = 2; goto out2; } } memcpy(&ns->address, address, addrlen); ns->addrlen = addrlen; ns->state = 1; event_assign(&ns->event, ns->base->event_base, ns->socket, EV_READ | EV_PERSIST, nameserver_ready_callback, ns); if (!base->disable_when_inactive && event_add(&ns->event, NULL) < 0) { err = 2; goto out2; } log(EVDNS_LOG_DEBUG, "Added nameserver %s as %p", evutil_format_sockaddr_port_(address, addrbuf, sizeof(addrbuf)), ns); /* insert this nameserver into the list of them */ if (!base->server_head) { ns->next = ns->prev = ns; base->server_head = ns; } else { ns->next = base->server_head->next; ns->prev = base->server_head; base->server_head->next = ns; ns->next->prev = ns; } base->global_good_nameservers++; return 0; out2: evutil_closesocket(ns->socket); out1: event_debug_unassign(&ns->event); mm_free(ns); log(EVDNS_LOG_WARN, "Unable to add nameserver %s: error %d", evutil_format_sockaddr_port_(address, addrbuf, sizeof(addrbuf)), err); return err; } /* exported function */ int evdns_base_nameserver_add(struct evdns_base *base, unsigned long int address) { struct sockaddr_in sin; int res; memset(&sin, 0, sizeof(sin)); sin.sin_addr.s_addr = address; sin.sin_port = htons(53); sin.sin_family = AF_INET; EVDNS_LOCK(base); res = evdns_nameserver_add_impl_(base, (struct sockaddr*)&sin, sizeof(sin)); EVDNS_UNLOCK(base); return res; } int evdns_nameserver_add(unsigned long int address) { if (!current_base) current_base = evdns_base_new(NULL, 0); return evdns_base_nameserver_add(current_base, address); } static void sockaddr_setport(struct sockaddr *sa, ev_uint16_t port) { if (sa->sa_family == AF_INET) { ((struct sockaddr_in *)sa)->sin_port = htons(port); } else if (sa->sa_family == AF_INET6) { ((struct sockaddr_in6 *)sa)->sin6_port = htons(port); } } static ev_uint16_t sockaddr_getport(struct sockaddr *sa) { if (sa->sa_family == AF_INET) { return ntohs(((struct sockaddr_in *)sa)->sin_port); } else if (sa->sa_family == AF_INET6) { return ntohs(((struct sockaddr_in6 *)sa)->sin6_port); } else { return 0; } } /* exported function */ int evdns_base_nameserver_ip_add(struct evdns_base *base, const char *ip_as_string) { struct sockaddr_storage ss; struct sockaddr *sa; int len = sizeof(ss); int res; if (evutil_parse_sockaddr_port(ip_as_string, (struct sockaddr *)&ss, &len)) { log(EVDNS_LOG_WARN, "Unable to parse nameserver address %s", ip_as_string); return 4; } sa = (struct sockaddr *) &ss; if (sockaddr_getport(sa) == 0) sockaddr_setport(sa, 53); EVDNS_LOCK(base); res = evdns_nameserver_add_impl_(base, sa, len); EVDNS_UNLOCK(base); return res; } int evdns_nameserver_ip_add(const char *ip_as_string) { if (!current_base) current_base = evdns_base_new(NULL, 0); return evdns_base_nameserver_ip_add(current_base, ip_as_string); } int evdns_base_nameserver_sockaddr_add(struct evdns_base *base, const struct sockaddr *sa, ev_socklen_t len, unsigned flags) { int res; EVUTIL_ASSERT(base); EVDNS_LOCK(base); res = evdns_nameserver_add_impl_(base, sa, len); EVDNS_UNLOCK(base); return res; } int evdns_base_get_nameserver_addr(struct evdns_base *base, int idx, struct sockaddr *sa, ev_socklen_t len) { int result = -1; int i; struct nameserver *server; EVDNS_LOCK(base); server = base->server_head; for (i = 0; i < idx && server; ++i, server = server->next) { if (server->next == base->server_head) goto done; } if (! server) goto done; if (server->addrlen > len) { result = (int) server->addrlen; goto done; } memcpy(sa, &server->address, server->addrlen); result = (int) server->addrlen; done: EVDNS_UNLOCK(base); return result; } /* remove from the queue */ static void evdns_request_remove(struct request *req, struct request **head) { ASSERT_LOCKED(req->base); ASSERT_VALID_REQUEST(req); #if 0 { struct request *ptr; int found = 0; EVUTIL_ASSERT(*head != NULL); ptr = *head; do { if (ptr == req) { found = 1; break; } ptr = ptr->next; } while (ptr != *head); EVUTIL_ASSERT(found); EVUTIL_ASSERT(req->next); } #endif if (req->next == req) { /* only item in the list */ *head = NULL; } else { req->next->prev = req->prev; req->prev->next = req->next; if (*head == req) *head = req->next; } req->next = req->prev = NULL; } /* insert into the tail of the queue */ static void evdns_request_insert(struct request *req, struct request **head) { ASSERT_LOCKED(req->base); ASSERT_VALID_REQUEST(req); if (!*head) { *head = req; req->next = req->prev = req; return; } req->prev = (*head)->prev; req->prev->next = req; req->next = *head; (*head)->prev = req; } static int string_num_dots(const char *s) { int count = 0; while ((s = strchr(s, '.'))) { s++; count++; } return count; } static struct request * request_new(struct evdns_base *base, struct evdns_request *handle, int type, const char *name, int flags, evdns_callback_type callback, void *user_ptr) { const char issuing_now = (base->global_requests_inflight < base->global_max_requests_inflight) ? 1 : 0; const size_t name_len = strlen(name); const size_t request_max_len = evdns_request_len(name_len); const u16 trans_id = issuing_now ? transaction_id_pick(base) : 0xffff; /* the request data is alloced in a single block with the header */ struct request *const req = mm_malloc(sizeof(struct request) + request_max_len); int rlen; char namebuf[256]; (void) flags; ASSERT_LOCKED(base); if (!req) return NULL; if (name_len >= sizeof(namebuf)) { mm_free(req); return NULL; } memset(req, 0, sizeof(struct request)); req->base = base; evtimer_assign(&req->timeout_event, req->base->event_base, evdns_request_timeout_callback, req); if (base->global_randomize_case) { unsigned i; char randbits[(sizeof(namebuf)+7)/8]; strlcpy(namebuf, name, sizeof(namebuf)); evutil_secure_rng_get_bytes(randbits, (name_len+7)/8); for (i = 0; i < name_len; ++i) { if (EVUTIL_ISALPHA_(namebuf[i])) { if ((randbits[i >> 3] & (1<<(i & 7)))) namebuf[i] |= 0x20; else namebuf[i] &= ~0x20; } } name = namebuf; } /* request data lives just after the header */ req->request = ((u8 *) req) + sizeof(struct request); /* denotes that the request data shouldn't be free()ed */ req->request_appended = 1; rlen = evdns_request_data_build(name, name_len, trans_id, type, CLASS_INET, req->request, request_max_len); if (rlen < 0) goto err1; req->request_len = rlen; req->trans_id = trans_id; req->tx_count = 0; req->request_type = type; req->user_pointer = user_ptr; req->user_callback = callback; req->ns = issuing_now ? nameserver_pick(base) : NULL; req->next = req->prev = NULL; req->handle = handle; if (handle) { handle->current_req = req; handle->base = base; } return req; err1: mm_free(req); return NULL; } static void request_submit(struct request *const req) { struct evdns_base *base = req->base; ASSERT_LOCKED(base); ASSERT_VALID_REQUEST(req); if (req->ns) { /* if it has a nameserver assigned then this is going */ /* straight into the inflight queue */ evdns_request_insert(req, &REQ_HEAD(base, req->trans_id)); base->global_requests_inflight++; req->ns->requests_inflight++; evdns_request_transmit(req); } else { evdns_request_insert(req, &base->req_waiting_head); base->global_requests_waiting++; } } /* exported function */ void evdns_cancel_request(struct evdns_base *base, struct evdns_request *handle) { struct request *req; if (!handle->current_req) return; if (!base) { /* This redundancy is silly; can we fix it? (Not for 2.0) XXXX */ base = handle->base; if (!base) base = handle->current_req->base; } EVDNS_LOCK(base); if (handle->pending_cb) { EVDNS_UNLOCK(base); return; } req = handle->current_req; ASSERT_VALID_REQUEST(req); reply_schedule_callback(req, 0, DNS_ERR_CANCEL, NULL); if (req->ns) { /* remove from inflight queue */ request_finished(req, &REQ_HEAD(base, req->trans_id), 1); } else { /* remove from global_waiting head */ request_finished(req, &base->req_waiting_head, 1); } EVDNS_UNLOCK(base); } /* exported function */ struct evdns_request * evdns_base_resolve_ipv4(struct evdns_base *base, const char *name, int flags, evdns_callback_type callback, void *ptr) { struct evdns_request *handle; struct request *req; log(EVDNS_LOG_DEBUG, "Resolve requested for %s", name); handle = mm_calloc(1, sizeof(*handle)); if (handle == NULL) return NULL; EVDNS_LOCK(base); if (flags & DNS_QUERY_NO_SEARCH) { req = request_new(base, handle, TYPE_A, name, flags, callback, ptr); if (req) request_submit(req); } else { search_request_new(base, handle, TYPE_A, name, flags, callback, ptr); } if (handle->current_req == NULL) { mm_free(handle); handle = NULL; } EVDNS_UNLOCK(base); return handle; } int evdns_resolve_ipv4(const char *name, int flags, evdns_callback_type callback, void *ptr) { return evdns_base_resolve_ipv4(current_base, name, flags, callback, ptr) ? 0 : -1; } /* exported function */ struct evdns_request * evdns_base_resolve_ipv6(struct evdns_base *base, const char *name, int flags, evdns_callback_type callback, void *ptr) { struct evdns_request *handle; struct request *req; log(EVDNS_LOG_DEBUG, "Resolve requested for %s", name); handle = mm_calloc(1, sizeof(*handle)); if (handle == NULL) return NULL; EVDNS_LOCK(base); if (flags & DNS_QUERY_NO_SEARCH) { req = request_new(base, handle, TYPE_AAAA, name, flags, callback, ptr); if (req) request_submit(req); } else { search_request_new(base, handle, TYPE_AAAA, name, flags, callback, ptr); } if (handle->current_req == NULL) { mm_free(handle); handle = NULL; } EVDNS_UNLOCK(base); return handle; } int evdns_resolve_ipv6(const char *name, int flags, evdns_callback_type callback, void *ptr) { return evdns_base_resolve_ipv6(current_base, name, flags, callback, ptr) ? 0 : -1; } struct evdns_request * evdns_base_resolve_reverse(struct evdns_base *base, const struct in_addr *in, int flags, evdns_callback_type callback, void *ptr) { char buf[32]; struct evdns_request *handle; struct request *req; u32 a; EVUTIL_ASSERT(in); a = ntohl(in->s_addr); evutil_snprintf(buf, sizeof(buf), "%d.%d.%d.%d.in-addr.arpa", (int)(u8)((a )&0xff), (int)(u8)((a>>8 )&0xff), (int)(u8)((a>>16)&0xff), (int)(u8)((a>>24)&0xff)); handle = mm_calloc(1, sizeof(*handle)); if (handle == NULL) return NULL; log(EVDNS_LOG_DEBUG, "Resolve requested for %s (reverse)", buf); EVDNS_LOCK(base); req = request_new(base, handle, TYPE_PTR, buf, flags, callback, ptr); if (req) request_submit(req); if (handle->current_req == NULL) { mm_free(handle); handle = NULL; } EVDNS_UNLOCK(base); return (handle); } int evdns_resolve_reverse(const struct in_addr *in, int flags, evdns_callback_type callback, void *ptr) { return evdns_base_resolve_reverse(current_base, in, flags, callback, ptr) ? 0 : -1; } struct evdns_request * evdns_base_resolve_reverse_ipv6(struct evdns_base *base, const struct in6_addr *in, int flags, evdns_callback_type callback, void *ptr) { /* 32 nybbles, 32 periods, "ip6.arpa", NUL. */ char buf[73]; char *cp; struct evdns_request *handle; struct request *req; int i; EVUTIL_ASSERT(in); cp = buf; for (i=15; i >= 0; --i) { u8 byte = in->s6_addr[i]; *cp++ = "0123456789abcdef"[byte & 0x0f]; *cp++ = '.'; *cp++ = "0123456789abcdef"[byte >> 4]; *cp++ = '.'; } EVUTIL_ASSERT(cp + strlen("ip6.arpa") < buf+sizeof(buf)); memcpy(cp, "ip6.arpa", strlen("ip6.arpa")+1); handle = mm_calloc(1, sizeof(*handle)); if (handle == NULL) return NULL; log(EVDNS_LOG_DEBUG, "Resolve requested for %s (reverse)", buf); EVDNS_LOCK(base); req = request_new(base, handle, TYPE_PTR, buf, flags, callback, ptr); if (req) request_submit(req); if (handle->current_req == NULL) { mm_free(handle); handle = NULL; } EVDNS_UNLOCK(base); return (handle); } int evdns_resolve_reverse_ipv6(const struct in6_addr *in, int flags, evdns_callback_type callback, void *ptr) { return evdns_base_resolve_reverse_ipv6(current_base, in, flags, callback, ptr) ? 0 : -1; } /* ================================================================= */ /* Search support */ /* */ /* the libc resolver has support for searching a number of domains */ /* to find a name. If nothing else then it takes the single domain */ /* from the gethostname() call. */ /* */ /* It can also be configured via the domain and search options in a */ /* resolv.conf. */ /* */ /* The ndots option controls how many dots it takes for the resolver */ /* to decide that a name is non-local and so try a raw lookup first. */ struct search_domain { int len; struct search_domain *next; /* the text string is appended to this structure */ }; struct search_state { int refcount; int ndots; int num_domains; struct search_domain *head; }; static void search_state_decref(struct search_state *const state) { if (!state) return; state->refcount--; if (!state->refcount) { struct search_domain *next, *dom; for (dom = state->head; dom; dom = next) { next = dom->next; mm_free(dom); } mm_free(state); } } static struct search_state * search_state_new(void) { struct search_state *state = (struct search_state *) mm_malloc(sizeof(struct search_state)); if (!state) return NULL; memset(state, 0, sizeof(struct search_state)); state->refcount = 1; state->ndots = 1; return state; } static void search_postfix_clear(struct evdns_base *base) { search_state_decref(base->global_search_state); base->global_search_state = search_state_new(); } /* exported function */ void evdns_base_search_clear(struct evdns_base *base) { EVDNS_LOCK(base); search_postfix_clear(base); EVDNS_UNLOCK(base); } void evdns_search_clear(void) { evdns_base_search_clear(current_base); } static void search_postfix_add(struct evdns_base *base, const char *domain) { size_t domain_len; struct search_domain *sdomain; while (domain[0] == '.') domain++; domain_len = strlen(domain); ASSERT_LOCKED(base); if (!base->global_search_state) base->global_search_state = search_state_new(); if (!base->global_search_state) return; base->global_search_state->num_domains++; sdomain = (struct search_domain *) mm_malloc(sizeof(struct search_domain) + domain_len); if (!sdomain) return; memcpy( ((u8 *) sdomain) + sizeof(struct search_domain), domain, domain_len); sdomain->next = base->global_search_state->head; sdomain->len = (int) domain_len; base->global_search_state->head = sdomain; } /* reverse the order of members in the postfix list. This is needed because, */ /* when parsing resolv.conf we push elements in the wrong order */ static void search_reverse(struct evdns_base *base) { struct search_domain *cur, *prev = NULL, *next; ASSERT_LOCKED(base); cur = base->global_search_state->head; while (cur) { next = cur->next; cur->next = prev; prev = cur; cur = next; } base->global_search_state->head = prev; } /* exported function */ void evdns_base_search_add(struct evdns_base *base, const char *domain) { EVDNS_LOCK(base); search_postfix_add(base, domain); EVDNS_UNLOCK(base); } void evdns_search_add(const char *domain) { evdns_base_search_add(current_base, domain); } /* exported function */ void evdns_base_search_ndots_set(struct evdns_base *base, const int ndots) { EVDNS_LOCK(base); if (!base->global_search_state) base->global_search_state = search_state_new(); if (base->global_search_state) base->global_search_state->ndots = ndots; EVDNS_UNLOCK(base); } void evdns_search_ndots_set(const int ndots) { evdns_base_search_ndots_set(current_base, ndots); } static void search_set_from_hostname(struct evdns_base *base) { char hostname[HOST_NAME_MAX + 1], *domainname; ASSERT_LOCKED(base); search_postfix_clear(base); if (gethostname(hostname, sizeof(hostname))) return; domainname = strchr(hostname, '.'); if (!domainname) return; search_postfix_add(base, domainname); } /* warning: returns malloced string */ static char * search_make_new(const struct search_state *const state, int n, const char *const base_name) { const size_t base_len = strlen(base_name); char need_to_append_dot; struct search_domain *dom; if (!base_len) return NULL; need_to_append_dot = base_name[base_len - 1] == '.' ? 0 : 1; for (dom = state->head; dom; dom = dom->next) { if (!n--) { /* this is the postfix we want */ /* the actual postfix string is kept at the end of the structure */ const u8 *const postfix = ((u8 *) dom) + sizeof(struct search_domain); const int postfix_len = dom->len; char *const newname = (char *) mm_malloc(base_len + need_to_append_dot + postfix_len + 1); if (!newname) return NULL; memcpy(newname, base_name, base_len); if (need_to_append_dot) newname[base_len] = '.'; memcpy(newname + base_len + need_to_append_dot, postfix, postfix_len); newname[base_len + need_to_append_dot + postfix_len] = 0; return newname; } } /* we ran off the end of the list and still didn't find the requested string */ EVUTIL_ASSERT(0); return NULL; /* unreachable; stops warnings in some compilers. */ } static struct request * search_request_new(struct evdns_base *base, struct evdns_request *handle, int type, const char *const name, int flags, evdns_callback_type user_callback, void *user_arg) { ASSERT_LOCKED(base); EVUTIL_ASSERT(type == TYPE_A || type == TYPE_AAAA); EVUTIL_ASSERT(handle->current_req == NULL); if ( ((flags & DNS_QUERY_NO_SEARCH) == 0) && base->global_search_state && base->global_search_state->num_domains) { /* we have some domains to search */ struct request *req; if (string_num_dots(name) >= base->global_search_state->ndots) { req = request_new(base, handle, type, name, flags, user_callback, user_arg); if (!req) return NULL; handle->search_index = -1; } else { char *const new_name = search_make_new(base->global_search_state, 0, name); if (!new_name) return NULL; req = request_new(base, handle, type, new_name, flags, user_callback, user_arg); mm_free(new_name); if (!req) return NULL; handle->search_index = 0; } EVUTIL_ASSERT(handle->search_origname == NULL); handle->search_origname = mm_strdup(name); if (handle->search_origname == NULL) { /* XXX Should we dealloc req? If yes, how? */ if (req) mm_free(req); return NULL; } handle->search_state = base->global_search_state; handle->search_flags = flags; base->global_search_state->refcount++; request_submit(req); return req; } else { struct request *const req = request_new(base, handle, type, name, flags, user_callback, user_arg); if (!req) return NULL; request_submit(req); return req; } } /* this is called when a request has failed to find a name. We need to check */ /* if it is part of a search and, if so, try the next name in the list */ /* returns: */ /* 0 another request has been submitted */ /* 1 no more requests needed */ static int search_try_next(struct evdns_request *const handle) { struct request *req = handle->current_req; struct evdns_base *base = req->base; struct request *newreq; ASSERT_LOCKED(base); if (handle->search_state) { /* it is part of a search */ char *new_name; handle->search_index++; if (handle->search_index >= handle->search_state->num_domains) { /* no more postfixes to try, however we may need to try */ /* this name without a postfix */ if (string_num_dots(handle->search_origname) < handle->search_state->ndots) { /* yep, we need to try it raw */ newreq = request_new(base, NULL, req->request_type, handle->search_origname, handle->search_flags, req->user_callback, req->user_pointer); log(EVDNS_LOG_DEBUG, "Search: trying raw query %s", handle->search_origname); if (newreq) { search_request_finished(handle); goto submit_next; } } return 1; } new_name = search_make_new(handle->search_state, handle->search_index, handle->search_origname); if (!new_name) return 1; log(EVDNS_LOG_DEBUG, "Search: now trying %s (%d)", new_name, handle->search_index); newreq = request_new(base, NULL, req->request_type, new_name, handle->search_flags, req->user_callback, req->user_pointer); mm_free(new_name); if (!newreq) return 1; goto submit_next; } return 1; submit_next: request_finished(req, &REQ_HEAD(req->base, req->trans_id), 0); handle->current_req = newreq; newreq->handle = handle; request_submit(newreq); return 0; } static void search_request_finished(struct evdns_request *const handle) { ASSERT_LOCKED(handle->current_req->base); if (handle->search_state) { search_state_decref(handle->search_state); handle->search_state = NULL; } if (handle->search_origname) { mm_free(handle->search_origname); handle->search_origname = NULL; } } /* ================================================================= */ /* Parsing resolv.conf files */ static void evdns_resolv_set_defaults(struct evdns_base *base, int flags) { /* if the file isn't found then we assume a local resolver */ ASSERT_LOCKED(base); if (flags & DNS_OPTION_SEARCH) search_set_from_hostname(base); if (flags & DNS_OPTION_NAMESERVERS) evdns_base_nameserver_ip_add(base,"127.0.0.1"); } #ifndef EVENT__HAVE_STRTOK_R static char * strtok_r(char *s, const char *delim, char **state) { char *cp, *start; start = cp = s ? s : *state; if (!cp) return NULL; while (*cp && !strchr(delim, *cp)) ++cp; if (!*cp) { if (cp == start) return NULL; *state = NULL; return start; } else { *cp++ = '\0'; *state = cp; return start; } } #endif /* helper version of atoi which returns -1 on error */ static int strtoint(const char *const str) { char *endptr; const int r = strtol(str, &endptr, 10); if (*endptr) return -1; return r; } /* Parse a number of seconds into a timeval; return -1 on error. */ static int evdns_strtotimeval(const char *const str, struct timeval *out) { double d; char *endptr; d = strtod(str, &endptr); if (*endptr) return -1; if (d < 0) return -1; out->tv_sec = (int) d; out->tv_usec = (int) ((d - (int) d)*1000000); if (out->tv_sec == 0 && out->tv_usec < 1000) /* less than 1 msec */ return -1; return 0; } /* helper version of atoi that returns -1 on error and clips to bounds. */ static int strtoint_clipped(const char *const str, int min, int max) { int r = strtoint(str); if (r == -1) return r; else if (r<min) return min; else if (r>max) return max; else return r; } static int evdns_base_set_max_requests_inflight(struct evdns_base *base, int maxinflight) { int old_n_heads = base->n_req_heads, n_heads; struct request **old_heads = base->req_heads, **new_heads, *req; int i; ASSERT_LOCKED(base); if (maxinflight < 1) maxinflight = 1; n_heads = (maxinflight+4) / 5; EVUTIL_ASSERT(n_heads > 0); new_heads = mm_calloc(n_heads, sizeof(struct request*)); if (!new_heads) return (-1); if (old_heads) { for (i = 0; i < old_n_heads; ++i) { while (old_heads[i]) { req = old_heads[i]; evdns_request_remove(req, &old_heads[i]); evdns_request_insert(req, &new_heads[req->trans_id % n_heads]); } } mm_free(old_heads); } base->req_heads = new_heads; base->n_req_heads = n_heads; base->global_max_requests_inflight = maxinflight; return (0); } /* exported function */ int evdns_base_set_option(struct evdns_base *base, const char *option, const char *val) { int res; EVDNS_LOCK(base); res = evdns_base_set_option_impl(base, option, val, DNS_OPTIONS_ALL); EVDNS_UNLOCK(base); return res; } static inline int str_matches_option(const char *s1, const char *optionname) { /* Option names are given as "option:" We accept either 'option' in * s1, or 'option:randomjunk'. The latter form is to implement the * resolv.conf parser. */ size_t optlen = strlen(optionname); size_t slen = strlen(s1); if (slen == optlen || slen == optlen - 1) return !strncmp(s1, optionname, slen); else if (slen > optlen) return !strncmp(s1, optionname, optlen); else return 0; } static int evdns_base_set_option_impl(struct evdns_base *base, const char *option, const char *val, int flags) { ASSERT_LOCKED(base); if (str_matches_option(option, "ndots:")) { const int ndots = strtoint(val); if (ndots == -1) return -1; if (!(flags & DNS_OPTION_SEARCH)) return 0; log(EVDNS_LOG_DEBUG, "Setting ndots to %d", ndots); if (!base->global_search_state) base->global_search_state = search_state_new(); if (!base->global_search_state) return -1; base->global_search_state->ndots = ndots; } else if (str_matches_option(option, "timeout:")) { struct timeval tv; if (evdns_strtotimeval(val, &tv) == -1) return -1; if (!(flags & DNS_OPTION_MISC)) return 0; log(EVDNS_LOG_DEBUG, "Setting timeout to %s", val); memcpy(&base->global_timeout, &tv, sizeof(struct timeval)); } else if (str_matches_option(option, "getaddrinfo-allow-skew:")) { struct timeval tv; if (evdns_strtotimeval(val, &tv) == -1) return -1; if (!(flags & DNS_OPTION_MISC)) return 0; log(EVDNS_LOG_DEBUG, "Setting getaddrinfo-allow-skew to %s", val); memcpy(&base->global_getaddrinfo_allow_skew, &tv, sizeof(struct timeval)); } else if (str_matches_option(option, "max-timeouts:")) { const int maxtimeout = strtoint_clipped(val, 1, 255); if (maxtimeout == -1) return -1; if (!(flags & DNS_OPTION_MISC)) return 0; log(EVDNS_LOG_DEBUG, "Setting maximum allowed timeouts to %d", maxtimeout); base->global_max_nameserver_timeout = maxtimeout; } else if (str_matches_option(option, "max-inflight:")) { const int maxinflight = strtoint_clipped(val, 1, 65000); if (maxinflight == -1) return -1; if (!(flags & DNS_OPTION_MISC)) return 0; log(EVDNS_LOG_DEBUG, "Setting maximum inflight requests to %d", maxinflight); evdns_base_set_max_requests_inflight(base, maxinflight); } else if (str_matches_option(option, "attempts:")) { int retries = strtoint(val); if (retries == -1) return -1; if (retries > 255) retries = 255; if (!(flags & DNS_OPTION_MISC)) return 0; log(EVDNS_LOG_DEBUG, "Setting retries to %d", retries); base->global_max_retransmits = retries; } else if (str_matches_option(option, "randomize-case:")) { int randcase = strtoint(val); if (!(flags & DNS_OPTION_MISC)) return 0; base->global_randomize_case = randcase; } else if (str_matches_option(option, "bind-to:")) { /* XXX This only applies to successive nameservers, not * to already-configured ones. We might want to fix that. */ int len = sizeof(base->global_outgoing_address); if (!(flags & DNS_OPTION_NAMESERVERS)) return 0; if (evutil_parse_sockaddr_port(val, (struct sockaddr*)&base->global_outgoing_address, &len)) return -1; base->global_outgoing_addrlen = len; } else if (str_matches_option(option, "initial-probe-timeout:")) { struct timeval tv; if (evdns_strtotimeval(val, &tv) == -1) return -1; if (tv.tv_sec > 3600) tv.tv_sec = 3600; if (!(flags & DNS_OPTION_MISC)) return 0; log(EVDNS_LOG_DEBUG, "Setting initial probe timeout to %s", val); memcpy(&base->global_nameserver_probe_initial_timeout, &tv, sizeof(tv)); } return 0; } int evdns_set_option(const char *option, const char *val, int flags) { if (!current_base) current_base = evdns_base_new(NULL, 0); return evdns_base_set_option(current_base, option, val); } static void resolv_conf_parse_line(struct evdns_base *base, char *const start, int flags) { char *strtok_state; static const char *const delims = " \t"; #define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) char *const first_token = strtok_r(start, delims, &strtok_state); ASSERT_LOCKED(base); if (!first_token) return; if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { const char *const nameserver = NEXT_TOKEN; if (nameserver) evdns_base_nameserver_ip_add(base, nameserver); } else if (!strcmp(first_token, "domain") && (flags & DNS_OPTION_SEARCH)) { const char *const domain = NEXT_TOKEN; if (domain) { search_postfix_clear(base); search_postfix_add(base, domain); } } else if (!strcmp(first_token, "search") && (flags & DNS_OPTION_SEARCH)) { const char *domain; search_postfix_clear(base); while ((domain = NEXT_TOKEN)) { search_postfix_add(base, domain); } search_reverse(base); } else if (!strcmp(first_token, "options")) { const char *option; while ((option = NEXT_TOKEN)) { const char *val = strchr(option, ':'); evdns_base_set_option_impl(base, option, val ? val+1 : "", flags); } } #undef NEXT_TOKEN } /* exported function */ /* returns: */ /* 0 no errors */ /* 1 failed to open file */ /* 2 failed to stat file */ /* 3 file too large */ /* 4 out of memory */ /* 5 short read from file */ int evdns_base_resolv_conf_parse(struct evdns_base *base, int flags, const char *const filename) { int res; EVDNS_LOCK(base); res = evdns_base_resolv_conf_parse_impl(base, flags, filename); EVDNS_UNLOCK(base); return res; } static char * evdns_get_default_hosts_filename(void) { #ifdef _WIN32 /* Windows is a little coy about where it puts its configuration * files. Sure, they're _usually_ in C:\windows\system32, but * there's no reason in principle they couldn't be in * W:\hoboken chicken emergency\ */ char path[MAX_PATH+1]; static const char hostfile[] = "\\drivers\\etc\\hosts"; char *path_out; size_t len_out; if (! SHGetSpecialFolderPathA(NULL, path, CSIDL_SYSTEM, 0)) return NULL; len_out = strlen(path)+strlen(hostfile)+1; path_out = mm_malloc(len_out); evutil_snprintf(path_out, len_out, "%s%s", path, hostfile); return path_out; #else return mm_strdup("/etc/hosts"); #endif } static int evdns_base_resolv_conf_parse_impl(struct evdns_base *base, int flags, const char *const filename) { size_t n; char *resolv; char *start; int err = 0; log(EVDNS_LOG_DEBUG, "Parsing resolv.conf file %s", filename); if (flags & DNS_OPTION_HOSTSFILE) { char *fname = evdns_get_default_hosts_filename(); evdns_base_load_hosts(base, fname); if (fname) mm_free(fname); } if ((err = evutil_read_file_(filename, &resolv, &n, 0)) < 0) { if (err == -1) { /* No file. */ evdns_resolv_set_defaults(base, flags); return 1; } else { return 2; } } start = resolv; for (;;) { char *const newline = strchr(start, '\n'); if (!newline) { resolv_conf_parse_line(base, start, flags); break; } else { *newline = 0; resolv_conf_parse_line(base, start, flags); start = newline + 1; } } if (!base->server_head && (flags & DNS_OPTION_NAMESERVERS)) { /* no nameservers were configured. */ evdns_base_nameserver_ip_add(base, "127.0.0.1"); err = 6; } if (flags & DNS_OPTION_SEARCH && (!base->global_search_state || base->global_search_state->num_domains == 0)) { search_set_from_hostname(base); } mm_free(resolv); return err; } int evdns_resolv_conf_parse(int flags, const char *const filename) { if (!current_base) current_base = evdns_base_new(NULL, 0); return evdns_base_resolv_conf_parse(current_base, flags, filename); } #ifdef _WIN32 /* Add multiple nameservers from a space-or-comma-separated list. */ static int evdns_nameserver_ip_add_line(struct evdns_base *base, const char *ips) { const char *addr; char *buf; int r; ASSERT_LOCKED(base); while (*ips) { while (isspace(*ips) || *ips == ',' || *ips == '\t') ++ips; addr = ips; while (isdigit(*ips) || *ips == '.' || *ips == ':' || *ips=='[' || *ips==']') ++ips; buf = mm_malloc(ips-addr+1); if (!buf) return 4; memcpy(buf, addr, ips-addr); buf[ips-addr] = '\0'; r = evdns_base_nameserver_ip_add(base, buf); mm_free(buf); if (r) return r; } return 0; } typedef DWORD(WINAPI *GetNetworkParams_fn_t)(FIXED_INFO *, DWORD*); /* Use the windows GetNetworkParams interface in iphlpapi.dll to */ /* figure out what our nameservers are. */ static int load_nameservers_with_getnetworkparams(struct evdns_base *base) { /* Based on MSDN examples and inspection of c-ares code. */ FIXED_INFO *fixed; HMODULE handle = 0; ULONG size = sizeof(FIXED_INFO); void *buf = NULL; int status = 0, r, added_any; IP_ADDR_STRING *ns; GetNetworkParams_fn_t fn; ASSERT_LOCKED(base); if (!(handle = evutil_load_windows_system_library_( TEXT("iphlpapi.dll")))) { log(EVDNS_LOG_WARN, "Could not open iphlpapi.dll"); status = -1; goto done; } if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, "GetNetworkParams"))) { log(EVDNS_LOG_WARN, "Could not get address of function."); status = -1; goto done; } buf = mm_malloc(size); if (!buf) { status = 4; goto done; } fixed = buf; r = fn(fixed, &size); if (r != ERROR_SUCCESS && r != ERROR_BUFFER_OVERFLOW) { status = -1; goto done; } if (r != ERROR_SUCCESS) { mm_free(buf); buf = mm_malloc(size); if (!buf) { status = 4; goto done; } fixed = buf; r = fn(fixed, &size); if (r != ERROR_SUCCESS) { log(EVDNS_LOG_DEBUG, "fn() failed."); status = -1; goto done; } } EVUTIL_ASSERT(fixed); added_any = 0; ns = &(fixed->DnsServerList); while (ns) { r = evdns_nameserver_ip_add_line(base, ns->IpAddress.String); if (r) { log(EVDNS_LOG_DEBUG,"Could not add nameserver %s to list,error: %d", (ns->IpAddress.String),(int)GetLastError()); status = r; } else { ++added_any; log(EVDNS_LOG_DEBUG,"Successfully added %s as nameserver",ns->IpAddress.String); } ns = ns->Next; } if (!added_any) { log(EVDNS_LOG_DEBUG, "No nameservers added."); if (status == 0) status = -1; } else { status = 0; } done: if (buf) mm_free(buf); if (handle) FreeLibrary(handle); return status; } static int config_nameserver_from_reg_key(struct evdns_base *base, HKEY key, const TCHAR *subkey) { char *buf; DWORD bufsz = 0, type = 0; int status = 0; ASSERT_LOCKED(base); if (RegQueryValueEx(key, subkey, 0, &type, NULL, &bufsz) != ERROR_MORE_DATA) return -1; if (!(buf = mm_malloc(bufsz))) return -1; if (RegQueryValueEx(key, subkey, 0, &type, (LPBYTE)buf, &bufsz) == ERROR_SUCCESS && bufsz > 1) { status = evdns_nameserver_ip_add_line(base,buf); } mm_free(buf); return status; } #define SERVICES_KEY TEXT("System\\CurrentControlSet\\Services\\") #define WIN_NS_9X_KEY SERVICES_KEY TEXT("VxD\\MSTCP") #define WIN_NS_NT_KEY SERVICES_KEY TEXT("Tcpip\\Parameters") static int load_nameservers_from_registry(struct evdns_base *base) { int found = 0; int r; #define TRY(k, name) \ if (!found && config_nameserver_from_reg_key(base,k,TEXT(name)) == 0) { \ log(EVDNS_LOG_DEBUG,"Found nameservers in %s/%s",#k,name); \ found = 1; \ } else if (!found) { \ log(EVDNS_LOG_DEBUG,"Didn't find nameservers in %s/%s", \ #k,#name); \ } ASSERT_LOCKED(base); if (((int)GetVersion()) > 0) { /* NT */ HKEY nt_key = 0, interfaces_key = 0; if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, KEY_READ, &nt_key) != ERROR_SUCCESS) { log(EVDNS_LOG_DEBUG,"Couldn't open nt key, %d",(int)GetLastError()); return -1; } r = RegOpenKeyEx(nt_key, TEXT("Interfaces"), 0, KEY_QUERY_VALUE|KEY_ENUMERATE_SUB_KEYS, &interfaces_key); if (r != ERROR_SUCCESS) { log(EVDNS_LOG_DEBUG,"Couldn't open interfaces key, %d",(int)GetLastError()); return -1; } TRY(nt_key, "NameServer"); TRY(nt_key, "DhcpNameServer"); TRY(interfaces_key, "NameServer"); TRY(interfaces_key, "DhcpNameServer"); RegCloseKey(interfaces_key); RegCloseKey(nt_key); } else { HKEY win_key = 0; if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_9X_KEY, 0, KEY_READ, &win_key) != ERROR_SUCCESS) { log(EVDNS_LOG_DEBUG, "Couldn't open registry key, %d", (int)GetLastError()); return -1; } TRY(win_key, "NameServer"); RegCloseKey(win_key); } if (found == 0) { log(EVDNS_LOG_WARN,"Didn't find any nameservers."); } return found ? 0 : -1; #undef TRY } int evdns_base_config_windows_nameservers(struct evdns_base *base) { int r; char *fname; if (base == NULL) base = current_base; if (base == NULL) return -1; EVDNS_LOCK(base); fname = evdns_get_default_hosts_filename(); log(EVDNS_LOG_DEBUG, "Loading hosts entries from %s", fname); evdns_base_load_hosts(base, fname); if (fname) mm_free(fname); if (load_nameservers_with_getnetworkparams(base) == 0) { EVDNS_UNLOCK(base); return 0; } r = load_nameservers_from_registry(base); EVDNS_UNLOCK(base); return r; } int evdns_config_windows_nameservers(void) { if (!current_base) { current_base = evdns_base_new(NULL, 1); return current_base == NULL ? -1 : 0; } else { return evdns_base_config_windows_nameservers(current_base); } } #endif struct evdns_base * evdns_base_new(struct event_base *event_base, int flags) { struct evdns_base *base; if (evutil_secure_rng_init() < 0) { log(EVDNS_LOG_WARN, "Unable to seed random number generator; " "DNS can't run."); return NULL; } /* Give the evutil library a hook into its evdns-enabled * functionality. We can't just call evdns_getaddrinfo directly or * else libevent-core will depend on libevent-extras. */ evutil_set_evdns_getaddrinfo_fn_(evdns_getaddrinfo); evutil_set_evdns_getaddrinfo_cancel_fn_(evdns_getaddrinfo_cancel); base = mm_malloc(sizeof(struct evdns_base)); if (base == NULL) return (NULL); memset(base, 0, sizeof(struct evdns_base)); base->req_waiting_head = NULL; EVTHREAD_ALLOC_LOCK(base->lock, EVTHREAD_LOCKTYPE_RECURSIVE); EVDNS_LOCK(base); /* Set max requests inflight and allocate req_heads. */ base->req_heads = NULL; evdns_base_set_max_requests_inflight(base, 64); base->server_head = NULL; base->event_base = event_base; base->global_good_nameservers = base->global_requests_inflight = base->global_requests_waiting = 0; base->global_timeout.tv_sec = 5; base->global_timeout.tv_usec = 0; base->global_max_reissues = 1; base->global_max_retransmits = 3; base->global_max_nameserver_timeout = 3; base->global_search_state = NULL; base->global_randomize_case = 1; base->global_getaddrinfo_allow_skew.tv_sec = 3; base->global_getaddrinfo_allow_skew.tv_usec = 0; base->global_nameserver_probe_initial_timeout.tv_sec = 10; base->global_nameserver_probe_initial_timeout.tv_usec = 0; TAILQ_INIT(&base->hostsdb); #define EVDNS_BASE_ALL_FLAGS (0x8001) if (flags & ~EVDNS_BASE_ALL_FLAGS) { flags = EVDNS_BASE_INITIALIZE_NAMESERVERS; log(EVDNS_LOG_WARN, "Unrecognized flag passed to evdns_base_new(). Assuming " "you meant EVDNS_BASE_INITIALIZE_NAMESERVERS."); } #undef EVDNS_BASE_ALL_FLAGS if (flags & EVDNS_BASE_INITIALIZE_NAMESERVERS) { int r; #ifdef _WIN32 r = evdns_base_config_windows_nameservers(base); #else r = evdns_base_resolv_conf_parse(base, DNS_OPTIONS_ALL, "/etc/resolv.conf"); #endif if (r == -1) { evdns_base_free_and_unlock(base, 0); return NULL; } } if (flags & EVDNS_BASE_DISABLE_WHEN_INACTIVE) { base->disable_when_inactive = 1; } EVDNS_UNLOCK(base); return base; } int evdns_init(void) { struct evdns_base *base = evdns_base_new(NULL, 1); if (base) { current_base = base; return 0; } else { return -1; } } const char * evdns_err_to_string(int err) { switch (err) { case DNS_ERR_NONE: return "no error"; case DNS_ERR_FORMAT: return "misformatted query"; case DNS_ERR_SERVERFAILED: return "server failed"; case DNS_ERR_NOTEXIST: return "name does not exist"; case DNS_ERR_NOTIMPL: return "query not implemented"; case DNS_ERR_REFUSED: return "refused"; case DNS_ERR_TRUNCATED: return "reply truncated or ill-formed"; case DNS_ERR_UNKNOWN: return "unknown"; case DNS_ERR_TIMEOUT: return "request timed out"; case DNS_ERR_SHUTDOWN: return "dns subsystem shut down"; case DNS_ERR_CANCEL: return "dns request canceled"; case DNS_ERR_NODATA: return "no records in the reply"; default: return "[Unknown error code]"; } } static void evdns_nameserver_free(struct nameserver *server) { if (server->socket >= 0) evutil_closesocket(server->socket); (void) event_del(&server->event); event_debug_unassign(&server->event); if (server->state == 0) (void) event_del(&server->timeout_event); if (server->probe_request) { evdns_cancel_request(server->base, server->probe_request); server->probe_request = NULL; } event_debug_unassign(&server->timeout_event); mm_free(server); } static void evdns_base_free_and_unlock(struct evdns_base *base, int fail_requests) { struct nameserver *server, *server_next; struct search_domain *dom, *dom_next; int i; /* Requires that we hold the lock. */ /* TODO(nickm) we might need to refcount here. */ for (i = 0; i < base->n_req_heads; ++i) { while (base->req_heads[i]) { if (fail_requests) reply_schedule_callback(base->req_heads[i], 0, DNS_ERR_SHUTDOWN, NULL); request_finished(base->req_heads[i], &REQ_HEAD(base, base->req_heads[i]->trans_id), 1); } } while (base->req_waiting_head) { if (fail_requests) reply_schedule_callback(base->req_waiting_head, 0, DNS_ERR_SHUTDOWN, NULL); request_finished(base->req_waiting_head, &base->req_waiting_head, 1); } base->global_requests_inflight = base->global_requests_waiting = 0; for (server = base->server_head; server; server = server_next) { server_next = server->next; /** already done something before */ server->probe_request = NULL; evdns_nameserver_free(server); if (server_next == base->server_head) break; } base->server_head = NULL; base->global_good_nameservers = 0; if (base->global_search_state) { for (dom = base->global_search_state->head; dom; dom = dom_next) { dom_next = dom->next; mm_free(dom); } mm_free(base->global_search_state); base->global_search_state = NULL; } { struct hosts_entry *victim; while ((victim = TAILQ_FIRST(&base->hostsdb))) { TAILQ_REMOVE(&base->hostsdb, victim, next); mm_free(victim); } } mm_free(base->req_heads); EVDNS_UNLOCK(base); EVTHREAD_FREE_LOCK(base->lock, EVTHREAD_LOCKTYPE_RECURSIVE); mm_free(base); } void evdns_base_free(struct evdns_base *base, int fail_requests) { EVDNS_LOCK(base); evdns_base_free_and_unlock(base, fail_requests); } void evdns_base_clear_host_addresses(struct evdns_base *base) { struct hosts_entry *victim; EVDNS_LOCK(base); while ((victim = TAILQ_FIRST(&base->hostsdb))) { TAILQ_REMOVE(&base->hostsdb, victim, next); mm_free(victim); } EVDNS_UNLOCK(base); } void evdns_shutdown(int fail_requests) { if (current_base) { struct evdns_base *b = current_base; current_base = NULL; evdns_base_free(b, fail_requests); } evdns_log_fn = NULL; } static int evdns_base_parse_hosts_line(struct evdns_base *base, char *line) { char *strtok_state; static const char *const delims = " \t"; char *const addr = strtok_r(line, delims, &strtok_state); char *hostname, *hash; struct sockaddr_storage ss; int socklen = sizeof(ss); ASSERT_LOCKED(base); #define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) if (!addr || *addr == '#') return 0; memset(&ss, 0, sizeof(ss)); if (evutil_parse_sockaddr_port(addr, (struct sockaddr*)&ss, &socklen)<0) return -1; if (socklen > (int)sizeof(struct sockaddr_in6)) return -1; if (sockaddr_getport((struct sockaddr*)&ss)) return -1; while ((hostname = NEXT_TOKEN)) { struct hosts_entry *he; size_t namelen; if ((hash = strchr(hostname, '#'))) { if (hash == hostname) return 0; *hash = '\0'; } namelen = strlen(hostname); he = mm_calloc(1, sizeof(struct hosts_entry)+namelen); if (!he) return -1; EVUTIL_ASSERT(socklen <= (int)sizeof(he->addr)); memcpy(&he->addr, &ss, socklen); memcpy(he->hostname, hostname, namelen+1); he->addrlen = socklen; TAILQ_INSERT_TAIL(&base->hostsdb, he, next); if (hash) return 0; } return 0; #undef NEXT_TOKEN } static int evdns_base_load_hosts_impl(struct evdns_base *base, const char *hosts_fname) { char *str=NULL, *cp, *eol; size_t len; int err=0; ASSERT_LOCKED(base); if (hosts_fname == NULL || (err = evutil_read_file_(hosts_fname, &str, &len, 0)) < 0) { char tmp[64]; strlcpy(tmp, "127.0.0.1 localhost", sizeof(tmp)); evdns_base_parse_hosts_line(base, tmp); strlcpy(tmp, "::1 localhost", sizeof(tmp)); evdns_base_parse_hosts_line(base, tmp); return err ? -1 : 0; } /* This will break early if there is a NUL in the hosts file. * Probably not a problem.*/ cp = str; for (;;) { eol = strchr(cp, '\n'); if (eol) { *eol = '\0'; evdns_base_parse_hosts_line(base, cp); cp = eol+1; } else { evdns_base_parse_hosts_line(base, cp); break; } } mm_free(str); return 0; } int evdns_base_load_hosts(struct evdns_base *base, const char *hosts_fname) { int res; if (!base) base = current_base; EVDNS_LOCK(base); res = evdns_base_load_hosts_impl(base, hosts_fname); EVDNS_UNLOCK(base); return res; } /* A single request for a getaddrinfo, either v4 or v6. */ struct getaddrinfo_subrequest { struct evdns_request *r; ev_uint32_t type; }; /* State data used to implement an in-progress getaddrinfo. */ struct evdns_getaddrinfo_request { struct evdns_base *evdns_base; /* Copy of the modified 'hints' data that we'll use to build * answers. */ struct evutil_addrinfo hints; /* The callback to invoke when we're done */ evdns_getaddrinfo_cb user_cb; /* User-supplied data to give to the callback. */ void *user_data; /* The port to use when building sockaddrs. */ ev_uint16_t port; /* The sub_request for an A record (if any) */ struct getaddrinfo_subrequest ipv4_request; /* The sub_request for an AAAA record (if any) */ struct getaddrinfo_subrequest ipv6_request; /* The cname result that we were told (if any) */ char *cname_result; /* If we have one request answered and one request still inflight, * then this field holds the answer from the first request... */ struct evutil_addrinfo *pending_result; /* And this event is a timeout that will tell us to cancel the second * request if it's taking a long time. */ struct event timeout; /* And this field holds the error code from the first request... */ int pending_error; /* If this is set, the user canceled this request. */ unsigned user_canceled : 1; /* If this is set, the user can no longer cancel this request; we're * just waiting for the free. */ unsigned request_done : 1; }; /* Convert an evdns errors to the equivalent getaddrinfo error. */ static int evdns_err_to_getaddrinfo_err(int e1) { /* XXX Do this better! */ if (e1 == DNS_ERR_NONE) return 0; else if (e1 == DNS_ERR_NOTEXIST) return EVUTIL_EAI_NONAME; else return EVUTIL_EAI_FAIL; } /* Return the more informative of two getaddrinfo errors. */ static int getaddrinfo_merge_err(int e1, int e2) { /* XXXX be cleverer here. */ if (e1 == 0) return e2; else return e1; } static void free_getaddrinfo_request(struct evdns_getaddrinfo_request *data) { /* DO NOT CALL this if either of the requests is pending. Only once * both callbacks have been invoked is it safe to free the request */ if (data->pending_result) evutil_freeaddrinfo(data->pending_result); if (data->cname_result) mm_free(data->cname_result); event_del(&data->timeout); mm_free(data); return; } static void add_cname_to_reply(struct evdns_getaddrinfo_request *data, struct evutil_addrinfo *ai) { if (data->cname_result && ai) { ai->ai_canonname = data->cname_result; data->cname_result = NULL; } } /* Callback: invoked when one request in a mixed-format A/AAAA getaddrinfo * request has finished, but the other one took too long to answer. Pass * along the answer we got, and cancel the other request. */ static void evdns_getaddrinfo_timeout_cb(evutil_socket_t fd, short what, void *ptr) { int v4_timedout = 0, v6_timedout = 0; struct evdns_getaddrinfo_request *data = ptr; /* Cancel any pending requests, and note which one */ if (data->ipv4_request.r) { /* XXXX This does nothing if the request's callback is already * running (pending_cb is set). */ evdns_cancel_request(NULL, data->ipv4_request.r); v4_timedout = 1; EVDNS_LOCK(data->evdns_base); ++data->evdns_base->getaddrinfo_ipv4_timeouts; EVDNS_UNLOCK(data->evdns_base); } if (data->ipv6_request.r) { /* XXXX This does nothing if the request's callback is already * running (pending_cb is set). */ evdns_cancel_request(NULL, data->ipv6_request.r); v6_timedout = 1; EVDNS_LOCK(data->evdns_base); ++data->evdns_base->getaddrinfo_ipv6_timeouts; EVDNS_UNLOCK(data->evdns_base); } /* We only use this timeout callback when we have an answer for * one address. */ EVUTIL_ASSERT(!v4_timedout || !v6_timedout); /* Report the outcome of the other request that didn't time out. */ if (data->pending_result) { add_cname_to_reply(data, data->pending_result); data->user_cb(0, data->pending_result, data->user_data); data->pending_result = NULL; } else { int e = data->pending_error; if (!e) e = EVUTIL_EAI_AGAIN; data->user_cb(e, NULL, data->user_data); } data->user_cb = NULL; /* prevent double-call if evdns callbacks are * in-progress. XXXX It would be better if this * weren't necessary. */ if (!v4_timedout && !v6_timedout) { /* should be impossible? XXXX */ free_getaddrinfo_request(data); } } static int evdns_getaddrinfo_set_timeout(struct evdns_base *evdns_base, struct evdns_getaddrinfo_request *data) { return event_add(&data->timeout, &evdns_base->global_getaddrinfo_allow_skew); } static inline int evdns_result_is_answer(int result) { return (result != DNS_ERR_NOTIMPL && result != DNS_ERR_REFUSED && result != DNS_ERR_SERVERFAILED && result != DNS_ERR_CANCEL); } static void evdns_getaddrinfo_gotresolve(int result, char type, int count, int ttl, void *addresses, void *arg) { int i; struct getaddrinfo_subrequest *req = arg; struct getaddrinfo_subrequest *other_req; struct evdns_getaddrinfo_request *data; struct evutil_addrinfo *res; struct sockaddr_in sin; struct sockaddr_in6 sin6; struct sockaddr *sa; int socklen, addrlen; void *addrp; int err; int user_canceled; EVUTIL_ASSERT(req->type == DNS_IPv4_A || req->type == DNS_IPv6_AAAA); if (req->type == DNS_IPv4_A) { data = EVUTIL_UPCAST(req, struct evdns_getaddrinfo_request, ipv4_request); other_req = &data->ipv6_request; } else { data = EVUTIL_UPCAST(req, struct evdns_getaddrinfo_request, ipv6_request); other_req = &data->ipv4_request; } /** Called from evdns_base_free() with @fail_requests == 1 */ if (result != DNS_ERR_SHUTDOWN) { EVDNS_LOCK(data->evdns_base); if (evdns_result_is_answer(result)) { if (req->type == DNS_IPv4_A) ++data->evdns_base->getaddrinfo_ipv4_answered; else ++data->evdns_base->getaddrinfo_ipv6_answered; } user_canceled = data->user_canceled; if (other_req->r == NULL) data->request_done = 1; EVDNS_UNLOCK(data->evdns_base); } else { data->evdns_base = NULL; user_canceled = data->user_canceled; } req->r = NULL; if (result == DNS_ERR_CANCEL && ! user_canceled) { /* Internal cancel request from timeout or internal error. * we already answered the user. */ if (other_req->r == NULL) free_getaddrinfo_request(data); return; } if (data->user_cb == NULL) { /* We already answered. XXXX This shouldn't be needed; see * comments in evdns_getaddrinfo_timeout_cb */ free_getaddrinfo_request(data); return; } if (result == DNS_ERR_NONE) { if (count == 0) err = EVUTIL_EAI_NODATA; else err = 0; } else { err = evdns_err_to_getaddrinfo_err(result); } if (err) { /* Looks like we got an error. */ if (other_req->r) { /* The other request is still working; maybe it will * succeed. */ /* XXXX handle failure from set_timeout */ if (result != DNS_ERR_SHUTDOWN) { evdns_getaddrinfo_set_timeout(data->evdns_base, data); } data->pending_error = err; return; } if (user_canceled) { data->user_cb(EVUTIL_EAI_CANCEL, NULL, data->user_data); } else if (data->pending_result) { /* If we have an answer waiting, and we weren't * canceled, ignore this error. */ add_cname_to_reply(data, data->pending_result); data->user_cb(0, data->pending_result, data->user_data); data->pending_result = NULL; } else { if (data->pending_error) err = getaddrinfo_merge_err(err, data->pending_error); data->user_cb(err, NULL, data->user_data); } free_getaddrinfo_request(data); return; } else if (user_canceled) { if (other_req->r) { /* The other request is still working; let it hit this * callback with EVUTIL_EAI_CANCEL callback and report * the failure. */ return; } data->user_cb(EVUTIL_EAI_CANCEL, NULL, data->user_data); free_getaddrinfo_request(data); return; } /* Looks like we got some answers. We should turn them into addrinfos * and then either queue those or return them all. */ EVUTIL_ASSERT(type == DNS_IPv4_A || type == DNS_IPv6_AAAA); if (type == DNS_IPv4_A) { memset(&sin, 0, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_port = htons(data->port); sa = (struct sockaddr *)&sin; socklen = sizeof(sin); addrlen = 4; addrp = &sin.sin_addr.s_addr; } else { memset(&sin6, 0, sizeof(sin6)); sin6.sin6_family = AF_INET6; sin6.sin6_port = htons(data->port); sa = (struct sockaddr *)&sin6; socklen = sizeof(sin6); addrlen = 16; addrp = &sin6.sin6_addr.s6_addr; } res = NULL; for (i=0; i < count; ++i) { struct evutil_addrinfo *ai; memcpy(addrp, ((char*)addresses)+i*addrlen, addrlen); ai = evutil_new_addrinfo_(sa, socklen, &data->hints); if (!ai) { if (other_req->r) { evdns_cancel_request(NULL, other_req->r); } data->user_cb(EVUTIL_EAI_MEMORY, NULL, data->user_data); if (res) evutil_freeaddrinfo(res); if (other_req->r == NULL) free_getaddrinfo_request(data); return; } res = evutil_addrinfo_append_(res, ai); } if (other_req->r) { /* The other request is still in progress; wait for it */ /* XXXX handle failure from set_timeout */ evdns_getaddrinfo_set_timeout(data->evdns_base, data); data->pending_result = res; return; } else { /* The other request is done or never started; append its * results (if any) and return them. */ if (data->pending_result) { if (req->type == DNS_IPv4_A) res = evutil_addrinfo_append_(res, data->pending_result); else res = evutil_addrinfo_append_( data->pending_result, res); data->pending_result = NULL; } /* Call the user callback. */ add_cname_to_reply(data, res); data->user_cb(0, res, data->user_data); /* Free data. */ free_getaddrinfo_request(data); } } static struct hosts_entry * find_hosts_entry(struct evdns_base *base, const char *hostname, struct hosts_entry *find_after) { struct hosts_entry *e; if (find_after) e = TAILQ_NEXT(find_after, next); else e = TAILQ_FIRST(&base->hostsdb); for (; e; e = TAILQ_NEXT(e, next)) { if (!evutil_ascii_strcasecmp(e->hostname, hostname)) return e; } return NULL; } static int evdns_getaddrinfo_fromhosts(struct evdns_base *base, const char *nodename, struct evutil_addrinfo *hints, ev_uint16_t port, struct evutil_addrinfo **res) { int n_found = 0; struct hosts_entry *e; struct evutil_addrinfo *ai=NULL; int f = hints->ai_family; EVDNS_LOCK(base); for (e = find_hosts_entry(base, nodename, NULL); e; e = find_hosts_entry(base, nodename, e)) { struct evutil_addrinfo *ai_new; ++n_found; if ((e->addr.sa.sa_family == AF_INET && f == PF_INET6) || (e->addr.sa.sa_family == AF_INET6 && f == PF_INET)) continue; ai_new = evutil_new_addrinfo_(&e->addr.sa, e->addrlen, hints); if (!ai_new) { n_found = 0; goto out; } sockaddr_setport(ai_new->ai_addr, port); ai = evutil_addrinfo_append_(ai, ai_new); } EVDNS_UNLOCK(base); out: if (n_found) { /* Note that we return an empty answer if we found entries for * this hostname but none were of the right address type. */ *res = ai; return 0; } else { if (ai) evutil_freeaddrinfo(ai); return -1; } } struct evdns_getaddrinfo_request * evdns_getaddrinfo(struct evdns_base *dns_base, const char *nodename, const char *servname, const struct evutil_addrinfo *hints_in, evdns_getaddrinfo_cb cb, void *arg) { struct evdns_getaddrinfo_request *data; struct evutil_addrinfo hints; struct evutil_addrinfo *res = NULL; int err; int port = 0; int want_cname = 0; if (!dns_base) { dns_base = current_base; if (!dns_base) { log(EVDNS_LOG_WARN, "Call to getaddrinfo_async with no " "evdns_base configured."); cb(EVUTIL_EAI_FAIL, NULL, arg); /* ??? better error? */ return NULL; } } /* If we _must_ answer this immediately, do so. */ if ((hints_in && (hints_in->ai_flags & EVUTIL_AI_NUMERICHOST))) { res = NULL; err = evutil_getaddrinfo(nodename, servname, hints_in, &res); cb(err, res, arg); return NULL; } if (hints_in) { memcpy(&hints, hints_in, sizeof(hints)); } else { memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; } evutil_adjust_hints_for_addrconfig_(&hints); /* Now try to see if we _can_ answer immediately. */ /* (It would be nice to do this by calling getaddrinfo directly, with * AI_NUMERICHOST, on plaforms that have it, but we can't: there isn't * a reliable way to distinguish the "that wasn't a numeric host!" case * from any other EAI_NONAME cases.) */ err = evutil_getaddrinfo_common_(nodename, servname, &hints, &res, &port); if (err != EVUTIL_EAI_NEED_RESOLVE) { cb(err, res, arg); return NULL; } /* If there is an entry in the hosts file, we should give it now. */ if (!evdns_getaddrinfo_fromhosts(dns_base, nodename, &hints, port, &res)) { cb(0, res, arg); return NULL; } /* Okay, things are serious now. We're going to need to actually * launch a request. */ data = mm_calloc(1,sizeof(struct evdns_getaddrinfo_request)); if (!data) { cb(EVUTIL_EAI_MEMORY, NULL, arg); return NULL; } memcpy(&data->hints, &hints, sizeof(data->hints)); data->port = (ev_uint16_t)port; data->ipv4_request.type = DNS_IPv4_A; data->ipv6_request.type = DNS_IPv6_AAAA; data->user_cb = cb; data->user_data = arg; data->evdns_base = dns_base; want_cname = (hints.ai_flags & EVUTIL_AI_CANONNAME); /* If we are asked for a PF_UNSPEC address, we launch two requests in * parallel: one for an A address and one for an AAAA address. We * can't send just one request, since many servers only answer one * question per DNS request. * * Once we have the answer to one request, we allow for a short * timeout before we report it, to see if the other one arrives. If * they both show up in time, then we report both the answers. * * If too many addresses of one type time out or fail, we should stop * launching those requests. (XXX we don't do that yet.) */ if (hints.ai_family != PF_INET6) { log(EVDNS_LOG_DEBUG, "Sending request for %s on ipv4 as %p", nodename, &data->ipv4_request); data->ipv4_request.r = evdns_base_resolve_ipv4(dns_base, nodename, 0, evdns_getaddrinfo_gotresolve, &data->ipv4_request); if (want_cname && data->ipv4_request.r) data->ipv4_request.r->current_req->put_cname_in_ptr = &data->cname_result; } if (hints.ai_family != PF_INET) { log(EVDNS_LOG_DEBUG, "Sending request for %s on ipv6 as %p", nodename, &data->ipv6_request); data->ipv6_request.r = evdns_base_resolve_ipv6(dns_base, nodename, 0, evdns_getaddrinfo_gotresolve, &data->ipv6_request); if (want_cname && data->ipv6_request.r) data->ipv6_request.r->current_req->put_cname_in_ptr = &data->cname_result; } evtimer_assign(&data->timeout, dns_base->event_base, evdns_getaddrinfo_timeout_cb, data); if (data->ipv4_request.r || data->ipv6_request.r) { return data; } else { mm_free(data); cb(EVUTIL_EAI_FAIL, NULL, arg); return NULL; } } void evdns_getaddrinfo_cancel(struct evdns_getaddrinfo_request *data) { EVDNS_LOCK(data->evdns_base); if (data->request_done) { EVDNS_UNLOCK(data->evdns_base); return; } event_del(&data->timeout); data->user_canceled = 1; if (data->ipv4_request.r) evdns_cancel_request(data->evdns_base, data->ipv4_request.r); if (data->ipv6_request.r) evdns_cancel_request(data->evdns_base, data->ipv6_request.r); EVDNS_UNLOCK(data->evdns_base); } ������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/event-config.h.cmake�����������������������������������������0000664�0000000�0000000�00000040700�15270523712�0023613�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* event-config.h * * This file was generated by cmake when the makefiles were generated. * * DO NOT EDIT THIS FILE. * * Do not rely on macros in this file existing in later versions. */ #ifndef EVENT2_EVENT_CONFIG_H_INCLUDED_ #define EVENT2_EVENT_CONFIG_H_INCLUDED_ /* Numeric representation of the version */ #define EVENT__NUMERIC_VERSION @EVENT_NUMERIC_VERSION@ #define EVENT__PACKAGE_VERSION "@EVENT_PACKAGE_VERSION@" #define EVENT__VERSION_MAJOR @EVENT_VERSION_MAJOR@ #define EVENT__VERSION_MINOR @EVENT_VERSION_MINOR@ #define EVENT__VERSION_PATCH @EVENT_VERSION_PATCH@ /* Version number of package */ #define EVENT__VERSION "@EVENT_VERSION@" /* Name of package */ #define EVENT__PACKAGE "libevent" /* Define to the address where bug reports for this package should be sent. */ #define EVENT__PACKAGE_BUGREPORT "" /* Define to the full name of this package. */ #define EVENT__PACKAGE_NAME "" /* Define to the full name and version of this package. */ #define EVENT__PACKAGE_STRING "" /* Define to the one symbol short name of this package. */ #define EVENT__PACKAGE_TARNAME "" /* Define if libevent should build without support for a debug mode */ #cmakedefine EVENT__DISABLE_DEBUG_MODE /* Define if libevent should not allow replacing the mm functions */ #cmakedefine EVENT__DISABLE_MM_REPLACEMENT /* Define if libevent should not be compiled with thread support */ #cmakedefine EVENT__DISABLE_THREAD_SUPPORT /* Define to 1 if you have the `accept4' function. */ #cmakedefine EVENT__HAVE_ACCEPT4 /* Define to 1 if you have the `arc4random' function. */ #cmakedefine EVENT__HAVE_ARC4RANDOM /* Define to 1 if you have the `arc4random_buf' function. */ #cmakedefine EVENT__HAVE_ARC4RANDOM_BUF /* Define if clock_gettime is available in libc */ #cmakedefine EVENT__DNS_USE_CPU_CLOCK_FOR_ID /* Define is no secure id variant is available */ #cmakedefine EVENT__DNS_USE_GETTIMEOFDAY_FOR_ID #cmakedefine EVENT__DNS_USE_FTIME_FOR_ID /* Define to 1 if you have the <arpa/inet.h> header file. */ #cmakedefine EVENT__HAVE_ARPA_INET_H /* Define to 1 if you have the `clock_gettime' function. */ #cmakedefine EVENT__HAVE_CLOCK_GETTIME /* Define to 1 if you have the declaration of `CTL_KERN'. */ #cmakedefine EVENT__HAVE_DECL_CTL_KERN /* Define to 1 if you have the declaration of `KERN_ARND'. */ #cmakedefine EVENT__HAVE_DECL_KERN_ARND /* Define to 1 if you have the declaration of `KERN_RANDOM'. */ #cmakedefine EVENT__HAVE_DECL_KERN_RANDOM /* Define if /dev/poll is available */ #cmakedefine EVENT__HAVE_DEVPOLL /* Define to 1 if you have the <netdb.h> header file. */ #cmakedefine EVENT__HAVE_NETDB_H /* Define to 1 if fd_mask type is defined */ #cmakedefine EVENT__HAVE_FD_MASK /* Define to 1 if the <sys/queue.h> header file defines TAILQ_FOREACH. */ #cmakedefine EVENT__HAVE_TAILQFOREACH /* Define to 1 if you have the <dlfcn.h> header file. */ #cmakedefine EVENT__HAVE_DLFCN_H /* Define if your system supports the epoll system calls */ #cmakedefine EVENT__HAVE_EPOLL /* Define to 1 if you have the `epoll_create1' function. */ #cmakedefine EVENT__HAVE_EPOLL_CREATE1 /* Define to 1 if you have the `epoll_ctl' function. */ #cmakedefine EVENT__HAVE_EPOLL_CTL /* Define to 1 if you have the `eventfd' function. */ #cmakedefine EVENT__HAVE_EVENTFD /* Define if your system supports event ports */ #cmakedefine EVENT__HAVE_EVENT_PORTS /* Define to 1 if you have the `fcntl' function. */ #cmakedefine EVENT__HAVE_FCNTL /* Define to 1 if you have the <fcntl.h> header file. */ #cmakedefine EVENT__HAVE_FCNTL_H /* Define to 1 if you have the `getaddrinfo' function. */ #cmakedefine EVENT__HAVE_GETADDRINFO /* Define to 1 if you have the `getegid' function. */ #cmakedefine EVENT__HAVE_GETEGID /* Define to 1 if you have the `geteuid' function. */ #cmakedefine EVENT__HAVE_GETEUID /* TODO: Check for different gethostname argument counts. CheckPrototypeDefinition.cmake can be used. */ /* Define this if you have any gethostbyname_r() */ #cmakedefine EVENT__HAVE_GETHOSTBYNAME_R /* Define this if gethostbyname_r takes 3 arguments */ #cmakedefine EVENT__HAVE_GETHOSTBYNAME_R_3_ARG /* Define this if gethostbyname_r takes 5 arguments */ #cmakedefine EVENT__HAVE_GETHOSTBYNAME_R_5_ARG /* Define this if gethostbyname_r takes 6 arguments */ #cmakedefine EVENT__HAVE_GETHOSTBYNAME_R_6_ARG /* Define to 1 if you have the `getifaddrs' function. */ #cmakedefine EVENT__HAVE_GETIFADDRS /* Define to 1 if you have the `getnameinfo' function. */ #cmakedefine EVENT__HAVE_GETNAMEINFO /* Define to 1 if you have the `getprotobynumber' function. */ #cmakedefine EVENT__HAVE_GETPROTOBYNUMBER /* Define to 1 if you have the `getservbyname' function. */ #cmakedefine EVENT__HAVE_GETSERVBYNAME /* Define to 1 if you have the `gettimeofday' function. */ #cmakedefine EVENT__HAVE_GETTIMEOFDAY /* Define to 1 if you have the <ifaddrs.h> header file. */ #cmakedefine EVENT__HAVE_IFADDRS_H /* Define to 1 if you have the `inet_ntop' function. */ #cmakedefine EVENT__HAVE_INET_NTOP /* Define to 1 if you have the `inet_pton' function. */ #cmakedefine EVENT__HAVE_INET_PTON /* Define to 1 if you have the <inttypes.h> header file. */ #cmakedefine EVENT__HAVE_INTTYPES_H /* Define to 1 if you have the `issetugid' function. */ #cmakedefine EVENT__HAVE_ISSETUGID /* Define to 1 if you have the `kqueue' function. */ #cmakedefine EVENT__HAVE_KQUEUE /* Define if the system has zlib */ #cmakedefine EVENT__HAVE_LIBZ /* Define to 1 if you have the `mach_absolute_time' function. */ #cmakedefine EVENT__HAVE_MACH_ABSOLUTE_TIME /* Define to 1 if you have the <mach/mach_time.h> header file. */ #cmakedefine EVENT__HAVE_MACH_MACH_TIME_H /* Define to 1 if you have the <memory.h> header file. */ #cmakedefine EVENT__HAVE_MEMORY_H /* Define to 1 if you have the `mmap' function. */ #cmakedefine EVENT__HAVE_MMAP /* Define to 1 if you have the `nanosleep' function. */ #cmakedefine EVENT__HAVE_NANOSLEEP /* Define to 1 if you have the `usleep' function. */ #cmakedefine EVENT__HAVE_USLEEP /* Define to 1 if you have the <netinet/in6.h> header file. */ #cmakedefine EVENT__HAVE_NETINET_IN6_H /* Define to 1 if you have the <netinet/in.h> header file. */ #cmakedefine EVENT__HAVE_NETINET_IN_H /* Define to 1 if you have the <netinet/tcp.h> header file. */ #cmakedefine EVENT__HAVE_NETINET_TCP_H /* Define if the system has openssl */ #cmakedefine EVENT__HAVE_OPENSSL /* Define to 1 if you have the `pipe' function. */ #cmakedefine EVENT__HAVE_PIPE /* Define to 1 if you have the `pipe2' function. */ #cmakedefine EVENT__HAVE_PIPE2 /* Define to 1 if you have the `poll' function. */ #cmakedefine EVENT__HAVE_POLL /* Define to 1 if you have the <poll.h> header file. */ #cmakedefine EVENT__HAVE_POLL_H /* Define to 1 if you have the `port_create' function. */ #cmakedefine EVENT__HAVE_PORT_CREATE /* Define to 1 if you have the <port.h> header file. */ #cmakedefine EVENT__HAVE_PORT_H /* Define if we have pthreads on this system */ #cmakedefine EVENT__HAVE_PTHREADS /* Define to 1 if you have the `putenv' function. */ #cmakedefine EVENT__HAVE_PUTENV /* Define to 1 if the system has the type `sa_family_t'. */ #cmakedefine EVENT__HAVE_SA_FAMILY_T /* Define to 1 if you have the `select' function. */ #cmakedefine EVENT__HAVE_SELECT /* Define to 1 if you have the `setenv' function. */ #cmakedefine EVENT__HAVE_SETENV /* Define if F_SETFD is defined in <fcntl.h> */ #cmakedefine EVENT__HAVE_SETFD /* Define to 1 if you have the `setrlimit' function. */ #cmakedefine EVENT__HAVE_SETRLIMIT /* Define to 1 if you have the `sendfile' function. */ #cmakedefine EVENT__HAVE_SENDFILE /* Define if F_SETFD is defined in <fcntl.h> */ #cmakedefine EVENT__HAVE_SETFD /* Define to 1 if you have the `sigaction' function. */ #cmakedefine EVENT__HAVE_SIGACTION /* Define to 1 if you have the `signal' function. */ #cmakedefine EVENT__HAVE_SIGNAL /* Define to 1 if you have the `splice' function. */ #cmakedefine EVENT__HAVE_SPLICE /* Define to 1 if you have the <stdarg.h> header file. */ #cmakedefine EVENT__HAVE_STDARG_H /* Define to 1 if you have the <stddef.h> header file. */ #cmakedefine EVENT__HAVE_STDDEF_H /* Define to 1 if you have the <stdint.h> header file. */ #cmakedefine EVENT__HAVE_STDINT_H /* Define to 1 if you have the <stdlib.h> header file. */ #cmakedefine EVENT__HAVE_STDLIB_H /* Define to 1 if you have the <strings.h> header file. */ #cmakedefine EVENT__HAVE_STRINGS_H /* Define to 1 if you have the <string.h> header file. */ #cmakedefine EVENT__HAVE_STRING_H /* Define to 1 if you have the `strlcpy' function. */ #cmakedefine EVENT__HAVE_STRLCPY /* Define to 1 if you have the `strsep' function. */ #cmakedefine EVENT__HAVE_STRSEP /* Define to 1 if you have the `strtok_r' function. */ #cmakedefine EVENT__HAVE_STRTOK_R /* Define to 1 if you have the `strtoll' function. */ #cmakedefine EVENT__HAVE_STRTOLL /* Define to 1 if the system has the type `struct addrinfo'. */ #cmakedefine EVENT__HAVE_STRUCT_ADDRINFO /* Define to 1 if the system has the type `struct in6_addr'. */ #cmakedefine EVENT__HAVE_STRUCT_IN6_ADDR /* Define to 1 if `s6_addr16' is member of `struct in6_addr'. */ #cmakedefine EVENT__HAVE_STRUCT_IN6_ADDR_S6_ADDR16 /* Define to 1 if `s6_addr32' is member of `struct in6_addr'. */ #cmakedefine EVENT__HAVE_STRUCT_IN6_ADDR_S6_ADDR32 /* Define to 1 if the system has the type `struct sockaddr_in6'. */ #cmakedefine EVENT__HAVE_STRUCT_SOCKADDR_IN6 /* Define to 1 if `sin6_len' is member of `struct sockaddr_in6'. */ #cmakedefine EVENT__HAVE_STRUCT_SOCKADDR_IN6_SIN6_LEN /* Define to 1 if `sin_len' is member of `struct sockaddr_in'. */ #cmakedefine EVENT__HAVE_STRUCT_SOCKADDR_IN_SIN_LEN /* Define to 1 if the system has the type `struct sockaddr_storage'. */ #cmakedefine EVENT__HAVE_STRUCT_SOCKADDR_STORAGE /* Define to 1 if `ss_family' is a member of `struct sockaddr_storage'. */ #cmakedefine EVENT__HAVE_STRUCT_SOCKADDR_STORAGE_SS_FAMILY /* Define to 1 if `__ss_family' is a member of `struct sockaddr_storage'. */ #cmakedefine EVENT__HAVE_STRUCT_SOCKADDR_STORAGE___SS_FAMILY /* Define to 1 if you have the `sysctl' function. */ #cmakedefine EVENT__HAVE_SYSCTL /* Define to 1 if you have the <sys/devpoll.h> header file. */ #cmakedefine EVENT__HAVE_SYS_DEVPOLL_H /* Define to 1 if you have the <sys/epoll.h> header file. */ #cmakedefine EVENT__HAVE_SYS_EPOLL_H /* Define to 1 if you have the <sys/eventfd.h> header file. */ #cmakedefine EVENT__HAVE_SYS_EVENTFD_H /* Define to 1 if you have the <sys/event.h> header file. */ #cmakedefine EVENT__HAVE_SYS_EVENT_H /* Define to 1 if you have the <sys/ioctl.h> header file. */ #cmakedefine EVENT__HAVE_SYS_IOCTL_H /* Define to 1 if you have the <sys/mman.h> header file. */ #cmakedefine EVENT__HAVE_SYS_MMAN_H /* Define to 1 if you have the <sys/param.h> header file. */ #cmakedefine EVENT__HAVE_SYS_PARAM_H /* Define to 1 if you have the <sys/queue.h> header file. */ #cmakedefine EVENT__HAVE_SYS_QUEUE_H /* Define to 1 if you have the <sys/resource.h> header file. */ #cmakedefine EVENT__HAVE_SYS_RESOURCE_H /* Define to 1 if you have the <sys/select.h> header file. */ #cmakedefine EVENT__HAVE_SYS_SELECT_H /* Define to 1 if you have the <sys/sendfile.h> header file. */ #cmakedefine EVENT__HAVE_SYS_SENDFILE_H /* Define to 1 if you have the <sys/socket.h> header file. */ #cmakedefine EVENT__HAVE_SYS_SOCKET_H /* Define to 1 if you have the <sys/stat.h> header file. */ #cmakedefine EVENT__HAVE_SYS_STAT_H /* Define to 1 if you have the <sys/sysctl.h> header file. */ #cmakedefine EVENT__HAVE_SYS_SYSCTL_H /* Define to 1 if you have the <sys/timerfd.h> header file. */ #cmakedefine EVENT__HAVE_SYS_TIMERFD_H */ /* Define to 1 if you have the <sys/time.h> header file. */ #cmakedefine EVENT__HAVE_SYS_TIME_H /* Define to 1 if you have the <sys/types.h> header file. */ #cmakedefine EVENT__HAVE_SYS_TYPES_H /* Define to 1 if you have the <sys/uio.h> header file. */ #cmakedefine EVENT__HAVE_SYS_UIO_H /* Define to 1 if you have the <sys/wait.h> header file. */ #cmakedefine EVENT__HAVE_SYS_WAIT_H /* Define to 1 if you have the <errno.h> header file. */ #cmakedefine EVENT__HAVE_ERRNO_H /* Define if TAILQ_FOREACH is defined in <sys/queue.h> */ #cmakedefine EVENT__HAVE_TAILQFOREACH /* Define if timeradd is defined in <sys/time.h> */ #cmakedefine EVENT__HAVE_TIMERADD /* Define if timerclear is defined in <sys/time.h> */ #cmakedefine EVENT__HAVE_TIMERCLEAR /* Define if timercmp is defined in <sys/time.h> */ #cmakedefine EVENT__HAVE_TIMERCMP /* Define to 1 if you have the `timerfd_create' function. */ #cmakedefine EVENT__HAVE_TIMERFD_CREATE /* Define if timerisset is defined in <sys/time.h> */ #cmakedefine EVENT__HAVE_TIMERISSET /* Define to 1 if the system has the type `uint8_t'. */ #cmakedefine EVENT__HAVE_UINT8_T /* Define to 1 if the system has the type `uint16_t'. */ #cmakedefine EVENT__HAVE_UINT16_T /* Define to 1 if the system has the type `uint32_t'. */ #cmakedefine EVENT__HAVE_UINT32_T /* Define to 1 if the system has the type `uint64_t'. */ #cmakedefine EVENT__HAVE_UINT64_T /* Define to 1 if the system has the type `uintptr_t'. */ #cmakedefine EVENT__HAVE_UINTPTR_T /* Define to 1 if you have the `umask' function. */ #cmakedefine EVENT__HAVE_UMASK /* Define to 1 if you have the <unistd.h> header file. */ #cmakedefine EVENT__HAVE_UNISTD_H /* Define to 1 if you have the `unsetenv' function. */ #cmakedefine EVENT__HAVE_UNSETENV /* Define to 1 if you have the `vasprintf' function. */ #cmakedefine EVENT__HAVE_VASPRINTF /* Define if kqueue works correctly with pipes */ #cmakedefine EVENT__HAVE_WORKING_KQUEUE #ifdef __USE_UNUSED_DEFINITIONS__ /* Define to necessary symbol if this constant uses a non-standard name on your system. */ /* XXX: Hello, this isn't even used, nor is it defined anywhere... - Ellzey */ #define EVENT__PTHREAD_CREATE_JOINABLE ${EVENT__PTHREAD_CREATE_JOINABLE} #endif /* The size of `pthread_t', as computed by sizeof. */ #define EVENT__SIZEOF_PTHREAD_T @EVENT__SIZEOF_PTHREAD_T@ /* The size of a `int', as computed by sizeof. */ #define EVENT__SIZEOF_INT @EVENT__SIZEOF_INT@ /* The size of a `long', as computed by sizeof. */ #define EVENT__SIZEOF_LONG @EVENT__SIZEOF_LONG@ /* The size of a `long long', as computed by sizeof. */ #define EVENT__SIZEOF_LONG_LONG @EVENT__SIZEOF_LONG_LONG@ /* The size of `off_t', as computed by sizeof. */ #define EVENT__SIZEOF_OFF_T @EVENT__SIZEOF_OFF_T@ #define EVENT__SIZEOF_SSIZE_T @EVENT__SIZEOF_SSIZE_T@ /* The size of a `short', as computed by sizeof. */ #define EVENT__SIZEOF_SHORT @EVENT__SIZEOF_SHORT@ /* The size of `size_t', as computed by sizeof. */ #define EVENT__SIZEOF_SIZE_T @EVENT__SIZEOF_SIZE_T@ /* Define to 1 if you have the ANSI C header files. */ #cmakedefine EVENT__STDC_HEADERS /* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */ #cmakedefine EVENT__TIME_WITH_SYS_TIME /* The size of `socklen_t', as computed by sizeof. */ #define EVENT__SIZEOF_SOCKLEN_T @EVENT__SIZEOF_SOCKLEN_T@ /* The size of 'void *', as computer by sizeof */ #define EVENT__SIZEOF_VOID_P @EVENT__SIZEOF_VOID_P@ /* set an alias for whatever __func__ __FUNCTION__ is, what sillyness */ #if defined (__func__) #define EVENT____func__ __func__ #elif defined(__FUNCTION__) #define EVENT____func__ __FUNCTION__ #else #define EVENT____func__ __FILE__ #endif #ifdef __THESE_ARE_NOT_CONFIG_H_THINGS_THEY_ARE_DASH_D_THINGS__ /* Number of bits in a file offset, on hosts where this is settable. */ /* Ellzey is not satisfied */ #define EVENT___FILE_OFFSET_BITS @EVENT___FILE_OFFSET_BITS@ /* Define for large files, on AIX-style hosts. */ #define @_LARGE_FILES@ #endif #ifdef _WhAT_DOES_THIS_EVEN_DO_ /* Define to empty if `const' does not conform to ANSI C. */ /* lolwut? - ellzey */ #undef EVENT__const #endif /* Define to `__inline__' or `__inline' if that's what the C compiler calls it, or to nothing if 'inline' is not supported under any name. */ #ifndef __cplusplus /* why not c++? * * and are we really expected to use EVENT__inline everywhere, * shouldn't we just do: * ifdef EVENT__inline * define inline EVENT__inline * * - Ellzey */ #define EVENT__inline @EVENT__inline@ #endif /* Define to `int' if <sys/tyes.h> does not define. */ #define EVENT__pid_t @EVENT__pid_t@ /* Define to `unsigned' if <sys/types.h> does not define. */ #define EVENT__size_t @EVENT__size_t@ /* Define to unsigned int if you dont have it */ #define EVENT__socklen_t @EVENT__socklen_t@ /* Define to `int' if <sys/types.h> does not define. */ #define EVENT__ssize_t @EVENT__ssize_t@ #cmakedefine EVENT__NEED_DLLIMPORT /* Define to 1 if you have ERR_remove_thread_stat(). */ #cmakedefine EVENT__HAVE_ERR_REMOVE_THREAD_STATE /* Define if waitpid() supports WNOWAIT */ #cmakedefine EVENT__HAVE_WAITPID_WITH_WNOWAIT #endif ����������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/event-internal.h���������������������������������������������0000664�0000000�0000000�00000041544�15270523712�0023112�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2000-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT_INTERNAL_H_INCLUDED_ #define EVENT_INTERNAL_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif #include "event2/event-config.h" #include "evconfig-private.h" #include <time.h> #include <sys/queue.h> #include "event2/event_struct.h" #include "minheap-internal.h" #include "evsignal-internal.h" #include "mm-internal.h" #include "defer-internal.h" /* map union members back */ /* mutually exclusive */ #define ev_signal_next ev_.ev_signal.ev_signal_next #define ev_io_next ev_.ev_io.ev_io_next #define ev_io_timeout ev_.ev_io.ev_timeout /* used only by signals */ #define ev_ncalls ev_.ev_signal.ev_ncalls #define ev_pncalls ev_.ev_signal.ev_pncalls #define ev_pri ev_evcallback.evcb_pri #define ev_flags ev_evcallback.evcb_flags #define ev_closure ev_evcallback.evcb_closure #define ev_callback ev_evcallback.evcb_cb_union.evcb_callback #define ev_arg ev_evcallback.evcb_arg /** @name Event closure codes Possible values for evcb_closure in struct event_callback @{ */ /** A regular event. Uses the evcb_callback callback */ #define EV_CLOSURE_EVENT 0 /** A signal event. Uses the evcb_callback callback */ #define EV_CLOSURE_EVENT_SIGNAL 1 /** A persistent non-signal event. Uses the evcb_callback callback */ #define EV_CLOSURE_EVENT_PERSIST 2 /** A simple callback. Uses the evcb_selfcb callback. */ #define EV_CLOSURE_CB_SELF 3 /** A finalizing callback. Uses the evcb_cbfinalize callback. */ #define EV_CLOSURE_CB_FINALIZE 4 /** A finalizing event. Uses the evcb_evfinalize callback. */ #define EV_CLOSURE_EVENT_FINALIZE 5 /** A finalizing event that should get freed after. Uses the evcb_evfinalize * callback. */ #define EV_CLOSURE_EVENT_FINALIZE_FREE 6 /** @} */ /** Structure to define the backend of a given event_base. */ struct eventop { /** The name of this backend. */ const char *name; /** Function to set up an event_base to use this backend. It should * create a new structure holding whatever information is needed to * run the backend, and return it. The returned pointer will get * stored by event_init into the event_base.evbase field. On failure, * this function should return NULL. */ void *(*init)(struct event_base *); /** Enable reading/writing on a given fd or signal. 'events' will be * the events that we're trying to enable: one or more of EV_READ, * EV_WRITE, EV_SIGNAL, and EV_ET. 'old' will be those events that * were enabled on this fd previously. 'fdinfo' will be a structure * associated with the fd by the evmap; its size is defined by the * fdinfo field below. It will be set to 0 the first time the fd is * added. The function should return 0 on success and -1 on error. */ int (*add)(struct event_base *, evutil_socket_t fd, short old, short events, void *fdinfo); /** As "add", except 'events' contains the events we mean to disable. */ int (*del)(struct event_base *, evutil_socket_t fd, short old, short events, void *fdinfo); /** Function to implement the core of an event loop. It must see which added events are ready, and cause event_active to be called for each active event (usually via event_io_active or such). It should return 0 on success and -1 on error. */ int (*dispatch)(struct event_base *, struct timeval *); /** Function to clean up and free our data from the event_base. */ void (*dealloc)(struct event_base *); /** Flag: set if we need to reinitialize the event base after we fork. */ int need_reinit; /** Bit-array of supported event_method_features that this backend can * provide. */ enum event_method_feature features; /** Length of the extra information we should record for each fd that has one or more active events. This information is recorded as part of the evmap entry for each fd, and passed as an argument to the add and del functions above. */ size_t fdinfo_len; }; #ifdef _WIN32 /* If we're on win32, then file descriptors are not nice low densely packed integers. Instead, they are pointer-like windows handles, and we want to use a hashtable instead of an array to map fds to events. */ #define EVMAP_USE_HT #endif /* #define HT_CACHE_HASH_VALS */ #ifdef EVMAP_USE_HT #define HT_NO_CACHE_HASH_VALUES #include "ht-internal.h" struct event_map_entry; HT_HEAD(event_io_map, event_map_entry); #else #define event_io_map event_signal_map #endif /* Used to map signal numbers to a list of events. If EVMAP_USE_HT is not defined, this structure is also used as event_io_map, which maps fds to a list of events. */ struct event_signal_map { /* An array of evmap_io * or of evmap_signal *; empty entries are * set to NULL. */ void **entries; /* The number of entries available in entries */ int nentries; }; /* A list of events waiting on a given 'common' timeout value. Ordinarily, * events waiting for a timeout wait on a minheap. Sometimes, however, a * queue can be faster. **/ struct common_timeout_list { /* List of events currently waiting in the queue. */ struct event_list events; /* 'magic' timeval used to indicate the duration of events in this * queue. */ struct timeval duration; /* Event that triggers whenever one of the events in the queue is * ready to activate */ struct event timeout_event; /* The event_base that this timeout list is part of */ struct event_base *base; }; /** Mask used to get the real tv_usec value from a common timeout. */ #define COMMON_TIMEOUT_MICROSECONDS_MASK 0x000fffff struct event_change; /* List of 'changes' since the last call to eventop.dispatch. Only maintained * if the backend is using changesets. */ struct event_changelist { struct event_change *changes; int n_changes; int changes_size; }; #ifndef EVENT__DISABLE_DEBUG_MODE /* Global internal flag: set to one if debug mode is on. */ extern int event_debug_mode_on_; #define EVENT_DEBUG_MODE_IS_ON() (event_debug_mode_on_) #else #define EVENT_DEBUG_MODE_IS_ON() (0) #endif TAILQ_HEAD(evcallback_list, event_callback); /* Sets up an event for processing once */ struct event_once { LIST_ENTRY(event_once) next_once; struct event ev; void (*cb)(evutil_socket_t, short, void *); void *arg; }; struct event_base { /** Function pointers and other data to describe this event_base's * backend. */ const struct eventop *evsel; /** Pointer to backend-specific data. */ void *evbase; /** List of changes to tell backend about at next dispatch. Only used * by the O(1) backends. */ struct event_changelist changelist; /** Function pointers used to describe the backend that this event_base * uses for signals */ const struct eventop *evsigsel; /** Data to implement the common signal handelr code. */ struct evsig_info sig; /** Number of virtual events */ int virtual_event_count; /** Maximum number of virtual events active */ int virtual_event_count_max; /** Number of total events added to this event_base */ int event_count; /** Maximum number of total events added to this event_base */ int event_count_max; /** Number of total events active in this event_base */ int event_count_active; /** Maximum number of total events active in this event_base */ int event_count_active_max; /** Set if we should terminate the loop once we're done processing * events. */ int event_gotterm; /** Set if we should terminate the loop immediately */ int event_break; /** Set if we should start a new instance of the loop immediately. */ int event_continue; /** The currently running priority of events */ int event_running_priority; /** Set if we're running the event_base_loop function, to prevent * reentrant invocation. */ int running_loop; /** Set to the number of deferred_cbs we've made 'active' in the * loop. This is a hack to prevent starvation; it would be smarter * to just use event_config_set_max_dispatch_interval's max_callbacks * feature */ int n_deferreds_queued; /* Active event management. */ /** An array of nactivequeues queues for active event_callbacks (ones * that have triggered, and whose callbacks need to be called). Low * priority numbers are more important, and stall higher ones. */ struct evcallback_list *activequeues; /** The length of the activequeues array */ int nactivequeues; /** A list of event_callbacks that should become active the next time * we process events, but not this time. */ struct evcallback_list active_later_queue; /* common timeout logic */ /** An array of common_timeout_list* for all of the common timeout * values we know. */ struct common_timeout_list **common_timeout_queues; /** The number of entries used in common_timeout_queues */ int n_common_timeouts; /** The total size of common_timeout_queues. */ int n_common_timeouts_allocated; /** Mapping from file descriptors to enabled (added) events */ struct event_io_map io; /** Mapping from signal numbers to enabled (added) events. */ struct event_signal_map sigmap; /** Priority queue of events with timeouts. */ struct min_heap timeheap; /** Stored timeval: used to avoid calling gettimeofday/clock_gettime * too often. */ struct timeval tv_cache; struct evutil_monotonic_timer monotonic_timer; /** Difference between internal time (maybe from clock_gettime) and * gettimeofday. */ struct timeval tv_clock_diff; /** Second in which we last updated tv_clock_diff, in monotonic time. */ time_t last_updated_clock_diff; #ifndef EVENT__DISABLE_THREAD_SUPPORT /* threading support */ /** The thread currently running the event_loop for this base */ unsigned long th_owner_id; /** A lock to prevent conflicting accesses to this event_base */ void *th_base_lock; /** A condition that gets signalled when we're done processing an * event with waiters on it. */ void *current_event_cond; /** Number of threads blocking on current_event_cond. */ int current_event_waiters; #endif /** The event whose callback is executing right now */ struct event_callback *current_event; #ifdef _WIN32 /** IOCP support structure, if IOCP is enabled. */ struct event_iocp_port *iocp; #endif /** Flags that this base was configured with */ enum event_base_config_flag flags; struct timeval max_dispatch_time; int max_dispatch_callbacks; int limit_callbacks_after_prio; /* Notify main thread to wake up break, etc. */ /** True if the base already has a pending notify, and we don't need * to add any more. */ int is_notify_pending; /** A socketpair used by some th_notify functions to wake up the main * thread. */ evutil_socket_t th_notify_fd[2]; /** An event used by some th_notify functions to wake up the main * thread. */ struct event th_notify; /** A function used to wake up the main thread from another thread. */ int (*th_notify_fn)(struct event_base *base); /** Saved seed for weak random number generator. Some backends use * this to produce fairness among sockets. Protected by th_base_lock. */ struct evutil_weakrand_state weakrand_seed; /** List of event_onces that have not yet fired. */ LIST_HEAD(once_event_list, event_once) once_events; }; struct event_config_entry { TAILQ_ENTRY(event_config_entry) next; const char *avoid_method; }; /** Internal structure: describes the configuration we want for an event_base * that we're about to allocate. */ struct event_config { TAILQ_HEAD(event_configq, event_config_entry) entries; int n_cpus_hint; struct timeval max_dispatch_interval; int max_dispatch_callbacks; int limit_callbacks_after_prio; enum event_method_feature require_features; enum event_base_config_flag flags; }; /* Internal use only: Functions that might be missing from <sys/queue.h> */ #ifndef TAILQ_FIRST #define TAILQ_FIRST(head) ((head)->tqh_first) #endif #ifndef TAILQ_END #define TAILQ_END(head) NULL #endif #ifndef TAILQ_NEXT #define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) #endif #ifndef TAILQ_FOREACH #define TAILQ_FOREACH(var, head, field) \ for ((var) = TAILQ_FIRST(head); \ (var) != TAILQ_END(head); \ (var) = TAILQ_NEXT(var, field)) #endif #ifndef TAILQ_INSERT_BEFORE #define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ (elm)->field.tqe_next = (listelm); \ *(listelm)->field.tqe_prev = (elm); \ (listelm)->field.tqe_prev = &(elm)->field.tqe_next; \ } while (0) #endif #define N_ACTIVE_CALLBACKS(base) \ ((base)->event_count_active) int evsig_set_handler_(struct event_base *base, int evsignal, void (*fn)(int)); int evsig_restore_handler_(struct event_base *base, int evsignal); int event_add_nolock_(struct event *ev, const struct timeval *tv, int tv_is_absolute); /** Argument for event_del_nolock_. Tells event_del not to block on the event * if it's running in another thread. */ #define EVENT_DEL_NOBLOCK 0 /** Argument for event_del_nolock_. Tells event_del to block on the event * if it's running in another thread, regardless of its value for EV_FINALIZE */ #define EVENT_DEL_BLOCK 1 /** Argument for event_del_nolock_. Tells event_del to block on the event * if it is running in another thread and it doesn't have EV_FINALIZE set. */ #define EVENT_DEL_AUTOBLOCK 2 /** Argument for event_del_nolock_. Tells event_del to procede even if the * event is set up for finalization rather for regular use.*/ #define EVENT_DEL_EVEN_IF_FINALIZING 3 int event_del_nolock_(struct event *ev, int blocking); int event_remove_timer_nolock_(struct event *ev); void event_active_nolock_(struct event *ev, int res, short count); int event_callback_activate_(struct event_base *, struct event_callback *); int event_callback_activate_nolock_(struct event_base *, struct event_callback *); int event_callback_cancel_(struct event_base *base, struct event_callback *evcb); void event_callback_finalize_nolock_(struct event_base *base, unsigned flags, struct event_callback *evcb, void (*cb)(struct event_callback *, void *)); void event_callback_finalize_(struct event_base *base, unsigned flags, struct event_callback *evcb, void (*cb)(struct event_callback *, void *)); int event_callback_finalize_many_(struct event_base *base, int n_cbs, struct event_callback **evcb, void (*cb)(struct event_callback *, void *)); void event_active_later_(struct event *ev, int res); void event_active_later_nolock_(struct event *ev, int res); int event_callback_activate_later_nolock_(struct event_base *base, struct event_callback *evcb); int event_callback_cancel_nolock_(struct event_base *base, struct event_callback *evcb, int even_if_finalizing); void event_callback_init_(struct event_base *base, struct event_callback *cb); /* FIXME document. */ void event_base_add_virtual_(struct event_base *base); void event_base_del_virtual_(struct event_base *base); /** For debugging: unless assertions are disabled, verify the referential integrity of the internal data structures of 'base'. This operation can be expensive. Returns on success; aborts on failure. */ void event_base_assert_ok_(struct event_base *base); void event_base_assert_ok_nolock_(struct event_base *base); /* Helper function: Call 'fn' exactly once every inserted or active event in * the event_base 'base'. * * If fn returns 0, continue on to the next event. Otherwise, return the same * value that fn returned. * * Requires that 'base' be locked. */ int event_base_foreach_event_nolock_(struct event_base *base, event_base_foreach_event_cb cb, void *arg); /* Cleanup function to reset debug mode during shutdown. * * Calling this function doesn't mean it'll be possible to re-enable * debug mode if any events were added. */ void event_disable_debug_mode(void); #ifdef __cplusplus } #endif #endif /* EVENT_INTERNAL_H_INCLUDED_ */ ������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/event.3������������������������������������������������������0000664�0000000�0000000�00000042476�15270523712�0021220�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������.\" $OpenBSD: event.3,v 1.4 2002/07/12 18:50:48 provos Exp $ .\" .\" Copyright (c) 2000 Artur Grabowski <art@openbsd.org> .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" 3. The name of the author may not be used to endorse or promote products .\" derived from this software without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, .\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY .\" AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL .\" THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, .\" EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, .\" PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; .\" OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, .\" WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF .\" ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .Dd August 8, 2000 .Dt EVENT 3 .Os .Sh NAME .Nm event_init , .Nm event_dispatch , .Nm event_loop , .Nm event_loopexit , .Nm event_loopbreak , .Nm event_set , .Nm event_base_dispatch , .Nm event_base_loop , .Nm event_base_loopexit , .Nm event_base_loopbreak , .Nm event_base_set , .Nm event_base_free , .Nm event_add , .Nm event_del , .Nm event_once , .Nm event_base_once , .Nm event_pending , .Nm event_initialized , .Nm event_priority_init , .Nm event_priority_set , .Nm evtimer_set , .Nm evtimer_add , .Nm evtimer_del , .Nm evtimer_pending , .Nm evtimer_initialized , .Nm signal_set , .Nm signal_add , .Nm signal_del , .Nm signal_pending , .Nm signal_initialized , .Nm bufferevent_new , .Nm bufferevent_free , .Nm bufferevent_write , .Nm bufferevent_write_buffer , .Nm bufferevent_read , .Nm bufferevent_enable , .Nm bufferevent_disable , .Nm bufferevent_settimeout , .Nm bufferevent_base_set , .Nm evbuffer_new , .Nm evbuffer_free , .Nm evbuffer_add , .Nm evbuffer_add_buffer , .Nm evbuffer_add_printf , .Nm evbuffer_add_vprintf , .Nm evbuffer_drain , .Nm evbuffer_write , .Nm evbuffer_read , .Nm evbuffer_find , .Nm evbuffer_readline , .Nm evhttp_new , .Nm evhttp_bind_socket , .Nm evhttp_free .Nd execute a function when a specific event occurs .Sh SYNOPSIS .Fd #include <sys/time.h> .Fd #include <event.h> .Ft "struct event_base *" .Fn "event_init" "void" .Ft int .Fn "event_dispatch" "void" .Ft int .Fn "event_loop" "int flags" .Ft int .Fn "event_loopexit" "struct timeval *tv" .Ft int .Fn "event_loopbreak" "void" .Ft void .Fn "event_set" "struct event *ev" "int fd" "short event" "void (*fn)(int, short, void *)" "void *arg" .Ft int .Fn "event_base_dispatch" "struct event_base *base" .Ft int .Fn "event_base_loop" "struct event_base *base" "int flags" .Ft int .Fn "event_base_loopexit" "struct event_base *base" "struct timeval *tv" .Ft int .Fn "event_base_loopbreak" "struct event_base *base" .Ft int .Fn "event_base_set" "struct event_base *base" "struct event *" .Ft void .Fn "event_base_free" "struct event_base *base" .Ft int .Fn "event_add" "struct event *ev" "struct timeval *tv" .Ft int .Fn "event_del" "struct event *ev" .Ft int .Fn "event_once" "int fd" "short event" "void (*fn)(int, short, void *)" "void *arg" "struct timeval *tv" .Ft int .Fn "event_base_once" "struct event_base *base" "int fd" "short event" "void (*fn)(int, short, void *)" "void *arg" "struct timeval *tv" .Ft int .Fn "event_pending" "struct event *ev" "short event" "struct timeval *tv" .Ft int .Fn "event_initialized" "struct event *ev" .Ft int .Fn "event_priority_init" "int npriorities" .Ft int .Fn "event_priority_set" "struct event *ev" "int priority" .Ft void .Fn "evtimer_set" "struct event *ev" "void (*fn)(int, short, void *)" "void *arg" .Ft void .Fn "evtimer_add" "struct event *ev" "struct timeval *" .Ft void .Fn "evtimer_del" "struct event *ev" .Ft int .Fn "evtimer_pending" "struct event *ev" "struct timeval *tv" .Ft int .Fn "evtimer_initialized" "struct event *ev" .Ft void .Fn "signal_set" "struct event *ev" "int signal" "void (*fn)(int, short, void *)" "void *arg" .Ft void .Fn "signal_add" "struct event *ev" "struct timeval *" .Ft void .Fn "signal_del" "struct event *ev" .Ft int .Fn "signal_pending" "struct event *ev" "struct timeval *tv" .Ft int .Fn "signal_initialized" "struct event *ev" .Ft "struct bufferevent *" .Fn "bufferevent_new" "int fd" "evbuffercb readcb" "evbuffercb writecb" "everrorcb" "void *cbarg" .Ft void .Fn "bufferevent_free" "struct bufferevent *bufev" .Ft int .Fn "bufferevent_write" "struct bufferevent *bufev" "void *data" "size_t size" .Ft int .Fn "bufferevent_write_buffer" "struct bufferevent *bufev" "struct evbuffer *buf" .Ft size_t .Fn "bufferevent_read" "struct bufferevent *bufev" "void *data" "size_t size" .Ft int .Fn "bufferevent_enable" "struct bufferevent *bufev" "short event" .Ft int .Fn "bufferevent_disable" "struct bufferevent *bufev" "short event" .Ft void .Fn "bufferevent_settimeout" "struct bufferevent *bufev" "int timeout_read" "int timeout_write" .Ft int .Fn "bufferevent_base_set" "struct event_base *base" "struct bufferevent *bufev" .Ft "struct evbuffer *" .Fn "evbuffer_new" "void" .Ft void .Fn "evbuffer_free" "struct evbuffer *buf" .Ft int .Fn "evbuffer_add" "struct evbuffer *buf" "const void *data" "size_t size" .Ft int .Fn "evbuffer_add_buffer" "struct evbuffer *dst" "struct evbuffer *src" .Ft int .Fn "evbuffer_add_printf" "struct evbuffer *buf" "const char *fmt" "..." .Ft int .Fn "evbuffer_add_vprintf" "struct evbuffer *buf" "const char *fmt" "va_list ap" .Ft void .Fn "evbuffer_drain" "struct evbuffer *buf" "size_t size" .Ft int .Fn "evbuffer_write" "struct evbuffer *buf" "int fd" .Ft int .Fn "evbuffer_read" "struct evbuffer *buf" "int fd" "int size" .Ft "unsigned char *" .Fn "evbuffer_find" "struct evbuffer *buf" "const unsigned char *data" "size_t size" .Ft "char *" .Fn "evbuffer_readline" "struct evbuffer *buf" .Ft "struct evhttp *" .Fn "evhttp_new" "struct event_base *base" .Ft int .Fn "evhttp_bind_socket" "struct evhttp *http" "const char *address" "unsigned short port" .Ft "void" .Fn "evhttp_free" "struct evhttp *http" .Ft int .Fa (*event_sigcb)(void) ; .Ft volatile sig_atomic_t .Fa event_gotsig ; .Sh DESCRIPTION The .Nm event API provides a mechanism to execute a function when a specific event on a file descriptor occurs or after a given time has passed. .Pp The .Nm event API needs to be initialized with .Fn event_init before it can be used. .Pp In order to process events, an application needs to call .Fn event_dispatch . This function only returns on error, and should replace the event core of the application program. .Pp The function .Fn event_set prepares the event structure .Fa ev to be used in future calls to .Fn event_add and .Fn event_del . The event will be prepared to call the function specified by the .Fa fn argument with an .Fa int argument indicating the file descriptor, a .Fa short argument indicating the type of event, and a .Fa void * argument given in the .Fa arg argument. The .Fa fd indicates the file descriptor that should be monitored for events. The events can be either .Va EV_READ , .Va EV_WRITE , or both, indicating that an application can read or write from the file descriptor respectively without blocking. .Pp The function .Fa fn will be called with the file descriptor that triggered the event and the type of event which will be either .Va EV_TIMEOUT , .Va EV_SIGNAL , .Va EV_READ , or .Va EV_WRITE . Additionally, an event which has registered interest in more than one of the preceeding events, via bitwise-OR to .Fn event_set , can provide its callback function with a bitwise-OR of more than one triggered event. The additional flag .Va EV_PERSIST makes an .Fn event_add persistent until .Fn event_del has been called. .Pp Once initialized, the .Fa ev structure can be used repeatedly with .Fn event_add and .Fn event_del and does not need to be reinitialized unless the function called and/or the argument to it are to be changed. However, when an .Fa ev structure has been added to libevent using .Fn event_add the structure must persist until the event occurs (assuming .Fa EV_PERSIST is not set) or is removed using .Fn event_del . You may not reuse the same .Fa ev structure for multiple monitored descriptors; each descriptor needs its own .Fa ev . .Pp The function .Fn event_add schedules the execution of the .Fa ev event when the event specified in .Fn event_set occurs or in at least the time specified in the .Fa tv . If .Fa tv is .Dv NULL , no timeout occurs and the function will only be called if a matching event occurs on the file descriptor. The event in the .Fa ev argument must be already initialized by .Fn event_set and may not be used in calls to .Fn event_set until it has timed out or been removed with .Fn event_del . If the event in the .Fa ev argument already has a scheduled timeout, the old timeout will be replaced by the new one. .Pp The function .Fn event_del will cancel the event in the argument .Fa ev . If the event has already executed or has never been added the call will have no effect. .Pp The functions .Fn evtimer_set , .Fn evtimer_add , .Fn evtimer_del , .Fn evtimer_initialized , and .Fn evtimer_pending are abbreviations for common situations where only a timeout is required. The file descriptor passed will be \-1, and the event type will be .Va EV_TIMEOUT . .Pp The functions .Fn signal_set , .Fn signal_add , .Fn signal_del , .Fn signal_initialized , and .Fn signal_pending are abbreviations. The event type will be a persistent .Va EV_SIGNAL . That means .Fn signal_set adds .Va EV_PERSIST . .Pp In order to avoid races in signal handlers, the .Nm event API provides two variables: .Va event_sigcb and .Va event_gotsig . A signal handler sets .Va event_gotsig to indicate that a signal has been received. The application sets .Va event_sigcb to a callback function. After the signal handler sets .Va event_gotsig , .Nm event_dispatch will execute the callback function to process received signals. The callback returns 1 when no events are registered any more. It can return \-1 to indicate an error to the .Nm event library, causing .Fn event_dispatch to terminate with .Va errno set to .Er EINTR . .Pp The function .Fn event_once is similar to .Fn event_set . However, it schedules a callback to be called exactly once and does not require the caller to prepare an .Fa event structure. This function supports .Fa EV_TIMEOUT , .Fa EV_READ , and .Fa EV_WRITE . .Pp The .Fn event_pending function can be used to check if the event specified by .Fa event is pending to run. If .Va EV_TIMEOUT was specified and .Fa tv is not .Dv NULL , the expiration time of the event will be returned in .Fa tv . .Pp The .Fn event_initialized macro can be used to check if an event has been initialized. .Pp The .Nm event_loop function provides an interface for single pass execution of pending events. The flags .Va EVLOOP_ONCE and .Va EVLOOP_NONBLOCK are recognized. The .Nm event_loopexit function exits from the event loop. The next .Fn event_loop iteration after the given timer expires will complete normally (handling all queued events) then exit without blocking for events again. Subsequent invocations of .Fn event_loop will proceed normally. The .Nm event_loopbreak function exits from the event loop immediately. .Fn event_loop will abort after the next event is completed; .Fn event_loopbreak is typically invoked from this event's callback. This behavior is analogous to the "break;" statement. Subsequent invocations of .Fn event_loop will proceed normally. .Pp It is the responsibility of the caller to provide these functions with pre-allocated event structures. .Pp .Sh EVENT PRIORITIES By default .Nm libevent schedules all active events with the same priority. However, sometimes it is desirable to process some events with a higher priority than others. For that reason, .Nm libevent supports strict priority queues. Active events with a lower priority are always processed before events with a higher priority. .Pp The number of different priorities can be set initially with the .Fn event_priority_init function. This function should be called before the first call to .Fn event_dispatch . The .Fn event_priority_set function can be used to assign a priority to an event. By default, .Nm libevent assigns the middle priority to all events unless their priority is explicitly set. .Sh THREAD SAFE EVENTS .Nm Libevent has experimental support for thread-safe events. When initializing the library via .Fn event_init , an event base is returned. This event base can be used in conjunction with calls to .Fn event_base_set , .Fn event_base_dispatch , .Fn event_base_loop , .Fn event_base_loopexit , .Fn bufferevent_base_set and .Fn event_base_free . .Fn event_base_set should be called after preparing an event with .Fn event_set , as .Fn event_set assigns the provided event to the most recently created event base. .Fn bufferevent_base_set should be called after preparing a bufferevent with .Fn bufferevent_new . .Fn event_base_free should be used to free memory associated with the event base when it is no longer needed. .Sh BUFFERED EVENTS .Nm libevent provides an abstraction on top of the regular event callbacks. This abstraction is called a .Va "buffered event" . A buffered event provides input and output buffers that get filled and drained automatically. The user of a buffered event no longer deals directly with the IO, but instead is reading from input and writing to output buffers. .Pp A new bufferevent is created by .Fn bufferevent_new . The parameter .Fa fd specifies the file descriptor from which data is read and written to. This file descriptor is not allowed to be a .Xr pipe 2 . The next three parameters are callbacks. The read and write callback have the following form: .Ft void .Fn "(*cb)" "struct bufferevent *bufev" "void *arg" . The error callback has the following form: .Ft void .Fn "(*cb)" "struct bufferevent *bufev" "short what" "void *arg" . The argument is specified by the fourth parameter .Fa "cbarg" . A .Fa bufferevent struct pointer is returned on success, NULL on error. Both the read and the write callback may be NULL. The error callback has to be always provided. .Pp Once initialized, the bufferevent structure can be used repeatedly with bufferevent_enable() and bufferevent_disable(). The flags parameter can be a combination of .Va EV_READ and .Va EV_WRITE . When read enabled the bufferevent will try to read from the file descriptor and call the read callback. The write callback is executed whenever the output buffer is drained below the write low watermark, which is .Va 0 by default. .Pp The .Fn bufferevent_write function can be used to write data to the file descriptor. The data is appended to the output buffer and written to the descriptor automatically as it becomes available for writing. .Fn bufferevent_write returns 0 on success or \-1 on failure. The .Fn bufferevent_read function is used to read data from the input buffer, returning the amount of data read. .Pp If multiple bases are in use, bufferevent_base_set() must be called before enabling the bufferevent for the first time. .Sh NON-BLOCKING HTTP SUPPORT .Nm libevent provides a very thin HTTP layer that can be used both to host an HTTP server and also to make HTTP requests. An HTTP server can be created by calling .Fn evhttp_new . It can be bound to any port and address with the .Fn evhttp_bind_socket function. When the HTTP server is no longer used, it can be freed via .Fn evhttp_free . .Pp To be notified of HTTP requests, a user needs to register callbacks with the HTTP server. This can be done by calling .Fn evhttp_set_cb . The second argument is the URI for which a callback is being registered. The corresponding callback will receive an .Va struct evhttp_request object that contains all information about the request. .Pp This section does not document all the possible function calls; please check .Va event.h for the public interfaces. .Sh ADDITIONAL NOTES It is possible to disable support for .Va epoll , kqueue , devpoll , poll or .Va select by setting the environment variable .Va EVENT_NOEPOLL , EVENT_NOKQUEUE , EVENT_NODEVPOLL , EVENT_NOPOLL or .Va EVENT_NOSELECT , respectively. By setting the environment variable .Va EVENT_SHOW_METHOD , .Nm libevent displays the kernel notification method that it uses. .Sh RETURN VALUES Upon successful completion .Fn event_add and .Fn event_del return 0. Otherwise, \-1 is returned and the global variable errno is set to indicate the error. .Sh SEE ALSO .Xr kqueue 2 , .Xr poll 2 , .Xr select 2 , .Xr evdns 3 , .Xr timeout 9 .Sh HISTORY The .Nm event API manpage is based on the .Xr timeout 9 manpage by Artur Grabowski. The port of .Nm libevent to Windows is due to Michael A. Davis. Support for real-time signals is due to Taral. .Sh AUTHORS The .Nm event library was written by Niels Provos. .Sh BUGS This documentation is neither complete nor authoritative. If you are in doubt about the usage of this API then check the source code to find out how it works, write up the missing piece of documentation and send it to me for inclusion in this man page. ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/event.c������������������������������������������������������0000664�0000000�0000000�00000310763�15270523712�0021275�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2000-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #include <winsock2.h> #define WIN32_LEAN_AND_MEAN #include <windows.h> #undef WIN32_LEAN_AND_MEAN #endif #include <sys/types.h> #if !defined(_WIN32) && defined(EVENT__HAVE_SYS_TIME_H) #include <sys/time.h> #endif #include <sys/queue.h> #ifdef EVENT__HAVE_SYS_SOCKET_H #include <sys/socket.h> #endif #include <stdio.h> #include <stdlib.h> #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #include <ctype.h> #include <errno.h> #include <signal.h> #include <string.h> #include <time.h> #include <limits.h> #include "event2/event.h" #include "event2/event_struct.h" #include "event2/event_compat.h" #include "event-internal.h" #include "defer-internal.h" #include "evthread-internal.h" #include "event2/thread.h" #include "event2/util.h" #include "log-internal.h" #include "evmap-internal.h" #include "iocp-internal.h" #include "changelist-internal.h" #define HT_NO_CACHE_HASH_VALUES #include "ht-internal.h" #include "util-internal.h" #ifdef EVENT__HAVE_WORKING_KQUEUE #include "kqueue-internal.h" #endif #ifdef EVENT__HAVE_EVENT_PORTS extern const struct eventop evportops; #endif #ifdef EVENT__HAVE_SELECT extern const struct eventop selectops; #endif #ifdef EVENT__HAVE_POLL extern const struct eventop pollops; #endif #ifdef EVENT__HAVE_EPOLL extern const struct eventop epollops; #endif #ifdef EVENT__HAVE_WORKING_KQUEUE extern const struct eventop kqops; #endif #ifdef EVENT__HAVE_DEVPOLL extern const struct eventop devpollops; #endif #ifdef _WIN32 extern const struct eventop win32ops; #endif /* Array of backends in order of preference. */ static const struct eventop *eventops[] = { #ifdef EVENT__HAVE_EVENT_PORTS &evportops, #endif #ifdef EVENT__HAVE_WORKING_KQUEUE &kqops, #endif #ifdef EVENT__HAVE_EPOLL &epollops, #endif #ifdef EVENT__HAVE_DEVPOLL &devpollops, #endif #ifdef EVENT__HAVE_POLL &pollops, #endif #ifdef EVENT__HAVE_SELECT &selectops, #endif #ifdef _WIN32 &win32ops, #endif NULL }; /* Global state; deprecated */ struct event_base *event_global_current_base_ = NULL; #define current_base event_global_current_base_ /* Global state */ static void *event_self_cbarg_ptr_ = NULL; /* Prototypes */ static void event_queue_insert_active(struct event_base *, struct event_callback *); static void event_queue_insert_active_later(struct event_base *, struct event_callback *); static void event_queue_insert_timeout(struct event_base *, struct event *); static void event_queue_insert_inserted(struct event_base *, struct event *); static void event_queue_remove_active(struct event_base *, struct event_callback *); static void event_queue_remove_active_later(struct event_base *, struct event_callback *); static void event_queue_remove_timeout(struct event_base *, struct event *); static void event_queue_remove_inserted(struct event_base *, struct event *); static void event_queue_make_later_events_active(struct event_base *base); static int evthread_make_base_notifiable_nolock_(struct event_base *base); static int event_del_(struct event *ev, int blocking); #ifdef USE_REINSERT_TIMEOUT /* This code seems buggy; only turn it on if we find out what the trouble is. */ static void event_queue_reinsert_timeout(struct event_base *,struct event *, int was_common, int is_common, int old_timeout_idx); #endif static int event_haveevents(struct event_base *); static int event_process_active(struct event_base *); static int timeout_next(struct event_base *, struct timeval **); static void timeout_process(struct event_base *); static inline void event_signal_closure(struct event_base *, struct event *ev); static inline void event_persist_closure(struct event_base *, struct event *ev); static int evthread_notify_base(struct event_base *base); static void insert_common_timeout_inorder(struct common_timeout_list *ctl, struct event *ev); #ifndef EVENT__DISABLE_DEBUG_MODE /* These functions implement a hashtable of which 'struct event *' structures * have been setup or added. We don't want to trust the content of the struct * event itself, since we're trying to work through cases where an event gets * clobbered or freed. Instead, we keep a hashtable indexed by the pointer. */ struct event_debug_entry { HT_ENTRY(event_debug_entry) node; const struct event *ptr; unsigned added : 1; }; static inline unsigned hash_debug_entry(const struct event_debug_entry *e) { /* We need to do this silliness to convince compilers that we * honestly mean to cast e->ptr to an integer, and discard any * part of it that doesn't fit in an unsigned. */ unsigned u = (unsigned) ((ev_uintptr_t) e->ptr); /* Our hashtable implementation is pretty sensitive to low bits, * and every struct event is over 64 bytes in size, so we can * just say >>6. */ return (u >> 6); } static inline int eq_debug_entry(const struct event_debug_entry *a, const struct event_debug_entry *b) { return a->ptr == b->ptr; } int event_debug_mode_on_ = 0; #if !defined(EVENT__DISABLE_THREAD_SUPPORT) && !defined(EVENT__DISABLE_DEBUG_MODE) /** * @brief debug mode variable which is set for any function/structure that needs * to be shared across threads (if thread support is enabled). * * When and if evthreads are initialized, this variable will be evaluated, * and if set to something other than zero, this means the evthread setup * functions were called out of order. * * See: "Locks and threading" in the documentation. */ int event_debug_created_threadable_ctx_ = 0; #endif /* Set if it's too late to enable event_debug_mode. */ static int event_debug_mode_too_late = 0; #ifndef EVENT__DISABLE_THREAD_SUPPORT static void *event_debug_map_lock_ = NULL; #endif static HT_HEAD(event_debug_map, event_debug_entry) global_debug_map = HT_INITIALIZER(); HT_PROTOTYPE(event_debug_map, event_debug_entry, node, hash_debug_entry, eq_debug_entry) HT_GENERATE(event_debug_map, event_debug_entry, node, hash_debug_entry, eq_debug_entry, 0.5, mm_malloc, mm_realloc, mm_free) /* Macro: record that ev is now setup (that is, ready for an add) */ #define event_debug_note_setup_(ev) do { \ if (event_debug_mode_on_) { \ struct event_debug_entry *dent,find; \ find.ptr = (ev); \ EVLOCK_LOCK(event_debug_map_lock_, 0); \ dent = HT_FIND(event_debug_map, &global_debug_map, &find); \ if (dent) { \ dent->added = 0; \ } else { \ dent = mm_malloc(sizeof(*dent)); \ if (!dent) \ event_err(1, \ "Out of memory in debugging code"); \ dent->ptr = (ev); \ dent->added = 0; \ HT_INSERT(event_debug_map, &global_debug_map, dent); \ } \ EVLOCK_UNLOCK(event_debug_map_lock_, 0); \ } \ event_debug_mode_too_late = 1; \ } while (0) /* Macro: record that ev is no longer setup */ #define event_debug_note_teardown_(ev) do { \ if (event_debug_mode_on_) { \ struct event_debug_entry *dent,find; \ find.ptr = (ev); \ EVLOCK_LOCK(event_debug_map_lock_, 0); \ dent = HT_REMOVE(event_debug_map, &global_debug_map, &find); \ if (dent) \ mm_free(dent); \ EVLOCK_UNLOCK(event_debug_map_lock_, 0); \ } \ event_debug_mode_too_late = 1; \ } while (0) /* Macro: record that ev is now added */ #define event_debug_note_add_(ev) do { \ if (event_debug_mode_on_) { \ struct event_debug_entry *dent,find; \ find.ptr = (ev); \ EVLOCK_LOCK(event_debug_map_lock_, 0); \ dent = HT_FIND(event_debug_map, &global_debug_map, &find); \ if (dent) { \ dent->added = 1; \ } else { \ event_errx(EVENT_ERR_ABORT_, \ "%s: noting an add on a non-setup event %p" \ " (events: 0x%x, fd: "EV_SOCK_FMT \ ", flags: 0x%x)", \ __func__, (ev), (ev)->ev_events, \ EV_SOCK_ARG((ev)->ev_fd), (ev)->ev_flags); \ } \ EVLOCK_UNLOCK(event_debug_map_lock_, 0); \ } \ event_debug_mode_too_late = 1; \ } while (0) /* Macro: record that ev is no longer added */ #define event_debug_note_del_(ev) do { \ if (event_debug_mode_on_) { \ struct event_debug_entry *dent,find; \ find.ptr = (ev); \ EVLOCK_LOCK(event_debug_map_lock_, 0); \ dent = HT_FIND(event_debug_map, &global_debug_map, &find); \ if (dent) { \ dent->added = 0; \ } else { \ event_errx(EVENT_ERR_ABORT_, \ "%s: noting a del on a non-setup event %p" \ " (events: 0x%x, fd: "EV_SOCK_FMT \ ", flags: 0x%x)", \ __func__, (ev), (ev)->ev_events, \ EV_SOCK_ARG((ev)->ev_fd), (ev)->ev_flags); \ } \ EVLOCK_UNLOCK(event_debug_map_lock_, 0); \ } \ event_debug_mode_too_late = 1; \ } while (0) /* Macro: assert that ev is setup (i.e., okay to add or inspect) */ #define event_debug_assert_is_setup_(ev) do { \ if (event_debug_mode_on_) { \ struct event_debug_entry *dent,find; \ find.ptr = (ev); \ EVLOCK_LOCK(event_debug_map_lock_, 0); \ dent = HT_FIND(event_debug_map, &global_debug_map, &find); \ if (!dent) { \ event_errx(EVENT_ERR_ABORT_, \ "%s called on a non-initialized event %p" \ " (events: 0x%x, fd: "EV_SOCK_FMT\ ", flags: 0x%x)", \ __func__, (ev), (ev)->ev_events, \ EV_SOCK_ARG((ev)->ev_fd), (ev)->ev_flags); \ } \ EVLOCK_UNLOCK(event_debug_map_lock_, 0); \ } \ } while (0) /* Macro: assert that ev is not added (i.e., okay to tear down or set * up again) */ #define event_debug_assert_not_added_(ev) do { \ if (event_debug_mode_on_) { \ struct event_debug_entry *dent,find; \ find.ptr = (ev); \ EVLOCK_LOCK(event_debug_map_lock_, 0); \ dent = HT_FIND(event_debug_map, &global_debug_map, &find); \ if (dent && dent->added) { \ event_errx(EVENT_ERR_ABORT_, \ "%s called on an already added event %p" \ " (events: 0x%x, fd: "EV_SOCK_FMT", " \ "flags: 0x%x)", \ __func__, (ev), (ev)->ev_events, \ EV_SOCK_ARG((ev)->ev_fd), (ev)->ev_flags); \ } \ EVLOCK_UNLOCK(event_debug_map_lock_, 0); \ } \ } while (0) #else #define event_debug_note_setup_(ev) \ ((void)0) #define event_debug_note_teardown_(ev) \ ((void)0) #define event_debug_note_add_(ev) \ ((void)0) #define event_debug_note_del_(ev) \ ((void)0) #define event_debug_assert_is_setup_(ev) \ ((void)0) #define event_debug_assert_not_added_(ev) \ ((void)0) #endif #define EVENT_BASE_ASSERT_LOCKED(base) \ EVLOCK_ASSERT_LOCKED((base)->th_base_lock) /* How often (in seconds) do we check for changes in wall clock time relative * to monotonic time? Set this to -1 for 'never.' */ #define CLOCK_SYNC_INTERVAL 5 /** Set 'tp' to the current time according to 'base'. We must hold the lock * on 'base'. If there is a cached time, return it. Otherwise, use * clock_gettime or gettimeofday as appropriate to find out the right time. * Return 0 on success, -1 on failure. */ static int gettime(struct event_base *base, struct timeval *tp) { EVENT_BASE_ASSERT_LOCKED(base); if (base->tv_cache.tv_sec) { *tp = base->tv_cache; return (0); } if (evutil_gettime_monotonic_(&base->monotonic_timer, tp) == -1) { return -1; } if (base->last_updated_clock_diff + CLOCK_SYNC_INTERVAL < tp->tv_sec) { struct timeval tv; evutil_gettimeofday(&tv,NULL); evutil_timersub(&tv, tp, &base->tv_clock_diff); base->last_updated_clock_diff = tp->tv_sec; } return 0; } int event_base_gettimeofday_cached(struct event_base *base, struct timeval *tv) { int r; if (!base) { base = current_base; if (!current_base) return evutil_gettimeofday(tv, NULL); } EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (base->tv_cache.tv_sec == 0) { r = evutil_gettimeofday(tv, NULL); } else { evutil_timeradd(&base->tv_cache, &base->tv_clock_diff, tv); r = 0; } EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } /** Make 'base' have no current cached time. */ static inline void clear_time_cache(struct event_base *base) { base->tv_cache.tv_sec = 0; } /** Replace the cached time in 'base' with the current time. */ static inline void update_time_cache(struct event_base *base) { base->tv_cache.tv_sec = 0; if (!(base->flags & EVENT_BASE_FLAG_NO_CACHE_TIME)) gettime(base, &base->tv_cache); } int event_base_update_cache_time(struct event_base *base) { if (!base) { base = current_base; if (!current_base) return -1; } EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (base->running_loop) update_time_cache(base); EVBASE_RELEASE_LOCK(base, th_base_lock); return 0; } static inline struct event * event_callback_to_event(struct event_callback *evcb) { EVUTIL_ASSERT((evcb->evcb_flags & EVLIST_INIT)); return EVUTIL_UPCAST(evcb, struct event, ev_evcallback); } static inline struct event_callback * event_to_event_callback(struct event *ev) { return &ev->ev_evcallback; } struct event_base * event_init(void) { struct event_base *base = event_base_new_with_config(NULL); if (base == NULL) { event_errx(1, "%s: Unable to construct event_base", __func__); return NULL; } current_base = base; return (base); } struct event_base * event_base_new(void) { struct event_base *base = NULL; struct event_config *cfg = event_config_new(); if (cfg) { base = event_base_new_with_config(cfg); event_config_free(cfg); } return base; } /** Return true iff 'method' is the name of a method that 'cfg' tells us to * avoid. */ static int event_config_is_avoided_method(const struct event_config *cfg, const char *method) { struct event_config_entry *entry; TAILQ_FOREACH(entry, &cfg->entries, next) { if (entry->avoid_method != NULL && strcmp(entry->avoid_method, method) == 0) return (1); } return (0); } /** Return true iff 'method' is disabled according to the environment. */ static int event_is_method_disabled(const char *name) { char environment[64]; int i; evutil_snprintf(environment, sizeof(environment), "EVENT_NO%s", name); for (i = 8; environment[i] != '\0'; ++i) environment[i] = EVUTIL_TOUPPER_(environment[i]); /* Note that evutil_getenv_() ignores the environment entirely if * we're setuid */ return (evutil_getenv_(environment) != NULL); } int event_base_get_features(const struct event_base *base) { return base->evsel->features; } void event_enable_debug_mode(void) { #ifndef EVENT__DISABLE_DEBUG_MODE if (event_debug_mode_on_) event_errx(1, "%s was called twice!", __func__); if (event_debug_mode_too_late) event_errx(1, "%s must be called *before* creating any events " "or event_bases",__func__); event_debug_mode_on_ = 1; HT_INIT(event_debug_map, &global_debug_map); #endif } void event_disable_debug_mode(void) { #ifndef EVENT__DISABLE_DEBUG_MODE struct event_debug_entry **ent, *victim; EVLOCK_LOCK(event_debug_map_lock_, 0); for (ent = HT_START(event_debug_map, &global_debug_map); ent; ) { victim = *ent; ent = HT_NEXT_RMV(event_debug_map, &global_debug_map, ent); mm_free(victim); } HT_CLEAR(event_debug_map, &global_debug_map); EVLOCK_UNLOCK(event_debug_map_lock_ , 0); event_debug_mode_on_ = 0; #endif } struct event_base * event_base_new_with_config(const struct event_config *cfg) { int i; struct event_base *base; int should_check_environment; #ifndef EVENT__DISABLE_DEBUG_MODE event_debug_mode_too_late = 1; #endif if ((base = mm_calloc(1, sizeof(struct event_base))) == NULL) { event_warn("%s: calloc", __func__); return NULL; } if (cfg) base->flags = cfg->flags; should_check_environment = !(cfg && (cfg->flags & EVENT_BASE_FLAG_IGNORE_ENV)); { struct timeval tmp; int precise_time = cfg && (cfg->flags & EVENT_BASE_FLAG_PRECISE_TIMER); int flags; if (should_check_environment && !precise_time) { precise_time = evutil_getenv_("EVENT_PRECISE_TIMER") != NULL; base->flags |= EVENT_BASE_FLAG_PRECISE_TIMER; } flags = precise_time ? EV_MONOT_PRECISE : 0; evutil_configure_monotonic_time_(&base->monotonic_timer, flags); gettime(base, &tmp); } min_heap_ctor_(&base->timeheap); base->sig.ev_signal_pair[0] = -1; base->sig.ev_signal_pair[1] = -1; base->th_notify_fd[0] = -1; base->th_notify_fd[1] = -1; TAILQ_INIT(&base->active_later_queue); evmap_io_initmap_(&base->io); evmap_signal_initmap_(&base->sigmap); event_changelist_init_(&base->changelist); base->evbase = NULL; if (cfg) { memcpy(&base->max_dispatch_time, &cfg->max_dispatch_interval, sizeof(struct timeval)); base->limit_callbacks_after_prio = cfg->limit_callbacks_after_prio; } else { base->max_dispatch_time.tv_sec = -1; base->limit_callbacks_after_prio = 1; } if (cfg && cfg->max_dispatch_callbacks >= 0) { base->max_dispatch_callbacks = cfg->max_dispatch_callbacks; } else { base->max_dispatch_callbacks = INT_MAX; } if (base->max_dispatch_callbacks == INT_MAX && base->max_dispatch_time.tv_sec == -1) base->limit_callbacks_after_prio = INT_MAX; for (i = 0; eventops[i] && !base->evbase; i++) { if (cfg != NULL) { /* determine if this backend should be avoided */ if (event_config_is_avoided_method(cfg, eventops[i]->name)) continue; if ((eventops[i]->features & cfg->require_features) != cfg->require_features) continue; } /* also obey the environment variables */ if (should_check_environment && event_is_method_disabled(eventops[i]->name)) continue; base->evsel = eventops[i]; base->evbase = base->evsel->init(base); } if (base->evbase == NULL) { event_warnx("%s: no event mechanism available", __func__); base->evsel = NULL; event_base_free(base); return NULL; } if (evutil_getenv_("EVENT_SHOW_METHOD")) event_msgx("libevent using: %s", base->evsel->name); /* allocate a single active event queue */ if (event_base_priority_init(base, 1) < 0) { event_base_free(base); return NULL; } /* prepare for threading */ #if !defined(EVENT__DISABLE_THREAD_SUPPORT) && !defined(EVENT__DISABLE_DEBUG_MODE) event_debug_created_threadable_ctx_ = 1; #endif #ifndef EVENT__DISABLE_THREAD_SUPPORT if (EVTHREAD_LOCKING_ENABLED() && (!cfg || !(cfg->flags & EVENT_BASE_FLAG_NOLOCK))) { int r; EVTHREAD_ALLOC_LOCK(base->th_base_lock, 0); EVTHREAD_ALLOC_COND(base->current_event_cond); r = evthread_make_base_notifiable(base); if (r<0) { event_warnx("%s: Unable to make base notifiable.", __func__); event_base_free(base); return NULL; } } #endif #ifdef _WIN32 if (cfg && (cfg->flags & EVENT_BASE_FLAG_STARTUP_IOCP)) event_base_start_iocp_(base, cfg->n_cpus_hint); #endif return (base); } int event_base_start_iocp_(struct event_base *base, int n_cpus) { #ifdef _WIN32 if (base->iocp) return 0; base->iocp = event_iocp_port_launch_(n_cpus); if (!base->iocp) { event_warnx("%s: Couldn't launch IOCP", __func__); return -1; } return 0; #else return -1; #endif } void event_base_stop_iocp_(struct event_base *base) { #ifdef _WIN32 int rv; if (!base->iocp) return; rv = event_iocp_shutdown_(base->iocp, -1); EVUTIL_ASSERT(rv >= 0); base->iocp = NULL; #endif } static int event_base_cancel_single_callback_(struct event_base *base, struct event_callback *evcb, int run_finalizers) { int result = 0; if (evcb->evcb_flags & EVLIST_INIT) { struct event *ev = event_callback_to_event(evcb); if (!(ev->ev_flags & EVLIST_INTERNAL)) { event_del_(ev, EVENT_DEL_EVEN_IF_FINALIZING); result = 1; } } else { EVBASE_ACQUIRE_LOCK(base, th_base_lock); event_callback_cancel_nolock_(base, evcb, 1); EVBASE_RELEASE_LOCK(base, th_base_lock); result = 1; } if (run_finalizers && (evcb->evcb_flags & EVLIST_FINALIZING)) { switch (evcb->evcb_closure) { case EV_CLOSURE_EVENT_FINALIZE: case EV_CLOSURE_EVENT_FINALIZE_FREE: { struct event *ev = event_callback_to_event(evcb); ev->ev_evcallback.evcb_cb_union.evcb_evfinalize(ev, ev->ev_arg); if (evcb->evcb_closure == EV_CLOSURE_EVENT_FINALIZE_FREE) mm_free(ev); break; } case EV_CLOSURE_CB_FINALIZE: evcb->evcb_cb_union.evcb_cbfinalize(evcb, evcb->evcb_arg); break; default: break; } } return result; } static int event_base_free_queues_(struct event_base *base, int run_finalizers) { int deleted = 0, i; for (i = 0; i < base->nactivequeues; ++i) { struct event_callback *evcb, *next; for (evcb = TAILQ_FIRST(&base->activequeues[i]); evcb; ) { next = TAILQ_NEXT(evcb, evcb_active_next); deleted += event_base_cancel_single_callback_(base, evcb, run_finalizers); evcb = next; } } { struct event_callback *evcb; while ((evcb = TAILQ_FIRST(&base->active_later_queue))) { deleted += event_base_cancel_single_callback_(base, evcb, run_finalizers); } } return deleted; } static void event_base_free_(struct event_base *base, int run_finalizers) { int i, n_deleted=0; struct event *ev; /* XXXX grab the lock? If there is contention when one thread frees * the base, then the contending thread will be very sad soon. */ /* event_base_free(NULL) is how to free the current_base if we * made it with event_init and forgot to hold a reference to it. */ if (base == NULL && current_base) base = current_base; /* Don't actually free NULL. */ if (base == NULL) { event_warnx("%s: no base to free", __func__); return; } /* XXX(niels) - check for internal events first */ #ifdef _WIN32 event_base_stop_iocp_(base); #endif /* threading fds if we have them */ if (base->th_notify_fd[0] != -1) { event_del(&base->th_notify); EVUTIL_CLOSESOCKET(base->th_notify_fd[0]); if (base->th_notify_fd[1] != -1) EVUTIL_CLOSESOCKET(base->th_notify_fd[1]); base->th_notify_fd[0] = -1; base->th_notify_fd[1] = -1; event_debug_unassign(&base->th_notify); } /* Delete all non-internal events. */ evmap_delete_all_(base); while ((ev = min_heap_top_(&base->timeheap)) != NULL) { event_del(ev); ++n_deleted; } for (i = 0; i < base->n_common_timeouts; ++i) { struct common_timeout_list *ctl = base->common_timeout_queues[i]; event_del(&ctl->timeout_event); /* Internal; doesn't count */ event_debug_unassign(&ctl->timeout_event); for (ev = TAILQ_FIRST(&ctl->events); ev; ) { struct event *next = TAILQ_NEXT(ev, ev_timeout_pos.ev_next_with_common_timeout); if (!(ev->ev_flags & EVLIST_INTERNAL)) { event_del(ev); ++n_deleted; } ev = next; } mm_free(ctl); } if (base->common_timeout_queues) mm_free(base->common_timeout_queues); for (;;) { /* For finalizers we can register yet another finalizer out from * finalizer, and iff finalizer will be in active_later_queue we can * add finalizer to activequeues, and we will have events in * activequeues after this function returns, which is not what we want * (we even have an assertion for this). * * A simple case is bufferevent with underlying (i.e. filters). */ int i = event_base_free_queues_(base, run_finalizers); if (!i) { break; } n_deleted += i; } if (n_deleted) event_debug(("%s: %d events were still set in base", __func__, n_deleted)); while (LIST_FIRST(&base->once_events)) { struct event_once *eonce = LIST_FIRST(&base->once_events); LIST_REMOVE(eonce, next_once); mm_free(eonce); } if (base->evsel != NULL && base->evsel->dealloc != NULL) base->evsel->dealloc(base); for (i = 0; i < base->nactivequeues; ++i) EVUTIL_ASSERT(TAILQ_EMPTY(&base->activequeues[i])); EVUTIL_ASSERT(min_heap_empty_(&base->timeheap)); min_heap_dtor_(&base->timeheap); mm_free(base->activequeues); evmap_io_clear_(&base->io); evmap_signal_clear_(&base->sigmap); event_changelist_freemem_(&base->changelist); EVTHREAD_FREE_LOCK(base->th_base_lock, 0); EVTHREAD_FREE_COND(base->current_event_cond); /* If we're freeing current_base, there won't be a current_base. */ if (base == current_base) current_base = NULL; mm_free(base); } void event_base_free_nofinalize(struct event_base *base) { event_base_free_(base, 0); } void event_base_free(struct event_base *base) { event_base_free_(base, 1); } /* Fake eventop; used to disable the backend temporarily inside event_reinit * so that we can call event_del() on an event without telling the backend. */ static int nil_backend_del(struct event_base *b, evutil_socket_t fd, short old, short events, void *fdinfo) { return 0; } const struct eventop nil_eventop = { "nil", NULL, /* init: unused. */ NULL, /* add: unused. */ nil_backend_del, /* del: used, so needs to be killed. */ NULL, /* dispatch: unused. */ NULL, /* dealloc: unused. */ 0, 0, 0 }; /* reinitialize the event base after a fork */ int event_reinit(struct event_base *base) { const struct eventop *evsel; int res = 0; int was_notifiable = 0; int had_signal_added = 0; EVBASE_ACQUIRE_LOCK(base, th_base_lock); evsel = base->evsel; /* check if this event mechanism requires reinit on the backend */ if (evsel->need_reinit) { /* We're going to call event_del() on our notify events (the * ones that tell about signals and wakeup events). But we * don't actually want to tell the backend to change its * state, since it might still share some resource (a kqueue, * an epoll fd) with the parent process, and we don't want to * delete the fds from _that_ backend, we temporarily stub out * the evsel with a replacement. */ base->evsel = &nil_eventop; } /* We need to re-create a new signal-notification fd and a new * thread-notification fd. Otherwise, we'll still share those with * the parent process, which would make any notification sent to them * get received by one or both of the event loops, more or less at * random. */ if (base->sig.ev_signal_added) { event_del_nolock_(&base->sig.ev_signal, EVENT_DEL_AUTOBLOCK); event_debug_unassign(&base->sig.ev_signal); memset(&base->sig.ev_signal, 0, sizeof(base->sig.ev_signal)); had_signal_added = 1; base->sig.ev_signal_added = 0; } if (base->sig.ev_signal_pair[0] != -1) EVUTIL_CLOSESOCKET(base->sig.ev_signal_pair[0]); if (base->sig.ev_signal_pair[1] != -1) EVUTIL_CLOSESOCKET(base->sig.ev_signal_pair[1]); if (base->th_notify_fn != NULL) { was_notifiable = 1; base->th_notify_fn = NULL; } if (base->th_notify_fd[0] != -1) { event_del_nolock_(&base->th_notify, EVENT_DEL_AUTOBLOCK); EVUTIL_CLOSESOCKET(base->th_notify_fd[0]); if (base->th_notify_fd[1] != -1) EVUTIL_CLOSESOCKET(base->th_notify_fd[1]); base->th_notify_fd[0] = -1; base->th_notify_fd[1] = -1; event_debug_unassign(&base->th_notify); } /* Replace the original evsel. */ base->evsel = evsel; if (evsel->need_reinit) { /* Reconstruct the backend through brute-force, so that we do * not share any structures with the parent process. For some * backends, this is necessary: epoll and kqueue, for * instance, have events associated with a kernel * structure. If didn't reinitialize, we'd share that * structure with the parent process, and any changes made by * the parent would affect our backend's behavior (and vice * versa). */ if (base->evsel->dealloc != NULL) base->evsel->dealloc(base); base->evbase = evsel->init(base); if (base->evbase == NULL) { event_errx(1, "%s: could not reinitialize event mechanism", __func__); res = -1; goto done; } /* Empty out the changelist (if any): we are starting from a * blank slate. */ event_changelist_freemem_(&base->changelist); /* Tell the event maps to re-inform the backend about all * pending events. This will make the signal notification * event get re-created if necessary. */ if (evmap_reinit_(base) < 0) res = -1; } else { res = evsig_init_(base); if (res == 0 && had_signal_added) { res = event_add_nolock_(&base->sig.ev_signal, NULL, 0); if (res == 0) base->sig.ev_signal_added = 1; } } /* If we were notifiable before, and nothing just exploded, become * notifiable again. */ if (was_notifiable && res == 0) res = evthread_make_base_notifiable_nolock_(base); done: EVBASE_RELEASE_LOCK(base, th_base_lock); return (res); } /* Get the monotonic time for this event_base' timer */ int event_gettime_monotonic(struct event_base *base, struct timeval *tv) { int rv = -1; if (base && tv) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); rv = evutil_gettime_monotonic_(&(base->monotonic_timer), tv); EVBASE_RELEASE_LOCK(base, th_base_lock); } return rv; } const char ** event_get_supported_methods(void) { static const char **methods = NULL; const struct eventop **method; const char **tmp; int i = 0, k; /* count all methods */ for (method = &eventops[0]; *method != NULL; ++method) { ++i; } /* allocate one more than we need for the NULL pointer */ tmp = mm_calloc((i + 1), sizeof(char *)); if (tmp == NULL) return (NULL); /* populate the array with the supported methods */ for (k = 0, i = 0; eventops[k] != NULL; ++k) { tmp[i++] = eventops[k]->name; } tmp[i] = NULL; if (methods != NULL) mm_free((char**)methods); methods = tmp; return (methods); } struct event_config * event_config_new(void) { struct event_config *cfg = mm_calloc(1, sizeof(*cfg)); if (cfg == NULL) return (NULL); TAILQ_INIT(&cfg->entries); cfg->max_dispatch_interval.tv_sec = -1; cfg->max_dispatch_callbacks = INT_MAX; cfg->limit_callbacks_after_prio = 1; return (cfg); } static void event_config_entry_free(struct event_config_entry *entry) { if (entry->avoid_method != NULL) mm_free((char *)entry->avoid_method); mm_free(entry); } void event_config_free(struct event_config *cfg) { struct event_config_entry *entry; while ((entry = TAILQ_FIRST(&cfg->entries)) != NULL) { TAILQ_REMOVE(&cfg->entries, entry, next); event_config_entry_free(entry); } mm_free(cfg); } int event_config_set_flag(struct event_config *cfg, int flag) { if (!cfg) return -1; cfg->flags |= flag; return 0; } int event_config_avoid_method(struct event_config *cfg, const char *method) { struct event_config_entry *entry = mm_malloc(sizeof(*entry)); if (entry == NULL) return (-1); if ((entry->avoid_method = mm_strdup(method)) == NULL) { mm_free(entry); return (-1); } TAILQ_INSERT_TAIL(&cfg->entries, entry, next); return (0); } int event_config_require_features(struct event_config *cfg, int features) { if (!cfg) return (-1); cfg->require_features = features; return (0); } int event_config_set_num_cpus_hint(struct event_config *cfg, int cpus) { if (!cfg) return (-1); cfg->n_cpus_hint = cpus; return (0); } int event_config_set_max_dispatch_interval(struct event_config *cfg, const struct timeval *max_interval, int max_callbacks, int min_priority) { if (max_interval) memcpy(&cfg->max_dispatch_interval, max_interval, sizeof(struct timeval)); else cfg->max_dispatch_interval.tv_sec = -1; cfg->max_dispatch_callbacks = max_callbacks >= 0 ? max_callbacks : INT_MAX; if (min_priority < 0) min_priority = 0; cfg->limit_callbacks_after_prio = min_priority; return (0); } int event_priority_init(int npriorities) { return event_base_priority_init(current_base, npriorities); } int event_base_priority_init(struct event_base *base, int npriorities) { int i, r; r = -1; EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (N_ACTIVE_CALLBACKS(base) || npriorities < 1 || npriorities >= EVENT_MAX_PRIORITIES) goto err; if (npriorities == base->nactivequeues) goto ok; if (base->nactivequeues) { mm_free(base->activequeues); base->nactivequeues = 0; } /* Allocate our priority queues */ base->activequeues = (struct evcallback_list *) mm_calloc(npriorities, sizeof(struct evcallback_list)); if (base->activequeues == NULL) { event_warn("%s: calloc", __func__); goto err; } base->nactivequeues = npriorities; for (i = 0; i < base->nactivequeues; ++i) { TAILQ_INIT(&base->activequeues[i]); } ok: r = 0; err: EVBASE_RELEASE_LOCK(base, th_base_lock); return (r); } int event_base_get_npriorities(struct event_base *base) { int n; if (base == NULL) base = current_base; EVBASE_ACQUIRE_LOCK(base, th_base_lock); n = base->nactivequeues; EVBASE_RELEASE_LOCK(base, th_base_lock); return (n); } int event_base_get_num_events(struct event_base *base, unsigned int type) { int r = 0; EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (type & EVENT_BASE_COUNT_ACTIVE) r += base->event_count_active; if (type & EVENT_BASE_COUNT_VIRTUAL) r += base->virtual_event_count; if (type & EVENT_BASE_COUNT_ADDED) r += base->event_count; EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } int event_base_get_max_events(struct event_base *base, unsigned int type, int clear) { int r = 0; EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (type & EVENT_BASE_COUNT_ACTIVE) { r += base->event_count_active_max; if (clear) base->event_count_active_max = 0; } if (type & EVENT_BASE_COUNT_VIRTUAL) { r += base->virtual_event_count_max; if (clear) base->virtual_event_count_max = 0; } if (type & EVENT_BASE_COUNT_ADDED) { r += base->event_count_max; if (clear) base->event_count_max = 0; } EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } /* Returns true iff we're currently watching any events. */ static int event_haveevents(struct event_base *base) { /* Caller must hold th_base_lock */ return (base->virtual_event_count > 0 || base->event_count > 0); } /* "closure" function called when processing active signal events */ static inline void event_signal_closure(struct event_base *base, struct event *ev) { short ncalls; int should_break; /* Allows deletes to work */ ncalls = ev->ev_ncalls; if (ncalls != 0) ev->ev_pncalls = &ncalls; EVBASE_RELEASE_LOCK(base, th_base_lock); while (ncalls) { ncalls--; ev->ev_ncalls = ncalls; if (ncalls == 0) ev->ev_pncalls = NULL; (*ev->ev_callback)(ev->ev_fd, ev->ev_res, ev->ev_arg); EVBASE_ACQUIRE_LOCK(base, th_base_lock); should_break = base->event_break; EVBASE_RELEASE_LOCK(base, th_base_lock); if (should_break) { if (ncalls != 0) ev->ev_pncalls = NULL; return; } } } /* Common timeouts are special timeouts that are handled as queues rather than * in the minheap. This is more efficient than the minheap if we happen to * know that we're going to get several thousands of timeout events all with * the same timeout value. * * Since all our timeout handling code assumes timevals can be copied, * assigned, etc, we can't use "magic pointer" to encode these common * timeouts. Searching through a list to see if every timeout is common could * also get inefficient. Instead, we take advantage of the fact that tv_usec * is 32 bits long, but only uses 20 of those bits (since it can never be over * 999999.) We use the top bits to encode 4 bites of magic number, and 8 bits * of index into the event_base's aray of common timeouts. */ #define MICROSECONDS_MASK COMMON_TIMEOUT_MICROSECONDS_MASK #define COMMON_TIMEOUT_IDX_MASK 0x0ff00000 #define COMMON_TIMEOUT_IDX_SHIFT 20 #define COMMON_TIMEOUT_MASK 0xf0000000 #define COMMON_TIMEOUT_MAGIC 0x50000000 #define COMMON_TIMEOUT_IDX(tv) \ (((tv)->tv_usec & COMMON_TIMEOUT_IDX_MASK)>>COMMON_TIMEOUT_IDX_SHIFT) /** Return true iff if 'tv' is a common timeout in 'base' */ static inline int is_common_timeout(const struct timeval *tv, const struct event_base *base) { int idx; if ((tv->tv_usec & COMMON_TIMEOUT_MASK) != COMMON_TIMEOUT_MAGIC) return 0; idx = COMMON_TIMEOUT_IDX(tv); return idx < base->n_common_timeouts; } /* True iff tv1 and tv2 have the same common-timeout index, or if neither * one is a common timeout. */ static inline int is_same_common_timeout(const struct timeval *tv1, const struct timeval *tv2) { return (tv1->tv_usec & ~MICROSECONDS_MASK) == (tv2->tv_usec & ~MICROSECONDS_MASK); } /** Requires that 'tv' is a common timeout. Return the corresponding * common_timeout_list. */ static inline struct common_timeout_list * get_common_timeout_list(struct event_base *base, const struct timeval *tv) { return base->common_timeout_queues[COMMON_TIMEOUT_IDX(tv)]; } #if 0 static inline int common_timeout_ok(const struct timeval *tv, struct event_base *base) { const struct timeval *expect = &get_common_timeout_list(base, tv)->duration; return tv->tv_sec == expect->tv_sec && tv->tv_usec == expect->tv_usec; } #endif /* Add the timeout for the first event in given common timeout list to the * event_base's minheap. */ static void common_timeout_schedule(struct common_timeout_list *ctl, const struct timeval *now, struct event *head) { struct timeval timeout = head->ev_timeout; timeout.tv_usec &= MICROSECONDS_MASK; event_add_nolock_(&ctl->timeout_event, &timeout, 1); } /* Callback: invoked when the timeout for a common timeout queue triggers. * This means that (at least) the first event in that queue should be run, * and the timeout should be rescheduled if there are more events. */ static void common_timeout_callback(evutil_socket_t fd, short what, void *arg) { struct timeval now; struct common_timeout_list *ctl = arg; struct event_base *base = ctl->base; struct event *ev = NULL; EVBASE_ACQUIRE_LOCK(base, th_base_lock); gettime(base, &now); while (1) { ev = TAILQ_FIRST(&ctl->events); if (!ev || ev->ev_timeout.tv_sec > now.tv_sec || (ev->ev_timeout.tv_sec == now.tv_sec && (ev->ev_timeout.tv_usec&MICROSECONDS_MASK) > now.tv_usec)) break; event_del_nolock_(ev, EVENT_DEL_NOBLOCK); event_active_nolock_(ev, EV_TIMEOUT, 1); } if (ev) common_timeout_schedule(ctl, &now, ev); EVBASE_RELEASE_LOCK(base, th_base_lock); } #define MAX_COMMON_TIMEOUTS 256 const struct timeval * event_base_init_common_timeout(struct event_base *base, const struct timeval *duration) { int i; struct timeval tv; const struct timeval *result=NULL; struct common_timeout_list *new_ctl; EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (duration->tv_usec > 1000000) { memcpy(&tv, duration, sizeof(struct timeval)); if (is_common_timeout(duration, base)) tv.tv_usec &= MICROSECONDS_MASK; tv.tv_sec += tv.tv_usec / 1000000; tv.tv_usec %= 1000000; duration = &tv; } for (i = 0; i < base->n_common_timeouts; ++i) { const struct common_timeout_list *ctl = base->common_timeout_queues[i]; if (duration->tv_sec == ctl->duration.tv_sec && duration->tv_usec == (ctl->duration.tv_usec & MICROSECONDS_MASK)) { EVUTIL_ASSERT(is_common_timeout(&ctl->duration, base)); result = &ctl->duration; goto done; } } if (base->n_common_timeouts == MAX_COMMON_TIMEOUTS) { event_warnx("%s: Too many common timeouts already in use; " "we only support %d per event_base", __func__, MAX_COMMON_TIMEOUTS); goto done; } if (base->n_common_timeouts_allocated == base->n_common_timeouts) { int n = base->n_common_timeouts < 16 ? 16 : base->n_common_timeouts*2; struct common_timeout_list **newqueues = mm_realloc(base->common_timeout_queues, n*sizeof(struct common_timeout_queue *)); if (!newqueues) { event_warn("%s: realloc",__func__); goto done; } base->n_common_timeouts_allocated = n; base->common_timeout_queues = newqueues; } new_ctl = mm_calloc(1, sizeof(struct common_timeout_list)); if (!new_ctl) { event_warn("%s: calloc",__func__); goto done; } TAILQ_INIT(&new_ctl->events); new_ctl->duration.tv_sec = duration->tv_sec; new_ctl->duration.tv_usec = duration->tv_usec | COMMON_TIMEOUT_MAGIC | (base->n_common_timeouts << COMMON_TIMEOUT_IDX_SHIFT); evtimer_assign(&new_ctl->timeout_event, base, common_timeout_callback, new_ctl); new_ctl->timeout_event.ev_flags |= EVLIST_INTERNAL; event_priority_set(&new_ctl->timeout_event, 0); new_ctl->base = base; base->common_timeout_queues[base->n_common_timeouts++] = new_ctl; result = &new_ctl->duration; done: if (result) EVUTIL_ASSERT(is_common_timeout(result, base)); EVBASE_RELEASE_LOCK(base, th_base_lock); return result; } /* Closure function invoked when we're activating a persistent event. */ static inline void event_persist_closure(struct event_base *base, struct event *ev) { void (*evcb_callback)(evutil_socket_t, short, void *); // Other fields of *ev that must be stored before executing evutil_socket_t evcb_fd; short evcb_res; void *evcb_arg; /* reschedule the persistent event if we have a timeout. */ if (ev->ev_io_timeout.tv_sec || ev->ev_io_timeout.tv_usec) { /* If there was a timeout, we want it to run at an interval of * ev_io_timeout after the last time it was _scheduled_ for, * not ev_io_timeout after _now_. If it fired for another * reason, though, the timeout ought to start ticking _now_. */ struct timeval run_at, relative_to, delay, now; ev_uint32_t usec_mask = 0; EVUTIL_ASSERT(is_same_common_timeout(&ev->ev_timeout, &ev->ev_io_timeout)); gettime(base, &now); if (is_common_timeout(&ev->ev_timeout, base)) { delay = ev->ev_io_timeout; usec_mask = delay.tv_usec & ~MICROSECONDS_MASK; delay.tv_usec &= MICROSECONDS_MASK; if (ev->ev_res & EV_TIMEOUT) { relative_to = ev->ev_timeout; relative_to.tv_usec &= MICROSECONDS_MASK; } else { relative_to = now; } } else { delay = ev->ev_io_timeout; if (ev->ev_res & EV_TIMEOUT) { relative_to = ev->ev_timeout; } else { relative_to = now; } } evutil_timeradd(&relative_to, &delay, &run_at); if (evutil_timercmp(&run_at, &now, <)) { /* Looks like we missed at least one invocation due to * a clock jump, not running the event loop for a * while, really slow callbacks, or * something. Reschedule relative to now. */ evutil_timeradd(&now, &delay, &run_at); } run_at.tv_usec |= usec_mask; event_add_nolock_(ev, &run_at, 1); } // Save our callback before we release the lock evcb_callback = ev->ev_callback; evcb_fd = ev->ev_fd; evcb_res = ev->ev_res; evcb_arg = ev->ev_arg; // Release the lock EVBASE_RELEASE_LOCK(base, th_base_lock); // Execute the callback (evcb_callback)(evcb_fd, evcb_res, evcb_arg); } /* Helper for event_process_active to process all the events in a single queue, releasing the lock as we go. This function requires that the lock be held when it's invoked. Returns -1 if we get a signal or an event_break that means we should stop processing any active events now. Otherwise returns the number of non-internal event_callbacks that we processed. */ static int event_process_active_single_queue(struct event_base *base, struct evcallback_list *activeq, int max_to_process, const struct timeval *endtime) { struct event_callback *evcb; int count = 0; EVUTIL_ASSERT(activeq != NULL); for (evcb = TAILQ_FIRST(activeq); evcb; evcb = TAILQ_FIRST(activeq)) { struct event *ev=NULL; if (evcb->evcb_flags & EVLIST_INIT) { ev = event_callback_to_event(evcb); if (ev->ev_events & EV_PERSIST || ev->ev_flags & EVLIST_FINALIZING) event_queue_remove_active(base, evcb); else event_del_nolock_(ev, EVENT_DEL_NOBLOCK); event_debug(( "event_process_active: event: %p, %s%s%scall %p", ev, ev->ev_res & EV_READ ? "EV_READ " : " ", ev->ev_res & EV_WRITE ? "EV_WRITE " : " ", ev->ev_res & EV_CLOSED ? "EV_CLOSED " : " ", ev->ev_callback)); } else { event_queue_remove_active(base, evcb); event_debug(("event_process_active: event_callback %p, " "closure %d, call %p", evcb, evcb->evcb_closure, evcb->evcb_cb_union.evcb_callback)); } if (!(evcb->evcb_flags & EVLIST_INTERNAL)) ++count; base->current_event = evcb; #ifndef EVENT__DISABLE_THREAD_SUPPORT base->current_event_waiters = 0; #endif switch (evcb->evcb_closure) { case EV_CLOSURE_EVENT_SIGNAL: EVUTIL_ASSERT(ev != NULL); event_signal_closure(base, ev); break; case EV_CLOSURE_EVENT_PERSIST: EVUTIL_ASSERT(ev != NULL); event_persist_closure(base, ev); break; case EV_CLOSURE_EVENT: { void (*evcb_callback)(evutil_socket_t, short, void *); EVUTIL_ASSERT(ev != NULL); evcb_callback = *ev->ev_callback; EVBASE_RELEASE_LOCK(base, th_base_lock); evcb_callback(ev->ev_fd, ev->ev_res, ev->ev_arg); } break; case EV_CLOSURE_CB_SELF: { void (*evcb_selfcb)(struct event_callback *, void *) = evcb->evcb_cb_union.evcb_selfcb; EVBASE_RELEASE_LOCK(base, th_base_lock); evcb_selfcb(evcb, evcb->evcb_arg); } break; case EV_CLOSURE_EVENT_FINALIZE: case EV_CLOSURE_EVENT_FINALIZE_FREE: { void (*evcb_evfinalize)(struct event *, void *); int evcb_closure = evcb->evcb_closure; EVUTIL_ASSERT(ev != NULL); base->current_event = NULL; evcb_evfinalize = ev->ev_evcallback.evcb_cb_union.evcb_evfinalize; EVUTIL_ASSERT((evcb->evcb_flags & EVLIST_FINALIZING)); EVBASE_RELEASE_LOCK(base, th_base_lock); evcb_evfinalize(ev, ev->ev_arg); event_debug_note_teardown_(ev); if (evcb_closure == EV_CLOSURE_EVENT_FINALIZE_FREE) mm_free(ev); } break; case EV_CLOSURE_CB_FINALIZE: { void (*evcb_cbfinalize)(struct event_callback *, void *) = evcb->evcb_cb_union.evcb_cbfinalize; base->current_event = NULL; EVUTIL_ASSERT((evcb->evcb_flags & EVLIST_FINALIZING)); EVBASE_RELEASE_LOCK(base, th_base_lock); evcb_cbfinalize(evcb, evcb->evcb_arg); } break; default: EVUTIL_ASSERT(0); } EVBASE_ACQUIRE_LOCK(base, th_base_lock); base->current_event = NULL; #ifndef EVENT__DISABLE_THREAD_SUPPORT if (base->current_event_waiters) { base->current_event_waiters = 0; EVTHREAD_COND_BROADCAST(base->current_event_cond); } #endif if (base->event_break) return -1; if (count >= max_to_process) return count; if (count && endtime) { struct timeval now; update_time_cache(base); gettime(base, &now); if (evutil_timercmp(&now, endtime, >=)) return count; } if (base->event_continue) break; } return count; } /* * Active events are stored in priority queues. Lower priorities are always * process before higher priorities. Low priority events can starve high * priority ones. */ static int event_process_active(struct event_base *base) { /* Caller must hold th_base_lock */ struct evcallback_list *activeq = NULL; int i, c = 0; const struct timeval *endtime; struct timeval tv; const int maxcb = base->max_dispatch_callbacks; const int limit_after_prio = base->limit_callbacks_after_prio; if (base->max_dispatch_time.tv_sec >= 0) { update_time_cache(base); gettime(base, &tv); evutil_timeradd(&base->max_dispatch_time, &tv, &tv); endtime = &tv; } else { endtime = NULL; } for (i = 0; i < base->nactivequeues; ++i) { if (TAILQ_FIRST(&base->activequeues[i]) != NULL) { base->event_running_priority = i; activeq = &base->activequeues[i]; if (i < limit_after_prio) c = event_process_active_single_queue(base, activeq, INT_MAX, NULL); else c = event_process_active_single_queue(base, activeq, maxcb, endtime); if (c < 0) { goto done; } else if (c > 0) break; /* Processed a real event; do not * consider lower-priority events */ /* If we get here, all of the events we processed * were internal. Continue. */ } } done: base->event_running_priority = -1; return c; } /* * Wait continuously for events. We exit only if no events are left. */ int event_dispatch(void) { return (event_loop(0)); } int event_base_dispatch(struct event_base *event_base) { return (event_base_loop(event_base, 0)); } const char * event_base_get_method(const struct event_base *base) { EVUTIL_ASSERT(base); return (base->evsel->name); } /** Callback: used to implement event_base_loopexit by telling the event_base * that it's time to exit its loop. */ static void event_loopexit_cb(evutil_socket_t fd, short what, void *arg) { struct event_base *base = arg; base->event_gotterm = 1; } int event_loopexit(const struct timeval *tv) { return (event_once(-1, EV_TIMEOUT, event_loopexit_cb, current_base, tv)); } int event_base_loopexit(struct event_base *event_base, const struct timeval *tv) { return (event_base_once(event_base, -1, EV_TIMEOUT, event_loopexit_cb, event_base, tv)); } int event_loopbreak(void) { return (event_base_loopbreak(current_base)); } int event_base_loopbreak(struct event_base *event_base) { int r = 0; if (event_base == NULL) return (-1); EVBASE_ACQUIRE_LOCK(event_base, th_base_lock); event_base->event_break = 1; if (EVBASE_NEED_NOTIFY(event_base)) { r = evthread_notify_base(event_base); } else { r = (0); } EVBASE_RELEASE_LOCK(event_base, th_base_lock); return r; } int event_base_loopcontinue(struct event_base *event_base) { int r = 0; if (event_base == NULL) return (-1); EVBASE_ACQUIRE_LOCK(event_base, th_base_lock); event_base->event_continue = 1; if (EVBASE_NEED_NOTIFY(event_base)) { r = evthread_notify_base(event_base); } else { r = (0); } EVBASE_RELEASE_LOCK(event_base, th_base_lock); return r; } int event_base_got_break(struct event_base *event_base) { int res; EVBASE_ACQUIRE_LOCK(event_base, th_base_lock); res = event_base->event_break; EVBASE_RELEASE_LOCK(event_base, th_base_lock); return res; } int event_base_got_exit(struct event_base *event_base) { int res; EVBASE_ACQUIRE_LOCK(event_base, th_base_lock); res = event_base->event_gotterm; EVBASE_RELEASE_LOCK(event_base, th_base_lock); return res; } /* not thread safe */ int event_loop(int flags) { return event_base_loop(current_base, flags); } int event_base_loop(struct event_base *base, int flags) { const struct eventop *evsel = base->evsel; struct timeval tv; struct timeval *tv_p; int res, done, retval = 0; /* Grab the lock. We will release it inside evsel.dispatch, and again * as we invoke user callbacks. */ EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (base->running_loop) { event_warnx("%s: reentrant invocation. Only one event_base_loop" " can run on each event_base at once.", __func__); EVBASE_RELEASE_LOCK(base, th_base_lock); return -1; } base->running_loop = 1; clear_time_cache(base); if (base->sig.ev_signal_added && base->sig.ev_n_signals_added) evsig_set_base_(base); done = 0; #ifndef EVENT__DISABLE_THREAD_SUPPORT base->th_owner_id = EVTHREAD_GET_ID(); #endif base->event_gotterm = base->event_break = 0; while (!done) { base->event_continue = 0; base->n_deferreds_queued = 0; /* Terminate the loop if we have been asked to */ if (base->event_gotterm) { break; } if (base->event_break) { break; } tv_p = &tv; if (!N_ACTIVE_CALLBACKS(base) && !(flags & EVLOOP_NONBLOCK)) { timeout_next(base, &tv_p); } else { /* * if we have active events, we just poll new events * without waiting. */ evutil_timerclear(&tv); } /* If we have no events, we just exit */ if (0==(flags&EVLOOP_NO_EXIT_ON_EMPTY) && !event_haveevents(base) && !N_ACTIVE_CALLBACKS(base)) { event_debug(("%s: no events registered.", __func__)); retval = 1; goto done; } event_queue_make_later_events_active(base); clear_time_cache(base); res = evsel->dispatch(base, tv_p); if (res == -1) { event_debug(("%s: dispatch returned unsuccessfully.", __func__)); retval = -1; goto done; } update_time_cache(base); timeout_process(base); if (N_ACTIVE_CALLBACKS(base)) { int n = event_process_active(base); if ((flags & EVLOOP_ONCE) && N_ACTIVE_CALLBACKS(base) == 0 && n != 0) done = 1; } else if (flags & EVLOOP_NONBLOCK) done = 1; } event_debug(("%s: asked to terminate loop.", __func__)); done: clear_time_cache(base); base->running_loop = 0; EVBASE_RELEASE_LOCK(base, th_base_lock); return (retval); } /* One-time callback to implement event_base_once: invokes the user callback, * then deletes the allocated storage */ static void event_once_cb(evutil_socket_t fd, short events, void *arg) { struct event_once *eonce = arg; (*eonce->cb)(fd, events, eonce->arg); EVBASE_ACQUIRE_LOCK(eonce->ev.ev_base, th_base_lock); LIST_REMOVE(eonce, next_once); EVBASE_RELEASE_LOCK(eonce->ev.ev_base, th_base_lock); event_debug_unassign(&eonce->ev); mm_free(eonce); } /* not threadsafe, event scheduled once. */ int event_once(evutil_socket_t fd, short events, void (*callback)(evutil_socket_t, short, void *), void *arg, const struct timeval *tv) { return event_base_once(current_base, fd, events, callback, arg, tv); } /* Schedules an event once */ int event_base_once(struct event_base *base, evutil_socket_t fd, short events, void (*callback)(evutil_socket_t, short, void *), void *arg, const struct timeval *tv) { struct event_once *eonce; int res = 0; int activate = 0; /* We cannot support signals that just fire once, or persistent * events. */ if (events & (EV_SIGNAL|EV_PERSIST)) return (-1); if ((eonce = mm_calloc(1, sizeof(struct event_once))) == NULL) return (-1); eonce->cb = callback; eonce->arg = arg; if ((events & (EV_TIMEOUT|EV_SIGNAL|EV_READ|EV_WRITE|EV_CLOSED)) == EV_TIMEOUT) { evtimer_assign(&eonce->ev, base, event_once_cb, eonce); if (tv == NULL || ! evutil_timerisset(tv)) { /* If the event is going to become active immediately, * don't put it on the timeout queue. This is one * idiom for scheduling a callback, so let's make * it fast (and order-preserving). */ activate = 1; } } else if (events & (EV_READ|EV_WRITE|EV_CLOSED)) { events &= EV_READ|EV_WRITE|EV_CLOSED; event_assign(&eonce->ev, base, fd, events, event_once_cb, eonce); } else { /* Bad event combination */ mm_free(eonce); return (-1); } if (res == 0) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (activate) event_active_nolock_(&eonce->ev, EV_TIMEOUT, 1); else res = event_add_nolock_(&eonce->ev, tv, 0); if (res != 0) { mm_free(eonce); return (res); } else { LIST_INSERT_HEAD(&base->once_events, eonce, next_once); } EVBASE_RELEASE_LOCK(base, th_base_lock); } return (0); } int event_assign(struct event *ev, struct event_base *base, evutil_socket_t fd, short events, void (*callback)(evutil_socket_t, short, void *), void *arg) { if (!base) base = current_base; if (arg == &event_self_cbarg_ptr_) arg = ev; event_debug_assert_not_added_(ev); ev->ev_base = base; ev->ev_callback = callback; ev->ev_arg = arg; ev->ev_fd = fd; ev->ev_events = events; ev->ev_res = 0; ev->ev_flags = EVLIST_INIT; ev->ev_ncalls = 0; ev->ev_pncalls = NULL; if (events & EV_SIGNAL) { if ((events & (EV_READ|EV_WRITE|EV_CLOSED)) != 0) { event_warnx("%s: EV_SIGNAL is not compatible with " "EV_READ, EV_WRITE or EV_CLOSED", __func__); return -1; } ev->ev_closure = EV_CLOSURE_EVENT_SIGNAL; } else { if (events & EV_PERSIST) { evutil_timerclear(&ev->ev_io_timeout); ev->ev_closure = EV_CLOSURE_EVENT_PERSIST; } else { ev->ev_closure = EV_CLOSURE_EVENT; } } min_heap_elem_init_(ev); if (base != NULL) { /* by default, we put new events into the middle priority */ ev->ev_pri = base->nactivequeues / 2; } event_debug_note_setup_(ev); return 0; } int event_base_set(struct event_base *base, struct event *ev) { /* Only innocent events may be assigned to a different base */ if (ev->ev_flags != EVLIST_INIT) return (-1); event_debug_assert_is_setup_(ev); ev->ev_base = base; ev->ev_pri = base->nactivequeues/2; return (0); } void event_set(struct event *ev, evutil_socket_t fd, short events, void (*callback)(evutil_socket_t, short, void *), void *arg) { int r; r = event_assign(ev, current_base, fd, events, callback, arg); EVUTIL_ASSERT(r == 0); } void * event_self_cbarg(void) { return &event_self_cbarg_ptr_; } struct event * event_base_get_running_event(struct event_base *base) { struct event *ev = NULL; EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (EVBASE_IN_THREAD(base)) { struct event_callback *evcb = base->current_event; if (evcb->evcb_flags & EVLIST_INIT) ev = event_callback_to_event(evcb); } EVBASE_RELEASE_LOCK(base, th_base_lock); return ev; } struct event * event_new(struct event_base *base, evutil_socket_t fd, short events, void (*cb)(evutil_socket_t, short, void *), void *arg) { struct event *ev; ev = mm_malloc(sizeof(struct event)); if (ev == NULL) return (NULL); if (event_assign(ev, base, fd, events, cb, arg) < 0) { mm_free(ev); return (NULL); } return (ev); } void event_free(struct event *ev) { /* This is disabled, so that events which have been finalized be a * valid target for event_free(). That's */ // event_debug_assert_is_setup_(ev); /* make sure that this event won't be coming back to haunt us. */ event_del(ev); event_debug_note_teardown_(ev); mm_free(ev); } void event_debug_unassign(struct event *ev) { event_debug_assert_not_added_(ev); event_debug_note_teardown_(ev); ev->ev_flags &= ~EVLIST_INIT; } #define EVENT_FINALIZE_FREE_ 0x10000 static int event_finalize_nolock_(struct event_base *base, unsigned flags, struct event *ev, event_finalize_callback_fn cb) { ev_uint8_t closure = (flags & EVENT_FINALIZE_FREE_) ? EV_CLOSURE_EVENT_FINALIZE_FREE : EV_CLOSURE_EVENT_FINALIZE; event_del_nolock_(ev, EVENT_DEL_NOBLOCK); ev->ev_closure = closure; ev->ev_evcallback.evcb_cb_union.evcb_evfinalize = cb; event_active_nolock_(ev, EV_FINALIZE, 1); ev->ev_flags |= EVLIST_FINALIZING; return 0; } static int event_finalize_impl_(unsigned flags, struct event *ev, event_finalize_callback_fn cb) { int r; struct event_base *base = ev->ev_base; if (EVUTIL_FAILURE_CHECK(!base)) { event_warnx("%s: event has no event_base set.", __func__); return -1; } EVBASE_ACQUIRE_LOCK(base, th_base_lock); r = event_finalize_nolock_(base, flags, ev, cb); EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } int event_finalize(unsigned flags, struct event *ev, event_finalize_callback_fn cb) { return event_finalize_impl_(flags, ev, cb); } int event_free_finalize(unsigned flags, struct event *ev, event_finalize_callback_fn cb) { return event_finalize_impl_(flags|EVENT_FINALIZE_FREE_, ev, cb); } void event_callback_finalize_nolock_(struct event_base *base, unsigned flags, struct event_callback *evcb, void (*cb)(struct event_callback *, void *)) { struct event *ev = NULL; if (evcb->evcb_flags & EVLIST_INIT) { ev = event_callback_to_event(evcb); event_del_nolock_(ev, EVENT_DEL_NOBLOCK); } else { event_callback_cancel_nolock_(base, evcb, 0); /*XXX can this fail?*/ } evcb->evcb_closure = EV_CLOSURE_CB_FINALIZE; evcb->evcb_cb_union.evcb_cbfinalize = cb; event_callback_activate_nolock_(base, evcb); /* XXX can this really fail?*/ evcb->evcb_flags |= EVLIST_FINALIZING; } void event_callback_finalize_(struct event_base *base, unsigned flags, struct event_callback *evcb, void (*cb)(struct event_callback *, void *)) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); event_callback_finalize_nolock_(base, flags, evcb, cb); EVBASE_RELEASE_LOCK(base, th_base_lock); } /** Internal: Finalize all of the n_cbs callbacks in evcbs. The provided * callback will be invoked on *one of them*, after they have *all* been * finalized. */ int event_callback_finalize_many_(struct event_base *base, int n_cbs, struct event_callback **evcbs, void (*cb)(struct event_callback *, void *)) { int n_pending = 0, i; if (base == NULL) base = current_base; EVBASE_ACQUIRE_LOCK(base, th_base_lock); event_debug(("%s: %d events finalizing", __func__, n_cbs)); /* At most one can be currently executing; the rest we just * cancel... But we always make sure that the finalize callback * runs. */ for (i = 0; i < n_cbs; ++i) { struct event_callback *evcb = evcbs[i]; if (evcb == base->current_event) { event_callback_finalize_nolock_(base, 0, evcb, cb); ++n_pending; } else { event_callback_cancel_nolock_(base, evcb, 0); } } if (n_pending == 0) { /* Just do the first one. */ event_callback_finalize_nolock_(base, 0, evcbs[0], cb); } EVBASE_RELEASE_LOCK(base, th_base_lock); return 0; } /* * Set's the priority of an event - if an event is already scheduled * changing the priority is going to fail. */ int event_priority_set(struct event *ev, int pri) { event_debug_assert_is_setup_(ev); if (ev->ev_flags & EVLIST_ACTIVE) return (-1); if (pri < 0 || pri >= ev->ev_base->nactivequeues) return (-1); ev->ev_pri = pri; return (0); } /* * Checks if a specific event is pending or scheduled. */ int event_pending(const struct event *ev, short event, struct timeval *tv) { int flags = 0; if (EVUTIL_FAILURE_CHECK(ev->ev_base == NULL)) { event_warnx("%s: event has no event_base set.", __func__); return 0; } EVBASE_ACQUIRE_LOCK(ev->ev_base, th_base_lock); event_debug_assert_is_setup_(ev); if (ev->ev_flags & EVLIST_INSERTED) flags |= (ev->ev_events & (EV_READ|EV_WRITE|EV_CLOSED|EV_SIGNAL)); if (ev->ev_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER)) flags |= ev->ev_res; if (ev->ev_flags & EVLIST_TIMEOUT) flags |= EV_TIMEOUT; event &= (EV_TIMEOUT|EV_READ|EV_WRITE|EV_CLOSED|EV_SIGNAL); /* See if there is a timeout that we should report */ if (tv != NULL && (flags & event & EV_TIMEOUT)) { struct timeval tmp = ev->ev_timeout; tmp.tv_usec &= MICROSECONDS_MASK; /* correctly remamp to real time */ evutil_timeradd(&ev->ev_base->tv_clock_diff, &tmp, tv); } EVBASE_RELEASE_LOCK(ev->ev_base, th_base_lock); return (flags & event); } int event_initialized(const struct event *ev) { if (!(ev->ev_flags & EVLIST_INIT)) return 0; return 1; } void event_get_assignment(const struct event *event, struct event_base **base_out, evutil_socket_t *fd_out, short *events_out, event_callback_fn *callback_out, void **arg_out) { event_debug_assert_is_setup_(event); if (base_out) *base_out = event->ev_base; if (fd_out) *fd_out = event->ev_fd; if (events_out) *events_out = event->ev_events; if (callback_out) *callback_out = event->ev_callback; if (arg_out) *arg_out = event->ev_arg; } size_t event_get_struct_event_size(void) { return sizeof(struct event); } evutil_socket_t event_get_fd(const struct event *ev) { event_debug_assert_is_setup_(ev); return ev->ev_fd; } struct event_base * event_get_base(const struct event *ev) { event_debug_assert_is_setup_(ev); return ev->ev_base; } short event_get_events(const struct event *ev) { event_debug_assert_is_setup_(ev); return ev->ev_events; } event_callback_fn event_get_callback(const struct event *ev) { event_debug_assert_is_setup_(ev); return ev->ev_callback; } void * event_get_callback_arg(const struct event *ev) { event_debug_assert_is_setup_(ev); return ev->ev_arg; } int event_get_priority(const struct event *ev) { event_debug_assert_is_setup_(ev); return ev->ev_pri; } int event_add(struct event *ev, const struct timeval *tv) { int res; if (EVUTIL_FAILURE_CHECK(!ev->ev_base)) { event_warnx("%s: event has no event_base set.", __func__); return -1; } EVBASE_ACQUIRE_LOCK(ev->ev_base, th_base_lock); res = event_add_nolock_(ev, tv, 0); EVBASE_RELEASE_LOCK(ev->ev_base, th_base_lock); return (res); } /* Helper callback: wake an event_base from another thread. This version * works by writing a byte to one end of a socketpair, so that the event_base * listening on the other end will wake up as the corresponding event * triggers */ static int evthread_notify_base_default(struct event_base *base) { char buf[1]; int r; buf[0] = (char) 0; #ifdef _WIN32 r = send(base->th_notify_fd[1], buf, 1, 0); #else r = write(base->th_notify_fd[1], buf, 1); #endif return (r < 0 && ! EVUTIL_ERR_IS_EAGAIN(errno)) ? -1 : 0; } #ifdef EVENT__HAVE_EVENTFD /* Helper callback: wake an event_base from another thread. This version * assumes that you have a working eventfd() implementation. */ static int evthread_notify_base_eventfd(struct event_base *base) { ev_uint64_t msg = 1; int r; do { r = write(base->th_notify_fd[0], (void*) &msg, sizeof(msg)); } while (r < 0 && errno == EAGAIN); return (r < 0) ? -1 : 0; } #endif /** Tell the thread currently running the event_loop for base (if any) that it * needs to stop waiting in its dispatch function (if it is) and process all * active callbacks. */ static int evthread_notify_base(struct event_base *base) { EVENT_BASE_ASSERT_LOCKED(base); if (!base->th_notify_fn) return -1; if (base->is_notify_pending) return 0; base->is_notify_pending = 1; return base->th_notify_fn(base); } /* Implementation function to remove a timeout on a currently pending event. */ int event_remove_timer_nolock_(struct event *ev) { struct event_base *base = ev->ev_base; EVENT_BASE_ASSERT_LOCKED(base); event_debug_assert_is_setup_(ev); event_debug(("event_remove_timer_nolock: event: %p", ev)); /* If it's not pending on a timeout, we don't need to do anything. */ if (ev->ev_flags & EVLIST_TIMEOUT) { event_queue_remove_timeout(base, ev); evutil_timerclear(&ev->ev_.ev_io.ev_timeout); } return (0); } int event_remove_timer(struct event *ev) { int res; if (EVUTIL_FAILURE_CHECK(!ev->ev_base)) { event_warnx("%s: event has no event_base set.", __func__); return -1; } EVBASE_ACQUIRE_LOCK(ev->ev_base, th_base_lock); res = event_remove_timer_nolock_(ev); EVBASE_RELEASE_LOCK(ev->ev_base, th_base_lock); return (res); } /* Implementation function to add an event. Works just like event_add, * except: 1) it requires that we have the lock. 2) if tv_is_absolute is set, * we treat tv as an absolute time, not as an interval to add to the current * time */ int event_add_nolock_(struct event *ev, const struct timeval *tv, int tv_is_absolute) { struct event_base *base = ev->ev_base; int res = 0; int notify = 0; EVENT_BASE_ASSERT_LOCKED(base); event_debug_assert_is_setup_(ev); event_debug(( "event_add: event: %p (fd "EV_SOCK_FMT"), %s%s%s%scall %p", ev, EV_SOCK_ARG(ev->ev_fd), ev->ev_events & EV_READ ? "EV_READ " : " ", ev->ev_events & EV_WRITE ? "EV_WRITE " : " ", ev->ev_events & EV_CLOSED ? "EV_CLOSED " : " ", tv ? "EV_TIMEOUT " : " ", ev->ev_callback)); EVUTIL_ASSERT(!(ev->ev_flags & ~EVLIST_ALL)); if (ev->ev_flags & EVLIST_FINALIZING) { /* XXXX debug */ return (-1); } /* * prepare for timeout insertion further below, if we get a * failure on any step, we should not change any state. */ if (tv != NULL && !(ev->ev_flags & EVLIST_TIMEOUT)) { if (min_heap_reserve_(&base->timeheap, 1 + min_heap_size_(&base->timeheap)) == -1) return (-1); /* ENOMEM == errno */ } /* If the main thread is currently executing a signal event's * callback, and we are not the main thread, then we want to wait * until the callback is done before we mess with the event, or else * we can race on ev_ncalls and ev_pncalls below. */ #ifndef EVENT__DISABLE_THREAD_SUPPORT if (base->current_event == event_to_event_callback(ev) && (ev->ev_events & EV_SIGNAL) && !EVBASE_IN_THREAD(base)) { ++base->current_event_waiters; EVTHREAD_COND_WAIT(base->current_event_cond, base->th_base_lock); } #endif if ((ev->ev_events & (EV_READ|EV_WRITE|EV_CLOSED|EV_SIGNAL)) && !(ev->ev_flags & (EVLIST_INSERTED|EVLIST_ACTIVE|EVLIST_ACTIVE_LATER))) { if (ev->ev_events & (EV_READ|EV_WRITE|EV_CLOSED)) res = evmap_io_add_(base, ev->ev_fd, ev); else if (ev->ev_events & EV_SIGNAL) res = evmap_signal_add_(base, (int)ev->ev_fd, ev); if (res != -1) event_queue_insert_inserted(base, ev); if (res == 1) { /* evmap says we need to notify the main thread. */ notify = 1; res = 0; } } /* * we should change the timeout state only if the previous event * addition succeeded. */ if (res != -1 && tv != NULL) { struct timeval now; int common_timeout; #ifdef USE_REINSERT_TIMEOUT int was_common; int old_timeout_idx; #endif /* * for persistent timeout events, we remember the * timeout value and re-add the event. * * If tv_is_absolute, this was already set. */ if (ev->ev_closure == EV_CLOSURE_EVENT_PERSIST && !tv_is_absolute) ev->ev_io_timeout = *tv; #ifndef USE_REINSERT_TIMEOUT if (ev->ev_flags & EVLIST_TIMEOUT) { event_queue_remove_timeout(base, ev); } #endif /* Check if it is active due to a timeout. Rescheduling * this timeout before the callback can be executed * removes it from the active list. */ if ((ev->ev_flags & EVLIST_ACTIVE) && (ev->ev_res & EV_TIMEOUT)) { if (ev->ev_events & EV_SIGNAL) { /* See if we are just active executing * this event in a loop */ if (ev->ev_ncalls && ev->ev_pncalls) { /* Abort loop */ *ev->ev_pncalls = 0; } } event_queue_remove_active(base, event_to_event_callback(ev)); } gettime(base, &now); common_timeout = is_common_timeout(tv, base); #ifdef USE_REINSERT_TIMEOUT was_common = is_common_timeout(&ev->ev_timeout, base); old_timeout_idx = COMMON_TIMEOUT_IDX(&ev->ev_timeout); #endif if (tv_is_absolute) { ev->ev_timeout = *tv; } else if (common_timeout) { struct timeval tmp = *tv; tmp.tv_usec &= MICROSECONDS_MASK; evutil_timeradd(&now, &tmp, &ev->ev_timeout); ev->ev_timeout.tv_usec |= (tv->tv_usec & ~MICROSECONDS_MASK); } else { evutil_timeradd(&now, tv, &ev->ev_timeout); } event_debug(( "event_add: event %p, timeout in %d seconds %d useconds, call %p", ev, (int)tv->tv_sec, (int)tv->tv_usec, ev->ev_callback)); #ifdef USE_REINSERT_TIMEOUT event_queue_reinsert_timeout(base, ev, was_common, common_timeout, old_timeout_idx); #else event_queue_insert_timeout(base, ev); #endif if (common_timeout) { struct common_timeout_list *ctl = get_common_timeout_list(base, &ev->ev_timeout); if (ev == TAILQ_FIRST(&ctl->events)) { common_timeout_schedule(ctl, &now, ev); } } else { struct event* top = NULL; /* See if the earliest timeout is now earlier than it * was before: if so, we will need to tell the main * thread to wake up earlier than it would otherwise. * We double check the timeout of the top element to * handle time distortions due to system suspension. */ if (min_heap_elt_is_top_(ev)) notify = 1; else if ((top = min_heap_top_(&base->timeheap)) != NULL && evutil_timercmp(&top->ev_timeout, &now, <)) notify = 1; } } /* if we are not in the right thread, we need to wake up the loop */ if (res != -1 && notify && EVBASE_NEED_NOTIFY(base)) evthread_notify_base(base); event_debug_note_add_(ev); return (res); } static int event_del_(struct event *ev, int blocking) { int res; if (EVUTIL_FAILURE_CHECK(!ev->ev_base)) { event_warnx("%s: event has no event_base set.", __func__); return -1; } EVBASE_ACQUIRE_LOCK(ev->ev_base, th_base_lock); res = event_del_nolock_(ev, blocking); EVBASE_RELEASE_LOCK(ev->ev_base, th_base_lock); return (res); } int event_del(struct event *ev) { return event_del_(ev, EVENT_DEL_AUTOBLOCK); } int event_del_block(struct event *ev) { return event_del_(ev, EVENT_DEL_BLOCK); } int event_del_noblock(struct event *ev) { return event_del_(ev, EVENT_DEL_NOBLOCK); } /** Helper for event_del: always called with th_base_lock held. * * "blocking" must be one of the EVENT_DEL_{BLOCK, NOBLOCK, AUTOBLOCK, * EVEN_IF_FINALIZING} values. See those for more information. */ int event_del_nolock_(struct event *ev, int blocking) { struct event_base *base; int res = 0, notify = 0; event_debug(("event_del: %p (fd "EV_SOCK_FMT"), callback %p", ev, EV_SOCK_ARG(ev->ev_fd), ev->ev_callback)); /* An event without a base has not been added */ if (ev->ev_base == NULL) return (-1); EVENT_BASE_ASSERT_LOCKED(ev->ev_base); if (blocking != EVENT_DEL_EVEN_IF_FINALIZING) { if (ev->ev_flags & EVLIST_FINALIZING) { /* XXXX Debug */ return 0; } } /* If the main thread is currently executing this event's callback, * and we are not the main thread, then we want to wait until the * callback is done before we start removing the event. That way, * when this function returns, it will be safe to free the * user-supplied argument. */ base = ev->ev_base; #ifndef EVENT__DISABLE_THREAD_SUPPORT if (blocking != EVENT_DEL_NOBLOCK && base->current_event == event_to_event_callback(ev) && !EVBASE_IN_THREAD(base) && (blocking == EVENT_DEL_BLOCK || !(ev->ev_events & EV_FINALIZE))) { ++base->current_event_waiters; EVTHREAD_COND_WAIT(base->current_event_cond, base->th_base_lock); } #endif EVUTIL_ASSERT(!(ev->ev_flags & ~EVLIST_ALL)); /* See if we are just active executing this event in a loop */ if (ev->ev_events & EV_SIGNAL) { if (ev->ev_ncalls && ev->ev_pncalls) { /* Abort loop */ *ev->ev_pncalls = 0; } } if (ev->ev_flags & EVLIST_TIMEOUT) { /* NOTE: We never need to notify the main thread because of a * deleted timeout event: all that could happen if we don't is * that the dispatch loop might wake up too early. But the * point of notifying the main thread _is_ to wake up the * dispatch loop early anyway, so we wouldn't gain anything by * doing it. */ event_queue_remove_timeout(base, ev); } if (ev->ev_flags & EVLIST_ACTIVE) event_queue_remove_active(base, event_to_event_callback(ev)); else if (ev->ev_flags & EVLIST_ACTIVE_LATER) event_queue_remove_active_later(base, event_to_event_callback(ev)); if (ev->ev_flags & EVLIST_INSERTED) { event_queue_remove_inserted(base, ev); if (ev->ev_events & (EV_READ|EV_WRITE|EV_CLOSED)) res = evmap_io_del_(base, ev->ev_fd, ev); else res = evmap_signal_del_(base, (int)ev->ev_fd, ev); if (res == 1) { /* evmap says we need to notify the main thread. */ notify = 1; res = 0; } } /* if we are not in the right thread, we need to wake up the loop */ if (res != -1 && notify && EVBASE_NEED_NOTIFY(base)) evthread_notify_base(base); event_debug_note_del_(ev); return (res); } void event_active(struct event *ev, int res, short ncalls) { if (EVUTIL_FAILURE_CHECK(!ev->ev_base)) { event_warnx("%s: event has no event_base set.", __func__); return; } EVBASE_ACQUIRE_LOCK(ev->ev_base, th_base_lock); event_debug_assert_is_setup_(ev); event_active_nolock_(ev, res, ncalls); EVBASE_RELEASE_LOCK(ev->ev_base, th_base_lock); } void event_active_nolock_(struct event *ev, int res, short ncalls) { struct event_base *base; event_debug(("event_active: %p (fd "EV_SOCK_FMT"), res %d, callback %p", ev, EV_SOCK_ARG(ev->ev_fd), (int)res, ev->ev_callback)); base = ev->ev_base; EVENT_BASE_ASSERT_LOCKED(base); if (ev->ev_flags & EVLIST_FINALIZING) { /* XXXX debug */ return; } switch ((ev->ev_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER))) { default: case EVLIST_ACTIVE|EVLIST_ACTIVE_LATER: EVUTIL_ASSERT(0); break; case EVLIST_ACTIVE: /* We get different kinds of events, add them together */ ev->ev_res |= res; return; case EVLIST_ACTIVE_LATER: ev->ev_res |= res; break; case 0: ev->ev_res = res; break; } if (ev->ev_pri < base->event_running_priority) base->event_continue = 1; if (ev->ev_events & EV_SIGNAL) { #ifndef EVENT__DISABLE_THREAD_SUPPORT if (base->current_event == event_to_event_callback(ev) && !EVBASE_IN_THREAD(base)) { ++base->current_event_waiters; EVTHREAD_COND_WAIT(base->current_event_cond, base->th_base_lock); } #endif ev->ev_ncalls = ncalls; ev->ev_pncalls = NULL; } event_callback_activate_nolock_(base, event_to_event_callback(ev)); } void event_active_later_(struct event *ev, int res) { EVBASE_ACQUIRE_LOCK(ev->ev_base, th_base_lock); event_active_later_nolock_(ev, res); EVBASE_RELEASE_LOCK(ev->ev_base, th_base_lock); } void event_active_later_nolock_(struct event *ev, int res) { struct event_base *base = ev->ev_base; EVENT_BASE_ASSERT_LOCKED(base); if (ev->ev_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER)) { /* We get different kinds of events, add them together */ ev->ev_res |= res; return; } ev->ev_res = res; event_callback_activate_later_nolock_(base, event_to_event_callback(ev)); } int event_callback_activate_(struct event_base *base, struct event_callback *evcb) { int r; EVBASE_ACQUIRE_LOCK(base, th_base_lock); r = event_callback_activate_nolock_(base, evcb); EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } int event_callback_activate_nolock_(struct event_base *base, struct event_callback *evcb) { int r = 1; if (evcb->evcb_flags & EVLIST_FINALIZING) return 0; switch (evcb->evcb_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER)) { default: EVUTIL_ASSERT(0); case EVLIST_ACTIVE_LATER: event_queue_remove_active_later(base, evcb); r = 0; break; case EVLIST_ACTIVE: return 0; case 0: break; } event_queue_insert_active(base, evcb); if (EVBASE_NEED_NOTIFY(base)) evthread_notify_base(base); return r; } int event_callback_activate_later_nolock_(struct event_base *base, struct event_callback *evcb) { if (evcb->evcb_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER)) return 0; event_queue_insert_active_later(base, evcb); if (EVBASE_NEED_NOTIFY(base)) evthread_notify_base(base); return 1; } void event_callback_init_(struct event_base *base, struct event_callback *cb) { memset(cb, 0, sizeof(*cb)); cb->evcb_pri = base->nactivequeues - 1; } int event_callback_cancel_(struct event_base *base, struct event_callback *evcb) { int r; EVBASE_ACQUIRE_LOCK(base, th_base_lock); r = event_callback_cancel_nolock_(base, evcb, 0); EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } int event_callback_cancel_nolock_(struct event_base *base, struct event_callback *evcb, int even_if_finalizing) { if ((evcb->evcb_flags & EVLIST_FINALIZING) && !even_if_finalizing) return 0; if (evcb->evcb_flags & EVLIST_INIT) return event_del_nolock_(event_callback_to_event(evcb), even_if_finalizing ? EVENT_DEL_EVEN_IF_FINALIZING : EVENT_DEL_AUTOBLOCK); switch ((evcb->evcb_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER))) { default: case EVLIST_ACTIVE|EVLIST_ACTIVE_LATER: EVUTIL_ASSERT(0); break; case EVLIST_ACTIVE: /* We get different kinds of events, add them together */ event_queue_remove_active(base, evcb); return 0; case EVLIST_ACTIVE_LATER: event_queue_remove_active_later(base, evcb); break; case 0: break; } return 0; } void event_deferred_cb_init_(struct event_callback *cb, ev_uint8_t priority, deferred_cb_fn fn, void *arg) { memset(cb, 0, sizeof(*cb)); cb->evcb_cb_union.evcb_selfcb = fn; cb->evcb_arg = arg; cb->evcb_pri = priority; cb->evcb_closure = EV_CLOSURE_CB_SELF; } void event_deferred_cb_set_priority_(struct event_callback *cb, ev_uint8_t priority) { cb->evcb_pri = priority; } void event_deferred_cb_cancel_(struct event_base *base, struct event_callback *cb) { if (!base) base = current_base; event_callback_cancel_(base, cb); } #define MAX_DEFERREDS_QUEUED 32 int event_deferred_cb_schedule_(struct event_base *base, struct event_callback *cb) { int r = 1; if (!base) base = current_base; EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (base->n_deferreds_queued > MAX_DEFERREDS_QUEUED) { r = event_callback_activate_later_nolock_(base, cb); } else { r = event_callback_activate_nolock_(base, cb); if (r) { ++base->n_deferreds_queued; } } EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } static int timeout_next(struct event_base *base, struct timeval **tv_p) { /* Caller must hold th_base_lock */ struct timeval now; struct event *ev; struct timeval *tv = *tv_p; int res = 0; ev = min_heap_top_(&base->timeheap); if (ev == NULL) { /* if no time-based events are active wait for I/O */ *tv_p = NULL; goto out; } if (gettime(base, &now) == -1) { res = -1; goto out; } if (evutil_timercmp(&ev->ev_timeout, &now, <=)) { evutil_timerclear(tv); goto out; } evutil_timersub(&ev->ev_timeout, &now, tv); EVUTIL_ASSERT(tv->tv_sec >= 0); EVUTIL_ASSERT(tv->tv_usec >= 0); event_debug(("timeout_next: event: %p, in %d seconds, %d useconds", ev, (int)tv->tv_sec, (int)tv->tv_usec)); out: return (res); } /* Activate every event whose timeout has elapsed. */ static void timeout_process(struct event_base *base) { /* Caller must hold lock. */ struct timeval now; struct event *ev; if (min_heap_empty_(&base->timeheap)) { return; } gettime(base, &now); while ((ev = min_heap_top_(&base->timeheap))) { if (evutil_timercmp(&ev->ev_timeout, &now, >)) break; /* delete this event from the I/O queues */ event_del_nolock_(ev, EVENT_DEL_NOBLOCK); event_debug(("timeout_process: event: %p, call %p", ev, ev->ev_callback)); event_active_nolock_(ev, EV_TIMEOUT, 1); } } #if (EVLIST_INTERNAL >> 4) != 1 #error "Mismatch for value of EVLIST_INTERNAL" #endif #ifndef MAX #define MAX(a,b) (((a)>(b))?(a):(b)) #endif #define MAX_EVENT_COUNT(var, v) var = MAX(var, v) /* These are a fancy way to spell if (flags & EVLIST_INTERNAL) base->event_count--/++; */ #define DECR_EVENT_COUNT(base,flags) \ ((base)->event_count -= (~((flags) >> 4) & 1)) #define INCR_EVENT_COUNT(base,flags) do { \ ((base)->event_count += (~((flags) >> 4) & 1)); \ MAX_EVENT_COUNT((base)->event_count_max, (base)->event_count); \ } while (0) static void event_queue_remove_inserted(struct event_base *base, struct event *ev) { EVENT_BASE_ASSERT_LOCKED(base); if (EVUTIL_FAILURE_CHECK(!(ev->ev_flags & EVLIST_INSERTED))) { event_errx(1, "%s: %p(fd "EV_SOCK_FMT") not on queue %x", __func__, ev, EV_SOCK_ARG(ev->ev_fd), EVLIST_INSERTED); return; } DECR_EVENT_COUNT(base, ev->ev_flags); ev->ev_flags &= ~EVLIST_INSERTED; } static void event_queue_remove_active(struct event_base *base, struct event_callback *evcb) { EVENT_BASE_ASSERT_LOCKED(base); if (EVUTIL_FAILURE_CHECK(!(evcb->evcb_flags & EVLIST_ACTIVE))) { event_errx(1, "%s: %p not on queue %x", __func__, evcb, EVLIST_ACTIVE); return; } DECR_EVENT_COUNT(base, evcb->evcb_flags); evcb->evcb_flags &= ~EVLIST_ACTIVE; base->event_count_active--; TAILQ_REMOVE(&base->activequeues[evcb->evcb_pri], evcb, evcb_active_next); } static void event_queue_remove_active_later(struct event_base *base, struct event_callback *evcb) { EVENT_BASE_ASSERT_LOCKED(base); if (EVUTIL_FAILURE_CHECK(!(evcb->evcb_flags & EVLIST_ACTIVE_LATER))) { event_errx(1, "%s: %p not on queue %x", __func__, evcb, EVLIST_ACTIVE_LATER); return; } DECR_EVENT_COUNT(base, evcb->evcb_flags); evcb->evcb_flags &= ~EVLIST_ACTIVE_LATER; base->event_count_active--; TAILQ_REMOVE(&base->active_later_queue, evcb, evcb_active_next); } static void event_queue_remove_timeout(struct event_base *base, struct event *ev) { EVENT_BASE_ASSERT_LOCKED(base); if (EVUTIL_FAILURE_CHECK(!(ev->ev_flags & EVLIST_TIMEOUT))) { event_errx(1, "%s: %p(fd "EV_SOCK_FMT") not on queue %x", __func__, ev, EV_SOCK_ARG(ev->ev_fd), EVLIST_TIMEOUT); return; } DECR_EVENT_COUNT(base, ev->ev_flags); ev->ev_flags &= ~EVLIST_TIMEOUT; if (is_common_timeout(&ev->ev_timeout, base)) { struct common_timeout_list *ctl = get_common_timeout_list(base, &ev->ev_timeout); TAILQ_REMOVE(&ctl->events, ev, ev_timeout_pos.ev_next_with_common_timeout); } else { min_heap_erase_(&base->timeheap, ev); } } #ifdef USE_REINSERT_TIMEOUT /* Remove and reinsert 'ev' into the timeout queue. */ static void event_queue_reinsert_timeout(struct event_base *base, struct event *ev, int was_common, int is_common, int old_timeout_idx) { struct common_timeout_list *ctl; if (!(ev->ev_flags & EVLIST_TIMEOUT)) { event_queue_insert_timeout(base, ev); return; } switch ((was_common<<1) | is_common) { case 3: /* Changing from one common timeout to another */ ctl = base->common_timeout_queues[old_timeout_idx]; TAILQ_REMOVE(&ctl->events, ev, ev_timeout_pos.ev_next_with_common_timeout); ctl = get_common_timeout_list(base, &ev->ev_timeout); insert_common_timeout_inorder(ctl, ev); break; case 2: /* Was common; is no longer common */ ctl = base->common_timeout_queues[old_timeout_idx]; TAILQ_REMOVE(&ctl->events, ev, ev_timeout_pos.ev_next_with_common_timeout); min_heap_push_(&base->timeheap, ev); break; case 1: /* Wasn't common; has become common. */ min_heap_erase_(&base->timeheap, ev); ctl = get_common_timeout_list(base, &ev->ev_timeout); insert_common_timeout_inorder(ctl, ev); break; case 0: /* was in heap; is still on heap. */ min_heap_adjust_(&base->timeheap, ev); break; default: EVUTIL_ASSERT(0); /* unreachable */ break; } } #endif /* Add 'ev' to the common timeout list in 'ev'. */ static void insert_common_timeout_inorder(struct common_timeout_list *ctl, struct event *ev) { struct event *e; /* By all logic, we should just be able to append 'ev' to the end of * ctl->events, since the timeout on each 'ev' is set to {the common * timeout} + {the time when we add the event}, and so the events * should arrive in order of their timeeouts. But just in case * there's some wacky threading issue going on, we do a search from * the end of 'ev' to find the right insertion point. */ TAILQ_FOREACH_REVERSE(e, &ctl->events, event_list, ev_timeout_pos.ev_next_with_common_timeout) { /* This timercmp is a little sneaky, since both ev and e have * magic values in tv_usec. Fortunately, they ought to have * the _same_ magic values in tv_usec. Let's assert for that. */ EVUTIL_ASSERT( is_same_common_timeout(&e->ev_timeout, &ev->ev_timeout)); if (evutil_timercmp(&ev->ev_timeout, &e->ev_timeout, >=)) { TAILQ_INSERT_AFTER(&ctl->events, e, ev, ev_timeout_pos.ev_next_with_common_timeout); return; } } TAILQ_INSERT_HEAD(&ctl->events, ev, ev_timeout_pos.ev_next_with_common_timeout); } static void event_queue_insert_inserted(struct event_base *base, struct event *ev) { EVENT_BASE_ASSERT_LOCKED(base); if (EVUTIL_FAILURE_CHECK(ev->ev_flags & EVLIST_INSERTED)) { event_errx(1, "%s: %p(fd "EV_SOCK_FMT") already inserted", __func__, ev, EV_SOCK_ARG(ev->ev_fd)); return; } INCR_EVENT_COUNT(base, ev->ev_flags); ev->ev_flags |= EVLIST_INSERTED; } static void event_queue_insert_active(struct event_base *base, struct event_callback *evcb) { EVENT_BASE_ASSERT_LOCKED(base); if (evcb->evcb_flags & EVLIST_ACTIVE) { /* Double insertion is possible for active events */ return; } INCR_EVENT_COUNT(base, evcb->evcb_flags); evcb->evcb_flags |= EVLIST_ACTIVE; base->event_count_active++; MAX_EVENT_COUNT(base->event_count_active_max, base->event_count_active); EVUTIL_ASSERT(evcb->evcb_pri < base->nactivequeues); TAILQ_INSERT_TAIL(&base->activequeues[evcb->evcb_pri], evcb, evcb_active_next); } static void event_queue_insert_active_later(struct event_base *base, struct event_callback *evcb) { EVENT_BASE_ASSERT_LOCKED(base); if (evcb->evcb_flags & (EVLIST_ACTIVE_LATER|EVLIST_ACTIVE)) { /* Double insertion is possible */ return; } INCR_EVENT_COUNT(base, evcb->evcb_flags); evcb->evcb_flags |= EVLIST_ACTIVE_LATER; base->event_count_active++; MAX_EVENT_COUNT(base->event_count_active_max, base->event_count_active); EVUTIL_ASSERT(evcb->evcb_pri < base->nactivequeues); TAILQ_INSERT_TAIL(&base->active_later_queue, evcb, evcb_active_next); } static void event_queue_insert_timeout(struct event_base *base, struct event *ev) { EVENT_BASE_ASSERT_LOCKED(base); if (EVUTIL_FAILURE_CHECK(ev->ev_flags & EVLIST_TIMEOUT)) { event_errx(1, "%s: %p(fd "EV_SOCK_FMT") already on timeout", __func__, ev, EV_SOCK_ARG(ev->ev_fd)); return; } INCR_EVENT_COUNT(base, ev->ev_flags); ev->ev_flags |= EVLIST_TIMEOUT; if (is_common_timeout(&ev->ev_timeout, base)) { struct common_timeout_list *ctl = get_common_timeout_list(base, &ev->ev_timeout); insert_common_timeout_inorder(ctl, ev); } else { min_heap_push_(&base->timeheap, ev); } } static void event_queue_make_later_events_active(struct event_base *base) { struct event_callback *evcb; EVENT_BASE_ASSERT_LOCKED(base); while ((evcb = TAILQ_FIRST(&base->active_later_queue))) { TAILQ_REMOVE(&base->active_later_queue, evcb, evcb_active_next); evcb->evcb_flags = (evcb->evcb_flags & ~EVLIST_ACTIVE_LATER) | EVLIST_ACTIVE; EVUTIL_ASSERT(evcb->evcb_pri < base->nactivequeues); TAILQ_INSERT_TAIL(&base->activequeues[evcb->evcb_pri], evcb, evcb_active_next); base->n_deferreds_queued += (evcb->evcb_closure == EV_CLOSURE_CB_SELF); } } /* Functions for debugging */ const char * event_get_version(void) { return (EVENT__VERSION); } ev_uint32_t event_get_version_number(void) { return (EVENT__NUMERIC_VERSION); } /* * No thread-safe interface needed - the information should be the same * for all threads. */ const char * event_get_method(void) { return (current_base->evsel->name); } #ifndef EVENT__DISABLE_MM_REPLACEMENT static void *(*mm_malloc_fn_)(size_t sz) = NULL; static void *(*mm_realloc_fn_)(void *p, size_t sz) = NULL; static void (*mm_free_fn_)(void *p) = NULL; void * event_mm_malloc_(size_t sz) { if (sz == 0) return NULL; if (mm_malloc_fn_) return mm_malloc_fn_(sz); else return malloc(sz); } void * event_mm_calloc_(size_t count, size_t size) { if (count == 0 || size == 0) return NULL; if (mm_malloc_fn_) { size_t sz = count * size; void *p = NULL; if (count > EV_SIZE_MAX / size) goto error; p = mm_malloc_fn_(sz); if (p) return memset(p, 0, sz); } else { void *p = calloc(count, size); #ifdef _WIN32 /* Windows calloc doesn't reliably set ENOMEM */ if (p == NULL) goto error; #endif return p; } error: errno = ENOMEM; return NULL; } char * event_mm_strdup_(const char *str) { if (!str) { errno = EINVAL; return NULL; } if (mm_malloc_fn_) { size_t ln = strlen(str); void *p = NULL; if (ln == EV_SIZE_MAX) goto error; p = mm_malloc_fn_(ln+1); if (p) return memcpy(p, str, ln+1); } else #ifdef _WIN32 return _strdup(str); #else return strdup(str); #endif error: errno = ENOMEM; return NULL; } void * event_mm_realloc_(void *ptr, size_t sz) { if (mm_realloc_fn_) return mm_realloc_fn_(ptr, sz); else return realloc(ptr, sz); } void event_mm_free_(void *ptr) { if (mm_free_fn_) mm_free_fn_(ptr); else free(ptr); } void event_set_mem_functions(void *(*malloc_fn)(size_t sz), void *(*realloc_fn)(void *ptr, size_t sz), void (*free_fn)(void *ptr)) { mm_malloc_fn_ = malloc_fn; mm_realloc_fn_ = realloc_fn; mm_free_fn_ = free_fn; } #endif #ifdef EVENT__HAVE_EVENTFD static void evthread_notify_drain_eventfd(evutil_socket_t fd, short what, void *arg) { ev_uint64_t msg; ev_ssize_t r; struct event_base *base = arg; r = read(fd, (void*) &msg, sizeof(msg)); if (r<0 && errno != EAGAIN) { event_sock_warn(fd, "Error reading from eventfd"); } EVBASE_ACQUIRE_LOCK(base, th_base_lock); base->is_notify_pending = 0; EVBASE_RELEASE_LOCK(base, th_base_lock); } #endif static void evthread_notify_drain_default(evutil_socket_t fd, short what, void *arg) { unsigned char buf[1024]; struct event_base *base = arg; #ifdef _WIN32 while (recv(fd, (char*)buf, sizeof(buf), 0) > 0) ; #else while (read(fd, (char*)buf, sizeof(buf)) > 0) ; #endif EVBASE_ACQUIRE_LOCK(base, th_base_lock); base->is_notify_pending = 0; EVBASE_RELEASE_LOCK(base, th_base_lock); } int evthread_make_base_notifiable(struct event_base *base) { int r; if (!base) return -1; EVBASE_ACQUIRE_LOCK(base, th_base_lock); r = evthread_make_base_notifiable_nolock_(base); EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } static int evthread_make_base_notifiable_nolock_(struct event_base *base) { void (*cb)(evutil_socket_t, short, void *); int (*notify)(struct event_base *); if (base->th_notify_fn != NULL) { /* The base is already notifiable: we're doing fine. */ return 0; } #if defined(EVENT__HAVE_WORKING_KQUEUE) if (base->evsel == &kqops && event_kq_add_notify_event_(base) == 0) { base->th_notify_fn = event_kq_notify_base_; /* No need to add an event here; the backend can wake * itself up just fine. */ return 0; } #endif #ifdef EVENT__HAVE_EVENTFD base->th_notify_fd[0] = evutil_eventfd_(0, EVUTIL_EFD_CLOEXEC|EVUTIL_EFD_NONBLOCK); if (base->th_notify_fd[0] >= 0) { base->th_notify_fd[1] = -1; notify = evthread_notify_base_eventfd; cb = evthread_notify_drain_eventfd; } else #endif if (evutil_make_internal_pipe_(base->th_notify_fd) == 0) { notify = evthread_notify_base_default; cb = evthread_notify_drain_default; } else { return -1; } base->th_notify_fn = notify; /* prepare an event that we can use for wakeup */ event_assign(&base->th_notify, base, base->th_notify_fd[0], EV_READ|EV_PERSIST, cb, base); /* we need to mark this as internal event */ base->th_notify.ev_flags |= EVLIST_INTERNAL; event_priority_set(&base->th_notify, 0); return event_add_nolock_(&base->th_notify, NULL, 0); } int event_base_foreach_event_nolock_(struct event_base *base, event_base_foreach_event_cb fn, void *arg) { int r, i; unsigned u; struct event *ev; /* Start out with all the EVLIST_INSERTED events. */ if ((r = evmap_foreach_event_(base, fn, arg))) return r; /* Okay, now we deal with those events that have timeouts and are in * the min-heap. */ for (u = 0; u < base->timeheap.n; ++u) { ev = base->timeheap.p[u]; if (ev->ev_flags & EVLIST_INSERTED) { /* we already processed this one */ continue; } if ((r = fn(base, ev, arg))) return r; } /* Now for the events in one of the timeout queues. * the min-heap. */ for (i = 0; i < base->n_common_timeouts; ++i) { struct common_timeout_list *ctl = base->common_timeout_queues[i]; TAILQ_FOREACH(ev, &ctl->events, ev_timeout_pos.ev_next_with_common_timeout) { if (ev->ev_flags & EVLIST_INSERTED) { /* we already processed this one */ continue; } if ((r = fn(base, ev, arg))) return r; } } /* Finally, we deal wit all the active events that we haven't touched * yet. */ for (i = 0; i < base->nactivequeues; ++i) { struct event_callback *evcb; TAILQ_FOREACH(evcb, &base->activequeues[i], evcb_active_next) { if ((evcb->evcb_flags & (EVLIST_INIT|EVLIST_INSERTED|EVLIST_TIMEOUT)) != EVLIST_INIT) { /* This isn't an event (evlist_init clear), or * we already processed it. (inserted or * timeout set */ continue; } ev = event_callback_to_event(evcb); if ((r = fn(base, ev, arg))) return r; } } return 0; } /* Helper for event_base_dump_events: called on each event in the event base; * dumps only the inserted events. */ static int dump_inserted_event_fn(const struct event_base *base, const struct event *e, void *arg) { FILE *output = arg; const char *gloss = (e->ev_events & EV_SIGNAL) ? "sig" : "fd "; if (! (e->ev_flags & (EVLIST_INSERTED|EVLIST_TIMEOUT))) return 0; fprintf(output, " %p [%s "EV_SOCK_FMT"]%s%s%s%s%s%s", (void*)e, gloss, EV_SOCK_ARG(e->ev_fd), (e->ev_events&EV_READ)?" Read":"", (e->ev_events&EV_WRITE)?" Write":"", (e->ev_events&EV_CLOSED)?" EOF":"", (e->ev_events&EV_SIGNAL)?" Signal":"", (e->ev_events&EV_PERSIST)?" Persist":"", (e->ev_flags&EVLIST_INTERNAL)?" Internal":""); if (e->ev_flags & EVLIST_TIMEOUT) { struct timeval tv; tv.tv_sec = e->ev_timeout.tv_sec; tv.tv_usec = e->ev_timeout.tv_usec & MICROSECONDS_MASK; evutil_timeradd(&tv, &base->tv_clock_diff, &tv); fprintf(output, " Timeout=%ld.%06d", (long)tv.tv_sec, (int)(tv.tv_usec & MICROSECONDS_MASK)); } fputc('\n', output); return 0; } /* Helper for event_base_dump_events: called on each event in the event base; * dumps only the active events. */ static int dump_active_event_fn(const struct event_base *base, const struct event *e, void *arg) { FILE *output = arg; const char *gloss = (e->ev_events & EV_SIGNAL) ? "sig" : "fd "; if (! (e->ev_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER))) return 0; fprintf(output, " %p [%s "EV_SOCK_FMT", priority=%d]%s%s%s%s%s active%s%s\n", (void*)e, gloss, EV_SOCK_ARG(e->ev_fd), e->ev_pri, (e->ev_res&EV_READ)?" Read":"", (e->ev_res&EV_WRITE)?" Write":"", (e->ev_res&EV_CLOSED)?" EOF":"", (e->ev_res&EV_SIGNAL)?" Signal":"", (e->ev_res&EV_TIMEOUT)?" Timeout":"", (e->ev_flags&EVLIST_INTERNAL)?" [Internal]":"", (e->ev_flags&EVLIST_ACTIVE_LATER)?" [NextTime]":""); return 0; } int event_base_foreach_event(struct event_base *base, event_base_foreach_event_cb fn, void *arg) { int r; if ((!fn) || (!base)) { return -1; } EVBASE_ACQUIRE_LOCK(base, th_base_lock); r = event_base_foreach_event_nolock_(base, fn, arg); EVBASE_RELEASE_LOCK(base, th_base_lock); return r; } void event_base_dump_events(struct event_base *base, FILE *output) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); fprintf(output, "Inserted events:\n"); event_base_foreach_event_nolock_(base, dump_inserted_event_fn, output); fprintf(output, "Active events:\n"); event_base_foreach_event_nolock_(base, dump_active_event_fn, output); EVBASE_RELEASE_LOCK(base, th_base_lock); } void event_base_active_by_fd(struct event_base *base, evutil_socket_t fd, short events) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); evmap_io_active_(base, fd, events & (EV_READ|EV_WRITE|EV_CLOSED)); EVBASE_RELEASE_LOCK(base, th_base_lock); } void event_base_active_by_signal(struct event_base *base, int sig) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); evmap_signal_active_(base, sig, 1); EVBASE_RELEASE_LOCK(base, th_base_lock); } void event_base_add_virtual_(struct event_base *base) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); base->virtual_event_count++; MAX_EVENT_COUNT(base->virtual_event_count_max, base->virtual_event_count); EVBASE_RELEASE_LOCK(base, th_base_lock); } void event_base_del_virtual_(struct event_base *base) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); EVUTIL_ASSERT(base->virtual_event_count > 0); base->virtual_event_count--; if (base->virtual_event_count == 0 && EVBASE_NEED_NOTIFY(base)) evthread_notify_base(base); EVBASE_RELEASE_LOCK(base, th_base_lock); } static void event_free_debug_globals_locks(void) { #ifndef EVENT__DISABLE_THREAD_SUPPORT #ifndef EVENT__DISABLE_DEBUG_MODE if (event_debug_map_lock_ != NULL) { EVTHREAD_FREE_LOCK(event_debug_map_lock_, 0); event_debug_map_lock_ = NULL; evthreadimpl_disable_lock_debugging_(); } #endif /* EVENT__DISABLE_DEBUG_MODE */ #endif /* EVENT__DISABLE_THREAD_SUPPORT */ return; } static void event_free_debug_globals(void) { event_free_debug_globals_locks(); } static void event_free_evsig_globals(void) { evsig_free_globals_(); } static void event_free_evutil_globals(void) { evutil_free_globals_(); } static void event_free_globals(void) { event_free_debug_globals(); event_free_evsig_globals(); event_free_evutil_globals(); } void libevent_global_shutdown(void) { event_disable_debug_mode(); event_free_globals(); } #ifndef EVENT__DISABLE_THREAD_SUPPORT int event_global_setup_locks_(const int enable_locks) { #ifndef EVENT__DISABLE_DEBUG_MODE EVTHREAD_SETUP_GLOBAL_LOCK(event_debug_map_lock_, 0); #endif if (evsig_global_setup_locks_(enable_locks) < 0) return -1; if (evutil_global_setup_locks_(enable_locks) < 0) return -1; if (evutil_secure_rng_global_setup_locks_(enable_locks) < 0) return -1; return 0; } #endif void event_base_assert_ok_(struct event_base *base) { EVBASE_ACQUIRE_LOCK(base, th_base_lock); event_base_assert_ok_nolock_(base); EVBASE_RELEASE_LOCK(base, th_base_lock); } void event_base_assert_ok_nolock_(struct event_base *base) { int i; int count; /* First do checks on the per-fd and per-signal lists */ evmap_check_integrity_(base); /* Check the heap property */ for (i = 1; i < (int)base->timeheap.n; ++i) { int parent = (i - 1) / 2; struct event *ev, *p_ev; ev = base->timeheap.p[i]; p_ev = base->timeheap.p[parent]; EVUTIL_ASSERT(ev->ev_flags & EVLIST_TIMEOUT); EVUTIL_ASSERT(evutil_timercmp(&p_ev->ev_timeout, &ev->ev_timeout, <=)); EVUTIL_ASSERT(ev->ev_timeout_pos.min_heap_idx == i); } /* Check that the common timeouts are fine */ for (i = 0; i < base->n_common_timeouts; ++i) { struct common_timeout_list *ctl = base->common_timeout_queues[i]; struct event *last=NULL, *ev; EVUTIL_ASSERT_TAILQ_OK(&ctl->events, event, ev_timeout_pos.ev_next_with_common_timeout); TAILQ_FOREACH(ev, &ctl->events, ev_timeout_pos.ev_next_with_common_timeout) { if (last) EVUTIL_ASSERT(evutil_timercmp(&last->ev_timeout, &ev->ev_timeout, <=)); EVUTIL_ASSERT(ev->ev_flags & EVLIST_TIMEOUT); EVUTIL_ASSERT(is_common_timeout(&ev->ev_timeout,base)); EVUTIL_ASSERT(COMMON_TIMEOUT_IDX(&ev->ev_timeout) == i); last = ev; } } /* Check the active queues. */ count = 0; for (i = 0; i < base->nactivequeues; ++i) { struct event_callback *evcb; EVUTIL_ASSERT_TAILQ_OK(&base->activequeues[i], event_callback, evcb_active_next); TAILQ_FOREACH(evcb, &base->activequeues[i], evcb_active_next) { EVUTIL_ASSERT((evcb->evcb_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER)) == EVLIST_ACTIVE); EVUTIL_ASSERT(evcb->evcb_pri == i); ++count; } } { struct event_callback *evcb; TAILQ_FOREACH(evcb, &base->active_later_queue, evcb_active_next) { EVUTIL_ASSERT((evcb->evcb_flags & (EVLIST_ACTIVE|EVLIST_ACTIVE_LATER)) == EVLIST_ACTIVE_LATER); ++count; } } EVUTIL_ASSERT(count == base->event_count_active); } �������������kytensor-2.49.0.7/third_party/libevent/event_iocp.c�������������������������������������������������0000664�0000000�0000000�00000017023�15270523712�0022300�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2009-2012 Niels Provos, Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "evconfig-private.h" #ifndef _WIN32_WINNT /* Minimum required for InitializeCriticalSectionAndSpinCount */ #define _WIN32_WINNT 0x0403 #endif #include <winsock2.h> #include <windows.h> #include <process.h> #include <stdio.h> #include <mswsock.h> #include "event2/util.h" #include "util-internal.h" #include "iocp-internal.h" #include "log-internal.h" #include "mm-internal.h" #include "event-internal.h" #include "evthread-internal.h" #define NOTIFICATION_KEY ((ULONG_PTR)-1) void event_overlapped_init_(struct event_overlapped *o, iocp_callback cb) { memset(o, 0, sizeof(struct event_overlapped)); o->cb = cb; } static void handle_entry(OVERLAPPED *o, ULONG_PTR completion_key, DWORD nBytes, int ok) { struct event_overlapped *eo = EVUTIL_UPCAST(o, struct event_overlapped, overlapped); eo->cb(eo, completion_key, nBytes, ok); } static void loop(void *port_) { struct event_iocp_port *port = port_; long ms = port->ms; HANDLE p = port->port; if (ms <= 0) ms = INFINITE; while (1) { OVERLAPPED *overlapped=NULL; ULONG_PTR key=0; DWORD bytes=0; int ok = GetQueuedCompletionStatus(p, &bytes, &key, &overlapped, ms); EnterCriticalSection(&port->lock); if (port->shutdown) { if (--port->n_live_threads == 0) ReleaseSemaphore(port->shutdownSemaphore, 1, NULL); LeaveCriticalSection(&port->lock); return; } LeaveCriticalSection(&port->lock); if (key != NOTIFICATION_KEY && overlapped) handle_entry(overlapped, key, bytes, ok); else if (!overlapped) break; } event_warnx("GetQueuedCompletionStatus exited with no event."); EnterCriticalSection(&port->lock); if (--port->n_live_threads == 0) ReleaseSemaphore(port->shutdownSemaphore, 1, NULL); LeaveCriticalSection(&port->lock); } int event_iocp_port_associate_(struct event_iocp_port *port, evutil_socket_t fd, ev_uintptr_t key) { HANDLE h; h = CreateIoCompletionPort((HANDLE)fd, port->port, key, port->n_threads); if (!h) return -1; return 0; } static void * get_extension_function(SOCKET s, const GUID *which_fn) { void *ptr = NULL; DWORD bytes=0; WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, (GUID*)which_fn, sizeof(*which_fn), &ptr, sizeof(ptr), &bytes, NULL, NULL); /* No need to detect errors here: if ptr is set, then we have a good function pointer. Otherwise, we should behave as if we had no function pointer. */ return ptr; } /* Mingw doesn't have these in its mswsock.h. The values are copied from wine.h. Perhaps if we copy them exactly, the cargo will come again. */ #ifndef WSAID_ACCEPTEX #define WSAID_ACCEPTEX \ {0xb5367df1,0xcbac,0x11cf,{0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92}} #endif #ifndef WSAID_CONNECTEX #define WSAID_CONNECTEX \ {0x25a207b9,0xddf3,0x4660,{0x8e,0xe9,0x76,0xe5,0x8c,0x74,0x06,0x3e}} #endif #ifndef WSAID_GETACCEPTEXSOCKADDRS #define WSAID_GETACCEPTEXSOCKADDRS \ {0xb5367df2,0xcbac,0x11cf,{0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92}} #endif static int extension_fns_initialized = 0; static void init_extension_functions(struct win32_extension_fns *ext) { const GUID acceptex = WSAID_ACCEPTEX; const GUID connectex = WSAID_CONNECTEX; const GUID getacceptexsockaddrs = WSAID_GETACCEPTEXSOCKADDRS; SOCKET s = socket(AF_INET, SOCK_STREAM, 0); if (s == INVALID_SOCKET) return; ext->AcceptEx = get_extension_function(s, &acceptex); ext->ConnectEx = get_extension_function(s, &connectex); ext->GetAcceptExSockaddrs = get_extension_function(s, &getacceptexsockaddrs); closesocket(s); extension_fns_initialized = 1; } static struct win32_extension_fns the_extension_fns; const struct win32_extension_fns * event_get_win32_extension_fns_(void) { return &the_extension_fns; } #define N_CPUS_DEFAULT 2 struct event_iocp_port * event_iocp_port_launch_(int n_cpus) { struct event_iocp_port *port; int i; if (!extension_fns_initialized) init_extension_functions(&the_extension_fns); if (!(port = mm_calloc(1, sizeof(struct event_iocp_port)))) return NULL; if (n_cpus <= 0) n_cpus = N_CPUS_DEFAULT; port->n_threads = n_cpus * 2; port->threads = mm_calloc(port->n_threads, sizeof(HANDLE)); if (!port->threads) goto err; port->port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, n_cpus); port->ms = -1; if (!port->port) goto err; port->shutdownSemaphore = CreateSemaphore(NULL, 0, 1, NULL); if (!port->shutdownSemaphore) goto err; for (i=0; i<port->n_threads; ++i) { ev_uintptr_t th = _beginthread(loop, 0, port); if (th == (ev_uintptr_t)-1) goto err; port->threads[i] = (HANDLE)th; ++port->n_live_threads; } InitializeCriticalSectionAndSpinCount(&port->lock, 1000); return port; err: if (port->port) CloseHandle(port->port); if (port->threads) mm_free(port->threads); if (port->shutdownSemaphore) CloseHandle(port->shutdownSemaphore); mm_free(port); return NULL; } static void event_iocp_port_unlock_and_free_(struct event_iocp_port *port) { DeleteCriticalSection(&port->lock); CloseHandle(port->port); CloseHandle(port->shutdownSemaphore); mm_free(port->threads); mm_free(port); } static int event_iocp_notify_all(struct event_iocp_port *port) { int i, r, ok=1; for (i=0; i<port->n_threads; ++i) { r = PostQueuedCompletionStatus(port->port, 0, NOTIFICATION_KEY, NULL); if (!r) ok = 0; } return ok ? 0 : -1; } int event_iocp_shutdown_(struct event_iocp_port *port, long waitMsec) { DWORD ms = INFINITE; int n; EnterCriticalSection(&port->lock); port->shutdown = 1; LeaveCriticalSection(&port->lock); event_iocp_notify_all(port); if (waitMsec >= 0) ms = waitMsec; WaitForSingleObject(port->shutdownSemaphore, ms); EnterCriticalSection(&port->lock); n = port->n_live_threads; LeaveCriticalSection(&port->lock); if (n == 0) { event_iocp_port_unlock_and_free_(port); return 0; } else { return -1; } } int event_iocp_activate_overlapped_( struct event_iocp_port *port, struct event_overlapped *o, ev_uintptr_t key, ev_uint32_t n) { BOOL r; r = PostQueuedCompletionStatus(port->port, n, key, &o->overlapped); return (r==0) ? -1 : 0; } struct event_iocp_port * event_base_get_iocp_(struct event_base *base) { #ifdef _WIN32 return base->iocp; #else return NULL; #endif } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/event_rpcgen.py����������������������������������������������0000775�0000000�0000000�00000154523�15270523712�0023044�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/usr/bin/env python2 # # Copyright (c) 2005-2007 Niels Provos <provos@citi.umich.edu> # Copyright (c) 2007-2012 Niels Provos and Nick Mathewson # All rights reserved. # # Generates marshaling code based on libevent. # TODO: # 1) use optparse to allow the strategy shell to parse options, and # to allow the instantiated factory (for the specific output language) # to parse remaining options # 2) move the globals into a class that manages execution (including the # progress outputs that space stderr at the moment) # 3) emit other languages import sys import re _NAME = "event_rpcgen.py" _VERSION = "0.1" # Globals line_count = 0 white = re.compile(r'\s+') cppcomment = re.compile(r'\/\/.*$') nonident = re.compile(r'[^a-zA-Z0-9_]') structref = re.compile(r'^struct\[([a-zA-Z_][a-zA-Z0-9_]*)\]$') structdef = re.compile(r'^struct +[a-zA-Z_][a-zA-Z0-9_]* *{$') headerdirect = [] cppdirect = [] QUIETLY = 0 def declare(s): if not QUIETLY: print s def TranslateList(mylist, mydict): return map(lambda x: x % mydict, mylist) # Exception class for parse errors class RpcGenError(Exception): def __init__(self, why): self.why = why def __str__(self): return str(self.why) # Holds everything that makes a struct class Struct: def __init__(self, name): self._name = name self._entries = [] self._tags = {} declare(' Created struct: %s' % name) def AddEntry(self, entry): if self._tags.has_key(entry.Tag()): raise RpcGenError( 'Entry "%s" duplicates tag number %d from "%s" ' 'around line %d' % (entry.Name(), entry.Tag(), self._tags[entry.Tag()], line_count)) self._entries.append(entry) self._tags[entry.Tag()] = entry.Name() declare(' Added entry: %s' % entry.Name()) def Name(self): return self._name def EntryTagName(self, entry): """Creates the name inside an enumeration for distinguishing data types.""" name = "%s_%s" % (self._name, entry.Name()) return name.upper() def PrintIndented(self, file, ident, code): """Takes an array, add indentation to each entry and prints it.""" for entry in code: print >>file, '%s%s' % (ident, entry) class StructCCode(Struct): """ Knows how to generate C code for a struct """ def __init__(self, name): Struct.__init__(self, name) def PrintTags(self, file): """Prints the tag definitions for a structure.""" print >>file, '/* Tag definition for %s */' % self._name print >>file, 'enum %s_ {' % self._name.lower() for entry in self._entries: print >>file, ' %s=%d,' % (self.EntryTagName(entry), entry.Tag()) print >>file, ' %s_MAX_TAGS' % (self._name.upper()) print >>file, '};\n' def PrintForwardDeclaration(self, file): print >>file, 'struct %s;' % self._name def PrintDeclaration(self, file): print >>file, '/* Structure declaration for %s */' % self._name print >>file, 'struct %s_access_ {' % self._name for entry in self._entries: dcl = entry.AssignDeclaration('(*%s_assign)' % entry.Name()) dcl.extend( entry.GetDeclaration('(*%s_get)' % entry.Name())) if entry.Array(): dcl.extend( entry.AddDeclaration('(*%s_add)' % entry.Name())) self.PrintIndented(file, ' ', dcl) print >>file, '};\n' print >>file, 'struct %s {' % self._name print >>file, ' struct %s_access_ *base;\n' % self._name for entry in self._entries: dcl = entry.Declaration() self.PrintIndented(file, ' ', dcl) print >>file, '' for entry in self._entries: print >>file, ' ev_uint8_t %s_set;' % entry.Name() print >>file, '};\n' print >>file, \ """struct %(name)s *%(name)s_new(void); struct %(name)s *%(name)s_new_with_arg(void *); void %(name)s_free(struct %(name)s *); void %(name)s_clear(struct %(name)s *); void %(name)s_marshal(struct evbuffer *, const struct %(name)s *); int %(name)s_unmarshal(struct %(name)s *, struct evbuffer *); int %(name)s_complete(struct %(name)s *); void evtag_marshal_%(name)s(struct evbuffer *, ev_uint32_t, const struct %(name)s *); int evtag_unmarshal_%(name)s(struct evbuffer *, ev_uint32_t, struct %(name)s *);""" % { 'name' : self._name } # Write a setting function of every variable for entry in self._entries: self.PrintIndented(file, '', entry.AssignDeclaration( entry.AssignFuncName())) self.PrintIndented(file, '', entry.GetDeclaration( entry.GetFuncName())) if entry.Array(): self.PrintIndented(file, '', entry.AddDeclaration( entry.AddFuncName())) print >>file, '/* --- %s done --- */\n' % self._name def PrintCode(self, file): print >>file, ('/*\n' ' * Implementation of %s\n' ' */\n') % self._name print >>file, \ 'static struct %(name)s_access_ %(name)s_base__ = {' % \ { 'name' : self._name } for entry in self._entries: self.PrintIndented(file, ' ', entry.CodeBase()) print >>file, '};\n' # Creation print >>file, ( 'struct %(name)s *\n' '%(name)s_new(void)\n' '{\n' ' return %(name)s_new_with_arg(NULL);\n' '}\n' '\n' 'struct %(name)s *\n' '%(name)s_new_with_arg(void *unused)\n' '{\n' ' struct %(name)s *tmp;\n' ' if ((tmp = malloc(sizeof(struct %(name)s))) == NULL) {\n' ' event_warn("%%s: malloc", __func__);\n' ' return (NULL);\n' ' }\n' ' tmp->base = &%(name)s_base__;\n') % { 'name' : self._name } for entry in self._entries: self.PrintIndented(file, ' ', entry.CodeInitialize('tmp')) print >>file, ' tmp->%s_set = 0;\n' % entry.Name() print >>file, ( ' return (tmp);\n' '}\n') # Adding for entry in self._entries: if entry.Array(): self.PrintIndented(file, '', entry.CodeAdd()) print >>file, '' # Assigning for entry in self._entries: self.PrintIndented(file, '', entry.CodeAssign()) print >>file, '' # Getting for entry in self._entries: self.PrintIndented(file, '', entry.CodeGet()) print >>file, '' # Clearing print >>file, ( 'void\n' '%(name)s_clear(struct %(name)s *tmp)\n' '{' ) % { 'name' : self._name } for entry in self._entries: self.PrintIndented(file, ' ', entry.CodeClear('tmp')) print >>file, '}\n' # Freeing print >>file, ( 'void\n' '%(name)s_free(struct %(name)s *tmp)\n' '{' ) % { 'name' : self._name } for entry in self._entries: self.PrintIndented(file, ' ', entry.CodeFree('tmp')) print >>file, (' free(tmp);\n' '}\n') # Marshaling print >>file, ('void\n' '%(name)s_marshal(struct evbuffer *evbuf, ' 'const struct %(name)s *tmp)' '{') % { 'name' : self._name } for entry in self._entries: indent = ' ' # Optional entries do not have to be set if entry.Optional(): indent += ' ' print >>file, ' if (tmp->%s_set) {' % entry.Name() self.PrintIndented( file, indent, entry.CodeMarshal('evbuf', self.EntryTagName(entry), entry.GetVarName('tmp'), entry.GetVarLen('tmp'))) if entry.Optional(): print >>file, ' }' print >>file, '}\n' # Unmarshaling print >>file, ('int\n' '%(name)s_unmarshal(struct %(name)s *tmp, ' ' struct evbuffer *evbuf)\n' '{\n' ' ev_uint32_t tag;\n' ' while (evbuffer_get_length(evbuf) > 0) {\n' ' if (evtag_peek(evbuf, &tag) == -1)\n' ' return (-1);\n' ' switch (tag) {\n' ) % { 'name' : self._name } for entry in self._entries: print >>file, ' case %s:\n' % self.EntryTagName(entry) if not entry.Array(): print >>file, ( ' if (tmp->%s_set)\n' ' return (-1);' ) % (entry.Name()) self.PrintIndented( file, ' ', entry.CodeUnmarshal('evbuf', self.EntryTagName(entry), entry.GetVarName('tmp'), entry.GetVarLen('tmp'))) print >>file, ( ' tmp->%s_set = 1;\n' % entry.Name() + ' break;\n' ) print >>file, ( ' default:\n' ' return -1;\n' ' }\n' ' }\n' ) # Check if it was decoded completely print >>file, ( ' if (%(name)s_complete(tmp) == -1)\n' ' return (-1);' ) % { 'name' : self._name } # Successfully decoded print >>file, ( ' return (0);\n' '}\n') # Checking if a structure has all the required data print >>file, ( 'int\n' '%(name)s_complete(struct %(name)s *msg)\n' '{' ) % { 'name' : self._name } for entry in self._entries: if not entry.Optional(): code = [ 'if (!msg->%(name)s_set)', ' return (-1);' ] code = TranslateList(code, entry.GetTranslation()) self.PrintIndented( file, ' ', code) self.PrintIndented( file, ' ', entry.CodeComplete('msg', entry.GetVarName('msg'))) print >>file, ( ' return (0);\n' '}\n' ) # Complete message unmarshaling print >>file, ( 'int\n' 'evtag_unmarshal_%(name)s(struct evbuffer *evbuf, ' 'ev_uint32_t need_tag, struct %(name)s *msg)\n' '{\n' ' ev_uint32_t tag;\n' ' int res = -1;\n' '\n' ' struct evbuffer *tmp = evbuffer_new();\n' '\n' ' if (evtag_unmarshal(evbuf, &tag, tmp) == -1' ' || tag != need_tag)\n' ' goto error;\n' '\n' ' if (%(name)s_unmarshal(msg, tmp) == -1)\n' ' goto error;\n' '\n' ' res = 0;\n' '\n' ' error:\n' ' evbuffer_free(tmp);\n' ' return (res);\n' '}\n' ) % { 'name' : self._name } # Complete message marshaling print >>file, ( 'void\n' 'evtag_marshal_%(name)s(struct evbuffer *evbuf, ev_uint32_t tag, ' 'const struct %(name)s *msg)\n' '{\n' ' struct evbuffer *buf_ = evbuffer_new();\n' ' assert(buf_ != NULL);\n' ' %(name)s_marshal(buf_, msg);\n' ' evtag_marshal_buffer(evbuf, tag, buf_);\n ' ' evbuffer_free(buf_);\n' '}\n' ) % { 'name' : self._name } class Entry: def __init__(self, type, name, tag): self._type = type self._name = name self._tag = int(tag) self._ctype = type self._optional = 0 self._can_be_array = 0 self._array = 0 self._line_count = -1 self._struct = None self._refname = None self._optpointer = True self._optaddarg = True def GetInitializer(self): assert 0, "Entry does not provide initializer" def SetStruct(self, struct): self._struct = struct def LineCount(self): assert self._line_count != -1 return self._line_count def SetLineCount(self, number): self._line_count = number def Array(self): return self._array def Optional(self): return self._optional def Tag(self): return self._tag def Name(self): return self._name def Type(self): return self._type def MakeArray(self, yes=1): self._array = yes def MakeOptional(self): self._optional = 1 def Verify(self): if self.Array() and not self._can_be_array: raise RpcGenError( 'Entry "%s" cannot be created as an array ' 'around line %d' % (self._name, self.LineCount())) if not self._struct: raise RpcGenError( 'Entry "%s" does not know which struct it belongs to ' 'around line %d' % (self._name, self.LineCount())) if self._optional and self._array: raise RpcGenError( 'Entry "%s" has illegal combination of optional and array ' 'around line %d' % (self._name, self.LineCount())) def GetTranslation(self, extradict = {}): mapping = { "parent_name" : self._struct.Name(), "name" : self._name, "ctype" : self._ctype, "refname" : self._refname, "optpointer" : self._optpointer and "*" or "", "optreference" : self._optpointer and "&" or "", "optaddarg" : self._optaddarg and ", const %s value" % self._ctype or "" } for (k, v) in extradict.items(): mapping[k] = v return mapping def GetVarName(self, var): return '%(var)s->%(name)s_data' % self.GetTranslation({ 'var' : var }) def GetVarLen(self, var): return 'sizeof(%s)' % self._ctype def GetFuncName(self): return '%s_%s_get' % (self._struct.Name(), self._name) def GetDeclaration(self, funcname): code = [ 'int %s(struct %s *, %s *);' % ( funcname, self._struct.Name(), self._ctype ) ] return code def CodeGet(self): code = ( 'int', '%(parent_name)s_%(name)s_get(struct %(parent_name)s *msg, ' '%(ctype)s *value)', '{', ' if (msg->%(name)s_set != 1)', ' return (-1);', ' *value = msg->%(name)s_data;', ' return (0);', '}' ) code = '\n'.join(code) code = code % self.GetTranslation() return code.split('\n') def AssignFuncName(self): return '%s_%s_assign' % (self._struct.Name(), self._name) def AddFuncName(self): return '%s_%s_add' % (self._struct.Name(), self._name) def AssignDeclaration(self, funcname): code = [ 'int %s(struct %s *, const %s);' % ( funcname, self._struct.Name(), self._ctype ) ] return code def CodeAssign(self): code = [ 'int', '%(parent_name)s_%(name)s_assign(struct %(parent_name)s *msg,' ' const %(ctype)s value)', '{', ' msg->%(name)s_set = 1;', ' msg->%(name)s_data = value;', ' return (0);', '}' ] code = '\n'.join(code) code = code % self.GetTranslation() return code.split('\n') def CodeClear(self, structname): code = [ '%s->%s_set = 0;' % (structname, self.Name()) ] return code def CodeComplete(self, structname, var_name): return [] def CodeFree(self, name): return [] def CodeBase(self): code = [ '%(parent_name)s_%(name)s_assign,', '%(parent_name)s_%(name)s_get,' ] if self.Array(): code.append('%(parent_name)s_%(name)s_add,') code = '\n'.join(code) code = code % self.GetTranslation() return code.split('\n') class EntryBytes(Entry): def __init__(self, type, name, tag, length): # Init base class Entry.__init__(self, type, name, tag) self._length = length self._ctype = 'ev_uint8_t' def GetInitializer(self): return "NULL" def GetVarLen(self, var): return '(%s)' % self._length def CodeArrayAdd(self, varname, value): # XXX: copy here return [ '%(varname)s = NULL;' % { 'varname' : varname } ] def GetDeclaration(self, funcname): code = [ 'int %s(struct %s *, %s **);' % ( funcname, self._struct.Name(), self._ctype ) ] return code def AssignDeclaration(self, funcname): code = [ 'int %s(struct %s *, const %s *);' % ( funcname, self._struct.Name(), self._ctype ) ] return code def Declaration(self): dcl = ['ev_uint8_t %s_data[%s];' % (self._name, self._length)] return dcl def CodeGet(self): name = self._name code = [ 'int', '%s_%s_get(struct %s *msg, %s **value)' % ( self._struct.Name(), name, self._struct.Name(), self._ctype), '{', ' if (msg->%s_set != 1)' % name, ' return (-1);', ' *value = msg->%s_data;' % name, ' return (0);', '}' ] return code def CodeAssign(self): name = self._name code = [ 'int', '%s_%s_assign(struct %s *msg, const %s *value)' % ( self._struct.Name(), name, self._struct.Name(), self._ctype), '{', ' msg->%s_set = 1;' % name, ' memcpy(msg->%s_data, value, %s);' % ( name, self._length), ' return (0);', '}' ] return code def CodeUnmarshal(self, buf, tag_name, var_name, var_len): code = [ 'if (evtag_unmarshal_fixed(%(buf)s, %(tag)s, ' '%(var)s, %(varlen)s) == -1) {', ' event_warnx("%%s: failed to unmarshal %(name)s", __func__);', ' return (-1);', '}' ] return TranslateList(code, self.GetTranslation({ 'var' : var_name, 'varlen' : var_len, 'buf' : buf, 'tag' : tag_name })) def CodeMarshal(self, buf, tag_name, var_name, var_len): code = ['evtag_marshal(%s, %s, %s, %s);' % ( buf, tag_name, var_name, var_len)] return code def CodeClear(self, structname): code = [ '%s->%s_set = 0;' % (structname, self.Name()), 'memset(%s->%s_data, 0, sizeof(%s->%s_data));' % ( structname, self._name, structname, self._name)] return code def CodeInitialize(self, name): code = ['memset(%s->%s_data, 0, sizeof(%s->%s_data));' % ( name, self._name, name, self._name)] return code def Verify(self): if not self._length: raise RpcGenError( 'Entry "%s" needs a length ' 'around line %d' % (self._name, self.LineCount())) Entry.Verify(self) class EntryInt(Entry): def __init__(self, type, name, tag, bits=32): # Init base class Entry.__init__(self, type, name, tag) self._can_be_array = 1 if bits == 32: self._ctype = 'ev_uint32_t' self._marshal_type = 'int' if bits == 64: self._ctype = 'ev_uint64_t' self._marshal_type = 'int64' def GetInitializer(self): return "0" def CodeArrayFree(self, var): return [] def CodeArrayAssign(self, varname, srcvar): return [ '%(varname)s = %(srcvar)s;' % { 'varname' : varname, 'srcvar' : srcvar } ] def CodeArrayAdd(self, varname, value): """Returns a new entry of this type.""" return [ '%(varname)s = %(value)s;' % { 'varname' : varname, 'value' : value } ] def CodeUnmarshal(self, buf, tag_name, var_name, var_len): code = [ 'if (evtag_unmarshal_%(ma)s(%(buf)s, %(tag)s, &%(var)s) == -1) {', ' event_warnx("%%s: failed to unmarshal %(name)s", __func__);', ' return (-1);', '}' ] code = '\n'.join(code) % self.GetTranslation({ 'ma' : self._marshal_type, 'buf' : buf, 'tag' : tag_name, 'var' : var_name }) return code.split('\n') def CodeMarshal(self, buf, tag_name, var_name, var_len): code = [ 'evtag_marshal_%s(%s, %s, %s);' % ( self._marshal_type, buf, tag_name, var_name)] return code def Declaration(self): dcl = ['%s %s_data;' % (self._ctype, self._name)] return dcl def CodeInitialize(self, name): code = ['%s->%s_data = 0;' % (name, self._name)] return code class EntryString(Entry): def __init__(self, type, name, tag): # Init base class Entry.__init__(self, type, name, tag) self._can_be_array = 1 self._ctype = 'char *' def GetInitializer(self): return "NULL" def CodeArrayFree(self, varname): code = [ 'if (%(var)s != NULL) free(%(var)s);' ] return TranslateList(code, { 'var' : varname }) def CodeArrayAssign(self, varname, srcvar): code = [ 'if (%(var)s != NULL)', ' free(%(var)s);', '%(var)s = strdup(%(srcvar)s);', 'if (%(var)s == NULL) {', ' event_warnx("%%s: strdup", __func__);', ' return (-1);', '}' ] return TranslateList(code, { 'var' : varname, 'srcvar' : srcvar }) def CodeArrayAdd(self, varname, value): code = [ 'if (%(value)s != NULL) {', ' %(var)s = strdup(%(value)s);', ' if (%(var)s == NULL) {', ' goto error;', ' }', '} else {', ' %(var)s = NULL;', '}' ] return TranslateList(code, { 'var' : varname, 'value' : value }) def GetVarLen(self, var): return 'strlen(%s)' % self.GetVarName(var) def CodeMakeInitalize(self, varname): return '%(varname)s = NULL;' % { 'varname' : varname } def CodeAssign(self): name = self._name code = """int %(parent_name)s_%(name)s_assign(struct %(parent_name)s *msg, const %(ctype)s value) { if (msg->%(name)s_data != NULL) free(msg->%(name)s_data); if ((msg->%(name)s_data = strdup(value)) == NULL) return (-1); msg->%(name)s_set = 1; return (0); }""" % self.GetTranslation() return code.split('\n') def CodeUnmarshal(self, buf, tag_name, var_name, var_len): code = ['if (evtag_unmarshal_string(%(buf)s, %(tag)s, &%(var)s) == -1) {', ' event_warnx("%%s: failed to unmarshal %(name)s", __func__);', ' return (-1);', '}' ] code = '\n'.join(code) % self.GetTranslation({ 'buf' : buf, 'tag' : tag_name, 'var' : var_name }) return code.split('\n') def CodeMarshal(self, buf, tag_name, var_name, var_len): code = ['evtag_marshal_string(%s, %s, %s);' % ( buf, tag_name, var_name)] return code def CodeClear(self, structname): code = [ 'if (%s->%s_set == 1) {' % (structname, self.Name()), ' free(%s->%s_data);' % (structname, self.Name()), ' %s->%s_data = NULL;' % (structname, self.Name()), ' %s->%s_set = 0;' % (structname, self.Name()), '}' ] return code def CodeInitialize(self, name): code = ['%s->%s_data = NULL;' % (name, self._name)] return code def CodeFree(self, name): code = ['if (%s->%s_data != NULL)' % (name, self._name), ' free (%s->%s_data);' % (name, self._name)] return code def Declaration(self): dcl = ['char *%s_data;' % self._name] return dcl class EntryStruct(Entry): def __init__(self, type, name, tag, refname): # Init base class Entry.__init__(self, type, name, tag) self._optpointer = False self._can_be_array = 1 self._refname = refname self._ctype = 'struct %s*' % refname self._optaddarg = False def GetInitializer(self): return "NULL" def GetVarLen(self, var): return '-1' def CodeArrayAdd(self, varname, value): code = [ '%(varname)s = %(refname)s_new();', 'if (%(varname)s == NULL)', ' goto error;' ] return TranslateList(code, self.GetTranslation({ 'varname' : varname })) def CodeArrayFree(self, var): code = [ '%(refname)s_free(%(var)s);' % self.GetTranslation( { 'var' : var }) ] return code def CodeArrayAssign(self, var, srcvar): code = [ 'int had_error = 0;', 'struct evbuffer *tmp = NULL;', '%(refname)s_clear(%(var)s);', 'if ((tmp = evbuffer_new()) == NULL) {', ' event_warn("%%s: evbuffer_new()", __func__);', ' had_error = 1;', ' goto done;', '}', '%(refname)s_marshal(tmp, %(srcvar)s);', 'if (%(refname)s_unmarshal(%(var)s, tmp) == -1) {', ' event_warnx("%%s: %(refname)s_unmarshal", __func__);', ' had_error = 1;', ' goto done;', '}', 'done:' 'if (tmp != NULL)', ' evbuffer_free(tmp);', 'if (had_error) {', ' %(refname)s_clear(%(var)s);', ' return (-1);', '}' ] return TranslateList(code, self.GetTranslation({ 'var' : var, 'srcvar' : srcvar})) def CodeGet(self): name = self._name code = [ 'int', '%s_%s_get(struct %s *msg, %s *value)' % ( self._struct.Name(), name, self._struct.Name(), self._ctype), '{', ' if (msg->%s_set != 1) {' % name, ' msg->%s_data = %s_new();' % (name, self._refname), ' if (msg->%s_data == NULL)' % name, ' return (-1);', ' msg->%s_set = 1;' % name, ' }', ' *value = msg->%s_data;' % name, ' return (0);', '}' ] return code def CodeAssign(self): name = self._name code = """int %(parent_name)s_%(name)s_assign(struct %(parent_name)s *msg, const %(ctype)s value) { struct evbuffer *tmp = NULL; if (msg->%(name)s_set) { %(refname)s_clear(msg->%(name)s_data); msg->%(name)s_set = 0; } else { msg->%(name)s_data = %(refname)s_new(); if (msg->%(name)s_data == NULL) { event_warn("%%s: %(refname)s_new()", __func__); goto error; } } if ((tmp = evbuffer_new()) == NULL) { event_warn("%%s: evbuffer_new()", __func__); goto error; } %(refname)s_marshal(tmp, value); if (%(refname)s_unmarshal(msg->%(name)s_data, tmp) == -1) { event_warnx("%%s: %(refname)s_unmarshal", __func__); goto error; } msg->%(name)s_set = 1; evbuffer_free(tmp); return (0); error: if (tmp != NULL) evbuffer_free(tmp); if (msg->%(name)s_data != NULL) { %(refname)s_free(msg->%(name)s_data); msg->%(name)s_data = NULL; } return (-1); }""" % self.GetTranslation() return code.split('\n') def CodeComplete(self, structname, var_name): code = [ 'if (%(structname)s->%(name)s_set && ' '%(refname)s_complete(%(var)s) == -1)', ' return (-1);' ] return TranslateList(code, self.GetTranslation({ 'structname' : structname, 'var' : var_name })) def CodeUnmarshal(self, buf, tag_name, var_name, var_len): code = ['%(var)s = %(refname)s_new();', 'if (%(var)s == NULL)', ' return (-1);', 'if (evtag_unmarshal_%(refname)s(%(buf)s, %(tag)s, ' '%(var)s) == -1) {', ' event_warnx("%%s: failed to unmarshal %(name)s", __func__);', ' return (-1);', '}' ] code = '\n'.join(code) % self.GetTranslation({ 'buf' : buf, 'tag' : tag_name, 'var' : var_name }) return code.split('\n') def CodeMarshal(self, buf, tag_name, var_name, var_len): code = ['evtag_marshal_%s(%s, %s, %s);' % ( self._refname, buf, tag_name, var_name)] return code def CodeClear(self, structname): code = [ 'if (%s->%s_set == 1) {' % (structname, self.Name()), ' %s_free(%s->%s_data);' % ( self._refname, structname, self.Name()), ' %s->%s_data = NULL;' % (structname, self.Name()), ' %s->%s_set = 0;' % (structname, self.Name()), '}' ] return code def CodeInitialize(self, name): code = ['%s->%s_data = NULL;' % (name, self._name)] return code def CodeFree(self, name): code = ['if (%s->%s_data != NULL)' % (name, self._name), ' %s_free(%s->%s_data);' % ( self._refname, name, self._name)] return code def Declaration(self): dcl = ['%s %s_data;' % (self._ctype, self._name)] return dcl class EntryVarBytes(Entry): def __init__(self, type, name, tag): # Init base class Entry.__init__(self, type, name, tag) self._ctype = 'ev_uint8_t *' def GetInitializer(self): return "NULL" def GetVarLen(self, var): return '%(var)s->%(name)s_length' % self.GetTranslation({ 'var' : var }) def CodeArrayAdd(self, varname, value): # xxx: copy return [ '%(varname)s = NULL;' % { 'varname' : varname } ] def GetDeclaration(self, funcname): code = [ 'int %s(struct %s *, %s *, ev_uint32_t *);' % ( funcname, self._struct.Name(), self._ctype ) ] return code def AssignDeclaration(self, funcname): code = [ 'int %s(struct %s *, const %s, ev_uint32_t);' % ( funcname, self._struct.Name(), self._ctype ) ] return code def CodeAssign(self): name = self._name code = [ 'int', '%s_%s_assign(struct %s *msg, ' 'const %s value, ev_uint32_t len)' % ( self._struct.Name(), name, self._struct.Name(), self._ctype), '{', ' if (msg->%s_data != NULL)' % name, ' free (msg->%s_data);' % name, ' msg->%s_data = malloc(len);' % name, ' if (msg->%s_data == NULL)' % name, ' return (-1);', ' msg->%s_set = 1;' % name, ' msg->%s_length = len;' % name, ' memcpy(msg->%s_data, value, len);' % name, ' return (0);', '}' ] return code def CodeGet(self): name = self._name code = [ 'int', '%s_%s_get(struct %s *msg, %s *value, ev_uint32_t *plen)' % ( self._struct.Name(), name, self._struct.Name(), self._ctype), '{', ' if (msg->%s_set != 1)' % name, ' return (-1);', ' *value = msg->%s_data;' % name, ' *plen = msg->%s_length;' % name, ' return (0);', '}' ] return code def CodeUnmarshal(self, buf, tag_name, var_name, var_len): code = ['if (evtag_payload_length(%(buf)s, &%(varlen)s) == -1)', ' return (-1);', # We do not want DoS opportunities 'if (%(varlen)s > evbuffer_get_length(%(buf)s))', ' return (-1);', 'if ((%(var)s = malloc(%(varlen)s)) == NULL)', ' return (-1);', 'if (evtag_unmarshal_fixed(%(buf)s, %(tag)s, %(var)s, ' '%(varlen)s) == -1) {', ' event_warnx("%%s: failed to unmarshal %(name)s", __func__);', ' return (-1);', '}' ] code = '\n'.join(code) % self.GetTranslation({ 'buf' : buf, 'tag' : tag_name, 'var' : var_name, 'varlen' : var_len }) return code.split('\n') def CodeMarshal(self, buf, tag_name, var_name, var_len): code = ['evtag_marshal(%s, %s, %s, %s);' % ( buf, tag_name, var_name, var_len)] return code def CodeClear(self, structname): code = [ 'if (%s->%s_set == 1) {' % (structname, self.Name()), ' free (%s->%s_data);' % (structname, self.Name()), ' %s->%s_data = NULL;' % (structname, self.Name()), ' %s->%s_length = 0;' % (structname, self.Name()), ' %s->%s_set = 0;' % (structname, self.Name()), '}' ] return code def CodeInitialize(self, name): code = ['%s->%s_data = NULL;' % (name, self._name), '%s->%s_length = 0;' % (name, self._name) ] return code def CodeFree(self, name): code = ['if (%s->%s_data != NULL)' % (name, self._name), ' free(%s->%s_data);' % (name, self._name)] return code def Declaration(self): dcl = ['ev_uint8_t *%s_data;' % self._name, 'ev_uint32_t %s_length;' % self._name] return dcl class EntryArray(Entry): def __init__(self, entry): # Init base class Entry.__init__(self, entry._type, entry._name, entry._tag) self._entry = entry self._refname = entry._refname self._ctype = self._entry._ctype self._optional = True self._optpointer = self._entry._optpointer self._optaddarg = self._entry._optaddarg # provide a new function for accessing the variable name def GetVarName(var_name): return '%(var)s->%(name)s_data[%(index)s]' % \ self._entry.GetTranslation({'var' : var_name, 'index' : self._index}) self._entry.GetVarName = GetVarName def GetInitializer(self): return "NULL" def GetVarName(self, var_name): return var_name def GetVarLen(self, var_name): return '-1' def GetDeclaration(self, funcname): """Allows direct access to elements of the array.""" code = [ 'int %(funcname)s(struct %(parent_name)s *, int, %(ctype)s *);' % self.GetTranslation({ 'funcname' : funcname }) ] return code def AssignDeclaration(self, funcname): code = [ 'int %s(struct %s *, int, const %s);' % ( funcname, self._struct.Name(), self._ctype ) ] return code def AddDeclaration(self, funcname): code = [ '%(ctype)s %(optpointer)s ' '%(funcname)s(struct %(parent_name)s *msg%(optaddarg)s);' % \ self.GetTranslation({ 'funcname' : funcname }) ] return code def CodeGet(self): code = """int %(parent_name)s_%(name)s_get(struct %(parent_name)s *msg, int offset, %(ctype)s *value) { if (!msg->%(name)s_set || offset < 0 || offset >= msg->%(name)s_length) return (-1); *value = msg->%(name)s_data[offset]; return (0); }""" % self.GetTranslation() return code.split('\n') def CodeAssign(self): code = [ 'int', '%(parent_name)s_%(name)s_assign(struct %(parent_name)s *msg, int off,', ' const %(ctype)s value)', '{', ' if (!msg->%(name)s_set || off < 0 || off >= msg->%(name)s_length)', ' return (-1);\n', ' {' ] code = TranslateList(code, self.GetTranslation()) codearrayassign = self._entry.CodeArrayAssign( 'msg->%(name)s_data[off]' % self.GetTranslation(), 'value') code += map(lambda x: ' ' + x, codearrayassign) code += TranslateList([ ' }', ' return (0);', '}' ], self.GetTranslation()) return code def CodeAdd(self): codearrayadd = self._entry.CodeArrayAdd( 'msg->%(name)s_data[msg->%(name)s_length - 1]' % self.GetTranslation(), 'value') code = [ 'static int', '%(parent_name)s_%(name)s_expand_to_hold_more(' 'struct %(parent_name)s *msg)', '{', ' int tobe_allocated = msg->%(name)s_num_allocated;', ' %(ctype)s* new_data = NULL;', ' tobe_allocated = !tobe_allocated ? 1 : tobe_allocated << 1;', ' new_data = (%(ctype)s*) realloc(msg->%(name)s_data,', ' tobe_allocated * sizeof(%(ctype)s));', ' if (new_data == NULL)', ' return -1;', ' msg->%(name)s_data = new_data;', ' msg->%(name)s_num_allocated = tobe_allocated;', ' return 0;' '}', '', '%(ctype)s %(optpointer)s', '%(parent_name)s_%(name)s_add(' 'struct %(parent_name)s *msg%(optaddarg)s)', '{', ' if (++msg->%(name)s_length >= msg->%(name)s_num_allocated) {', ' if (%(parent_name)s_%(name)s_expand_to_hold_more(msg)<0)', ' goto error;', ' }' ] code = TranslateList(code, self.GetTranslation()) code += map(lambda x: ' ' + x, codearrayadd) code += TranslateList([ ' msg->%(name)s_set = 1;', ' return %(optreference)s(msg->%(name)s_data[' 'msg->%(name)s_length - 1]);', 'error:', ' --msg->%(name)s_length;', ' return (NULL);', '}' ], self.GetTranslation()) return code def CodeComplete(self, structname, var_name): self._index = 'i' tmp = self._entry.CodeComplete(structname, self._entry.GetVarName(var_name)) # skip the whole loop if there is nothing to check if not tmp: return [] translate = self.GetTranslation({ 'structname' : structname }) code = [ '{', ' int i;', ' for (i = 0; i < %(structname)s->%(name)s_length; ++i) {' ] code = TranslateList(code, translate) code += map(lambda x: ' ' + x, tmp) code += [ ' }', '}' ] return code def CodeUnmarshal(self, buf, tag_name, var_name, var_len): translate = self.GetTranslation({ 'var' : var_name, 'buf' : buf, 'tag' : tag_name, 'init' : self._entry.GetInitializer()}) code = [ 'if (%(var)s->%(name)s_length >= %(var)s->%(name)s_num_allocated &&', ' %(parent_name)s_%(name)s_expand_to_hold_more(%(var)s) < 0) {', ' puts("HEY NOW");', ' return (-1);', '}'] # the unmarshal code directly returns code = TranslateList(code, translate) self._index = '%(var)s->%(name)s_length' % translate code += self._entry.CodeUnmarshal(buf, tag_name, self._entry.GetVarName(var_name), self._entry.GetVarLen(var_name)) code += [ '++%(var)s->%(name)s_length;' % translate ] return code def CodeMarshal(self, buf, tag_name, var_name, var_len): code = ['{', ' int i;', ' for (i = 0; i < %(var)s->%(name)s_length; ++i) {' ] self._index = 'i' code += self._entry.CodeMarshal(buf, tag_name, self._entry.GetVarName(var_name), self._entry.GetVarLen(var_name)) code += [' }', '}' ] code = "\n".join(code) % self.GetTranslation({ 'var' : var_name }) return code.split('\n') def CodeClear(self, structname): translate = self.GetTranslation({ 'structname' : structname }) codearrayfree = self._entry.CodeArrayFree( '%(structname)s->%(name)s_data[i]' % self.GetTranslation( { 'structname' : structname } )) code = [ 'if (%(structname)s->%(name)s_set == 1) {' ] if codearrayfree: code += [ ' int i;', ' for (i = 0; i < %(structname)s->%(name)s_length; ++i) {' ] code = TranslateList(code, translate) if codearrayfree: code += map(lambda x: ' ' + x, codearrayfree) code += [ ' }' ] code += TranslateList([ ' free(%(structname)s->%(name)s_data);', ' %(structname)s->%(name)s_data = NULL;', ' %(structname)s->%(name)s_set = 0;', ' %(structname)s->%(name)s_length = 0;', ' %(structname)s->%(name)s_num_allocated = 0;', '}' ], translate) return code def CodeInitialize(self, name): code = ['%s->%s_data = NULL;' % (name, self._name), '%s->%s_length = 0;' % (name, self._name), '%s->%s_num_allocated = 0;' % (name, self._name)] return code def CodeFree(self, structname): code = self.CodeClear(structname); code += TranslateList([ 'free(%(structname)s->%(name)s_data);' ], self.GetTranslation({'structname' : structname })) return code def Declaration(self): dcl = ['%s *%s_data;' % (self._ctype, self._name), 'int %s_length;' % self._name, 'int %s_num_allocated;' % self._name ] return dcl def NormalizeLine(line): global white global cppcomment line = cppcomment.sub('', line) line = line.strip() line = white.sub(' ', line) return line def ProcessOneEntry(factory, newstruct, entry): optional = 0 array = 0 entry_type = '' name = '' tag = '' tag_set = None separator = '' fixed_length = '' tokens = entry.split(' ') while tokens: token = tokens[0] tokens = tokens[1:] if not entry_type: if not optional and token == 'optional': optional = 1 continue if not array and token == 'array': array = 1 continue if not entry_type: entry_type = token continue if not name: res = re.match(r'^([^\[\]]+)(\[.*\])?$', token) if not res: raise RpcGenError( 'Cannot parse name: \"%s\" ' 'around line %d' % (entry, line_count)) name = res.group(1) fixed_length = res.group(2) if fixed_length: fixed_length = fixed_length[1:-1] continue if not separator: separator = token if separator != '=': raise RpcGenError('Expected "=" after name \"%s\" got %s' % (name, token)) continue if not tag_set: tag_set = 1 if not re.match(r'^(0x)?[0-9]+$', token): raise RpcGenError('Expected tag number: \"%s\"' % entry) tag = int(token, 0) continue raise RpcGenError('Cannot parse \"%s\"' % entry) if not tag_set: raise RpcGenError('Need tag number: \"%s\"' % entry) # Create the right entry if entry_type == 'bytes': if fixed_length: newentry = factory.EntryBytes(entry_type, name, tag, fixed_length) else: newentry = factory.EntryVarBytes(entry_type, name, tag) elif entry_type == 'int' and not fixed_length: newentry = factory.EntryInt(entry_type, name, tag) elif entry_type == 'int64' and not fixed_length: newentry = factory.EntryInt(entry_type, name, tag, bits=64) elif entry_type == 'string' and not fixed_length: newentry = factory.EntryString(entry_type, name, tag) else: res = structref.match(entry_type) if res: # References another struct defined in our file newentry = factory.EntryStruct(entry_type, name, tag, res.group(1)) else: raise RpcGenError('Bad type: "%s" in "%s"' % (entry_type, entry)) structs = [] if optional: newentry.MakeOptional() if array: newentry.MakeArray() newentry.SetStruct(newstruct) newentry.SetLineCount(line_count) newentry.Verify() if array: # We need to encapsulate this entry into a struct newname = newentry.Name()+ '_array' # Now borgify the new entry. newentry = factory.EntryArray(newentry) newentry.SetStruct(newstruct) newentry.SetLineCount(line_count) newentry.MakeArray() newstruct.AddEntry(newentry) return structs def ProcessStruct(factory, data): tokens = data.split(' ') # First three tokens are: 'struct' 'name' '{' newstruct = factory.Struct(tokens[1]) inside = ' '.join(tokens[3:-1]) tokens = inside.split(';') structs = [] for entry in tokens: entry = NormalizeLine(entry) if not entry: continue # It's possible that new structs get defined in here structs.extend(ProcessOneEntry(factory, newstruct, entry)) structs.append(newstruct) return structs def GetNextStruct(file): global line_count global cppdirect got_struct = 0 processed_lines = [] have_c_comment = 0 data = '' while 1: line = file.readline() if not line: break line_count += 1 line = line[:-1] if not have_c_comment and re.search(r'/\*', line): if re.search(r'/\*.*?\*/', line): line = re.sub(r'/\*.*?\*/', '', line) else: line = re.sub(r'/\*.*$', '', line) have_c_comment = 1 if have_c_comment: if not re.search(r'\*/', line): continue have_c_comment = 0 line = re.sub(r'^.*\*/', '', line) line = NormalizeLine(line) if not line: continue if not got_struct: if re.match(r'#include ["<].*[>"]', line): cppdirect.append(line) continue if re.match(r'^#(if( |def)|endif)', line): cppdirect.append(line) continue if re.match(r'^#define', line): headerdirect.append(line) continue if not structdef.match(line): raise RpcGenError('Missing struct on line %d: %s' % (line_count, line)) else: got_struct = 1 data += line continue # We are inside the struct tokens = line.split('}') if len(tokens) == 1: data += ' ' + line continue if len(tokens[1]): raise RpcGenError('Trailing garbage after struct on line %d' % line_count) # We found the end of the struct data += ' %s}' % tokens[0] break # Remove any comments, that might be in there data = re.sub(r'/\*.*\*/', '', data) return data def Parse(factory, file): """ Parses the input file and returns C code and corresponding header file. """ entities = [] while 1: # Just gets the whole struct nicely formatted data = GetNextStruct(file) if not data: break entities.extend(ProcessStruct(factory, data)) return entities class CCodeGenerator: def __init__(self): pass def GuardName(self, name): # Use the complete provided path to the input file, with all # non-identifier characters replaced with underscores, to # reduce the chance of a collision between guard macros. return 'EVENT_RPCOUT_' + nonident.sub('_', name).upper() + '_' def HeaderPreamble(self, name): guard = self.GuardName(name) pre = ( '/*\n' ' * Automatically generated from %s\n' ' */\n\n' '#ifndef %s\n' '#define %s\n\n' ) % ( name, guard, guard) for statement in headerdirect: pre += '%s\n' % statement if headerdirect: pre += '\n' pre += ( '#include <event2/util.h> /* for ev_uint*_t */\n' '#include <event2/rpc.h>\n' ) return pre def HeaderPostamble(self, name): guard = self.GuardName(name) return '#endif /* %s */' % guard def BodyPreamble(self, name, header_file): global _NAME global _VERSION slash = header_file.rfind('/') if slash != -1: header_file = header_file[slash+1:] pre = ( '/*\n' ' * Automatically generated from %s\n' ' * by %s/%s. DO NOT EDIT THIS FILE.\n' ' */\n\n' ) % (name, _NAME, _VERSION) pre += ( '#include <stdlib.h>\n' '#include <string.h>\n' '#include <assert.h>\n' '#include <event2/event-config.h>\n' '#include <event2/event.h>\n' '#include <event2/buffer.h>\n' '#include <event2/tag.h>\n\n' '#if defined(EVENT____func__) && !defined(__func__)\n' '#define __func__ EVENT____func__\n' '#endif\n\n' ) for statement in cppdirect: pre += '%s\n' % statement pre += '\n#include "%s"\n\n' % header_file pre += 'void event_warn(const char *fmt, ...);\n' pre += 'void event_warnx(const char *fmt, ...);\n\n' return pre def HeaderFilename(self, filename): return '.'.join(filename.split('.')[:-1]) + '.h' def CodeFilename(self, filename): return '.'.join(filename.split('.')[:-1]) + '.gen.c' def Struct(self, name): return StructCCode(name) def EntryBytes(self, entry_type, name, tag, fixed_length): return EntryBytes(entry_type, name, tag, fixed_length) def EntryVarBytes(self, entry_type, name, tag): return EntryVarBytes(entry_type, name, tag) def EntryInt(self, entry_type, name, tag, bits=32): return EntryInt(entry_type, name, tag, bits) def EntryString(self, entry_type, name, tag): return EntryString(entry_type, name, tag) def EntryStruct(self, entry_type, name, tag, struct_name): return EntryStruct(entry_type, name, tag, struct_name) def EntryArray(self, entry): return EntryArray(entry) class Usage(RpcGenError): def __init__(self, argv0): RpcGenError.__init__("usage: %s input.rpc [[output.h] output.c]" % argv0) class CommandLine: def __init__(self, argv): """Initialize a command-line to launch event_rpcgen, as if from a command-line with CommandLine(sys.argv). If you're calling this directly, remember to provide a dummy value for sys.argv[0] """ self.filename = None self.header_file = None self.impl_file = None self.factory = CCodeGenerator() if len(argv) >= 2 and argv[1] == '--quiet': global QUIETLY QUIETLY = 1 del argv[1] if len(argv) < 2 or len(argv) > 4: raise Usage(argv[0]) self.filename = argv[1].replace('\\', '/') if len(argv) == 3: self.impl_file = argv[2].replace('\\', '/') if len(argv) == 4: self.header_file = argv[2].replace('\\', '/') self.impl_file = argv[3].replace('\\', '/') if not self.filename: raise Usage(argv[0]) if not self.impl_file: self.impl_file = self.factory.CodeFilename(self.filename) if not self.header_file: self.header_file = self.factory.HeaderFilename(self.impl_file) if not self.impl_file.endswith('.c'): raise RpcGenError("can only generate C implementation files") if not self.header_file.endswith('.h'): raise RpcGenError("can only generate C header files") def run(self): filename = self.filename header_file = self.header_file impl_file = self.impl_file factory = self.factory declare('Reading \"%s\"' % filename) fp = open(filename, 'r') entities = Parse(factory, fp) fp.close() declare('... creating "%s"' % header_file) header_fp = open(header_file, 'w') print >>header_fp, factory.HeaderPreamble(filename) # Create forward declarations: allows other structs to reference # each other for entry in entities: entry.PrintForwardDeclaration(header_fp) print >>header_fp, '' for entry in entities: entry.PrintTags(header_fp) entry.PrintDeclaration(header_fp) print >>header_fp, factory.HeaderPostamble(filename) header_fp.close() declare('... creating "%s"' % impl_file) impl_fp = open(impl_file, 'w') print >>impl_fp, factory.BodyPreamble(filename, header_file) for entry in entities: entry.PrintCode(impl_fp) impl_fp.close() if __name__ == '__main__': try: CommandLine(sys.argv).run() sys.exit(0) except RpcGenError, e: print >>sys.stderr, e sys.exit(1) except EnvironmentError, e: if e.filename and e.strerror: print >>sys.stderr, "%s: %s" % (e.filename, e.strerror) sys.exit(1) elif e.strerror: print >> sys.stderr, e.strerror sys.exit(1) else: raise �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/event_tagging.c����������������������������������������������0000664�0000000�0000000�00000034466�15270523712�0023000�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2003-2009 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_SYS_TYPES_H #include <sys/types.h> #endif #ifdef EVENT__HAVE_SYS_PARAM_H #include <sys/param.h> #endif #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include <winsock2.h> #include <windows.h> #undef WIN32_LEAN_AND_MEAN #endif #ifdef EVENT__HAVE_SYS_IOCTL_H #include <sys/ioctl.h> #endif #include <sys/queue.h> #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #ifndef _WIN32 #include <syslog.h> #endif #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #include <limits.h> #include "event2/event.h" #include "event2/tag.h" #include "event2/buffer.h" #include "log-internal.h" #include "mm-internal.h" #include "util-internal.h" /* Here's our wire format: Stream = TaggedData* TaggedData = Tag Length Data where the integer value of 'Length' is the length of 'data'. Tag = HByte* LByte where HByte is a byte with the high bit set, and LByte is a byte with the high bit clear. The integer value of the tag is taken by concatenating the lower 7 bits from all the tags. So for example, the tag 0x66 is encoded as [66], whereas the tag 0x166 is encoded as [82 66] Length = Integer Integer = NNibbles Nibble* Padding? where NNibbles is a 4-bit value encoding the number of nibbles-1, and each Nibble is 4 bits worth of encoded integer, in big-endian order. If the total encoded integer size is an odd number of nibbles, a final padding nibble with value 0 is appended. */ int evtag_decode_int(ev_uint32_t *pnumber, struct evbuffer *evbuf); int evtag_decode_int64(ev_uint64_t *pnumber, struct evbuffer *evbuf); int evtag_encode_tag(struct evbuffer *evbuf, ev_uint32_t tag); int evtag_decode_tag(ev_uint32_t *ptag, struct evbuffer *evbuf); void evtag_init(void) { } /* * We encode integers by nibbles; the first nibble contains the number * of significant nibbles - 1; this allows us to encode up to 64-bit * integers. This function is byte-order independent. * * @param number a 32-bit unsigned integer to encode * @param data a pointer to where the data should be written. Must * have at least 5 bytes free. * @return the number of bytes written into data. */ #define ENCODE_INT_INTERNAL(data, number) do { \ int off = 1, nibbles = 0; \ \ memset(data, 0, sizeof(number)+1); \ while (number) { \ if (off & 0x1) \ data[off/2] = (data[off/2] & 0xf0) | (number & 0x0f); \ else \ data[off/2] = (data[off/2] & 0x0f) | \ ((number & 0x0f) << 4); \ number >>= 4; \ off++; \ } \ \ if (off > 2) \ nibbles = off - 2; \ \ /* Off - 1 is the number of encoded nibbles */ \ data[0] = (data[0] & 0x0f) | ((nibbles & 0x0f) << 4); \ \ return ((off + 1) / 2); \ } while (0) static inline int encode_int_internal(ev_uint8_t *data, ev_uint32_t number) { ENCODE_INT_INTERNAL(data, number); } static inline int encode_int64_internal(ev_uint8_t *data, ev_uint64_t number) { ENCODE_INT_INTERNAL(data, number); } void evtag_encode_int(struct evbuffer *evbuf, ev_uint32_t number) { ev_uint8_t data[5]; int len = encode_int_internal(data, number); evbuffer_add(evbuf, data, len); } void evtag_encode_int64(struct evbuffer *evbuf, ev_uint64_t number) { ev_uint8_t data[9]; int len = encode_int64_internal(data, number); evbuffer_add(evbuf, data, len); } /* * Support variable length encoding of tags; we use the high bit in each * octet as a continuation signal. */ int evtag_encode_tag(struct evbuffer *evbuf, ev_uint32_t tag) { int bytes = 0; ev_uint8_t data[5]; memset(data, 0, sizeof(data)); do { ev_uint8_t lower = tag & 0x7f; tag >>= 7; if (tag) lower |= 0x80; data[bytes++] = lower; } while (tag); if (evbuf != NULL) evbuffer_add(evbuf, data, bytes); return (bytes); } static int decode_tag_internal(ev_uint32_t *ptag, struct evbuffer *evbuf, int dodrain) { ev_uint32_t number = 0; size_t len = evbuffer_get_length(evbuf); ev_uint8_t *data; size_t count = 0; int shift = 0, done = 0; /* * the encoding of a number is at most one byte more than its * storage size. however, it may also be much smaller. */ data = evbuffer_pullup( evbuf, len < sizeof(number) + 1 ? len : sizeof(number) + 1); if (!data) return (-1); while (count++ < len) { ev_uint8_t lower = *data++; if (shift >= 28) { /* Make sure it fits into 32 bits */ if (shift > 28) return (-1); if ((lower & 0x7f) > 15) return (-1); } number |= (lower & (unsigned)0x7f) << shift; shift += 7; if (!(lower & 0x80)) { done = 1; break; } } if (!done) return (-1); if (dodrain) evbuffer_drain(evbuf, count); if (ptag != NULL) *ptag = number; return count > INT_MAX ? INT_MAX : (int)(count); } int evtag_decode_tag(ev_uint32_t *ptag, struct evbuffer *evbuf) { return (decode_tag_internal(ptag, evbuf, 1 /* dodrain */)); } /* * Marshal a data type, the general format is as follows: * * tag number: one byte; length: var bytes; payload: var bytes */ void evtag_marshal(struct evbuffer *evbuf, ev_uint32_t tag, const void *data, ev_uint32_t len) { evtag_encode_tag(evbuf, tag); evtag_encode_int(evbuf, len); evbuffer_add(evbuf, (void *)data, len); } void evtag_marshal_buffer(struct evbuffer *evbuf, ev_uint32_t tag, struct evbuffer *data) { evtag_encode_tag(evbuf, tag); /* XXX support more than UINT32_MAX data */ evtag_encode_int(evbuf, (ev_uint32_t)evbuffer_get_length(data)); evbuffer_add_buffer(evbuf, data); } /* Marshaling for integers */ void evtag_marshal_int(struct evbuffer *evbuf, ev_uint32_t tag, ev_uint32_t integer) { ev_uint8_t data[5]; int len = encode_int_internal(data, integer); evtag_encode_tag(evbuf, tag); evtag_encode_int(evbuf, len); evbuffer_add(evbuf, data, len); } void evtag_marshal_int64(struct evbuffer *evbuf, ev_uint32_t tag, ev_uint64_t integer) { ev_uint8_t data[9]; int len = encode_int64_internal(data, integer); evtag_encode_tag(evbuf, tag); evtag_encode_int(evbuf, len); evbuffer_add(evbuf, data, len); } void evtag_marshal_string(struct evbuffer *buf, ev_uint32_t tag, const char *string) { /* TODO support strings longer than UINT32_MAX ? */ evtag_marshal(buf, tag, string, (ev_uint32_t)strlen(string)); } void evtag_marshal_timeval(struct evbuffer *evbuf, ev_uint32_t tag, struct timeval *tv) { ev_uint8_t data[10]; int len = encode_int_internal(data, tv->tv_sec); len += encode_int_internal(data + len, tv->tv_usec); evtag_marshal(evbuf, tag, data, len); } #define DECODE_INT_INTERNAL(number, maxnibbles, pnumber, evbuf, offset) \ do { \ ev_uint8_t *data; \ ev_ssize_t len = evbuffer_get_length(evbuf) - offset; \ int nibbles = 0; \ \ if (len <= 0) \ return (-1); \ \ /* XXX(niels): faster? */ \ data = evbuffer_pullup(evbuf, offset + 1) + offset; \ if (!data) \ return (-1); \ \ nibbles = ((data[0] & 0xf0) >> 4) + 1; \ if (nibbles > maxnibbles || (nibbles >> 1) + 1 > len) \ return (-1); \ len = (nibbles >> 1) + 1; \ \ data = evbuffer_pullup(evbuf, offset + len) + offset; \ if (!data) \ return (-1); \ \ while (nibbles > 0) { \ number <<= 4; \ if (nibbles & 0x1) \ number |= data[nibbles >> 1] & 0x0f; \ else \ number |= (data[nibbles >> 1] & 0xf0) >> 4; \ nibbles--; \ } \ \ *pnumber = number; \ \ return (int)(len); \ } while (0) /* Internal: decode an integer from an evbuffer, without draining it. * Only integers up to 32-bits are supported. * * @param evbuf the buffer to read from * @param offset an index into the buffer at which we should start reading. * @param pnumber a pointer to receive the integer. * @return The length of the number as encoded, or -1 on error. */ static int decode_int_internal(ev_uint32_t *pnumber, struct evbuffer *evbuf, int offset) { ev_uint32_t number = 0; DECODE_INT_INTERNAL(number, 8, pnumber, evbuf, offset); } static int decode_int64_internal(ev_uint64_t *pnumber, struct evbuffer *evbuf, int offset) { ev_uint64_t number = 0; DECODE_INT_INTERNAL(number, 16, pnumber, evbuf, offset); } int evtag_decode_int(ev_uint32_t *pnumber, struct evbuffer *evbuf) { int res = decode_int_internal(pnumber, evbuf, 0); if (res != -1) evbuffer_drain(evbuf, res); return (res == -1 ? -1 : 0); } int evtag_decode_int64(ev_uint64_t *pnumber, struct evbuffer *evbuf) { int res = decode_int64_internal(pnumber, evbuf, 0); if (res != -1) evbuffer_drain(evbuf, res); return (res == -1 ? -1 : 0); } int evtag_peek(struct evbuffer *evbuf, ev_uint32_t *ptag) { return (decode_tag_internal(ptag, evbuf, 0 /* dodrain */)); } int evtag_peek_length(struct evbuffer *evbuf, ev_uint32_t *plength) { int res, len; len = decode_tag_internal(NULL, evbuf, 0 /* dodrain */); if (len == -1) return (-1); res = decode_int_internal(plength, evbuf, len); if (res == -1) return (-1); *plength += res + len; return (0); } int evtag_payload_length(struct evbuffer *evbuf, ev_uint32_t *plength) { int res, len; len = decode_tag_internal(NULL, evbuf, 0 /* dodrain */); if (len == -1) return (-1); res = decode_int_internal(plength, evbuf, len); if (res == -1) return (-1); return (0); } /* just unmarshals the header and returns the length of the remaining data */ int evtag_unmarshal_header(struct evbuffer *evbuf, ev_uint32_t *ptag) { ev_uint32_t len; if (decode_tag_internal(ptag, evbuf, 1 /* dodrain */) == -1) return (-1); if (evtag_decode_int(&len, evbuf) == -1) return (-1); if (evbuffer_get_length(evbuf) < len) return (-1); return (len); } int evtag_consume(struct evbuffer *evbuf) { int len; if ((len = evtag_unmarshal_header(evbuf, NULL)) == -1) return (-1); evbuffer_drain(evbuf, len); return (0); } /* Reads the data type from an event buffer */ int evtag_unmarshal(struct evbuffer *src, ev_uint32_t *ptag, struct evbuffer *dst) { int len; if ((len = evtag_unmarshal_header(src, ptag)) == -1) return (-1); if (evbuffer_add(dst, evbuffer_pullup(src, len), len) == -1) return (-1); evbuffer_drain(src, len); return (len); } /* Marshaling for integers */ int evtag_unmarshal_int(struct evbuffer *evbuf, ev_uint32_t need_tag, ev_uint32_t *pinteger) { ev_uint32_t tag; ev_uint32_t len; int result; if (decode_tag_internal(&tag, evbuf, 1 /* dodrain */) == -1) return (-1); if (need_tag != tag) return (-1); if (evtag_decode_int(&len, evbuf) == -1) return (-1); if (evbuffer_get_length(evbuf) < len) return (-1); result = decode_int_internal(pinteger, evbuf, 0); evbuffer_drain(evbuf, len); if (result < 0 || (size_t)result > len) /* XXX Should this be != rather than > ?*/ return (-1); else return result; } int evtag_unmarshal_int64(struct evbuffer *evbuf, ev_uint32_t need_tag, ev_uint64_t *pinteger) { ev_uint32_t tag; ev_uint32_t len; int result; if (decode_tag_internal(&tag, evbuf, 1 /* dodrain */) == -1) return (-1); if (need_tag != tag) return (-1); if (evtag_decode_int(&len, evbuf) == -1) return (-1); if (evbuffer_get_length(evbuf) < len) return (-1); result = decode_int64_internal(pinteger, evbuf, 0); evbuffer_drain(evbuf, len); if (result < 0 || (size_t)result > len) /* XXX Should this be != rather than > ?*/ return (-1); else return result; } /* Unmarshal a fixed length tag */ int evtag_unmarshal_fixed(struct evbuffer *src, ev_uint32_t need_tag, void *data, size_t len) { ev_uint32_t tag; int tag_len; /* Now unmarshal a tag and check that it matches the tag we want */ if ((tag_len = evtag_unmarshal_header(src, &tag)) < 0 || tag != need_tag) return (-1); if ((size_t)tag_len != len) return (-1); evbuffer_remove(src, data, len); return (0); } int evtag_unmarshal_string(struct evbuffer *evbuf, ev_uint32_t need_tag, char **pstring) { ev_uint32_t tag; int tag_len; if ((tag_len = evtag_unmarshal_header(evbuf, &tag)) == -1 || tag != need_tag) return (-1); *pstring = mm_malloc(tag_len + 1); if (*pstring == NULL) { event_warn("%s: malloc", __func__); return -1; } evbuffer_remove(evbuf, *pstring, tag_len); (*pstring)[tag_len] = '\0'; return (0); } int evtag_unmarshal_timeval(struct evbuffer *evbuf, ev_uint32_t need_tag, struct timeval *ptv) { ev_uint32_t tag; ev_uint32_t integer; int len, offset, offset2; int result = -1; if ((len = evtag_unmarshal_header(evbuf, &tag)) == -1) return (-1); if (tag != need_tag) goto done; if ((offset = decode_int_internal(&integer, evbuf, 0)) == -1) goto done; ptv->tv_sec = integer; if ((offset2 = decode_int_internal(&integer, evbuf, offset)) == -1) goto done; ptv->tv_usec = integer; if (offset + offset2 > len) /* XXX Should this be != instead of > ? */ goto done; result = 0; done: evbuffer_drain(evbuf, len); return result; } ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evmap-internal.h���������������������������������������������0000664�0000000�0000000�00000011435�15270523712�0023075�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVMAP_INTERNAL_H_INCLUDED_ #define EVMAP_INTERNAL_H_INCLUDED_ /** @file evmap-internal.h * * An event_map is a utility structure to map each fd or signal to zero or * more events. Functions to manipulate event_maps should only be used from * inside libevent. They generally need to hold the lock on the corresponding * event_base. **/ struct event_base; struct event; /** Initialize an event_map for use. */ void evmap_io_initmap_(struct event_io_map* ctx); void evmap_signal_initmap_(struct event_signal_map* ctx); /** Remove all entries from an event_map. @param ctx the map to clear. */ void evmap_io_clear_(struct event_io_map* ctx); void evmap_signal_clear_(struct event_signal_map* ctx); /** Add an IO event (some combination of EV_READ or EV_WRITE) to an event_base's list of events on a given file descriptor, and tell the underlying eventops about the fd if its state has changed. Requires that ev is not already added. @param base the event_base to operate on. @param fd the file descriptor corresponding to ev. @param ev the event to add. */ int evmap_io_add_(struct event_base *base, evutil_socket_t fd, struct event *ev); /** Remove an IO event (some combination of EV_READ or EV_WRITE) to an event_base's list of events on a given file descriptor, and tell the underlying eventops about the fd if its state has changed. @param base the event_base to operate on. @param fd the file descriptor corresponding to ev. @param ev the event to remove. */ int evmap_io_del_(struct event_base *base, evutil_socket_t fd, struct event *ev); /** Active the set of events waiting on an event_base for a given fd. @param base the event_base to operate on. @param fd the file descriptor that has become active. @param events a bitmask of EV_READ|EV_WRITE|EV_ET. */ void evmap_io_active_(struct event_base *base, evutil_socket_t fd, short events); /* These functions behave in the same way as evmap_io_*, except they work on * signals rather than fds. signals use a linear map everywhere; fds use * either a linear map or a hashtable. */ int evmap_signal_add_(struct event_base *base, int signum, struct event *ev); int evmap_signal_del_(struct event_base *base, int signum, struct event *ev); void evmap_signal_active_(struct event_base *base, evutil_socket_t signum, int ncalls); /* Return the fdinfo object associated with a given fd. If the fd has no * events associated with it, the result may be NULL. */ void *evmap_io_get_fdinfo_(struct event_io_map *ctx, evutil_socket_t fd); /* Helper for event_reinit(): Tell the backend to re-add every fd and signal * for which we have a pending event. */ int evmap_reinit_(struct event_base *base); /* Helper for event_base_free(): Call event_del() on every pending fd and * signal event. */ void evmap_delete_all_(struct event_base *base); /* Helper for event_base_assert_ok_(): Check referential integrity of the * evmaps. */ void evmap_check_integrity_(struct event_base *base); /* Helper: Call fn on every fd or signal event, passing as its arguments the * provided event_base, the event, and arg. If fn returns 0, process the next * event. If it returns any other value, return that value and process no * more events. */ int evmap_foreach_event_(struct event_base *base, event_base_foreach_event_cb fn, void *arg); #endif /* EVMAP_INTERNAL_H_INCLUDED_ */ �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evmap.c������������������������������������������������������0000664�0000000�0000000�00000070311�15270523712�0021254�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #include <winsock2.h> #define WIN32_LEAN_AND_MEAN #include <windows.h> #undef WIN32_LEAN_AND_MEAN #endif #include <sys/types.h> #if !defined(_WIN32) && defined(EVENT__HAVE_SYS_TIME_H) #include <sys/time.h> #endif #include <sys/queue.h> #include <stdio.h> #include <stdlib.h> #ifndef _WIN32 #include <unistd.h> #endif #include <errno.h> #include <signal.h> #include <string.h> #include <time.h> #include "event-internal.h" #include "evmap-internal.h" #include "mm-internal.h" #include "changelist-internal.h" /** An entry for an evmap_io list: notes all the events that want to read or write on a given fd, and the number of each. */ struct evmap_io { struct event_dlist events; ev_uint16_t nread; ev_uint16_t nwrite; ev_uint16_t nclose; }; /* An entry for an evmap_signal list: notes all the events that want to know when a signal triggers. */ struct evmap_signal { struct event_dlist events; }; /* On some platforms, fds start at 0 and increment by 1 as they are allocated, and old numbers get used. For these platforms, we implement io maps just like signal maps: as an array of pointers to struct evmap_io. But on other platforms (windows), sockets are not 0-indexed, not necessarily consecutive, and not necessarily reused. There, we use a hashtable to implement evmap_io. */ #ifdef EVMAP_USE_HT struct event_map_entry { HT_ENTRY(event_map_entry) map_node; evutil_socket_t fd; union { /* This is a union in case we need to make more things that can be in the hashtable. */ struct evmap_io evmap_io; } ent; }; /* Helper used by the event_io_map hashtable code; tries to return a good hash * of the fd in e->fd. */ static inline unsigned hashsocket(struct event_map_entry *e) { /* On win32, in practice, the low 2-3 bits of a SOCKET seem not to * matter. Our hashtable implementation really likes low-order bits, * though, so let's do the rotate-and-add trick. */ unsigned h = (unsigned) e->fd; h += (h >> 2) | (h << 30); return h; } /* Helper used by the event_io_map hashtable code; returns true iff e1 and e2 * have the same e->fd. */ static inline int eqsocket(struct event_map_entry *e1, struct event_map_entry *e2) { return e1->fd == e2->fd; } HT_PROTOTYPE(event_io_map, event_map_entry, map_node, hashsocket, eqsocket) HT_GENERATE(event_io_map, event_map_entry, map_node, hashsocket, eqsocket, 0.5, mm_malloc, mm_realloc, mm_free) #define GET_IO_SLOT(x, map, slot, type) \ do { \ struct event_map_entry key_, *ent_; \ key_.fd = slot; \ ent_ = HT_FIND(event_io_map, map, &key_); \ (x) = ent_ ? &ent_->ent.type : NULL; \ } while (0); #define GET_IO_SLOT_AND_CTOR(x, map, slot, type, ctor, fdinfo_len) \ do { \ struct event_map_entry key_, *ent_; \ key_.fd = slot; \ HT_FIND_OR_INSERT_(event_io_map, map_node, hashsocket, map, \ event_map_entry, &key_, ptr, \ { \ ent_ = *ptr; \ }, \ { \ ent_ = mm_calloc(1,sizeof(struct event_map_entry)+fdinfo_len); \ if (EVUTIL_UNLIKELY(ent_ == NULL)) \ return (-1); \ ent_->fd = slot; \ (ctor)(&ent_->ent.type); \ HT_FOI_INSERT_(map_node, map, &key_, ent_, ptr) \ }); \ (x) = &ent_->ent.type; \ } while (0) void evmap_io_initmap_(struct event_io_map *ctx) { HT_INIT(event_io_map, ctx); } void evmap_io_clear_(struct event_io_map *ctx) { struct event_map_entry **ent, **next, *this; for (ent = HT_START(event_io_map, ctx); ent; ent = next) { this = *ent; next = HT_NEXT_RMV(event_io_map, ctx, ent); mm_free(this); } HT_CLEAR(event_io_map, ctx); /* remove all storage held by the ctx. */ } #endif /* Set the variable 'x' to the field in event_map 'map' with fields of type 'struct type *' corresponding to the fd or signal 'slot'. Set 'x' to NULL if there are no entries for 'slot'. Does no bounds-checking. */ #define GET_SIGNAL_SLOT(x, map, slot, type) \ (x) = (struct type *)((map)->entries[slot]) /* As GET_SLOT, but construct the entry for 'slot' if it is not present, by allocating enough memory for a 'struct type', and initializing the new value by calling the function 'ctor' on it. Makes the function return -1 on allocation failure. */ #define GET_SIGNAL_SLOT_AND_CTOR(x, map, slot, type, ctor, fdinfo_len) \ do { \ if ((map)->entries[slot] == NULL) { \ (map)->entries[slot] = \ mm_calloc(1,sizeof(struct type)+fdinfo_len); \ if (EVUTIL_UNLIKELY((map)->entries[slot] == NULL)) \ return (-1); \ (ctor)((struct type *)(map)->entries[slot]); \ } \ (x) = (struct type *)((map)->entries[slot]); \ } while (0) /* If we aren't using hashtables, then define the IO_SLOT macros and functions as thin aliases over the SIGNAL_SLOT versions. */ #ifndef EVMAP_USE_HT #define GET_IO_SLOT(x,map,slot,type) GET_SIGNAL_SLOT(x,map,slot,type) #define GET_IO_SLOT_AND_CTOR(x,map,slot,type,ctor,fdinfo_len) \ GET_SIGNAL_SLOT_AND_CTOR(x,map,slot,type,ctor,fdinfo_len) #define FDINFO_OFFSET sizeof(struct evmap_io) void evmap_io_initmap_(struct event_io_map* ctx) { evmap_signal_initmap_(ctx); } void evmap_io_clear_(struct event_io_map* ctx) { evmap_signal_clear_(ctx); } #endif /** Expand 'map' with new entries of width 'msize' until it is big enough to store a value in 'slot'. */ static int evmap_make_space(struct event_signal_map *map, int slot, int msize) { if (map->nentries <= slot) { int nentries = map->nentries ? map->nentries : 32; void **tmp; while (nentries <= slot) nentries <<= 1; tmp = (void **)mm_realloc(map->entries, nentries * msize); if (tmp == NULL) return (-1); memset(&tmp[map->nentries], 0, (nentries - map->nentries) * msize); map->nentries = nentries; map->entries = tmp; } return (0); } void evmap_signal_initmap_(struct event_signal_map *ctx) { ctx->nentries = 0; ctx->entries = NULL; } void evmap_signal_clear_(struct event_signal_map *ctx) { if (ctx->entries != NULL) { int i; for (i = 0; i < ctx->nentries; ++i) { if (ctx->entries[i] != NULL) mm_free(ctx->entries[i]); } mm_free(ctx->entries); ctx->entries = NULL; } ctx->nentries = 0; } /* code specific to file descriptors */ /** Constructor for struct evmap_io */ static void evmap_io_init(struct evmap_io *entry) { LIST_INIT(&entry->events); entry->nread = 0; entry->nwrite = 0; entry->nclose = 0; } /* return -1 on error, 0 on success if nothing changed in the event backend, * and 1 on success if something did. */ int evmap_io_add_(struct event_base *base, evutil_socket_t fd, struct event *ev) { const struct eventop *evsel = base->evsel; struct event_io_map *io = &base->io; struct evmap_io *ctx = NULL; int nread, nwrite, nclose, retval = 0; short res = 0, old = 0; struct event *old_ev; EVUTIL_ASSERT(fd == ev->ev_fd); if (fd < 0) return 0; #ifndef EVMAP_USE_HT if (fd >= io->nentries) { if (evmap_make_space(io, fd, sizeof(struct evmap_io *)) == -1) return (-1); } #endif GET_IO_SLOT_AND_CTOR(ctx, io, fd, evmap_io, evmap_io_init, evsel->fdinfo_len); nread = ctx->nread; nwrite = ctx->nwrite; nclose = ctx->nclose; if (nread) old |= EV_READ; if (nwrite) old |= EV_WRITE; if (nclose) old |= EV_CLOSED; if (ev->ev_events & EV_READ) { if (++nread == 1) res |= EV_READ; } if (ev->ev_events & EV_WRITE) { if (++nwrite == 1) res |= EV_WRITE; } if (ev->ev_events & EV_CLOSED) { if (++nclose == 1) res |= EV_CLOSED; } if (EVUTIL_UNLIKELY(nread > 0xffff || nwrite > 0xffff || nclose > 0xffff)) { event_warnx("Too many events reading or writing on fd %d", (int)fd); return -1; } if (EVENT_DEBUG_MODE_IS_ON() && (old_ev = LIST_FIRST(&ctx->events)) && (old_ev->ev_events&EV_ET) != (ev->ev_events&EV_ET)) { event_warnx("Tried to mix edge-triggered and non-edge-triggered" " events on fd %d", (int)fd); return -1; } if (res) { void *extra = ((char*)ctx) + sizeof(struct evmap_io); /* XXX(niels): we cannot mix edge-triggered and * level-triggered, we should probably assert on * this. */ if (evsel->add(base, ev->ev_fd, old, (ev->ev_events & EV_ET) | res, extra) == -1) return (-1); retval = 1; } ctx->nread = (ev_uint16_t) nread; ctx->nwrite = (ev_uint16_t) nwrite; ctx->nclose = (ev_uint16_t) nclose; LIST_INSERT_HEAD(&ctx->events, ev, ev_io_next); return (retval); } /* return -1 on error, 0 on success if nothing changed in the event backend, * and 1 on success if something did. */ int evmap_io_del_(struct event_base *base, evutil_socket_t fd, struct event *ev) { const struct eventop *evsel = base->evsel; struct event_io_map *io = &base->io; struct evmap_io *ctx; int nread, nwrite, nclose, retval = 0; short res = 0, old = 0; if (fd < 0) return 0; EVUTIL_ASSERT(fd == ev->ev_fd); #ifndef EVMAP_USE_HT if (fd >= io->nentries) return (-1); #endif GET_IO_SLOT(ctx, io, fd, evmap_io); nread = ctx->nread; nwrite = ctx->nwrite; nclose = ctx->nclose; if (nread) old |= EV_READ; if (nwrite) old |= EV_WRITE; if (nclose) old |= EV_CLOSED; if (ev->ev_events & EV_READ) { if (--nread == 0) res |= EV_READ; EVUTIL_ASSERT(nread >= 0); } if (ev->ev_events & EV_WRITE) { if (--nwrite == 0) res |= EV_WRITE; EVUTIL_ASSERT(nwrite >= 0); } if (ev->ev_events & EV_CLOSED) { if (--nclose == 0) res |= EV_CLOSED; EVUTIL_ASSERT(nclose >= 0); } if (res) { void *extra = ((char*)ctx) + sizeof(struct evmap_io); if (evsel->del(base, ev->ev_fd, old, res, extra) == -1) { retval = -1; } else { retval = 1; } } ctx->nread = nread; ctx->nwrite = nwrite; ctx->nclose = nclose; LIST_REMOVE(ev, ev_io_next); return (retval); } void evmap_io_active_(struct event_base *base, evutil_socket_t fd, short events) { struct event_io_map *io = &base->io; struct evmap_io *ctx; struct event *ev; #ifndef EVMAP_USE_HT if (fd < 0 || fd >= io->nentries) return; #endif GET_IO_SLOT(ctx, io, fd, evmap_io); if (NULL == ctx) return; LIST_FOREACH(ev, &ctx->events, ev_io_next) { if (ev->ev_events & events) event_active_nolock_(ev, ev->ev_events & events, 1); } } /* code specific to signals */ static void evmap_signal_init(struct evmap_signal *entry) { LIST_INIT(&entry->events); } int evmap_signal_add_(struct event_base *base, int sig, struct event *ev) { const struct eventop *evsel = base->evsigsel; struct event_signal_map *map = &base->sigmap; struct evmap_signal *ctx = NULL; if (sig >= map->nentries) { if (evmap_make_space( map, sig, sizeof(struct evmap_signal *)) == -1) return (-1); } GET_SIGNAL_SLOT_AND_CTOR(ctx, map, sig, evmap_signal, evmap_signal_init, base->evsigsel->fdinfo_len); if (LIST_EMPTY(&ctx->events)) { if (evsel->add(base, ev->ev_fd, 0, EV_SIGNAL, NULL) == -1) return (-1); } LIST_INSERT_HEAD(&ctx->events, ev, ev_signal_next); return (1); } int evmap_signal_del_(struct event_base *base, int sig, struct event *ev) { const struct eventop *evsel = base->evsigsel; struct event_signal_map *map = &base->sigmap; struct evmap_signal *ctx; if (sig >= map->nentries) return (-1); GET_SIGNAL_SLOT(ctx, map, sig, evmap_signal); LIST_REMOVE(ev, ev_signal_next); if (LIST_FIRST(&ctx->events) == NULL) { if (evsel->del(base, ev->ev_fd, 0, EV_SIGNAL, NULL) == -1) return (-1); } return (1); } void evmap_signal_active_(struct event_base *base, evutil_socket_t sig, int ncalls) { struct event_signal_map *map = &base->sigmap; struct evmap_signal *ctx; struct event *ev; if (sig < 0 || sig >= map->nentries) return; GET_SIGNAL_SLOT(ctx, map, sig, evmap_signal); if (!ctx) return; LIST_FOREACH(ev, &ctx->events, ev_signal_next) event_active_nolock_(ev, EV_SIGNAL, ncalls); } void * evmap_io_get_fdinfo_(struct event_io_map *map, evutil_socket_t fd) { struct evmap_io *ctx; GET_IO_SLOT(ctx, map, fd, evmap_io); if (ctx) return ((char*)ctx) + sizeof(struct evmap_io); else return NULL; } /* Callback type for evmap_io_foreach_fd */ typedef int (*evmap_io_foreach_fd_cb)( struct event_base *, evutil_socket_t, struct evmap_io *, void *); /* Multipurpose helper function: Iterate over every file descriptor event_base * for which we could have EV_READ or EV_WRITE events. For each such fd, call * fn(base, signum, evmap_io, arg), where fn is the user-provided * function, base is the event_base, signum is the signal number, evmap_io * is an evmap_io structure containing a list of events pending on the * file descriptor, and arg is the user-supplied argument. * * If fn returns 0, continue on to the next signal. Otherwise, return the same * value that fn returned. * * Note that there is no guarantee that the file descriptors will be processed * in any particular order. */ static int evmap_io_foreach_fd(struct event_base *base, evmap_io_foreach_fd_cb fn, void *arg) { evutil_socket_t fd; struct event_io_map *iomap = &base->io; int r = 0; #ifdef EVMAP_USE_HT struct event_map_entry **mapent; HT_FOREACH(mapent, event_io_map, iomap) { struct evmap_io *ctx = &(*mapent)->ent.evmap_io; fd = (*mapent)->fd; #else for (fd = 0; fd < iomap->nentries; ++fd) { struct evmap_io *ctx = iomap->entries[fd]; if (!ctx) continue; #endif if ((r = fn(base, fd, ctx, arg))) break; } return r; } /* Callback type for evmap_signal_foreach_signal */ typedef int (*evmap_signal_foreach_signal_cb)( struct event_base *, int, struct evmap_signal *, void *); /* Multipurpose helper function: Iterate over every signal number in the * event_base for which we could have signal events. For each such signal, * call fn(base, signum, evmap_signal, arg), where fn is the user-provided * function, base is the event_base, signum is the signal number, evmap_signal * is an evmap_signal structure containing a list of events pending on the * signal, and arg is the user-supplied argument. * * If fn returns 0, continue on to the next signal. Otherwise, return the same * value that fn returned. */ static int evmap_signal_foreach_signal(struct event_base *base, evmap_signal_foreach_signal_cb fn, void *arg) { struct event_signal_map *sigmap = &base->sigmap; int r = 0; int signum; for (signum = 0; signum < sigmap->nentries; ++signum) { struct evmap_signal *ctx = sigmap->entries[signum]; if (!ctx) continue; if ((r = fn(base, signum, ctx, arg))) break; } return r; } /* Helper for evmap_reinit_: tell the backend to add every fd for which we have * pending events, with the appropriate combination of EV_READ, EV_WRITE, and * EV_ET. */ static int evmap_io_reinit_iter_fn(struct event_base *base, evutil_socket_t fd, struct evmap_io *ctx, void *arg) { const struct eventop *evsel = base->evsel; void *extra; int *result = arg; short events = 0; struct event *ev; EVUTIL_ASSERT(ctx); extra = ((char*)ctx) + sizeof(struct evmap_io); if (ctx->nread) events |= EV_READ; if (ctx->nwrite) events |= EV_WRITE; if (ctx->nclose) events |= EV_CLOSED; if (evsel->fdinfo_len) memset(extra, 0, evsel->fdinfo_len); if (events && (ev = LIST_FIRST(&ctx->events)) && (ev->ev_events & EV_ET)) events |= EV_ET; if (evsel->add(base, fd, 0, events, extra) == -1) *result = -1; return 0; } /* Helper for evmap_reinit_: tell the backend to add every signal for which we * have pending events. */ static int evmap_signal_reinit_iter_fn(struct event_base *base, int signum, struct evmap_signal *ctx, void *arg) { const struct eventop *evsel = base->evsigsel; int *result = arg; if (!LIST_EMPTY(&ctx->events)) { if (evsel->add(base, signum, 0, EV_SIGNAL, NULL) == -1) *result = -1; } return 0; } int evmap_reinit_(struct event_base *base) { int result = 0; evmap_io_foreach_fd(base, evmap_io_reinit_iter_fn, &result); if (result < 0) return -1; evmap_signal_foreach_signal(base, evmap_signal_reinit_iter_fn, &result); if (result < 0) return -1; return 0; } /* Helper for evmap_delete_all_: delete every event in an event_dlist. */ static int delete_all_in_dlist(struct event_dlist *dlist) { struct event *ev; while ((ev = LIST_FIRST(dlist))) event_del(ev); return 0; } /* Helper for evmap_delete_all_: delete every event pending on an fd. */ static int evmap_io_delete_all_iter_fn(struct event_base *base, evutil_socket_t fd, struct evmap_io *io_info, void *arg) { return delete_all_in_dlist(&io_info->events); } /* Helper for evmap_delete_all_: delete every event pending on a signal. */ static int evmap_signal_delete_all_iter_fn(struct event_base *base, int signum, struct evmap_signal *sig_info, void *arg) { return delete_all_in_dlist(&sig_info->events); } void evmap_delete_all_(struct event_base *base) { evmap_signal_foreach_signal(base, evmap_signal_delete_all_iter_fn, NULL); evmap_io_foreach_fd(base, evmap_io_delete_all_iter_fn, NULL); } /** Per-fd structure for use with changelists. It keeps track, for each fd or * signal using the changelist, of where its entry in the changelist is. */ struct event_changelist_fdinfo { int idxplus1; /* this is the index +1, so that memset(0) will make it * a no-such-element */ }; void event_changelist_init_(struct event_changelist *changelist) { changelist->changes = NULL; changelist->changes_size = 0; changelist->n_changes = 0; } /** Helper: return the changelist_fdinfo corresponding to a given change. */ static inline struct event_changelist_fdinfo * event_change_get_fdinfo(struct event_base *base, const struct event_change *change) { char *ptr; if (change->read_change & EV_CHANGE_SIGNAL) { struct evmap_signal *ctx; GET_SIGNAL_SLOT(ctx, &base->sigmap, change->fd, evmap_signal); ptr = ((char*)ctx) + sizeof(struct evmap_signal); } else { struct evmap_io *ctx; GET_IO_SLOT(ctx, &base->io, change->fd, evmap_io); ptr = ((char*)ctx) + sizeof(struct evmap_io); } return (void*)ptr; } /** Callback helper for event_changelist_assert_ok */ static int event_changelist_assert_ok_foreach_iter_fn( struct event_base *base, evutil_socket_t fd, struct evmap_io *io, void *arg) { struct event_changelist *changelist = &base->changelist; struct event_changelist_fdinfo *f; f = (void*) ( ((char*)io) + sizeof(struct evmap_io) ); if (f->idxplus1) { struct event_change *c = &changelist->changes[f->idxplus1 - 1]; EVUTIL_ASSERT(c->fd == fd); } return 0; } /** Make sure that the changelist is consistent with the evmap structures. */ static void event_changelist_assert_ok(struct event_base *base) { int i; struct event_changelist *changelist = &base->changelist; EVUTIL_ASSERT(changelist->changes_size >= changelist->n_changes); for (i = 0; i < changelist->n_changes; ++i) { struct event_change *c = &changelist->changes[i]; struct event_changelist_fdinfo *f; EVUTIL_ASSERT(c->fd >= 0); f = event_change_get_fdinfo(base, c); EVUTIL_ASSERT(f); EVUTIL_ASSERT(f->idxplus1 == i + 1); } evmap_io_foreach_fd(base, event_changelist_assert_ok_foreach_iter_fn, NULL); } #ifdef DEBUG_CHANGELIST #define event_changelist_check(base) event_changelist_assert_ok((base)) #else #define event_changelist_check(base) ((void)0) #endif void event_changelist_remove_all_(struct event_changelist *changelist, struct event_base *base) { int i; event_changelist_check(base); for (i = 0; i < changelist->n_changes; ++i) { struct event_change *ch = &changelist->changes[i]; struct event_changelist_fdinfo *fdinfo = event_change_get_fdinfo(base, ch); EVUTIL_ASSERT(fdinfo->idxplus1 == i + 1); fdinfo->idxplus1 = 0; } changelist->n_changes = 0; event_changelist_check(base); } void event_changelist_freemem_(struct event_changelist *changelist) { if (changelist->changes) mm_free(changelist->changes); event_changelist_init_(changelist); /* zero it all out. */ } /** Increase the size of 'changelist' to hold more changes. */ static int event_changelist_grow(struct event_changelist *changelist) { int new_size; struct event_change *new_changes; if (changelist->changes_size < 64) new_size = 64; else new_size = changelist->changes_size * 2; new_changes = mm_realloc(changelist->changes, new_size * sizeof(struct event_change)); if (EVUTIL_UNLIKELY(new_changes == NULL)) return (-1); changelist->changes = new_changes; changelist->changes_size = new_size; return (0); } /** Return a pointer to the changelist entry for the file descriptor or signal * 'fd', whose fdinfo is 'fdinfo'. If none exists, construct it, setting its * old_events field to old_events. */ static struct event_change * event_changelist_get_or_construct(struct event_changelist *changelist, evutil_socket_t fd, short old_events, struct event_changelist_fdinfo *fdinfo) { struct event_change *change; if (fdinfo->idxplus1 == 0) { int idx; EVUTIL_ASSERT(changelist->n_changes <= changelist->changes_size); if (changelist->n_changes == changelist->changes_size) { if (event_changelist_grow(changelist) < 0) return NULL; } idx = changelist->n_changes++; change = &changelist->changes[idx]; fdinfo->idxplus1 = idx + 1; memset(change, 0, sizeof(struct event_change)); change->fd = fd; change->old_events = old_events; } else { change = &changelist->changes[fdinfo->idxplus1 - 1]; EVUTIL_ASSERT(change->fd == fd); } return change; } int event_changelist_add_(struct event_base *base, evutil_socket_t fd, short old, short events, void *p) { struct event_changelist *changelist = &base->changelist; struct event_changelist_fdinfo *fdinfo = p; struct event_change *change; event_changelist_check(base); change = event_changelist_get_or_construct(changelist, fd, old, fdinfo); if (!change) return -1; /* An add replaces any previous delete, but doesn't result in a no-op, * since the delete might fail (because the fd had been closed since * the last add, for instance. */ if (events & (EV_READ|EV_SIGNAL)) { change->read_change = EV_CHANGE_ADD | (events & (EV_ET|EV_PERSIST|EV_SIGNAL)); } if (events & EV_WRITE) { change->write_change = EV_CHANGE_ADD | (events & (EV_ET|EV_PERSIST|EV_SIGNAL)); } if (events & EV_CLOSED) { change->close_change = EV_CHANGE_ADD | (events & (EV_ET|EV_PERSIST|EV_SIGNAL)); } event_changelist_check(base); return (0); } int event_changelist_del_(struct event_base *base, evutil_socket_t fd, short old, short events, void *p) { struct event_changelist *changelist = &base->changelist; struct event_changelist_fdinfo *fdinfo = p; struct event_change *change; event_changelist_check(base); change = event_changelist_get_or_construct(changelist, fd, old, fdinfo); event_changelist_check(base); if (!change) return -1; /* A delete on an event set that doesn't contain the event to be deleted produces a no-op. This effectively emoves any previous uncommitted add, rather than replacing it: on those platforms where "add, delete, dispatch" is not the same as "no-op, dispatch", we want the no-op behavior. If we have a no-op item, we could remove it it from the list entirely, but really there's not much point: skipping the no-op change when we do the dispatch later is far cheaper than rejuggling the array now. As this stands, it also lets through deletions of events that are not currently set. */ if (events & (EV_READ|EV_SIGNAL)) { if (!(change->old_events & (EV_READ | EV_SIGNAL))) change->read_change = 0; else change->read_change = EV_CHANGE_DEL; } if (events & EV_WRITE) { if (!(change->old_events & EV_WRITE)) change->write_change = 0; else change->write_change = EV_CHANGE_DEL; } if (events & EV_CLOSED) { if (!(change->old_events & EV_CLOSED)) change->close_change = 0; else change->close_change = EV_CHANGE_DEL; } event_changelist_check(base); return (0); } /* Helper for evmap_check_integrity_: verify that all of the events pending on * given fd are set up correctly, and that the nread and nwrite counts on that * fd are correct. */ static int evmap_io_check_integrity_fn(struct event_base *base, evutil_socket_t fd, struct evmap_io *io_info, void *arg) { struct event *ev; int n_read = 0, n_write = 0, n_close = 0; /* First, make sure the list itself isn't corrupt. Otherwise, * running LIST_FOREACH could be an exciting adventure. */ EVUTIL_ASSERT_LIST_OK(&io_info->events, event, ev_io_next); LIST_FOREACH(ev, &io_info->events, ev_io_next) { EVUTIL_ASSERT(ev->ev_flags & EVLIST_INSERTED); EVUTIL_ASSERT(ev->ev_fd == fd); EVUTIL_ASSERT(!(ev->ev_events & EV_SIGNAL)); EVUTIL_ASSERT((ev->ev_events & (EV_READ|EV_WRITE|EV_CLOSED))); if (ev->ev_events & EV_READ) ++n_read; if (ev->ev_events & EV_WRITE) ++n_write; if (ev->ev_events & EV_CLOSED) ++n_close; } EVUTIL_ASSERT(n_read == io_info->nread); EVUTIL_ASSERT(n_write == io_info->nwrite); EVUTIL_ASSERT(n_close == io_info->nclose); return 0; } /* Helper for evmap_check_integrity_: verify that all of the events pending * on given signal are set up correctly. */ static int evmap_signal_check_integrity_fn(struct event_base *base, int signum, struct evmap_signal *sig_info, void *arg) { struct event *ev; /* First, make sure the list itself isn't corrupt. */ EVUTIL_ASSERT_LIST_OK(&sig_info->events, event, ev_signal_next); LIST_FOREACH(ev, &sig_info->events, ev_io_next) { EVUTIL_ASSERT(ev->ev_flags & EVLIST_INSERTED); EVUTIL_ASSERT(ev->ev_fd == signum); EVUTIL_ASSERT((ev->ev_events & EV_SIGNAL)); EVUTIL_ASSERT(!(ev->ev_events & (EV_READ|EV_WRITE|EV_CLOSED))); } return 0; } void evmap_check_integrity_(struct event_base *base) { evmap_io_foreach_fd(base, evmap_io_check_integrity_fn, NULL); evmap_signal_foreach_signal(base, evmap_signal_check_integrity_fn, NULL); if (base->evsel->add == event_changelist_add_) event_changelist_assert_ok(base); } /* Helper type for evmap_foreach_event_: Bundles a function to call on every * event, and the user-provided void* to use as its third argument. */ struct evmap_foreach_event_helper { event_base_foreach_event_cb fn; void *arg; }; /* Helper for evmap_foreach_event_: calls a provided function on every event * pending on a given fd. */ static int evmap_io_foreach_event_fn(struct event_base *base, evutil_socket_t fd, struct evmap_io *io_info, void *arg) { struct evmap_foreach_event_helper *h = arg; struct event *ev; int r; LIST_FOREACH(ev, &io_info->events, ev_io_next) { if ((r = h->fn(base, ev, h->arg))) return r; } return 0; } /* Helper for evmap_foreach_event_: calls a provided function on every event * pending on a given signal. */ static int evmap_signal_foreach_event_fn(struct event_base *base, int signum, struct evmap_signal *sig_info, void *arg) { struct event *ev; struct evmap_foreach_event_helper *h = arg; int r; LIST_FOREACH(ev, &sig_info->events, ev_signal_next) { if ((r = h->fn(base, ev, h->arg))) return r; } return 0; } int evmap_foreach_event_(struct event_base *base, event_base_foreach_event_cb fn, void *arg) { struct evmap_foreach_event_helper h; int r; h.fn = fn; h.arg = arg; if ((r = evmap_io_foreach_fd(base, evmap_io_foreach_event_fn, &h))) return r; return evmap_signal_foreach_signal(base, evmap_signal_foreach_event_fn, &h); } �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evport.c�����������������������������������������������������0000664�0000000�0000000�00000027345�15270523712�0021474�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Submitted by David Pacheco (dp.spambait@gmail.com) * * Copyright 2006-2007 Niels Provos * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY SUN MICROSYSTEMS, INC. ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL SUN MICROSYSTEMS, INC. BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2007 Sun Microsystems. All rights reserved. * Use is subject to license terms. */ /* * evport.c: event backend using Solaris 10 event ports. See port_create(3C). * This implementation is loosely modeled after the one used for select(2) (in * select.c). * * The outstanding events are tracked in a data structure called evport_data. * Each entry in the ed_fds array corresponds to a file descriptor, and contains * pointers to the read and write events that correspond to that fd. (That is, * when the file is readable, the "read" event should handle it, etc.) * * evport_add and evport_del update this data structure. evport_dispatch uses it * to determine where to callback when an event occurs (which it gets from * port_getn). * * Helper functions are used: grow() grows the file descriptor array as * necessary when large fd's come in. reassociate() takes care of maintaining * the proper file-descriptor/event-port associations. * * As in the select(2) implementation, signals are handled by evsignal. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_EVENT_PORTS #include <sys/time.h> #include <sys/queue.h> #include <errno.h> #include <poll.h> #include <port.h> #include <signal.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <time.h> #include <unistd.h> #include "event2/thread.h" #include "evthread-internal.h" #include "event-internal.h" #include "log-internal.h" #include "evsignal-internal.h" #include "evmap-internal.h" #define INITIAL_EVENTS_PER_GETN 8 #define MAX_EVENTS_PER_GETN 4096 /* * Per-file-descriptor information about what events we're subscribed to. These * fields are NULL if no event is subscribed to either of them. */ struct fd_info { /* combinations of EV_READ and EV_WRITE */ short fdi_what; /* Index of this fd within ed_pending, plus 1. Zero if this fd is * not in ed_pending. (The +1 is a hack so that memset(0) will set * it to a nil index. */ int pending_idx_plus_1; }; #define FDI_HAS_READ(fdi) ((fdi)->fdi_what & EV_READ) #define FDI_HAS_WRITE(fdi) ((fdi)->fdi_what & EV_WRITE) #define FDI_HAS_EVENTS(fdi) (FDI_HAS_READ(fdi) || FDI_HAS_WRITE(fdi)) #define FDI_TO_SYSEVENTS(fdi) (FDI_HAS_READ(fdi) ? POLLIN : 0) | \ (FDI_HAS_WRITE(fdi) ? POLLOUT : 0) struct evport_data { int ed_port; /* event port for system events */ /* How many elements of ed_pending should we look at? */ int ed_npending; /* How many elements are allocated in ed_pending and pevtlist? */ int ed_maxevents; /* fdi's that we need to reassoc */ int *ed_pending; /* storage space for incoming events. */ port_event_t *ed_pevtlist; }; static void* evport_init(struct event_base *); static int evport_add(struct event_base *, int fd, short old, short events, void *); static int evport_del(struct event_base *, int fd, short old, short events, void *); static int evport_dispatch(struct event_base *, struct timeval *); static void evport_dealloc(struct event_base *); static int grow(struct evport_data *, int min_events); const struct eventop evportops = { "evport", evport_init, evport_add, evport_del, evport_dispatch, evport_dealloc, 1, /* need reinit */ 0, /* features */ sizeof(struct fd_info), /* fdinfo length */ }; /* * Initialize the event port implementation. */ static void* evport_init(struct event_base *base) { struct evport_data *evpd; if (!(evpd = mm_calloc(1, sizeof(struct evport_data)))) return (NULL); if ((evpd->ed_port = port_create()) == -1) { mm_free(evpd); return (NULL); } if (grow(evpd, INITIAL_EVENTS_PER_GETN) < 0) { close(evpd->ed_port); mm_free(evpd); return NULL; } evpd->ed_npending = 0; evsig_init_(base); return (evpd); } static int grow(struct evport_data *data, int min_events) { int newsize; int *new_pending; port_event_t *new_pevtlist; if (data->ed_maxevents) { newsize = data->ed_maxevents; do { newsize *= 2; } while (newsize < min_events); } else { newsize = min_events; } new_pending = mm_realloc(data->ed_pending, sizeof(int)*newsize); if (new_pending == NULL) return -1; data->ed_pending = new_pending; new_pevtlist = mm_realloc(data->ed_pevtlist, sizeof(port_event_t)*newsize); if (new_pevtlist == NULL) return -1; data->ed_pevtlist = new_pevtlist; data->ed_maxevents = newsize; return 0; } #ifdef CHECK_INVARIANTS /* * Checks some basic properties about the evport_data structure. Because it * checks all file descriptors, this function can be expensive when the maximum * file descriptor ever used is rather large. */ static void check_evportop(struct evport_data *evpd) { EVUTIL_ASSERT(evpd); EVUTIL_ASSERT(evpd->ed_port > 0); } /* * Verifies very basic integrity of a given port_event. */ static void check_event(port_event_t* pevt) { /* * We've only registered for PORT_SOURCE_FD events. The only * other thing we can legitimately receive is PORT_SOURCE_ALERT, * but since we're not using port_alert either, we can assume * PORT_SOURCE_FD. */ EVUTIL_ASSERT(pevt->portev_source == PORT_SOURCE_FD); } #else #define check_evportop(epop) #define check_event(pevt) #endif /* CHECK_INVARIANTS */ /* * (Re)associates the given file descriptor with the event port. The OS events * are specified (implicitly) from the fd_info struct. */ static int reassociate(struct evport_data *epdp, struct fd_info *fdip, int fd) { int sysevents = FDI_TO_SYSEVENTS(fdip); if (sysevents != 0) { if (port_associate(epdp->ed_port, PORT_SOURCE_FD, fd, sysevents, fdip) == -1) { event_warn("port_associate"); return (-1); } } check_evportop(epdp); return (0); } /* * Main event loop - polls port_getn for some number of events, and processes * them. */ static int evport_dispatch(struct event_base *base, struct timeval *tv) { int i, res; struct evport_data *epdp = base->evbase; port_event_t *pevtlist = epdp->ed_pevtlist; /* * port_getn will block until it has at least nevents events. It will * also return how many it's given us (which may be more than we asked * for, as long as it's less than our maximum (ed_maxevents)) in * nevents. */ int nevents = 1; /* * We have to convert a struct timeval to a struct timespec * (only difference is nanoseconds vs. microseconds). If no time-based * events are active, we should wait for I/O (and tv == NULL). */ struct timespec ts; struct timespec *ts_p = NULL; if (tv != NULL) { ts.tv_sec = tv->tv_sec; ts.tv_nsec = tv->tv_usec * 1000; ts_p = &ts; } /* * Before doing anything else, we need to reassociate the events we hit * last time which need reassociation. See comment at the end of the * loop below. */ for (i = 0; i < epdp->ed_npending; ++i) { struct fd_info *fdi = NULL; const int fd = epdp->ed_pending[i]; if (fd != -1) { /* We might have cleared out this event; we need * to be sure that it's still set. */ fdi = evmap_io_get_fdinfo_(&base->io, fd); } if (fdi != NULL && FDI_HAS_EVENTS(fdi)) { reassociate(epdp, fdi, fd); /* epdp->ed_pending[i] = -1; */ fdi->pending_idx_plus_1 = 0; } } EVBASE_RELEASE_LOCK(base, th_base_lock); res = port_getn(epdp->ed_port, pevtlist, epdp->ed_maxevents, (unsigned int *) &nevents, ts_p); EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (res == -1) { if (errno == EINTR || errno == EAGAIN) { return (0); } else if (errno == ETIME) { if (nevents == 0) return (0); } else { event_warn("port_getn"); return (-1); } } event_debug(("%s: port_getn reports %d events", __func__, nevents)); for (i = 0; i < nevents; ++i) { port_event_t *pevt = &pevtlist[i]; int fd = (int) pevt->portev_object; struct fd_info *fdi = pevt->portev_user; /*EVUTIL_ASSERT(evmap_io_get_fdinfo_(&base->io, fd) == fdi);*/ check_evportop(epdp); check_event(pevt); epdp->ed_pending[i] = fd; fdi->pending_idx_plus_1 = i + 1; /* * Figure out what kind of event it was * (because we have to pass this to the callback) */ res = 0; if (pevt->portev_events & (POLLERR|POLLHUP)) { res = EV_READ | EV_WRITE; } else { if (pevt->portev_events & POLLIN) res |= EV_READ; if (pevt->portev_events & POLLOUT) res |= EV_WRITE; } /* * Check for the error situations or a hangup situation */ if (pevt->portev_events & (POLLERR|POLLHUP|POLLNVAL)) res |= EV_READ|EV_WRITE; evmap_io_active_(base, fd, res); } /* end of all events gotten */ epdp->ed_npending = nevents; if (nevents == epdp->ed_maxevents && epdp->ed_maxevents < MAX_EVENTS_PER_GETN) { /* we used all the space this time. We should be ready * for more events next time around. */ grow(epdp, epdp->ed_maxevents * 2); } check_evportop(epdp); return (0); } /* * Adds the given event (so that you will be notified when it happens via * the callback function). */ static int evport_add(struct event_base *base, int fd, short old, short events, void *p) { struct evport_data *evpd = base->evbase; struct fd_info *fdi = p; check_evportop(evpd); fdi->fdi_what |= events; return reassociate(evpd, fdi, fd); } /* * Removes the given event from the list of events to wait for. */ static int evport_del(struct event_base *base, int fd, short old, short events, void *p) { struct evport_data *evpd = base->evbase; struct fd_info *fdi = p; int associated = ! fdi->pending_idx_plus_1; check_evportop(evpd); fdi->fdi_what &= ~(events &(EV_READ|EV_WRITE)); if (associated) { if (!FDI_HAS_EVENTS(fdi) && port_dissociate(evpd->ed_port, PORT_SOURCE_FD, fd) == -1) { /* * Ignore EBADFD error the fd could have been closed * before event_del() was called. */ if (errno != EBADFD) { event_warn("port_dissociate"); return (-1); } } else { if (FDI_HAS_EVENTS(fdi)) { return (reassociate(evpd, fdi, fd)); } } } else { if ((fdi->fdi_what & (EV_READ|EV_WRITE)) == 0) { const int i = fdi->pending_idx_plus_1 - 1; EVUTIL_ASSERT(evpd->ed_pending[i] == fd); evpd->ed_pending[i] = -1; fdi->pending_idx_plus_1 = 0; } } return 0; } static void evport_dealloc(struct event_base *base) { struct evport_data *evpd = base->evbase; evsig_dealloc_(base); close(evpd->ed_port); if (evpd->ed_pending) mm_free(evpd->ed_pending); if (evpd->ed_pevtlist) mm_free(evpd->ed_pevtlist); mm_free(evpd); } #endif /* EVENT__HAVE_EVENT_PORTS */ �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evrpc-internal.h���������������������������������������������0000664�0000000�0000000�00000013117�15270523712�0023103�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2006-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVRPC_INTERNAL_H_INCLUDED_ #define EVRPC_INTERNAL_H_INCLUDED_ #include "event2/http.h" #include "http-internal.h" struct evrpc; struct evrpc_request_wrapper; #define EVRPC_URI_PREFIX "/.rpc." struct evrpc_hook { TAILQ_ENTRY(evrpc_hook) next; /* returns EVRPC_TERMINATE; if the rpc should be aborted. * a hook is is allowed to rewrite the evbuffer */ int (*process)(void *, struct evhttp_request *, struct evbuffer *, void *); void *process_arg; }; TAILQ_HEAD(evrpc_hook_list, evrpc_hook); /* * this is shared between the base and the pool, so that we can reuse * the hook adding functions; we alias both evrpc_pool and evrpc_base * to this common structure. */ struct evrpc_hook_ctx; TAILQ_HEAD(evrpc_pause_list, evrpc_hook_ctx); struct evrpc_hooks_ { /* hooks for processing outbound and inbound rpcs */ struct evrpc_hook_list in_hooks; struct evrpc_hook_list out_hooks; struct evrpc_pause_list pause_requests; }; #define input_hooks common.in_hooks #define output_hooks common.out_hooks #define paused_requests common.pause_requests struct evrpc_base { struct evrpc_hooks_ common; /* the HTTP server under which we register our RPC calls */ struct evhttp* http_server; /* a list of all RPCs registered with us */ TAILQ_HEAD(evrpc_list, evrpc) registered_rpcs; }; struct evrpc_req_generic; void evrpc_reqstate_free_(struct evrpc_req_generic* rpc_state); /* A pool for holding evhttp_connection objects */ struct evrpc_pool { struct evrpc_hooks_ common; struct event_base *base; struct evconq connections; int timeout; TAILQ_HEAD(evrpc_requestq, evrpc_request_wrapper) (requests); }; struct evrpc_hook_ctx { TAILQ_ENTRY(evrpc_hook_ctx) next; void *ctx; void (*cb)(void *, enum EVRPC_HOOK_RESULT); }; struct evrpc_meta { TAILQ_ENTRY(evrpc_meta) next; char *key; void *data; size_t data_size; }; TAILQ_HEAD(evrpc_meta_list, evrpc_meta); struct evrpc_hook_meta { struct evrpc_meta_list meta_data; struct evhttp_connection *evcon; }; /* allows association of meta data with a request */ static void evrpc_hook_associate_meta_(struct evrpc_hook_meta **pctx, struct evhttp_connection *evcon); /* creates a new meta data store */ static struct evrpc_hook_meta *evrpc_hook_meta_new_(void); /* frees the meta data associated with a request */ static void evrpc_hook_context_free_(struct evrpc_hook_meta *ctx); /* the server side of an rpc */ /* We alias the RPC specific structs to this voided one */ struct evrpc_req_generic { /* * allows association of meta data via hooks - needs to be * synchronized with evrpc_request_wrapper */ struct evrpc_hook_meta *hook_meta; /* the unmarshaled request object */ void *request; /* the empty reply object that needs to be filled in */ void *reply; /* * the static structure for this rpc; that can be used to * automatically unmarshal and marshal the http buffers. */ struct evrpc *rpc; /* * the http request structure on which we need to answer. */ struct evhttp_request* http_req; /* * Temporary data store for marshaled data */ struct evbuffer* rpc_data; }; /* the client side of an rpc request */ struct evrpc_request_wrapper { /* * allows association of meta data via hooks - needs to be * synchronized with evrpc_req_generic. */ struct evrpc_hook_meta *hook_meta; TAILQ_ENTRY(evrpc_request_wrapper) next; /* pool on which this rpc request is being made */ struct evrpc_pool *pool; /* connection on which the request is being sent */ struct evhttp_connection *evcon; /* the actual request */ struct evhttp_request *req; /* event for implementing request timeouts */ struct event ev_timeout; /* the name of the rpc */ char *name; /* callback */ void (*cb)(struct evrpc_status*, void *request, void *reply, void *arg); void *cb_arg; void *request; void *reply; /* unmarshals the buffer into the proper request structure */ void (*request_marshal)(struct evbuffer *, void *); /* removes all stored state in the reply */ void (*reply_clear)(void *); /* marshals the reply into a buffer */ int (*reply_unmarshal)(void *, struct evbuffer*); }; #endif /* EVRPC_INTERNAL_H_INCLUDED_ */ �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evrpc.c������������������������������������������������������0000664�0000000�0000000�00000071275�15270523712�0021275�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2000-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include <winsock2.h> #include <windows.h> #undef WIN32_LEAN_AND_MEAN #endif #include <sys/types.h> #ifndef _WIN32 #include <sys/socket.h> #endif #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #include <sys/queue.h> #include <stdio.h> #include <stdlib.h> #ifndef _WIN32 #include <unistd.h> #endif #include <errno.h> #include <signal.h> #include <string.h> #include <sys/queue.h> #include "event2/event.h" #include "event2/event_struct.h" #include "event2/rpc.h" #include "event2/rpc_struct.h" #include "evrpc-internal.h" #include "event2/http.h" #include "event2/buffer.h" #include "event2/tag.h" #include "event2/http_struct.h" #include "event2/http_compat.h" #include "event2/util.h" #include "util-internal.h" #include "log-internal.h" #include "mm-internal.h" struct evrpc_base * evrpc_init(struct evhttp *http_server) { struct evrpc_base* base = mm_calloc(1, sizeof(struct evrpc_base)); if (base == NULL) return (NULL); /* we rely on the tagging sub system */ evtag_init(); TAILQ_INIT(&base->registered_rpcs); TAILQ_INIT(&base->input_hooks); TAILQ_INIT(&base->output_hooks); TAILQ_INIT(&base->paused_requests); base->http_server = http_server; return (base); } void evrpc_free(struct evrpc_base *base) { struct evrpc *rpc; struct evrpc_hook *hook; struct evrpc_hook_ctx *pause; int r; while ((rpc = TAILQ_FIRST(&base->registered_rpcs)) != NULL) { r = evrpc_unregister_rpc(base, rpc->uri); EVUTIL_ASSERT(r == 0); } while ((pause = TAILQ_FIRST(&base->paused_requests)) != NULL) { TAILQ_REMOVE(&base->paused_requests, pause, next); mm_free(pause); } while ((hook = TAILQ_FIRST(&base->input_hooks)) != NULL) { r = evrpc_remove_hook(base, EVRPC_INPUT, hook); EVUTIL_ASSERT(r); } while ((hook = TAILQ_FIRST(&base->output_hooks)) != NULL) { r = evrpc_remove_hook(base, EVRPC_OUTPUT, hook); EVUTIL_ASSERT(r); } mm_free(base); } void * evrpc_add_hook(void *vbase, enum EVRPC_HOOK_TYPE hook_type, int (*cb)(void *, struct evhttp_request *, struct evbuffer *, void *), void *cb_arg) { struct evrpc_hooks_ *base = vbase; struct evrpc_hook_list *head = NULL; struct evrpc_hook *hook = NULL; switch (hook_type) { case EVRPC_INPUT: head = &base->in_hooks; break; case EVRPC_OUTPUT: head = &base->out_hooks; break; default: EVUTIL_ASSERT(hook_type == EVRPC_INPUT || hook_type == EVRPC_OUTPUT); } hook = mm_calloc(1, sizeof(struct evrpc_hook)); EVUTIL_ASSERT(hook != NULL); hook->process = cb; hook->process_arg = cb_arg; TAILQ_INSERT_TAIL(head, hook, next); return (hook); } static int evrpc_remove_hook_internal(struct evrpc_hook_list *head, void *handle) { struct evrpc_hook *hook = NULL; TAILQ_FOREACH(hook, head, next) { if (hook == handle) { TAILQ_REMOVE(head, hook, next); mm_free(hook); return (1); } } return (0); } /* * remove the hook specified by the handle */ int evrpc_remove_hook(void *vbase, enum EVRPC_HOOK_TYPE hook_type, void *handle) { struct evrpc_hooks_ *base = vbase; struct evrpc_hook_list *head = NULL; switch (hook_type) { case EVRPC_INPUT: head = &base->in_hooks; break; case EVRPC_OUTPUT: head = &base->out_hooks; break; default: EVUTIL_ASSERT(hook_type == EVRPC_INPUT || hook_type == EVRPC_OUTPUT); } return (evrpc_remove_hook_internal(head, handle)); } static int evrpc_process_hooks(struct evrpc_hook_list *head, void *ctx, struct evhttp_request *req, struct evbuffer *evbuf) { struct evrpc_hook *hook; TAILQ_FOREACH(hook, head, next) { int res = hook->process(ctx, req, evbuf, hook->process_arg); if (res != EVRPC_CONTINUE) return (res); } return (EVRPC_CONTINUE); } static void evrpc_pool_schedule(struct evrpc_pool *pool); static void evrpc_request_cb(struct evhttp_request *, void *); /* * Registers a new RPC with the HTTP server. The evrpc object is expected * to have been filled in via the EVRPC_REGISTER_OBJECT macro which in turn * calls this function. */ static char * evrpc_construct_uri(const char *uri) { char *constructed_uri; size_t constructed_uri_len; constructed_uri_len = strlen(EVRPC_URI_PREFIX) + strlen(uri) + 1; if ((constructed_uri = mm_malloc(constructed_uri_len)) == NULL) event_err(1, "%s: failed to register rpc at %s", __func__, uri); memcpy(constructed_uri, EVRPC_URI_PREFIX, strlen(EVRPC_URI_PREFIX)); memcpy(constructed_uri + strlen(EVRPC_URI_PREFIX), uri, strlen(uri)); constructed_uri[constructed_uri_len - 1] = '\0'; return (constructed_uri); } int evrpc_register_rpc(struct evrpc_base *base, struct evrpc *rpc, void (*cb)(struct evrpc_req_generic *, void *), void *cb_arg) { char *constructed_uri = evrpc_construct_uri(rpc->uri); rpc->base = base; rpc->cb = cb; rpc->cb_arg = cb_arg; TAILQ_INSERT_TAIL(&base->registered_rpcs, rpc, next); evhttp_set_cb(base->http_server, constructed_uri, evrpc_request_cb, rpc); mm_free(constructed_uri); return (0); } int evrpc_unregister_rpc(struct evrpc_base *base, const char *name) { char *registered_uri = NULL; struct evrpc *rpc; int r; /* find the right rpc; linear search might be slow */ TAILQ_FOREACH(rpc, &base->registered_rpcs, next) { if (strcmp(rpc->uri, name) == 0) break; } if (rpc == NULL) { /* We did not find an RPC with this name */ return (-1); } TAILQ_REMOVE(&base->registered_rpcs, rpc, next); registered_uri = evrpc_construct_uri(name); /* remove the http server callback */ r = evhttp_del_cb(base->http_server, registered_uri); EVUTIL_ASSERT(r == 0); mm_free(registered_uri); mm_free((char *)rpc->uri); mm_free(rpc); return (0); } static int evrpc_pause_request(void *vbase, void *ctx, void (*cb)(void *, enum EVRPC_HOOK_RESULT)); static void evrpc_request_cb_closure(void *, enum EVRPC_HOOK_RESULT); static void evrpc_request_cb(struct evhttp_request *req, void *arg) { struct evrpc *rpc = arg; struct evrpc_req_generic *rpc_state = NULL; /* let's verify the outside parameters */ if (req->type != EVHTTP_REQ_POST || evbuffer_get_length(req->input_buffer) <= 0) goto error; rpc_state = mm_calloc(1, sizeof(struct evrpc_req_generic)); if (rpc_state == NULL) goto error; rpc_state->rpc = rpc; rpc_state->http_req = req; rpc_state->rpc_data = NULL; if (TAILQ_FIRST(&rpc->base->input_hooks) != NULL) { int hook_res; evrpc_hook_associate_meta_(&rpc_state->hook_meta, req->evcon); /* * allow hooks to modify the outgoing request */ hook_res = evrpc_process_hooks(&rpc->base->input_hooks, rpc_state, req, req->input_buffer); switch (hook_res) { case EVRPC_TERMINATE: goto error; case EVRPC_PAUSE: evrpc_pause_request(rpc->base, rpc_state, evrpc_request_cb_closure); return; case EVRPC_CONTINUE: break; default: EVUTIL_ASSERT(hook_res == EVRPC_TERMINATE || hook_res == EVRPC_CONTINUE || hook_res == EVRPC_PAUSE); } } evrpc_request_cb_closure(rpc_state, EVRPC_CONTINUE); return; error: evrpc_reqstate_free_(rpc_state); evhttp_send_error(req, HTTP_SERVUNAVAIL, NULL); return; } static void evrpc_request_cb_closure(void *arg, enum EVRPC_HOOK_RESULT hook_res) { struct evrpc_req_generic *rpc_state = arg; struct evrpc *rpc; struct evhttp_request *req; EVUTIL_ASSERT(rpc_state); rpc = rpc_state->rpc; req = rpc_state->http_req; if (hook_res == EVRPC_TERMINATE) goto error; /* let's check that we can parse the request */ rpc_state->request = rpc->request_new(rpc->request_new_arg); if (rpc_state->request == NULL) goto error; if (rpc->request_unmarshal( rpc_state->request, req->input_buffer) == -1) { /* we failed to parse the request; that's a bummer */ goto error; } /* at this point, we have a well formed request, prepare the reply */ rpc_state->reply = rpc->reply_new(rpc->reply_new_arg); if (rpc_state->reply == NULL) goto error; /* give the rpc to the user; they can deal with it */ rpc->cb(rpc_state, rpc->cb_arg); return; error: evrpc_reqstate_free_(rpc_state); evhttp_send_error(req, HTTP_SERVUNAVAIL, NULL); return; } void evrpc_reqstate_free_(struct evrpc_req_generic* rpc_state) { struct evrpc *rpc; EVUTIL_ASSERT(rpc_state != NULL); rpc = rpc_state->rpc; /* clean up all memory */ if (rpc_state->hook_meta != NULL) evrpc_hook_context_free_(rpc_state->hook_meta); if (rpc_state->request != NULL) rpc->request_free(rpc_state->request); if (rpc_state->reply != NULL) rpc->reply_free(rpc_state->reply); if (rpc_state->rpc_data != NULL) evbuffer_free(rpc_state->rpc_data); mm_free(rpc_state); } static void evrpc_request_done_closure(void *, enum EVRPC_HOOK_RESULT); void evrpc_request_done(struct evrpc_req_generic *rpc_state) { struct evhttp_request *req; struct evrpc *rpc; EVUTIL_ASSERT(rpc_state); req = rpc_state->http_req; rpc = rpc_state->rpc; if (rpc->reply_complete(rpc_state->reply) == -1) { /* the reply was not completely filled in. error out */ goto error; } if ((rpc_state->rpc_data = evbuffer_new()) == NULL) { /* out of memory */ goto error; } /* serialize the reply */ rpc->reply_marshal(rpc_state->rpc_data, rpc_state->reply); if (TAILQ_FIRST(&rpc->base->output_hooks) != NULL) { int hook_res; evrpc_hook_associate_meta_(&rpc_state->hook_meta, req->evcon); /* do hook based tweaks to the request */ hook_res = evrpc_process_hooks(&rpc->base->output_hooks, rpc_state, req, rpc_state->rpc_data); switch (hook_res) { case EVRPC_TERMINATE: goto error; case EVRPC_PAUSE: if (evrpc_pause_request(rpc->base, rpc_state, evrpc_request_done_closure) == -1) goto error; return; case EVRPC_CONTINUE: break; default: EVUTIL_ASSERT(hook_res == EVRPC_TERMINATE || hook_res == EVRPC_CONTINUE || hook_res == EVRPC_PAUSE); } } evrpc_request_done_closure(rpc_state, EVRPC_CONTINUE); return; error: evrpc_reqstate_free_(rpc_state); evhttp_send_error(req, HTTP_SERVUNAVAIL, NULL); return; } void * evrpc_get_request(struct evrpc_req_generic *req) { return req->request; } void * evrpc_get_reply(struct evrpc_req_generic *req) { return req->reply; } static void evrpc_request_done_closure(void *arg, enum EVRPC_HOOK_RESULT hook_res) { struct evrpc_req_generic *rpc_state = arg; struct evhttp_request *req; EVUTIL_ASSERT(rpc_state); req = rpc_state->http_req; if (hook_res == EVRPC_TERMINATE) goto error; /* on success, we are going to transmit marshaled binary data */ if (evhttp_find_header(req->output_headers, "Content-Type") == NULL) { evhttp_add_header(req->output_headers, "Content-Type", "application/octet-stream"); } evhttp_send_reply(req, HTTP_OK, "OK", rpc_state->rpc_data); evrpc_reqstate_free_(rpc_state); return; error: evrpc_reqstate_free_(rpc_state); evhttp_send_error(req, HTTP_SERVUNAVAIL, NULL); return; } /* Client implementation of RPC site */ static int evrpc_schedule_request(struct evhttp_connection *connection, struct evrpc_request_wrapper *ctx); struct evrpc_pool * evrpc_pool_new(struct event_base *base) { struct evrpc_pool *pool = mm_calloc(1, sizeof(struct evrpc_pool)); if (pool == NULL) return (NULL); TAILQ_INIT(&pool->connections); TAILQ_INIT(&pool->requests); TAILQ_INIT(&pool->paused_requests); TAILQ_INIT(&pool->input_hooks); TAILQ_INIT(&pool->output_hooks); pool->base = base; pool->timeout = -1; return (pool); } static void evrpc_request_wrapper_free(struct evrpc_request_wrapper *request) { if (request->hook_meta != NULL) evrpc_hook_context_free_(request->hook_meta); mm_free(request->name); mm_free(request); } void evrpc_pool_free(struct evrpc_pool *pool) { struct evhttp_connection *connection; struct evrpc_request_wrapper *request; struct evrpc_hook_ctx *pause; struct evrpc_hook *hook; int r; while ((request = TAILQ_FIRST(&pool->requests)) != NULL) { TAILQ_REMOVE(&pool->requests, request, next); evrpc_request_wrapper_free(request); } while ((pause = TAILQ_FIRST(&pool->paused_requests)) != NULL) { TAILQ_REMOVE(&pool->paused_requests, pause, next); mm_free(pause); } while ((connection = TAILQ_FIRST(&pool->connections)) != NULL) { TAILQ_REMOVE(&pool->connections, connection, next); evhttp_connection_free(connection); } while ((hook = TAILQ_FIRST(&pool->input_hooks)) != NULL) { r = evrpc_remove_hook(pool, EVRPC_INPUT, hook); EVUTIL_ASSERT(r); } while ((hook = TAILQ_FIRST(&pool->output_hooks)) != NULL) { r = evrpc_remove_hook(pool, EVRPC_OUTPUT, hook); EVUTIL_ASSERT(r); } mm_free(pool); } /* * Add a connection to the RPC pool. A request scheduled on the pool * may use any available connection. */ void evrpc_pool_add_connection(struct evrpc_pool *pool, struct evhttp_connection *connection) { EVUTIL_ASSERT(connection->http_server == NULL); TAILQ_INSERT_TAIL(&pool->connections, connection, next); /* * associate an event base with this connection */ if (pool->base != NULL) evhttp_connection_set_base(connection, pool->base); /* * unless a timeout was specifically set for a connection, * the connection inherits the timeout from the pool. */ if (!evutil_timerisset(&connection->timeout)) evhttp_connection_set_timeout(connection, pool->timeout); /* * if we have any requests pending, schedule them with the new * connections. */ if (TAILQ_FIRST(&pool->requests) != NULL) { struct evrpc_request_wrapper *request = TAILQ_FIRST(&pool->requests); TAILQ_REMOVE(&pool->requests, request, next); evrpc_schedule_request(connection, request); } } void evrpc_pool_remove_connection(struct evrpc_pool *pool, struct evhttp_connection *connection) { TAILQ_REMOVE(&pool->connections, connection, next); } void evrpc_pool_set_timeout(struct evrpc_pool *pool, int timeout_in_secs) { struct evhttp_connection *evcon; TAILQ_FOREACH(evcon, &pool->connections, next) { evhttp_connection_set_timeout(evcon, timeout_in_secs); } pool->timeout = timeout_in_secs; } static void evrpc_reply_done(struct evhttp_request *, void *); static void evrpc_request_timeout(evutil_socket_t, short, void *); /* * Finds a connection object associated with the pool that is currently * idle and can be used to make a request. */ static struct evhttp_connection * evrpc_pool_find_connection(struct evrpc_pool *pool) { struct evhttp_connection *connection; TAILQ_FOREACH(connection, &pool->connections, next) { if (TAILQ_FIRST(&connection->requests) == NULL) return (connection); } return (NULL); } /* * Prototypes responsible for evrpc scheduling and hooking */ static void evrpc_schedule_request_closure(void *ctx, enum EVRPC_HOOK_RESULT); /* * We assume that the ctx is no longer queued on the pool. */ static int evrpc_schedule_request(struct evhttp_connection *connection, struct evrpc_request_wrapper *ctx) { struct evhttp_request *req = NULL; struct evrpc_pool *pool = ctx->pool; struct evrpc_status status; if ((req = evhttp_request_new(evrpc_reply_done, ctx)) == NULL) goto error; /* serialize the request data into the output buffer */ ctx->request_marshal(req->output_buffer, ctx->request); /* we need to know the connection that we might have to abort */ ctx->evcon = connection; /* if we get paused we also need to know the request */ ctx->req = req; if (TAILQ_FIRST(&pool->output_hooks) != NULL) { int hook_res; evrpc_hook_associate_meta_(&ctx->hook_meta, connection); /* apply hooks to the outgoing request */ hook_res = evrpc_process_hooks(&pool->output_hooks, ctx, req, req->output_buffer); switch (hook_res) { case EVRPC_TERMINATE: goto error; case EVRPC_PAUSE: /* we need to be explicitly resumed */ if (evrpc_pause_request(pool, ctx, evrpc_schedule_request_closure) == -1) goto error; return (0); case EVRPC_CONTINUE: /* we can just continue */ break; default: EVUTIL_ASSERT(hook_res == EVRPC_TERMINATE || hook_res == EVRPC_CONTINUE || hook_res == EVRPC_PAUSE); } } evrpc_schedule_request_closure(ctx, EVRPC_CONTINUE); return (0); error: memset(&status, 0, sizeof(status)); status.error = EVRPC_STATUS_ERR_UNSTARTED; (*ctx->cb)(&status, ctx->request, ctx->reply, ctx->cb_arg); evrpc_request_wrapper_free(ctx); return (-1); } static void evrpc_schedule_request_closure(void *arg, enum EVRPC_HOOK_RESULT hook_res) { struct evrpc_request_wrapper *ctx = arg; struct evhttp_connection *connection = ctx->evcon; struct evhttp_request *req = ctx->req; struct evrpc_pool *pool = ctx->pool; struct evrpc_status status; char *uri = NULL; int res = 0; if (hook_res == EVRPC_TERMINATE) goto error; uri = evrpc_construct_uri(ctx->name); if (uri == NULL) goto error; if (pool->timeout > 0) { /* * a timeout after which the whole rpc is going to be aborted. */ struct timeval tv; evutil_timerclear(&tv); tv.tv_sec = pool->timeout; evtimer_add(&ctx->ev_timeout, &tv); } /* start the request over the connection */ res = evhttp_make_request(connection, req, EVHTTP_REQ_POST, uri); mm_free(uri); if (res == -1) goto error; return; error: memset(&status, 0, sizeof(status)); status.error = EVRPC_STATUS_ERR_UNSTARTED; (*ctx->cb)(&status, ctx->request, ctx->reply, ctx->cb_arg); evrpc_request_wrapper_free(ctx); } /* we just queue the paused request on the pool under the req object */ static int evrpc_pause_request(void *vbase, void *ctx, void (*cb)(void *, enum EVRPC_HOOK_RESULT)) { struct evrpc_hooks_ *base = vbase; struct evrpc_hook_ctx *pause = mm_malloc(sizeof(*pause)); if (pause == NULL) return (-1); pause->ctx = ctx; pause->cb = cb; TAILQ_INSERT_TAIL(&base->pause_requests, pause, next); return (0); } int evrpc_resume_request(void *vbase, void *ctx, enum EVRPC_HOOK_RESULT res) { struct evrpc_hooks_ *base = vbase; struct evrpc_pause_list *head = &base->pause_requests; struct evrpc_hook_ctx *pause; TAILQ_FOREACH(pause, head, next) { if (pause->ctx == ctx) break; } if (pause == NULL) return (-1); (*pause->cb)(pause->ctx, res); TAILQ_REMOVE(head, pause, next); mm_free(pause); return (0); } int evrpc_make_request(struct evrpc_request_wrapper *ctx) { struct evrpc_pool *pool = ctx->pool; /* initialize the event structure for this rpc */ evtimer_assign(&ctx->ev_timeout, pool->base, evrpc_request_timeout, ctx); /* we better have some available connections on the pool */ EVUTIL_ASSERT(TAILQ_FIRST(&pool->connections) != NULL); /* * if no connection is available, we queue the request on the pool, * the next time a connection is empty, the rpc will be send on that. */ TAILQ_INSERT_TAIL(&pool->requests, ctx, next); evrpc_pool_schedule(pool); return (0); } struct evrpc_request_wrapper * evrpc_make_request_ctx( struct evrpc_pool *pool, void *request, void *reply, const char *rpcname, void (*req_marshal)(struct evbuffer*, void *), void (*rpl_clear)(void *), int (*rpl_unmarshal)(void *, struct evbuffer *), void (*cb)(struct evrpc_status *, void *, void *, void *), void *cbarg) { struct evrpc_request_wrapper *ctx = (struct evrpc_request_wrapper *) mm_malloc(sizeof(struct evrpc_request_wrapper)); if (ctx == NULL) return (NULL); ctx->pool = pool; ctx->hook_meta = NULL; ctx->evcon = NULL; ctx->name = mm_strdup(rpcname); if (ctx->name == NULL) { mm_free(ctx); return (NULL); } ctx->cb = cb; ctx->cb_arg = cbarg; ctx->request = request; ctx->reply = reply; ctx->request_marshal = req_marshal; ctx->reply_clear = rpl_clear; ctx->reply_unmarshal = rpl_unmarshal; return (ctx); } static void evrpc_reply_done_closure(void *, enum EVRPC_HOOK_RESULT); static void evrpc_reply_done(struct evhttp_request *req, void *arg) { struct evrpc_request_wrapper *ctx = arg; struct evrpc_pool *pool = ctx->pool; int hook_res = EVRPC_CONTINUE; /* cancel any timeout we might have scheduled */ event_del(&ctx->ev_timeout); ctx->req = req; /* we need to get the reply now */ if (req == NULL) { evrpc_reply_done_closure(ctx, EVRPC_CONTINUE); return; } if (TAILQ_FIRST(&pool->input_hooks) != NULL) { evrpc_hook_associate_meta_(&ctx->hook_meta, ctx->evcon); /* apply hooks to the incoming request */ hook_res = evrpc_process_hooks(&pool->input_hooks, ctx, req, req->input_buffer); switch (hook_res) { case EVRPC_TERMINATE: case EVRPC_CONTINUE: break; case EVRPC_PAUSE: /* * if we get paused we also need to know the * request. unfortunately, the underlying * layer is going to free it. we need to * request ownership explicitly */ if (req != NULL) evhttp_request_own(req); evrpc_pause_request(pool, ctx, evrpc_reply_done_closure); return; default: EVUTIL_ASSERT(hook_res == EVRPC_TERMINATE || hook_res == EVRPC_CONTINUE || hook_res == EVRPC_PAUSE); } } evrpc_reply_done_closure(ctx, hook_res); /* http request is being freed by underlying layer */ } static void evrpc_reply_done_closure(void *arg, enum EVRPC_HOOK_RESULT hook_res) { struct evrpc_request_wrapper *ctx = arg; struct evhttp_request *req = ctx->req; struct evrpc_pool *pool = ctx->pool; struct evrpc_status status; int res = -1; memset(&status, 0, sizeof(status)); status.http_req = req; /* we need to get the reply now */ if (req == NULL) { status.error = EVRPC_STATUS_ERR_TIMEOUT; } else if (hook_res == EVRPC_TERMINATE) { status.error = EVRPC_STATUS_ERR_HOOKABORTED; } else { res = ctx->reply_unmarshal(ctx->reply, req->input_buffer); if (res == -1) status.error = EVRPC_STATUS_ERR_BADPAYLOAD; } if (res == -1) { /* clear everything that we might have written previously */ ctx->reply_clear(ctx->reply); } (*ctx->cb)(&status, ctx->request, ctx->reply, ctx->cb_arg); evrpc_request_wrapper_free(ctx); /* the http layer owned the original request structure, but if we * got paused, we asked for ownership and need to free it here. */ if (req != NULL && evhttp_request_is_owned(req)) evhttp_request_free(req); /* see if we can schedule another request */ evrpc_pool_schedule(pool); } static void evrpc_pool_schedule(struct evrpc_pool *pool) { struct evrpc_request_wrapper *ctx = TAILQ_FIRST(&pool->requests); struct evhttp_connection *evcon; /* if no requests are pending, we have no work */ if (ctx == NULL) return; if ((evcon = evrpc_pool_find_connection(pool)) != NULL) { TAILQ_REMOVE(&pool->requests, ctx, next); evrpc_schedule_request(evcon, ctx); } } static void evrpc_request_timeout(evutil_socket_t fd, short what, void *arg) { struct evrpc_request_wrapper *ctx = arg; struct evhttp_connection *evcon = ctx->evcon; EVUTIL_ASSERT(evcon != NULL); evhttp_connection_fail_(evcon, EVREQ_HTTP_TIMEOUT); } /* * frees potential meta data associated with a request. */ static void evrpc_meta_data_free(struct evrpc_meta_list *meta_data) { struct evrpc_meta *entry; EVUTIL_ASSERT(meta_data != NULL); while ((entry = TAILQ_FIRST(meta_data)) != NULL) { TAILQ_REMOVE(meta_data, entry, next); mm_free(entry->key); mm_free(entry->data); mm_free(entry); } } static struct evrpc_hook_meta * evrpc_hook_meta_new_(void) { struct evrpc_hook_meta *ctx; ctx = mm_malloc(sizeof(struct evrpc_hook_meta)); EVUTIL_ASSERT(ctx != NULL); TAILQ_INIT(&ctx->meta_data); ctx->evcon = NULL; return (ctx); } static void evrpc_hook_associate_meta_(struct evrpc_hook_meta **pctx, struct evhttp_connection *evcon) { struct evrpc_hook_meta *ctx = *pctx; if (ctx == NULL) *pctx = ctx = evrpc_hook_meta_new_(); ctx->evcon = evcon; } static void evrpc_hook_context_free_(struct evrpc_hook_meta *ctx) { evrpc_meta_data_free(&ctx->meta_data); mm_free(ctx); } /* Adds meta data */ void evrpc_hook_add_meta(void *ctx, const char *key, const void *data, size_t data_size) { struct evrpc_request_wrapper *req = ctx; struct evrpc_hook_meta *store = NULL; struct evrpc_meta *meta = NULL; if ((store = req->hook_meta) == NULL) store = req->hook_meta = evrpc_hook_meta_new_(); meta = mm_malloc(sizeof(struct evrpc_meta)); EVUTIL_ASSERT(meta != NULL); meta->key = mm_strdup(key); EVUTIL_ASSERT(meta->key != NULL); meta->data_size = data_size; meta->data = mm_malloc(data_size); EVUTIL_ASSERT(meta->data != NULL); memcpy(meta->data, data, data_size); TAILQ_INSERT_TAIL(&store->meta_data, meta, next); } int evrpc_hook_find_meta(void *ctx, const char *key, void **data, size_t *data_size) { struct evrpc_request_wrapper *req = ctx; struct evrpc_meta *meta = NULL; if (req->hook_meta == NULL) return (-1); TAILQ_FOREACH(meta, &req->hook_meta->meta_data, next) { if (strcmp(meta->key, key) == 0) { *data = meta->data; *data_size = meta->data_size; return (0); } } return (-1); } struct evhttp_connection * evrpc_hook_get_connection(void *ctx) { struct evrpc_request_wrapper *req = ctx; return (req->hook_meta != NULL ? req->hook_meta->evcon : NULL); } int evrpc_send_request_generic(struct evrpc_pool *pool, void *request, void *reply, void (*cb)(struct evrpc_status *, void *, void *, void *), void *cb_arg, const char *rpcname, void (*req_marshal)(struct evbuffer *, void *), void (*rpl_clear)(void *), int (*rpl_unmarshal)(void *, struct evbuffer *)) { struct evrpc_status status; struct evrpc_request_wrapper *ctx; ctx = evrpc_make_request_ctx(pool, request, reply, rpcname, req_marshal, rpl_clear, rpl_unmarshal, cb, cb_arg); if (ctx == NULL) goto error; return (evrpc_make_request(ctx)); error: memset(&status, 0, sizeof(status)); status.error = EVRPC_STATUS_ERR_UNSTARTED; (*(cb))(&status, request, reply, cb_arg); return (-1); } /** Takes a request object and fills it in with the right magic */ static struct evrpc * evrpc_register_object(const char *name, void *(*req_new)(void*), void *req_new_arg, void (*req_free)(void *), int (*req_unmarshal)(void *, struct evbuffer *), void *(*rpl_new)(void*), void *rpl_new_arg, void (*rpl_free)(void *), int (*rpl_complete)(void *), void (*rpl_marshal)(struct evbuffer *, void *)) { struct evrpc* rpc = (struct evrpc *)mm_calloc(1, sizeof(struct evrpc)); if (rpc == NULL) return (NULL); rpc->uri = mm_strdup(name); if (rpc->uri == NULL) { mm_free(rpc); return (NULL); } rpc->request_new = req_new; rpc->request_new_arg = req_new_arg; rpc->request_free = req_free; rpc->request_unmarshal = req_unmarshal; rpc->reply_new = rpl_new; rpc->reply_new_arg = rpl_new_arg; rpc->reply_free = rpl_free; rpc->reply_complete = rpl_complete; rpc->reply_marshal = rpl_marshal; return (rpc); } int evrpc_register_generic(struct evrpc_base *base, const char *name, void (*callback)(struct evrpc_req_generic *, void *), void *cbarg, void *(*req_new)(void *), void *req_new_arg, void (*req_free)(void *), int (*req_unmarshal)(void *, struct evbuffer *), void *(*rpl_new)(void *), void *rpl_new_arg, void (*rpl_free)(void *), int (*rpl_complete)(void *), void (*rpl_marshal)(struct evbuffer *, void *)) { struct evrpc* rpc = evrpc_register_object(name, req_new, req_new_arg, req_free, req_unmarshal, rpl_new, rpl_new_arg, rpl_free, rpl_complete, rpl_marshal); if (rpc == NULL) return (-1); evrpc_register_rpc(base, rpc, (void (*)(struct evrpc_req_generic*, void *))callback, cbarg); return (0); } /** accessors for obscure and undocumented functionality */ struct evrpc_pool * evrpc_request_get_pool(struct evrpc_request_wrapper *ctx) { return (ctx->pool); } void evrpc_request_set_pool(struct evrpc_request_wrapper *ctx, struct evrpc_pool *pool) { ctx->pool = pool; } void evrpc_request_set_cb(struct evrpc_request_wrapper *ctx, void (*cb)(struct evrpc_status*, void *request, void *reply, void *arg), void *cb_arg) { ctx->cb = cb; ctx->cb_arg = cb_arg; } �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evsignal-internal.h������������������������������������������0000664�0000000�0000000�00000005106�15270523712�0023573�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright 2000-2007 Niels Provos <provos@citi.umich.edu> * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVSIGNAL_INTERNAL_H_INCLUDED_ #define EVSIGNAL_INTERNAL_H_INCLUDED_ #ifndef evutil_socket_t #include "event2/util.h" #endif #include <signal.h> typedef void (*ev_sighandler_t)(int); /* Data structure for the default signal-handling implementation in signal.c */ struct evsig_info { /* Event watching ev_signal_pair[1] */ struct event ev_signal; /* Socketpair used to send notifications from the signal handler */ evutil_socket_t ev_signal_pair[2]; /* True iff we've added the ev_signal event yet. */ int ev_signal_added; /* Count of the number of signals we're currently watching. */ int ev_n_signals_added; /* Array of previous signal handler objects before Libevent started * messing with them. Used to restore old signal handlers. */ #ifdef EVENT__HAVE_SIGACTION struct sigaction **sh_old; #else ev_sighandler_t **sh_old; #endif /* Size of sh_old. */ int sh_old_max; }; int evsig_init_(struct event_base *); void evsig_dealloc_(struct event_base *); void evsig_set_base_(struct event_base *base); void evsig_free_globals_(void); #endif /* EVSIGNAL_INTERNAL_H_INCLUDED_ */ ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evthread-internal.h������������������������������������������0000664�0000000�0000000�00000033770�15270523712�0023575�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2008-2012 Niels Provos, Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVTHREAD_INTERNAL_H_INCLUDED_ #define EVTHREAD_INTERNAL_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif #include "event2/event-config.h" #include "evconfig-private.h" #include "event2/thread.h" #include "util-internal.h" struct event_base; #ifndef _WIN32 /* On Windows, the way we currently make DLLs, it's not allowed for us to * have shared global structures. Thus, we only do the direct-call-to-function * code path if we know that the local shared library system supports it. */ #define EVTHREAD_EXPOSE_STRUCTS #endif #if ! defined(EVENT__DISABLE_THREAD_SUPPORT) && defined(EVTHREAD_EXPOSE_STRUCTS) /* Global function pointers to lock-related functions. NULL if locking isn't enabled. */ extern struct evthread_lock_callbacks evthread_lock_fns_; extern struct evthread_condition_callbacks evthread_cond_fns_; extern unsigned long (*evthread_id_fn_)(void); extern int evthread_lock_debugging_enabled_; /** Return the ID of the current thread, or 1 if threading isn't enabled. */ #define EVTHREAD_GET_ID() \ (evthread_id_fn_ ? evthread_id_fn_() : 1) /** Return true iff we're in the thread that is currently (or most recently) * running a given event_base's loop. Requires lock. */ #define EVBASE_IN_THREAD(base) \ (evthread_id_fn_ == NULL || \ (base)->th_owner_id == evthread_id_fn_()) /** Return true iff we need to notify the base's main thread about changes to * its state, because it's currently running the main loop in another * thread. Requires lock. */ #define EVBASE_NEED_NOTIFY(base) \ (evthread_id_fn_ != NULL && \ (base)->running_loop && \ (base)->th_owner_id != evthread_id_fn_()) /** Allocate a new lock, and store it in lockvar, a void*. Sets lockvar to NULL if locking is not enabled. */ #define EVTHREAD_ALLOC_LOCK(lockvar, locktype) \ ((lockvar) = evthread_lock_fns_.alloc ? \ evthread_lock_fns_.alloc(locktype) : NULL) /** Free a given lock, if it is present and locking is enabled. */ #define EVTHREAD_FREE_LOCK(lockvar, locktype) \ do { \ void *lock_tmp_ = (lockvar); \ if (lock_tmp_ && evthread_lock_fns_.free) \ evthread_lock_fns_.free(lock_tmp_, (locktype)); \ } while (0) /** Acquire a lock. */ #define EVLOCK_LOCK(lockvar,mode) \ do { \ if (lockvar) \ evthread_lock_fns_.lock(mode, lockvar); \ } while (0) /** Release a lock */ #define EVLOCK_UNLOCK(lockvar,mode) \ do { \ if (lockvar) \ evthread_lock_fns_.unlock(mode, lockvar); \ } while (0) /** Helper: put lockvar1 and lockvar2 into pointerwise ascending order. */ #define EVLOCK_SORTLOCKS_(lockvar1, lockvar2) \ do { \ if (lockvar1 && lockvar2 && lockvar1 > lockvar2) { \ void *tmp = lockvar1; \ lockvar1 = lockvar2; \ lockvar2 = tmp; \ } \ } while (0) /** Lock an event_base, if it is set up for locking. Acquires the lock in the base structure whose field is named 'lockvar'. */ #define EVBASE_ACQUIRE_LOCK(base, lockvar) do { \ EVLOCK_LOCK((base)->lockvar, 0); \ } while (0) /** Unlock an event_base, if it is set up for locking. */ #define EVBASE_RELEASE_LOCK(base, lockvar) do { \ EVLOCK_UNLOCK((base)->lockvar, 0); \ } while (0) /** If lock debugging is enabled, and lock is non-null, assert that 'lock' is * locked and held by us. */ #define EVLOCK_ASSERT_LOCKED(lock) \ do { \ if ((lock) && evthread_lock_debugging_enabled_) { \ EVUTIL_ASSERT(evthread_is_debug_lock_held_(lock)); \ } \ } while (0) /** Try to grab the lock for 'lockvar' without blocking, and return 1 if we * manage to get it. */ static inline int EVLOCK_TRY_LOCK_(void *lock); static inline int EVLOCK_TRY_LOCK_(void *lock) { if (lock && evthread_lock_fns_.lock) { int r = evthread_lock_fns_.lock(EVTHREAD_TRY, lock); return !r; } else { /* Locking is disabled either globally or for this thing; * of course we count as having the lock. */ return 1; } } /** Allocate a new condition variable and store it in the void *, condvar */ #define EVTHREAD_ALLOC_COND(condvar) \ do { \ (condvar) = evthread_cond_fns_.alloc_condition ? \ evthread_cond_fns_.alloc_condition(0) : NULL; \ } while (0) /** Deallocate and free a condition variable in condvar */ #define EVTHREAD_FREE_COND(cond) \ do { \ if (cond) \ evthread_cond_fns_.free_condition((cond)); \ } while (0) /** Signal one thread waiting on cond */ #define EVTHREAD_COND_SIGNAL(cond) \ ( (cond) ? evthread_cond_fns_.signal_condition((cond), 0) : 0 ) /** Signal all threads waiting on cond */ #define EVTHREAD_COND_BROADCAST(cond) \ ( (cond) ? evthread_cond_fns_.signal_condition((cond), 1) : 0 ) /** Wait until the condition 'cond' is signalled. Must be called while * holding 'lock'. The lock will be released until the condition is * signalled, at which point it will be acquired again. Returns 0 for * success, -1 for failure. */ #define EVTHREAD_COND_WAIT(cond, lock) \ ( (cond) ? evthread_cond_fns_.wait_condition((cond), (lock), NULL) : 0 ) /** As EVTHREAD_COND_WAIT, but gives up after 'tv' has elapsed. Returns 1 * on timeout. */ #define EVTHREAD_COND_WAIT_TIMED(cond, lock, tv) \ ( (cond) ? evthread_cond_fns_.wait_condition((cond), (lock), (tv)) : 0 ) /** True iff locking functions have been configured. */ #define EVTHREAD_LOCKING_ENABLED() \ (evthread_lock_fns_.lock != NULL) #elif ! defined(EVENT__DISABLE_THREAD_SUPPORT) unsigned long evthreadimpl_get_id_(void); int evthreadimpl_is_lock_debugging_enabled_(void); void *evthreadimpl_lock_alloc_(unsigned locktype); void evthreadimpl_lock_free_(void *lock, unsigned locktype); int evthreadimpl_lock_lock_(unsigned mode, void *lock); int evthreadimpl_lock_unlock_(unsigned mode, void *lock); void *evthreadimpl_cond_alloc_(unsigned condtype); void evthreadimpl_cond_free_(void *cond); int evthreadimpl_cond_signal_(void *cond, int broadcast); int evthreadimpl_cond_wait_(void *cond, void *lock, const struct timeval *tv); int evthreadimpl_locking_enabled_(void); #define EVTHREAD_GET_ID() evthreadimpl_get_id_() #define EVBASE_IN_THREAD(base) \ ((base)->th_owner_id == evthreadimpl_get_id_()) #define EVBASE_NEED_NOTIFY(base) \ ((base)->running_loop && \ ((base)->th_owner_id != evthreadimpl_get_id_())) #define EVTHREAD_ALLOC_LOCK(lockvar, locktype) \ ((lockvar) = evthreadimpl_lock_alloc_(locktype)) #define EVTHREAD_FREE_LOCK(lockvar, locktype) \ do { \ void *lock_tmp_ = (lockvar); \ if (lock_tmp_) \ evthreadimpl_lock_free_(lock_tmp_, (locktype)); \ } while (0) /** Acquire a lock. */ #define EVLOCK_LOCK(lockvar,mode) \ do { \ if (lockvar) \ evthreadimpl_lock_lock_(mode, lockvar); \ } while (0) /** Release a lock */ #define EVLOCK_UNLOCK(lockvar,mode) \ do { \ if (lockvar) \ evthreadimpl_lock_unlock_(mode, lockvar); \ } while (0) /** Lock an event_base, if it is set up for locking. Acquires the lock in the base structure whose field is named 'lockvar'. */ #define EVBASE_ACQUIRE_LOCK(base, lockvar) do { \ EVLOCK_LOCK((base)->lockvar, 0); \ } while (0) /** Unlock an event_base, if it is set up for locking. */ #define EVBASE_RELEASE_LOCK(base, lockvar) do { \ EVLOCK_UNLOCK((base)->lockvar, 0); \ } while (0) /** If lock debugging is enabled, and lock is non-null, assert that 'lock' is * locked and held by us. */ #define EVLOCK_ASSERT_LOCKED(lock) \ do { \ if ((lock) && evthreadimpl_is_lock_debugging_enabled_()) { \ EVUTIL_ASSERT(evthread_is_debug_lock_held_(lock)); \ } \ } while (0) /** Try to grab the lock for 'lockvar' without blocking, and return 1 if we * manage to get it. */ static inline int EVLOCK_TRY_LOCK_(void *lock); static inline int EVLOCK_TRY_LOCK_(void *lock) { if (lock) { int r = evthreadimpl_lock_lock_(EVTHREAD_TRY, lock); return !r; } else { /* Locking is disabled either globally or for this thing; * of course we count as having the lock. */ return 1; } } /** Allocate a new condition variable and store it in the void *, condvar */ #define EVTHREAD_ALLOC_COND(condvar) \ do { \ (condvar) = evthreadimpl_cond_alloc_(0); \ } while (0) /** Deallocate and free a condition variable in condvar */ #define EVTHREAD_FREE_COND(cond) \ do { \ if (cond) \ evthreadimpl_cond_free_((cond)); \ } while (0) /** Signal one thread waiting on cond */ #define EVTHREAD_COND_SIGNAL(cond) \ ( (cond) ? evthreadimpl_cond_signal_((cond), 0) : 0 ) /** Signal all threads waiting on cond */ #define EVTHREAD_COND_BROADCAST(cond) \ ( (cond) ? evthreadimpl_cond_signal_((cond), 1) : 0 ) /** Wait until the condition 'cond' is signalled. Must be called while * holding 'lock'. The lock will be released until the condition is * signalled, at which point it will be acquired again. Returns 0 for * success, -1 for failure. */ #define EVTHREAD_COND_WAIT(cond, lock) \ ( (cond) ? evthreadimpl_cond_wait_((cond), (lock), NULL) : 0 ) /** As EVTHREAD_COND_WAIT, but gives up after 'tv' has elapsed. Returns 1 * on timeout. */ #define EVTHREAD_COND_WAIT_TIMED(cond, lock, tv) \ ( (cond) ? evthreadimpl_cond_wait_((cond), (lock), (tv)) : 0 ) #define EVTHREAD_LOCKING_ENABLED() \ (evthreadimpl_locking_enabled_()) #else /* EVENT__DISABLE_THREAD_SUPPORT */ #define EVTHREAD_GET_ID() 1 #define EVTHREAD_ALLOC_LOCK(lockvar, locktype) EVUTIL_NIL_STMT_ #define EVTHREAD_FREE_LOCK(lockvar, locktype) EVUTIL_NIL_STMT_ #define EVLOCK_LOCK(lockvar, mode) EVUTIL_NIL_STMT_ #define EVLOCK_UNLOCK(lockvar, mode) EVUTIL_NIL_STMT_ #define EVLOCK_LOCK2(lock1,lock2,mode1,mode2) EVUTIL_NIL_STMT_ #define EVLOCK_UNLOCK2(lock1,lock2,mode1,mode2) EVUTIL_NIL_STMT_ #define EVBASE_IN_THREAD(base) 1 #define EVBASE_NEED_NOTIFY(base) 0 #define EVBASE_ACQUIRE_LOCK(base, lock) EVUTIL_NIL_STMT_ #define EVBASE_RELEASE_LOCK(base, lock) EVUTIL_NIL_STMT_ #define EVLOCK_ASSERT_LOCKED(lock) EVUTIL_NIL_STMT_ #define EVLOCK_TRY_LOCK_(lock) 1 #define EVTHREAD_ALLOC_COND(condvar) EVUTIL_NIL_STMT_ #define EVTHREAD_FREE_COND(cond) EVUTIL_NIL_STMT_ #define EVTHREAD_COND_SIGNAL(cond) EVUTIL_NIL_STMT_ #define EVTHREAD_COND_BROADCAST(cond) EVUTIL_NIL_STMT_ #define EVTHREAD_COND_WAIT(cond, lock) EVUTIL_NIL_STMT_ #define EVTHREAD_COND_WAIT_TIMED(cond, lock, howlong) EVUTIL_NIL_STMT_ #define EVTHREAD_LOCKING_ENABLED() 0 #endif /* This code is shared between both lock impls */ #if ! defined(EVENT__DISABLE_THREAD_SUPPORT) /** Helper: put lockvar1 and lockvar2 into pointerwise ascending order. */ #define EVLOCK_SORTLOCKS_(lockvar1, lockvar2) \ do { \ if (lockvar1 && lockvar2 && lockvar1 > lockvar2) { \ void *tmp = lockvar1; \ lockvar1 = lockvar2; \ lockvar2 = tmp; \ } \ } while (0) /** Acquire both lock1 and lock2. Always allocates locks in the same order, * so that two threads locking two locks with LOCK2 will not deadlock. */ #define EVLOCK_LOCK2(lock1,lock2,mode1,mode2) \ do { \ void *lock1_tmplock_ = (lock1); \ void *lock2_tmplock_ = (lock2); \ EVLOCK_SORTLOCKS_(lock1_tmplock_,lock2_tmplock_); \ EVLOCK_LOCK(lock1_tmplock_,mode1); \ if (lock2_tmplock_ != lock1_tmplock_) \ EVLOCK_LOCK(lock2_tmplock_,mode2); \ } while (0) /** Release both lock1 and lock2. */ #define EVLOCK_UNLOCK2(lock1,lock2,mode1,mode2) \ do { \ void *lock1_tmplock_ = (lock1); \ void *lock2_tmplock_ = (lock2); \ EVLOCK_SORTLOCKS_(lock1_tmplock_,lock2_tmplock_); \ if (lock2_tmplock_ != lock1_tmplock_) \ EVLOCK_UNLOCK(lock2_tmplock_,mode2); \ EVLOCK_UNLOCK(lock1_tmplock_,mode1); \ } while (0) int evthread_is_debug_lock_held_(void *lock); void *evthread_debug_get_real_lock_(void *lock); void *evthread_setup_global_lock_(void *lock_, unsigned locktype, int enable_locks); #define EVTHREAD_SETUP_GLOBAL_LOCK(lockvar, locktype) \ do { \ lockvar = evthread_setup_global_lock_(lockvar, \ (locktype), enable_locks); \ if (!lockvar) { \ event_warn("Couldn't allocate %s", #lockvar); \ return -1; \ } \ } while (0); int event_global_setup_locks_(const int enable_locks); int evsig_global_setup_locks_(const int enable_locks); int evutil_global_setup_locks_(const int enable_locks); int evutil_secure_rng_global_setup_locks_(const int enable_locks); /** Return current evthread_lock_callbacks */ struct evthread_lock_callbacks *evthread_get_lock_callbacks(void); /** Return current evthread_condition_callbacks */ struct evthread_condition_callbacks *evthread_get_condition_callbacks(void); /** Disable locking for internal usage (like global shutdown) */ void evthreadimpl_disable_lock_debugging_(void); #endif #ifdef __cplusplus } #endif #endif /* EVTHREAD_INTERNAL_H_INCLUDED_ */ ��������kytensor-2.49.0.7/third_party/libevent/evthread.c���������������������������������������������������0000664�0000000�0000000�00000032651�15270523712�0021753�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2008-2012 Niels Provos, Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifndef EVENT__DISABLE_THREAD_SUPPORT #include "event2/thread.h" #include <stdlib.h> #include <string.h> #include "log-internal.h" #include "mm-internal.h" #include "util-internal.h" #include "evthread-internal.h" #ifdef EVTHREAD_EXPOSE_STRUCTS #define GLOBAL #else #define GLOBAL static #endif #ifndef EVENT__DISABLE_DEBUG_MODE extern int event_debug_created_threadable_ctx_; extern int event_debug_mode_on_; #endif /* globals */ GLOBAL int evthread_lock_debugging_enabled_ = 0; GLOBAL struct evthread_lock_callbacks evthread_lock_fns_ = { 0, 0, NULL, NULL, NULL, NULL }; GLOBAL unsigned long (*evthread_id_fn_)(void) = NULL; GLOBAL struct evthread_condition_callbacks evthread_cond_fns_ = { 0, NULL, NULL, NULL, NULL }; /* Used for debugging */ static struct evthread_lock_callbacks original_lock_fns_ = { 0, 0, NULL, NULL, NULL, NULL }; static struct evthread_condition_callbacks original_cond_fns_ = { 0, NULL, NULL, NULL, NULL }; void evthread_set_id_callback(unsigned long (*id_fn)(void)) { evthread_id_fn_ = id_fn; } struct evthread_lock_callbacks *evthread_get_lock_callbacks() { return evthread_lock_debugging_enabled_ ? &original_lock_fns_ : &evthread_lock_fns_; } struct evthread_condition_callbacks *evthread_get_condition_callbacks() { return evthread_lock_debugging_enabled_ ? &original_cond_fns_ : &evthread_cond_fns_; } void evthreadimpl_disable_lock_debugging_(void) { evthread_lock_debugging_enabled_ = 0; } int evthread_set_lock_callbacks(const struct evthread_lock_callbacks *cbs) { struct evthread_lock_callbacks *target = evthread_get_lock_callbacks(); #ifndef EVENT__DISABLE_DEBUG_MODE if (event_debug_mode_on_) { if (event_debug_created_threadable_ctx_) { event_errx(1, "evthread initialization must be called BEFORE anything else!"); } } #endif if (!cbs) { if (target->alloc) event_warnx("Trying to disable lock functions after " "they have been set up will probaby not work."); memset(target, 0, sizeof(evthread_lock_fns_)); return 0; } if (target->alloc) { /* Uh oh; we already had locking callbacks set up.*/ if (target->lock_api_version == cbs->lock_api_version && target->supported_locktypes == cbs->supported_locktypes && target->alloc == cbs->alloc && target->free == cbs->free && target->lock == cbs->lock && target->unlock == cbs->unlock) { /* no change -- allow this. */ return 0; } event_warnx("Can't change lock callbacks once they have been " "initialized."); return -1; } if (cbs->alloc && cbs->free && cbs->lock && cbs->unlock) { memcpy(target, cbs, sizeof(evthread_lock_fns_)); return event_global_setup_locks_(1); } else { return -1; } } int evthread_set_condition_callbacks(const struct evthread_condition_callbacks *cbs) { struct evthread_condition_callbacks *target = evthread_get_condition_callbacks(); #ifndef EVENT__DISABLE_DEBUG_MODE if (event_debug_mode_on_) { if (event_debug_created_threadable_ctx_) { event_errx(1, "evthread initialization must be called BEFORE anything else!"); } } #endif if (!cbs) { if (target->alloc_condition) event_warnx("Trying to disable condition functions " "after they have been set up will probaby not " "work."); memset(target, 0, sizeof(evthread_cond_fns_)); return 0; } if (target->alloc_condition) { /* Uh oh; we already had condition callbacks set up.*/ if (target->condition_api_version == cbs->condition_api_version && target->alloc_condition == cbs->alloc_condition && target->free_condition == cbs->free_condition && target->signal_condition == cbs->signal_condition && target->wait_condition == cbs->wait_condition) { /* no change -- allow this. */ return 0; } event_warnx("Can't change condition callbacks once they " "have been initialized."); return -1; } if (cbs->alloc_condition && cbs->free_condition && cbs->signal_condition && cbs->wait_condition) { memcpy(target, cbs, sizeof(evthread_cond_fns_)); } if (evthread_lock_debugging_enabled_) { evthread_cond_fns_.alloc_condition = cbs->alloc_condition; evthread_cond_fns_.free_condition = cbs->free_condition; evthread_cond_fns_.signal_condition = cbs->signal_condition; } return 0; } #define DEBUG_LOCK_SIG 0xdeb0b10c struct debug_lock { unsigned signature; unsigned locktype; unsigned long held_by; /* XXXX if we ever use read-write locks, we will need a separate * lock to protect count. */ int count; void *lock; }; static void * debug_lock_alloc(unsigned locktype) { struct debug_lock *result = mm_malloc(sizeof(struct debug_lock)); if (!result) return NULL; if (original_lock_fns_.alloc) { if (!(result->lock = original_lock_fns_.alloc( locktype|EVTHREAD_LOCKTYPE_RECURSIVE))) { mm_free(result); return NULL; } } else { result->lock = NULL; } result->signature = DEBUG_LOCK_SIG; result->locktype = locktype; result->count = 0; result->held_by = 0; return result; } static void debug_lock_free(void *lock_, unsigned locktype) { struct debug_lock *lock = lock_; EVUTIL_ASSERT(lock->count == 0); EVUTIL_ASSERT(locktype == lock->locktype); EVUTIL_ASSERT(DEBUG_LOCK_SIG == lock->signature); if (original_lock_fns_.free) { original_lock_fns_.free(lock->lock, lock->locktype|EVTHREAD_LOCKTYPE_RECURSIVE); } lock->lock = NULL; lock->count = -100; lock->signature = 0x12300fda; mm_free(lock); } static void evthread_debug_lock_mark_locked(unsigned mode, struct debug_lock *lock) { EVUTIL_ASSERT(DEBUG_LOCK_SIG == lock->signature); ++lock->count; if (!(lock->locktype & EVTHREAD_LOCKTYPE_RECURSIVE)) EVUTIL_ASSERT(lock->count == 1); if (evthread_id_fn_) { unsigned long me; me = evthread_id_fn_(); if (lock->count > 1) EVUTIL_ASSERT(lock->held_by == me); lock->held_by = me; } } static int debug_lock_lock(unsigned mode, void *lock_) { struct debug_lock *lock = lock_; int res = 0; if (lock->locktype & EVTHREAD_LOCKTYPE_READWRITE) EVUTIL_ASSERT(mode & (EVTHREAD_READ|EVTHREAD_WRITE)); else EVUTIL_ASSERT((mode & (EVTHREAD_READ|EVTHREAD_WRITE)) == 0); if (original_lock_fns_.lock) res = original_lock_fns_.lock(mode, lock->lock); if (!res) { evthread_debug_lock_mark_locked(mode, lock); } return res; } static void evthread_debug_lock_mark_unlocked(unsigned mode, struct debug_lock *lock) { EVUTIL_ASSERT(DEBUG_LOCK_SIG == lock->signature); if (lock->locktype & EVTHREAD_LOCKTYPE_READWRITE) EVUTIL_ASSERT(mode & (EVTHREAD_READ|EVTHREAD_WRITE)); else EVUTIL_ASSERT((mode & (EVTHREAD_READ|EVTHREAD_WRITE)) == 0); if (evthread_id_fn_) { unsigned long me; me = evthread_id_fn_(); EVUTIL_ASSERT(lock->held_by == me); if (lock->count == 1) lock->held_by = 0; } --lock->count; EVUTIL_ASSERT(lock->count >= 0); } static int debug_lock_unlock(unsigned mode, void *lock_) { struct debug_lock *lock = lock_; int res = 0; evthread_debug_lock_mark_unlocked(mode, lock); if (original_lock_fns_.unlock) res = original_lock_fns_.unlock(mode, lock->lock); return res; } static int debug_cond_wait(void *cond_, void *lock_, const struct timeval *tv) { int r; struct debug_lock *lock = lock_; EVUTIL_ASSERT(lock); EVUTIL_ASSERT(DEBUG_LOCK_SIG == lock->signature); EVLOCK_ASSERT_LOCKED(lock_); evthread_debug_lock_mark_unlocked(0, lock); r = original_cond_fns_.wait_condition(cond_, lock->lock, tv); evthread_debug_lock_mark_locked(0, lock); return r; } /* misspelled version for backward compatibility */ void evthread_enable_lock_debuging(void) { evthread_enable_lock_debugging(); } void evthread_enable_lock_debugging(void) { struct evthread_lock_callbacks cbs = { EVTHREAD_LOCK_API_VERSION, EVTHREAD_LOCKTYPE_RECURSIVE, debug_lock_alloc, debug_lock_free, debug_lock_lock, debug_lock_unlock }; if (evthread_lock_debugging_enabled_) return; memcpy(&original_lock_fns_, &evthread_lock_fns_, sizeof(struct evthread_lock_callbacks)); memcpy(&evthread_lock_fns_, &cbs, sizeof(struct evthread_lock_callbacks)); memcpy(&original_cond_fns_, &evthread_cond_fns_, sizeof(struct evthread_condition_callbacks)); evthread_cond_fns_.wait_condition = debug_cond_wait; evthread_lock_debugging_enabled_ = 1; /* XXX return value should get checked. */ event_global_setup_locks_(0); } int evthread_is_debug_lock_held_(void *lock_) { struct debug_lock *lock = lock_; if (! lock->count) return 0; if (evthread_id_fn_) { unsigned long me = evthread_id_fn_(); if (lock->held_by != me) return 0; } return 1; } void * evthread_debug_get_real_lock_(void *lock_) { struct debug_lock *lock = lock_; return lock->lock; } void * evthread_setup_global_lock_(void *lock_, unsigned locktype, int enable_locks) { /* there are four cases here: 1) we're turning on debugging; locking is not on. 2) we're turning on debugging; locking is on. 3) we're turning on locking; debugging is not on. 4) we're turning on locking; debugging is on. */ if (!enable_locks && original_lock_fns_.alloc == NULL) { /* Case 1: allocate a debug lock. */ EVUTIL_ASSERT(lock_ == NULL); return debug_lock_alloc(locktype); } else if (!enable_locks && original_lock_fns_.alloc != NULL) { /* Case 2: wrap the lock in a debug lock. */ struct debug_lock *lock; EVUTIL_ASSERT(lock_ != NULL); if (!(locktype & EVTHREAD_LOCKTYPE_RECURSIVE)) { /* We can't wrap it: We need a recursive lock */ original_lock_fns_.free(lock_, locktype); return debug_lock_alloc(locktype); } lock = mm_malloc(sizeof(struct debug_lock)); if (!lock) { original_lock_fns_.free(lock_, locktype); return NULL; } lock->lock = lock_; lock->locktype = locktype; lock->count = 0; lock->held_by = 0; return lock; } else if (enable_locks && ! evthread_lock_debugging_enabled_) { /* Case 3: allocate a regular lock */ EVUTIL_ASSERT(lock_ == NULL); return evthread_lock_fns_.alloc(locktype); } else { /* Case 4: Fill in a debug lock with a real lock */ struct debug_lock *lock = lock_ ? lock_ : debug_lock_alloc(locktype); EVUTIL_ASSERT(enable_locks && evthread_lock_debugging_enabled_); EVUTIL_ASSERT(lock->locktype == locktype); if (!lock->lock) { lock->lock = original_lock_fns_.alloc( locktype|EVTHREAD_LOCKTYPE_RECURSIVE); if (!lock->lock) { lock->count = -200; mm_free(lock); return NULL; } } return lock; } } #ifndef EVTHREAD_EXPOSE_STRUCTS unsigned long evthreadimpl_get_id_() { return evthread_id_fn_ ? evthread_id_fn_() : 1; } void * evthreadimpl_lock_alloc_(unsigned locktype) { #ifndef EVENT__DISABLE_DEBUG_MODE if (event_debug_mode_on_) { event_debug_created_threadable_ctx_ = 1; } #endif return evthread_lock_fns_.alloc ? evthread_lock_fns_.alloc(locktype) : NULL; } void evthreadimpl_lock_free_(void *lock, unsigned locktype) { if (evthread_lock_fns_.free) evthread_lock_fns_.free(lock, locktype); } int evthreadimpl_lock_lock_(unsigned mode, void *lock) { if (evthread_lock_fns_.lock) return evthread_lock_fns_.lock(mode, lock); else return 0; } int evthreadimpl_lock_unlock_(unsigned mode, void *lock) { if (evthread_lock_fns_.unlock) return evthread_lock_fns_.unlock(mode, lock); else return 0; } void * evthreadimpl_cond_alloc_(unsigned condtype) { #ifndef EVENT__DISABLE_DEBUG_MODE if (event_debug_mode_on_) { event_debug_created_threadable_ctx_ = 1; } #endif return evthread_cond_fns_.alloc_condition ? evthread_cond_fns_.alloc_condition(condtype) : NULL; } void evthreadimpl_cond_free_(void *cond) { if (evthread_cond_fns_.free_condition) evthread_cond_fns_.free_condition(cond); } int evthreadimpl_cond_signal_(void *cond, int broadcast) { if (evthread_cond_fns_.signal_condition) return evthread_cond_fns_.signal_condition(cond, broadcast); else return 0; } int evthreadimpl_cond_wait_(void *cond, void *lock, const struct timeval *tv) { if (evthread_cond_fns_.wait_condition) return evthread_cond_fns_.wait_condition(cond, lock, tv); else return 0; } int evthreadimpl_is_lock_debugging_enabled_(void) { return evthread_lock_debugging_enabled_; } int evthreadimpl_locking_enabled_(void) { return evthread_lock_fns_.lock != NULL; } #endif #endif ���������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evthread_pthread.c�������������������������������������������0000664�0000000�0000000�00000011535�15270523712�0023460�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" /* With glibc we need to define _GNU_SOURCE to get PTHREAD_MUTEX_RECURSIVE. * This comes from evconfig-private.h */ #include <pthread.h> struct event_base; #include "event2/thread.h" #include <stdlib.h> #include <string.h> #include "mm-internal.h" #include "evthread-internal.h" static pthread_mutexattr_t attr_recursive; static void * evthread_posix_lock_alloc(unsigned locktype) { pthread_mutexattr_t *attr = NULL; pthread_mutex_t *lock = mm_malloc(sizeof(pthread_mutex_t)); if (!lock) return NULL; if (locktype & EVTHREAD_LOCKTYPE_RECURSIVE) attr = &attr_recursive; if (pthread_mutex_init(lock, attr)) { mm_free(lock); return NULL; } return lock; } static void evthread_posix_lock_free(void *lock_, unsigned locktype) { pthread_mutex_t *lock = lock_; pthread_mutex_destroy(lock); mm_free(lock); } static int evthread_posix_lock(unsigned mode, void *lock_) { pthread_mutex_t *lock = lock_; if (mode & EVTHREAD_TRY) return pthread_mutex_trylock(lock); else return pthread_mutex_lock(lock); } static int evthread_posix_unlock(unsigned mode, void *lock_) { pthread_mutex_t *lock = lock_; return pthread_mutex_unlock(lock); } static unsigned long evthread_posix_get_id(void) { union { pthread_t thr; #if EVENT__SIZEOF_PTHREAD_T > EVENT__SIZEOF_LONG ev_uint64_t id; #else unsigned long id; #endif } r; #if EVENT__SIZEOF_PTHREAD_T < EVENT__SIZEOF_LONG memset(&r, 0, sizeof(r)); #endif r.thr = pthread_self(); return (unsigned long)r.id; } static void * evthread_posix_cond_alloc(unsigned condflags) { pthread_cond_t *cond = mm_malloc(sizeof(pthread_cond_t)); if (!cond) return NULL; if (pthread_cond_init(cond, NULL)) { mm_free(cond); return NULL; } return cond; } static void evthread_posix_cond_free(void *cond_) { pthread_cond_t *cond = cond_; pthread_cond_destroy(cond); mm_free(cond); } static int evthread_posix_cond_signal(void *cond_, int broadcast) { pthread_cond_t *cond = cond_; int r; if (broadcast) r = pthread_cond_broadcast(cond); else r = pthread_cond_signal(cond); return r ? -1 : 0; } static int evthread_posix_cond_wait(void *cond_, void *lock_, const struct timeval *tv) { int r; pthread_cond_t *cond = cond_; pthread_mutex_t *lock = lock_; if (tv) { struct timeval now, abstime; struct timespec ts; evutil_gettimeofday(&now, NULL); evutil_timeradd(&now, tv, &abstime); ts.tv_sec = abstime.tv_sec; ts.tv_nsec = abstime.tv_usec*1000; r = pthread_cond_timedwait(cond, lock, &ts); if (r == ETIMEDOUT) return 1; else if (r) return -1; else return 0; } else { r = pthread_cond_wait(cond, lock); return r ? -1 : 0; } } int evthread_use_pthreads(void) { struct evthread_lock_callbacks cbs = { EVTHREAD_LOCK_API_VERSION, EVTHREAD_LOCKTYPE_RECURSIVE, evthread_posix_lock_alloc, evthread_posix_lock_free, evthread_posix_lock, evthread_posix_unlock }; struct evthread_condition_callbacks cond_cbs = { EVTHREAD_CONDITION_API_VERSION, evthread_posix_cond_alloc, evthread_posix_cond_free, evthread_posix_cond_signal, evthread_posix_cond_wait }; /* Set ourselves up to get recursive locks. */ if (pthread_mutexattr_init(&attr_recursive)) return -1; if (pthread_mutexattr_settype(&attr_recursive, PTHREAD_MUTEX_RECURSIVE)) return -1; evthread_set_lock_callbacks(&cbs); evthread_set_condition_callbacks(&cond_cbs); evthread_set_id_callback(evthread_posix_get_id); return 0; } �������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evthread_win32.c���������������������������������������������0000664�0000000�0000000�00000020623�15270523712�0022771�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #ifndef _WIN32_WINNT /* Minimum required for InitializeCriticalSectionAndSpinCount */ #define _WIN32_WINNT 0x0403 #endif #include <winsock2.h> #define WIN32_LEAN_AND_MEAN #include <windows.h> #undef WIN32_LEAN_AND_MEAN #include <sys/locking.h> #endif struct event_base; #include "event2/thread.h" #include "mm-internal.h" #include "evthread-internal.h" #include "time-internal.h" #define SPIN_COUNT 2000 static void * evthread_win32_lock_create(unsigned locktype) { CRITICAL_SECTION *lock = mm_malloc(sizeof(CRITICAL_SECTION)); if (!lock) return NULL; if (InitializeCriticalSectionAndSpinCount(lock, SPIN_COUNT) == 0) { mm_free(lock); return NULL; } return lock; } static void evthread_win32_lock_free(void *lock_, unsigned locktype) { CRITICAL_SECTION *lock = lock_; DeleteCriticalSection(lock); mm_free(lock); } static int evthread_win32_lock(unsigned mode, void *lock_) { CRITICAL_SECTION *lock = lock_; if ((mode & EVTHREAD_TRY)) { return ! TryEnterCriticalSection(lock); } else { EnterCriticalSection(lock); return 0; } } static int evthread_win32_unlock(unsigned mode, void *lock_) { CRITICAL_SECTION *lock = lock_; LeaveCriticalSection(lock); return 0; } static unsigned long evthread_win32_get_id(void) { return (unsigned long) GetCurrentThreadId(); } #ifdef WIN32_HAVE_CONDITION_VARIABLES static void WINAPI (*InitializeConditionVariable_fn)(PCONDITION_VARIABLE) = NULL; static BOOL WINAPI (*SleepConditionVariableCS_fn)( PCONDITION_VARIABLE, PCRITICAL_SECTION, DWORD) = NULL; static void WINAPI (*WakeAllConditionVariable_fn)(PCONDITION_VARIABLE) = NULL; static void WINAPI (*WakeConditionVariable_fn)(PCONDITION_VARIABLE) = NULL; static int evthread_win32_condvar_init(void) { HANDLE lib; lib = GetModuleHandle(TEXT("kernel32.dll")); if (lib == NULL) return 0; #define LOAD(name) \ name##_fn = GetProcAddress(lib, #name) LOAD(InitializeConditionVariable); LOAD(SleepConditionVariableCS); LOAD(WakeAllConditionVariable); LOAD(WakeConditionVariable); return InitializeConditionVariable_fn && SleepConditionVariableCS_fn && WakeAllConditionVariable_fn && WakeConditionVariable_fn; } /* XXXX Even if we can build this, we don't necessarily want to: the functions * in question didn't exist before Vista, so we'd better LoadProc them. */ static void * evthread_win32_condvar_alloc(unsigned condflags) { CONDITION_VARIABLE *cond = mm_malloc(sizeof(CONDITION_VARIABLE)); if (!cond) return NULL; InitializeConditionVariable_fn(cond); return cond; } static void evthread_win32_condvar_free(void *cond_) { CONDITION_VARIABLE *cond = cond_; /* There doesn't _seem_ to be a cleaup fn here... */ mm_free(cond); } static int evthread_win32_condvar_signal(void *cond, int broadcast) { CONDITION_VARIABLE *cond = cond_; if (broadcast) WakeAllConditionVariable_fn(cond); else WakeConditionVariable_fn(cond); return 0; } static int evthread_win32_condvar_wait(void *cond_, void *lock_, const struct timeval *tv) { CONDITION_VARIABLE *cond = cond_; CRITICAL_SECTION *lock = lock_; DWORD ms, err; BOOL result; if (tv) ms = evutil_tv_to_msec_(tv); else ms = INFINITE; result = SleepConditionVariableCS_fn(cond, lock, ms); if (result) { if (GetLastError() == WAIT_TIMEOUT) return 1; else return -1; } else { return 0; } } #endif struct evthread_win32_cond { HANDLE event; CRITICAL_SECTION lock; int n_waiting; int n_to_wake; int generation; }; static void * evthread_win32_cond_alloc(unsigned flags) { struct evthread_win32_cond *cond; if (!(cond = mm_malloc(sizeof(struct evthread_win32_cond)))) return NULL; if (InitializeCriticalSectionAndSpinCount(&cond->lock, SPIN_COUNT)==0) { mm_free(cond); return NULL; } if ((cond->event = CreateEvent(NULL,TRUE,FALSE,NULL)) == NULL) { DeleteCriticalSection(&cond->lock); mm_free(cond); return NULL; } cond->n_waiting = cond->n_to_wake = cond->generation = 0; return cond; } static void evthread_win32_cond_free(void *cond_) { struct evthread_win32_cond *cond = cond_; DeleteCriticalSection(&cond->lock); CloseHandle(cond->event); mm_free(cond); } static int evthread_win32_cond_signal(void *cond_, int broadcast) { struct evthread_win32_cond *cond = cond_; EnterCriticalSection(&cond->lock); if (broadcast) cond->n_to_wake = cond->n_waiting; else ++cond->n_to_wake; cond->generation++; SetEvent(cond->event); LeaveCriticalSection(&cond->lock); return 0; } static int evthread_win32_cond_wait(void *cond_, void *lock_, const struct timeval *tv) { struct evthread_win32_cond *cond = cond_; CRITICAL_SECTION *lock = lock_; int generation_at_start; int waiting = 1; int result = -1; DWORD ms = INFINITE, ms_orig = INFINITE, startTime, endTime; if (tv) ms_orig = ms = evutil_tv_to_msec_(tv); EnterCriticalSection(&cond->lock); ++cond->n_waiting; generation_at_start = cond->generation; LeaveCriticalSection(&cond->lock); LeaveCriticalSection(lock); startTime = GetTickCount(); do { DWORD res; res = WaitForSingleObject(cond->event, ms); EnterCriticalSection(&cond->lock); if (cond->n_to_wake && cond->generation != generation_at_start) { --cond->n_to_wake; --cond->n_waiting; result = 0; waiting = 0; goto out; } else if (res != WAIT_OBJECT_0) { result = (res==WAIT_TIMEOUT) ? 1 : -1; --cond->n_waiting; waiting = 0; goto out; } else if (ms != INFINITE) { endTime = GetTickCount(); if (startTime + ms_orig <= endTime) { result = 1; /* Timeout */ --cond->n_waiting; waiting = 0; goto out; } else { ms = startTime + ms_orig - endTime; } } /* If we make it here, we are still waiting. */ if (cond->n_to_wake == 0) { /* There is nobody else who should wake up; reset * the event. */ ResetEvent(cond->event); } out: LeaveCriticalSection(&cond->lock); } while (waiting); EnterCriticalSection(lock); EnterCriticalSection(&cond->lock); if (!cond->n_waiting) ResetEvent(cond->event); LeaveCriticalSection(&cond->lock); return result; } int evthread_use_windows_threads(void) { struct evthread_lock_callbacks cbs = { EVTHREAD_LOCK_API_VERSION, EVTHREAD_LOCKTYPE_RECURSIVE, evthread_win32_lock_create, evthread_win32_lock_free, evthread_win32_lock, evthread_win32_unlock }; struct evthread_condition_callbacks cond_cbs = { EVTHREAD_CONDITION_API_VERSION, evthread_win32_cond_alloc, evthread_win32_cond_free, evthread_win32_cond_signal, evthread_win32_cond_wait }; #ifdef WIN32_HAVE_CONDITION_VARIABLES struct evthread_condition_callbacks condvar_cbs = { EVTHREAD_CONDITION_API_VERSION, evthread_win32_condvar_alloc, evthread_win32_condvar_free, evthread_win32_condvar_signal, evthread_win32_condvar_wait }; #endif evthread_set_lock_callbacks(&cbs); evthread_set_id_callback(evthread_win32_get_id); #ifdef WIN32_HAVE_CONDITION_VARIABLES if (evthread_win32_condvar_init()) { evthread_set_condition_callbacks(&condvar_cbs); return 0; } #endif evthread_set_condition_callbacks(&cond_cbs); return 0; } �������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evutil.c�����������������������������������������������������0000664�0000000�0000000�00000211776�15270523712�0021470�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #include <winsock2.h> #include <ws2tcpip.h> #define WIN32_LEAN_AND_MEAN #include <windows.h> #undef WIN32_LEAN_AND_MEAN #include <io.h> #include <tchar.h> #include <process.h> #undef _WIN32_WINNT /* For structs needed by GetAdaptersAddresses */ #define _WIN32_WINNT 0x0501 #include <iphlpapi.h> #endif #include <sys/types.h> #ifdef EVENT__HAVE_SYS_SOCKET_H #include <sys/socket.h> #endif #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #ifdef EVENT__HAVE_FCNTL_H #include <fcntl.h> #endif #ifdef EVENT__HAVE_STDLIB_H #include <stdlib.h> #endif #include <errno.h> #include <limits.h> #include <stdio.h> #include <string.h> #ifdef EVENT__HAVE_NETINET_IN_H #include <netinet/in.h> #endif #ifdef EVENT__HAVE_NETINET_IN6_H #include <netinet/in6.h> #endif #ifdef EVENT__HAVE_NETINET_TCP_H #include <netinet/tcp.h> #endif #ifdef EVENT__HAVE_ARPA_INET_H #include <arpa/inet.h> #endif #include <time.h> #include <sys/stat.h> #ifdef EVENT__HAVE_IFADDRS_H #include <ifaddrs.h> #endif #include "event2/util.h" #include "util-internal.h" #include "log-internal.h" #include "mm-internal.h" #include "evthread-internal.h" #include "strlcpy-internal.h" #include "ipv6-internal.h" #ifdef _WIN32 #define HT_NO_CACHE_HASH_VALUES #include "ht-internal.h" #define open _open #define read _read #define close _close #ifndef fstat #define fstat _fstati64 #endif #ifndef stat #define stat _stati64 #endif #define mode_t int #endif int evutil_open_closeonexec_(const char *pathname, int flags, unsigned mode) { int fd; #ifdef O_CLOEXEC fd = open(pathname, flags|O_CLOEXEC, (mode_t)mode); if (fd >= 0 || errno == EINVAL) return fd; /* If we got an EINVAL, fall through and try without O_CLOEXEC */ #endif fd = open(pathname, flags, (mode_t)mode); if (fd < 0) return -1; #if defined(FD_CLOEXEC) if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { close(fd); return -1; } #endif return fd; } /** Read the contents of 'filename' into a newly allocated NUL-terminated string. Set *content_out to hold this string, and *len_out to hold its length (not including the appended NUL). If 'is_binary', open the file in binary mode. Returns 0 on success, -1 if the open fails, and -2 for all other failures. Used internally only; may go away in a future version. */ int evutil_read_file_(const char *filename, char **content_out, size_t *len_out, int is_binary) { int fd, r; struct stat st; char *mem; size_t read_so_far=0; int mode = O_RDONLY; EVUTIL_ASSERT(content_out); EVUTIL_ASSERT(len_out); *content_out = NULL; *len_out = 0; #ifdef O_BINARY if (is_binary) mode |= O_BINARY; #endif fd = evutil_open_closeonexec_(filename, mode, 0); if (fd < 0) return -1; if (fstat(fd, &st) || st.st_size < 0 || st.st_size > EV_SSIZE_MAX-1 ) { close(fd); return -2; } mem = mm_malloc((size_t)st.st_size + 1); if (!mem) { close(fd); return -2; } read_so_far = 0; #ifdef _WIN32 #define N_TO_READ(x) ((x) > INT_MAX) ? INT_MAX : ((int)(x)) #else #define N_TO_READ(x) (x) #endif while ((r = read(fd, mem+read_so_far, N_TO_READ(st.st_size - read_so_far))) > 0) { read_so_far += r; if (read_so_far >= (size_t)st.st_size) break; EVUTIL_ASSERT(read_so_far < (size_t)st.st_size); } close(fd); if (r < 0) { mm_free(mem); return -2; } mem[read_so_far] = 0; *len_out = read_so_far; *content_out = mem; return 0; } int evutil_socketpair(int family, int type, int protocol, evutil_socket_t fd[2]) { #ifndef _WIN32 return socketpair(family, type, protocol, fd); #else return evutil_ersatz_socketpair_(family, type, protocol, fd); #endif } int evutil_ersatz_socketpair_(int family, int type, int protocol, evutil_socket_t fd[2]) { /* This code is originally from Tor. Used with permission. */ /* This socketpair does not work when localhost is down. So * it's really not the same thing at all. But it's close enough * for now, and really, when localhost is down sometimes, we * have other problems too. */ #ifdef _WIN32 #define ERR(e) WSA##e #else #define ERR(e) e #endif evutil_socket_t listener = -1; evutil_socket_t connector = -1; evutil_socket_t acceptor = -1; struct sockaddr_in listen_addr; struct sockaddr_in connect_addr; ev_socklen_t size; int saved_errno = -1; int family_test; family_test = family != AF_INET; #ifdef AF_UNIX family_test = family_test && (family != AF_UNIX); #endif if (protocol || family_test) { EVUTIL_SET_SOCKET_ERROR(ERR(EAFNOSUPPORT)); return -1; } if (!fd) { EVUTIL_SET_SOCKET_ERROR(ERR(EINVAL)); return -1; } listener = socket(AF_INET, type, 0); if (listener < 0) return -1; memset(&listen_addr, 0, sizeof(listen_addr)); listen_addr.sin_family = AF_INET; listen_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); listen_addr.sin_port = 0; /* kernel chooses port. */ if (bind(listener, (struct sockaddr *) &listen_addr, sizeof (listen_addr)) == -1) goto tidy_up_and_fail; if (listen(listener, 1) == -1) goto tidy_up_and_fail; connector = socket(AF_INET, type, 0); if (connector < 0) goto tidy_up_and_fail; memset(&connect_addr, 0, sizeof(connect_addr)); /* We want to find out the port number to connect to. */ size = sizeof(connect_addr); if (getsockname(listener, (struct sockaddr *) &connect_addr, &size) == -1) goto tidy_up_and_fail; if (size != sizeof (connect_addr)) goto abort_tidy_up_and_fail; if (connect(connector, (struct sockaddr *) &connect_addr, sizeof(connect_addr)) == -1) goto tidy_up_and_fail; size = sizeof(listen_addr); acceptor = accept(listener, (struct sockaddr *) &listen_addr, &size); if (acceptor < 0) goto tidy_up_and_fail; if (size != sizeof(listen_addr)) goto abort_tidy_up_and_fail; /* Now check we are talking to ourself by matching port and host on the two sockets. */ if (getsockname(connector, (struct sockaddr *) &connect_addr, &size) == -1) goto tidy_up_and_fail; if (size != sizeof (connect_addr) || listen_addr.sin_family != connect_addr.sin_family || listen_addr.sin_addr.s_addr != connect_addr.sin_addr.s_addr || listen_addr.sin_port != connect_addr.sin_port) goto abort_tidy_up_and_fail; evutil_closesocket(listener); fd[0] = connector; fd[1] = acceptor; return 0; abort_tidy_up_and_fail: saved_errno = ERR(ECONNABORTED); tidy_up_and_fail: if (saved_errno < 0) saved_errno = EVUTIL_SOCKET_ERROR(); if (listener != -1) evutil_closesocket(listener); if (connector != -1) evutil_closesocket(connector); if (acceptor != -1) evutil_closesocket(acceptor); EVUTIL_SET_SOCKET_ERROR(saved_errno); return -1; #undef ERR } int evutil_make_socket_nonblocking(evutil_socket_t fd) { #ifdef _WIN32 { unsigned long nonblocking = 1; if (ioctlsocket(fd, FIONBIO, &nonblocking) == SOCKET_ERROR) { event_sock_warn(fd, "fcntl(%d, F_GETFL)", (int)fd); return -1; } } #else { int flags; if ((flags = fcntl(fd, F_GETFL, NULL)) < 0) { event_warn("fcntl(%d, F_GETFL)", fd); return -1; } if (!(flags & O_NONBLOCK)) { if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) { event_warn("fcntl(%d, F_SETFL)", fd); return -1; } } } #endif return 0; } /* Faster version of evutil_make_socket_nonblocking for internal use. * * Requires that no F_SETFL flags were previously set on the fd. */ static int evutil_fast_socket_nonblocking(evutil_socket_t fd) { #ifdef _WIN32 return evutil_make_socket_nonblocking(fd); #else if (fcntl(fd, F_SETFL, O_NONBLOCK) == -1) { event_warn("fcntl(%d, F_SETFL)", fd); return -1; } return 0; #endif } int evutil_make_listen_socket_reuseable(evutil_socket_t sock) { #if defined(SO_REUSEADDR) && !defined(_WIN32) int one = 1; /* REUSEADDR on Unix means, "don't hang on to this address after the * listener is closed." On Windows, though, it means "don't keep other * processes from binding to this address while we're using it. */ return setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*) &one, (ev_socklen_t)sizeof(one)); #else return 0; #endif } int evutil_make_listen_socket_reuseable_port(evutil_socket_t sock) { #if defined __linux__ && defined(SO_REUSEPORT) int one = 1; /* REUSEPORT on Linux 3.9+ means, "Multiple servers (processes or * threads) can bind to the same port if they each set the option. */ return setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, (void*) &one, (ev_socklen_t)sizeof(one)); #else return 0; #endif } int evutil_make_tcp_listen_socket_deferred(evutil_socket_t sock) { #if defined(EVENT__HAVE_NETINET_TCP_H) && defined(TCP_DEFER_ACCEPT) int one = 1; /* TCP_DEFER_ACCEPT tells the kernel to call defer accept() only after data * has arrived and ready to read */ return setsockopt(sock, IPPROTO_TCP, TCP_DEFER_ACCEPT, &one, (ev_socklen_t)sizeof(one)); #endif return 0; } int evutil_make_socket_closeonexec(evutil_socket_t fd) { #if !defined(_WIN32) && defined(EVENT__HAVE_SETFD) int flags; if ((flags = fcntl(fd, F_GETFD, NULL)) < 0) { event_warn("fcntl(%d, F_GETFD)", fd); return -1; } if (!(flags & FD_CLOEXEC)) { if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) == -1) { event_warn("fcntl(%d, F_SETFD)", fd); return -1; } } #endif return 0; } /* Faster version of evutil_make_socket_closeonexec for internal use. * * Requires that no F_SETFD flags were previously set on the fd. */ static int evutil_fast_socket_closeonexec(evutil_socket_t fd) { #if !defined(_WIN32) && defined(EVENT__HAVE_SETFD) if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) { event_warn("fcntl(%d, F_SETFD)", fd); return -1; } #endif return 0; } int evutil_closesocket(evutil_socket_t sock) { #ifndef _WIN32 return close(sock); #else return closesocket(sock); #endif } ev_int64_t evutil_strtoll(const char *s, char **endptr, int base) { #ifdef EVENT__HAVE_STRTOLL return (ev_int64_t)strtoll(s, endptr, base); #elif EVENT__SIZEOF_LONG == 8 return (ev_int64_t)strtol(s, endptr, base); #elif defined(_WIN32) && defined(_MSC_VER) && _MSC_VER < 1300 /* XXXX on old versions of MS APIs, we only support base * 10. */ ev_int64_t r; if (base != 10) return 0; r = (ev_int64_t) _atoi64(s); while (isspace(*s)) ++s; if (*s == '-') ++s; while (isdigit(*s)) ++s; if (endptr) *endptr = (char*) s; return r; #elif defined(_WIN32) return (ev_int64_t) _strtoi64(s, endptr, base); #elif defined(EVENT__SIZEOF_LONG_LONG) && EVENT__SIZEOF_LONG_LONG == 8 long long r; int n; if (base != 10 && base != 16) return 0; if (base == 10) { n = sscanf(s, "%lld", &r); } else { unsigned long long ru=0; n = sscanf(s, "%llx", &ru); if (ru > EV_INT64_MAX) return 0; r = (long long) ru; } if (n != 1) return 0; while (EVUTIL_ISSPACE_(*s)) ++s; if (*s == '-') ++s; if (base == 10) { while (EVUTIL_ISDIGIT_(*s)) ++s; } else { while (EVUTIL_ISXDIGIT_(*s)) ++s; } if (endptr) *endptr = (char*) s; return r; #else #error "I don't know how to parse 64-bit integers." #endif } #ifdef _WIN32 int evutil_socket_geterror(evutil_socket_t sock) { int optval, optvallen=sizeof(optval); int err = WSAGetLastError(); if (err == WSAEWOULDBLOCK && sock >= 0) { if (getsockopt(sock, SOL_SOCKET, SO_ERROR, (void*)&optval, &optvallen)) return err; if (optval) return optval; } return err; } #endif /* XXX we should use an enum here. */ /* 2 for connection refused, 1 for connected, 0 for not yet, -1 for error. */ int evutil_socket_connect_(evutil_socket_t *fd_ptr, const struct sockaddr *sa, int socklen) { int made_fd = 0; if (*fd_ptr < 0) { if ((*fd_ptr = socket(sa->sa_family, SOCK_STREAM, 0)) < 0) goto err; made_fd = 1; if (evutil_make_socket_nonblocking(*fd_ptr) < 0) { goto err; } } if (connect(*fd_ptr, sa, socklen) < 0) { int e = evutil_socket_geterror(*fd_ptr); if (EVUTIL_ERR_CONNECT_RETRIABLE(e)) return 0; if (EVUTIL_ERR_CONNECT_REFUSED(e)) return 2; goto err; } else { return 1; } err: if (made_fd) { evutil_closesocket(*fd_ptr); *fd_ptr = -1; } return -1; } /* Check whether a socket on which we called connect() is done connecting. Return 1 for connected, 0 for not yet, -1 for error. In the error case, set the current socket errno to the error that happened during the connect operation. */ int evutil_socket_finished_connecting_(evutil_socket_t fd) { int e; ev_socklen_t elen = sizeof(e); if (getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&e, &elen) < 0) return -1; if (e) { if (EVUTIL_ERR_CONNECT_RETRIABLE(e)) return 0; EVUTIL_SET_SOCKET_ERROR(e); return -1; } return 1; } #if (EVUTIL_AI_PASSIVE|EVUTIL_AI_CANONNAME|EVUTIL_AI_NUMERICHOST| \ EVUTIL_AI_NUMERICSERV|EVUTIL_AI_V4MAPPED|EVUTIL_AI_ALL| \ EVUTIL_AI_ADDRCONFIG) != \ (EVUTIL_AI_PASSIVE^EVUTIL_AI_CANONNAME^EVUTIL_AI_NUMERICHOST^ \ EVUTIL_AI_NUMERICSERV^EVUTIL_AI_V4MAPPED^EVUTIL_AI_ALL^ \ EVUTIL_AI_ADDRCONFIG) #error "Some of our EVUTIL_AI_* flags seem to overlap with system AI_* flags" #endif /* We sometimes need to know whether we have an ipv4 address and whether we have an ipv6 address. If 'have_checked_interfaces', then we've already done the test. If 'had_ipv4_address', then it turns out we had an ipv4 address. If 'had_ipv6_address', then it turns out we had an ipv6 address. These are set by evutil_check_interfaces. */ static int have_checked_interfaces, had_ipv4_address, had_ipv6_address; /* Macro: True iff the IPv4 address 'addr', in host order, is in 127.0.0.0/8 */ #define EVUTIL_V4ADDR_IS_LOCALHOST(addr) (((addr)>>24) == 127) /* Macro: True iff the IPv4 address 'addr', in host order, is a class D * (multiclass) address. */ #define EVUTIL_V4ADDR_IS_CLASSD(addr) ((((addr)>>24) & 0xf0) == 0xe0) static void evutil_found_ifaddr(const struct sockaddr *sa) { const char ZEROES[] = "\x00\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00"; if (sa->sa_family == AF_INET) { const struct sockaddr_in *sin = (struct sockaddr_in *)sa; ev_uint32_t addr = ntohl(sin->sin_addr.s_addr); if (addr == 0 || EVUTIL_V4ADDR_IS_LOCALHOST(addr) || EVUTIL_V4ADDR_IS_CLASSD(addr)) { /* Not actually a usable external address. */ } else { event_debug(("Detected an IPv4 interface")); had_ipv4_address = 1; } } else if (sa->sa_family == AF_INET6) { const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sa; const unsigned char *addr = (unsigned char*)sin6->sin6_addr.s6_addr; if (!memcmp(addr, ZEROES, 8) || ((addr[0] & 0xfe) == 0xfc) || (addr[0] == 0xfe && (addr[1] & 0xc0) == 0x80) || (addr[0] == 0xfe && (addr[1] & 0xc0) == 0xc0) || (addr[0] == 0xff)) { /* This is a reserved, ipv4compat, ipv4map, loopback, * link-local, multicast, or unspecified address. */ } else { event_debug(("Detected an IPv6 interface")); had_ipv6_address = 1; } } } #ifdef _WIN32 typedef ULONG (WINAPI *GetAdaptersAddresses_fn_t)( ULONG, ULONG, PVOID, PIP_ADAPTER_ADDRESSES, PULONG); #endif static int evutil_check_ifaddrs(void) { #if defined(EVENT__HAVE_GETIFADDRS) /* Most free Unixy systems provide getifaddrs, which gives us a linked list * of struct ifaddrs. */ struct ifaddrs *ifa = NULL; const struct ifaddrs *i; if (getifaddrs(&ifa) < 0) { event_warn("Unable to call getifaddrs()"); return -1; } for (i = ifa; i; i = i->ifa_next) { if (!i->ifa_addr) continue; evutil_found_ifaddr(i->ifa_addr); } freeifaddrs(ifa); return 0; #elif defined(_WIN32) /* Windows XP began to provide GetAdaptersAddresses. Windows 2000 had a "GetAdaptersInfo", but that's deprecated; let's just try GetAdaptersAddresses and fall back to connect+getsockname. */ HMODULE lib = evutil_load_windows_system_library_(TEXT("ihplapi.dll")); GetAdaptersAddresses_fn_t fn; ULONG size, res; IP_ADAPTER_ADDRESSES *addresses = NULL, *address; int result = -1; #define FLAGS (GAA_FLAG_SKIP_ANYCAST | \ GAA_FLAG_SKIP_MULTICAST | \ GAA_FLAG_SKIP_DNS_SERVER) if (!lib) goto done; if (!(fn = (GetAdaptersAddresses_fn_t) GetProcAddress(lib, "GetAdaptersAddresses"))) goto done; /* Guess how much space we need. */ size = 15*1024; addresses = mm_malloc(size); if (!addresses) goto done; res = fn(AF_UNSPEC, FLAGS, NULL, addresses, &size); if (res == ERROR_BUFFER_OVERFLOW) { /* we didn't guess that we needed enough space; try again */ mm_free(addresses); addresses = mm_malloc(size); if (!addresses) goto done; res = fn(AF_UNSPEC, FLAGS, NULL, addresses, &size); } if (res != NO_ERROR) goto done; for (address = addresses; address; address = address->Next) { IP_ADAPTER_UNICAST_ADDRESS *a; for (a = address->FirstUnicastAddress; a; a = a->Next) { /* Yes, it's a linked list inside a linked list */ struct sockaddr *sa = a->Address.lpSockaddr; evutil_found_ifaddr(sa); } } result = 0; done: if (lib) FreeLibrary(lib); if (addresses) mm_free(addresses); return result; #else return -1; #endif } /* Test whether we have an ipv4 interface and an ipv6 interface. Return 0 if * the test seemed successful. */ static int evutil_check_interfaces(int force_recheck) { evutil_socket_t fd = -1; struct sockaddr_in sin, sin_out; struct sockaddr_in6 sin6, sin6_out; ev_socklen_t sin_out_len = sizeof(sin_out); ev_socklen_t sin6_out_len = sizeof(sin6_out); int r; if (have_checked_interfaces && !force_recheck) return 0; if (evutil_check_ifaddrs() == 0) { /* Use a nice sane interface, if this system has one. */ return 0; } /* Ugh. There was no nice sane interface. So to check whether we have * an interface open for a given protocol, will try to make a UDP * 'connection' to a remote host on the internet. We don't actually * use it, so the address doesn't matter, but we want to pick one that * keep us from using a host- or link-local interface. */ memset(&sin, 0, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_port = htons(53); r = evutil_inet_pton(AF_INET, "18.244.0.188", &sin.sin_addr); EVUTIL_ASSERT(r); memset(&sin6, 0, sizeof(sin6)); sin6.sin6_family = AF_INET6; sin6.sin6_port = htons(53); r = evutil_inet_pton(AF_INET6, "2001:4860:b002::68", &sin6.sin6_addr); EVUTIL_ASSERT(r); memset(&sin_out, 0, sizeof(sin_out)); memset(&sin6_out, 0, sizeof(sin6_out)); /* XXX some errnos mean 'no address'; some mean 'not enough sockets'. */ if ((fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) >= 0 && connect(fd, (struct sockaddr*)&sin, sizeof(sin)) == 0 && getsockname(fd, (struct sockaddr*)&sin_out, &sin_out_len) == 0) { /* We might have an IPv4 interface. */ evutil_found_ifaddr((struct sockaddr*) &sin_out); } if (fd >= 0) evutil_closesocket(fd); if ((fd = socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP)) >= 0 && connect(fd, (struct sockaddr*)&sin6, sizeof(sin6)) == 0 && getsockname(fd, (struct sockaddr*)&sin6_out, &sin6_out_len) == 0) { /* We might have an IPv6 interface. */ evutil_found_ifaddr((struct sockaddr*) &sin6_out); } if (fd >= 0) evutil_closesocket(fd); return 0; } /* Internal addrinfo flag. This one is set when we allocate the addrinfo from * inside libevent. Otherwise, the built-in getaddrinfo() function allocated * it, and we should trust what they said. **/ #define EVUTIL_AI_LIBEVENT_ALLOCATED 0x80000000 /* Helper: construct a new addrinfo containing the socket address in * 'sa', which must be a sockaddr_in or a sockaddr_in6. Take the * socktype and protocol info from hints. If they weren't set, then * allocate both a TCP and a UDP addrinfo. */ struct evutil_addrinfo * evutil_new_addrinfo_(struct sockaddr *sa, ev_socklen_t socklen, const struct evutil_addrinfo *hints) { struct evutil_addrinfo *res; EVUTIL_ASSERT(hints); if (hints->ai_socktype == 0 && hints->ai_protocol == 0) { /* Indecisive user! Give them a UDP and a TCP. */ struct evutil_addrinfo *r1, *r2; struct evutil_addrinfo tmp; memcpy(&tmp, hints, sizeof(tmp)); tmp.ai_socktype = SOCK_STREAM; tmp.ai_protocol = IPPROTO_TCP; r1 = evutil_new_addrinfo_(sa, socklen, &tmp); if (!r1) return NULL; tmp.ai_socktype = SOCK_DGRAM; tmp.ai_protocol = IPPROTO_UDP; r2 = evutil_new_addrinfo_(sa, socklen, &tmp); if (!r2) { evutil_freeaddrinfo(r1); return NULL; } r1->ai_next = r2; return r1; } /* We're going to allocate extra space to hold the sockaddr. */ res = mm_calloc(1,sizeof(struct evutil_addrinfo)+socklen); if (!res) return NULL; res->ai_addr = (struct sockaddr*) (((char*)res) + sizeof(struct evutil_addrinfo)); memcpy(res->ai_addr, sa, socklen); res->ai_addrlen = socklen; res->ai_family = sa->sa_family; /* Same or not? XXX */ res->ai_flags = EVUTIL_AI_LIBEVENT_ALLOCATED; res->ai_socktype = hints->ai_socktype; res->ai_protocol = hints->ai_protocol; return res; } /* Append the addrinfo 'append' to the end of 'first', and return the start of * the list. Either element can be NULL, in which case we return the element * that is not NULL. */ struct evutil_addrinfo * evutil_addrinfo_append_(struct evutil_addrinfo *first, struct evutil_addrinfo *append) { struct evutil_addrinfo *ai = first; if (!ai) return append; while (ai->ai_next) ai = ai->ai_next; ai->ai_next = append; return first; } static int parse_numeric_servname(const char *servname) { int n; char *endptr=NULL; n = (int) strtol(servname, &endptr, 10); if (n>=0 && n <= 65535 && servname[0] && endptr && !endptr[0]) return n; else return -1; } /** Parse a service name in 'servname', which can be a decimal port. * Return the port number, or -1 on error. */ static int evutil_parse_servname(const char *servname, const char *protocol, const struct evutil_addrinfo *hints) { int n = parse_numeric_servname(servname); if (n>=0) return n; #if defined(EVENT__HAVE_GETSERVBYNAME) || defined(_WIN32) if (!(hints->ai_flags & EVUTIL_AI_NUMERICSERV)) { struct servent *ent = getservbyname(servname, protocol); if (ent) { return ntohs(ent->s_port); } } #endif return -1; } /* Return a string corresponding to a protocol number that we can pass to * getservyname. */ static const char * evutil_unparse_protoname(int proto) { switch (proto) { case 0: return NULL; case IPPROTO_TCP: return "tcp"; case IPPROTO_UDP: return "udp"; #ifdef IPPROTO_SCTP case IPPROTO_SCTP: return "sctp"; #endif default: #ifdef EVENT__HAVE_GETPROTOBYNUMBER { struct protoent *ent = getprotobynumber(proto); if (ent) return ent->p_name; } #endif return NULL; } } static void evutil_getaddrinfo_infer_protocols(struct evutil_addrinfo *hints) { /* If we can guess the protocol from the socktype, do so. */ if (!hints->ai_protocol && hints->ai_socktype) { if (hints->ai_socktype == SOCK_DGRAM) hints->ai_protocol = IPPROTO_UDP; else if (hints->ai_socktype == SOCK_STREAM) hints->ai_protocol = IPPROTO_TCP; } /* Set the socktype if it isn't set. */ if (!hints->ai_socktype && hints->ai_protocol) { if (hints->ai_protocol == IPPROTO_UDP) hints->ai_socktype = SOCK_DGRAM; else if (hints->ai_protocol == IPPROTO_TCP) hints->ai_socktype = SOCK_STREAM; #ifdef IPPROTO_SCTP else if (hints->ai_protocol == IPPROTO_SCTP) hints->ai_socktype = SOCK_STREAM; #endif } } #if AF_UNSPEC != PF_UNSPEC #error "I cannot build on a system where AF_UNSPEC != PF_UNSPEC" #endif /** Implements the part of looking up hosts by name that's common to both * the blocking and nonblocking resolver: * - Adjust 'hints' to have a reasonable socktype and protocol. * - Look up the port based on 'servname', and store it in *portnum, * - Handle the nodename==NULL case * - Handle some invalid arguments cases. * - Handle the cases where nodename is an IPv4 or IPv6 address. * * If we need the resolver to look up the hostname, we return * EVUTIL_EAI_NEED_RESOLVE. Otherwise, we can completely implement * getaddrinfo: we return 0 or an appropriate EVUTIL_EAI_* error, and * set *res as getaddrinfo would. */ int evutil_getaddrinfo_common_(const char *nodename, const char *servname, struct evutil_addrinfo *hints, struct evutil_addrinfo **res, int *portnum) { int port = 0; const char *pname; if (nodename == NULL && servname == NULL) return EVUTIL_EAI_NONAME; /* We only understand 3 families */ if (hints->ai_family != PF_UNSPEC && hints->ai_family != PF_INET && hints->ai_family != PF_INET6) return EVUTIL_EAI_FAMILY; evutil_getaddrinfo_infer_protocols(hints); /* Look up the port number and protocol, if possible. */ pname = evutil_unparse_protoname(hints->ai_protocol); if (servname) { /* XXXX We could look at the protocol we got back from * getservbyname, but it doesn't seem too useful. */ port = evutil_parse_servname(servname, pname, hints); if (port < 0) { return EVUTIL_EAI_NONAME; } } /* If we have no node name, then we're supposed to bind to 'any' and * connect to localhost. */ if (nodename == NULL) { struct evutil_addrinfo *res4=NULL, *res6=NULL; if (hints->ai_family != PF_INET) { /* INET6 or UNSPEC. */ struct sockaddr_in6 sin6; memset(&sin6, 0, sizeof(sin6)); sin6.sin6_family = AF_INET6; sin6.sin6_port = htons(port); if (hints->ai_flags & EVUTIL_AI_PASSIVE) { /* Bind to :: */ } else { /* connect to ::1 */ sin6.sin6_addr.s6_addr[15] = 1; } res6 = evutil_new_addrinfo_((struct sockaddr*)&sin6, sizeof(sin6), hints); if (!res6) return EVUTIL_EAI_MEMORY; } if (hints->ai_family != PF_INET6) { /* INET or UNSPEC */ struct sockaddr_in sin; memset(&sin, 0, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_port = htons(port); if (hints->ai_flags & EVUTIL_AI_PASSIVE) { /* Bind to 0.0.0.0 */ } else { /* connect to 127.0.0.1 */ sin.sin_addr.s_addr = htonl(0x7f000001); } res4 = evutil_new_addrinfo_((struct sockaddr*)&sin, sizeof(sin), hints); if (!res4) { if (res6) evutil_freeaddrinfo(res6); return EVUTIL_EAI_MEMORY; } } *res = evutil_addrinfo_append_(res4, res6); return 0; } /* If we can, we should try to parse the hostname without resolving * it. */ /* Try ipv6. */ if (hints->ai_family == PF_INET6 || hints->ai_family == PF_UNSPEC) { struct sockaddr_in6 sin6; memset(&sin6, 0, sizeof(sin6)); if (1==evutil_inet_pton(AF_INET6, nodename, &sin6.sin6_addr)) { /* Got an ipv6 address. */ sin6.sin6_family = AF_INET6; sin6.sin6_port = htons(port); *res = evutil_new_addrinfo_((struct sockaddr*)&sin6, sizeof(sin6), hints); if (!*res) return EVUTIL_EAI_MEMORY; return 0; } } /* Try ipv4. */ if (hints->ai_family == PF_INET || hints->ai_family == PF_UNSPEC) { struct sockaddr_in sin; memset(&sin, 0, sizeof(sin)); if (1==evutil_inet_pton(AF_INET, nodename, &sin.sin_addr)) { /* Got an ipv6 address. */ sin.sin_family = AF_INET; sin.sin_port = htons(port); *res = evutil_new_addrinfo_((struct sockaddr*)&sin, sizeof(sin), hints); if (!*res) return EVUTIL_EAI_MEMORY; return 0; } } /* If we have reached this point, we definitely need to do a DNS * lookup. */ if ((hints->ai_flags & EVUTIL_AI_NUMERICHOST)) { /* If we're not allowed to do one, then say so. */ return EVUTIL_EAI_NONAME; } *portnum = port; return EVUTIL_EAI_NEED_RESOLVE; } #ifdef EVENT__HAVE_GETADDRINFO #define USE_NATIVE_GETADDRINFO #endif #ifdef USE_NATIVE_GETADDRINFO /* A mask of all the flags that we declare, so we can clear them before calling * the native getaddrinfo */ static const unsigned int ALL_NONNATIVE_AI_FLAGS = #ifndef AI_PASSIVE EVUTIL_AI_PASSIVE | #endif #ifndef AI_CANONNAME EVUTIL_AI_CANONNAME | #endif #ifndef AI_NUMERICHOST EVUTIL_AI_NUMERICHOST | #endif #ifndef AI_NUMERICSERV EVUTIL_AI_NUMERICSERV | #endif #ifndef AI_ADDRCONFIG EVUTIL_AI_ADDRCONFIG | #endif #ifndef AI_ALL EVUTIL_AI_ALL | #endif #ifndef AI_V4MAPPED EVUTIL_AI_V4MAPPED | #endif EVUTIL_AI_LIBEVENT_ALLOCATED; static const unsigned int ALL_NATIVE_AI_FLAGS = #ifdef AI_PASSIVE AI_PASSIVE | #endif #ifdef AI_CANONNAME AI_CANONNAME | #endif #ifdef AI_NUMERICHOST AI_NUMERICHOST | #endif #ifdef AI_NUMERICSERV AI_NUMERICSERV | #endif #ifdef AI_ADDRCONFIG AI_ADDRCONFIG | #endif #ifdef AI_ALL AI_ALL | #endif #ifdef AI_V4MAPPED AI_V4MAPPED | #endif 0; #endif #ifndef USE_NATIVE_GETADDRINFO /* Helper for systems with no getaddrinfo(): make one or more addrinfos out of * a struct hostent. */ static struct evutil_addrinfo * addrinfo_from_hostent(const struct hostent *ent, int port, const struct evutil_addrinfo *hints) { int i; struct sockaddr_in sin; struct sockaddr_in6 sin6; struct sockaddr *sa; int socklen; struct evutil_addrinfo *res=NULL, *ai; void *addrp; if (ent->h_addrtype == PF_INET) { memset(&sin, 0, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_port = htons(port); sa = (struct sockaddr *)&sin; socklen = sizeof(struct sockaddr_in); addrp = &sin.sin_addr; if (ent->h_length != sizeof(sin.sin_addr)) { event_warnx("Weird h_length from gethostbyname"); return NULL; } } else if (ent->h_addrtype == PF_INET6) { memset(&sin6, 0, sizeof(sin6)); sin6.sin6_family = AF_INET6; sin6.sin6_port = htons(port); sa = (struct sockaddr *)&sin6; socklen = sizeof(struct sockaddr_in6); addrp = &sin6.sin6_addr; if (ent->h_length != sizeof(sin6.sin6_addr)) { event_warnx("Weird h_length from gethostbyname"); return NULL; } } else return NULL; for (i = 0; ent->h_addr_list[i]; ++i) { memcpy(addrp, ent->h_addr_list[i], ent->h_length); ai = evutil_new_addrinfo_(sa, socklen, hints); if (!ai) { evutil_freeaddrinfo(res); return NULL; } res = evutil_addrinfo_append_(res, ai); } if (res && ((hints->ai_flags & EVUTIL_AI_CANONNAME) && ent->h_name)) { res->ai_canonname = mm_strdup(ent->h_name); if (res->ai_canonname == NULL) { evutil_freeaddrinfo(res); return NULL; } } return res; } #endif /* If the EVUTIL_AI_ADDRCONFIG flag is set on hints->ai_flags, and * hints->ai_family is PF_UNSPEC, then revise the value of hints->ai_family so * that we'll only get addresses we could maybe connect to. */ void evutil_adjust_hints_for_addrconfig_(struct evutil_addrinfo *hints) { if (!(hints->ai_flags & EVUTIL_AI_ADDRCONFIG)) return; if (hints->ai_family != PF_UNSPEC) return; if (!have_checked_interfaces) evutil_check_interfaces(0); if (had_ipv4_address && !had_ipv6_address) { hints->ai_family = PF_INET; } else if (!had_ipv4_address && had_ipv6_address) { hints->ai_family = PF_INET6; } } #ifdef USE_NATIVE_GETADDRINFO static int need_numeric_port_hack_=0; static int need_socktype_protocol_hack_=0; static int tested_for_getaddrinfo_hacks=0; /* Some older BSDs (like OpenBSD up to 4.6) used to believe that giving a numeric port without giving an ai_socktype was verboten. We test for this so we can apply an appropriate workaround. If it turns out that the bug is present, then: - If nodename==NULL and servname is numeric, we build an answer ourselves using evutil_getaddrinfo_common_(). - If nodename!=NULL and servname is numeric, then we set servname=NULL when calling getaddrinfo, and post-process the result to set the ports on it. We test for this bug at runtime, since otherwise we can't have the same binary run on multiple BSD versions. - Some versions of Solaris believe that it's nice to leave to protocol field set to 0. We test for this so we can apply an appropriate workaround. */ static struct evutil_addrinfo *ai_find_protocol(struct evutil_addrinfo *ai) { while (ai) { if (ai->ai_protocol) return ai; ai = ai->ai_next; } return NULL; } static void test_for_getaddrinfo_hacks(void) { int r, r2; struct evutil_addrinfo *ai=NULL, *ai2=NULL, *ai3=NULL; struct evutil_addrinfo hints; memset(&hints,0,sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_flags = #ifdef AI_NUMERICHOST AI_NUMERICHOST | #endif #ifdef AI_NUMERICSERV AI_NUMERICSERV | #endif 0; r = getaddrinfo("1.2.3.4", "80", &hints, &ai); getaddrinfo("1.2.3.4", NULL, &hints, &ai3); hints.ai_socktype = SOCK_STREAM; r2 = getaddrinfo("1.2.3.4", "80", &hints, &ai2); if (r2 == 0 && r != 0) { need_numeric_port_hack_=1; } if (!ai_find_protocol(ai2) || !ai_find_protocol(ai3)) { need_socktype_protocol_hack_=1; } if (ai) freeaddrinfo(ai); if (ai2) freeaddrinfo(ai2); if (ai3) freeaddrinfo(ai3); tested_for_getaddrinfo_hacks=1; } static inline int need_numeric_port_hack(void) { if (!tested_for_getaddrinfo_hacks) test_for_getaddrinfo_hacks(); return need_numeric_port_hack_; } static inline int need_socktype_protocol_hack(void) { if (!tested_for_getaddrinfo_hacks) test_for_getaddrinfo_hacks(); return need_socktype_protocol_hack_; } static void apply_numeric_port_hack(int port, struct evutil_addrinfo **ai) { /* Now we run through the list and set the ports on all of the * results where ports would make sense. */ for ( ; *ai; ai = &(*ai)->ai_next) { struct sockaddr *sa = (*ai)->ai_addr; if (sa && sa->sa_family == AF_INET) { struct sockaddr_in *sin = (struct sockaddr_in*)sa; sin->sin_port = htons(port); } else if (sa && sa->sa_family == AF_INET6) { struct sockaddr_in6 *sin6 = (struct sockaddr_in6*)sa; sin6->sin6_port = htons(port); } else { /* A numeric port makes no sense here; remove this one * from the list. */ struct evutil_addrinfo *victim = *ai; *ai = victim->ai_next; victim->ai_next = NULL; freeaddrinfo(victim); } } } static int apply_socktype_protocol_hack(struct evutil_addrinfo *ai) { struct evutil_addrinfo *ai_new; for (; ai; ai = ai->ai_next) { evutil_getaddrinfo_infer_protocols(ai); if (ai->ai_socktype || ai->ai_protocol) continue; ai_new = mm_malloc(sizeof(*ai_new)); if (!ai_new) return -1; memcpy(ai_new, ai, sizeof(*ai_new)); ai->ai_socktype = SOCK_STREAM; ai->ai_protocol = IPPROTO_TCP; ai_new->ai_socktype = SOCK_DGRAM; ai_new->ai_protocol = IPPROTO_UDP; ai_new->ai_next = ai->ai_next; ai->ai_next = ai_new; } return 0; } #endif int evutil_getaddrinfo(const char *nodename, const char *servname, const struct evutil_addrinfo *hints_in, struct evutil_addrinfo **res) { #ifdef USE_NATIVE_GETADDRINFO struct evutil_addrinfo hints; int portnum=-1, need_np_hack, err; if (hints_in) { memcpy(&hints, hints_in, sizeof(hints)); } else { memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; } #ifndef AI_ADDRCONFIG /* Not every system has AI_ADDRCONFIG, so fake it. */ if (hints.ai_family == PF_UNSPEC && (hints.ai_flags & EVUTIL_AI_ADDRCONFIG)) { evutil_adjust_hints_for_addrconfig_(&hints); } #endif #ifndef AI_NUMERICSERV /* Not every system has AI_NUMERICSERV, so fake it. */ if (hints.ai_flags & EVUTIL_AI_NUMERICSERV) { if (servname && parse_numeric_servname(servname)<0) return EVUTIL_EAI_NONAME; } #endif /* Enough operating systems handle enough common non-resolve * cases here weirdly enough that we are better off just * overriding them. For example: * * - Windows doesn't like to infer the protocol from the * socket type, or fill in socket or protocol types much at * all. It also seems to do its own broken implicit * always-on version of AI_ADDRCONFIG that keeps it from * ever resolving even a literal IPv6 address when * ai_addrtype is PF_UNSPEC. */ #ifdef _WIN32 { int tmp_port; err = evutil_getaddrinfo_common_(nodename,servname,&hints, res, &tmp_port); if (err == 0 || err == EVUTIL_EAI_MEMORY || err == EVUTIL_EAI_NONAME) return err; /* If we make it here, the system getaddrinfo can * have a crack at it. */ } #endif /* See documentation for need_numeric_port_hack above.*/ need_np_hack = need_numeric_port_hack() && servname && !hints.ai_socktype && ((portnum=parse_numeric_servname(servname)) >= 0); if (need_np_hack) { if (!nodename) return evutil_getaddrinfo_common_( NULL,servname,&hints, res, &portnum); servname = NULL; } if (need_socktype_protocol_hack()) { evutil_getaddrinfo_infer_protocols(&hints); } /* Make sure that we didn't actually steal any AI_FLAGS values that * the system is using. (This is a constant expression, and should ge * optimized out.) * * XXXX Turn this into a compile-time failure rather than a run-time * failure. */ EVUTIL_ASSERT((ALL_NONNATIVE_AI_FLAGS & ALL_NATIVE_AI_FLAGS) == 0); /* Clear any flags that only libevent understands. */ hints.ai_flags &= ~ALL_NONNATIVE_AI_FLAGS; err = getaddrinfo(nodename, servname, &hints, res); if (need_np_hack) apply_numeric_port_hack(portnum, res); if (need_socktype_protocol_hack()) { if (apply_socktype_protocol_hack(*res) < 0) { evutil_freeaddrinfo(*res); *res = NULL; return EVUTIL_EAI_MEMORY; } } return err; #else int port=0, err; struct hostent *ent = NULL; struct evutil_addrinfo hints; if (hints_in) { memcpy(&hints, hints_in, sizeof(hints)); } else { memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; } evutil_adjust_hints_for_addrconfig_(&hints); err = evutil_getaddrinfo_common_(nodename, servname, &hints, res, &port); if (err != EVUTIL_EAI_NEED_RESOLVE) { /* We either succeeded or failed. No need to continue */ return err; } err = 0; /* Use any of the various gethostbyname_r variants as available. */ { #ifdef EVENT__HAVE_GETHOSTBYNAME_R_6_ARG /* This one is what glibc provides. */ char buf[2048]; struct hostent hostent; int r; r = gethostbyname_r(nodename, &hostent, buf, sizeof(buf), &ent, &err); #elif defined(EVENT__HAVE_GETHOSTBYNAME_R_5_ARG) char buf[2048]; struct hostent hostent; ent = gethostbyname_r(nodename, &hostent, buf, sizeof(buf), &err); #elif defined(EVENT__HAVE_GETHOSTBYNAME_R_3_ARG) struct hostent_data data; struct hostent hostent; memset(&data, 0, sizeof(data)); err = gethostbyname_r(nodename, &hostent, &data); ent = err ? NULL : &hostent; #else /* fall back to gethostbyname. */ /* XXXX This needs a lock everywhere but Windows. */ ent = gethostbyname(nodename); #ifdef _WIN32 err = WSAGetLastError(); #else err = h_errno; #endif #endif /* Now we have either ent or err set. */ if (!ent) { /* XXX is this right for windows ? */ switch (err) { case TRY_AGAIN: return EVUTIL_EAI_AGAIN; case NO_RECOVERY: default: return EVUTIL_EAI_FAIL; case HOST_NOT_FOUND: return EVUTIL_EAI_NONAME; case NO_ADDRESS: #if NO_DATA != NO_ADDRESS case NO_DATA: #endif return EVUTIL_EAI_NODATA; } } if (ent->h_addrtype != hints.ai_family && hints.ai_family != PF_UNSPEC) { /* This wasn't the type we were hoping for. Too bad * we never had a chance to ask gethostbyname for what * we wanted. */ return EVUTIL_EAI_NONAME; } /* Make sure we got _some_ answers. */ if (ent->h_length == 0) return EVUTIL_EAI_NODATA; /* If we got an address type we don't know how to make a sockaddr for, give up. */ if (ent->h_addrtype != PF_INET && ent->h_addrtype != PF_INET6) return EVUTIL_EAI_FAMILY; *res = addrinfo_from_hostent(ent, port, &hints); if (! *res) return EVUTIL_EAI_MEMORY; } return 0; #endif } void evutil_freeaddrinfo(struct evutil_addrinfo *ai) { #ifdef EVENT__HAVE_GETADDRINFO if (!(ai->ai_flags & EVUTIL_AI_LIBEVENT_ALLOCATED)) { freeaddrinfo(ai); return; } #endif while (ai) { struct evutil_addrinfo *next = ai->ai_next; if (ai->ai_canonname) mm_free(ai->ai_canonname); mm_free(ai); ai = next; } } static evdns_getaddrinfo_fn evdns_getaddrinfo_impl = NULL; static evdns_getaddrinfo_cancel_fn evdns_getaddrinfo_cancel_impl = NULL; void evutil_set_evdns_getaddrinfo_fn_(evdns_getaddrinfo_fn fn) { if (!evdns_getaddrinfo_impl) evdns_getaddrinfo_impl = fn; } void evutil_set_evdns_getaddrinfo_cancel_fn_(evdns_getaddrinfo_cancel_fn fn) { if (!evdns_getaddrinfo_cancel_impl) evdns_getaddrinfo_cancel_impl = fn; } /* Internal helper function: act like evdns_getaddrinfo if dns_base is set; * otherwise do a blocking resolve and pass the result to the callback in the * way that evdns_getaddrinfo would. */ struct evdns_getaddrinfo_request *evutil_getaddrinfo_async_( struct evdns_base *dns_base, const char *nodename, const char *servname, const struct evutil_addrinfo *hints_in, void (*cb)(int, struct evutil_addrinfo *, void *), void *arg) { if (dns_base && evdns_getaddrinfo_impl) { return evdns_getaddrinfo_impl( dns_base, nodename, servname, hints_in, cb, arg); } else { struct evutil_addrinfo *ai=NULL; int err; err = evutil_getaddrinfo(nodename, servname, hints_in, &ai); cb(err, ai, arg); return NULL; } } void evutil_getaddrinfo_cancel_async_(struct evdns_getaddrinfo_request *data) { if (evdns_getaddrinfo_cancel_impl && data) { evdns_getaddrinfo_cancel_impl(data); } } const char * evutil_gai_strerror(int err) { /* As a sneaky side-benefit, this case statement will get most * compilers to tell us if any of the error codes we defined * conflict with the platform's native error codes. */ switch (err) { case EVUTIL_EAI_CANCEL: return "Request canceled"; case 0: return "No error"; case EVUTIL_EAI_ADDRFAMILY: return "address family for nodename not supported"; case EVUTIL_EAI_AGAIN: return "temporary failure in name resolution"; case EVUTIL_EAI_BADFLAGS: return "invalid value for ai_flags"; case EVUTIL_EAI_FAIL: return "non-recoverable failure in name resolution"; case EVUTIL_EAI_FAMILY: return "ai_family not supported"; case EVUTIL_EAI_MEMORY: return "memory allocation failure"; case EVUTIL_EAI_NODATA: return "no address associated with nodename"; case EVUTIL_EAI_NONAME: return "nodename nor servname provided, or not known"; case EVUTIL_EAI_SERVICE: return "servname not supported for ai_socktype"; case EVUTIL_EAI_SOCKTYPE: return "ai_socktype not supported"; case EVUTIL_EAI_SYSTEM: return "system error"; default: #if defined(USE_NATIVE_GETADDRINFO) && defined(_WIN32) return gai_strerrorA(err); #elif defined(USE_NATIVE_GETADDRINFO) return gai_strerror(err); #else return "Unknown error code"; #endif } } #ifdef _WIN32 /* destructively remove a trailing line terminator from s */ static void chomp (char *s) { size_t len; if (s && (len = strlen (s)) > 0 && s[len - 1] == '\n') { s[--len] = 0; if (len > 0 && s[len - 1] == '\r') s[--len] = 0; } } /* FormatMessage returns allocated strings, but evutil_socket_error_to_string * is supposed to return a string which is good indefinitely without having * to be freed. To make this work without leaking memory, we cache the * string the first time FormatMessage is called on a particular error * code, and then return the cached string on subsequent calls with the * same code. The strings aren't freed until libevent_global_shutdown * (or never). We use a linked list to cache the errors, because we * only expect there to be a few dozen, and that should be fast enough. */ struct cached_sock_errs_entry { HT_ENTRY(cached_sock_errs_entry) node; DWORD code; char *msg; /* allocated with LocalAlloc; free with LocalFree */ }; static inline unsigned hash_cached_sock_errs(const struct cached_sock_errs_entry *e) { /* Use Murmur3's 32-bit finalizer as an integer hash function */ DWORD h = e->code; h ^= h >> 16; h *= 0x85ebca6b; h ^= h >> 13; h *= 0xc2b2ae35; h ^= h >> 16; return h; } static inline int eq_cached_sock_errs(const struct cached_sock_errs_entry *a, const struct cached_sock_errs_entry *b) { return a->code == b->code; } #ifndef EVENT__DISABLE_THREAD_SUPPORT static void *windows_socket_errors_lock_ = NULL; #endif static HT_HEAD(cached_sock_errs_map, cached_sock_errs_entry) windows_socket_errors = HT_INITIALIZER(); HT_PROTOTYPE(cached_sock_errs_map, cached_sock_errs_entry, node, hash_cached_sock_errs, eq_cached_sock_errs); HT_GENERATE(cached_sock_errs_map, cached_sock_errs_entry, node, hash_cached_sock_errs, eq_cached_sock_errs, 0.5, mm_malloc, mm_realloc, mm_free); /** Equivalent to strerror, but for windows socket errors. */ const char * evutil_socket_error_to_string(int errcode) { struct cached_sock_errs_entry *errs, *newerr, find; char *msg = NULL; EVLOCK_LOCK(windows_socket_errors_lock_, 0); find.code = errcode; errs = HT_FIND(cached_sock_errs_map, &windows_socket_errors, &find); if (errs) { msg = errs->msg; goto done; } if (0 != FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_ALLOCATE_BUFFER, NULL, errcode, 0, (char *)&msg, 0, NULL)) chomp (msg); /* because message has trailing newline */ else { size_t len = 50; /* use LocalAlloc because FormatMessage does */ msg = LocalAlloc(LMEM_FIXED, len); if (!msg) { msg = (char *)"LocalAlloc failed during Winsock error"; goto done; } evutil_snprintf(msg, len, "winsock error 0x%08x", errcode); } newerr = (struct cached_sock_errs_entry *) mm_malloc(sizeof (struct cached_sock_errs_entry)); if (!newerr) { LocalFree(msg); msg = (char *)"malloc failed during Winsock error"; goto done; } newerr->code = errcode; newerr->msg = msg; HT_INSERT(cached_sock_errs_map, &windows_socket_errors, newerr); done: EVLOCK_UNLOCK(windows_socket_errors_lock_, 0); return msg; } #ifndef EVENT__DISABLE_THREAD_SUPPORT int evutil_global_setup_locks_(const int enable_locks) { EVTHREAD_SETUP_GLOBAL_LOCK(windows_socket_errors_lock_, 0); return 0; } #endif static void evutil_free_sock_err_globals(void) { struct cached_sock_errs_entry **errs, *tofree; for (errs = HT_START(cached_sock_errs_map, &windows_socket_errors) ; errs; ) { tofree = *errs; errs = HT_NEXT_RMV(cached_sock_errs_map, &windows_socket_errors, errs); LocalFree(tofree->msg); mm_free(tofree); } HT_CLEAR(cached_sock_errs_map, &windows_socket_errors); #ifndef EVENT__DISABLE_THREAD_SUPPORT if (windows_socket_errors_lock_ != NULL) { EVTHREAD_FREE_LOCK(windows_socket_errors_lock_, 0); windows_socket_errors_lock_ = NULL; } #endif } #else #ifndef EVENT__DISABLE_THREAD_SUPPORT int evutil_global_setup_locks_(const int enable_locks) { return 0; } #endif static void evutil_free_sock_err_globals(void) { } #endif int evutil_snprintf(char *buf, size_t buflen, const char *format, ...) { int r; va_list ap; va_start(ap, format); r = evutil_vsnprintf(buf, buflen, format, ap); va_end(ap); return r; } int evutil_vsnprintf(char *buf, size_t buflen, const char *format, va_list ap) { int r; if (!buflen) return 0; #if defined(_MSC_VER) || defined(_WIN32) r = _vsnprintf(buf, buflen, format, ap); if (r < 0) r = _vscprintf(format, ap); #elif defined(sgi) /* Make sure we always use the correct vsnprintf on IRIX */ extern int _xpg5_vsnprintf(char * __restrict, __SGI_LIBC_NAMESPACE_QUALIFIER size_t, const char * __restrict, /* va_list */ char *); r = _xpg5_vsnprintf(buf, buflen, format, ap); #else r = vsnprintf(buf, buflen, format, ap); #endif buf[buflen-1] = '\0'; return r; } #define USE_INTERNAL_NTOP #define USE_INTERNAL_PTON const char * evutil_inet_ntop(int af, const void *src, char *dst, size_t len) { #if defined(EVENT__HAVE_INET_NTOP) && !defined(USE_INTERNAL_NTOP) return inet_ntop(af, src, dst, len); #else if (af == AF_INET) { const struct in_addr *in = src; const ev_uint32_t a = ntohl(in->s_addr); int r; r = evutil_snprintf(dst, len, "%d.%d.%d.%d", (int)(ev_uint8_t)((a>>24)&0xff), (int)(ev_uint8_t)((a>>16)&0xff), (int)(ev_uint8_t)((a>>8 )&0xff), (int)(ev_uint8_t)((a )&0xff)); if (r<0||(size_t)r>=len) return NULL; else return dst; #ifdef AF_INET6 } else if (af == AF_INET6) { const struct in6_addr *addr = src; char buf[64], *cp; int longestGapLen = 0, longestGapPos = -1, i, curGapPos = -1, curGapLen = 0; ev_uint16_t words[8]; for (i = 0; i < 8; ++i) { words[i] = (((ev_uint16_t)addr->s6_addr[2*i])<<8) + addr->s6_addr[2*i+1]; } if (words[0] == 0 && words[1] == 0 && words[2] == 0 && words[3] == 0 && words[4] == 0 && ((words[5] == 0 && words[6] && words[7]) || (words[5] == 0xffff))) { /* This is an IPv4 address. */ if (words[5] == 0) { evutil_snprintf(buf, sizeof(buf), "::%d.%d.%d.%d", addr->s6_addr[12], addr->s6_addr[13], addr->s6_addr[14], addr->s6_addr[15]); } else { evutil_snprintf(buf, sizeof(buf), "::%x:%d.%d.%d.%d", words[5], addr->s6_addr[12], addr->s6_addr[13], addr->s6_addr[14], addr->s6_addr[15]); } if (strlen(buf) > len) return NULL; strlcpy(dst, buf, len); return dst; } i = 0; while (i < 8) { if (words[i] == 0) { curGapPos = i++; curGapLen = 1; while (i<8 && words[i] == 0) { ++i; ++curGapLen; } if (curGapLen > longestGapLen) { longestGapPos = curGapPos; longestGapLen = curGapLen; } } else { ++i; } } if (longestGapLen<=1) longestGapPos = -1; cp = buf; for (i = 0; i < 8; ++i) { if (words[i] == 0 && longestGapPos == i) { if (i == 0) *cp++ = ':'; *cp++ = ':'; while (i < 8 && words[i] == 0) ++i; --i; /* to compensate for loop increment. */ } else { evutil_snprintf(cp, sizeof(buf)-(cp-buf), "%x", (unsigned)words[i]); cp += strlen(cp); if (i != 7) *cp++ = ':'; } } *cp = '\0'; if (strlen(buf) > len) return NULL; strlcpy(dst, buf, len); return dst; #endif } else { return NULL; } #endif } int evutil_inet_pton(int af, const char *src, void *dst) { #if defined(EVENT__HAVE_INET_PTON) && !defined(USE_INTERNAL_PTON) return inet_pton(af, src, dst); #else if (af == AF_INET) { unsigned a,b,c,d; char more; struct in_addr *addr = dst; if (sscanf(src, "%u.%u.%u.%u%c", &a,&b,&c,&d,&more) != 4) return 0; if (a > 255) return 0; if (b > 255) return 0; if (c > 255) return 0; if (d > 255) return 0; addr->s_addr = htonl((a<<24) | (b<<16) | (c<<8) | d); return 1; #ifdef AF_INET6 } else if (af == AF_INET6) { struct in6_addr *out = dst; ev_uint16_t words[8]; int gapPos = -1, i, setWords=0; const char *dot = strchr(src, '.'); const char *eow; /* end of words. */ if (dot == src) return 0; else if (!dot) eow = src+strlen(src); else { unsigned byte1,byte2,byte3,byte4; char more; for (eow = dot-1; eow >= src && EVUTIL_ISDIGIT_(*eow); --eow) ; ++eow; /* We use "scanf" because some platform inet_aton()s are too lax * about IPv4 addresses of the form "1.2.3" */ if (sscanf(eow, "%u.%u.%u.%u%c", &byte1,&byte2,&byte3,&byte4,&more) != 4) return 0; if (byte1 > 255 || byte2 > 255 || byte3 > 255 || byte4 > 255) return 0; words[6] = (byte1<<8) | byte2; words[7] = (byte3<<8) | byte4; setWords += 2; } i = 0; while (src < eow) { if (i > 7) return 0; if (EVUTIL_ISXDIGIT_(*src)) { char *next; long r = strtol(src, &next, 16); if (next > 4+src) return 0; if (next == src) return 0; if (r<0 || r>65536) return 0; words[i++] = (ev_uint16_t)r; setWords++; src = next; if (*src != ':' && src != eow) return 0; ++src; } else if (*src == ':' && i > 0 && gapPos==-1) { gapPos = i; ++src; } else if (*src == ':' && i == 0 && src[1] == ':' && gapPos==-1) { gapPos = i; src += 2; } else { return 0; } } if (setWords > 8 || (setWords == 8 && gapPos != -1) || (setWords < 8 && gapPos == -1)) return 0; if (gapPos >= 0) { int nToMove = setWords - (dot ? 2 : 0) - gapPos; int gapLen = 8 - setWords; /* assert(nToMove >= 0); */ if (nToMove < 0) return -1; /* should be impossible */ memmove(&words[gapPos+gapLen], &words[gapPos], sizeof(ev_uint16_t)*nToMove); memset(&words[gapPos], 0, sizeof(ev_uint16_t)*gapLen); } for (i = 0; i < 8; ++i) { out->s6_addr[2*i ] = words[i] >> 8; out->s6_addr[2*i+1] = words[i] & 0xff; } return 1; #endif } else { return -1; } #endif } int evutil_parse_sockaddr_port(const char *ip_as_string, struct sockaddr *out, int *outlen) { int port; char buf[128]; const char *cp, *addr_part, *port_part; int is_ipv6; /* recognized formats are: * [ipv6]:port * ipv6 * [ipv6] * ipv4:port * ipv4 */ cp = strchr(ip_as_string, ':'); if (*ip_as_string == '[') { size_t len; if (!(cp = strchr(ip_as_string, ']'))) { return -1; } len = ( cp-(ip_as_string + 1) ); if (len > sizeof(buf)-1) { return -1; } memcpy(buf, ip_as_string+1, len); buf[len] = '\0'; addr_part = buf; if (cp[1] == ':') port_part = cp+2; else port_part = NULL; is_ipv6 = 1; } else if (cp && strchr(cp+1, ':')) { is_ipv6 = 1; addr_part = ip_as_string; port_part = NULL; } else if (cp) { is_ipv6 = 0; if (cp - ip_as_string > (int)sizeof(buf)-1) { return -1; } memcpy(buf, ip_as_string, cp-ip_as_string); buf[cp-ip_as_string] = '\0'; addr_part = buf; port_part = cp+1; } else { addr_part = ip_as_string; port_part = NULL; is_ipv6 = 0; } if (port_part == NULL) { port = 0; } else { port = atoi(port_part); if (port <= 0 || port > 65535) { return -1; } } if (!addr_part) return -1; /* Should be impossible. */ #ifdef AF_INET6 if (is_ipv6) { struct sockaddr_in6 sin6; memset(&sin6, 0, sizeof(sin6)); #ifdef EVENT__HAVE_STRUCT_SOCKADDR_IN6_SIN6_LEN sin6.sin6_len = sizeof(sin6); #endif sin6.sin6_family = AF_INET6; sin6.sin6_port = htons(port); if (1 != evutil_inet_pton(AF_INET6, addr_part, &sin6.sin6_addr)) return -1; if ((int)sizeof(sin6) > *outlen) return -1; memset(out, 0, *outlen); memcpy(out, &sin6, sizeof(sin6)); *outlen = sizeof(sin6); return 0; } else #endif { struct sockaddr_in sin; memset(&sin, 0, sizeof(sin)); #ifdef EVENT__HAVE_STRUCT_SOCKADDR_IN_SIN_LEN sin.sin_len = sizeof(sin); #endif sin.sin_family = AF_INET; sin.sin_port = htons(port); if (1 != evutil_inet_pton(AF_INET, addr_part, &sin.sin_addr)) return -1; if ((int)sizeof(sin) > *outlen) return -1; memset(out, 0, *outlen); memcpy(out, &sin, sizeof(sin)); *outlen = sizeof(sin); return 0; } } const char * evutil_format_sockaddr_port_(const struct sockaddr *sa, char *out, size_t outlen) { char b[128]; const char *res=NULL; int port; if (sa->sa_family == AF_INET) { const struct sockaddr_in *sin = (const struct sockaddr_in*)sa; res = evutil_inet_ntop(AF_INET, &sin->sin_addr,b,sizeof(b)); port = ntohs(sin->sin_port); if (res) { evutil_snprintf(out, outlen, "%s:%d", b, port); return out; } } else if (sa->sa_family == AF_INET6) { const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6*)sa; res = evutil_inet_ntop(AF_INET6, &sin6->sin6_addr,b,sizeof(b)); port = ntohs(sin6->sin6_port); if (res) { evutil_snprintf(out, outlen, "[%s]:%d", b, port); return out; } } evutil_snprintf(out, outlen, "<addr with socktype %d>", (int)sa->sa_family); return out; } int evutil_sockaddr_cmp(const struct sockaddr *sa1, const struct sockaddr *sa2, int include_port) { int r; if (0 != (r = (sa1->sa_family - sa2->sa_family))) return r; if (sa1->sa_family == AF_INET) { const struct sockaddr_in *sin1, *sin2; sin1 = (const struct sockaddr_in *)sa1; sin2 = (const struct sockaddr_in *)sa2; if (sin1->sin_addr.s_addr < sin2->sin_addr.s_addr) return -1; else if (sin1->sin_addr.s_addr > sin2->sin_addr.s_addr) return 1; else if (include_port && (r = ((int)sin1->sin_port - (int)sin2->sin_port))) return r; else return 0; } #ifdef AF_INET6 else if (sa1->sa_family == AF_INET6) { const struct sockaddr_in6 *sin1, *sin2; sin1 = (const struct sockaddr_in6 *)sa1; sin2 = (const struct sockaddr_in6 *)sa2; if ((r = memcmp(sin1->sin6_addr.s6_addr, sin2->sin6_addr.s6_addr, 16))) return r; else if (include_port && (r = ((int)sin1->sin6_port - (int)sin2->sin6_port))) return r; else return 0; } #endif return 1; } /* Tables to implement ctypes-replacement EVUTIL_IS*() functions. Each table * has 256 bits to look up whether a character is in some set or not. This * fails on non-ASCII platforms, but so does every other place where we * take a char and write it onto the network. **/ static const ev_uint32_t EVUTIL_ISALPHA_TABLE[8] = { 0, 0, 0x7fffffe, 0x7fffffe, 0, 0, 0, 0 }; static const ev_uint32_t EVUTIL_ISALNUM_TABLE[8] = { 0, 0x3ff0000, 0x7fffffe, 0x7fffffe, 0, 0, 0, 0 }; static const ev_uint32_t EVUTIL_ISSPACE_TABLE[8] = { 0x3e00, 0x1, 0, 0, 0, 0, 0, 0 }; static const ev_uint32_t EVUTIL_ISXDIGIT_TABLE[8] = { 0, 0x3ff0000, 0x7e, 0x7e, 0, 0, 0, 0 }; static const ev_uint32_t EVUTIL_ISDIGIT_TABLE[8] = { 0, 0x3ff0000, 0, 0, 0, 0, 0, 0 }; static const ev_uint32_t EVUTIL_ISPRINT_TABLE[8] = { 0, 0xffffffff, 0xffffffff, 0x7fffffff, 0, 0, 0, 0x0 }; static const ev_uint32_t EVUTIL_ISUPPER_TABLE[8] = { 0, 0, 0x7fffffe, 0, 0, 0, 0, 0 }; static const ev_uint32_t EVUTIL_ISLOWER_TABLE[8] = { 0, 0, 0, 0x7fffffe, 0, 0, 0, 0 }; /* Upper-casing and lowercasing tables to map characters to upper/lowercase * equivalents. */ static const unsigned char EVUTIL_TOUPPER_TABLE[256] = { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31, 32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47, 48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63, 64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79, 80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95, 96,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79, 80,81,82,83,84,85,86,87,88,89,90,123,124,125,126,127, 128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159, 160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175, 176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191, 192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207, 208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223, 224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239, 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, }; static const unsigned char EVUTIL_TOLOWER_TABLE[256] = { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31, 32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47, 48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63, 64,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111, 112,113,114,115,116,117,118,119,120,121,122,91,92,93,94,95, 96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111, 112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127, 128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159, 160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175, 176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191, 192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207, 208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223, 224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239, 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, }; #define IMPL_CTYPE_FN(name) \ int EVUTIL_##name##_(char c) { \ ev_uint8_t u = c; \ return !!(EVUTIL_##name##_TABLE[(u >> 5) & 7] & (1 << (u & 31))); \ } IMPL_CTYPE_FN(ISALPHA) IMPL_CTYPE_FN(ISALNUM) IMPL_CTYPE_FN(ISSPACE) IMPL_CTYPE_FN(ISDIGIT) IMPL_CTYPE_FN(ISXDIGIT) IMPL_CTYPE_FN(ISPRINT) IMPL_CTYPE_FN(ISLOWER) IMPL_CTYPE_FN(ISUPPER) char EVUTIL_TOLOWER_(char c) { return ((char)EVUTIL_TOLOWER_TABLE[(ev_uint8_t)c]); } char EVUTIL_TOUPPER_(char c) { return ((char)EVUTIL_TOUPPER_TABLE[(ev_uint8_t)c]); } int evutil_ascii_strcasecmp(const char *s1, const char *s2) { char c1, c2; while (1) { c1 = EVUTIL_TOLOWER_(*s1++); c2 = EVUTIL_TOLOWER_(*s2++); if (c1 < c2) return -1; else if (c1 > c2) return 1; else if (c1 == 0) return 0; } } int evutil_ascii_strncasecmp(const char *s1, const char *s2, size_t n) { char c1, c2; while (n--) { c1 = EVUTIL_TOLOWER_(*s1++); c2 = EVUTIL_TOLOWER_(*s2++); if (c1 < c2) return -1; else if (c1 > c2) return 1; else if (c1 == 0) return 0; } return 0; } void evutil_rtrim_lws_(char *str) { char *cp; if (str == NULL) return; if ((cp = strchr(str, '\0')) == NULL || (cp == str)) return; --cp; while (*cp == ' ' || *cp == '\t') { *cp = '\0'; if (cp == str) break; --cp; } } static int evutil_issetugid(void) { #ifdef EVENT__HAVE_ISSETUGID return issetugid(); #else #ifdef EVENT__HAVE_GETEUID if (getuid() != geteuid()) return 1; #endif #ifdef EVENT__HAVE_GETEGID if (getgid() != getegid()) return 1; #endif return 0; #endif } const char * evutil_getenv_(const char *varname) { if (evutil_issetugid()) return NULL; return getenv(varname); } ev_uint32_t evutil_weakrand_seed_(struct evutil_weakrand_state *state, ev_uint32_t seed) { if (seed == 0) { struct timeval tv; evutil_gettimeofday(&tv, NULL); seed = (ev_uint32_t)tv.tv_sec + (ev_uint32_t)tv.tv_usec; #ifdef _WIN32 seed += (ev_uint32_t) _getpid(); #else seed += (ev_uint32_t) getpid(); #endif } state->seed = seed; return seed; } ev_int32_t evutil_weakrand_(struct evutil_weakrand_state *state) { /* This RNG implementation is a linear congruential generator, with * modulus 2^31, multiplier 1103515245, and addend 12345. It's also * used by OpenBSD, and by Glibc's TYPE_0 RNG. * * The linear congruential generator is not an industrial-strength * RNG! It's fast, but it can have higher-order patterns. Notably, * the low bits tend to have periodicity. */ state->seed = ((state->seed) * 1103515245 + 12345) & 0x7fffffff; return (ev_int32_t)(state->seed); } ev_int32_t evutil_weakrand_range_(struct evutil_weakrand_state *state, ev_int32_t top) { ev_int32_t divisor, result; /* We can't just do weakrand() % top, since the low bits of the LCG * are less random than the high ones. (Specifically, since the LCG * modulus is 2^N, every 2^m for m<N will divide the modulus, and so * therefore the low m bits of the LCG will have period 2^m.) */ divisor = EVUTIL_WEAKRAND_MAX / top; do { result = evutil_weakrand_(state) / divisor; } while (result >= top); return result; } /** * Volatile pointer to memset: we use this to keep the compiler from * eliminating our call to memset. */ void * (*volatile evutil_memset_volatile_)(void *, int, size_t) = memset; void evutil_memclear_(void *mem, size_t len) { evutil_memset_volatile_(mem, 0, len); } int evutil_sockaddr_is_loopback_(const struct sockaddr *addr) { static const char LOOPBACK_S6[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"; if (addr->sa_family == AF_INET) { struct sockaddr_in *sin = (struct sockaddr_in *)addr; return (ntohl(sin->sin_addr.s_addr) & 0xff000000) == 0x7f000000; } else if (addr->sa_family == AF_INET6) { struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; return !memcmp(sin6->sin6_addr.s6_addr, LOOPBACK_S6, 16); } return 0; } int evutil_hex_char_to_int_(char c) { switch(c) { case '0': return 0; case '1': return 1; case '2': return 2; case '3': return 3; case '4': return 4; case '5': return 5; case '6': return 6; case '7': return 7; case '8': return 8; case '9': return 9; case 'A': case 'a': return 10; case 'B': case 'b': return 11; case 'C': case 'c': return 12; case 'D': case 'd': return 13; case 'E': case 'e': return 14; case 'F': case 'f': return 15; } return -1; } #ifdef _WIN32 HMODULE evutil_load_windows_system_library_(const TCHAR *library_name) { TCHAR path[MAX_PATH]; unsigned n; n = GetSystemDirectory(path, MAX_PATH); if (n == 0 || n + _tcslen(library_name) + 2 >= MAX_PATH) return 0; _tcscat(path, TEXT("\\")); _tcscat(path, library_name); return LoadLibrary(path); } #endif /* Internal wrapper around 'socket' to provide Linux-style support for * syscall-saving methods where available. * * In addition to regular socket behavior, you can use a bitwise or to set the * flags EVUTIL_SOCK_NONBLOCK and EVUTIL_SOCK_CLOEXEC in the 'type' argument, * to make the socket nonblocking or close-on-exec with as few syscalls as * possible. */ evutil_socket_t evutil_socket_(int domain, int type, int protocol) { evutil_socket_t r; #if defined(SOCK_NONBLOCK) && defined(SOCK_CLOEXEC) r = socket(domain, type, protocol); if (r >= 0) return r; else if ((type & (SOCK_NONBLOCK|SOCK_CLOEXEC)) == 0) return -1; #endif #define SOCKET_TYPE_MASK (~(EVUTIL_SOCK_NONBLOCK|EVUTIL_SOCK_CLOEXEC)) r = socket(domain, type & SOCKET_TYPE_MASK, protocol); if (r < 0) return -1; if (type & EVUTIL_SOCK_NONBLOCK) { if (evutil_fast_socket_nonblocking(r) < 0) { evutil_closesocket(r); return -1; } } if (type & EVUTIL_SOCK_CLOEXEC) { if (evutil_fast_socket_closeonexec(r) < 0) { evutil_closesocket(r); return -1; } } return r; } /* Internal wrapper around 'accept' or 'accept4' to provide Linux-style * support for syscall-saving methods where available. * * In addition to regular accept behavior, you can set one or more of flags * EVUTIL_SOCK_NONBLOCK and EVUTIL_SOCK_CLOEXEC in the 'flags' argument, to * make the socket nonblocking or close-on-exec with as few syscalls as * possible. */ evutil_socket_t evutil_accept4_(evutil_socket_t sockfd, struct sockaddr *addr, ev_socklen_t *addrlen, int flags) { evutil_socket_t result; #if defined(EVENT__HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) result = accept4(sockfd, addr, addrlen, flags); if (result >= 0 || (errno != EINVAL && errno != ENOSYS)) { /* A nonnegative result means that we succeeded, so return. * Failing with EINVAL means that an option wasn't supported, * and failing with ENOSYS means that the syscall wasn't * there: in those cases we want to fall back. Otherwise, we * got a real error, and we should return. */ return result; } #endif result = accept(sockfd, addr, addrlen); if (result < 0) return result; if (flags & EVUTIL_SOCK_CLOEXEC) { if (evutil_fast_socket_closeonexec(result) < 0) { evutil_closesocket(result); return -1; } } if (flags & EVUTIL_SOCK_NONBLOCK) { if (evutil_fast_socket_nonblocking(result) < 0) { evutil_closesocket(result); return -1; } } return result; } /* Internal function: Set fd[0] and fd[1] to a pair of fds such that writes on * fd[0] get read from fd[1]. Make both fds nonblocking and close-on-exec. * Return 0 on success, -1 on failure. */ int evutil_make_internal_pipe_(evutil_socket_t fd[2]) { /* Making the second socket nonblocking is a bit subtle, given that we ignore any EAGAIN returns when writing to it, and you don't usally do that for a nonblocking socket. But if the kernel gives us EAGAIN, then there's no need to add any more data to the buffer, since the main thread is already either about to wake up and drain it, or woken up and in the process of draining it. */ #if defined(EVENT__HAVE_PIPE2) if (pipe2(fd, O_NONBLOCK|O_CLOEXEC) == 0) return 0; #endif #if defined(EVENT__HAVE_PIPE) if (pipe(fd) == 0) { if (evutil_fast_socket_nonblocking(fd[0]) < 0 || evutil_fast_socket_nonblocking(fd[1]) < 0 || evutil_fast_socket_closeonexec(fd[0]) < 0 || evutil_fast_socket_closeonexec(fd[1]) < 0) { close(fd[0]); close(fd[1]); fd[0] = fd[1] = -1; return -1; } return 0; } else { event_warn("%s: pipe", __func__); } #endif #ifdef _WIN32 #define LOCAL_SOCKETPAIR_AF AF_INET #else #define LOCAL_SOCKETPAIR_AF AF_UNIX #endif if (evutil_socketpair(LOCAL_SOCKETPAIR_AF, SOCK_STREAM, 0, fd) == 0) { if (evutil_fast_socket_nonblocking(fd[0]) < 0 || evutil_fast_socket_nonblocking(fd[1]) < 0 || evutil_fast_socket_closeonexec(fd[0]) < 0 || evutil_fast_socket_closeonexec(fd[1]) < 0) { evutil_closesocket(fd[0]); evutil_closesocket(fd[1]); fd[0] = fd[1] = -1; return -1; } return 0; } fd[0] = fd[1] = -1; return -1; } /* Wrapper around eventfd on systems that provide it. Unlike the system * eventfd, it always supports EVUTIL_EFD_CLOEXEC and EVUTIL_EFD_NONBLOCK as * flags. Returns -1 on error or if eventfd is not supported. */ evutil_socket_t evutil_eventfd_(unsigned initval, int flags) { #if defined(EVENT__HAVE_EVENTFD) && defined(EVENT__HAVE_SYS_EVENTFD_H) int r; #if defined(EFD_CLOEXEC) && defined(EFD_NONBLOCK) r = eventfd(initval, flags); if (r >= 0 || flags == 0) return r; #endif r = eventfd(initval, 0); if (r < 0) return r; if (flags & EVUTIL_EFD_CLOEXEC) { if (evutil_fast_socket_closeonexec(r) < 0) { evutil_closesocket(r); return -1; } } if (flags & EVUTIL_EFD_NONBLOCK) { if (evutil_fast_socket_nonblocking(r) < 0) { evutil_closesocket(r); return -1; } } return r; #else return -1; #endif } void evutil_free_globals_(void) { evutil_free_secure_rng_globals_(); evutil_free_sock_err_globals(); } ��kytensor-2.49.0.7/third_party/libevent/evutil_rand.c������������������������������������������������0000664�0000000�0000000�00000012251�15270523712�0022457�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* This file has our secure PRNG code. On platforms that have arc4random(), * we just use that. Otherwise, we include arc4random.c as a bunch of static * functions, and wrap it lightly. We don't expose the arc4random*() APIs * because A) they aren't in our namespace, and B) it's not nice to name your * APIs after their implementations. We keep them in a separate file * so that other people can rip it out and use it for whatever. */ #include "event2/event-config.h" #include "evconfig-private.h" #include <limits.h> #include "util-internal.h" #include "evthread-internal.h" #ifdef EVENT__HAVE_ARC4RANDOM #include <stdlib.h> #include <string.h> int evutil_secure_rng_set_urandom_device_file(char *fname) { (void) fname; return -1; } int evutil_secure_rng_init(void) { /* call arc4random() now to force it to self-initialize */ (void) arc4random(); return 0; } #ifndef EVENT__DISABLE_THREAD_SUPPORT int evutil_secure_rng_global_setup_locks_(const int enable_locks) { return 0; } #endif static void evutil_free_secure_rng_globals_locks(void) { } static void ev_arc4random_buf(void *buf, size_t n) { #if defined(EVENT__HAVE_ARC4RANDOM_BUF) && !defined(__APPLE__) arc4random_buf(buf, n); return; #else unsigned char *b = buf; #if defined(EVENT__HAVE_ARC4RANDOM_BUF) /* OSX 10.7 introducd arc4random_buf, so if you build your program * there, you'll get surprised when older versions of OSX fail to run. * To solve this, we can check whether the function pointer is set, * and fall back otherwise. (OSX does this using some linker * trickery.) */ { void (*tptr)(void *,size_t) = (void (*)(void*,size_t))arc4random_buf; if (tptr != NULL) { arc4random_buf(buf, n); return; } } #endif /* Make sure that we start out with b at a 4-byte alignment; plenty * of CPUs care about this for 32-bit access. */ if (n >= 4 && ((ev_uintptr_t)b) & 3) { ev_uint32_t u = arc4random(); int n_bytes = 4 - (((ev_uintptr_t)b) & 3); memcpy(b, &u, n_bytes); b += n_bytes; n -= n_bytes; } while (n >= 4) { *(ev_uint32_t*)b = arc4random(); b += 4; n -= 4; } if (n) { ev_uint32_t u = arc4random(); memcpy(b, &u, n); } #endif } #else /* !EVENT__HAVE_ARC4RANDOM { */ #ifdef EVENT__ssize_t #define ssize_t EVENT__ssize_t #endif #define ARC4RANDOM_EXPORT static #define ARC4_LOCK_() EVLOCK_LOCK(arc4rand_lock, 0) #define ARC4_UNLOCK_() EVLOCK_UNLOCK(arc4rand_lock, 0) #ifndef EVENT__DISABLE_THREAD_SUPPORT static void *arc4rand_lock; #endif #define ARC4RANDOM_UINT32 ev_uint32_t #define ARC4RANDOM_NOSTIR #define ARC4RANDOM_NORANDOM #define ARC4RANDOM_NOUNIFORM #include "./arc4random.c" #ifndef EVENT__DISABLE_THREAD_SUPPORT int evutil_secure_rng_global_setup_locks_(const int enable_locks) { EVTHREAD_SETUP_GLOBAL_LOCK(arc4rand_lock, 0); return 0; } #endif static void evutil_free_secure_rng_globals_locks(void) { #ifndef EVENT__DISABLE_THREAD_SUPPORT if (arc4rand_lock != NULL) { EVTHREAD_FREE_LOCK(arc4rand_lock, 0); arc4rand_lock = NULL; } #endif return; } int evutil_secure_rng_set_urandom_device_file(char *fname) { #ifdef TRY_SEED_URANDOM ARC4_LOCK_(); arc4random_urandom_filename = fname; ARC4_UNLOCK_(); #endif return 0; } int evutil_secure_rng_init(void) { int val; ARC4_LOCK_(); if (!arc4_seeded_ok) arc4_stir(); val = arc4_seeded_ok ? 0 : -1; ARC4_UNLOCK_(); return val; } static void ev_arc4random_buf(void *buf, size_t n) { arc4random_buf(buf, n); } #endif /* } !EVENT__HAVE_ARC4RANDOM */ void evutil_secure_rng_get_bytes(void *buf, size_t n) { ev_arc4random_buf(buf, n); } void evutil_secure_rng_add_bytes(const char *buf, size_t n) { arc4random_addrandom((unsigned char*)buf, n>(size_t)INT_MAX ? INT_MAX : (int)n); } void evutil_free_secure_rng_globals_(void) { evutil_free_secure_rng_globals_locks(); } �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/evutil_time.c������������������������������������������������0000664�0000000�0000000�00000041307�15270523712�0022475�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #include <winsock2.h> #define WIN32_LEAN_AND_MEAN #include <windows.h> #undef WIN32_LEAN_AND_MEAN #endif #include <sys/types.h> #ifdef EVENT__HAVE_STDLIB_H #include <stdlib.h> #endif #include <errno.h> #include <limits.h> #ifndef EVENT__HAVE_GETTIMEOFDAY #include <sys/timeb.h> #endif #if !defined(EVENT__HAVE_NANOSLEEP) && !defined(EVENT_HAVE_USLEEP) && \ !defined(_WIN32) #include <sys/select.h> #endif #include <time.h> #include <sys/stat.h> #include <string.h> /** evutil_usleep_() */ #if defined(_WIN32) #elif defined(EVENT__HAVE_NANOSLEEP) #elif defined(EVENT__HAVE_USLEEP) #include <unistd.h> #endif #include "event2/util.h" #include "util-internal.h" #include "log-internal.h" #include "mm-internal.h" #ifndef EVENT__HAVE_GETTIMEOFDAY /* No gettimeofday; this must be windows. */ int evutil_gettimeofday(struct timeval *tv, struct timezone *tz) { #ifdef _MSC_VER #define U64_LITERAL(n) n##ui64 #else #define U64_LITERAL(n) n##llu #endif /* Conversion logic taken from Tor, which in turn took it * from Perl. GetSystemTimeAsFileTime returns its value as * an unaligned (!) 64-bit value containing the number of * 100-nanosecond intervals since 1 January 1601 UTC. */ #define EPOCH_BIAS U64_LITERAL(116444736000000000) #define UNITS_PER_SEC U64_LITERAL(10000000) #define USEC_PER_SEC U64_LITERAL(1000000) #define UNITS_PER_USEC U64_LITERAL(10) union { FILETIME ft_ft; ev_uint64_t ft_64; } ft; if (tv == NULL) return -1; GetSystemTimeAsFileTime(&ft.ft_ft); if (EVUTIL_UNLIKELY(ft.ft_64 < EPOCH_BIAS)) { /* Time before the unix epoch. */ return -1; } ft.ft_64 -= EPOCH_BIAS; tv->tv_sec = (long) (ft.ft_64 / UNITS_PER_SEC); tv->tv_usec = (long) ((ft.ft_64 / UNITS_PER_USEC) % USEC_PER_SEC); return 0; } #endif #define MAX_SECONDS_IN_MSEC_LONG \ (((LONG_MAX) - 999) / 1000) long evutil_tv_to_msec_(const struct timeval *tv) { if (tv->tv_usec > 1000000 || tv->tv_sec > MAX_SECONDS_IN_MSEC_LONG) return -1; return (tv->tv_sec * 1000) + ((tv->tv_usec + 999) / 1000); } /* Replacement for usleep on platforms that don't have one. Not guaranteed to be any more finegrained than 1 msec. */ void evutil_usleep_(const struct timeval *tv) { if (!tv) return; #if defined(_WIN32) { long msec = evutil_tv_to_msec_(tv); Sleep((DWORD)msec); } #elif defined(EVENT__HAVE_NANOSLEEP) { struct timespec ts; ts.tv_sec = tv->tv_sec; ts.tv_nsec = tv->tv_usec*1000; nanosleep(&ts, NULL); } #elif defined(EVENT__HAVE_USLEEP) /* Some systems don't like to usleep more than 999999 usec */ sleep(tv->tv_sec); usleep(tv->tv_usec); #else select(0, NULL, NULL, NULL, tv); #endif } int evutil_date_rfc1123(char *date, const size_t datelen, const struct tm *tm) { static const char *DAYS[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; static const char *MONTHS[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; time_t t = time(NULL); #ifndef _WIN32 struct tm sys; #endif /* If `tm` is null, set system's current time. */ if (tm == NULL) { #ifdef _WIN32 /** TODO: detect _gmtime64()/_gmtime64_s() */ tm = gmtime(&t); #else gmtime_r(&t, &sys); tm = &sys; #endif } return evutil_snprintf( date, datelen, "%s, %02d %s %4d %02d:%02d:%02d GMT", DAYS[tm->tm_wday], tm->tm_mday, MONTHS[tm->tm_mon], 1900 + tm->tm_year, tm->tm_hour, tm->tm_min, tm->tm_sec); } /* This function assumes it's called repeatedly with a not-actually-so-monotonic time source whose outputs are in 'tv'. It implements a trivial ratcheting mechanism so that the values never go backwards. */ static void adjust_monotonic_time(struct evutil_monotonic_timer *base, struct timeval *tv) { evutil_timeradd(tv, &base->adjust_monotonic_clock, tv); if (evutil_timercmp(tv, &base->last_time, <)) { /* Guess it wasn't monotonic after all. */ struct timeval adjust; evutil_timersub(&base->last_time, tv, &adjust); evutil_timeradd(&adjust, &base->adjust_monotonic_clock, &base->adjust_monotonic_clock); *tv = base->last_time; } base->last_time = *tv; } /* Allocate a new struct evutil_monotonic_timer */ struct evutil_monotonic_timer * evutil_monotonic_timer_new(void) { struct evutil_monotonic_timer *p = NULL; p = mm_malloc(sizeof(*p)); if (!p) goto done; memset(p, 0, sizeof(*p)); done: return p; } /* Free a struct evutil_monotonic_timer */ void evutil_monotonic_timer_free(struct evutil_monotonic_timer *timer) { if (timer) { mm_free(timer); } } /* Set up a struct evutil_monotonic_timer for initial use */ int evutil_configure_monotonic_time(struct evutil_monotonic_timer *timer, int flags) { return evutil_configure_monotonic_time_(timer, flags); } /* Query the current monotonic time */ int evutil_gettime_monotonic(struct evutil_monotonic_timer *timer, struct timeval *tp) { return evutil_gettime_monotonic_(timer, tp); } #if defined(HAVE_POSIX_MONOTONIC) /* ===== The POSIX clock_gettime() interface provides a few ways to get at a monotonic clock. CLOCK_MONOTONIC is most widely supported. Linux also provides a CLOCK_MONOTONIC_COARSE with accuracy of about 1-4 msec. On all platforms I'm aware of, CLOCK_MONOTONIC really is monotonic. Platforms don't agree about whether it should jump on a sleep/resume. */ int evutil_configure_monotonic_time_(struct evutil_monotonic_timer *base, int flags) { /* CLOCK_MONOTONIC exists on FreeBSD, Linux, and Solaris. You need to * check for it at runtime, because some older kernel versions won't * have it working. */ #ifdef CLOCK_MONOTONIC_COARSE const int precise = flags & EV_MONOT_PRECISE; #endif const int fallback = flags & EV_MONOT_FALLBACK; struct timespec ts; #ifdef CLOCK_MONOTONIC_COARSE if (CLOCK_MONOTONIC_COARSE < 0) { /* Technically speaking, nothing keeps CLOCK_* from being * negative (as far as I know). This check and the one below * make sure that it's safe for us to use -1 as an "unset" * value. */ event_errx(1,"I didn't expect CLOCK_MONOTONIC_COARSE to be < 0"); } if (! precise && ! fallback) { if (clock_gettime(CLOCK_MONOTONIC_COARSE, &ts) == 0) { base->monotonic_clock = CLOCK_MONOTONIC_COARSE; return 0; } } #endif if (!fallback && clock_gettime(CLOCK_MONOTONIC, &ts) == 0) { base->monotonic_clock = CLOCK_MONOTONIC; return 0; } if (CLOCK_MONOTONIC < 0) { event_errx(1,"I didn't expect CLOCK_MONOTONIC to be < 0"); } base->monotonic_clock = -1; return 0; } int evutil_gettime_monotonic_(struct evutil_monotonic_timer *base, struct timeval *tp) { struct timespec ts; if (base->monotonic_clock < 0) { if (evutil_gettimeofday(tp, NULL) < 0) return -1; adjust_monotonic_time(base, tp); return 0; } if (clock_gettime(base->monotonic_clock, &ts) == -1) return -1; tp->tv_sec = ts.tv_sec; tp->tv_usec = ts.tv_nsec / 1000; return 0; } #endif #if defined(HAVE_MACH_MONOTONIC) /* ====== Apple is a little late to the POSIX party. And why not? Instead of clock_gettime(), they provide mach_absolute_time(). Its units are not fixed; we need to use mach_timebase_info() to get the right functions to convert its units into nanoseconds. To all appearances, mach_absolute_time() seems to be honest-to-goodness monotonic. Whether it stops during sleep or not is unspecified in principle, and dependent on CPU architecture in practice. */ int evutil_configure_monotonic_time_(struct evutil_monotonic_timer *base, int flags) { const int fallback = flags & EV_MONOT_FALLBACK; struct mach_timebase_info mi; memset(base, 0, sizeof(*base)); /* OSX has mach_absolute_time() */ if (!fallback && mach_timebase_info(&mi) == 0 && mach_absolute_time() != 0) { /* mach_timebase_info tells us how to convert * mach_absolute_time() into nanoseconds, but we * want to use microseconds instead. */ mi.denom *= 1000; memcpy(&base->mach_timebase_units, &mi, sizeof(mi)); } else { base->mach_timebase_units.numer = 0; } return 0; } int evutil_gettime_monotonic_(struct evutil_monotonic_timer *base, struct timeval *tp) { ev_uint64_t abstime, usec; if (base->mach_timebase_units.numer == 0) { if (evutil_gettimeofday(tp, NULL) < 0) return -1; adjust_monotonic_time(base, tp); return 0; } abstime = mach_absolute_time(); usec = (abstime * base->mach_timebase_units.numer) / (base->mach_timebase_units.denom); tp->tv_sec = usec / 1000000; tp->tv_usec = usec % 1000000; return 0; } #endif #if defined(HAVE_WIN32_MONOTONIC) /* ===== Turn we now to Windows. Want monontonic time on Windows? Windows has QueryPerformanceCounter(), which gives time most high- resolution time. It's a pity it's not so monotonic in practice; it's also got some fun bugs, especially: with older Windowses, under virtualizations, with funny hardware, on multiprocessor systems, and so on. PEP418 [1] has a nice roundup of the issues here. There's GetTickCount64() on Vista and later, which gives a number of 1-msec ticks since startup. The accuracy here might be as bad as 10-20 msec, I hear. There's an undocumented function (NtSetTimerResolution) that allegedly increases the accuracy. Good luck! There's also GetTickCount(), which is only 32 bits, but seems to be supported on pre-Vista versions of Windows. Apparently, you can coax another 14 bits out of it, giving you 2231 years before rollover. The less said about timeGetTime() the better. "We don't care. We don't have to. We're the Phone Company." -- Lily Tomlin, SNL Our strategy, if precise timers are turned off, is to just use the best GetTickCount equivalent available. If we've been asked for precise timing, then we mostly[2] assume that GetTickCount is monotonic, and correct GetPerformanceCounter to approximate it. [1] http://www.python.org/dev/peps/pep-0418 [2] Of course, we feed the Windows stuff into adjust_monotonic_time() anyway, just in case it isn't. */ /* Parts of our logic in the win32 timer code here are closely based on BitTorrent's libUTP library. That code is subject to the following license: Copyright (c) 2010 BitTorrent, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ static ev_uint64_t evutil_GetTickCount_(struct evutil_monotonic_timer *base) { if (base->GetTickCount64_fn) { /* Let's just use GetTickCount64 if we can. */ return base->GetTickCount64_fn(); } else if (base->GetTickCount_fn) { /* Greg Hazel assures me that this works, that BitTorrent has * done it for years, and this it won't turn around and * bite us. He says they found it on some game programmers' * forum some time around 2007. */ ev_uint64_t v = base->GetTickCount_fn(); return (DWORD)v | ((v >> 18) & 0xFFFFFFFF00000000); } else { /* Here's the fallback implementation. We have to use * GetTickCount() with its given signature, so we only get * 32 bits worth of milliseconds, which will roll ove every * 49 days or so. */ DWORD ticks = GetTickCount(); if (ticks < base->last_tick_count) { base->adjust_tick_count += ((ev_uint64_t)1) << 32; } base->last_tick_count = ticks; return ticks + base->adjust_tick_count; } } int evutil_configure_monotonic_time_(struct evutil_monotonic_timer *base, int flags) { const int precise = flags & EV_MONOT_PRECISE; const int fallback = flags & EV_MONOT_FALLBACK; HANDLE h; memset(base, 0, sizeof(*base)); h = evutil_load_windows_system_library_(TEXT("kernel32.dll")); if (h != NULL && !fallback) { base->GetTickCount64_fn = (ev_GetTickCount_func)GetProcAddress(h, "GetTickCount64"); base->GetTickCount_fn = (ev_GetTickCount_func)GetProcAddress(h, "GetTickCount"); } base->first_tick = base->last_tick_count = evutil_GetTickCount_(base); if (precise && !fallback) { LARGE_INTEGER freq; if (QueryPerformanceFrequency(&freq)) { LARGE_INTEGER counter; QueryPerformanceCounter(&counter); base->first_counter = counter.QuadPart; base->usec_per_count = 1.0e6 / freq.QuadPart; base->use_performance_counter = 1; } } return 0; } static inline ev_int64_t abs64(ev_int64_t i) { return i < 0 ? -i : i; } int evutil_gettime_monotonic_(struct evutil_monotonic_timer *base, struct timeval *tp) { ev_uint64_t ticks = evutil_GetTickCount_(base); if (base->use_performance_counter) { /* Here's a trick we took from BitTorrent's libutp, at Greg * Hazel's recommendation. We use QueryPerformanceCounter for * our high-resolution timer, but use GetTickCount*() to keep * it sane, and adjust_monotonic_time() to keep it monotonic. */ LARGE_INTEGER counter; ev_int64_t counter_elapsed, counter_usec_elapsed, ticks_elapsed; QueryPerformanceCounter(&counter); counter_elapsed = (ev_int64_t) (counter.QuadPart - base->first_counter); ticks_elapsed = ticks - base->first_tick; /* TODO: This may upset VC6. If you need this to work with * VC6, please supply an appropriate patch. */ counter_usec_elapsed = (ev_int64_t) (counter_elapsed * base->usec_per_count); if (abs64(ticks_elapsed*1000 - counter_usec_elapsed) > 1000000) { /* It appears that the QueryPerformanceCounter() * result is more than 1 second away from * GetTickCount() result. Let's adjust it to be as * accurate as we can; adjust_monotnonic_time() below * will keep it monotonic. */ counter_usec_elapsed = ticks_elapsed * 1000; base->first_counter = (ev_uint64_t) (counter.QuadPart - counter_usec_elapsed / base->usec_per_count); } tp->tv_sec = (time_t) (counter_usec_elapsed / 1000000); tp->tv_usec = counter_usec_elapsed % 1000000; } else { /* We're just using GetTickCount(). */ tp->tv_sec = (time_t) (ticks / 1000); tp->tv_usec = (ticks % 1000) * 1000; } adjust_monotonic_time(base, tp); return 0; } #endif #if defined(HAVE_FALLBACK_MONOTONIC) /* ===== And if none of the other options work, let's just use gettimeofday(), and ratchet it forward so that it acts like a monotonic timer, whether it wants to or not. */ int evutil_configure_monotonic_time_(struct evutil_monotonic_timer *base, int precise) { memset(base, 0, sizeof(*base)); return 0; } int evutil_gettime_monotonic_(struct evutil_monotonic_timer *base, struct timeval *tp) { if (evutil_gettimeofday(tp, NULL) < 0) return -1; adjust_monotonic_time(base, tp); return 0; } #endif �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/ht-internal.h������������������������������������������������0000664�0000000�0000000�00000071001�15270523712�0022373�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* Copyright 2002 Christopher Clark */ /* Copyright 2005-2012 Nick Mathewson */ /* Copyright 2009-2012 Niels Provos and Nick Mathewson */ /* See license at end. */ /* Based on ideas by Christopher Clark and interfaces from Niels Provos. */ #ifndef HT_INTERNAL_H_INCLUDED_ #define HT_INTERNAL_H_INCLUDED_ #define HT_HEAD(name, type) \ struct name { \ /* The hash table itself. */ \ struct type **hth_table; \ /* How long is the hash table? */ \ unsigned hth_table_length; \ /* How many elements does the table contain? */ \ unsigned hth_n_entries; \ /* How many elements will we allow in the table before resizing it? */ \ unsigned hth_load_limit; \ /* Position of hth_table_length in the primes table. */ \ int hth_prime_idx; \ } #define HT_INITIALIZER() \ { NULL, 0, 0, 0, -1 } #ifdef HT_NO_CACHE_HASH_VALUES #define HT_ENTRY(type) \ struct { \ struct type *hte_next; \ } #else #define HT_ENTRY(type) \ struct { \ struct type *hte_next; \ unsigned hte_hash; \ } #endif #define HT_EMPTY(head) \ ((head)->hth_n_entries == 0) /* How many elements in 'head'? */ #define HT_SIZE(head) \ ((head)->hth_n_entries) /* Return memory usage for a hashtable (not counting the entries themselves) */ #define HT_MEM_USAGE(head) \ (sizeof(*head) + (head)->hth_table_length * sizeof(void*)) #define HT_FIND(name, head, elm) name##_HT_FIND((head), (elm)) #define HT_INSERT(name, head, elm) name##_HT_INSERT((head), (elm)) #define HT_REPLACE(name, head, elm) name##_HT_REPLACE((head), (elm)) #define HT_REMOVE(name, head, elm) name##_HT_REMOVE((head), (elm)) #define HT_START(name, head) name##_HT_START(head) #define HT_NEXT(name, head, elm) name##_HT_NEXT((head), (elm)) #define HT_NEXT_RMV(name, head, elm) name##_HT_NEXT_RMV((head), (elm)) #define HT_CLEAR(name, head) name##_HT_CLEAR(head) #define HT_INIT(name, head) name##_HT_INIT(head) /* Helper: */ static inline unsigned ht_improve_hash_(unsigned h) { /* Aim to protect against poor hash functions by adding logic here * - logic taken from java 1.4 hashtable source */ h += ~(h << 9); h ^= ((h >> 14) | (h << 18)); /* >>> */ h += (h << 4); h ^= ((h >> 10) | (h << 22)); /* >>> */ return h; } #if 0 /** Basic string hash function, from Java standard String.hashCode(). */ static inline unsigned ht_string_hash_(const char *s) { unsigned h = 0; int m = 1; while (*s) { h += ((signed char)*s++)*m; m = (m<<5)-1; /* m *= 31 */ } return h; } #endif /** Basic string hash function, from Python's str.__hash__() */ static inline unsigned ht_string_hash_(const char *s) { unsigned h; const unsigned char *cp = (const unsigned char *)s; h = *cp << 7; while (*cp) { h = (1000003*h) ^ *cp++; } /* This conversion truncates the length of the string, but that's ok. */ h ^= (unsigned)(cp-(const unsigned char*)s); return h; } #ifndef HT_NO_CACHE_HASH_VALUES #define HT_SET_HASH_(elm, field, hashfn) \ do { (elm)->field.hte_hash = hashfn(elm); } while (0) #define HT_SET_HASHVAL_(elm, field, val) \ do { (elm)->field.hte_hash = (val); } while (0) #define HT_ELT_HASH_(elm, field, hashfn) \ ((elm)->field.hte_hash) #else #define HT_SET_HASH_(elm, field, hashfn) \ ((void)0) #define HT_ELT_HASH_(elm, field, hashfn) \ (hashfn(elm)) #define HT_SET_HASHVAL_(elm, field, val) \ ((void)0) #endif /* Helper: alias for the bucket containing 'elm'. */ #define HT_BUCKET_(head, field, elm, hashfn) \ ((head)->hth_table[HT_ELT_HASH_(elm,field,hashfn) % head->hth_table_length]) #define HT_FOREACH(x, name, head) \ for ((x) = HT_START(name, head); \ (x) != NULL; \ (x) = HT_NEXT(name, head, x)) #define HT_PROTOTYPE(name, type, field, hashfn, eqfn) \ int name##_HT_GROW(struct name *ht, unsigned min_capacity); \ void name##_HT_CLEAR(struct name *ht); \ int name##_HT_REP_IS_BAD_(const struct name *ht); \ static inline void \ name##_HT_INIT(struct name *head) { \ head->hth_table_length = 0; \ head->hth_table = NULL; \ head->hth_n_entries = 0; \ head->hth_load_limit = 0; \ head->hth_prime_idx = -1; \ } \ /* Helper: returns a pointer to the right location in the table \ * 'head' to find or insert the element 'elm'. */ \ static inline struct type ** \ name##_HT_FIND_P_(struct name *head, struct type *elm) \ { \ struct type **p; \ if (!head->hth_table) \ return NULL; \ p = &HT_BUCKET_(head, field, elm, hashfn); \ while (*p) { \ if (eqfn(*p, elm)) \ return p; \ p = &(*p)->field.hte_next; \ } \ return p; \ } \ /* Return a pointer to the element in the table 'head' matching 'elm', \ * or NULL if no such element exists */ \ static inline struct type * \ name##_HT_FIND(const struct name *head, struct type *elm) \ { \ struct type **p; \ struct name *h = (struct name *) head; \ HT_SET_HASH_(elm, field, hashfn); \ p = name##_HT_FIND_P_(h, elm); \ return p ? *p : NULL; \ } \ /* Insert the element 'elm' into the table 'head'. Do not call this \ * function if the table might already contain a matching element. */ \ static inline void \ name##_HT_INSERT(struct name *head, struct type *elm) \ { \ struct type **p; \ if (!head->hth_table || head->hth_n_entries >= head->hth_load_limit) \ name##_HT_GROW(head, head->hth_n_entries+1); \ ++head->hth_n_entries; \ HT_SET_HASH_(elm, field, hashfn); \ p = &HT_BUCKET_(head, field, elm, hashfn); \ elm->field.hte_next = *p; \ *p = elm; \ } \ /* Insert the element 'elm' into the table 'head'. If there already \ * a matching element in the table, replace that element and return \ * it. */ \ static inline struct type * \ name##_HT_REPLACE(struct name *head, struct type *elm) \ { \ struct type **p, *r; \ if (!head->hth_table || head->hth_n_entries >= head->hth_load_limit) \ name##_HT_GROW(head, head->hth_n_entries+1); \ HT_SET_HASH_(elm, field, hashfn); \ p = name##_HT_FIND_P_(head, elm); \ r = *p; \ *p = elm; \ if (r && (r!=elm)) { \ elm->field.hte_next = r->field.hte_next; \ r->field.hte_next = NULL; \ return r; \ } else { \ ++head->hth_n_entries; \ return NULL; \ } \ } \ /* Remove any element matching 'elm' from the table 'head'. If such \ * an element is found, return it; otherwise return NULL. */ \ static inline struct type * \ name##_HT_REMOVE(struct name *head, struct type *elm) \ { \ struct type **p, *r; \ HT_SET_HASH_(elm, field, hashfn); \ p = name##_HT_FIND_P_(head,elm); \ if (!p || !*p) \ return NULL; \ r = *p; \ *p = r->field.hte_next; \ r->field.hte_next = NULL; \ --head->hth_n_entries; \ return r; \ } \ /* Invoke the function 'fn' on every element of the table 'head', \ * using 'data' as its second argument. If the function returns \ * nonzero, remove the most recently examined element before invoking \ * the function again. */ \ static inline void \ name##_HT_FOREACH_FN(struct name *head, \ int (*fn)(struct type *, void *), \ void *data) \ { \ unsigned idx; \ struct type **p, **nextp, *next; \ if (!head->hth_table) \ return; \ for (idx=0; idx < head->hth_table_length; ++idx) { \ p = &head->hth_table[idx]; \ while (*p) { \ nextp = &(*p)->field.hte_next; \ next = *nextp; \ if (fn(*p, data)) { \ --head->hth_n_entries; \ *p = next; \ } else { \ p = nextp; \ } \ } \ } \ } \ /* Return a pointer to the first element in the table 'head', under \ * an arbitrary order. This order is stable under remove operations, \ * but not under others. If the table is empty, return NULL. */ \ static inline struct type ** \ name##_HT_START(struct name *head) \ { \ unsigned b = 0; \ while (b < head->hth_table_length) { \ if (head->hth_table[b]) \ return &head->hth_table[b]; \ ++b; \ } \ return NULL; \ } \ /* Return the next element in 'head' after 'elm', under the arbitrary \ * order used by HT_START. If there are no more elements, return \ * NULL. If 'elm' is to be removed from the table, you must call \ * this function for the next value before you remove it. \ */ \ static inline struct type ** \ name##_HT_NEXT(struct name *head, struct type **elm) \ { \ if ((*elm)->field.hte_next) { \ return &(*elm)->field.hte_next; \ } else { \ unsigned b = (HT_ELT_HASH_(*elm, field, hashfn) % head->hth_table_length)+1; \ while (b < head->hth_table_length) { \ if (head->hth_table[b]) \ return &head->hth_table[b]; \ ++b; \ } \ return NULL; \ } \ } \ static inline struct type ** \ name##_HT_NEXT_RMV(struct name *head, struct type **elm) \ { \ unsigned h = HT_ELT_HASH_(*elm, field, hashfn); \ *elm = (*elm)->field.hte_next; \ --head->hth_n_entries; \ if (*elm) { \ return elm; \ } else { \ unsigned b = (h % head->hth_table_length)+1; \ while (b < head->hth_table_length) { \ if (head->hth_table[b]) \ return &head->hth_table[b]; \ ++b; \ } \ return NULL; \ } \ } #define HT_GENERATE(name, type, field, hashfn, eqfn, load, mallocfn, \ reallocfn, freefn) \ static unsigned name##_PRIMES[] = { \ 53, 97, 193, 389, \ 769, 1543, 3079, 6151, \ 12289, 24593, 49157, 98317, \ 196613, 393241, 786433, 1572869, \ 3145739, 6291469, 12582917, 25165843, \ 50331653, 100663319, 201326611, 402653189, \ 805306457, 1610612741 \ }; \ static unsigned name##_N_PRIMES = \ (unsigned)(sizeof(name##_PRIMES)/sizeof(name##_PRIMES[0])); \ /* Expand the internal table of 'head' until it is large enough to \ * hold 'size' elements. Return 0 on success, -1 on allocation \ * failure. */ \ int \ name##_HT_GROW(struct name *head, unsigned size) \ { \ unsigned new_len, new_load_limit; \ int prime_idx; \ struct type **new_table; \ if (head->hth_prime_idx == (int)name##_N_PRIMES - 1) \ return 0; \ if (head->hth_load_limit > size) \ return 0; \ prime_idx = head->hth_prime_idx; \ do { \ new_len = name##_PRIMES[++prime_idx]; \ new_load_limit = (unsigned)(load*new_len); \ } while (new_load_limit <= size && \ prime_idx < (int)name##_N_PRIMES); \ if ((new_table = mallocfn(new_len*sizeof(struct type*)))) { \ unsigned b; \ memset(new_table, 0, new_len*sizeof(struct type*)); \ for (b = 0; b < head->hth_table_length; ++b) { \ struct type *elm, *next; \ unsigned b2; \ elm = head->hth_table[b]; \ while (elm) { \ next = elm->field.hte_next; \ b2 = HT_ELT_HASH_(elm, field, hashfn) % new_len; \ elm->field.hte_next = new_table[b2]; \ new_table[b2] = elm; \ elm = next; \ } \ } \ if (head->hth_table) \ freefn(head->hth_table); \ head->hth_table = new_table; \ } else { \ unsigned b, b2; \ new_table = reallocfn(head->hth_table, new_len*sizeof(struct type*)); \ if (!new_table) return -1; \ memset(new_table + head->hth_table_length, 0, \ (new_len - head->hth_table_length)*sizeof(struct type*)); \ for (b=0; b < head->hth_table_length; ++b) { \ struct type *e, **pE; \ for (pE = &new_table[b], e = *pE; e != NULL; e = *pE) { \ b2 = HT_ELT_HASH_(e, field, hashfn) % new_len; \ if (b2 == b) { \ pE = &e->field.hte_next; \ } else { \ *pE = e->field.hte_next; \ e->field.hte_next = new_table[b2]; \ new_table[b2] = e; \ } \ } \ } \ head->hth_table = new_table; \ } \ head->hth_table_length = new_len; \ head->hth_prime_idx = prime_idx; \ head->hth_load_limit = new_load_limit; \ return 0; \ } \ /* Free all storage held by 'head'. Does not free 'head' itself, or \ * individual elements. */ \ void \ name##_HT_CLEAR(struct name *head) \ { \ if (head->hth_table) \ freefn(head->hth_table); \ name##_HT_INIT(head); \ } \ /* Debugging helper: return false iff the representation of 'head' is \ * internally consistent. */ \ int \ name##_HT_REP_IS_BAD_(const struct name *head) \ { \ unsigned n, i; \ struct type *elm; \ if (!head->hth_table_length) { \ if (!head->hth_table && !head->hth_n_entries && \ !head->hth_load_limit && head->hth_prime_idx == -1) \ return 0; \ else \ return 1; \ } \ if (!head->hth_table || head->hth_prime_idx < 0 || \ !head->hth_load_limit) \ return 2; \ if (head->hth_n_entries > head->hth_load_limit) \ return 3; \ if (head->hth_table_length != name##_PRIMES[head->hth_prime_idx]) \ return 4; \ if (head->hth_load_limit != (unsigned)(load*head->hth_table_length)) \ return 5; \ for (n = i = 0; i < head->hth_table_length; ++i) { \ for (elm = head->hth_table[i]; elm; elm = elm->field.hte_next) { \ if (HT_ELT_HASH_(elm, field, hashfn) != hashfn(elm)) \ return 1000 + i; \ if ((HT_ELT_HASH_(elm, field, hashfn) % head->hth_table_length) != i) \ return 10000 + i; \ ++n; \ } \ } \ if (n != head->hth_n_entries) \ return 6; \ return 0; \ } /** Implements an over-optimized "find and insert if absent" block; * not meant for direct usage by typical code, or usage outside the critical * path.*/ #define HT_FIND_OR_INSERT_(name, field, hashfn, head, eltype, elm, var, y, n) \ { \ struct name *var##_head_ = head; \ struct eltype **var; \ if (!var##_head_->hth_table || \ var##_head_->hth_n_entries >= var##_head_->hth_load_limit) \ name##_HT_GROW(var##_head_, var##_head_->hth_n_entries+1); \ HT_SET_HASH_((elm), field, hashfn); \ var = name##_HT_FIND_P_(var##_head_, (elm)); \ if (*var) { \ y; \ } else { \ n; \ } \ } #define HT_FOI_INSERT_(field, head, elm, newent, var) \ { \ HT_SET_HASHVAL_(newent, field, (elm)->field.hte_hash); \ newent->field.hte_next = NULL; \ *var = newent; \ ++((head)->hth_n_entries); \ } /* * Copyright 2005, Nick Mathewson. Implementation logic is adapted from code * by Christopher Clark, retrofit to allow drop-in memory management, and to * use the same interface as Niels Provos's tree.h. This is probably still * a derived work, so the original license below still applies. * * Copyright (c) 2002, Christopher Clark * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * * Neither the name of the original author; nor the names of any contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #endif �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/http-internal.h����������������������������������������������0000664�0000000�0000000�00000013435�15270523712�0022746�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright 2001-2007 Niels Provos <provos@citi.umich.edu> * Copyright 2007-2012 Niels Provos and Nick Mathewson * * This header file contains definitions for dealing with HTTP requests * that are internal to libevent. As user of the library, you should not * need to know about these. */ #ifndef HTTP_INTERNAL_H_INCLUDED_ #define HTTP_INTERNAL_H_INCLUDED_ #include "event2/event_struct.h" #include "util-internal.h" #include "defer-internal.h" #define HTTP_CONNECT_TIMEOUT 45 #define HTTP_WRITE_TIMEOUT 50 #define HTTP_READ_TIMEOUT 50 #define HTTP_PREFIX "http://" #define HTTP_DEFAULTPORT 80 enum message_read_status { ALL_DATA_READ = 1, MORE_DATA_EXPECTED = 0, DATA_CORRUPTED = -1, REQUEST_CANCELED = -2, DATA_TOO_LONG = -3 }; struct evbuffer; struct addrinfo; struct evhttp_request; /* Indicates an unknown request method. */ #define EVHTTP_REQ_UNKNOWN_ (1<<15) enum evhttp_connection_state { EVCON_DISCONNECTED, /**< not currently connected not trying either*/ EVCON_CONNECTING, /**< tries to currently connect */ EVCON_IDLE, /**< connection is established */ EVCON_READING_FIRSTLINE,/**< reading Request-Line (incoming conn) or **< Status-Line (outgoing conn) */ EVCON_READING_HEADERS, /**< reading request/response headers */ EVCON_READING_BODY, /**< reading request/response body */ EVCON_READING_TRAILER, /**< reading request/response chunked trailer */ EVCON_WRITING /**< writing request/response headers/body */ }; struct event_base; /* A client or server connection. */ struct evhttp_connection { /* we use this tailq only if this connection was created for an http * server */ TAILQ_ENTRY(evhttp_connection) next; evutil_socket_t fd; struct bufferevent *bufev; struct event retry_ev; /* for retrying connects */ char *bind_address; /* address to use for binding the src */ ev_uint16_t bind_port; /* local port for binding the src */ char *address; /* address to connect to */ ev_uint16_t port; size_t max_headers_size; ev_uint64_t max_body_size; int flags; #define EVHTTP_CON_INCOMING 0x0001 /* only one request on it ever */ #define EVHTTP_CON_OUTGOING 0x0002 /* multiple requests possible */ #define EVHTTP_CON_CLOSEDETECT 0x0004 /* detecting if persistent close */ /* set when we want to auto free the connection */ #define EVHTTP_CON_AUTOFREE EVHTTP_CON_PUBLIC_FLAGS_END /* Installed when attempt to read HTTP error after write failed, see * EVHTTP_CON_READ_ON_WRITE_ERROR */ #define EVHTTP_CON_READING_ERROR (EVHTTP_CON_AUTOFREE << 1) struct timeval timeout; /* timeout for events */ int retry_cnt; /* retry count */ int retry_max; /* maximum number of retries */ struct timeval initial_retry_timeout; /* Timeout for low long to wait * after first failing attempt * before retry */ enum evhttp_connection_state state; /* for server connections, the http server they are connected with */ struct evhttp *http_server; TAILQ_HEAD(evcon_requestq, evhttp_request) requests; void (*cb)(struct evhttp_connection *, void *); void *cb_arg; void (*closecb)(struct evhttp_connection *, void *); void *closecb_arg; struct event_callback read_more_deferred_cb; struct event_base *base; struct evdns_base *dns_base; int ai_family; }; /* A callback for an http server */ struct evhttp_cb { TAILQ_ENTRY(evhttp_cb) next; char *what; void (*cb)(struct evhttp_request *req, void *); void *cbarg; }; /* both the http server as well as the rpc system need to queue connections */ TAILQ_HEAD(evconq, evhttp_connection); /* each bound socket is stored in one of these */ struct evhttp_bound_socket { TAILQ_ENTRY(evhttp_bound_socket) next; struct evconnlistener *listener; }; /* server alias list item. */ struct evhttp_server_alias { TAILQ_ENTRY(evhttp_server_alias) next; char *alias; /* the server alias. */ }; struct evhttp { /* Next vhost, if this is a vhost. */ TAILQ_ENTRY(evhttp) next_vhost; /* All listeners for this host */ TAILQ_HEAD(boundq, evhttp_bound_socket) sockets; TAILQ_HEAD(httpcbq, evhttp_cb) callbacks; /* All live connections on this host. */ struct evconq connections; TAILQ_HEAD(vhostsq, evhttp) virtualhosts; TAILQ_HEAD(aliasq, evhttp_server_alias) aliases; /* NULL if this server is not a vhost */ char *vhost_pattern; struct timeval timeout; size_t default_max_headers_size; ev_uint64_t default_max_body_size; int flags; const char *default_content_type; /* Bitmask of all HTTP methods that we accept and pass to user * callbacks. */ ev_uint16_t allowed_methods; /* Fallback callback if all the other callbacks for this connection don't match. */ void (*gencb)(struct evhttp_request *req, void *); void *gencbarg; struct bufferevent* (*bevcb)(struct event_base *, void *); void *bevcbarg; struct event_base *base; }; /* XXX most of these functions could be static. */ /* resets the connection; can be reused for more requests */ void evhttp_connection_reset_(struct evhttp_connection *); /* connects if necessary */ int evhttp_connection_connect_(struct evhttp_connection *); enum evhttp_request_error; /* notifies the current request that it failed; resets connection */ void evhttp_connection_fail_(struct evhttp_connection *, enum evhttp_request_error error); enum message_read_status; enum message_read_status evhttp_parse_firstline_(struct evhttp_request *, struct evbuffer*); enum message_read_status evhttp_parse_headers_(struct evhttp_request *, struct evbuffer*); void evhttp_start_read_(struct evhttp_connection *); void evhttp_start_write_(struct evhttp_connection *); /* response sending HTML the data in the buffer */ void evhttp_response_code_(struct evhttp_request *, int, const char *); void evhttp_send_page_(struct evhttp_request *, struct evbuffer *); int evhttp_decode_uri_internal(const char *uri, size_t length, char *ret, int decode_plus); #endif /* _HTTP_H */ �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������kytensor-2.49.0.7/third_party/libevent/http.c�������������������������������������������������������0000664�0000000�0000000�00000361626�15270523712�0021137�0����������������������������������������������������������������������������������������������������ustar�00root����������������������������root����������������������������0000000�0000000������������������������������������������������������������������������������������������������������������������������������������������������������������������������/* * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu> * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_SYS_PARAM_H #include <sys/param.h> #endif #ifdef EVENT__HAVE_SYS_TYPES_H #include <sys/types.h> #endif #ifdef HAVE_SYS_IOCCOM_H #include <sys/ioccom.h> #endif #ifdef EVENT__HAVE_SYS_RESOURCE_H #include <sys/resource.h> #endif #ifdef EVENT__HAVE_SYS_TIME_H #include <sys/time.h> #endif #ifdef EVENT__HAVE_SYS_WAIT_H #include <sys/wait.h> #endif #ifndef _WIN32 #include <sys/socket.h> #include <sys/stat.h> #else #include <winsock2.h> #include <ws2tcpip.h> #endif #include <sys/queue.h> #ifdef EVENT__HAVE_NETINET_IN_H #include <netinet/in.h> #endif #ifdef EVENT__HAVE_ARPA_INET_H #include <arpa/inet.h> #endif #ifdef EVENT__HAVE_NETDB_H #include <netdb.h> #endif #ifdef _WIN32 #include <winsock2.h> #endif #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #ifndef _WIN32 #include <syslog.h> #endif #include <signal.h> #ifdef EVENT__HAVE_UNISTD_H #include <unistd.h> #endif #ifdef EVENT__HAVE_FCNTL_H #include <fcntl.h> #endif #undef timeout_pending #undef timeout_initialized #include "strlcpy-internal.h" #include "event2/http.h" #include "event2/event.h" #include "event2/buffer.h" #include "event2/bufferevent.h" #include "event2/http_struct.h" #include "event2/http_compat.h" #include "event2/util.h" #include "event2/listener.h" #include "log-internal.h" #include "util-internal.h" #include "http-internal.h" #include "mm-internal.h" #include "bufferevent-internal.h" #ifndef EVENT__HAVE_GETNAMEINFO #define NI_MAXSERV 32 #define NI_MAXHOST 1025 #ifndef NI_NUMERICHOST #define NI_NUMERICHOST 1 #endif #ifndef NI_NUMERICSERV #define NI_NUMERICSERV 2 #endif static int fake_getnameinfo(const struct sockaddr *sa, size_t salen, char *host, size_t hostlen, char *serv, size_t servlen, int flags) { struct sockaddr_in *sin = (struct sockaddr_in *)sa; if (serv != NULL) { char tmpserv[16]; evutil_snprintf(tmpserv, sizeof(tmpserv), "%d", ntohs(sin->sin_port)); if (strlcpy(serv, tmpserv, servlen) >= servlen) return (-1); } if (host != NULL) { if (flags & NI_NUMERICHOST) { if (strlcpy(host, inet_ntoa(sin->sin_addr), hostlen) >= hostlen) return (-1); else return (0); } else { struct hostent *hp; hp = gethostbyaddr((char *)&sin->sin_addr, sizeof(struct in_addr), AF_INET); if (hp == NULL) return (-2); if (strlcpy(host, hp->h_name, hostlen) >= hostlen) return (-1); else return (0); } } return (0); } #endif #define REQ_VERSION_BEFORE(req, major_v, minor_v) \ ((req)->major < (major_v) || \ ((req)->major == (major_v) && (req)->minor < (minor_v))) #define REQ_VERSION_ATLEAST(req, major_v, minor_v) \ ((req)->major > (major_v) || \ ((req)->major == (major_v) && (req)->minor >= (minor_v))) #ifndef MIN #define MIN(a,b) (((a)<(b))?(a):(b)) #endif extern int debug; static evutil_socket_t bind_socket_ai(struct evutil_addrinfo *, int reuse); static evutil_socket_t bind_socket(const char *, ev_uint16_t, int reuse); static void name_from_addr(struct sockaddr *, ev_socklen_t, char **, char **); static int evhttp_associate_new_request_with_connection( struct evhttp_connection *evcon); static void evhttp_connection_start_detectclose( struct evhttp_connection *evcon); static void evhttp_connection_stop_detectclose( struct evhttp_connection *evcon); static void evhttp_request_dispatch(struct evhttp_connection* evcon); static void evhttp_read_firstline(struct evhttp_connection *evcon, struct evhttp_request *req); static void evhttp_read_header(struct evhttp_connection *evcon, struct evhttp_request *req); static int evhttp_add_header_internal(struct evkeyvalq *headers, const char *key, const char *value); static const char *evhttp_response_phrase_internal(int code); static void evhttp_get_request(struct evhttp *, evutil_socket_t, struct sockaddr *, ev_socklen_t); static void evhttp_write_buffer(struct evhttp_connection *, void (*)(struct evhttp_connection *, void *), void *); static void evhttp_make_header(struct evhttp_connection *, struct evhttp_request *); /* callbacks for bufferevent */ static void evhttp_read_cb(struct bufferevent *, void *); static void evhttp_write_cb(struct bufferevent *, void *); static void evhttp_error_cb(struct bufferevent *bufev, short what, void *arg); static int evhttp_find_vhost(struct evhttp *http, struct evhttp **outhttp, const char *hostname); #ifndef EVENT__HAVE_STRSEP /* strsep replacement for platforms that lack it. Only works if * del is one character long. */ static char * strsep(char **s, const char *del) { char *d, *tok; EVUTIL_ASSERT(strlen(del) == 1); if (!s || !*s) return NULL; tok = *s; d = strstr(tok, del); if (d) { *d = '\0'; *s = d + 1; } else *s = NULL; return tok; } #endif static size_t html_replace(const char ch, const char **escaped) { switch (ch) { case '<': *escaped = "<"; return 4; case '>': *escaped = ">"; return 4; case '"': *escaped = """; return 6; case '\'': *escaped = "'"; return 6; case '&': *escaped = "&"; return 5; default: break; } return 1; } /* * Replaces <, >, ", ' and & with <, >, ", * ' and & correspondingly. * * The returned string needs to be freed by the caller. */ char * evhttp_htmlescape(const char *html) { size_t i; size_t new_size = 0, old_size = 0; char *escaped_html, *p; if (html == NULL) return (NULL); old_size = strlen(html); for (i = 0; i < old_size; ++i) { const char *replaced = NULL; const size_t replace_size = html_replace(html[i], &replaced); if (replace_size > EV_SIZE_MAX - new_size) { event_warn("%s: html_replace overflow", __func__); return (NULL); } new_size += replace_size; } if (new_size == EV_SIZE_MAX) return (NULL); p = escaped_html = mm_malloc(new_size + 1); if (escaped_html == NULL) { event_warn("%s: malloc(%lu)", __func__, (unsigned long)(new_size + 1)); return (NULL); } for (i = 0; i < old_size; ++i) { const char *replaced = &html[i]; const size_t len = html_replace(html[i], &replaced); memcpy(p, replaced, len); p += len; } *p = '\0'; return (escaped_html); } /** Given an evhttp_cmd_type, returns a constant string containing the * equivalent HTTP command, or NULL if the evhttp_command_type is * unrecognized. */ static const char * evhttp_method(enum evhttp_cmd_type type) { const char *method; switch (type) { case EVHTTP_REQ_GET: method = "GET"; break; case EVHTTP_REQ_POST: method = "POST"; break; case EVHTTP_REQ_HEAD: method = "HEAD"; break; case EVHTTP_REQ_PUT: method = "PUT"; break; case EVHTTP_REQ_DELETE: method = "DELETE"; break; case EVHTTP_REQ_OPTIONS: method = "OPTIONS"; break; case EVHTTP_REQ_TRACE: method = "TRACE"; break; case EVHTTP_REQ_CONNECT: method = "CONNECT"; break; case EVHTTP_REQ_PATCH: method = "PATCH"; break; default: method = NULL; break; } return (method); } /** * Determines if a response should have a body. * Follows the rules in RFC 2616 section 4.3. * @return 1 if the response MUST have a body; 0 if the response MUST NOT have * a body. */ static int evhttp_response_needs_body(struct evhttp_request *req) { return (req->response_code != HTTP_NOCONTENT && req->response_code != HTTP_NOTMODIFIED && (req->response_code < 100 || req->response_code >= 200) && req->type != EVHTTP_REQ_HEAD); } /** Helper: called after we've added some data to an evcon's bufferevent's * output buffer. Sets the evconn's writing-is-done callback, and puts * the bufferevent into writing mode. */ static void evhttp_write_buffer(struct evhttp_connection *evcon, void (*cb)(struct evhttp_connection *, void *), void *arg) { event_debug(("%s: preparing to write buffer\n", __func__)); /* Set call back */ evcon->cb = cb; evcon->cb_arg = arg; /* Disable the read callback: we don't actually care about data; * we only care about close detection. (We don't disable reading, * since we *do* want to learn about any close events.) */ bufferevent_setcb(evcon->bufev, NULL, /*read*/ evhttp_write_cb, evhttp_error_cb, evcon); bufferevent_enable(evcon->bufev, EV_WRITE); } static void evhttp_send_continue_done(struct evhttp_connection *evcon, void *arg) { bufferevent_disable(evcon->bufev, EV_WRITE); } static void evhttp_send_continue(struct evhttp_connection *evcon, struct evhttp_request *req) { bufferevent_enable(evcon->bufev, EV_WRITE); evbuffer_add_printf(bufferevent_get_output(evcon->bufev), "HTTP/%d.%d 100 Continue\r\n\r\n", req->major, req->minor); evcon->cb = evhttp_send_continue_done; evcon->cb_arg = NULL; bufferevent_setcb(evcon->bufev, evhttp_read_cb, evhttp_write_cb, evhttp_error_cb, evcon); } /** Helper: returns true iff evconn is in any connected state. */ static int evhttp_connected(struct evhttp_connection *evcon) { switch (evcon->state) { case EVCON_DISCONNECTED: case EVCON_CONNECTING: return (0); case EVCON_IDLE: case EVCON_READING_FIRSTLINE: case EVCON_READING_HEADERS: case EVCON_READING_BODY: case EVCON_READING_TRAILER: case EVCON_WRITING: default: return (1); } } /* Create the headers needed for an outgoing HTTP request, adds them to * the request's header list, and writes the request line to the * connection's output buffer. */ static void evhttp_make_header_request(struct evhttp_connection *evcon, struct evhttp_request *req) { const char *method; evhttp_remove_header(req->output_headers, "Proxy-Connection"); /* Generate request line */ if (!(method = evhttp_method(req->type))) { method = "NULL"; } evbuffer_add_printf(bufferevent_get_output(evcon->bufev), "%s %s HTTP/%d.%d\r\n", method, req->uri, req->major, req->minor); /* Add the content length on a post or put request if missing */ if ((req->type == EVHTTP_REQ_POST || req->type == EVHTTP_REQ_PUT) && evhttp_find_header(req->output_headers, "Content-Length") == NULL){ char size[22]; evutil_snprintf(size, sizeof(size), EV_SIZE_FMT, EV_SIZE_ARG(evbuffer_get_length(req->output_buffer))); evhttp_add_header(req->output_headers, "Content-Length", size); } } /** Return true if the list of headers in 'headers', intepreted with respect * to flags, means that we should send a "connection: close" when the request * is done. */ static int evhttp_is_connection_close(int flags, struct evkeyvalq* headers) { if (flags & EVHTTP_PROXY_REQUEST) { /* proxy connection */ const char *connection = evhttp_find_header(headers, "Proxy-Connection"); return (connection == NULL || evutil_ascii_strcasecmp(connection, "keep-alive") != 0); } else { const char *connection = evhttp_find_header(headers, "Connection"); return (connection != NULL && evutil_ascii_strcasecmp(connection, "close") == 0); } } static int evhttp_is_request_connection_close(struct evhttp_request *req) { return evhttp_is_connection_close(req->flags, req->input_headers) || evhttp_is_connection_close(req->flags, req->output_headers); } /* Return true iff 'headers' contains 'Connection: keep-alive' */ static int evhttp_is_connection_keepalive(struct evkeyvalq* headers) { const char *connection = evhttp_find_header(headers, "Connection"); return (connection != NULL && evutil_ascii_strncasecmp(connection, "keep-alive", 10) == 0); } /* Add a correct "Date" header to headers, unless it already has one. */ static void evhttp_maybe_add_date_header(struct evkeyvalq *headers) { if (evhttp_find_header(headers, "Date") == NULL) { char date[50]; if (sizeof(date) - evutil_date_rfc1123(date, sizeof(date), NULL) > 0) { evhttp_add_header(headers, "Date", date); } } } /* Add a "Content-Length" header with value 'content_length' to headers, * unless it already has a content-length or transfer-encoding header. */ static void evhttp_maybe_add_content_length_header(struct evkeyvalq *headers, size_t content_length) { if (evhttp_find_header(headers, "Transfer-Encoding") == NULL && evhttp_find_header(headers, "Content-Length") == NULL) { char len[22]; evutil_snprintf(len, sizeof(len), EV_SIZE_FMT, EV_SIZE_ARG(content_length)); evhttp_add_header(headers, "Content-Length", len); } } /* * Create the headers needed for an HTTP reply in req->output_headers, * and write the first HTTP response for req line to evcon. */ static void evhttp_make_header_response(struct evhttp_connection *evcon, struct evhttp_request *req) { int is_keepalive = evhttp_is_connection_keepalive(req->input_headers); evbuffer_add_printf(bufferevent_get_output(evcon->bufev), "HTTP/%d.%d %d %s\r\n", req->major, req->minor, req->response_code, req->response_code_line); if (req->major == 1) { if (req->minor >= 1) evhttp_maybe_add_date_header(req->output_headers); /* * if the protocol is 1.0; and the connection was keep-alive * we need to add a keep-alive header, too. */ if (req->minor == 0 && is_keepalive) evhttp_add_header(req->output_headers, "Connection", "keep-alive"); if ((req->minor >= 1 || is_keepalive) && evhttp_response_needs_body(req)) { /* * we need to add the content length if the * user did not give it, this is required for * persistent connections to work. */ evhttp_maybe_add_content_length_header( req->output_headers, evbuffer_get_length(req->output_buffer)); } } /* Potentially add headers for unidentified content. */ if (evhttp_response_needs_body(req)) { if (evhttp_find_header(req->output_headers, "Content-Type") == NULL && evcon->http_server->default_content_type) { evhttp_add_header(req->output_headers, "Content-Type", evcon->http_server->default_content_type); } } /* if the request asked for a close, we send a close, too */ if (evhttp_is_connection_close(req->flags, req->input_headers)) { evhttp_remove_header(req->output_headers, "Connection"); if (!(req->flags & EVHTTP_PROXY_REQUEST)) evhttp_add_header(req->output_headers, "Connection", "close"); evhttp_remove_header(req->output_headers, "Proxy-Connection"); } } enum expect { NO, CONTINUE, OTHER }; static enum expect evhttp_have_expect(struct evhttp_request *req, int input) { const char *expect; struct evkeyvalq *h = input ? req->input_headers : req->output_headers; if (!(req->kind == EVHTTP_REQUEST) || !REQ_VERSION_ATLEAST(req, 1, 1)) return NO; expect = evhttp_find_header(h, "Expect"); if (!expect) return NO; return !evutil_ascii_strcasecmp(expect, "100-continue") ? CONTINUE : OTHER; } /** Generate all headers appropriate for sending the http request in req (or * the response, if we're sending a response), and write them to evcon's * bufferevent. Also writes all data from req->output_buffer */ static void evhttp_make_header(struct evhttp_connection *evcon, struct evhttp_request *req) { struct evkeyval *header; struct evbuffer *output = bufferevent_get_output(evcon->bufev); /* * Depending if this is a HTTP request or response, we might need to * add some new headers or remove existing headers. */ if (req->kind == EVHTTP_REQUEST) { evhttp_make_header_request(evcon, req); } else { evhttp_make_header_response(evcon, req); } TAILQ_FOREACH(header, req->output_headers, next) { evbuffer_add_printf(output, "%s: %s\r\n", header->key, header->value); } evbuffer_add(output, "\r\n", 2); if (evhttp_have_expect(req, 0) != CONTINUE && evbuffer_get_length(req->output_buffer)) { /* * For a request, we add the POST data, for a reply, this * is the regular data. */ evbuffer_add_buffer(output, req->output_buffer); } } void evhttp_connection_set_max_headers_size(struct evhttp_connection *evcon, ev_ssize_t new_max_headers_size) { if (new_max_headers_size<0) evcon->max_headers_size = EV_SIZE_MAX; else evcon->max_headers_size = new_max_headers_size; } void evhttp_connection_set_max_body_size(struct evhttp_connection* evcon, ev_ssize_t new_max_body_size) { if (new_max_body_size<0) evcon->max_body_size = EV_UINT64_MAX; else evcon->max_body_size = new_max_body_size; } static int evhttp_connection_incoming_fail(struct evhttp_request *req, enum evhttp_request_error error) { switch (error) { case EVREQ_HTTP_DATA_TOO_LONG: req->response_code = HTTP_ENTITYTOOLARGE; break; default: req->response_code = HTTP_BADREQUEST; } switch (error) { case EVREQ_HTTP_TIMEOUT: case EVREQ_HTTP_EOF: /* * these are cases in which we probably should just * close the connection and not send a reply. this * case may happen when a browser keeps a persistent * connection open and we timeout on the read. when * the request is still being used for sending, we * need to disassociated it from the connection here. */ if (!req->userdone) { /* remove it so that it will not be freed */ TAILQ_REMOVE(&req->evcon->requests, req, next); /* indicate that this request no longer has a * connection object */ req->evcon = NULL; } return (-1); case EVREQ_HTTP_INVALID_HEADER: case EVREQ_HTTP_BUFFER_ERROR: case EVREQ_HTTP_REQUEST_CANCEL: case EVREQ_HTTP_DATA_TOO_LONG: default: /* xxx: probably should just error on default */ /* the callback looks at the uri to determine errors */ if (req->uri) { mm_free(req->uri); req->uri = NULL; } if (req->uri_elems) { evhttp_uri_free(req->uri_elems); req->uri_elems = NULL; } /* * the callback needs to send a reply, once the reply has * been send, the connection should get freed. */ (*req->cb)(req, req->cb_arg); } return (0); } /* Free connection ownership of which can be acquired by user using * evhttp_request_own(). */ static inline void evhttp_request_free_auto(struct evhttp_request *req) { if (!(req->flags & EVHTTP_USER_OWNED)) evhttp_request_free(req); } static void evhttp_request_free_(struct evhttp_connection *evcon, struct evhttp_request *req) { TAILQ_REMOVE(&evcon->requests, req, next); evhttp_request_free_auto(req); } /* Called when evcon has experienced a (non-recoverable? -NM) error, as * given in error. If it's an outgoing connection, reset the connection, * retry any pending requests, and inform the user. If it's incoming, * delegates to evhttp_connection_incoming_fail(). */ void evhttp_connection_fail_(struct evhttp_connection *evcon, enum evhttp_request_error error) { const int errsave = EVUTIL_SOCKET_ERROR(); struct evhttp_request* req = TAILQ_FIRST(&evcon->requests); void (*cb)(struct evhttp_request *, void *); void *cb_arg; void (*error_cb)(enum evhttp_request_error, void *); void *error_cb_arg; EVUTIL_ASSERT(req != NULL); bufferevent_disable(evcon->bufev, EV_READ|EV_WRITE); if (evcon->flags & EVHTTP_CON_INCOMING) { /* * for incoming requests, there are two different * failure cases. it's either a network level error * or an http layer error. for problems on the network * layer like timeouts we just drop the connections. * For HTTP problems, we might have to send back a * reply before the connection can be freed. */ if (evhttp_connection_incoming_fail(req, error) == -1) evhttp_connection_free(evcon); return; } error_cb = req->error_cb; error_cb_arg = req->cb_arg; /* when the request was canceled, the callback is not executed */ if (error != EVREQ_HTTP_REQUEST_CANCEL) { /* save the callback for later; the cb might free our object */ cb = req->cb; cb_arg = req->cb_arg; } else { cb = NULL; cb_arg = NULL; } /* do not fail all requests; the next request is going to get * send over a new connection. when a user cancels a request, * all other pending requests should be processed as normal */ evhttp_request_free_(evcon, req); /* reset the connection */ evhttp_connection_reset_(evcon); /* We are trying the next request that was queued on us */ if (TAILQ_FIRST(&evcon->requests) != NULL) evhttp_connection_connect_(evcon); /* The call to evhttp_connection_reset_ overwrote errno. * Let's restore the original errno, so that the user's * callback can have a better idea of what the error was. */ EVUTIL_SET_SOCKET_ERROR(errsave); /* inform the user */ if (error_cb != NULL) error_cb(error, error_cb_arg); if (cb != NULL) (*cb)(NULL, cb_arg); } /* Bufferevent callback: invoked when any data has been written from an * http connection's bufferevent */ static void evhttp_write_cb(struct bufferevent *bufev, void *arg) { struct evhttp_connection *evcon = arg; /* Activate our call back */ if (evcon->cb != NULL) (*evcon->cb)(evcon, evcon->cb_arg); } /** * Advance the connection state. * - If this is an outgoing connection, we've just processed the response; * idle or close the connection. * - If this is an incoming connection, we've just processed the request; * respond. */ static void evhttp_connection_done(struct evhttp_connection *evcon) { struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); int con_outgoing = evcon->flags & EVHTTP_CON_OUTGOING; int free_evcon = 0; if (con_outgoing) { /* idle or close the connection */ int need_close = evhttp_is_request_connection_close(req); TAILQ_REMOVE(&evcon->requests, req, next); req->evcon = NULL; evcon->state = EVCON_IDLE; /* check if we got asked to close the connection */ if (need_close) evhttp_connection_reset_(evcon); if (TAILQ_FIRST(&evcon->requests) != NULL) { /* * We have more requests; reset the connection * and deal with the next request. */ if (!evhttp_connected(evcon)) evhttp_connection_connect_(evcon); else evhttp_request_dispatch(evcon); } else if (!need_close) { /* * The connection is going to be persistent, but we * need to detect if the other side closes it. */ evhttp_connection_start_detectclose(evcon); } else if ((evcon->flags & EVHTTP_CON_AUTOFREE)) { /* * If we have no more requests that need completion * and we're not waiting for the connection to close */ free_evcon = 1; } } else { /* * incoming connection - we need to leave the request on the * connection so that we can reply to it. */ evcon->state = EVCON_WRITING; } /* notify the user of the request */ (*req->cb)(req, req->cb_arg); /* if this was an outgoing request, we own and it's done. so free it. */ if (con_outgoing) { evhttp_request_free_auto(req); } /* If this was the last request of an outgoing connection and we're * not waiting to receive a connection close event and we want to * automatically free the connection. We check to ensure our request * list is empty one last time just in case our callback added a * new request. */ if (free_evcon && TAILQ_FIRST(&evcon->requests) == NULL) { evhttp_connection_free(evcon); } } /* * Handles reading from a chunked request. * return ALL_DATA_READ: * all data has been read * return MORE_DATA_EXPECTED: * more data is expected * return DATA_CORRUPTED: * data is corrupted * return REQUEST_CANCELED: * request was canceled by the user calling evhttp_cancel_request * return DATA_TOO_LONG: * ran over the maximum limit */ static enum message_read_status evhttp_handle_chunked_read(struct evhttp_request *req, struct evbuffer *buf) { if (req == NULL || buf == NULL) { return DATA_CORRUPTED; } while (1) { size_t buflen; if ((buflen = evbuffer_get_length(buf)) == 0) { break; } /* evbuffer_get_length returns size_t, but len variable is ssize_t, * check for overflow conditions */ if (buflen > EV_SSIZE_MAX) { return DATA_CORRUPTED; } if (req->ntoread < 0) { /* Read chunk size */ ev_int64_t ntoread; char *p = evbuffer_readln(buf, NULL, EVBUFFER_EOL_CRLF); char *endp; int error; if (p == NULL) break; /* the last chunk is on a new line? */ if (strlen(p) == 0) { mm_free(p); continue; } ntoread = evutil_strtoll(p, &endp, 16); error = (*p == '\0' || (*endp != '\0' && *endp != ' ') || ntoread < 0); mm_free(p); if (error) { /* could not get chunk size */ return (DATA_CORRUPTED); } /* ntoread is signed int64, body_size is unsigned size_t, check for under/overflow conditions */ if ((ev_uint64_t)ntoread > EV_SIZE_MAX - req->body_size) { return DATA_CORRUPTED; } if (req->body_size + (size_t)ntoread > req->evcon->max_body_size) { /* failed body length test */ event_debug(("Request body is too long")); return (DATA_TOO_LONG); } req->body_size += (size_t)ntoread; req->ntoread = ntoread; if (req->ntoread == 0) { /* Last chunk */ return (ALL_DATA_READ); } continue; } /* req->ntoread is signed int64, len is ssize_t, based on arch, * ssize_t could only be 32b, check for these conditions */ if (req->ntoread > EV_SSIZE_MAX) { return DATA_CORRUPTED; } /* don't have enough to complete a chunk; wait for more */ if (req->ntoread > 0 && buflen < (ev_uint64_t)req->ntoread) return (MORE_DATA_EXPECTED); /* Completed chunk */ evbuffer_remove_buffer(buf, req->input_buffer, (size_t)req->ntoread); req->ntoread = -1; if (req->chunk_cb != NULL) { req->flags |= EVHTTP_REQ_DEFER_FREE; (*req->chunk_cb)(req, req->cb_arg); evbuffer_drain(req->input_buffer, evbuffer_get_length(req->input_buffer)); req->flags &= ~EVHTTP_REQ_DEFER_FREE; if ((req->flags & EVHTTP_REQ_NEEDS_FREE) != 0) { return (REQUEST_CANCELED); } } } return (MORE_DATA_EXPECTED); } static void evhttp_read_trailer(struct evhttp_connection *evcon, struct evhttp_request *req) { struct evbuffer *buf = bufferevent_get_input(evcon->bufev); switch (evhttp_parse_headers_(req, buf)) { case DATA_CORRUPTED: case DATA_TOO_LONG: evhttp_connection_fail_(evcon, EVREQ_HTTP_DATA_TOO_LONG); break; case ALL_DATA_READ: bufferevent_disable(evcon->bufev, EV_READ); evhttp_connection_done(evcon); break; case MORE_DATA_EXPECTED: case REQUEST_CANCELED: /* ??? */ default: break; } } static void evhttp_lingering_close(struct evhttp_connection *evcon, struct evhttp_request *req) { struct evbuffer *buf = bufferevent_get_input(evcon->bufev); size_t n = evbuffer_get_length(buf); if (n > (size_t) req->ntoread) n = (size_t) req->ntoread; req->ntoread -= n; req->body_size += n; event_debug(("Request body is too long, left " EV_I64_FMT, EV_I64_ARG(req->ntoread))); evbuffer_drain(buf, n); if (!req->ntoread) evhttp_connection_fail_(evcon, EVREQ_HTTP_DATA_TOO_LONG); } static void evhttp_lingering_fail(struct evhttp_connection *evcon, struct evhttp_request *req) { if (evcon->flags & EVHTTP_CON_LINGERING_CLOSE) evhttp_lingering_close(evcon, req); else evhttp_connection_fail_(evcon, EVREQ_HTTP_DATA_TOO_LONG); } static void evhttp_read_body(struct evhttp_connection *evcon, struct evhttp_request *req) { struct evbuffer *buf = bufferevent_get_input(evcon->bufev); if (req->chunked) { switch (evhttp_handle_chunked_read(req, buf)) { case ALL_DATA_READ: /* finished last chunk */ evcon->state = EVCON_READING_TRAILER; evhttp_read_trailer(evcon, req); return; case DATA_CORRUPTED: case DATA_TOO_LONG: /* corrupted data */ evhttp_connection_fail_(evcon, EVREQ_HTTP_DATA_TOO_LONG); return; case REQUEST_CANCELED: /* request canceled */ evhttp_request_free_auto(req); return; case MORE_DATA_EXPECTED: default: break; } } else if (req->ntoread < 0) { /* Read until connection close. */ if ((size_t)(req->body_size + evbuffer_get_length(buf)) < req->body_size) { evhttp_connection_fail_(evcon, EVREQ_HTTP_INVALID_HEADER); return; } req->body_size += evbuffer_get_length(buf); evbuffer_add_buffer(req->input_buffer, buf); } else if (req->chunk_cb != NULL || evbuffer_get_length(buf) >= (size_t)req->ntoread) { /* XXX: the above get_length comparison has to be fixed for overflow conditions! */ /* We've postponed moving the data until now, but we're * about to use it. */ size_t n = evbuffer_get_length(buf); if (n > (size_t) req->ntoread) n = (size_t) req->ntoread; req->ntoread -= n; req->body_size += n; evbuffer_remove_buffer(buf, req->input_buffer, n); } if (req->body_size > req->evcon->max_body_size || (!req->chunked && req->ntoread >= 0 && (size_t)req->ntoread > req->evcon->max_body_size)) { /* XXX: The above casted comparison must checked for overflow */ /* failed body length test */ evhttp_lingering_fail(evcon, req); return; } if (evbuffer_get_length(req->input_buffer) > 0 && req->chunk_cb != NULL) { req->flags |= EVHTTP_REQ_DEFER_FREE; (*req->chunk_cb)(req, req->cb_arg); req->flags &= ~EVHTTP_REQ_DEFER_FREE; evbuffer_drain(req->input_buffer, evbuffer_get_length(req->input_buffer)); if ((req->flags & EVHTTP_REQ_NEEDS_FREE) != 0) { evhttp_request_free_auto(req); return; } } if (!req->ntoread) { bufferevent_disable(evcon->bufev, EV_READ); /* Completed content length */ evhttp_connection_done(evcon); return; } } #define get_deferred_queue(evcon) \ ((evcon)->base) /* * Gets called when more data becomes available */ static void evhttp_read_cb(struct bufferevent *bufev, void *arg) { struct evhttp_connection *evcon = arg; struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); /* Cancel if it's pending. */ event_deferred_cb_cancel_(get_deferred_queue(evcon), &evcon->read_more_deferred_cb); switch (evcon->state) { case EVCON_READING_FIRSTLINE: evhttp_read_firstline(evcon, req); /* note the request may have been freed in * evhttp_read_body */ break; case EVCON_READING_HEADERS: evhttp_read_header(evcon, req); /* note the request may have been freed in * evhttp_read_body */ break; case EVCON_READING_BODY: evhttp_read_body(evcon, req); /* note the request may have been freed in * evhttp_read_body */ break; case EVCON_READING_TRAILER: evhttp_read_trailer(evcon, req); break; case EVCON_IDLE: { #ifdef USE_DEBUG struct evbuffer *input; size_t total_len; input = bufferevent_get_input(evcon->bufev); total_len = evbuffer_get_length(input); event_debug(("%s: read "EV_SIZE_FMT " bytes in EVCON_IDLE state," " resetting connection", __func__, EV_SIZE_ARG(total_len))); #endif evhttp_connection_reset_(evcon); } break; case EVCON_DISCONNECTED: case EVCON_CONNECTING: case EVCON_WRITING: default: event_errx(1, "%s: illegal connection state %d", __func__, evcon->state); } } static void evhttp_deferred_read_cb(struct event_callback *cb, void *data) { struct evhttp_connection *evcon = data; evhttp_read_cb(evcon->bufev, evcon); } static void evhttp_write_connectioncb(struct evhttp_connection *evcon, void *arg) { /* This is after writing the request to the server */ struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); struct evbuffer *output = bufferevent_get_output(evcon->bufev); EVUTIL_ASSERT(req != NULL); EVUTIL_ASSERT(evcon->state == EVCON_WRITING); /* We need to wait until we've written all of our output data before we can * continue */ if (evbuffer_get_length(output) > 0) return; /* We are done writing our header and are now expecting the response */ req->kind = EVHTTP_RESPONSE; evhttp_start_read_(evcon); } /* * Clean up a connection object */ void evhttp_connection_free(struct evhttp_connection *evcon) { struct evhttp_request *req; /* notify interested parties that this connection is going down */ if (evcon->fd != -1) { if (evhttp_connected(evcon) && evcon->closecb != NULL) (*evcon->closecb)(evcon, evcon->closecb_arg); } /* remove all requests that might be queued on this * connection. for server connections, this should be empty. * because it gets dequeued either in evhttp_connection_done or * evhttp_connection_fail_. */ while ((req = TAILQ_FIRST(&evcon->requests)) != NULL) { evhttp_request_free_(evcon, req); } if (evcon->http_server != NULL) { struct evhttp *http = evcon->http_server; TAILQ_REMOVE(&http->connections, evcon, next); } if (event_initialized(&evcon->retry_ev)) { event_del(&evcon->retry_ev); event_debug_unassign(&evcon->retry_ev); } if (evcon->bufev != NULL) bufferevent_free(evcon->bufev); event_deferred_cb_cancel_(get_deferred_queue(evcon), &evcon->read_more_deferred_cb); if (evcon->fd == -1) evcon->fd = bufferevent_getfd(evcon->bufev); if (evcon->fd != -1) { bufferevent_disable(evcon->bufev, EV_READ|EV_WRITE); shutdown(evcon->fd, EVUTIL_SHUT_WR); if (!(bufferevent_get_options_(evcon->bufev) & BEV_OPT_CLOSE_ON_FREE)) { evutil_closesocket(evcon->fd); } } if (evcon->bind_address != NULL) mm_free(evcon->bind_address); if (evcon->address != NULL) mm_free(evcon->address); mm_free(evcon); } void evhttp_connection_free_on_completion(struct evhttp_connection *evcon) { evcon->flags |= EVHTTP_CON_AUTOFREE; } void evhttp_connection_set_local_address(struct evhttp_connection *evcon, const char *address) { EVUTIL_ASSERT(evcon->state == EVCON_DISCONNECTED); if (evcon->bind_address) mm_free(evcon->bind_address); if ((evcon->bind_address = mm_strdup(address)) == NULL) event_warn("%s: strdup", __func__); } void evhttp_connection_set_local_port(struct evhttp_connection *evcon, ev_uint16_t port) { EVUTIL_ASSERT(evcon->state == EVCON_DISCONNECTED); evcon->bind_port = port; } static void evhttp_request_dispatch(struct evhttp_connection* evcon) { struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); /* this should not usually happy but it's possible */ if (req == NULL) return; /* delete possible close detection events */ evhttp_connection_stop_detectclose(evcon); /* we assume that the connection is connected already */ EVUTIL_ASSERT(evcon->state == EVCON_IDLE); evcon->state = EVCON_WRITING; /* Create the header from the store arguments */ evhttp_make_header(evcon, req); evhttp_write_buffer(evcon, evhttp_write_connectioncb, NULL); } /* Reset our connection state: disables reading/writing, closes our fd (if * any), clears out buffers, and puts us in state DISCONNECTED. */ void evhttp_connection_reset_(struct evhttp_connection *evcon) { struct evbuffer *tmp; int err; /* XXXX This is not actually an optimal fix. Instead we ought to have an API for "stop connecting", or use bufferevent_setfd to turn off connecting. But for Libevent 2.0, this seems like a minimal change least likely to disrupt the rest of the bufferevent and http code. Why is this here? If the fd is set in the bufferevent, and the bufferevent is connecting, then you can't actually stop the bufferevent from trying to connect with bufferevent_disable(). The connect will never trigger, since we close the fd, but the timeout might. That caused an assertion failure in evhttp_connection_fail_. */ bufferevent_disable_hard_(evcon->bufev, EV_READ|EV_WRITE); if (evcon->fd == -1) evcon->fd = bufferevent_getfd(evcon->bufev); if (evcon->fd != -1) { /* inform interested parties about connection close */ if (evhttp_connected(evcon) && evcon->closecb != NULL) (*evcon->closecb)(evcon, evcon->closecb_arg); shutdown(evcon->fd, EVUTIL_SHUT_WR); evutil_closesocket(evcon->fd); evcon->fd = -1; } bufferevent_setfd(evcon->bufev, -1); /* we need to clean up any buffered data */ tmp = bufferevent_get_output(evcon->bufev); err = evbuffer_drain(tmp, -1); EVUTIL_ASSERT(!err && "drain output"); tmp = bufferevent_get_input(evcon->bufev); err = evbuffer_drain(tmp, -1); EVUTIL_ASSERT(!err && "drain input"); evcon->flags &= ~EVHTTP_CON_READING_ERROR; evcon->state = EVCON_DISCONNECTED; } static void evhttp_connection_start_detectclose(struct evhttp_connection *evcon) { evcon->flags |= EVHTTP_CON_CLOSEDETECT; bufferevent_enable(evcon->bufev, EV_READ); } static void evhttp_connection_stop_detectclose(struct evhttp_connection *evcon) { evcon->flags &= ~EVHTTP_CON_CLOSEDETECT; bufferevent_disable(evcon->bufev, EV_READ); } static void evhttp_connection_retry(evutil_socket_t fd, short what, void *arg) { struct evhttp_connection *evcon = arg; evcon->state = EVCON_DISCONNECTED; evhttp_connection_connect_(evcon); } static void evhttp_connection_cb_cleanup(struct evhttp_connection *evcon) { struct evcon_requestq requests; evhttp_connection_reset_(evcon); if (evcon->retry_max < 0 || evcon->retry_cnt < evcon->retry_max) { struct timeval tv_retry = evcon->initial_retry_timeout; int i; evtimer_assign(&evcon->retry_ev, evcon->base, evhttp_connection_retry, evcon); /* XXXX handle failure from evhttp_add_event */ for (i=0; i < evcon->retry_cnt; ++i) { tv_retry.tv_usec *= 2; if (tv_retry.tv_usec > 1000000) { tv_retry.tv_usec -= 1000000; tv_retry.tv_sec += 1; } tv_retry.tv_sec *= 2; if (tv_retry.tv_sec > 3600) { tv_retry.tv_sec = 3600; tv_retry.tv_usec = 0; } } event_add(&evcon->retry_ev, &tv_retry); evcon->retry_cnt++; return; } /* * User callback can do evhttp_make_request() on the same * evcon so new request will be added to evcon->requests. To * avoid freeing it prematurely we iterate over the copy of * the queue. */ TAILQ_INIT(&requests); while (TAILQ_FIRST(&evcon->requests) != NULL) { struct evhttp_request *request = TAILQ_FIRST(&evcon->requests); TAILQ_REMOVE(&evcon->requests, request, next); TAILQ_INSERT_TAIL(&requests, request, next); } /* for now, we just signal all requests by executing their callbacks */ while (TAILQ_FIRST(&requests) != NULL) { struct evhttp_request *request = TAILQ_FIRST(&requests); TAILQ_REMOVE(&requests, request, next); request->evcon = NULL; /* we might want to set an error here */ request->cb(request, request->cb_arg); evhttp_request_free_auto(request); } } static void evhttp_connection_read_on_write_error(struct evhttp_connection *evcon, struct evhttp_request *req) { struct evbuffer *buf; /** Second time, we can't read anything */ if (evcon->flags & EVHTTP_CON_READING_ERROR) { evcon->flags &= ~EVHTTP_CON_READING_ERROR; evhttp_connection_fail_(evcon, EVREQ_HTTP_EOF); return; } req->kind = EVHTTP_RESPONSE; buf = bufferevent_get_output(evcon->bufev); evbuffer_unfreeze(buf, 1); evbuffer_drain(buf, evbuffer_get_length(buf)); evbuffer_freeze(buf, 1); evhttp_start_read_(evcon); evcon->flags |= EVHTTP_CON_READING_ERROR; } static void evhttp_error_cb(struct bufferevent *bufev, short what, void *arg) { struct evhttp_connection *evcon = arg; struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); if (evcon->fd == -1) evcon->fd = bufferevent_getfd(bufev); switch (evcon->state) { case EVCON_CONNECTING: if (what & BEV_EVENT_TIMEOUT) { event_debug(("%s: connection timeout for \"%s:%d\" on " EV_SOCK_FMT, __func__, evcon->address, evcon->port, EV_SOCK_ARG(evcon->fd))); evhttp_connection_cb_cleanup(evcon); return; } break; case EVCON_READING_BODY: if (!req->chunked && req->ntoread < 0 && what == (BEV_EVENT_READING|BEV_EVENT_EOF)) { /* EOF on read can be benign */ evhttp_connection_done(evcon); return; } break; case EVCON_DISCONNECTED: case EVCON_IDLE: case EVCON_READING_FIRSTLINE: case EVCON_READING_HEADERS: case EVCON_READING_TRAILER: case EVCON_WRITING: default: break; } /* when we are in close detect mode, a read error means that * the other side closed their connection. */ if (evcon->flags & EVHTTP_CON_CLOSEDETECT) { evcon->flags &= ~EVHTTP_CON_CLOSEDETECT; EVUTIL_ASSERT(evcon->http_server == NULL); /* For connections from the client, we just * reset the connection so that it becomes * disconnected. */ EVUTIL_ASSERT(evcon->state == EVCON_IDLE); evhttp_connection_reset_(evcon); /* * If we have no more requests that need completion * and we want to auto-free the connection when all * requests have been completed. */ if (TAILQ_FIRST(&evcon->requests) == NULL && (evcon->flags & EVHTTP_CON_OUTGOING) && (evcon->flags & EVHTTP_CON_AUTOFREE)) { evhttp_connection_free(evcon); } return; } if (what & BEV_EVENT_TIMEOUT) { evhttp_connection_fail_(evcon, EVREQ_HTTP_TIMEOUT); } else if (what & (BEV_EVENT_EOF|BEV_EVENT_ERROR)) { if (what & BEV_EVENT_WRITING && evcon->flags & EVHTTP_CON_READ_ON_WRITE_ERROR) { evhttp_connection_read_on_write_error(evcon, req); return; } evhttp_connection_fail_(evcon, EVREQ_HTTP_EOF); } else if (what == BEV_EVENT_CONNECTED) { } else { evhttp_connection_fail_(evcon, EVREQ_HTTP_BUFFER_ERROR); } } /* * Event callback for asynchronous connection attempt. */ static void evhttp_connection_cb(struct bufferevent *bufev, short what, void *arg) { struct evhttp_connection *evcon = arg; int error; ev_socklen_t errsz = sizeof(error); if (evcon->fd == -1) evcon->fd = bufferevent_getfd(bufev); if (!(what & BEV_EVENT_CONNECTED)) { /* some operating systems return ECONNREFUSED immediately * when connecting to a local address. the cleanup is going * to reschedule this function call. */ #ifndef _WIN32 if (errno == ECONNREFUSED) goto cleanup; #endif evhttp_error_cb(bufev, what, arg); return; } if (evcon->fd == -1) { event_debug(("%s: bufferevent_getfd returned -1", __func__)); goto cleanup; } /* Check if the connection completed */ if (getsockopt(evcon->fd, SOL_SOCKET, SO_ERROR, (void*)&error, &errsz) == -1) { event_debug(("%s: getsockopt for \"%s:%d\" on "EV_SOCK_FMT, __func__, evcon->address, evcon->port, EV_SOCK_ARG(evcon->fd))); goto cleanup; } if (error) { event_debug(("%s: connect failed for \"%s:%d\" on " EV_SOCK_FMT": %s", __func__, evcon->address, evcon->port, EV_SOCK_ARG(evcon->fd), evutil_socket_error_to_string(error))); goto cleanup; } /* We are connected to the server now */ event_debug(("%s: connected to \"%s:%d\" on "EV_SOCK_FMT"\n", __func__, evcon->address, evcon->port, EV_SOCK_ARG(evcon->fd))); /* Reset the retry count as we were successful in connecting */ evcon->retry_cnt = 0; evcon->state = EVCON_IDLE; /* reset the bufferevent cbs */ bufferevent_setcb(evcon->bufev, evhttp_read_cb, evhttp_write_cb, evhttp_error_cb, evcon); if (!evutil_timerisset(&evcon->timeout)) { const struct timeval read_tv = { HTTP_READ_TIMEOUT, 0 }; const struct timeval write_tv = { HTTP_WRITE_TIMEOUT, 0 }; bufferevent_set_timeouts(evcon->bufev, &read_tv, &write_tv); } else { bufferevent_set_timeouts(evcon->bufev, &evcon->timeout, &evcon->timeout); } /* try to start requests that have queued up on this connection */ evhttp_request_dispatch(evcon); return; cleanup: evhttp_connection_cb_cleanup(evcon); } /* * Check if we got a valid response code. */ static int evhttp_valid_response_code(int code) { if (code == 0) return (0); return (1); } static int evhttp_parse_http_version(const char *version, struct evhttp_request *req) { int major, minor; char ch; int n = sscanf(version, "HTTP/%d.%d%c", &major, &minor, &ch); if (n != 2 || major > 1) { event_debug(("%s: bad version %s on message %p from %s", __func__, version, req, req->remote_host)); return (-1); } req->major = major; req->minor = minor; return (0); } /* Parses the status line of a web server */ static int evhttp_parse_response_line(struct evhttp_request *req, char *line) { char *protocol; char *number; const char *readable = ""; protocol = strsep(&line, " "); if (line == NULL) return (-1); number = strsep(&line, " "); if (line != NULL) readable = line; if (evhttp_parse_http_version(protocol, req) < 0) return (-1); req->response_code = atoi(number); if (!evhttp_valid_response_code(req->response_code)) { event_debug(("%s: bad response code \"%s\"", __func__, number)); return (-1); } if (req->response_code_line != NULL) mm_free(req->response_code_line); if ((req->response_code_line = mm_strdup(readable)) == NULL) { event_warn("%s: strdup", __func__); return (-1); } return (0); } /* Parse the first line of a HTTP request */ static int evhttp_parse_request_line(struct evhttp_request *req, char *line) { char *method; char *uri; char *version; const char *hostname; const char *scheme; size_t method_len; enum evhttp_cmd_type type; /* Parse the request line */ method = strsep(&line, " "); if (line == NULL) return (-1); uri = strsep(&line, " "); if (line == NULL) return (-1); version = strsep(&line, " "); if (line != NULL) return (-1); method_len = (uri - method) - 1; type = EVHTTP_REQ_UNKNOWN_; /* First line */ switch (method_len) { case 3: /* The length of the method string is 3, meaning it can only be one of two methods: GET or PUT */ /* Since both GET and PUT share the same character 'T' at the end, * if the string doesn't have 'T', we can immediately determine this * is an invalid HTTP method */ if (method[2] != 'T') { break; } switch (*method) { case 'G': /* This first byte is 'G', so make sure the next byte is * 'E', if it isn't then this isn't a valid method */ if (method[1] == 'E') { type = EVHTTP_REQ_GET; } break; case 'P': /* First byte is P, check second byte for 'U', if not, * we know it's an invalid method */ if (method[1] == 'U') { type = EVHTTP_REQ_PUT; } break; default: break; } break; case 4: /* The method length is 4 bytes, leaving only the methods "POST" and "HEAD" */ switch (*method) { case 'P': if (method[3] == 'T' && method[2] == 'S' && method[1] == 'O') { type = EVHTTP_REQ_POST; } break; case 'H': if (method[3] == 'D' && method[2] == 'A' && method[1] == 'E') { type = EVHTTP_REQ_HEAD; } break; default: break; } break; case 5: /* Method length is 5 bytes, which can only encompass PATCH and TRACE */ switch (*method) { case 'P': if (method[4] == 'H' && method[3] == 'C' && method[2] == 'T' && method[1] == 'A') { type = EVHTTP_REQ_PATCH; } break; case 'T': if (method[4] == 'E' && method[3] == 'C' && method[2] == 'A' && method[1] == 'R') { type = EVHTTP_REQ_TRACE; } break; default: break; } break; case 6: /* Method length is 6, only valid method 6 bytes in length is DELEte */ /* If the first byte isn't 'D' then it's invalid */ if (*method != 'D') { break; } if (method[5] == 'E' && method[4] == 'T' && method[3] == 'E' && method[2] == 'L' && method[1] == 'E') { type = EVHTTP_REQ_DELETE; } break; case 7: /* Method length is 7, only valid methods are "OPTIONS" and "CONNECT" */ switch (*method) { case 'O': if (method[6] == 'S' && method[5] == 'N' && method[4] == 'O' && method[3] == 'I' && method[2] == 'T' && method[1] == 'P') { type = EVHTTP_REQ_OPTIONS; } break; case 'C': if (method[6] == 'T' && method[5] == 'C' && method[4] == 'E' && method[3] == 'N' && method[2] == 'N' && method[1] == 'O') { type = EVHTTP_REQ_CONNECT; } break; default: break; } break; } /* switch */ if ((int)type == EVHTTP_REQ_UNKNOWN_) { event_debug(("%s: bad method %s on request %p from %s", __func__, method, req, req->remote_host)); /* No error yet; we'll give a better error later when * we see that req->type is unsupported. */ } req->type = type; if (evhttp_parse_http_version(version, req) < 0) return (-1); if ((req->uri = mm_strdup(uri)) == NULL) { event_debug(("%s: mm_strdup", __func__)); return (-1); } if ((req->uri_elems = evhttp_uri_parse_with_flags(req->uri, EVHTTP_URI_NONCONFORMANT)) == NULL) { return -1; } /* If we have an absolute-URI, check to see if it is an http request for a known vhost or server alias. If we don't know about this host, we consider it a proxy request. */ scheme = evhttp_uri_get_scheme(req->uri_elems); hostname = evhttp_uri_get_host(req->uri_elems); if (scheme && (!evutil_ascii_strcasecmp(scheme, "http") || !evutil_ascii_strcasecmp(scheme, "https")) && hostname && !evhttp_find_vhost(req->evcon->http_server, NULL, hostname)) req->flags |= EVHTTP_PROXY_REQUEST; return (0); } const char * evhttp_find_header(const struct evkeyvalq *headers, const char *key) { struct evkeyval *header; TAILQ_FOREACH(header, headers, next) { if (evutil_ascii_strcasecmp(header->key, key) == 0) return (header->value); } return (NULL); } void evhttp_clear_headers(struct evkeyvalq *headers) { struct evkeyval *header; for (header = TAILQ_FIRST(headers); header != NULL; header = TAILQ_FIRST(headers)) { TAILQ_REMOVE(headers, header, next); mm_free(header->key); mm_free(header->value); mm_free(header); } } /* * Returns 0, if the header was successfully removed. * Returns -1, if the header could not be found. */ int evhttp_remove_header(struct evkeyvalq *headers, const char *key) { struct evkeyval *header; TAILQ_FOREACH(header, headers, next) { if (evutil_ascii_strcasecmp(header->key, key) == 0) break; } if (header == NULL) return (-1); /* Free and remove the header that we found */ TAILQ_REMOVE(headers, header, next); mm_free(header->key); mm_free(header->value); mm_free(header); return (0); } static int evhttp_header_is_valid_value(const char *value) { const char *p = value; while ((p = strpbrk(p, "\r\n")) != NULL) { /* we really expect only one new line */ p += strspn(p, "\r\n"); /* we expect a space or tab for continuation */ if (*p != ' ' && *p != '\t') return (0); } return (1); } int evhttp_add_header(struct evkeyvalq *headers, const char *key, const char *value) { event_debug(("%s: key: %s val: %s\n", __func__, key, value)); if (strchr(key, '\r') != NULL || strchr(key, '\n') != NULL) { /* drop illegal headers */ event_debug(("%s: dropping illegal header key\n", __func__)); return (-1); } if (!evhttp_header_is_valid_value(value)) { event_debug(("%s: dropping illegal header value\n", __func__)); return (-1); } return (evhttp_add_header_internal(headers, key, value)); } static int evhttp_add_header_internal(struct evkeyvalq *headers, const char *key, const char *value) { struct evkeyval *header = mm_calloc(1, sizeof(struct evkeyval)); if (header == NULL) { event_warn("%s: calloc", __func__); return (-1); } if ((header->key = mm_strdup(key)) == NULL) { mm_free(header); event_warn("%s: strdup", __func__); return (-1); } if ((header->value = mm_strdup(value)) == NULL) { mm_free(header->key); mm_free(header); event_warn("%s: strdup", __func__); return (-1); } TAILQ_INSERT_TAIL(headers, header, next); return (0); } /* * Parses header lines from a request or a response into the specified * request object given an event buffer. * * Returns * DATA_CORRUPTED on error * MORE_DATA_EXPECTED when we need to read more headers * ALL_DATA_READ when all headers have been read. */ enum message_read_status evhttp_parse_firstline_(struct evhttp_request *req, struct evbuffer *buffer) { char *line; enum message_read_status status = ALL_DATA_READ; size_t line_length; /* XXX try */ line = evbuffer_readln(buffer, &line_length, EVBUFFER_EOL_CRLF); if (line == NULL) { if (req->evcon != NULL && evbuffer_get_length(buffer) > req->evcon->max_headers_size) return (DATA_TOO_LONG); else return (MORE_DATA_EXPECTED); } if (req->evcon != NULL && line_length > req->evcon->max_headers_size) { mm_free(line); return (DATA_TOO_LONG); } req->headers_size = line_length; switch (req->kind) { case EVHTTP_REQUEST: if (evhttp_parse_request_line(req, line) == -1) status = DATA_CORRUPTED; break; case EVHTTP_RESPONSE: if (evhttp_parse_response_line(req, line) == -1) status = DATA_CORRUPTED; break; default: status = DATA_CORRUPTED; } mm_free(line); return (status); } static int evhttp_append_to_last_header(struct evkeyvalq *headers, char *line) { struct evkeyval *header = TAILQ_LAST(headers, evkeyvalq); char *newval; size_t old_len, line_len; if (header == NULL) return (-1); old_len = strlen(header->value); /* Strip space from start and end of line. */ while (*line == ' ' || *line == '\t') ++line; evutil_rtrim_lws_(line); line_len = strlen(line); newval = mm_realloc(header->value, old_len + line_len + 2); if (newval == NULL) return (-1); newval[old_len] = ' '; memcpy(newval + old_len + 1, line, line_len + 1); header->value = newval; return (0); } enum message_read_status evhttp_parse_headers_(struct evhttp_request *req, struct evbuffer* buffer) { enum message_read_status errcode = DATA_CORRUPTED; char *line; enum message_read_status status = MORE_DATA_EXPECTED; struct evkeyvalq* headers = req->input_headers; size_t line_length; while ((line = evbuffer_readln(buffer, &line_length, EVBUFFER_EOL_CRLF)) != NULL) { char *skey, *svalue; req->headers_size += line_length; if (req->evcon != NULL && req->headers_size > req->evcon->max_headers_size) { errcode = DATA_TOO_LONG; goto error; } if (*line == '\0') { /* Last header - Done */ status = ALL_DATA_READ; mm_free(line); break; } /* Check if this is a continuation line */ if (*line == ' ' || *line == '\t') { if (evhttp_append_to_last_header(headers, line) == -1) goto error; mm_free(line); continue; } /* Processing of header lines */ svalue = line; skey = strsep(&svalue, ":"); if (svalue == NULL) goto error; svalue += strspn(svalue, " "); evutil_rtrim_lws_(svalue); if (evhttp_add_header(headers, skey, svalue) == -1) goto error; mm_free(line); } if (status == MORE_DATA_EXPECTED) { if (req->evcon != NULL && req->headers_size + evbuffer_get_length(buffer) > req->evcon->max_headers_size) return (DATA_TOO_LONG); } return (status); error: mm_free(line); return (errcode); } static int evhttp_get_body_length(struct evhttp_request *req) { struct evkeyvalq *headers = req->input_headers; const char *content_length; const char *connection; content_length = evhttp_find_header(headers, "Content-Length"); connection = evhttp_find_header(headers, "Connection"); if (content_length == NULL && connection == NULL) req->ntoread = -1; else if (content_length == NULL && evutil_ascii_strcasecmp(connection, "Close") != 0) { /* Bad combination, we don't know when it will end */ event_warnx("%s: we got no content length, but the " "server wants to keep the connection open: %s.", __func__, connection); return (-1); } else if (content_length == NULL) { req->ntoread = -1; } else { char *endp; ev_int64_t ntoread = evutil_strtoll(content_length, &endp, 10); if (*content_length == '\0' || *endp != '\0' || ntoread < 0) { event_debug(("%s: illegal content length: %s", __func__, content_length)); return (-1); } req->ntoread = ntoread; } event_debug(("%s: bytes to read: "EV_I64_FMT" (in buffer "EV_SIZE_FMT")\n", __func__, EV_I64_ARG(req->ntoread), EV_SIZE_ARG(evbuffer_get_length(bufferevent_get_input(req->evcon->bufev))))); return (0); } static int evhttp_method_may_have_body(enum evhttp_cmd_type type) { switch (type) { case EVHTTP_REQ_POST: case EVHTTP_REQ_PUT: case EVHTTP_REQ_PATCH: return 1; case EVHTTP_REQ_TRACE: return 0; /* XXX May any of the below methods have a body? */ case EVHTTP_REQ_GET: case EVHTTP_REQ_HEAD: case EVHTTP_REQ_DELETE: case EVHTTP_REQ_OPTIONS: case EVHTTP_REQ_CONNECT: return 0; default: return 0; } } static void evhttp_get_body(struct evhttp_connection *evcon, struct evhttp_request *req) { const char *xfer_enc; /* If this is a request without a body, then we are done */ if (req->kind == EVHTTP_REQUEST && !evhttp_method_may_have_body(req->type)) { evhttp_connection_done(evcon); return; } evcon->state = EVCON_READING_BODY; xfer_enc = evhttp_find_header(req->input_headers, "Transfer-Encoding"); if (xfer_enc != NULL && evutil_ascii_strcasecmp(xfer_enc, "chunked") == 0) { req->chunked = 1; req->ntoread = -1; } else { if (evhttp_get_body_length(req) == -1) { evhttp_connection_fail_(evcon, EVREQ_HTTP_INVALID_HEADER); return; } if (req->kind == EVHTTP_REQUEST && req->ntoread < 1) { /* An incoming request with no content-length and no * transfer-encoding has no body. */ evhttp_connection_done(evcon); return; } } /* Should we send a 100 Continue status line? */ switch (evhttp_have_expect(req, 1)) { case CONTINUE: /* XXX It would be nice to do some sanity checking here. Does the resource exist? Should the resource accept post requests? If no, we should respond with an error. For now, just optimistically tell the client to send their message body. */ if (req->ntoread > 0) { /* ntoread is ev_int64_t, max_body_size is ev_uint64_t */ if ((req->evcon->max_body_size <= EV_INT64_MAX) && (ev_uint64_t)req->ntoread > req->evcon->max_body_size) { evhttp_lingering_fail(evcon, req); return; } } if (!evbuffer_get_length(bufferevent_get_input(evcon->bufev))) evhttp_send_continue(evcon, req); break; case OTHER: evhttp_send_error(req, HTTP_EXPECTATIONFAILED, NULL); return; case NO: break; } evhttp_read_body(evcon, req); /* note the request may have been freed in evhttp_read_body */ } static void evhttp_read_firstline(struct evhttp_connection *evcon, struct evhttp_request *req) { enum message_read_status res; res = evhttp_parse_firstline_(req, bufferevent_get_input(evcon->bufev)); if (res == DATA_CORRUPTED || res == DATA_TOO_LONG) { /* Error while reading, terminate */ event_debug(("%s: bad header lines on "EV_SOCK_FMT"\n", __func__, EV_SOCK_ARG(evcon->fd))); evhttp_connection_fail_(evcon, EVREQ_HTTP_INVALID_HEADER); return; } else if (res == MORE_DATA_EXPECTED) { /* Need more header lines */ return; } evcon->state = EVCON_READING_HEADERS; evhttp_read_header(evcon, req); } static void evhttp_read_header(struct evhttp_connection *evcon, struct evhttp_request *req) { enum message_read_status res; evutil_socket_t fd = evcon->fd; res = evhttp_parse_headers_(req, bufferevent_get_input(evcon->bufev)); if (res == DATA_CORRUPTED || res == DATA_TOO_LONG) { /* Error while reading, terminate */ event_debug(("%s: bad header lines on "EV_SOCK_FMT"\n", __func__, EV_SOCK_ARG(fd))); evhttp_connection_fail_(evcon, EVREQ_HTTP_INVALID_HEADER); return; } else if (res == MORE_DATA_EXPECTED) { /* Need more header lines */ return; } /* Callback can shut down connection with negative return value */ if (req->header_cb != NULL) { if ((*req->header_cb)(req, req->cb_arg) < 0) { evhttp_connection_fail_(evcon, EVREQ_HTTP_EOF); return; } } /* Done reading headers, do the real work */ switch (req->kind) { case EVHTTP_REQUEST: event_debug(("%s: checking for post data on "EV_SOCK_FMT"\n", __func__, EV_SOCK_ARG(fd))); evhttp_get_body(evcon, req); /* note the request may have been freed in evhttp_get_body */ break; case EVHTTP_RESPONSE: /* Start over if we got a 100 Continue response. */ if (req->response_code == 100) { struct evbuffer *output = bufferevent_get_output(evcon->bufev); evbuffer_add_buffer(output, req->output_buffer); evhttp_start_write_(evcon); return; } if (!evhttp_response_needs_body(req)) { event_debug(("%s: skipping body for code %d\n", __func__, req->response_code)); evhttp_connection_done(evcon); } else { event_debug(("%s: start of read body for %s on " EV_SOCK_FMT"\n", __func__, req->remote_host, EV_SOCK_ARG(fd))); evhttp_get_body(evcon, req); /* note the request may have been freed in * evhttp_get_body */ } break; default: event_warnx("%s: bad header on "EV_SOCK_FMT, __func__, EV_SOCK_ARG(fd)); evhttp_connection_fail_(evcon, EVREQ_HTTP_INVALID_HEADER); break; } /* request may have been freed above */ } /* * Creates a TCP connection to the specified port and executes a callback * when finished. Failure or success is indicate by the passed connection * object. * * Although this interface accepts a hostname, it is intended to take * only numeric hostnames so that non-blocking DNS resolution can * happen elsewhere. */ struct evhttp_connection * evhttp_connection_new(const char *address, ev_uint16_t port) { return (evhttp_connection_base_new(NULL, NULL, address, port)); } struct evhttp_connection * evhttp_connection_base_bufferevent_new(struct event_base *base, struct evdns_base *dnsbase, struct bufferevent* bev, const char *address, ev_uint16_t port) { struct evhttp_connection *evcon = NULL; event_debug(("Attempting connection to %s:%d\n", address, port)); if ((evcon = mm_calloc(1, sizeof(struct evhttp_connection))) == NULL) { event_warn("%s: calloc failed", __func__); goto error; } evcon->fd = -1; evcon->port = port; evcon->max_headers_size = EV_SIZE_MAX; evcon->max_body_size = EV_SIZE_MAX; evutil_timerclear(&evcon->timeout); evcon->retry_cnt = evcon->retry_max = 0; if ((evcon->address = mm_strdup(address)) == NULL) { event_warn("%s: strdup failed", __func__); goto error; } if (bev == NULL) { if (!(bev = bufferevent_socket_new(base, -1, 0))) { event_warn("%s: bufferevent_socket_new failed", __func__); goto error; } } bufferevent_setcb(bev, evhttp_read_cb, evhttp_write_cb, evhttp_error_cb, evcon); evcon->bufev = bev; evcon->state = EVCON_DISCONNECTED; TAILQ_INIT(&evcon->requests); evcon->initial_retry_timeout.tv_sec = 2; evcon->initial_retry_timeout.tv_usec = 0; if (base != NULL) { evcon->base = base; if (bufferevent_get_base(bev) != base) bufferevent_base_set(base, evcon->bufev); } event_deferred_cb_init_( &evcon->read_more_deferred_cb, bufferevent_get_priority(bev), evhttp_deferred_read_cb, evcon); evcon->dns_base = dnsbase; evcon->ai_family = AF_UNSPEC; return (evcon); error: if (evcon != NULL) evhttp_connection_free(evcon); return (NULL); } struct bufferevent* evhttp_connection_get_bufferevent(struct evhttp_connection *evcon) { return evcon->bufev; } struct evhttp * evhttp_connection_get_server(struct evhttp_connection *evcon) { return evcon->http_server; } struct evhttp_connection * evhttp_connection_base_new(struct event_base *base, struct evdns_base *dnsbase, const char *address, ev_uint16_t port) { return evhttp_connection_base_bufferevent_new(base, dnsbase, NULL, address, port); } void evhttp_connection_set_family(struct evhttp_connection *evcon, int family) { evcon->ai_family = family; } int evhttp_connection_set_flags(struct evhttp_connection *evcon, int flags) { int avail_flags = 0; avail_flags |= EVHTTP_CON_REUSE_CONNECTED_ADDR; avail_flags |= EVHTTP_CON_READ_ON_WRITE_ERROR; if (flags & ~avail_flags || flags > EVHTTP_CON_PUBLIC_FLAGS_END) return 1; evcon->flags &= ~avail_flags; evcon->flags |= flags; return 0; } void evhttp_connection_set_base(struct evhttp_connection *evcon, struct event_base *base) { EVUTIL_ASSERT(evcon->base == NULL); EVUTIL_ASSERT(evcon->state == EVCON_DISCONNECTED); evcon->base = base; bufferevent_base_set(base, evcon->bufev); } void evhttp_connection_set_timeout(struct evhttp_connection *evcon, int timeout_in_secs) { if (timeout_in_secs == -1) evhttp_connection_set_timeout_tv(evcon, NULL); else { struct timeval tv; tv.tv_sec = timeout_in_secs; tv.tv_usec = 0; evhttp_connection_set_timeout_tv(evcon, &tv); } } void evhttp_connection_set_timeout_tv(struct evhttp_connection *evcon, const struct timeval* tv) { if (tv) { evcon->timeout = *tv; bufferevent_set_timeouts(evcon->bufev, &evcon->timeout, &evcon->timeout); } else { const struct timeval read_tv = { HTTP_READ_TIMEOUT, 0 }; const struct timeval write_tv = { HTTP_WRITE_TIMEOUT, 0 }; evutil_timerclear(&evcon->timeout); bufferevent_set_timeouts(evcon->bufev, &read_tv, &write_tv); } } void evhttp_connection_set_initial_retry_tv(struct evhttp_connection *evcon, const struct timeval *tv) { if (tv) { evcon->initial_retry_timeout = *tv; } else { evutil_timerclear(&evcon->initial_retry_timeout); evcon->initial_retry_timeout.tv_sec = 2; } } void evhttp_connection_set_retries(struct evhttp_connection *evcon, int retry_max) { evcon->retry_max = retry_max; } void evhttp_connection_set_closecb(struct evhttp_connection *evcon, void (*cb)(struct evhttp_connection *, void *), void *cbarg) { evcon->closecb = cb; evcon->closecb_arg = cbarg; } void evhttp_connection_get_peer(struct evhttp_connection *evcon, char **address, ev_uint16_t *port) { *address = evcon->address; *port = evcon->port; } const struct sockaddr* evhttp_connection_get_addr(struct evhttp_connection *evcon) { return bufferevent_socket_get_conn_address_(evcon->bufev); } int evhttp_connection_connect_(struct evhttp_connection *evcon) { int old_state = evcon->state; const char *address = evcon->address; const struct sockaddr *sa = evhttp_connection_get_addr(evcon); int ret; if (evcon->state == EVCON_CONNECTING) return (0); evhttp_connection_reset_(evcon); EVUTIL_ASSERT(!(evcon->flags & EVHTTP_CON_INCOMING)); evcon->flags |= EVHTTP_CON_OUTGOING; if (evcon->bind_address || evcon->bind_port) { evcon->fd = bind_socket( evcon->bind_address, evcon->bind_port, 0 /*reuse*/); if (evcon->fd == -1) { event_debug(("%s: failed to bind to \"%s\"", __func__, evcon->bind_address)); return (-1); } bufferevent_setfd(evcon->bufev, evcon->fd); } else { bufferevent_setfd(evcon->bufev, -1); } /* Set up a callback for successful connection setup */ bufferevent_setcb(evcon->bufev, NULL /* evhttp_read_cb */, NULL /* evhttp_write_cb */, evhttp_connection_cb, evcon); if (!evutil_timerisset(&evcon->timeout)) { const struct timeval conn_tv = { HTTP_CONNECT_TIMEOUT, 0 }; bufferevent_set_timeouts(evcon->bufev, &conn_tv, &conn_tv); } else { bufferevent_set_timeouts(evcon->bufev, &evcon->timeout, &evcon->timeout); } /* make sure that we get a write callback */ bufferevent_enable(evcon->bufev, EV_WRITE); evcon->state = EVCON_CONNECTING; if (evcon->flags & EVHTTP_CON_REUSE_CONNECTED_ADDR && sa && (sa->sa_family == AF_INET || sa->sa_family == AF_INET6)) { int socklen = sizeof(struct sockaddr_in); if (sa->sa_family == AF_INET6) { socklen = sizeof(struct sockaddr_in6); } ret = bufferevent_socket_connect(evcon->bufev, sa, socklen); } else { ret = bufferevent_socket_connect_hostname(evcon->bufev, evcon->dns_base, evcon->ai_family, address, evcon->port); } if (ret < 0) { evcon->state = old_state; event_sock_warn(evcon->fd, "%s: connection to \"%s\" failed", __func__, evcon->address); /* some operating systems return ECONNREFUSED immediately * when connecting to a local address. the cleanup is going * to reschedule this function call. */ evhttp_connection_cb_cleanup(evcon); return (0); } return (0); } /* * Starts an HTTP request on the provided evhttp_connection object. * If the connection object is not connected to the web server already, * this will start the connection. */ int evhttp_make_request(struct evhttp_connection *evcon, struct evhttp_request *req, enum evhttp_cmd_type type, const char *uri) { /* We are making a request */ req->kind = EVHTTP_REQUEST; req->type = type; if (req->uri != NULL) mm_free(req->uri); if ((req->uri = mm_strdup(uri)) == NULL) { event_warn("%s: strdup", __func__); evhttp_request_free_auto(req); return (-1); } /* Set the protocol version if it is not supplied */ if (!req->major && !req->minor) { req->major = 1; req->minor = 1; } EVUTIL_ASSERT(req->evcon == NULL); req->evcon = evcon; EVUTIL_ASSERT(!(req->flags & EVHTTP_REQ_OWN_CONNECTION)); TAILQ_INSERT_TAIL(&evcon->requests, req, next); /* If the connection object is not connected; make it so */ if (!evhttp_connected(evcon)) { int res = evhttp_connection_connect_(evcon); /* evhttp_connection_fail_(), which is called through * evhttp_connection_connect_(), assumes that req lies in * evcon->requests. Thus, enqueue the request in advance and * remove it in the error case. */ if (res != 0) TAILQ_REMOVE(&evcon->requests, req, next); return res; } /* * If it's connected already and we are the first in the queue, * then we can dispatch this request immediately. Otherwise, it * will be dispatched once the pending requests are completed. */ if (TAILQ_FIRST(&evcon->requests) == req) evhttp_request_dispatch(evcon); return (0); } void evhttp_cancel_request(struct evhttp_request *req) { struct evhttp_connection *evcon = req->evcon; if (evcon != NULL) { /* We need to remove it from the connection */ if (TAILQ_FIRST(&evcon->requests) == req) { /* it's currently being worked on, so reset * the connection. */ evhttp_connection_fail_(evcon, EVREQ_HTTP_REQUEST_CANCEL); /* connection fail freed the request */ return; } else { /* otherwise, we can just remove it from the * queue */ TAILQ_REMOVE(&evcon->requests, req, next); } } evhttp_request_free_auto(req); } /* * Reads data from file descriptor into request structure * Request structure needs to be set up correctly. */ void evhttp_start_read_(struct evhttp_connection *evcon) { bufferevent_disable(evcon->bufev, EV_WRITE); bufferevent_enable(evcon->bufev, EV_READ); evcon->state = EVCON_READING_FIRSTLINE; /* Reset the bufferevent callbacks */ bufferevent_setcb(evcon->bufev, evhttp_read_cb, evhttp_write_cb, evhttp_error_cb, evcon); /* If there's still data pending, process it next time through the * loop. Don't do it now; that could get recusive. */ if (evbuffer_get_length(bufferevent_get_input(evcon->bufev))) { event_deferred_cb_schedule_(get_deferred_queue(evcon), &evcon->read_more_deferred_cb); } } void evhttp_start_write_(struct evhttp_connection *evcon) { bufferevent_disable(evcon->bufev, EV_WRITE); bufferevent_enable(evcon->bufev, EV_READ); evcon->state = EVCON_WRITING; evhttp_write_buffer(evcon, evhttp_write_connectioncb, NULL); } static void evhttp_send_done(struct evhttp_connection *evcon, void *arg) { int need_close; struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); TAILQ_REMOVE(&evcon->requests, req, next); if (req->on_complete_cb != NULL) { req->on_complete_cb(req, req->on_complete_cb_arg); } need_close = (REQ_VERSION_BEFORE(req, 1, 1) && !evhttp_is_connection_keepalive(req->input_headers)) || evhttp_is_request_connection_close(req); EVUTIL_ASSERT(req->flags & EVHTTP_REQ_OWN_CONNECTION); evhttp_request_free(req); if (need_close) { evhttp_connection_free(evcon); return; } /* we have a persistent connection; try to accept another request. */ if (evhttp_associate_new_request_with_connection(evcon) == -1) { evhttp_connection_free(evcon); } } /* * Returns an error page. */ void evhttp_send_error(struct evhttp_request *req, int error, const char *reason) { #define ERR_FORMAT "<HTML><HEAD>\n" \ "<TITLE>%d %s\n" \ "\n" \ "

%s

\n" \ "\n" struct evbuffer *buf = evbuffer_new(); if (buf == NULL) { /* if we cannot allocate memory; we just drop the connection */ evhttp_connection_free(req->evcon); return; } if (reason == NULL) { reason = evhttp_response_phrase_internal(error); } evhttp_response_code_(req, error, reason); evbuffer_add_printf(buf, ERR_FORMAT, error, reason, reason); evhttp_send_page_(req, buf); evbuffer_free(buf); #undef ERR_FORMAT } /* Requires that headers and response code are already set up */ static inline void evhttp_send(struct evhttp_request *req, struct evbuffer *databuf) { struct evhttp_connection *evcon = req->evcon; if (evcon == NULL) { evhttp_request_free(req); return; } EVUTIL_ASSERT(TAILQ_FIRST(&evcon->requests) == req); /* we expect no more calls form the user on this request */ req->userdone = 1; /* xxx: not sure if we really should expose the data buffer this way */ if (databuf != NULL) evbuffer_add_buffer(req->output_buffer, databuf); /* Adds headers to the response */ evhttp_make_header(evcon, req); evhttp_write_buffer(evcon, evhttp_send_done, NULL); } void evhttp_send_reply(struct evhttp_request *req, int code, const char *reason, struct evbuffer *databuf) { evhttp_response_code_(req, code, reason); evhttp_send(req, databuf); } void evhttp_send_reply_start(struct evhttp_request *req, int code, const char *reason) { evhttp_response_code_(req, code, reason); if (evhttp_find_header(req->output_headers, "Content-Length") == NULL && REQ_VERSION_ATLEAST(req, 1, 1) && evhttp_response_needs_body(req)) { /* * prefer HTTP/1.1 chunked encoding to closing the connection; * note RFC 2616 section 4.4 forbids it with Content-Length: * and it's not necessary then anyway. */ evhttp_add_header(req->output_headers, "Transfer-Encoding", "chunked"); req->chunked = 1; } else { req->chunked = 0; } evhttp_make_header(req->evcon, req); evhttp_write_buffer(req->evcon, NULL, NULL); } void evhttp_send_reply_chunk_with_cb(struct evhttp_request *req, struct evbuffer *databuf, void (*cb)(struct evhttp_connection *, void *), void *arg) { struct evhttp_connection *evcon = req->evcon; struct evbuffer *output; if (evcon == NULL) return; output = bufferevent_get_output(evcon->bufev); if (evbuffer_get_length(databuf) == 0) return; if (!evhttp_response_needs_body(req)) return; if (req->chunked) { evbuffer_add_printf(output, "%x\r\n", (unsigned)evbuffer_get_length(databuf)); } evbuffer_add_buffer(output, databuf); if (req->chunked) { evbuffer_add(output, "\r\n", 2); } evhttp_write_buffer(evcon, cb, arg); } void evhttp_send_reply_chunk(struct evhttp_request *req, struct evbuffer *databuf) { evhttp_send_reply_chunk_with_cb(req, databuf, NULL, NULL); } void evhttp_send_reply_end(struct evhttp_request *req) { struct evhttp_connection *evcon = req->evcon; struct evbuffer *output; if (evcon == NULL) { evhttp_request_free(req); return; } output = bufferevent_get_output(evcon->bufev); /* we expect no more calls form the user on this request */ req->userdone = 1; if (req->chunked) { evbuffer_add(output, "0\r\n\r\n", 5); evhttp_write_buffer(req->evcon, evhttp_send_done, NULL); req->chunked = 0; } else if (evbuffer_get_length(output) == 0) { /* let the connection know that we are done with the request */ evhttp_send_done(evcon, NULL); } else { /* make the callback execute after all data has been written */ evcon->cb = evhttp_send_done; evcon->cb_arg = NULL; } } static const char *informational_phrases[] = { /* 100 */ "Continue", /* 101 */ "Switching Protocols" }; static const char *success_phrases[] = { /* 200 */ "OK", /* 201 */ "Created", /* 202 */ "Accepted", /* 203 */ "Non-Authoritative Information", /* 204 */ "No Content", /* 205 */ "Reset Content", /* 206 */ "Partial Content" }; static const char *redirection_phrases[] = { /* 300 */ "Multiple Choices", /* 301 */ "Moved Permanently", /* 302 */ "Found", /* 303 */ "See Other", /* 304 */ "Not Modified", /* 305 */ "Use Proxy", /* 307 */ "Temporary Redirect" }; static const char *client_error_phrases[] = { /* 400 */ "Bad Request", /* 401 */ "Unauthorized", /* 402 */ "Payment Required", /* 403 */ "Forbidden", /* 404 */ "Not Found", /* 405 */ "Method Not Allowed", /* 406 */ "Not Acceptable", /* 407 */ "Proxy Authentication Required", /* 408 */ "Request Time-out", /* 409 */ "Conflict", /* 410 */ "Gone", /* 411 */ "Length Required", /* 412 */ "Precondition Failed", /* 413 */ "Request Entity Too Large", /* 414 */ "Request-URI Too Large", /* 415 */ "Unsupported Media Type", /* 416 */ "Requested range not satisfiable", /* 417 */ "Expectation Failed" }; static const char *server_error_phrases[] = { /* 500 */ "Internal Server Error", /* 501 */ "Not Implemented", /* 502 */ "Bad Gateway", /* 503 */ "Service Unavailable", /* 504 */ "Gateway Time-out", /* 505 */ "HTTP Version not supported" }; struct response_class { const char *name; size_t num_responses; const char **responses; }; #ifndef MEMBERSOF #define MEMBERSOF(x) (sizeof(x)/sizeof(x[0])) #endif static const struct response_class response_classes[] = { /* 1xx */ { "Informational", MEMBERSOF(informational_phrases), informational_phrases }, /* 2xx */ { "Success", MEMBERSOF(success_phrases), success_phrases }, /* 3xx */ { "Redirection", MEMBERSOF(redirection_phrases), redirection_phrases }, /* 4xx */ { "Client Error", MEMBERSOF(client_error_phrases), client_error_phrases }, /* 5xx */ { "Server Error", MEMBERSOF(server_error_phrases), server_error_phrases } }; static const char * evhttp_response_phrase_internal(int code) { int klass = code / 100 - 1; int subcode = code % 100; /* Unknown class - can't do any better here */ if (klass < 0 || klass >= (int) MEMBERSOF(response_classes)) return "Unknown Status Class"; /* Unknown sub-code, return class name at least */ if (subcode >= (int) response_classes[klass].num_responses) return response_classes[klass].name; return response_classes[klass].responses[subcode]; } void evhttp_response_code_(struct evhttp_request *req, int code, const char *reason) { req->kind = EVHTTP_RESPONSE; req->response_code = code; if (req->response_code_line != NULL) mm_free(req->response_code_line); if (reason == NULL) reason = evhttp_response_phrase_internal(code); req->response_code_line = mm_strdup(reason); if (req->response_code_line == NULL) { event_warn("%s: strdup", __func__); /* XXX what else can we do? */ } } void evhttp_send_page_(struct evhttp_request *req, struct evbuffer *databuf) { if (!req->major || !req->minor) { req->major = 1; req->minor = 1; } if (req->kind != EVHTTP_RESPONSE) evhttp_response_code_(req, 200, "OK"); evhttp_clear_headers(req->output_headers); evhttp_add_header(req->output_headers, "Content-Type", "text/html"); evhttp_add_header(req->output_headers, "Connection", "close"); evhttp_send(req, databuf); } static const char uri_chars[256] = { /* 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 64 */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, /* 128 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 192 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; #define CHAR_IS_UNRESERVED(c) \ (uri_chars[(unsigned char)(c)]) /* * Helper functions to encode/decode a string for inclusion in a URI. * The returned string must be freed by the caller. */ char * evhttp_uriencode(const char *uri, ev_ssize_t len, int space_as_plus) { struct evbuffer *buf = evbuffer_new(); const char *p, *end; char *result; if (buf == NULL) { return (NULL); } if (len >= 0) { if (uri + len < uri) { return (NULL); } end = uri + len; } else { size_t slen = strlen(uri); if (slen >= EV_SSIZE_MAX) { /* we don't want to mix signed and unsigned */ return (NULL); } if (uri + slen < uri) { return (NULL); } end = uri + slen; } for (p = uri; p < end; p++) { if (CHAR_IS_UNRESERVED(*p)) { evbuffer_add(buf, p, 1); } else if (*p == ' ' && space_as_plus) { evbuffer_add(buf, "+", 1); } else { evbuffer_add_printf(buf, "%%%02X", (unsigned char)(*p)); } } evbuffer_add(buf, "", 1); /* NUL-terminator. */ result = mm_malloc(evbuffer_get_length(buf)); if (result) evbuffer_remove(buf, result, evbuffer_get_length(buf)); evbuffer_free(buf); return (result); } char * evhttp_encode_uri(const char *str) { return evhttp_uriencode(str, -1, 0); } /* * @param decode_plus_ctl: if 1, we decode plus into space. If 0, we don't. * If -1, when true we transform plus to space only after we've seen * a ?. -1 is deprecated. * @return the number of bytes written to 'ret'. */ int evhttp_decode_uri_internal( const char *uri, size_t length, char *ret, int decode_plus_ctl) { char c; int j; int decode_plus = (decode_plus_ctl == 1) ? 1: 0; unsigned i; for (i = j = 0; i < length; i++) { c = uri[i]; if (c == '?') { if (decode_plus_ctl < 0) decode_plus = 1; } else if (c == '+' && decode_plus) { c = ' '; } else if ((i + 2) < length && c == '%' && EVUTIL_ISXDIGIT_(uri[i+1]) && EVUTIL_ISXDIGIT_(uri[i+2])) { char tmp[3]; tmp[0] = uri[i+1]; tmp[1] = uri[i+2]; tmp[2] = '\0'; c = (char)strtol(tmp, NULL, 16); i += 2; } ret[j++] = c; } ret[j] = '\0'; return (j); } /* deprecated */ char * evhttp_decode_uri(const char *uri) { char *ret; if ((ret = mm_malloc(strlen(uri) + 1)) == NULL) { event_warn("%s: malloc(%lu)", __func__, (unsigned long)(strlen(uri) + 1)); return (NULL); } evhttp_decode_uri_internal(uri, strlen(uri), ret, -1 /*always_decode_plus*/); return (ret); } char * evhttp_uridecode(const char *uri, int decode_plus, size_t *size_out) { char *ret; int n; if ((ret = mm_malloc(strlen(uri) + 1)) == NULL) { event_warn("%s: malloc(%lu)", __func__, (unsigned long)(strlen(uri) + 1)); return (NULL); } n = evhttp_decode_uri_internal(uri, strlen(uri), ret, !!decode_plus/*always_decode_plus*/); if (size_out) { EVUTIL_ASSERT(n >= 0); *size_out = (size_t)n; } return (ret); } /* * Helper function to parse out arguments in a query. * The arguments are separated by key and value. */ static int evhttp_parse_query_impl(const char *str, struct evkeyvalq *headers, int is_whole_uri) { char *line=NULL; char *argument; char *p; const char *query_part; int result = -1; struct evhttp_uri *uri=NULL; TAILQ_INIT(headers); if (is_whole_uri) { uri = evhttp_uri_parse(str); if (!uri) goto error; query_part = evhttp_uri_get_query(uri); } else { query_part = str; } /* No arguments - we are done */ if (!query_part || !strlen(query_part)) { result = 0; goto done; } if ((line = mm_strdup(query_part)) == NULL) { event_warn("%s: strdup", __func__); goto error; } p = argument = line; while (p != NULL && *p != '\0') { char *key, *value, *decoded_value; argument = strsep(&p, "&"); value = argument; key = strsep(&value, "="); if (value == NULL || *key == '\0') { goto error; } if ((decoded_value = mm_malloc(strlen(value) + 1)) == NULL) { event_warn("%s: mm_malloc", __func__); goto error; } evhttp_decode_uri_internal(value, strlen(value), decoded_value, 1 /*always_decode_plus*/); event_debug(("Query Param: %s -> %s\n", key, decoded_value)); evhttp_add_header_internal(headers, key, decoded_value); mm_free(decoded_value); } result = 0; goto done; error: evhttp_clear_headers(headers); done: if (line) mm_free(line); if (uri) evhttp_uri_free(uri); return result; } int evhttp_parse_query(const char *uri, struct evkeyvalq *headers) { return evhttp_parse_query_impl(uri, headers, 1); } int evhttp_parse_query_str(const char *uri, struct evkeyvalq *headers) { return evhttp_parse_query_impl(uri, headers, 0); } static struct evhttp_cb * evhttp_dispatch_callback(struct httpcbq *callbacks, struct evhttp_request *req) { struct evhttp_cb *cb; size_t offset = 0; char *translated; const char *path; /* Test for different URLs */ path = evhttp_uri_get_path(req->uri_elems); offset = strlen(path); if ((translated = mm_malloc(offset + 1)) == NULL) return (NULL); evhttp_decode_uri_internal(path, offset, translated, 0 /* decode_plus */); TAILQ_FOREACH(cb, callbacks, next) { if (!strcmp(cb->what, translated)) { mm_free(translated); return (cb); } } mm_free(translated); return (NULL); } static int prefix_suffix_match(const char *pattern, const char *name, int ignorecase) { char c; while (1) { switch (c = *pattern++) { case '\0': return *name == '\0'; case '*': while (*name != '\0') { if (prefix_suffix_match(pattern, name, ignorecase)) return (1); ++name; } return (0); default: if (c != *name) { if (!ignorecase || EVUTIL_TOLOWER_(c) != EVUTIL_TOLOWER_(*name)) return (0); } ++name; } } /* NOTREACHED */ } /* Search the vhost hierarchy beginning with http for a server alias matching hostname. If a match is found, and outhttp is non-null, outhttp is set to the matching http object and 1 is returned. */ static int evhttp_find_alias(struct evhttp *http, struct evhttp **outhttp, const char *hostname) { struct evhttp_server_alias *alias; struct evhttp *vhost; TAILQ_FOREACH(alias, &http->aliases, next) { /* XXX Do we need to handle IP addresses? */ if (!evutil_ascii_strcasecmp(alias->alias, hostname)) { if (outhttp) *outhttp = http; return 1; } } /* XXX It might be good to avoid recursion here, but I don't see a way to do that w/o a list. */ TAILQ_FOREACH(vhost, &http->virtualhosts, next_vhost) { if (evhttp_find_alias(vhost, outhttp, hostname)) return 1; } return 0; } /* Attempts to find the best http object to handle a request for a hostname. All aliases for the root http object and vhosts are searched for an exact match. Then, the vhost hierarchy is traversed again for a matching pattern. If an alias or vhost is matched, 1 is returned, and outhttp, if non-null, is set with the best matching http object. If there are no matches, the root http object is stored in outhttp and 0 is returned. */ static int evhttp_find_vhost(struct evhttp *http, struct evhttp **outhttp, const char *hostname) { struct evhttp *vhost; struct evhttp *oldhttp; int match_found = 0; if (evhttp_find_alias(http, outhttp, hostname)) return 1; do { oldhttp = http; TAILQ_FOREACH(vhost, &http->virtualhosts, next_vhost) { if (prefix_suffix_match(vhost->vhost_pattern, hostname, 1 /* ignorecase */)) { http = vhost; match_found = 1; break; } } } while (oldhttp != http); if (outhttp) *outhttp = http; return match_found; } static void evhttp_handle_request(struct evhttp_request *req, void *arg) { struct evhttp *http = arg; struct evhttp_cb *cb = NULL; const char *hostname; /* we have a new request on which the user needs to take action */ req->userdone = 0; if (req->type == 0 || req->uri == NULL) { evhttp_send_error(req, req->response_code, NULL); return; } if ((http->allowed_methods & req->type) == 0) { event_debug(("Rejecting disallowed method %x (allowed: %x)\n", (unsigned)req->type, (unsigned)http->allowed_methods)); evhttp_send_error(req, HTTP_NOTIMPLEMENTED, NULL); return; } /* handle potential virtual hosts */ hostname = evhttp_request_get_host(req); if (hostname != NULL) { evhttp_find_vhost(http, &http, hostname); } if ((cb = evhttp_dispatch_callback(&http->callbacks, req)) != NULL) { (*cb->cb)(req, cb->cbarg); return; } /* Generic call back */ if (http->gencb) { (*http->gencb)(req, http->gencbarg); return; } else { /* We need to send a 404 here */ #define ERR_FORMAT "" \ "404 Not Found" \ "" \ "

Not Found

" \ "

The requested URL %s was not found on this server.

"\ "\n" char *escaped_html; struct evbuffer *buf; if ((escaped_html = evhttp_htmlescape(req->uri)) == NULL) { evhttp_connection_free(req->evcon); return; } if ((buf = evbuffer_new()) == NULL) { mm_free(escaped_html); evhttp_connection_free(req->evcon); return; } evhttp_response_code_(req, HTTP_NOTFOUND, "Not Found"); evbuffer_add_printf(buf, ERR_FORMAT, escaped_html); mm_free(escaped_html); evhttp_send_page_(req, buf); evbuffer_free(buf); #undef ERR_FORMAT } } /* Listener callback when a connection arrives at a server. */ static void accept_socket_cb(struct evconnlistener *listener, evutil_socket_t nfd, struct sockaddr *peer_sa, int peer_socklen, void *arg) { struct evhttp *http = arg; evhttp_get_request(http, nfd, peer_sa, peer_socklen); } int evhttp_bind_socket(struct evhttp *http, const char *address, ev_uint16_t port) { struct evhttp_bound_socket *bound = evhttp_bind_socket_with_handle(http, address, port); if (bound == NULL) return (-1); return (0); } struct evhttp_bound_socket * evhttp_bind_socket_with_handle(struct evhttp *http, const char *address, ev_uint16_t port) { evutil_socket_t fd; struct evhttp_bound_socket *bound; if ((fd = bind_socket(address, port, 1 /*reuse*/)) == -1) return (NULL); if (listen(fd, 128) == -1) { event_sock_warn(fd, "%s: listen", __func__); evutil_closesocket(fd); return (NULL); } bound = evhttp_accept_socket_with_handle(http, fd); if (bound != NULL) { event_debug(("Bound to port %d - Awaiting connections ... ", port)); return (bound); } return (NULL); } int evhttp_accept_socket(struct evhttp *http, evutil_socket_t fd) { struct evhttp_bound_socket *bound = evhttp_accept_socket_with_handle(http, fd); if (bound == NULL) return (-1); return (0); } void evhttp_foreach_bound_socket(struct evhttp *http, evhttp_bound_socket_foreach_fn *function, void *argument) { struct evhttp_bound_socket *bound; TAILQ_FOREACH(bound, &http->sockets, next) function(bound, argument); } struct evhttp_bound_socket * evhttp_accept_socket_with_handle(struct evhttp *http, evutil_socket_t fd) { struct evhttp_bound_socket *bound; struct evconnlistener *listener; const int flags = LEV_OPT_REUSEABLE|LEV_OPT_CLOSE_ON_EXEC|LEV_OPT_CLOSE_ON_FREE; listener = evconnlistener_new(http->base, NULL, NULL, flags, 0, /* Backlog is '0' because we already said 'listen' */ fd); if (!listener) return (NULL); bound = evhttp_bind_listener(http, listener); if (!bound) { evconnlistener_free(listener); return (NULL); } return (bound); } struct evhttp_bound_socket * evhttp_bind_listener(struct evhttp *http, struct evconnlistener *listener) { struct evhttp_bound_socket *bound; bound = mm_malloc(sizeof(struct evhttp_bound_socket)); if (bound == NULL) return (NULL); bound->listener = listener; TAILQ_INSERT_TAIL(&http->sockets, bound, next); evconnlistener_set_cb(listener, accept_socket_cb, http); return bound; } evutil_socket_t evhttp_bound_socket_get_fd(struct evhttp_bound_socket *bound) { return evconnlistener_get_fd(bound->listener); } struct evconnlistener * evhttp_bound_socket_get_listener(struct evhttp_bound_socket *bound) { return bound->listener; } void evhttp_del_accept_socket(struct evhttp *http, struct evhttp_bound_socket *bound) { TAILQ_REMOVE(&http->sockets, bound, next); evconnlistener_free(bound->listener); mm_free(bound); } static struct evhttp* evhttp_new_object(void) { struct evhttp *http = NULL; if ((http = mm_calloc(1, sizeof(struct evhttp))) == NULL) { event_warn("%s: calloc", __func__); return (NULL); } evutil_timerclear(&http->timeout); evhttp_set_max_headers_size(http, EV_SIZE_MAX); evhttp_set_max_body_size(http, EV_SIZE_MAX); evhttp_set_default_content_type(http, "text/html; charset=ISO-8859-1"); evhttp_set_allowed_methods(http, EVHTTP_REQ_GET | EVHTTP_REQ_POST | EVHTTP_REQ_HEAD | EVHTTP_REQ_PUT | EVHTTP_REQ_DELETE); TAILQ_INIT(&http->sockets); TAILQ_INIT(&http->callbacks); TAILQ_INIT(&http->connections); TAILQ_INIT(&http->virtualhosts); TAILQ_INIT(&http->aliases); return (http); } struct evhttp * evhttp_new(struct event_base *base) { struct evhttp *http = NULL; http = evhttp_new_object(); if (http == NULL) return (NULL); http->base = base; return (http); } /* * Start a web server on the specified address and port. */ struct evhttp * evhttp_start(const char *address, ev_uint16_t port) { struct evhttp *http = NULL; http = evhttp_new_object(); if (http == NULL) return (NULL); if (evhttp_bind_socket(http, address, port) == -1) { mm_free(http); return (NULL); } return (http); } void evhttp_free(struct evhttp* http) { struct evhttp_cb *http_cb; struct evhttp_connection *evcon; struct evhttp_bound_socket *bound; struct evhttp* vhost; struct evhttp_server_alias *alias; /* Remove the accepting part */ while ((bound = TAILQ_FIRST(&http->sockets)) != NULL) { TAILQ_REMOVE(&http->sockets, bound, next); evconnlistener_free(bound->listener); mm_free(bound); } while ((evcon = TAILQ_FIRST(&http->connections)) != NULL) { /* evhttp_connection_free removes the connection */ evhttp_connection_free(evcon); } while ((http_cb = TAILQ_FIRST(&http->callbacks)) != NULL) { TAILQ_REMOVE(&http->callbacks, http_cb, next); mm_free(http_cb->what); mm_free(http_cb); } while ((vhost = TAILQ_FIRST(&http->virtualhosts)) != NULL) { TAILQ_REMOVE(&http->virtualhosts, vhost, next_vhost); evhttp_free(vhost); } if (http->vhost_pattern != NULL) mm_free(http->vhost_pattern); while ((alias = TAILQ_FIRST(&http->aliases)) != NULL) { TAILQ_REMOVE(&http->aliases, alias, next); mm_free(alias->alias); mm_free(alias); } mm_free(http); } int evhttp_add_virtual_host(struct evhttp* http, const char *pattern, struct evhttp* vhost) { /* a vhost can only be a vhost once and should not have bound sockets */ if (vhost->vhost_pattern != NULL || TAILQ_FIRST(&vhost->sockets) != NULL) return (-1); vhost->vhost_pattern = mm_strdup(pattern); if (vhost->vhost_pattern == NULL) return (-1); TAILQ_INSERT_TAIL(&http->virtualhosts, vhost, next_vhost); return (0); } int evhttp_remove_virtual_host(struct evhttp* http, struct evhttp* vhost) { if (vhost->vhost_pattern == NULL) return (-1); TAILQ_REMOVE(&http->virtualhosts, vhost, next_vhost); mm_free(vhost->vhost_pattern); vhost->vhost_pattern = NULL; return (0); } int evhttp_add_server_alias(struct evhttp *http, const char *alias) { struct evhttp_server_alias *evalias; evalias = mm_calloc(1, sizeof(*evalias)); if (!evalias) return -1; evalias->alias = mm_strdup(alias); if (!evalias->alias) { mm_free(evalias); return -1; } TAILQ_INSERT_TAIL(&http->aliases, evalias, next); return 0; } int evhttp_remove_server_alias(struct evhttp *http, const char *alias) { struct evhttp_server_alias *evalias; TAILQ_FOREACH(evalias, &http->aliases, next) { if (evutil_ascii_strcasecmp(evalias->alias, alias) == 0) { TAILQ_REMOVE(&http->aliases, evalias, next); mm_free(evalias->alias); mm_free(evalias); return 0; } } return -1; } void evhttp_set_timeout(struct evhttp* http, int timeout_in_secs) { if (timeout_in_secs == -1) { evhttp_set_timeout_tv(http, NULL); } else { struct timeval tv; tv.tv_sec = timeout_in_secs; tv.tv_usec = 0; evhttp_set_timeout_tv(http, &tv); } } void evhttp_set_timeout_tv(struct evhttp* http, const struct timeval* tv) { if (tv) { http->timeout = *tv; } else { evutil_timerclear(&http->timeout); } } int evhttp_set_flags(struct evhttp *http, int flags) { int avail_flags = 0; avail_flags |= EVHTTP_SERVER_LINGERING_CLOSE; if (flags & ~avail_flags) return 1; http->flags &= ~avail_flags; http->flags |= flags; return 0; } void evhttp_set_max_headers_size(struct evhttp* http, ev_ssize_t max_headers_size) { if (max_headers_size < 0) http->default_max_headers_size = EV_SIZE_MAX; else http->default_max_headers_size = max_headers_size; } void evhttp_set_max_body_size(struct evhttp* http, ev_ssize_t max_body_size) { if (max_body_size < 0) http->default_max_body_size = EV_UINT64_MAX; else http->default_max_body_size = max_body_size; } void evhttp_set_default_content_type(struct evhttp *http, const char *content_type) { http->default_content_type = content_type; } void evhttp_set_allowed_methods(struct evhttp* http, ev_uint16_t methods) { http->allowed_methods = methods; } int evhttp_set_cb(struct evhttp *http, const char *uri, void (*cb)(struct evhttp_request *, void *), void *cbarg) { struct evhttp_cb *http_cb; TAILQ_FOREACH(http_cb, &http->callbacks, next) { if (strcmp(http_cb->what, uri) == 0) return (-1); } if ((http_cb = mm_calloc(1, sizeof(struct evhttp_cb))) == NULL) { event_warn("%s: calloc", __func__); return (-2); } http_cb->what = mm_strdup(uri); if (http_cb->what == NULL) { event_warn("%s: strdup", __func__); mm_free(http_cb); return (-3); } http_cb->cb = cb; http_cb->cbarg = cbarg; TAILQ_INSERT_TAIL(&http->callbacks, http_cb, next); return (0); } int evhttp_del_cb(struct evhttp *http, const char *uri) { struct evhttp_cb *http_cb; TAILQ_FOREACH(http_cb, &http->callbacks, next) { if (strcmp(http_cb->what, uri) == 0) break; } if (http_cb == NULL) return (-1); TAILQ_REMOVE(&http->callbacks, http_cb, next); mm_free(http_cb->what); mm_free(http_cb); return (0); } void evhttp_set_gencb(struct evhttp *http, void (*cb)(struct evhttp_request *, void *), void *cbarg) { http->gencb = cb; http->gencbarg = cbarg; } void evhttp_set_bevcb(struct evhttp *http, struct bufferevent* (*cb)(struct event_base *, void *), void *cbarg) { http->bevcb = cb; http->bevcbarg = cbarg; } /* * Request related functions */ struct evhttp_request * evhttp_request_new(void (*cb)(struct evhttp_request *, void *), void *arg) { struct evhttp_request *req = NULL; /* Allocate request structure */ if ((req = mm_calloc(1, sizeof(struct evhttp_request))) == NULL) { event_warn("%s: calloc", __func__); goto error; } req->headers_size = 0; req->body_size = 0; req->kind = EVHTTP_RESPONSE; req->input_headers = mm_calloc(1, sizeof(struct evkeyvalq)); if (req->input_headers == NULL) { event_warn("%s: calloc", __func__); goto error; } TAILQ_INIT(req->input_headers); req->output_headers = mm_calloc(1, sizeof(struct evkeyvalq)); if (req->output_headers == NULL) { event_warn("%s: calloc", __func__); goto error; } TAILQ_INIT(req->output_headers); if ((req->input_buffer = evbuffer_new()) == NULL) { event_warn("%s: evbuffer_new", __func__); goto error; } if ((req->output_buffer = evbuffer_new()) == NULL) { event_warn("%s: evbuffer_new", __func__); goto error; } req->cb = cb; req->cb_arg = arg; return (req); error: if (req != NULL) evhttp_request_free(req); return (NULL); } void evhttp_request_free(struct evhttp_request *req) { if ((req->flags & EVHTTP_REQ_DEFER_FREE) != 0) { req->flags |= EVHTTP_REQ_NEEDS_FREE; return; } if (req->remote_host != NULL) mm_free(req->remote_host); if (req->uri != NULL) mm_free(req->uri); if (req->uri_elems != NULL) evhttp_uri_free(req->uri_elems); if (req->response_code_line != NULL) mm_free(req->response_code_line); if (req->host_cache != NULL) mm_free(req->host_cache); evhttp_clear_headers(req->input_headers); mm_free(req->input_headers); evhttp_clear_headers(req->output_headers); mm_free(req->output_headers); if (req->input_buffer != NULL) evbuffer_free(req->input_buffer); if (req->output_buffer != NULL) evbuffer_free(req->output_buffer); mm_free(req); } void evhttp_request_own(struct evhttp_request *req) { req->flags |= EVHTTP_USER_OWNED; } int evhttp_request_is_owned(struct evhttp_request *req) { return (req->flags & EVHTTP_USER_OWNED) != 0; } struct evhttp_connection * evhttp_request_get_connection(struct evhttp_request *req) { return req->evcon; } struct event_base * evhttp_connection_get_base(struct evhttp_connection *conn) { return conn->base; } void evhttp_request_set_chunked_cb(struct evhttp_request *req, void (*cb)(struct evhttp_request *, void *)) { req->chunk_cb = cb; } void evhttp_request_set_header_cb(struct evhttp_request *req, int (*cb)(struct evhttp_request *, void *)) { req->header_cb = cb; } void evhttp_request_set_error_cb(struct evhttp_request *req, void (*cb)(enum evhttp_request_error, void *)) { req->error_cb = cb; } void evhttp_request_set_on_complete_cb(struct evhttp_request *req, void (*cb)(struct evhttp_request *, void *), void *cb_arg) { req->on_complete_cb = cb; req->on_complete_cb_arg = cb_arg; } /* * Allows for inspection of the request URI */ const char * evhttp_request_get_uri(const struct evhttp_request *req) { if (req->uri == NULL) event_debug(("%s: request %p has no uri\n", __func__, req)); return (req->uri); } const struct evhttp_uri * evhttp_request_get_evhttp_uri(const struct evhttp_request *req) { if (req->uri_elems == NULL) event_debug(("%s: request %p has no uri elems\n", __func__, req)); return (req->uri_elems); } const char * evhttp_request_get_host(struct evhttp_request *req) { const char *host = NULL; if (req->host_cache) return req->host_cache; if (req->uri_elems) host = evhttp_uri_get_host(req->uri_elems); if (!host && req->input_headers) { const char *p; size_t len; host = evhttp_find_header(req->input_headers, "Host"); /* The Host: header may include a port. Remove it here to be consistent with uri_elems case above. */ if (host) { p = host + strlen(host) - 1; while (p > host && EVUTIL_ISDIGIT_(*p)) --p; if (p > host && *p == ':') { len = p - host; req->host_cache = mm_malloc(len + 1); if (!req->host_cache) { event_warn("%s: malloc", __func__); return NULL; } memcpy(req->host_cache, host, len); req->host_cache[len] = '\0'; host = req->host_cache; } } } return host; } enum evhttp_cmd_type evhttp_request_get_command(const struct evhttp_request *req) { return (req->type); } int evhttp_request_get_response_code(const struct evhttp_request *req) { return req->response_code; } const char * evhttp_request_get_response_code_line(const struct evhttp_request *req) { return req->response_code_line; } /** Returns the input headers */ struct evkeyvalq *evhttp_request_get_input_headers(struct evhttp_request *req) { return (req->input_headers); } /** Returns the output headers */ struct evkeyvalq *evhttp_request_get_output_headers(struct evhttp_request *req) { return (req->output_headers); } /** Returns the input buffer */ struct evbuffer *evhttp_request_get_input_buffer(struct evhttp_request *req) { return (req->input_buffer); } /** Returns the output buffer */ struct evbuffer *evhttp_request_get_output_buffer(struct evhttp_request *req) { return (req->output_buffer); } /* * Takes a file descriptor to read a request from. * The callback is executed once the whole request has been read. */ static struct evhttp_connection* evhttp_get_request_connection( struct evhttp* http, evutil_socket_t fd, struct sockaddr *sa, ev_socklen_t salen) { struct evhttp_connection *evcon; char *hostname = NULL, *portname = NULL; struct bufferevent* bev = NULL; name_from_addr(sa, salen, &hostname, &portname); if (hostname == NULL || portname == NULL) { if (hostname) mm_free(hostname); if (portname) mm_free(portname); return (NULL); } event_debug(("%s: new request from %s:%s on "EV_SOCK_FMT"\n", __func__, hostname, portname, EV_SOCK_ARG(fd))); /* we need a connection object to put the http request on */ if (http->bevcb != NULL) { bev = (*http->bevcb)(http->base, http->bevcbarg); } evcon = evhttp_connection_base_bufferevent_new( http->base, NULL, bev, hostname, atoi(portname)); mm_free(hostname); mm_free(portname); if (evcon == NULL) return (NULL); evcon->max_headers_size = http->default_max_headers_size; evcon->max_body_size = http->default_max_body_size; if (http->flags & EVHTTP_SERVER_LINGERING_CLOSE) evcon->flags |= EVHTTP_CON_LINGERING_CLOSE; evcon->flags |= EVHTTP_CON_INCOMING; evcon->state = EVCON_READING_FIRSTLINE; evcon->fd = fd; bufferevent_enable(evcon->bufev, EV_READ); bufferevent_disable(evcon->bufev, EV_WRITE); bufferevent_setfd(evcon->bufev, fd); return (evcon); } static int evhttp_associate_new_request_with_connection(struct evhttp_connection *evcon) { struct evhttp *http = evcon->http_server; struct evhttp_request *req; if ((req = evhttp_request_new(evhttp_handle_request, http)) == NULL) return (-1); if ((req->remote_host = mm_strdup(evcon->address)) == NULL) { event_warn("%s: strdup", __func__); evhttp_request_free(req); return (-1); } req->remote_port = evcon->port; req->evcon = evcon; /* the request ends up owning the connection */ req->flags |= EVHTTP_REQ_OWN_CONNECTION; /* We did not present the request to the user user yet, so treat it as * if the user was done with the request. This allows us to free the * request on a persistent connection if the client drops it without * sending a request. */ req->userdone = 1; TAILQ_INSERT_TAIL(&evcon->requests, req, next); req->kind = EVHTTP_REQUEST; evhttp_start_read_(evcon); return (0); } static void evhttp_get_request(struct evhttp *http, evutil_socket_t fd, struct sockaddr *sa, ev_socklen_t salen) { struct evhttp_connection *evcon; evcon = evhttp_get_request_connection(http, fd, sa, salen); if (evcon == NULL) { event_sock_warn(fd, "%s: cannot get connection on "EV_SOCK_FMT, __func__, EV_SOCK_ARG(fd)); evutil_closesocket(fd); return; } /* the timeout can be used by the server to close idle connections */ if (evutil_timerisset(&http->timeout)) evhttp_connection_set_timeout_tv(evcon, &http->timeout); /* * if we want to accept more than one request on a connection, * we need to know which http server it belongs to. */ evcon->http_server = http; TAILQ_INSERT_TAIL(&http->connections, evcon, next); if (evhttp_associate_new_request_with_connection(evcon) == -1) evhttp_connection_free(evcon); } /* * Network helper functions that we do not want to export to the rest of * the world. */ static void name_from_addr(struct sockaddr *sa, ev_socklen_t salen, char **phost, char **pport) { char ntop[NI_MAXHOST]; char strport[NI_MAXSERV]; int ni_result; #ifdef EVENT__HAVE_GETNAMEINFO ni_result = getnameinfo(sa, salen, ntop, sizeof(ntop), strport, sizeof(strport), NI_NUMERICHOST|NI_NUMERICSERV); if (ni_result != 0) { #ifdef EAI_SYSTEM /* Windows doesn't have an EAI_SYSTEM. */ if (ni_result == EAI_SYSTEM) event_err(1, "getnameinfo failed"); else #endif event_errx(1, "getnameinfo failed: %s", gai_strerror(ni_result)); return; } #else ni_result = fake_getnameinfo(sa, salen, ntop, sizeof(ntop), strport, sizeof(strport), NI_NUMERICHOST|NI_NUMERICSERV); if (ni_result != 0) return; #endif *phost = mm_strdup(ntop); *pport = mm_strdup(strport); } /* Create a non-blocking socket and bind it */ /* todo: rename this function */ static evutil_socket_t bind_socket_ai(struct evutil_addrinfo *ai, int reuse) { evutil_socket_t fd; int on = 1, r; int serrno; /* Create listen socket */ fd = evutil_socket_(ai ? ai->ai_family : AF_INET, SOCK_STREAM|EVUTIL_SOCK_NONBLOCK|EVUTIL_SOCK_CLOEXEC, 0); if (fd == -1) { event_sock_warn(-1, "socket"); return (-1); } if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&on, sizeof(on))<0) goto out; if (reuse) { if (evutil_make_listen_socket_reuseable(fd) < 0) goto out; } if (ai != NULL) { r = bind(fd, ai->ai_addr, (ev_socklen_t)ai->ai_addrlen); if (r == -1) goto out; } return (fd); out: serrno = EVUTIL_SOCKET_ERROR(); evutil_closesocket(fd); EVUTIL_SET_SOCKET_ERROR(serrno); return (-1); } static struct evutil_addrinfo * make_addrinfo(const char *address, ev_uint16_t port) { struct evutil_addrinfo *ai = NULL; struct evutil_addrinfo hints; char strport[NI_MAXSERV]; int ai_result; memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; /* turn NULL hostname into INADDR_ANY, and skip looking up any address * types we don't have an interface to connect to. */ hints.ai_flags = EVUTIL_AI_PASSIVE|EVUTIL_AI_ADDRCONFIG; evutil_snprintf(strport, sizeof(strport), "%d", port); if ((ai_result = evutil_getaddrinfo(address, strport, &hints, &ai)) != 0) { if (ai_result == EVUTIL_EAI_SYSTEM) event_warn("getaddrinfo"); else event_warnx("getaddrinfo: %s", evutil_gai_strerror(ai_result)); return (NULL); } return (ai); } static evutil_socket_t bind_socket(const char *address, ev_uint16_t port, int reuse) { evutil_socket_t fd; struct evutil_addrinfo *aitop = NULL; /* just create an unbound socket */ if (address == NULL && port == 0) return bind_socket_ai(NULL, 0); aitop = make_addrinfo(address, port); if (aitop == NULL) return (-1); fd = bind_socket_ai(aitop, reuse); evutil_freeaddrinfo(aitop); return (fd); } struct evhttp_uri { unsigned flags; char *scheme; /* scheme; e.g http, ftp etc */ char *userinfo; /* userinfo (typically username:pass), or NULL */ char *host; /* hostname, IP address, or NULL */ int port; /* port, or zero */ char *path; /* path, or "". */ char *query; /* query, or NULL */ char *fragment; /* fragment or NULL */ }; struct evhttp_uri * evhttp_uri_new(void) { struct evhttp_uri *uri = mm_calloc(sizeof(struct evhttp_uri), 1); if (uri) uri->port = -1; return uri; } void evhttp_uri_set_flags(struct evhttp_uri *uri, unsigned flags) { uri->flags = flags; } /* Return true if the string starting at s and ending immediately before eos * is a valid URI scheme according to RFC3986 */ static int scheme_ok(const char *s, const char *eos) { /* scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) */ EVUTIL_ASSERT(eos >= s); if (s == eos) return 0; if (!EVUTIL_ISALPHA_(*s)) return 0; while (++s < eos) { if (! EVUTIL_ISALNUM_(*s) && *s != '+' && *s != '-' && *s != '.') return 0; } return 1; } #define SUBDELIMS "!$&'()*+,;=" /* Return true iff [s..eos) is a valid userinfo */ static int userinfo_ok(const char *s, const char *eos) { while (s < eos) { if (CHAR_IS_UNRESERVED(*s) || strchr(SUBDELIMS, *s) || *s == ':') ++s; else if (*s == '%' && s+2 < eos && EVUTIL_ISXDIGIT_(s[1]) && EVUTIL_ISXDIGIT_(s[2])) s += 3; else return 0; } return 1; } static int regname_ok(const char *s, const char *eos) { while (s && s 65535) return -1; ++s; } return portnum; } /* returns 0 for bad, 1 for ipv6, 2 for IPvFuture */ static int bracket_addr_ok(const char *s, const char *eos) { if (s + 3 > eos || *s != '[' || *(eos-1) != ']') return 0; if (s[1] == 'v') { /* IPvFuture, or junk. "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ) */ s += 2; /* skip [v */ --eos; if (!EVUTIL_ISXDIGIT_(*s)) /*require at least one*/ return 0; while (s < eos && *s != '.') { if (EVUTIL_ISXDIGIT_(*s)) ++s; else return 0; } if (*s != '.') return 0; ++s; while (s < eos) { if (CHAR_IS_UNRESERVED(*s) || strchr(SUBDELIMS, *s) || *s == ':') ++s; else return 0; } return 2; } else { /* IPv6, or junk */ char buf[64]; ev_ssize_t n_chars = eos-s-2; struct in6_addr in6; if (n_chars >= 64) /* way too long */ return 0; memcpy(buf, s+1, n_chars); buf[n_chars]='\0'; return (evutil_inet_pton(AF_INET6,buf,&in6)==1) ? 1 : 0; } } static int parse_authority(struct evhttp_uri *uri, char *s, char *eos) { char *cp, *port; EVUTIL_ASSERT(eos); if (eos == s) { uri->host = mm_strdup(""); if (uri->host == NULL) { event_warn("%s: strdup", __func__); return -1; } return 0; } /* Optionally, we start with "userinfo@" */ cp = strchr(s, '@'); if (cp && cp < eos) { if (! userinfo_ok(s,cp)) return -1; *cp++ = '\0'; uri->userinfo = mm_strdup(s); if (uri->userinfo == NULL) { event_warn("%s: strdup", __func__); return -1; } } else { cp = s; } /* Optionally, we end with ":port" */ for (port=eos-1; port >= cp && EVUTIL_ISDIGIT_(*port); --port) ; if (port >= cp && *port == ':') { if (port+1 == eos) /* Leave port unspecified; the RFC allows a * nil port */ uri->port = -1; else if ((uri->port = parse_port(port+1, eos))<0) return -1; eos = port; } /* Now, cp..eos holds the "host" port, which can be an IPv4Address, * an IP-Literal, or a reg-name */ EVUTIL_ASSERT(eos >= cp); if (*cp == '[' && eos >= cp+2 && *(eos-1) == ']') { /* IPv6address, IP-Literal, or junk. */ if (! bracket_addr_ok(cp, eos)) return -1; } else { /* Make sure the host part is ok. */ if (! regname_ok(cp,eos)) /* Match IPv4Address or reg-name */ return -1; } uri->host = mm_malloc(eos-cp+1); if (uri->host == NULL) { event_warn("%s: malloc", __func__); return -1; } memcpy(uri->host, cp, eos-cp); uri->host[eos-cp] = '\0'; return 0; } static char * end_of_authority(char *cp) { while (*cp) { if (*cp == '?' || *cp == '#' || *cp == '/') return cp; ++cp; } return cp; } enum uri_part { PART_PATH, PART_QUERY, PART_FRAGMENT }; /* Return the character after the longest prefix of 'cp' that matches... * *pchar / "/" if allow_qchars is false, or * *(pchar / "/" / "?") if allow_qchars is true. */ static char * end_of_path(char *cp, enum uri_part part, unsigned flags) { if (flags & EVHTTP_URI_NONCONFORMANT) { /* If NONCONFORMANT: * Path is everything up to a # or ? or nul. * Query is everything up a # or nul * Fragment is everything up to a nul. */ switch (part) { case PART_PATH: while (*cp && *cp != '#' && *cp != '?') ++cp; break; case PART_QUERY: while (*cp && *cp != '#') ++cp; break; case PART_FRAGMENT: cp += strlen(cp); break; }; return cp; } while (*cp) { if (CHAR_IS_UNRESERVED(*cp) || strchr(SUBDELIMS, *cp) || *cp == ':' || *cp == '@' || *cp == '/') ++cp; else if (*cp == '%' && EVUTIL_ISXDIGIT_(cp[1]) && EVUTIL_ISXDIGIT_(cp[2])) cp += 3; else if (*cp == '?' && part != PART_PATH) ++cp; else return cp; } return cp; } static int path_matches_noscheme(const char *cp) { while (*cp) { if (*cp == ':') return 0; else if (*cp == '/') return 1; ++cp; } return 1; } struct evhttp_uri * evhttp_uri_parse(const char *source_uri) { return evhttp_uri_parse_with_flags(source_uri, 0); } struct evhttp_uri * evhttp_uri_parse_with_flags(const char *source_uri, unsigned flags) { char *readbuf = NULL, *readp = NULL, *token = NULL, *query = NULL; char *path = NULL, *fragment = NULL; int got_authority = 0; struct evhttp_uri *uri = mm_calloc(1, sizeof(struct evhttp_uri)); if (uri == NULL) { event_warn("%s: calloc", __func__); goto err; } uri->port = -1; uri->flags = flags; readbuf = mm_strdup(source_uri); if (readbuf == NULL) { event_warn("%s: strdup", __func__); goto err; } readp = readbuf; token = NULL; /* We try to follow RFC3986 here as much as we can, and match the productions URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] relative-ref = relative-part [ "?" query ] [ "#" fragment ] */ /* 1. scheme: */ token = strchr(readp, ':'); if (token && scheme_ok(readp,token)) { *token = '\0'; uri->scheme = mm_strdup(readp); if (uri->scheme == NULL) { event_warn("%s: strdup", __func__); goto err; } readp = token+1; /* eat : */ } /* 2. Optionally, "//" then an 'authority' part. */ if (readp[0]=='/' && readp[1] == '/') { char *authority; readp += 2; authority = readp; path = end_of_authority(readp); if (parse_authority(uri, authority, path) < 0) goto err; readp = path; got_authority = 1; } /* 3. Query: path-abempty, path-absolute, path-rootless, or path-empty */ path = readp; readp = end_of_path(path, PART_PATH, flags); /* Query */ if (*readp == '?') { *readp = '\0'; ++readp; query = readp; readp = end_of_path(readp, PART_QUERY, flags); } /* fragment */ if (*readp == '#') { *readp = '\0'; ++readp; fragment = readp; readp = end_of_path(readp, PART_FRAGMENT, flags); } if (*readp != '\0') { goto err; } /* These next two cases may be unreachable; I'm leaving them * in to be defensive. */ /* If you didn't get an authority, the path can't begin with "//" */ if (!got_authority && path[0]=='/' && path[1]=='/') goto err; /* If you did get an authority, the path must begin with "/" or be * empty. */ if (got_authority && path[0] != '/' && path[0] != '\0') goto err; /* (End of maybe-unreachable cases) */ /* If there was no scheme, the first part of the path (if any) must * have no colon in it. */ if (! uri->scheme && !path_matches_noscheme(path)) goto err; EVUTIL_ASSERT(path); uri->path = mm_strdup(path); if (uri->path == NULL) { event_warn("%s: strdup", __func__); goto err; } if (query) { uri->query = mm_strdup(query); if (uri->query == NULL) { event_warn("%s: strdup", __func__); goto err; } } if (fragment) { uri->fragment = mm_strdup(fragment); if (uri->fragment == NULL) { event_warn("%s: strdup", __func__); goto err; } } mm_free(readbuf); return uri; err: if (uri) evhttp_uri_free(uri); if (readbuf) mm_free(readbuf); return NULL; } void evhttp_uri_free(struct evhttp_uri *uri) { #define URI_FREE_STR_(f) \ if (uri->f) { \ mm_free(uri->f); \ } URI_FREE_STR_(scheme); URI_FREE_STR_(userinfo); URI_FREE_STR_(host); URI_FREE_STR_(path); URI_FREE_STR_(query); URI_FREE_STR_(fragment); mm_free(uri); #undef URI_FREE_STR_ } char * evhttp_uri_join(struct evhttp_uri *uri, char *buf, size_t limit) { struct evbuffer *tmp = 0; size_t joined_size = 0; char *output = NULL; #define URI_ADD_(f) evbuffer_add(tmp, uri->f, strlen(uri->f)) if (!uri || !buf || !limit) return NULL; tmp = evbuffer_new(); if (!tmp) return NULL; if (uri->scheme) { URI_ADD_(scheme); evbuffer_add(tmp, ":", 1); } if (uri->host) { evbuffer_add(tmp, "//", 2); if (uri->userinfo) evbuffer_add_printf(tmp,"%s@", uri->userinfo); URI_ADD_(host); if (uri->port >= 0) evbuffer_add_printf(tmp,":%d", uri->port); if (uri->path && uri->path[0] != '/' && uri->path[0] != '\0') goto err; } if (uri->path) URI_ADD_(path); if (uri->query) { evbuffer_add(tmp, "?", 1); URI_ADD_(query); } if (uri->fragment) { evbuffer_add(tmp, "#", 1); URI_ADD_(fragment); } evbuffer_add(tmp, "\0", 1); /* NUL */ joined_size = evbuffer_get_length(tmp); if (joined_size > limit) { /* It doesn't fit. */ evbuffer_free(tmp); return NULL; } evbuffer_remove(tmp, buf, joined_size); output = buf; err: evbuffer_free(tmp); return output; #undef URI_ADD_ } const char * evhttp_uri_get_scheme(const struct evhttp_uri *uri) { return uri->scheme; } const char * evhttp_uri_get_userinfo(const struct evhttp_uri *uri) { return uri->userinfo; } const char * evhttp_uri_get_host(const struct evhttp_uri *uri) { return uri->host; } int evhttp_uri_get_port(const struct evhttp_uri *uri) { return uri->port; } const char * evhttp_uri_get_path(const struct evhttp_uri *uri) { return uri->path; } const char * evhttp_uri_get_query(const struct evhttp_uri *uri) { return uri->query; } const char * evhttp_uri_get_fragment(const struct evhttp_uri *uri) { return uri->fragment; } #define URI_SET_STR_(f) do { \ if (uri->f) \ mm_free(uri->f); \ if (f) { \ if ((uri->f = mm_strdup(f)) == NULL) { \ event_warn("%s: strdup()", __func__); \ return -1; \ } \ } else { \ uri->f = NULL; \ } \ } while(0) int evhttp_uri_set_scheme(struct evhttp_uri *uri, const char *scheme) { if (scheme && !scheme_ok(scheme, scheme+strlen(scheme))) return -1; URI_SET_STR_(scheme); return 0; } int evhttp_uri_set_userinfo(struct evhttp_uri *uri, const char *userinfo) { if (userinfo && !userinfo_ok(userinfo, userinfo+strlen(userinfo))) return -1; URI_SET_STR_(userinfo); return 0; } int evhttp_uri_set_host(struct evhttp_uri *uri, const char *host) { if (host) { if (host[0] == '[') { if (! bracket_addr_ok(host, host+strlen(host))) return -1; } else { if (! regname_ok(host, host+strlen(host))) return -1; } } URI_SET_STR_(host); return 0; } int evhttp_uri_set_port(struct evhttp_uri *uri, int port) { if (port < -1) return -1; uri->port = port; return 0; } #define end_of_cpath(cp,p,f) \ ((const char*)(end_of_path(((char*)(cp)), (p), (f)))) int evhttp_uri_set_path(struct evhttp_uri *uri, const char *path) { if (path && end_of_cpath(path, PART_PATH, uri->flags) != path+strlen(path)) return -1; URI_SET_STR_(path); return 0; } int evhttp_uri_set_query(struct evhttp_uri *uri, const char *query) { if (query && end_of_cpath(query, PART_QUERY, uri->flags) != query+strlen(query)) return -1; URI_SET_STR_(query); return 0; } int evhttp_uri_set_fragment(struct evhttp_uri *uri, const char *fragment) { if (fragment && end_of_cpath(fragment, PART_FRAGMENT, uri->flags) != fragment+strlen(fragment)) return -1; URI_SET_STR_(fragment); return 0; } kytensor-2.49.0.7/third_party/libevent/include/000077500000000000000000000000001527052371200214215ustar00rootroot00000000000000kytensor-2.49.0.7/third_party/libevent/include/evdns.h000066400000000000000000000037431527052371200227200ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT1_EVDNS_H_INCLUDED_ #define EVENT1_EVDNS_H_INCLUDED_ /** @file evdns.h A dns subsystem for Libevent. The header is deprecated in Libevent 2.0 and later; please use instead. Depending on what functionality you need, you may also want to include more of the other headers. */ #include #include #include #include #endif /* EVENT1_EVDNS_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event.h000066400000000000000000000052701527052371200227170ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT1_EVENT_H_INCLUDED_ #define EVENT1_EVENT_H_INCLUDED_ /** @file event.h A library for writing event-driven network servers. The header is deprecated in Libevent 2.0 and later; please use instead. Depending on what functionality you need, you may also want to include more of the other event2/ headers. */ #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #ifdef EVENT__HAVE_STDINT_H #include #endif #include /* For int types. */ #include #ifdef _WIN32 #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include #include #undef WIN32_LEAN_AND_MEAN #endif #include #include #include #include #include #include #include #include #include #include #ifdef __cplusplus } #endif #endif /* EVENT1_EVENT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/000077500000000000000000000000001527052371200226245ustar00rootroot00000000000000kytensor-2.49.0.7/third_party/libevent/include/event2/buffer.h000066400000000000000000001142461527052371200242560ustar00rootroot00000000000000/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_BUFFER_H_INCLUDED_ #define EVENT2_BUFFER_H_INCLUDED_ /** @file event2/buffer.h Functions for buffering data for network sending or receiving. An evbuffer can be used for preparing data before sending it to the network or conversely for reading data from the network. Evbuffers try to avoid memory copies as much as possible. As a result, evbuffers can be used to pass data around without actually incurring the overhead of copying the data. A new evbuffer can be allocated with evbuffer_new(), and can be freed with evbuffer_free(). Most users will be using evbuffers via the bufferevent interface. To access a bufferevent's evbuffers, use bufferevent_get_input() and bufferevent_get_output(). There are several guidelines for using evbuffers. - if you already know how much data you are going to add as a result of calling evbuffer_add() multiple times, it makes sense to use evbuffer_expand() first to make sure that enough memory is allocated before hand. - evbuffer_add_buffer() adds the contents of one buffer to the other without incurring any unnecessary memory copies. - evbuffer_add() and evbuffer_add_buffer() do not mix very well: if you use them, you will wind up with fragmented memory in your buffer. - For high-performance code, you may want to avoid copying data into and out of buffers. You can skip the copy step by using evbuffer_reserve_space()/evbuffer_commit_space() when writing into a buffer, and evbuffer_peek() when reading. In Libevent 2.0 and later, evbuffers are represented using a linked list of memory chunks, with pointers to the first and last chunk in the chain. As the contents of an evbuffer can be stored in multiple different memory blocks, it cannot be accessed directly. Instead, evbuffer_pullup() can be used to force a specified number of bytes to be contiguous. This will cause memory reallocation and memory copies if the data is split across multiple blocks. It is more efficient, however, to use evbuffer_peek() if you don't require that the memory to be contiguous. */ #include #ifdef __cplusplus extern "C" { #endif #include #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_UIO_H #include #endif #include /** An evbuffer is an opaque data type for efficiently buffering data to be sent or received on the network. @see event2/event.h for more information */ struct evbuffer #ifdef EVENT_IN_DOXYGEN_ {} #endif ; /** Pointer to a position within an evbuffer. Used when repeatedly searching through a buffer. Calling any function that modifies or re-packs the buffer contents may invalidate all evbuffer_ptrs for that buffer. Do not modify or contruct these values except with evbuffer_ptr_set. An evbuffer_ptr can represent any position from the start of a buffer up to a position immediately after the end of a buffer. @see evbuffer_ptr_set() */ struct evbuffer_ptr { ev_ssize_t pos; /* Do not alter or rely on the values of fields: they are for internal * use */ struct { void *chain; size_t pos_in_chain; } internal_; }; /** Describes a single extent of memory inside an evbuffer. Used for direct-access functions. @see evbuffer_reserve_space, evbuffer_commit_space, evbuffer_peek */ #ifdef EVENT__HAVE_SYS_UIO_H #define evbuffer_iovec iovec /* Internal use -- defined only if we are using the native struct iovec */ #define EVBUFFER_IOVEC_IS_NATIVE_ #else struct evbuffer_iovec { /** The start of the extent of memory. */ void *iov_base; /** The length of the extent of memory. */ size_t iov_len; }; #endif /** Allocate storage for a new evbuffer. @return a pointer to a newly allocated evbuffer struct, or NULL if an error occurred */ EVENT2_EXPORT_SYMBOL struct evbuffer *evbuffer_new(void); /** Deallocate storage for an evbuffer. @param buf pointer to the evbuffer to be freed */ EVENT2_EXPORT_SYMBOL void evbuffer_free(struct evbuffer *buf); /** Enable locking on an evbuffer so that it can safely be used by multiple threads at the same time. NOTE: when locking is enabled, the lock will be held when callbacks are invoked. This could result in deadlock if you aren't careful. Plan accordingly! @param buf An evbuffer to make lockable. @param lock A lock object, or NULL if we should allocate our own. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_enable_locking(struct evbuffer *buf, void *lock); /** Acquire the lock on an evbuffer. Has no effect if locking was not enabled with evbuffer_enable_locking. */ EVENT2_EXPORT_SYMBOL void evbuffer_lock(struct evbuffer *buf); /** Release the lock on an evbuffer. Has no effect if locking was not enabled with evbuffer_enable_locking. */ EVENT2_EXPORT_SYMBOL void evbuffer_unlock(struct evbuffer *buf); /** If this flag is set, then we will not use evbuffer_peek(), * evbuffer_remove(), evbuffer_remove_buffer(), and so on to read bytes * from this buffer: we'll only take bytes out of this buffer by * writing them to the network (as with evbuffer_write_atmost), by * removing them without observing them (as with evbuffer_drain), * or by copying them all out at once (as with evbuffer_add_buffer). * * Using this option allows the implementation to use sendfile-based * operations for evbuffer_add_file(); see that function for more * information. * * This flag is on by default for bufferevents that can take advantage * of it; you should never actually need to set it on a bufferevent's * output buffer. */ #define EVBUFFER_FLAG_DRAINS_TO_FD 1 /** Change the flags that are set for an evbuffer by adding more. * * @param buffer the evbuffer that the callback is watching. * @param cb the callback whose status we want to change. * @param flags One or more EVBUFFER_FLAG_* options * @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_set_flags(struct evbuffer *buf, ev_uint64_t flags); /** Change the flags that are set for an evbuffer by removing some. * * @param buffer the evbuffer that the callback is watching. * @param cb the callback whose status we want to change. * @param flags One or more EVBUFFER_FLAG_* options * @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_clear_flags(struct evbuffer *buf, ev_uint64_t flags); /** Returns the total number of bytes stored in the evbuffer @param buf pointer to the evbuffer @return the number of bytes stored in the evbuffer */ EVENT2_EXPORT_SYMBOL size_t evbuffer_get_length(const struct evbuffer *buf); /** Returns the number of contiguous available bytes in the first buffer chain. This is useful when processing data that might be split into multiple chains, or that might all be in the first chain. Calls to evbuffer_pullup() that cause reallocation and copying of data can thus be avoided. @param buf pointer to the evbuffer @return 0 if no data is available, otherwise the number of available bytes in the first buffer chain. */ EVENT2_EXPORT_SYMBOL size_t evbuffer_get_contiguous_space(const struct evbuffer *buf); /** Expands the available space in an evbuffer. Expands the available space in the evbuffer to at least datlen, so that appending datlen additional bytes will not require any new allocations. @param buf the evbuffer to be expanded @param datlen the new minimum length requirement @return 0 if successful, or -1 if an error occurred */ EVENT2_EXPORT_SYMBOL int evbuffer_expand(struct evbuffer *buf, size_t datlen); /** Reserves space in the last chain or chains of an evbuffer. Makes space available in the last chain or chains of an evbuffer that can be arbitrarily written to by a user. The space does not become available for reading until it has been committed with evbuffer_commit_space(). The space is made available as one or more extents, represented by an initial pointer and a length. You can force the memory to be available as only one extent. Allowing more extents, however, makes the function more efficient. Multiple subsequent calls to this function will make the same space available until evbuffer_commit_space() has been called. It is an error to do anything that moves around the buffer's internal memory structures before committing the space. NOTE: The code currently does not ever use more than two extents. This may change in future versions. @param buf the evbuffer in which to reserve space. @param size how much space to make available, at minimum. The total length of the extents may be greater than the requested length. @param vec an array of one or more evbuffer_iovec structures to hold pointers to the reserved extents of memory. @param n_vec The length of the vec array. Must be at least 1; 2 is more efficient. @return the number of provided extents, or -1 on error. @see evbuffer_commit_space() */ EVENT2_EXPORT_SYMBOL int evbuffer_reserve_space(struct evbuffer *buf, ev_ssize_t size, struct evbuffer_iovec *vec, int n_vec); /** Commits previously reserved space. Commits some of the space previously reserved with evbuffer_reserve_space(). It then becomes available for reading. This function may return an error if the pointer in the extents do not match those returned from evbuffer_reserve_space, or if data has been added to the buffer since the space was reserved. If you want to commit less data than you got reserved space for, modify the iov_len pointer of the appropriate extent to a smaller value. Note that you may have received more space than you requested if it was available! @param buf the evbuffer in which to reserve space. @param vec one or two extents returned by evbuffer_reserve_space. @param n_vecs the number of extents. @return 0 on success, -1 on error @see evbuffer_reserve_space() */ EVENT2_EXPORT_SYMBOL int evbuffer_commit_space(struct evbuffer *buf, struct evbuffer_iovec *vec, int n_vecs); /** Append data to the end of an evbuffer. @param buf the evbuffer to be appended to @param data pointer to the beginning of the data buffer @param datlen the number of bytes to be copied from the data buffer @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_add(struct evbuffer *buf, const void *data, size_t datlen); /** Read data from an evbuffer and drain the bytes read. If more bytes are requested than are available in the evbuffer, we only extract as many bytes as were available. @param buf the evbuffer to be read from @param data the destination buffer to store the result @param datlen the maximum size of the destination buffer @return the number of bytes read, or -1 if we can't drain the buffer. */ EVENT2_EXPORT_SYMBOL int evbuffer_remove(struct evbuffer *buf, void *data, size_t datlen); /** Read data from an evbuffer, and leave the buffer unchanged. If more bytes are requested than are available in the evbuffer, we only extract as many bytes as were available. @param buf the evbuffer to be read from @param data_out the destination buffer to store the result @param datlen the maximum size of the destination buffer @return the number of bytes read, or -1 if we can't drain the buffer. */ EVENT2_EXPORT_SYMBOL ev_ssize_t evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen); /** Read data from the middle of an evbuffer, and leave the buffer unchanged. If more bytes are requested than are available in the evbuffer, we only extract as many bytes as were available. @param buf the evbuffer to be read from @param pos the position to start reading from @param data_out the destination buffer to store the result @param datlen the maximum size of the destination buffer @return the number of bytes read, or -1 if we can't drain the buffer. */ EVENT2_EXPORT_SYMBOL ev_ssize_t evbuffer_copyout_from(struct evbuffer *buf, const struct evbuffer_ptr *pos, void *data_out, size_t datlen); /** Read data from an evbuffer into another evbuffer, draining the bytes from the source buffer. This function avoids copy operations to the extent possible. If more bytes are requested than are available in src, the src buffer is drained completely. @param src the evbuffer to be read from @param dst the destination evbuffer to store the result into @param datlen the maximum numbers of bytes to transfer @return the number of bytes read */ EVENT2_EXPORT_SYMBOL int evbuffer_remove_buffer(struct evbuffer *src, struct evbuffer *dst, size_t datlen); /** Used to tell evbuffer_readln what kind of line-ending to look for. */ enum evbuffer_eol_style { /** Any sequence of CR and LF characters is acceptable as an * EOL. * * Note that this style can produce ambiguous results: the * sequence "CRLF" will be treated as a single EOL if it is * all in the buffer at once, but if you first read a CR from * the network and later read an LF from the network, it will * be treated as two EOLs. */ EVBUFFER_EOL_ANY, /** An EOL is an LF, optionally preceded by a CR. This style is * most useful for implementing text-based internet protocols. */ EVBUFFER_EOL_CRLF, /** An EOL is a CR followed by an LF. */ EVBUFFER_EOL_CRLF_STRICT, /** An EOL is a LF. */ EVBUFFER_EOL_LF, /** An EOL is a NUL character (that is, a single byte with value 0) */ EVBUFFER_EOL_NUL }; /** * Read a single line from an evbuffer. * * Reads a line terminated by an EOL as determined by the evbuffer_eol_style * argument. Returns a newly allocated nul-terminated string; the caller must * free the returned value. The EOL is not included in the returned string. * * @param buffer the evbuffer to read from * @param n_read_out if non-NULL, points to a size_t that is set to the * number of characters in the returned string. This is useful for * strings that can contain NUL characters. * @param eol_style the style of line-ending to use. * @return pointer to a single line, or NULL if an error occurred */ EVENT2_EXPORT_SYMBOL char *evbuffer_readln(struct evbuffer *buffer, size_t *n_read_out, enum evbuffer_eol_style eol_style); /** Move all data from one evbuffer into another evbuffer. This is a destructive add. The data from one buffer moves into the other buffer. However, no unnecessary memory copies occur. @param outbuf the output buffer @param inbuf the input buffer @return 0 if successful, or -1 if an error occurred @see evbuffer_remove_buffer() */ EVENT2_EXPORT_SYMBOL int evbuffer_add_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf); /** Copy data from one evbuffer into another evbuffer. This is a non-destructive add. The data from one buffer is copied into the other buffer. However, no unnecessary memory copies occur. Note that buffers already containing buffer references can't be added to other buffers. @param outbuf the output buffer @param inbuf the input buffer @return 0 if successful, or -1 if an error occurred */ EVENT2_EXPORT_SYMBOL int evbuffer_add_buffer_reference(struct evbuffer *outbuf, struct evbuffer *inbuf); /** A cleanup function for a piece of memory added to an evbuffer by reference. @see evbuffer_add_reference() */ typedef void (*evbuffer_ref_cleanup_cb)(const void *data, size_t datalen, void *extra); /** Reference memory into an evbuffer without copying. The memory needs to remain valid until all the added data has been read. This function keeps just a reference to the memory without actually incurring the overhead of a copy. @param outbuf the output buffer @param data the memory to reference @param datlen how memory to reference @param cleanupfn callback to be invoked when the memory is no longer referenced by this evbuffer. @param cleanupfn_arg optional argument to the cleanup callback @return 0 if successful, or -1 if an error occurred */ EVENT2_EXPORT_SYMBOL int evbuffer_add_reference(struct evbuffer *outbuf, const void *data, size_t datlen, evbuffer_ref_cleanup_cb cleanupfn, void *cleanupfn_arg); /** Copy data from a file into the evbuffer for writing to a socket. This function avoids unnecessary data copies between userland and kernel. If sendfile is available and the EVBUFFER_FLAG_DRAINS_TO_FD flag is set, it uses those functions. Otherwise, it tries to use mmap (or CreateFileMapping on Windows). The function owns the resulting file descriptor and will close it when finished transferring data. The results of using evbuffer_remove() or evbuffer_pullup() on evbuffers whose data was added using this function are undefined. For more fine-grained control, use evbuffer_add_file_segment. @param outbuf the output buffer @param fd the file descriptor @param offset the offset from which to read data @param length how much data to read, or -1 to read as much as possible. (-1 requires that 'fd' support fstat.) @return 0 if successful, or -1 if an error occurred */ EVENT2_EXPORT_SYMBOL int evbuffer_add_file(struct evbuffer *outbuf, int fd, ev_off_t offset, ev_off_t length); /** An evbuffer_file_segment holds a reference to a range of a file -- possibly the whole file! -- for use in writing from an evbuffer to a socket. It could be implemented with mmap, sendfile, splice, or (if all else fails) by just pulling all the data into RAM. A single evbuffer_file_segment can be added more than once, and to more than one evbuffer. */ struct evbuffer_file_segment; /** Flag for creating evbuffer_file_segment: If this flag is set, then when the evbuffer_file_segment is freed and no longer in use by any evbuffer, the underlying fd is closed. */ #define EVBUF_FS_CLOSE_ON_FREE 0x01 /** Flag for creating evbuffer_file_segment: Disable memory-map based implementations. */ #define EVBUF_FS_DISABLE_MMAP 0x02 /** Flag for creating evbuffer_file_segment: Disable direct fd-to-fd implementations (including sendfile and splice). You might want to use this option if data needs to be taken from the evbuffer by any means other than writing it to the network: the sendfile backend is fast, but it only works for sending files directly to the network. */ #define EVBUF_FS_DISABLE_SENDFILE 0x04 /** Flag for creating evbuffer_file_segment: Do not allocate a lock for this segment. If this option is set, then neither the segment nor any evbuffer it is added to may ever be accessed from more than one thread at a time. */ #define EVBUF_FS_DISABLE_LOCKING 0x08 /** A cleanup function for a evbuffer_file_segment added to an evbuffer for reference. */ typedef void (*evbuffer_file_segment_cleanup_cb)( struct evbuffer_file_segment const* seg, int flags, void* arg); /** Create and return a new evbuffer_file_segment for reading data from a file and sending it out via an evbuffer. This function avoids unnecessary data copies between userland and kernel. Where available, it uses sendfile or splice. The file descriptor must not be closed so long as any evbuffer is using this segment. The results of using evbuffer_remove() or evbuffer_pullup() or any other function that reads bytes from an evbuffer on any evbuffer containing the newly returned segment are undefined, unless you pass the EVBUF_FS_DISABLE_SENDFILE flag to this function. @param fd an open file to read from. @param offset an index within the file at which to start reading @param length how much data to read, or -1 to read as much as possible. (-1 requires that 'fd' support fstat.) @param flags any number of the EVBUF_FS_* flags @return a new evbuffer_file_segment, or NULL on failure. **/ EVENT2_EXPORT_SYMBOL struct evbuffer_file_segment *evbuffer_file_segment_new( int fd, ev_off_t offset, ev_off_t length, unsigned flags); /** Free an evbuffer_file_segment It is safe to call this function even if the segment has been added to one or more evbuffers. The evbuffer_file_segment will not be freed until no more references to it exist. */ EVENT2_EXPORT_SYMBOL void evbuffer_file_segment_free(struct evbuffer_file_segment *seg); /** Add cleanup callback and argument for the callback to an evbuffer_file_segment. The cleanup callback will be invoked when no more references to the evbuffer_file_segment exist. **/ EVENT2_EXPORT_SYMBOL void evbuffer_file_segment_add_cleanup_cb(struct evbuffer_file_segment *seg, evbuffer_file_segment_cleanup_cb cb, void* arg); /** Insert some or all of an evbuffer_file_segment at the end of an evbuffer Note that the offset and length parameters of this function have a different meaning from those provided to evbuffer_file_segment_new: When you create the segment, the offset is the offset _within the file_, and the length is the length _of the segment_, whereas when you add a segment to an evbuffer, the offset is _within the segment_ and the length is the length of the _part of the segment you want to use. In other words, if you have a 10 KiB file, and you create an evbuffer_file_segment for it with offset 20 and length 1000, it will refer to bytes 20..1019 inclusive. If you then pass this segment to evbuffer_add_file_segment and specify an offset of 20 and a length of 50, you will be adding bytes 40..99 inclusive. @param buf the evbuffer to append to @param seg the segment to add @param offset the offset within the segment to start from @param length the amount of data to add, or -1 to add it all. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_add_file_segment(struct evbuffer *buf, struct evbuffer_file_segment *seg, ev_off_t offset, ev_off_t length); /** Append a formatted string to the end of an evbuffer. The string is formated as printf. @param buf the evbuffer that will be appended to @param fmt a format string @param ... arguments that will be passed to printf(3) @return The number of bytes added if successful, or -1 if an error occurred. @see evutil_printf(), evbuffer_add_vprintf() */ EVENT2_EXPORT_SYMBOL int evbuffer_add_printf(struct evbuffer *buf, const char *fmt, ...) #ifdef __GNUC__ __attribute__((format(printf, 2, 3))) #endif ; /** Append a va_list formatted string to the end of an evbuffer. @param buf the evbuffer that will be appended to @param fmt a format string @param ap a varargs va_list argument array that will be passed to vprintf(3) @return The number of bytes added if successful, or -1 if an error occurred. */ EVENT2_EXPORT_SYMBOL int evbuffer_add_vprintf(struct evbuffer *buf, const char *fmt, va_list ap) #ifdef __GNUC__ __attribute__((format(printf, 2, 0))) #endif ; /** Remove a specified number of bytes data from the beginning of an evbuffer. @param buf the evbuffer to be drained @param len the number of bytes to drain from the beginning of the buffer @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_drain(struct evbuffer *buf, size_t len); /** Write the contents of an evbuffer to a file descriptor. The evbuffer will be drained after the bytes have been successfully written. @param buffer the evbuffer to be written and drained @param fd the file descriptor to be written to @return the number of bytes written, or -1 if an error occurred @see evbuffer_read() */ EVENT2_EXPORT_SYMBOL int evbuffer_write(struct evbuffer *buffer, evutil_socket_t fd); /** Write some of the contents of an evbuffer to a file descriptor. The evbuffer will be drained after the bytes have been successfully written. @param buffer the evbuffer to be written and drained @param fd the file descriptor to be written to @param howmuch the largest allowable number of bytes to write, or -1 to write as many bytes as we can. @return the number of bytes written, or -1 if an error occurred @see evbuffer_read() */ EVENT2_EXPORT_SYMBOL int evbuffer_write_atmost(struct evbuffer *buffer, evutil_socket_t fd, ev_ssize_t howmuch); /** Read from a file descriptor and store the result in an evbuffer. @param buffer the evbuffer to store the result @param fd the file descriptor to read from @param howmuch the number of bytes to be read @return the number of bytes read, or -1 if an error occurred @see evbuffer_write() */ EVENT2_EXPORT_SYMBOL int evbuffer_read(struct evbuffer *buffer, evutil_socket_t fd, int howmuch); /** Search for a string within an evbuffer. @param buffer the evbuffer to be searched @param what the string to be searched for @param len the length of the search string @param start NULL or a pointer to a valid struct evbuffer_ptr. @return a struct evbuffer_ptr whose 'pos' field has the offset of the first occurrence of the string in the buffer after 'start'. The 'pos' field of the result is -1 if the string was not found. */ EVENT2_EXPORT_SYMBOL struct evbuffer_ptr evbuffer_search(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start); /** Search for a string within part of an evbuffer. @param buffer the evbuffer to be searched @param what the string to be searched for @param len the length of the search string @param start NULL or a pointer to a valid struct evbuffer_ptr that indicates where we should start searching. @param end NULL or a pointer to a valid struct evbuffer_ptr that indicates where we should stop searching. @return a struct evbuffer_ptr whose 'pos' field has the offset of the first occurrence of the string in the buffer after 'start'. The 'pos' field of the result is -1 if the string was not found. */ EVENT2_EXPORT_SYMBOL struct evbuffer_ptr evbuffer_search_range(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start, const struct evbuffer_ptr *end); /** Defines how to adjust an evbuffer_ptr by evbuffer_ptr_set() @see evbuffer_ptr_set() */ enum evbuffer_ptr_how { /** Sets the pointer to the position; can be called on with an uninitialized evbuffer_ptr. */ EVBUFFER_PTR_SET, /** Advances the pointer by adding to the current position. */ EVBUFFER_PTR_ADD }; /** Sets the search pointer in the buffer to position. There are two ways to use this function: you can call evbuffer_ptr_set(buf, &pos, N, EVBUFFER_PTR_SET) to move 'pos' to a position 'N' bytes after the start of the buffer, or evbuffer_ptr_set(buf, &pos, N, EVBUFFER_PTR_ADD) to move 'pos' forward by 'N' bytes. If evbuffer_ptr is not initialized, this function can only be called with EVBUFFER_PTR_SET. An evbuffer_ptr can represent any position from the start of the buffer to a position immediately after the end of the buffer. @param buffer the evbuffer to be search @param ptr a pointer to a struct evbuffer_ptr @param position the position at which to start the next search @param how determines how the pointer should be manipulated. @returns 0 on success or -1 otherwise */ EVENT2_EXPORT_SYMBOL int evbuffer_ptr_set(struct evbuffer *buffer, struct evbuffer_ptr *ptr, size_t position, enum evbuffer_ptr_how how); /** Search for an end-of-line string within an evbuffer. @param buffer the evbuffer to be searched @param start NULL or a pointer to a valid struct evbuffer_ptr to start searching at. @param eol_len_out If non-NULL, the pointed-to value will be set to the length of the end-of-line string. @param eol_style The kind of EOL to look for; see evbuffer_readln() for more information @return a struct evbuffer_ptr whose 'pos' field has the offset of the first occurrence EOL in the buffer after 'start'. The 'pos' field of the result is -1 if the string was not found. */ EVENT2_EXPORT_SYMBOL struct evbuffer_ptr evbuffer_search_eol(struct evbuffer *buffer, struct evbuffer_ptr *start, size_t *eol_len_out, enum evbuffer_eol_style eol_style); /** Function to peek at data inside an evbuffer without removing it or copying it out. Pointers to the data are returned by filling the 'vec_out' array with pointers to one or more extents of data inside the buffer. The total data in the extents that you get back may be more than you requested (if there is more data last extent than you asked for), or less (if you do not provide enough evbuffer_iovecs, or if the buffer does not have as much data as you asked to see). @param buffer the evbuffer to peek into, @param len the number of bytes to try to peek. If len is negative, we will try to fill as much of vec_out as we can. If len is negative and vec_out is not provided, we return the number of evbuffer_iovecs that would be needed to get all the data in the buffer. @param start_at an evbuffer_ptr indicating the point at which we should start looking for data. NULL means, "At the start of the buffer." @param vec_out an array of evbuffer_iovec @param n_vec the length of vec_out. If 0, we only count how many extents would be necessary to point to the requested amount of data. @return The number of extents needed. This may be less than n_vec if we didn't need all the evbuffer_iovecs we were given, or more than n_vec if we would need more to return all the data that was requested. */ EVENT2_EXPORT_SYMBOL int evbuffer_peek(struct evbuffer *buffer, ev_ssize_t len, struct evbuffer_ptr *start_at, struct evbuffer_iovec *vec_out, int n_vec); /** Structure passed to an evbuffer_cb_func evbuffer callback @see evbuffer_cb_func, evbuffer_add_cb() */ struct evbuffer_cb_info { /** The number of bytes in this evbuffer when callbacks were last * invoked. */ size_t orig_size; /** The number of bytes added since callbacks were last invoked. */ size_t n_added; /** The number of bytes removed since callbacks were last invoked. */ size_t n_deleted; }; /** Type definition for a callback that is invoked whenever data is added or removed from an evbuffer. An evbuffer may have one or more callbacks set at a time. The order in which they are executed is undefined. A callback function may add more callbacks, or remove itself from the list of callbacks, or add or remove data from the buffer. It may not remove another callback from the list. If a callback adds or removes data from the buffer or from another buffer, this can cause a recursive invocation of your callback or other callbacks. If you ask for an infinite loop, you might just get one: watch out! @param buffer the buffer whose size has changed @param info a structure describing how the buffer changed. @param arg a pointer to user data */ typedef void (*evbuffer_cb_func)(struct evbuffer *buffer, const struct evbuffer_cb_info *info, void *arg); struct evbuffer_cb_entry; /** Add a new callback to an evbuffer. Subsequent calls to evbuffer_add_cb() add new callbacks. To remove this callback, call evbuffer_remove_cb or evbuffer_remove_cb_entry. @param buffer the evbuffer to be monitored @param cb the callback function to invoke when the evbuffer is modified, or NULL to remove all callbacks. @param cbarg an argument to be provided to the callback function @return a handle to the callback on success, or NULL on failure. */ EVENT2_EXPORT_SYMBOL struct evbuffer_cb_entry *evbuffer_add_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg); /** Remove a callback from an evbuffer, given a handle returned from evbuffer_add_cb. Calling this function invalidates the handle. @return 0 if a callback was removed, or -1 if no matching callback was found. */ EVENT2_EXPORT_SYMBOL int evbuffer_remove_cb_entry(struct evbuffer *buffer, struct evbuffer_cb_entry *ent); /** Remove a callback from an evbuffer, given the function and argument used to add it. @return 0 if a callback was removed, or -1 if no matching callback was found. */ EVENT2_EXPORT_SYMBOL int evbuffer_remove_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg); /** If this flag is not set, then a callback is temporarily disabled, and * should not be invoked. * * @see evbuffer_cb_set_flags(), evbuffer_cb_clear_flags() */ #define EVBUFFER_CB_ENABLED 1 /** Change the flags that are set for a callback on a buffer by adding more. @param buffer the evbuffer that the callback is watching. @param cb the callback whose status we want to change. @param flags EVBUFFER_CB_ENABLED to re-enable the callback. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_cb_set_flags(struct evbuffer *buffer, struct evbuffer_cb_entry *cb, ev_uint32_t flags); /** Change the flags that are set for a callback on a buffer by removing some @param buffer the evbuffer that the callback is watching. @param cb the callback whose status we want to change. @param flags EVBUFFER_CB_ENABLED to disable the callback. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_cb_clear_flags(struct evbuffer *buffer, struct evbuffer_cb_entry *cb, ev_uint32_t flags); #if 0 /** Postpone calling a given callback until unsuspend is called later. This is different from disabling the callback, since the callback will get invoked later if the buffer size changes between now and when we unsuspend it. @param the buffer that the callback is watching. @param cb the callback we want to suspend. */ EVENT2_EXPORT_SYMBOL void evbuffer_cb_suspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb); /** Stop postponing a callback that we postponed with evbuffer_cb_suspend. If data was added to or removed from the buffer while the callback was suspended, the callback will get called once now. @param the buffer that the callback is watching. @param cb the callback we want to stop suspending. */ EVENT2_EXPORT_SYMBOL void evbuffer_cb_unsuspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb); #endif /** Makes the data at the beginning of an evbuffer contiguous. @param buf the evbuffer to make contiguous @param size the number of bytes to make contiguous, or -1 to make the entire buffer contiguous. @return a pointer to the contiguous memory array, or NULL if param size requested more data than is present in the buffer. */ EVENT2_EXPORT_SYMBOL unsigned char *evbuffer_pullup(struct evbuffer *buf, ev_ssize_t size); /** Prepends data to the beginning of the evbuffer @param buf the evbuffer to which to prepend data @param data a pointer to the memory to prepend @param size the number of bytes to prepend @return 0 if successful, or -1 otherwise */ EVENT2_EXPORT_SYMBOL int evbuffer_prepend(struct evbuffer *buf, const void *data, size_t size); /** Prepends all data from the src evbuffer to the beginning of the dst evbuffer. @param dst the evbuffer to which to prepend data @param src the evbuffer to prepend; it will be emptied as a result @return 0 if successful, or -1 otherwise */ EVENT2_EXPORT_SYMBOL int evbuffer_prepend_buffer(struct evbuffer *dst, struct evbuffer* src); /** Prevent calls that modify an evbuffer from succeeding. A buffer may frozen at the front, at the back, or at both the front and the back. If the front of a buffer is frozen, operations that drain data from the front of the buffer, or that prepend data to the buffer, will fail until it is unfrozen. If the back a buffer is frozen, operations that append data from the buffer will fail until it is unfrozen. @param buf The buffer to freeze @param at_front If true, we freeze the front of the buffer. If false, we freeze the back. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_freeze(struct evbuffer *buf, int at_front); /** Re-enable calls that modify an evbuffer. @param buf The buffer to un-freeze @param at_front If true, we unfreeze the front of the buffer. If false, we unfreeze the back. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evbuffer_unfreeze(struct evbuffer *buf, int at_front); struct event_base; /** Force all the callbacks on an evbuffer to be run, not immediately after the evbuffer is altered, but instead from inside the event loop. This can be used to serialize all the callbacks to a single thread of execution. */ EVENT2_EXPORT_SYMBOL int evbuffer_defer_callbacks(struct evbuffer *buffer, struct event_base *base); /** Append data from 1 or more iovec's to an evbuffer Calculates the number of bytes needed for an iovec structure and guarantees all data will fit into a single chain. Can be used in lieu of functionality which calls evbuffer_add() constantly before being used to increase performance. @param buffer the destination buffer @param vec the source iovec @param n_vec the number of iovec structures. @return the number of bytes successfully written to the output buffer. */ EVENT2_EXPORT_SYMBOL size_t evbuffer_add_iovec(struct evbuffer * buffer, struct evbuffer_iovec * vec, int n_vec); #ifdef __cplusplus } #endif #endif /* EVENT2_BUFFER_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/buffer_compat.h000066400000000000000000000111341527052371200256110ustar00rootroot00000000000000/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_BUFFER_COMPAT_H_INCLUDED_ #define EVENT2_BUFFER_COMPAT_H_INCLUDED_ #include /** @file event2/buffer_compat.h Obsolete and deprecated versions of the functions in buffer.h: provided only for backward compatibility. */ /** Obsolete alias for evbuffer_readln(buffer, NULL, EVBUFFER_EOL_ANY). @deprecated This function is deprecated because its behavior is not correct for almost any protocol, and also because it's wholly subsumed by evbuffer_readln(). @param buffer the evbuffer to read from @return pointer to a single line, or NULL if an error occurred */ EVENT2_EXPORT_SYMBOL char *evbuffer_readline(struct evbuffer *buffer); /** Type definition for a callback that is invoked whenever data is added or removed from an evbuffer. An evbuffer may have one or more callbacks set at a time. The order in which they are executed is undefined. A callback function may add more callbacks, or remove itself from the list of callbacks, or add or remove data from the buffer. It may not remove another callback from the list. If a callback adds or removes data from the buffer or from another buffer, this can cause a recursive invocation of your callback or other callbacks. If you ask for an infinite loop, you might just get one: watch out! @param buffer the buffer whose size has changed @param old_len the previous length of the buffer @param new_len the current length of the buffer @param arg a pointer to user data */ typedef void (*evbuffer_cb)(struct evbuffer *buffer, size_t old_len, size_t new_len, void *arg); /** Replace all callbacks on an evbuffer with a single new callback, or remove them. Subsequent calls to evbuffer_setcb() replace callbacks set by previous calls. Setting the callback to NULL removes any previously set callback. @deprecated This function is deprecated because it clears all previous callbacks set on the evbuffer, which can cause confusing behavior if multiple parts of the code all want to add their own callbacks on a buffer. Instead, use evbuffer_add(), evbuffer_del(), and evbuffer_setflags() to manage your own evbuffer callbacks without interfering with callbacks set by others. @param buffer the evbuffer to be monitored @param cb the callback function to invoke when the evbuffer is modified, or NULL to remove all callbacks. @param cbarg an argument to be provided to the callback function */ EVENT2_EXPORT_SYMBOL void evbuffer_setcb(struct evbuffer *buffer, evbuffer_cb cb, void *cbarg); /** Find a string within an evbuffer. @param buffer the evbuffer to be searched @param what the string to be searched for @param len the length of the search string @return a pointer to the beginning of the search string, or NULL if the search failed. */ EVENT2_EXPORT_SYMBOL unsigned char *evbuffer_find(struct evbuffer *buffer, const unsigned char *what, size_t len); /** deprecated in favor of calling the functions directly */ #define EVBUFFER_LENGTH(x) evbuffer_get_length(x) /** deprecated in favor of calling the functions directly */ #define EVBUFFER_DATA(x) evbuffer_pullup((x), -1) #endif kytensor-2.49.0.7/third_party/libevent/include/event2/bufferevent.h000066400000000000000000001033261527052371200253150ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_BUFFEREVENT_H_INCLUDED_ #define EVENT2_BUFFEREVENT_H_INCLUDED_ /** @file event2/bufferevent.h Functions for buffering data for network sending or receiving. Bufferevents are higher level than evbuffers: each has an underlying evbuffer for reading and one for writing, and callbacks that are invoked under certain circumstances. A bufferevent provides input and output buffers that get filled and drained automatically. The user of a bufferevent no longer deals directly with the I/O, but instead is reading from input and writing to output buffers. Once initialized, the bufferevent structure can be used repeatedly with bufferevent_enable() and bufferevent_disable(). When reading is enabled, the bufferevent will try to read from the file descriptor onto its input buffer, and call the read callback. When writing is enabled, the bufferevent will try to write data onto its file descriptor when the output buffer has enough data, and call the write callback when the output buffer is sufficiently drained. Bufferevents come in several flavors, including:
Socket-based bufferevents
A bufferevent that reads and writes data onto a network socket. Created with bufferevent_socket_new().
Paired bufferevents
A pair of bufferevents that send and receive data to one another without touching the network. Created with bufferevent_pair_new().
Filtering bufferevents
A bufferevent that transforms data, and sends or receives it over another underlying bufferevent. Created with bufferevent_filter_new().
SSL-backed bufferevents
A bufferevent that uses the openssl library to send and receive data over an encrypted connection. Created with bufferevent_openssl_socket_new() or bufferevent_openssl_filter_new().
*/ #include #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include /** @name Bufferevent event codes These flags are passed as arguments to a bufferevent's event callback. @{ */ #define BEV_EVENT_READING 0x01 /**< error encountered while reading */ #define BEV_EVENT_WRITING 0x02 /**< error encountered while writing */ #define BEV_EVENT_EOF 0x10 /**< eof file reached */ #define BEV_EVENT_ERROR 0x20 /**< unrecoverable error encountered */ #define BEV_EVENT_TIMEOUT 0x40 /**< user-specified timeout reached */ #define BEV_EVENT_CONNECTED 0x80 /**< connect operation finished. */ /**@}*/ /** An opaque type for handling buffered IO @see event2/bufferevent.h */ struct bufferevent #ifdef EVENT_IN_DOXYGEN_ {} #endif ; struct event_base; struct evbuffer; struct sockaddr; /** A read or write callback for a bufferevent. The read callback is triggered when new data arrives in the input buffer and the amount of readable data exceed the low watermark which is 0 by default. The write callback is triggered if the write buffer has been exhausted or fell below its low watermark. @param bev the bufferevent that triggered the callback @param ctx the user-specified context for this bufferevent */ typedef void (*bufferevent_data_cb)(struct bufferevent *bev, void *ctx); /** An event/error callback for a bufferevent. The event callback is triggered if either an EOF condition or another unrecoverable error was encountered. For bufferevents with deferred callbacks, this is a bitwise OR of all errors that have happened on the bufferevent since the last callback invocation. @param bev the bufferevent for which the error condition was reached @param what a conjunction of flags: BEV_EVENT_READING or BEV_EVENT_WRITING to indicate if the error was encountered on the read or write path, and one of the following flags: BEV_EVENT_EOF, BEV_EVENT_ERROR, BEV_EVENT_TIMEOUT, BEV_EVENT_CONNECTED. @param ctx the user-specified context for this bufferevent */ typedef void (*bufferevent_event_cb)(struct bufferevent *bev, short what, void *ctx); /** Options that can be specified when creating a bufferevent */ enum bufferevent_options { /** If set, we close the underlying file * descriptor/bufferevent/whatever when this bufferevent is freed. */ BEV_OPT_CLOSE_ON_FREE = (1<<0), /** If set, and threading is enabled, operations on this bufferevent * are protected by a lock */ BEV_OPT_THREADSAFE = (1<<1), /** If set, callbacks are run deferred in the event loop. */ BEV_OPT_DEFER_CALLBACKS = (1<<2), /** If set, callbacks are executed without locks being held on the * bufferevent. This option currently requires that * BEV_OPT_DEFER_CALLBACKS also be set; a future version of Libevent * might remove the requirement.*/ BEV_OPT_UNLOCK_CALLBACKS = (1<<3) }; /** Create a new socket bufferevent over an existing socket. @param base the event base to associate with the new bufferevent. @param fd the file descriptor from which data is read and written to. This file descriptor is not allowed to be a pipe(2). It is safe to set the fd to -1, so long as you later set it with bufferevent_setfd or bufferevent_socket_connect(). @param options Zero or more BEV_OPT_* flags @return a pointer to a newly allocated bufferevent struct, or NULL if an error occurred @see bufferevent_free() */ EVENT2_EXPORT_SYMBOL struct bufferevent *bufferevent_socket_new(struct event_base *base, evutil_socket_t fd, int options); /** Launch a connect() attempt with a socket-based bufferevent. When the connect succeeds, the eventcb will be invoked with BEV_EVENT_CONNECTED set. If the bufferevent does not already have a socket set, we allocate a new socket here and make it nonblocking before we begin. If no address is provided, we assume that the socket is already connecting, and configure the bufferevent so that a BEV_EVENT_CONNECTED event will be yielded when it is done connecting. @param bufev an existing bufferevent allocated with bufferevent_socket_new(). @param addr the address we should connect to @param socklen The length of the address @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int bufferevent_socket_connect(struct bufferevent *, const struct sockaddr *, int); struct evdns_base; /** Resolve the hostname 'hostname' and connect to it as with bufferevent_socket_connect(). @param bufev An existing bufferevent allocated with bufferevent_socket_new() @param evdns_base Optionally, an evdns_base to use for resolving hostnames asynchronously. May be set to NULL for a blocking resolve. @param family A preferred address family to resolve addresses to, or AF_UNSPEC for no preference. Only AF_INET, AF_INET6, and AF_UNSPEC are supported. @param hostname The hostname to resolve; see below for notes on recognized formats @param port The port to connect to on the resolved address. @return 0 if successful, -1 on failure. Recognized hostname formats are: www.example.com (hostname) 1.2.3.4 (ipv4address) ::1 (ipv6address) [::1] ([ipv6address]) Performance note: If you do not provide an evdns_base, this function may block while it waits for a DNS response. This is probably not what you want. */ EVENT2_EXPORT_SYMBOL int bufferevent_socket_connect_hostname(struct bufferevent *, struct evdns_base *, int, const char *, int); /** Return the error code for the last failed DNS lookup attempt made by bufferevent_socket_connect_hostname(). @param bev The bufferevent object. @return DNS error code. @see evutil_gai_strerror() */ EVENT2_EXPORT_SYMBOL int bufferevent_socket_get_dns_error(struct bufferevent *bev); /** Assign a bufferevent to a specific event_base. NOTE that only socket bufferevents support this function. @param base an event_base returned by event_init() @param bufev a bufferevent struct returned by bufferevent_new() or bufferevent_socket_new() @return 0 if successful, or -1 if an error occurred @see bufferevent_new() */ EVENT2_EXPORT_SYMBOL int bufferevent_base_set(struct event_base *base, struct bufferevent *bufev); /** Return the event_base used by a bufferevent */ EVENT2_EXPORT_SYMBOL struct event_base *bufferevent_get_base(struct bufferevent *bev); /** Assign a priority to a bufferevent. Only supported for socket bufferevents. @param bufev a bufferevent struct @param pri the priority to be assigned @return 0 if successful, or -1 if an error occurred */ EVENT2_EXPORT_SYMBOL int bufferevent_priority_set(struct bufferevent *bufev, int pri); /** Return the priority of a bufferevent. Only supported for socket bufferevents */ EVENT2_EXPORT_SYMBOL int bufferevent_get_priority(const struct bufferevent *bufev); /** Deallocate the storage associated with a bufferevent structure. If there is pending data to write on the bufferevent, it probably won't be flushed before the bufferevent is freed. @param bufev the bufferevent structure to be freed. */ EVENT2_EXPORT_SYMBOL void bufferevent_free(struct bufferevent *bufev); /** Changes the callbacks for a bufferevent. @param bufev the bufferevent object for which to change callbacks @param readcb callback to invoke when there is data to be read, or NULL if no callback is desired @param writecb callback to invoke when the file descriptor is ready for writing, or NULL if no callback is desired @param eventcb callback to invoke when there is an event on the file descriptor @param cbarg an argument that will be supplied to each of the callbacks (readcb, writecb, and errorcb) @see bufferevent_new() */ EVENT2_EXPORT_SYMBOL void bufferevent_setcb(struct bufferevent *bufev, bufferevent_data_cb readcb, bufferevent_data_cb writecb, bufferevent_event_cb eventcb, void *cbarg); /** Retrieves the callbacks for a bufferevent. @param bufev the bufferevent to examine. @param readcb_ptr if readcb_ptr is nonnull, *readcb_ptr is set to the current read callback for the bufferevent. @param writecb_ptr if writecb_ptr is nonnull, *writecb_ptr is set to the current write callback for the bufferevent. @param eventcb_ptr if eventcb_ptr is nonnull, *eventcb_ptr is set to the current event callback for the bufferevent. @param cbarg_ptr if cbarg_ptr is nonnull, *cbarg_ptr is set to the current callback argument for the bufferevent. @see buffervent_setcb() */ EVENT2_EXPORT_SYMBOL void bufferevent_getcb(struct bufferevent *bufev, bufferevent_data_cb *readcb_ptr, bufferevent_data_cb *writecb_ptr, bufferevent_event_cb *eventcb_ptr, void **cbarg_ptr); /** Changes the file descriptor on which the bufferevent operates. Not supported for all bufferevent types. @param bufev the bufferevent object for which to change the file descriptor @param fd the file descriptor to operate on */ EVENT2_EXPORT_SYMBOL int bufferevent_setfd(struct bufferevent *bufev, evutil_socket_t fd); /** Returns the file descriptor associated with a bufferevent, or -1 if no file descriptor is associated with the bufferevent. */ EVENT2_EXPORT_SYMBOL evutil_socket_t bufferevent_getfd(struct bufferevent *bufev); /** Returns the underlying bufferevent associated with a bufferevent (if the bufferevent is a wrapper), or NULL if there is no underlying bufferevent. */ EVENT2_EXPORT_SYMBOL struct bufferevent *bufferevent_get_underlying(struct bufferevent *bufev); /** Write data to a bufferevent buffer. The bufferevent_write() function can be used to write data to the file descriptor. The data is appended to the output buffer and written to the descriptor automatically as it becomes available for writing. @param bufev the bufferevent to be written to @param data a pointer to the data to be written @param size the length of the data, in bytes @return 0 if successful, or -1 if an error occurred @see bufferevent_write_buffer() */ EVENT2_EXPORT_SYMBOL int bufferevent_write(struct bufferevent *bufev, const void *data, size_t size); /** Write data from an evbuffer to a bufferevent buffer. The evbuffer is being drained as a result. @param bufev the bufferevent to be written to @param buf the evbuffer to be written @return 0 if successful, or -1 if an error occurred @see bufferevent_write() */ EVENT2_EXPORT_SYMBOL int bufferevent_write_buffer(struct bufferevent *bufev, struct evbuffer *buf); /** Read data from a bufferevent buffer. The bufferevent_read() function is used to read data from the input buffer. @param bufev the bufferevent to be read from @param data pointer to a buffer that will store the data @param size the size of the data buffer, in bytes @return the amount of data read, in bytes. */ EVENT2_EXPORT_SYMBOL size_t bufferevent_read(struct bufferevent *bufev, void *data, size_t size); /** Read data from a bufferevent buffer into an evbuffer. This avoids memory copies. @param bufev the bufferevent to be read from @param buf the evbuffer to which to add data @return 0 if successful, or -1 if an error occurred. */ EVENT2_EXPORT_SYMBOL int bufferevent_read_buffer(struct bufferevent *bufev, struct evbuffer *buf); /** Returns the input buffer. The user MUST NOT set the callback on this buffer. @param bufev the bufferevent from which to get the evbuffer @return the evbuffer object for the input buffer */ EVENT2_EXPORT_SYMBOL struct evbuffer *bufferevent_get_input(struct bufferevent *bufev); /** Returns the output buffer. The user MUST NOT set the callback on this buffer. When filters are being used, the filters need to be manually triggered if the output buffer was manipulated. @param bufev the bufferevent from which to get the evbuffer @return the evbuffer object for the output buffer */ EVENT2_EXPORT_SYMBOL struct evbuffer *bufferevent_get_output(struct bufferevent *bufev); /** Enable a bufferevent. @param bufev the bufferevent to be enabled @param event any combination of EV_READ | EV_WRITE. @return 0 if successful, or -1 if an error occurred @see bufferevent_disable() */ EVENT2_EXPORT_SYMBOL int bufferevent_enable(struct bufferevent *bufev, short event); /** Disable a bufferevent. @param bufev the bufferevent to be disabled @param event any combination of EV_READ | EV_WRITE. @return 0 if successful, or -1 if an error occurred @see bufferevent_enable() */ EVENT2_EXPORT_SYMBOL int bufferevent_disable(struct bufferevent *bufev, short event); /** Return the events that are enabled on a given bufferevent. @param bufev the bufferevent to inspect @return A combination of EV_READ | EV_WRITE */ EVENT2_EXPORT_SYMBOL short bufferevent_get_enabled(struct bufferevent *bufev); /** Set the read and write timeout for a bufferevent. A bufferevent's timeout will fire the first time that the indicated amount of time has elapsed since a successful read or write operation, during which the bufferevent was trying to read or write. (In other words, if reading or writing is disabled, or if the bufferevent's read or write operation has been suspended because there's no data to write, or not enough banwidth, or so on, the timeout isn't active. The timeout only becomes active when we we're willing to actually read or write.) Calling bufferevent_enable or setting a timeout for a bufferevent whose timeout is already pending resets its timeout. If the timeout elapses, the corresponding operation (EV_READ or EV_WRITE) becomes disabled until you re-enable it again. The bufferevent's event callback is called with the BEV_EVENT_TIMEOUT|BEV_EVENT_READING or BEV_EVENT_TIMEOUT|BEV_EVENT_WRITING. @param bufev the bufferevent to be modified @param timeout_read the read timeout, or NULL @param timeout_write the write timeout, or NULL */ EVENT2_EXPORT_SYMBOL int bufferevent_set_timeouts(struct bufferevent *bufev, const struct timeval *timeout_read, const struct timeval *timeout_write); /** Sets the watermarks for read and write events. On input, a bufferevent does not invoke the user read callback unless there is at least low watermark data in the buffer. If the read buffer is beyond the high watermark, the bufferevent stops reading from the network. On output, the user write callback is invoked whenever the buffered data falls below the low watermark. Filters that write to this bufev will try not to write more bytes to this buffer than the high watermark would allow, except when flushing. @param bufev the bufferevent to be modified @param events EV_READ, EV_WRITE or both @param lowmark the lower watermark to set @param highmark the high watermark to set */ EVENT2_EXPORT_SYMBOL void bufferevent_setwatermark(struct bufferevent *bufev, short events, size_t lowmark, size_t highmark); /** Retrieves the watermarks for read or write events. Returns non-zero if events contains not only EV_READ or EV_WRITE. Returns zero if events equal EV_READ or EV_WRITE @param bufev the bufferevent to be examined @param events EV_READ or EV_WRITE @param lowmark receives the lower watermark if not NULL @param highmark receives the high watermark if not NULL */ EVENT2_EXPORT_SYMBOL int bufferevent_getwatermark(struct bufferevent *bufev, short events, size_t *lowmark, size_t *highmark); /** Acquire the lock on a bufferevent. Has no effect if locking was not enabled with BEV_OPT_THREADSAFE. */ EVENT2_EXPORT_SYMBOL void bufferevent_lock(struct bufferevent *bufev); /** Release the lock on a bufferevent. Has no effect if locking was not enabled with BEV_OPT_THREADSAFE. */ EVENT2_EXPORT_SYMBOL void bufferevent_unlock(struct bufferevent *bufev); /** * Public interface to manually increase the reference count of a bufferevent * this is useful in situations where a user may reference the bufferevent * somewhere eles (unknown to libevent) * * @param bufev the bufferevent to increase the refcount on * */ EVENT2_EXPORT_SYMBOL void bufferevent_incref(struct bufferevent *bufev); /** * Public interface to manually decrement the reference count of a bufferevent * * Warning: make sure you know what you're doing. This is mainly used in * conjunction with bufferevent_incref(). This will free up all data associated * with a bufferevent if the reference count hits 0. * * @param bufev the bufferevent to decrement the refcount on * * @return 1 if the bufferevent was freed, otherwise 0 (still referenced) */ EVENT2_EXPORT_SYMBOL int bufferevent_decref(struct bufferevent *bufev); /** Flags that can be passed into filters to let them know how to deal with the incoming data. */ enum bufferevent_flush_mode { /** usually set when processing data */ BEV_NORMAL = 0, /** want to checkpoint all data sent. */ BEV_FLUSH = 1, /** encountered EOF on read or done sending data */ BEV_FINISHED = 2 }; /** Triggers the bufferevent to produce more data if possible. @param bufev the bufferevent object @param iotype either EV_READ or EV_WRITE or both. @param mode either BEV_NORMAL or BEV_FLUSH or BEV_FINISHED @return -1 on failure, 0 if no data was produces, 1 if data was produced */ EVENT2_EXPORT_SYMBOL int bufferevent_flush(struct bufferevent *bufev, short iotype, enum bufferevent_flush_mode mode); /** Flags for bufferevent_trigger(_event) that modify when and how to trigger the callback. */ enum bufferevent_trigger_options { /** trigger the callback regardless of the watermarks */ BEV_TRIG_IGNORE_WATERMARKS = (1<<16), /** defer even if the callbacks are not */ BEV_TRIG_DEFER_CALLBACKS = BEV_OPT_DEFER_CALLBACKS /* (Note: for internal reasons, these need to be disjoint from * bufferevent_options, except when they mean the same thing. */ }; /** Triggers bufferevent data callbacks. The function will honor watermarks unless options contain BEV_TRIG_IGNORE_WATERMARKS. If the options contain BEV_OPT_DEFER_CALLBACKS, the callbacks are deferred. @param bufev the bufferevent object @param iotype either EV_READ or EV_WRITE or both. @param options */ EVENT2_EXPORT_SYMBOL void bufferevent_trigger(struct bufferevent *bufev, short iotype, int options); /** Triggers the bufferevent event callback. If the options contain BEV_OPT_DEFER_CALLBACKS, the callbacks are deferred. @param bufev the bufferevent object @param what the flags to pass onto the event callback @param options */ EVENT2_EXPORT_SYMBOL void bufferevent_trigger_event(struct bufferevent *bufev, short what, int options); /** @name Filtering support @{ */ /** Values that filters can return. */ enum bufferevent_filter_result { /** everything is okay */ BEV_OK = 0, /** the filter needs to read more data before output */ BEV_NEED_MORE = 1, /** the filter encountered a critical error, no further data can be processed. */ BEV_ERROR = 2 }; /** A callback function to implement a filter for a bufferevent. @param src An evbuffer to drain data from. @param dst An evbuffer to add data to. @param limit A suggested upper bound of bytes to write to dst. The filter may ignore this value, but doing so means that it will overflow the high-water mark associated with dst. -1 means "no limit". @param mode Whether we should write data as may be convenient (BEV_NORMAL), or flush as much data as we can (BEV_FLUSH), or flush as much as we can, possibly including an end-of-stream marker (BEV_FINISH). @param ctx A user-supplied pointer. @return BEV_OK if we wrote some data; BEV_NEED_MORE if we can't produce any more output until we get some input; and BEV_ERROR on an error. */ typedef enum bufferevent_filter_result (*bufferevent_filter_cb)( struct evbuffer *src, struct evbuffer *dst, ev_ssize_t dst_limit, enum bufferevent_flush_mode mode, void *ctx); /** Allocate a new filtering bufferevent on top of an existing bufferevent. @param underlying the underlying bufferevent. @param input_filter The filter to apply to data we read from the underlying bufferevent @param output_filter The filer to apply to data we write to the underlying bufferevent @param options A bitfield of bufferevent options. @param free_context A function to use to free the filter context when this bufferevent is freed. @param ctx A context pointer to pass to the filter functions. */ EVENT2_EXPORT_SYMBOL struct bufferevent * bufferevent_filter_new(struct bufferevent *underlying, bufferevent_filter_cb input_filter, bufferevent_filter_cb output_filter, int options, void (*free_context)(void *), void *ctx); /**@}*/ /** Allocate a pair of linked bufferevents. The bufferevents behave as would two bufferevent_sock instances connected to opposite ends of a socketpair(), except that no internal socketpair is allocated. @param base The event base to associate with the socketpair. @param options A set of options for this bufferevent @param pair A pointer to an array to hold the two new bufferevent objects. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int bufferevent_pair_new(struct event_base *base, int options, struct bufferevent *pair[2]); /** Given one bufferevent returned by bufferevent_pair_new(), returns the other one if it still exists. Otherwise returns NULL. */ EVENT2_EXPORT_SYMBOL struct bufferevent *bufferevent_pair_get_partner(struct bufferevent *bev); /** Abstract type used to configure rate-limiting on a bufferevent or a group of bufferevents. */ struct ev_token_bucket_cfg; /** A group of bufferevents which are configured to respect the same rate limit. */ struct bufferevent_rate_limit_group; /** Maximum configurable rate- or burst-limit. */ #define EV_RATE_LIMIT_MAX EV_SSIZE_MAX /** Initialize and return a new object to configure the rate-limiting behavior of bufferevents. @param read_rate The maximum number of bytes to read per tick on average. @param read_burst The maximum number of bytes to read in any single tick. @param write_rate The maximum number of bytes to write per tick on average. @param write_burst The maximum number of bytes to write in any single tick. @param tick_len The length of a single tick. Defaults to one second. Any fractions of a millisecond are ignored. Note that all rate-limits hare are currently best-effort: future versions of Libevent may implement them more tightly. */ EVENT2_EXPORT_SYMBOL struct ev_token_bucket_cfg *ev_token_bucket_cfg_new( size_t read_rate, size_t read_burst, size_t write_rate, size_t write_burst, const struct timeval *tick_len); /** Free all storage held in 'cfg'. Note: 'cfg' is not currently reference-counted; it is not safe to free it until no bufferevent is using it. */ EVENT2_EXPORT_SYMBOL void ev_token_bucket_cfg_free(struct ev_token_bucket_cfg *cfg); /** Set the rate-limit of a the bufferevent 'bev' to the one specified in 'cfg'. If 'cfg' is NULL, disable any per-bufferevent rate-limiting on 'bev'. Note that only some bufferevent types currently respect rate-limiting. They are: socket-based bufferevents (normal and IOCP-based), and SSL-based bufferevents. Return 0 on sucess, -1 on failure. */ EVENT2_EXPORT_SYMBOL int bufferevent_set_rate_limit(struct bufferevent *bev, struct ev_token_bucket_cfg *cfg); /** Create a new rate-limit group for bufferevents. A rate-limit group constrains the maximum number of bytes sent and received, in toto, by all of its bufferevents. @param base An event_base to run any necessary timeouts for the group. Note that all bufferevents in the group do not necessarily need to share this event_base. @param cfg The rate-limit for this group. Note that all rate-limits hare are currently best-effort: future versions of Libevent may implement them more tightly. Note also that only some bufferevent types currently respect rate-limiting. They are: socket-based bufferevents (normal and IOCP-based), and SSL-based bufferevents. */ EVENT2_EXPORT_SYMBOL struct bufferevent_rate_limit_group *bufferevent_rate_limit_group_new( struct event_base *base, const struct ev_token_bucket_cfg *cfg); /** Change the rate-limiting settings for a given rate-limiting group. Return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int bufferevent_rate_limit_group_set_cfg( struct bufferevent_rate_limit_group *, const struct ev_token_bucket_cfg *); /** Change the smallest quantum we're willing to allocate to any single bufferevent in a group for reading or writing at a time. The rationale is that, because of TCP/IP protocol overheads and kernel behavior, if a rate-limiting group is so tight on bandwidth that you're only willing to send 1 byte per tick per bufferevent, you might instead want to batch up the reads and writes so that you send N bytes per 1/N of the bufferevents (chosen at random) each tick, so you still wind up send 1 byte per tick per bufferevent on average, but you don't send so many tiny packets. The default min-share is currently 64 bytes. Returns 0 on success, -1 on faulre. */ EVENT2_EXPORT_SYMBOL int bufferevent_rate_limit_group_set_min_share( struct bufferevent_rate_limit_group *, size_t); /** Free a rate-limiting group. The group must have no members when this function is called. */ EVENT2_EXPORT_SYMBOL void bufferevent_rate_limit_group_free(struct bufferevent_rate_limit_group *); /** Add 'bev' to the list of bufferevents whose aggregate reading and writing is restricted by 'g'. If 'g' is NULL, remove 'bev' from its current group. A bufferevent may belong to no more than one rate-limit group at a time. If 'bev' is already a member of a group, it will be removed from its old group before being added to 'g'. Return 0 on success and -1 on failure. */ EVENT2_EXPORT_SYMBOL int bufferevent_add_to_rate_limit_group(struct bufferevent *bev, struct bufferevent_rate_limit_group *g); /** Remove 'bev' from its current rate-limit group (if any). */ EVENT2_EXPORT_SYMBOL int bufferevent_remove_from_rate_limit_group(struct bufferevent *bev); /** Set the size limit for single read operation. Set to 0 for a reasonable default. Return 0 on success and -1 on failure. */ EVENT2_EXPORT_SYMBOL int bufferevent_set_max_single_read(struct bufferevent *bev, size_t size); /** Set the size limit for single write operation. Set to 0 for a reasonable default. Return 0 on success and -1 on failure. */ EVENT2_EXPORT_SYMBOL int bufferevent_set_max_single_write(struct bufferevent *bev, size_t size); /** Get the current size limit for single read operation. */ EVENT2_EXPORT_SYMBOL ev_ssize_t bufferevent_get_max_single_read(struct bufferevent *bev); /** Get the current size limit for single write operation. */ EVENT2_EXPORT_SYMBOL ev_ssize_t bufferevent_get_max_single_write(struct bufferevent *bev); /** @name Rate limit inspection Return the current read or write bucket size for a bufferevent. If it is not configured with a per-bufferevent ratelimit, return EV_SSIZE_MAX. This function does not inspect the group limit, if any. Note that it can return a negative value if the bufferevent has been made to read or write more than its limit. @{ */ EVENT2_EXPORT_SYMBOL ev_ssize_t bufferevent_get_read_limit(struct bufferevent *bev); EVENT2_EXPORT_SYMBOL ev_ssize_t bufferevent_get_write_limit(struct bufferevent *bev); /*@}*/ EVENT2_EXPORT_SYMBOL ev_ssize_t bufferevent_get_max_to_read(struct bufferevent *bev); EVENT2_EXPORT_SYMBOL ev_ssize_t bufferevent_get_max_to_write(struct bufferevent *bev); EVENT2_EXPORT_SYMBOL const struct ev_token_bucket_cfg *bufferevent_get_token_bucket_cfg(const struct bufferevent * bev); /** @name Group Rate limit inspection Return the read or write bucket size for a bufferevent rate limit group. Note that it can return a negative value if bufferevents in the group have been made to read or write more than their limits. @{ */ EVENT2_EXPORT_SYMBOL ev_ssize_t bufferevent_rate_limit_group_get_read_limit( struct bufferevent_rate_limit_group *); EVENT2_EXPORT_SYMBOL ev_ssize_t bufferevent_rate_limit_group_get_write_limit( struct bufferevent_rate_limit_group *); /*@}*/ /** @name Rate limit manipulation Subtract a number of bytes from a bufferevent's read or write bucket. The decrement value can be negative, if you want to manually refill the bucket. If the change puts the bucket above or below zero, the bufferevent will resume or suspend reading writing as appropriate. These functions make no change in the buckets for the bufferevent's group, if any. Returns 0 on success, -1 on internal error. @{ */ EVENT2_EXPORT_SYMBOL int bufferevent_decrement_read_limit(struct bufferevent *bev, ev_ssize_t decr); EVENT2_EXPORT_SYMBOL int bufferevent_decrement_write_limit(struct bufferevent *bev, ev_ssize_t decr); /*@}*/ /** @name Group rate limit manipulation Subtract a number of bytes from a bufferevent rate-limiting group's read or write bucket. The decrement value can be negative, if you want to manually refill the bucket. If the change puts the bucket above or below zero, the bufferevents in the group will resume or suspend reading writing as appropriate. Returns 0 on success, -1 on internal error. @{ */ EVENT2_EXPORT_SYMBOL int bufferevent_rate_limit_group_decrement_read( struct bufferevent_rate_limit_group *, ev_ssize_t); EVENT2_EXPORT_SYMBOL int bufferevent_rate_limit_group_decrement_write( struct bufferevent_rate_limit_group *, ev_ssize_t); /*@}*/ /** * Inspect the total bytes read/written on a group. * * Set the variable pointed to by total_read_out to the total number of bytes * ever read on grp, and the variable pointed to by total_written_out to the * total number of bytes ever written on grp. */ EVENT2_EXPORT_SYMBOL void bufferevent_rate_limit_group_get_totals( struct bufferevent_rate_limit_group *grp, ev_uint64_t *total_read_out, ev_uint64_t *total_written_out); /** * Reset the total bytes read/written on a group. * * Reset the number of bytes read or written on grp as given by * bufferevent_rate_limit_group_reset_totals(). */ EVENT2_EXPORT_SYMBOL void bufferevent_rate_limit_group_reset_totals( struct bufferevent_rate_limit_group *grp); #ifdef __cplusplus } #endif #endif /* EVENT2_BUFFEREVENT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/bufferevent_compat.h000066400000000000000000000105601527052371200266550ustar00rootroot00000000000000/* * Copyright (c) 2007-2012 Niels Provos, Nick Mathewson * Copyright (c) 2000-2007 Niels Provos * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_BUFFEREVENT_COMPAT_H_INCLUDED_ #define EVENT2_BUFFEREVENT_COMPAT_H_INCLUDED_ #define evbuffercb bufferevent_data_cb #define everrorcb bufferevent_event_cb /** Create a new bufferevent for an fd. This function is deprecated. Use bufferevent_socket_new and bufferevent_set_callbacks instead. Libevent provides an abstraction on top of the regular event callbacks. This abstraction is called a buffered event. A buffered event provides input and output buffers that get filled and drained automatically. The user of a buffered event no longer deals directly with the I/O, but instead is reading from input and writing to output buffers. Once initialized, the bufferevent structure can be used repeatedly with bufferevent_enable() and bufferevent_disable(). When read enabled the bufferevent will try to read from the file descriptor and call the read callback. The write callback is executed whenever the output buffer is drained below the write low watermark, which is 0 by default. If multiple bases are in use, bufferevent_base_set() must be called before enabling the bufferevent for the first time. @deprecated This function is deprecated because it uses the current event base, and as such can be error prone for multithreaded programs. Use bufferevent_socket_new() instead. @param fd the file descriptor from which data is read and written to. This file descriptor is not allowed to be a pipe(2). @param readcb callback to invoke when there is data to be read, or NULL if no callback is desired @param writecb callback to invoke when the file descriptor is ready for writing, or NULL if no callback is desired @param errorcb callback to invoke when there is an error on the file descriptor @param cbarg an argument that will be supplied to each of the callbacks (readcb, writecb, and errorcb) @return a pointer to a newly allocated bufferevent struct, or NULL if an error occurred @see bufferevent_base_set(), bufferevent_free() */ struct bufferevent *bufferevent_new(evutil_socket_t fd, evbuffercb readcb, evbuffercb writecb, everrorcb errorcb, void *cbarg); /** Set the read and write timeout for a buffered event. @param bufev the bufferevent to be modified @param timeout_read the read timeout @param timeout_write the write timeout */ void bufferevent_settimeout(struct bufferevent *bufev, int timeout_read, int timeout_write); #define EVBUFFER_READ BEV_EVENT_READING #define EVBUFFER_WRITE BEV_EVENT_WRITING #define EVBUFFER_EOF BEV_EVENT_EOF #define EVBUFFER_ERROR BEV_EVENT_ERROR #define EVBUFFER_TIMEOUT BEV_EVENT_TIMEOUT /** macro for getting access to the input buffer of a bufferevent */ #define EVBUFFER_INPUT(x) bufferevent_get_input(x) /** macro for getting access to the output buffer of a bufferevent */ #define EVBUFFER_OUTPUT(x) bufferevent_get_output(x) #endif kytensor-2.49.0.7/third_party/libevent/include/event2/bufferevent_ssl.h000066400000000000000000000113601527052371200261720ustar00rootroot00000000000000/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_BUFFEREVENT_SSL_H_INCLUDED_ #define EVENT2_BUFFEREVENT_SSL_H_INCLUDED_ /** @file event2/bufferevent_ssl.h OpenSSL support for bufferevents. */ #include #include #include #include #ifdef __cplusplus extern "C" { #endif /* This is what openssl's SSL objects are underneath. */ struct ssl_st; /** The state of an SSL object to be used when creating a new SSL bufferevent. */ enum bufferevent_ssl_state { BUFFEREVENT_SSL_OPEN = 0, BUFFEREVENT_SSL_CONNECTING = 1, BUFFEREVENT_SSL_ACCEPTING = 2 }; #if defined(EVENT__HAVE_OPENSSL) || defined(EVENT_IN_DOXYGEN_) /** Create a new SSL bufferevent to send its data over another bufferevent. @param base An event_base to use to detect reading and writing. It must also be the base for the underlying bufferevent. @param underlying A socket to use for this SSL @param ssl A SSL* object from openssl. @param state The current state of the SSL connection @param options One or more bufferevent_options @return A new bufferevent on success, or NULL on failure */ EVENT2_EXPORT_SYMBOL struct bufferevent * bufferevent_openssl_filter_new(struct event_base *base, struct bufferevent *underlying, struct ssl_st *ssl, enum bufferevent_ssl_state state, int options); /** Create a new SSL bufferevent to send its data over an SSL * on a socket. @param base An event_base to use to detect reading and writing @param fd A socket to use for this SSL @param ssl A SSL* object from openssl. @param state The current state of the SSL connection @param options One or more bufferevent_options @return A new bufferevent on success, or NULL on failure. */ EVENT2_EXPORT_SYMBOL struct bufferevent * bufferevent_openssl_socket_new(struct event_base *base, evutil_socket_t fd, struct ssl_st *ssl, enum bufferevent_ssl_state state, int options); /** Control how to report dirty SSL shutdowns. If the peer (or the network, or an attacker) closes the TCP connection before closing the SSL channel, and the protocol is SSL >= v3, this is a "dirty" shutdown. If allow_dirty_shutdown is 0 (default), this is reported as BEV_EVENT_ERROR. If instead allow_dirty_shutdown=1, a dirty shutdown is reported as BEV_EVENT_EOF. (Note that if the protocol is < SSLv3, you will always receive BEV_EVENT_EOF, since SSL 2 and earlier cannot distinguish a secure connection close from a dirty one. This is one reason (among many) not to use SSL 2.) */ EVENT2_EXPORT_SYMBOL int bufferevent_openssl_get_allow_dirty_shutdown(struct bufferevent *bev); EVENT2_EXPORT_SYMBOL void bufferevent_openssl_set_allow_dirty_shutdown(struct bufferevent *bev, int allow_dirty_shutdown); /** Return the underlying openssl SSL * object for an SSL bufferevent. */ EVENT2_EXPORT_SYMBOL struct ssl_st * bufferevent_openssl_get_ssl(struct bufferevent *bufev); /** Tells a bufferevent to begin SSL renegotiation. */ EVENT2_EXPORT_SYMBOL int bufferevent_ssl_renegotiate(struct bufferevent *bev); /** Return the most recent OpenSSL error reported on an SSL bufferevent. */ EVENT2_EXPORT_SYMBOL unsigned long bufferevent_get_openssl_error(struct bufferevent *bev); #endif #ifdef __cplusplus } #endif #endif /* EVENT2_BUFFEREVENT_SSL_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/bufferevent_struct.h000066400000000000000000000100471527052371200267160ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_BUFFEREVENT_STRUCT_H_INCLUDED_ #define EVENT2_BUFFEREVENT_STRUCT_H_INCLUDED_ /** @file event2/bufferevent_struct.h Data structures for bufferevents. Using these structures may hurt forward compatibility with later versions of Libevent: be careful! @deprecated Use of bufferevent_struct.h is completely deprecated; these structures are only exposed for backward compatibility with programs written before Libevent 2.0 that used them. */ #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include /* For struct event */ #include struct event_watermark { size_t low; size_t high; }; /** Shared implementation of a bufferevent. This type is exposed only because it was exposed in previous versions, and some people's code may rely on manipulating it. Otherwise, you should really not rely on the layout, size, or contents of this structure: it is fairly volatile, and WILL change in future versions of the code. **/ struct bufferevent { /** Event base for which this bufferevent was created. */ struct event_base *ev_base; /** Pointer to a table of function pointers to set up how this bufferevent behaves. */ const struct bufferevent_ops *be_ops; /** A read event that triggers when a timeout has happened or a socket is ready to read data. Only used by some subtypes of bufferevent. */ struct event ev_read; /** A write event that triggers when a timeout has happened or a socket is ready to write data. Only used by some subtypes of bufferevent. */ struct event ev_write; /** An input buffer. Only the bufferevent is allowed to add data to this buffer, though the user is allowed to drain it. */ struct evbuffer *input; /** An input buffer. Only the bufferevent is allowed to drain data from this buffer, though the user is allowed to add it. */ struct evbuffer *output; struct event_watermark wm_read; struct event_watermark wm_write; bufferevent_data_cb readcb; bufferevent_data_cb writecb; /* This should be called 'eventcb', but renaming it would break * backward compatibility */ bufferevent_event_cb errorcb; void *cbarg; struct timeval timeout_read; struct timeval timeout_write; /** Events that are currently enabled: currently EV_READ and EV_WRITE are supported. */ short enabled; }; #ifdef __cplusplus } #endif #endif /* EVENT2_BUFFEREVENT_STRUCT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/dns.h000066400000000000000000000643261527052371200235740ustar00rootroot00000000000000/* * Copyright (c) 2006-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * The original DNS code is due to Adam Langley with heavy * modifications by Nick Mathewson. Adam put his DNS software in the * public domain. You can find his original copyright below. Please, * aware that the code as part of Libevent is governed by the 3-clause * BSD license above. * * This software is Public Domain. To view a copy of the public domain dedication, * visit http://creativecommons.org/licenses/publicdomain/ or send a letter to * Creative Commons, 559 Nathan Abbott Way, Stanford, California 94305, USA. * * I ask and expect, but do not require, that all derivative works contain an * attribution similar to: * Parts developed by Adam Langley * * You may wish to replace the word "Parts" with something else depending on * the amount of original code. * * (Derivative works does not include programs which link against, run or include * the source verbatim in their source distributions) */ /** @file event2/dns.h * * Welcome, gentle reader * * Async DNS lookups are really a whole lot harder than they should be, * mostly stemming from the fact that the libc resolver has never been * very good at them. Before you use this library you should see if libc * can do the job for you with the modern async call getaddrinfo_a * (see http://www.imperialviolet.org/page25.html#e498). Otherwise, * please continue. * * The library keeps track of the state of nameservers and will avoid * them when they go down. Otherwise it will round robin between them. * * Quick start guide: * #include "evdns.h" * void callback(int result, char type, int count, int ttl, * void *addresses, void *arg); * evdns_resolv_conf_parse(DNS_OPTIONS_ALL, "/etc/resolv.conf"); * evdns_resolve("www.hostname.com", 0, callback, NULL); * * When the lookup is complete the callback function is called. The * first argument will be one of the DNS_ERR_* defines in evdns.h. * Hopefully it will be DNS_ERR_NONE, in which case type will be * DNS_IPv4_A, count will be the number of IP addresses, ttl is the time * which the data can be cached for (in seconds), addresses will point * to an array of uint32_t's and arg will be whatever you passed to * evdns_resolve. * * Searching: * * In order for this library to be a good replacement for glibc's resolver it * supports searching. This involves setting a list of default domains, in * which names will be queried for. The number of dots in the query name * determines the order in which this list is used. * * Searching appears to be a single lookup from the point of view of the API, * although many DNS queries may be generated from a single call to * evdns_resolve. Searching can also drastically slow down the resolution * of names. * * To disable searching: * 1. Never set it up. If you never call evdns_resolv_conf_parse or * evdns_search_add then no searching will occur. * * 2. If you do call evdns_resolv_conf_parse then don't pass * DNS_OPTION_SEARCH (or DNS_OPTIONS_ALL, which implies it). * * 3. When calling evdns_resolve, pass the DNS_QUERY_NO_SEARCH flag. * * The order of searches depends on the number of dots in the name. If the * number is greater than the ndots setting then the names is first tried * globally. Otherwise each search domain is appended in turn. * * The ndots setting can either be set from a resolv.conf, or by calling * evdns_search_ndots_set. * * For example, with ndots set to 1 (the default) and a search domain list of * ["myhome.net"]: * Query: www * Order: www.myhome.net, www. * * Query: www.abc * Order: www.abc., www.abc.myhome.net * * Internals: * * Requests are kept in two queues. The first is the inflight queue. In * this queue requests have an allocated transaction id and nameserver. * They will soon be transmitted if they haven't already been. * * The second is the waiting queue. The size of the inflight ring is * limited and all other requests wait in waiting queue for space. This * bounds the number of concurrent requests so that we don't flood the * nameserver. Several algorithms require a full walk of the inflight * queue and so bounding its size keeps thing going nicely under huge * (many thousands of requests) loads. * * If a nameserver loses too many requests it is considered down and we * try not to use it. After a while we send a probe to that nameserver * (a lookup for google.com) and, if it replies, we consider it working * again. If the nameserver fails a probe we wait longer to try again * with the next probe. */ #ifndef EVENT2_DNS_H_INCLUDED_ #define EVENT2_DNS_H_INCLUDED_ #include #ifdef __cplusplus extern "C" { #endif /* For integer types. */ #include /** Error codes 0-5 are as described in RFC 1035. */ #define DNS_ERR_NONE 0 /** The name server was unable to interpret the query */ #define DNS_ERR_FORMAT 1 /** The name server was unable to process this query due to a problem with the * name server */ #define DNS_ERR_SERVERFAILED 2 /** The domain name does not exist */ #define DNS_ERR_NOTEXIST 3 /** The name server does not support the requested kind of query */ #define DNS_ERR_NOTIMPL 4 /** The name server refuses to reform the specified operation for policy * reasons */ #define DNS_ERR_REFUSED 5 /** The reply was truncated or ill-formatted */ #define DNS_ERR_TRUNCATED 65 /** An unknown error occurred */ #define DNS_ERR_UNKNOWN 66 /** Communication with the server timed out */ #define DNS_ERR_TIMEOUT 67 /** The request was canceled because the DNS subsystem was shut down. */ #define DNS_ERR_SHUTDOWN 68 /** The request was canceled via a call to evdns_cancel_request */ #define DNS_ERR_CANCEL 69 /** There were no answers and no error condition in the DNS packet. * This can happen when you ask for an address that exists, but a record * type that doesn't. */ #define DNS_ERR_NODATA 70 #define DNS_IPv4_A 1 #define DNS_PTR 2 #define DNS_IPv6_AAAA 3 #define DNS_QUERY_NO_SEARCH 1 #define DNS_OPTION_SEARCH 1 #define DNS_OPTION_NAMESERVERS 2 #define DNS_OPTION_MISC 4 #define DNS_OPTION_HOSTSFILE 8 #define DNS_OPTIONS_ALL 15 /* Obsolete name for DNS_QUERY_NO_SEARCH */ #define DNS_NO_SEARCH DNS_QUERY_NO_SEARCH /** * The callback that contains the results from a lookup. * - result is one of the DNS_ERR_* values (DNS_ERR_NONE for success) * - type is either DNS_IPv4_A or DNS_PTR or DNS_IPv6_AAAA * - count contains the number of addresses of form type * - ttl is the number of seconds the resolution may be cached for. * - addresses needs to be cast according to type. It will be an array of * 4-byte sequences for ipv4, or an array of 16-byte sequences for ipv6, * or a nul-terminated string for PTR. */ typedef void (*evdns_callback_type) (int result, char type, int count, int ttl, void *addresses, void *arg); struct evdns_base; struct event_base; /** Flag for evdns_base_new: process resolv.conf. */ #define EVDNS_BASE_INITIALIZE_NAMESERVERS 1 /** Flag for evdns_base_new: Do not prevent the libevent event loop from * exiting when we have no active dns requests. */ #define EVDNS_BASE_DISABLE_WHEN_INACTIVE 0x8000 /** Initialize the asynchronous DNS library. This function initializes support for non-blocking name resolution by calling evdns_resolv_conf_parse() on UNIX and evdns_config_windows_nameservers() on Windows. @param event_base the event base to associate the dns client with @param flags any of EVDNS_BASE_INITIALIZE_NAMESERVERS| EVDNS_BASE_DISABLE_WHEN_INACTIVE @return evdns_base object if successful, or NULL if an error occurred. @see evdns_base_free() */ EVENT2_EXPORT_SYMBOL struct evdns_base * evdns_base_new(struct event_base *event_base, int initialize_nameservers); /** Shut down the asynchronous DNS resolver and terminate all active requests. If the 'fail_requests' option is enabled, all active requests will return an empty result with the error flag set to DNS_ERR_SHUTDOWN. Otherwise, the requests will be silently discarded. @param evdns_base the evdns base to free @param fail_requests if zero, active requests will be aborted; if non-zero, active requests will return DNS_ERR_SHUTDOWN. @see evdns_base_new() */ EVENT2_EXPORT_SYMBOL void evdns_base_free(struct evdns_base *base, int fail_requests); /** Remove all hosts entries that have been loaded into the event_base via evdns_base_load_hosts or via event_base_resolv_conf_parse. @param evdns_base the evdns base to remove outdated host addresses from */ EVENT2_EXPORT_SYMBOL void evdns_base_clear_host_addresses(struct evdns_base *base); /** Convert a DNS error code to a string. @param err the DNS error code @return a string containing an explanation of the error code */ EVENT2_EXPORT_SYMBOL const char *evdns_err_to_string(int err); /** Add a nameserver. The address should be an IPv4 address in network byte order. The type of address is chosen so that it matches in_addr.s_addr. @param base the evdns_base to which to add the name server @param address an IP address in network byte order @return 0 if successful, or -1 if an error occurred @see evdns_base_nameserver_ip_add() */ EVENT2_EXPORT_SYMBOL int evdns_base_nameserver_add(struct evdns_base *base, unsigned long int address); /** Get the number of configured nameservers. This returns the number of configured nameservers (not necessarily the number of running nameservers). This is useful for double-checking whether our calls to the various nameserver configuration functions have been successful. @param base the evdns_base to which to apply this operation @return the number of configured nameservers @see evdns_base_nameserver_add() */ EVENT2_EXPORT_SYMBOL int evdns_base_count_nameservers(struct evdns_base *base); /** Remove all configured nameservers, and suspend all pending resolves. Resolves will not necessarily be re-attempted until evdns_base_resume() is called. @param base the evdns_base to which to apply this operation @return 0 if successful, or -1 if an error occurred @see evdns_base_resume() */ EVENT2_EXPORT_SYMBOL int evdns_base_clear_nameservers_and_suspend(struct evdns_base *base); /** Resume normal operation and continue any suspended resolve requests. Re-attempt resolves left in limbo after an earlier call to evdns_base_clear_nameservers_and_suspend(). @param base the evdns_base to which to apply this operation @return 0 if successful, or -1 if an error occurred @see evdns_base_clear_nameservers_and_suspend() */ EVENT2_EXPORT_SYMBOL int evdns_base_resume(struct evdns_base *base); /** Add a nameserver by string address. This function parses a n IPv4 or IPv6 address from a string and adds it as a nameserver. It supports the following formats: - [IPv6Address]:port - [IPv6Address] - IPv6Address - IPv4Address:port - IPv4Address If no port is specified, it defaults to 53. @param base the evdns_base to which to apply this operation @return 0 if successful, or -1 if an error occurred @see evdns_base_nameserver_add() */ EVENT2_EXPORT_SYMBOL int evdns_base_nameserver_ip_add(struct evdns_base *base, const char *ip_as_string); /** Add a nameserver by sockaddr. **/ EVENT2_EXPORT_SYMBOL int evdns_base_nameserver_sockaddr_add(struct evdns_base *base, const struct sockaddr *sa, ev_socklen_t len, unsigned flags); struct evdns_request; /** Lookup an A record for a given name. @param base the evdns_base to which to apply this operation @param name a DNS hostname @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. @param callback a callback function to invoke when the request is completed @param ptr an argument to pass to the callback function @return an evdns_request object if successful, or NULL if an error occurred. @see evdns_resolve_ipv6(), evdns_resolve_reverse(), evdns_resolve_reverse_ipv6(), evdns_cancel_request() */ EVENT2_EXPORT_SYMBOL struct evdns_request *evdns_base_resolve_ipv4(struct evdns_base *base, const char *name, int flags, evdns_callback_type callback, void *ptr); /** Lookup an AAAA record for a given name. @param base the evdns_base to which to apply this operation @param name a DNS hostname @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. @param callback a callback function to invoke when the request is completed @param ptr an argument to pass to the callback function @return an evdns_request object if successful, or NULL if an error occurred. @see evdns_resolve_ipv4(), evdns_resolve_reverse(), evdns_resolve_reverse_ipv6(), evdns_cancel_request() */ EVENT2_EXPORT_SYMBOL struct evdns_request *evdns_base_resolve_ipv6(struct evdns_base *base, const char *name, int flags, evdns_callback_type callback, void *ptr); struct in_addr; struct in6_addr; /** Lookup a PTR record for a given IP address. @param base the evdns_base to which to apply this operation @param in an IPv4 address @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. @param callback a callback function to invoke when the request is completed @param ptr an argument to pass to the callback function @return an evdns_request object if successful, or NULL if an error occurred. @see evdns_resolve_reverse_ipv6(), evdns_cancel_request() */ EVENT2_EXPORT_SYMBOL struct evdns_request *evdns_base_resolve_reverse(struct evdns_base *base, const struct in_addr *in, int flags, evdns_callback_type callback, void *ptr); /** Lookup a PTR record for a given IPv6 address. @param base the evdns_base to which to apply this operation @param in an IPv6 address @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. @param callback a callback function to invoke when the request is completed @param ptr an argument to pass to the callback function @return an evdns_request object if successful, or NULL if an error occurred. @see evdns_resolve_reverse_ipv6(), evdns_cancel_request() */ EVENT2_EXPORT_SYMBOL struct evdns_request *evdns_base_resolve_reverse_ipv6(struct evdns_base *base, const struct in6_addr *in, int flags, evdns_callback_type callback, void *ptr); /** Cancels a pending DNS resolution request. @param base the evdns_base that was used to make the request @param req the evdns_request that was returned by calling a resolve function @see evdns_base_resolve_ipv4(), evdns_base_resolve_ipv6, evdns_base_resolve_reverse */ EVENT2_EXPORT_SYMBOL void evdns_cancel_request(struct evdns_base *base, struct evdns_request *req); /** Set the value of a configuration option. The currently available configuration options are: ndots, timeout, max-timeouts, max-inflight, attempts, randomize-case, bind-to, initial-probe-timeout, getaddrinfo-allow-skew. In versions before Libevent 2.0.3-alpha, the option name needed to end with a colon. @param base the evdns_base to which to apply this operation @param option the name of the configuration option to be modified @param val the value to be set @return 0 if successful, or -1 if an error occurred */ EVENT2_EXPORT_SYMBOL int evdns_base_set_option(struct evdns_base *base, const char *option, const char *val); /** Parse a resolv.conf file. The 'flags' parameter determines what information is parsed from the resolv.conf file. See the man page for resolv.conf for the format of this file. The following directives are not parsed from the file: sortlist, rotate, no-check-names, inet6, debug. If this function encounters an error, the possible return values are: 1 = failed to open file, 2 = failed to stat file, 3 = file too large, 4 = out of memory, 5 = short read from file, 6 = no nameservers listed in the file @param base the evdns_base to which to apply this operation @param flags any of DNS_OPTION_NAMESERVERS|DNS_OPTION_SEARCH|DNS_OPTION_MISC| DNS_OPTION_HOSTSFILE|DNS_OPTIONS_ALL @param filename the path to the resolv.conf file @return 0 if successful, or various positive error codes if an error occurred (see above) @see resolv.conf(3), evdns_config_windows_nameservers() */ EVENT2_EXPORT_SYMBOL int evdns_base_resolv_conf_parse(struct evdns_base *base, int flags, const char *const filename); /** Load an /etc/hosts-style file from 'hosts_fname' into 'base'. If hosts_fname is NULL, add minimal entries for localhost, and nothing else. Note that only evdns_getaddrinfo uses the /etc/hosts entries. This function does not replace previously loaded hosts entries; to do that, call evdns_base_clear_host_addresses first. Return 0 on success, negative on failure. */ EVENT2_EXPORT_SYMBOL int evdns_base_load_hosts(struct evdns_base *base, const char *hosts_fname); /** Obtain nameserver information using the Windows API. Attempt to configure a set of nameservers based on platform settings on a win32 host. Preferentially tries to use GetNetworkParams; if that fails, looks in the registry. @return 0 if successful, or -1 if an error occurred @see evdns_resolv_conf_parse() */ #ifdef _WIN32 EVENT2_EXPORT_SYMBOL int evdns_base_config_windows_nameservers(struct evdns_base *); #define EVDNS_BASE_CONFIG_WINDOWS_NAMESERVERS_IMPLEMENTED #endif /** Clear the list of search domains. */ EVENT2_EXPORT_SYMBOL void evdns_base_search_clear(struct evdns_base *base); /** Add a domain to the list of search domains @param domain the domain to be added to the search list */ EVENT2_EXPORT_SYMBOL void evdns_base_search_add(struct evdns_base *base, const char *domain); /** Set the 'ndots' parameter for searches. Sets the number of dots which, when found in a name, causes the first query to be without any search domain. @param ndots the new ndots parameter */ EVENT2_EXPORT_SYMBOL void evdns_base_search_ndots_set(struct evdns_base *base, const int ndots); /** A callback that is invoked when a log message is generated @param is_warning indicates if the log message is a 'warning' @param msg the content of the log message */ typedef void (*evdns_debug_log_fn_type)(int is_warning, const char *msg); /** Set the callback function to handle DNS log messages. If this callback is not set, evdns log messages are handled with the regular Libevent logging system. @param fn the callback to be invoked when a log message is generated */ EVENT2_EXPORT_SYMBOL void evdns_set_log_fn(evdns_debug_log_fn_type fn); /** Set a callback that will be invoked to generate transaction IDs. By default, we pick transaction IDs based on the current clock time, which is bad for security. @param fn the new callback, or NULL to use the default. NOTE: This function has no effect in Libevent 2.0.4-alpha and later, since Libevent now provides its own secure RNG. */ EVENT2_EXPORT_SYMBOL void evdns_set_transaction_id_fn(ev_uint16_t (*fn)(void)); /** Set a callback used to generate random bytes. By default, we use the same function as passed to evdns_set_transaction_id_fn to generate bytes two at a time. If a function is provided here, it's also used to generate transaction IDs. NOTE: This function has no effect in Libevent 2.0.4-alpha and later, since Libevent now provides its own secure RNG. */ EVENT2_EXPORT_SYMBOL void evdns_set_random_bytes_fn(void (*fn)(char *, size_t)); /* * Functions used to implement a DNS server. */ struct evdns_server_request; struct evdns_server_question; /** A callback to implement a DNS server. The callback function receives a DNS request. It should then optionally add a number of answers to the reply using the evdns_server_request_add_*_reply functions, before calling either evdns_server_request_respond to send the reply back, or evdns_server_request_drop to decline to answer the request. @param req A newly received request @param user_data A pointer that was passed to evdns_add_server_port_with_base(). */ typedef void (*evdns_request_callback_fn_type)(struct evdns_server_request *, void *); #define EVDNS_ANSWER_SECTION 0 #define EVDNS_AUTHORITY_SECTION 1 #define EVDNS_ADDITIONAL_SECTION 2 #define EVDNS_TYPE_A 1 #define EVDNS_TYPE_NS 2 #define EVDNS_TYPE_CNAME 5 #define EVDNS_TYPE_SOA 6 #define EVDNS_TYPE_PTR 12 #define EVDNS_TYPE_MX 15 #define EVDNS_TYPE_TXT 16 #define EVDNS_TYPE_AAAA 28 #define EVDNS_QTYPE_AXFR 252 #define EVDNS_QTYPE_ALL 255 #define EVDNS_CLASS_INET 1 /* flags that can be set in answers; as part of the err parameter */ #define EVDNS_FLAGS_AA 0x400 #define EVDNS_FLAGS_RD 0x080 /** Create a new DNS server port. @param base The event base to handle events for the server port. @param socket A UDP socket to accept DNS requests. @param flags Always 0 for now. @param callback A function to invoke whenever we get a DNS request on the socket. @param user_data Data to pass to the callback. @return an evdns_server_port structure for this server port. */ EVENT2_EXPORT_SYMBOL struct evdns_server_port *evdns_add_server_port_with_base(struct event_base *base, evutil_socket_t socket, int flags, evdns_request_callback_fn_type callback, void *user_data); /** Close down a DNS server port, and free associated structures. */ EVENT2_EXPORT_SYMBOL void evdns_close_server_port(struct evdns_server_port *port); /** Sets some flags in a reply we're building. Allows setting of the AA or RD flags */ EVENT2_EXPORT_SYMBOL void evdns_server_request_set_flags(struct evdns_server_request *req, int flags); /* Functions to add an answer to an in-progress DNS reply. */ EVENT2_EXPORT_SYMBOL int evdns_server_request_add_reply(struct evdns_server_request *req, int section, const char *name, int type, int dns_class, int ttl, int datalen, int is_name, const char *data); EVENT2_EXPORT_SYMBOL int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl); EVENT2_EXPORT_SYMBOL int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl); EVENT2_EXPORT_SYMBOL int evdns_server_request_add_ptr_reply(struct evdns_server_request *req, struct in_addr *in, const char *inaddr_name, const char *hostname, int ttl); EVENT2_EXPORT_SYMBOL int evdns_server_request_add_cname_reply(struct evdns_server_request *req, const char *name, const char *cname, int ttl); /** Send back a response to a DNS request, and free the request structure. */ EVENT2_EXPORT_SYMBOL int evdns_server_request_respond(struct evdns_server_request *req, int err); /** Free a DNS request without sending back a reply. */ EVENT2_EXPORT_SYMBOL int evdns_server_request_drop(struct evdns_server_request *req); struct sockaddr; /** Get the address that made a DNS request. */ EVENT2_EXPORT_SYMBOL int evdns_server_request_get_requesting_addr(struct evdns_server_request *req, struct sockaddr *sa, int addr_len); /** Callback for evdns_getaddrinfo. */ typedef void (*evdns_getaddrinfo_cb)(int result, struct evutil_addrinfo *res, void *arg); struct evdns_base; struct evdns_getaddrinfo_request; /** Make a non-blocking getaddrinfo request using the dns_base in 'dns_base'. * * If we can answer the request immediately (with an error or not!), then we * invoke cb immediately and return NULL. Otherwise we return * an evdns_getaddrinfo_request and invoke cb later. * * When the callback is invoked, we pass as its first argument the error code * that getaddrinfo would return (or 0 for no error). As its second argument, * we pass the evutil_addrinfo structures we found (or NULL on error). We * pass 'arg' as the third argument. * * Limitations: * * - The AI_V4MAPPED and AI_ALL flags are not currently implemented. * - For ai_socktype, we only handle SOCKTYPE_STREAM, SOCKTYPE_UDP, and 0. * - For ai_protocol, we only handle IPPROTO_TCP, IPPROTO_UDP, and 0. */ EVENT2_EXPORT_SYMBOL struct evdns_getaddrinfo_request *evdns_getaddrinfo( struct evdns_base *dns_base, const char *nodename, const char *servname, const struct evutil_addrinfo *hints_in, evdns_getaddrinfo_cb cb, void *arg); /* Cancel an in-progress evdns_getaddrinfo. This MUST NOT be called after the * getaddrinfo's callback has been invoked. The resolves will be canceled, * and the callback will be invoked with the error EVUTIL_EAI_CANCEL. */ EVENT2_EXPORT_SYMBOL void evdns_getaddrinfo_cancel(struct evdns_getaddrinfo_request *req); /** Retrieve the address of the 'idx'th configured nameserver. @param base The evdns_base to examine. @param idx The index of the nameserver to get the address of. @param sa A location to receive the server's address. @param len The number of bytes available at sa. @return the number of bytes written into sa on success. On failure, returns -1 if idx is greater than the number of configured nameservers, or a value greater than 'len' if len was not high enough. */ EVENT2_EXPORT_SYMBOL int evdns_base_get_nameserver_addr(struct evdns_base *base, int idx, struct sockaddr *sa, ev_socklen_t len); #ifdef __cplusplus } #endif #endif /* !EVENT2_DNS_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/dns_compat.h000066400000000000000000000275741527052371200251430ustar00rootroot00000000000000/* * Copyright (c) 2006-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_DNS_COMPAT_H_INCLUDED_ #define EVENT2_DNS_COMPAT_H_INCLUDED_ /** @file event2/dns_compat.h Potentially non-threadsafe versions of the functions in dns.h: provided only for backwards compatibility. */ #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include /** Initialize the asynchronous DNS library. This function initializes support for non-blocking name resolution by calling evdns_resolv_conf_parse() on UNIX and evdns_config_windows_nameservers() on Windows. @deprecated This function is deprecated because it always uses the current event base, and is easily confused by multiple calls to event_init(), and so is not safe for multithreaded use. Additionally, it allocates a global structure that only one thread can use. The replacement is evdns_base_new(). @return 0 if successful, or -1 if an error occurred @see evdns_shutdown() */ int evdns_init(void); struct evdns_base; /** Return the global evdns_base created by event_init() and used by the other deprecated functions. @deprecated This function is deprecated because use of the global evdns_base is error-prone. */ struct evdns_base *evdns_get_global_base(void); /** Shut down the asynchronous DNS resolver and terminate all active requests. If the 'fail_requests' option is enabled, all active requests will return an empty result with the error flag set to DNS_ERR_SHUTDOWN. Otherwise, the requests will be silently discarded. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_shutdown(). @param fail_requests if zero, active requests will be aborted; if non-zero, active requests will return DNS_ERR_SHUTDOWN. @see evdns_init() */ void evdns_shutdown(int fail_requests); /** Add a nameserver. The address should be an IPv4 address in network byte order. The type of address is chosen so that it matches in_addr.s_addr. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_nameserver_add(). @param address an IP address in network byte order @return 0 if successful, or -1 if an error occurred @see evdns_nameserver_ip_add() */ int evdns_nameserver_add(unsigned long int address); /** Get the number of configured nameservers. This returns the number of configured nameservers (not necessarily the number of running nameservers). This is useful for double-checking whether our calls to the various nameserver configuration functions have been successful. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_count_nameservers(). @return the number of configured nameservers @see evdns_nameserver_add() */ int evdns_count_nameservers(void); /** Remove all configured nameservers, and suspend all pending resolves. Resolves will not necessarily be re-attempted until evdns_resume() is called. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_clear_nameservers_and_suspend(). @return 0 if successful, or -1 if an error occurred @see evdns_resume() */ int evdns_clear_nameservers_and_suspend(void); /** Resume normal operation and continue any suspended resolve requests. Re-attempt resolves left in limbo after an earlier call to evdns_clear_nameservers_and_suspend(). @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_resume(). @return 0 if successful, or -1 if an error occurred @see evdns_clear_nameservers_and_suspend() */ int evdns_resume(void); /** Add a nameserver. This wraps the evdns_nameserver_add() function by parsing a string as an IP address and adds it as a nameserver. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_nameserver_ip_add(). @return 0 if successful, or -1 if an error occurred @see evdns_nameserver_add() */ int evdns_nameserver_ip_add(const char *ip_as_string); /** Lookup an A record for a given name. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_resolve_ipv4(). @param name a DNS hostname @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. @param callback a callback function to invoke when the request is completed @param ptr an argument to pass to the callback function @return 0 if successful, or -1 if an error occurred @see evdns_resolve_ipv6(), evdns_resolve_reverse(), evdns_resolve_reverse_ipv6() */ int evdns_resolve_ipv4(const char *name, int flags, evdns_callback_type callback, void *ptr); /** Lookup an AAAA record for a given name. @param name a DNS hostname @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. @param callback a callback function to invoke when the request is completed @param ptr an argument to pass to the callback function @return 0 if successful, or -1 if an error occurred @see evdns_resolve_ipv4(), evdns_resolve_reverse(), evdns_resolve_reverse_ipv6() */ int evdns_resolve_ipv6(const char *name, int flags, evdns_callback_type callback, void *ptr); struct in_addr; struct in6_addr; /** Lookup a PTR record for a given IP address. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_resolve_reverse(). @param in an IPv4 address @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. @param callback a callback function to invoke when the request is completed @param ptr an argument to pass to the callback function @return 0 if successful, or -1 if an error occurred @see evdns_resolve_reverse_ipv6() */ int evdns_resolve_reverse(const struct in_addr *in, int flags, evdns_callback_type callback, void *ptr); /** Lookup a PTR record for a given IPv6 address. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_resolve_reverse_ipv6(). @param in an IPv6 address @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. @param callback a callback function to invoke when the request is completed @param ptr an argument to pass to the callback function @return 0 if successful, or -1 if an error occurred @see evdns_resolve_reverse_ipv6() */ int evdns_resolve_reverse_ipv6(const struct in6_addr *in, int flags, evdns_callback_type callback, void *ptr); /** Set the value of a configuration option. The currently available configuration options are: ndots, timeout, max-timeouts, max-inflight, and attempts @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_set_option(). @param option the name of the configuration option to be modified @param val the value to be set @param flags Ignored. @return 0 if successful, or -1 if an error occurred */ int evdns_set_option(const char *option, const char *val, int flags); /** Parse a resolv.conf file. The 'flags' parameter determines what information is parsed from the resolv.conf file. See the man page for resolv.conf for the format of this file. The following directives are not parsed from the file: sortlist, rotate, no-check-names, inet6, debug. If this function encounters an error, the possible return values are: 1 = failed to open file, 2 = failed to stat file, 3 = file too large, 4 = out of memory, 5 = short read from file, 6 = no nameservers listed in the file @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_resolv_conf_parse(). @param flags any of DNS_OPTION_NAMESERVERS|DNS_OPTION_SEARCH|DNS_OPTION_MISC| DNS_OPTIONS_ALL @param filename the path to the resolv.conf file @return 0 if successful, or various positive error codes if an error occurred (see above) @see resolv.conf(3), evdns_config_windows_nameservers() */ int evdns_resolv_conf_parse(int flags, const char *const filename); /** Clear the list of search domains. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_search_clear(). */ void evdns_search_clear(void); /** Add a domain to the list of search domains @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_search_add(). @param domain the domain to be added to the search list */ void evdns_search_add(const char *domain); /** Set the 'ndots' parameter for searches. Sets the number of dots which, when found in a name, causes the first query to be without any search domain. @deprecated This function is deprecated because it does not allow the caller to specify which evdns_base it applies to. The recommended function is evdns_base_search_ndots_set(). @param ndots the new ndots parameter */ void evdns_search_ndots_set(const int ndots); /** As evdns_server_new_with_base. @deprecated This function is deprecated because it does not allow the caller to specify which even_base it uses. The recommended function is evdns_add_server_port_with_base(). */ struct evdns_server_port *evdns_add_server_port(evutil_socket_t socket, int flags, evdns_request_callback_fn_type callback, void *user_data); #ifdef _WIN32 int evdns_config_windows_nameservers(void); #define EVDNS_CONFIG_WINDOWS_NAMESERVERS_IMPLEMENTED #endif #ifdef __cplusplus } #endif #endif /* EVENT2_EVENT_COMPAT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/dns_struct.h000066400000000000000000000050441527052371200251700ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_DNS_STRUCT_H_INCLUDED_ #define EVENT2_DNS_STRUCT_H_INCLUDED_ /** @file event2/dns_struct.h Data structures for dns. Using these structures may hurt forward compatibility with later versions of Libevent: be careful! */ #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include /* * Structures used to implement a DNS server. */ struct evdns_server_request { int flags; int nquestions; struct evdns_server_question **questions; }; struct evdns_server_question { int type; #ifdef __cplusplus int dns_question_class; #else /* You should refer to this field as "dns_question_class". The * name "class" works in C for backward compatibility, and will be * removed in a future version. (1.5 or later). */ int class; #define dns_question_class class #endif char name[1]; }; #ifdef __cplusplus } #endif #endif /* EVENT2_DNS_STRUCT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/event.h000066400000000000000000001716711527052371200241330ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_EVENT_H_INCLUDED_ #define EVENT2_EVENT_H_INCLUDED_ /** @mainpage @section intro Introduction Libevent is an event notification library for developing scalable network servers. The Libevent API provides a mechanism to execute a callback function when a specific event occurs on a file descriptor or after a timeout has been reached. Furthermore, Libevent also support callbacks due to signals or regular timeouts. Libevent is meant to replace the event loop found in event driven network servers. An application just needs to call event_base_dispatch() and then add or remove events dynamically without having to change the event loop. Currently, Libevent supports /dev/poll, kqueue(2), select(2), poll(2), epoll(4), and evports. The internal event mechanism is completely independent of the exposed event API, and a simple update of Libevent can provide new functionality without having to redesign the applications. As a result, Libevent allows for portable application development and provides the most scalable event notification mechanism available on an operating system. Libevent can also be used for multithreaded programs. Libevent should compile on Linux, *BSD, Mac OS X, Solaris and, Windows. @section usage Standard usage Every program that uses Libevent must include the header, and pass the -levent flag to the linker. (You can instead link -levent_core if you only want the main event and buffered IO-based code, and don't want to link any protocol code.) @section setup Library setup Before you call any other Libevent functions, you need to set up the library. If you're going to use Libevent from multiple threads in a multithreaded application, you need to initialize thread support -- typically by using evthread_use_pthreads() or evthread_use_windows_threads(). See for more information. This is also the point where you can replace Libevent's memory management functions with event_set_mem_functions, and enable debug mode with event_enable_debug_mode(). @section base Creating an event base Next, you need to create an event_base structure, using event_base_new() or event_base_new_with_config(). The event_base is responsible for keeping track of which events are "pending" (that is to say, being watched to see if they become active) and which events are "active". Every event is associated with a single event_base. @section event Event notification For each file descriptor that you wish to monitor, you must create an event structure with event_new(). (You may also declare an event structure and call event_assign() to initialize the members of the structure.) To enable notification, you add the structure to the list of monitored events by calling event_add(). The event structure must remain allocated as long as it is active, so it should generally be allocated on the heap. @section loop Dispatching events. Finally, you call event_base_dispatch() to loop and dispatch events. You can also use event_base_loop() for more fine-grained control. Currently, only one thread can be dispatching a given event_base at a time. If you want to run events in multiple threads at once, you can either have a single event_base whose events add work to a work queue, or you can create multiple event_base objects. @section bufferevent I/O Buffers Libevent provides a buffered I/O abstraction on top of the regular event callbacks. This abstraction is called a bufferevent. A bufferevent provides input and output buffers that get filled and drained automatically. The user of a buffered event no longer deals directly with the I/O, but instead is reading from input and writing to output buffers. Once initialized via bufferevent_socket_new(), the bufferevent structure can be used repeatedly with bufferevent_enable() and bufferevent_disable(). Instead of reading and writing directly to a socket, you would call bufferevent_read() and bufferevent_write(). When read enabled the bufferevent will try to read from the file descriptor and call the read callback. The write callback is executed whenever the output buffer is drained below the write low watermark, which is 0 by default. See for more information. @section timers Timers Libevent can also be used to create timers that invoke a callback after a certain amount of time has expired. The evtimer_new() macro returns an event struct to use as a timer. To activate the timer, call evtimer_add(). Timers can be deactivated by calling evtimer_del(). (These macros are thin wrappers around event_new(), event_add(), and event_del(); you can also use those instead.) @section evdns Asynchronous DNS resolution Libevent provides an asynchronous DNS resolver that should be used instead of the standard DNS resolver functions. See the functions for more detail. @section evhttp Event-driven HTTP servers Libevent provides a very simple event-driven HTTP server that can be embedded in your program and used to service HTTP requests. To use this capability, you need to include the header in your program. See that header for more information. @section evrpc A framework for RPC servers and clients Libevent provides a framework for creating RPC servers and clients. It takes care of marshaling and unmarshaling all data structures. @section api API Reference To browse the complete documentation of the libevent API, click on any of the following links. event2/event.h The primary libevent header event2/thread.h Functions for use by multithreaded programs event2/buffer.h and event2/bufferevent.h Buffer management for network reading and writing event2/util.h Utility functions for portable nonblocking network code event2/dns.h Asynchronous DNS resolution event2/http.h An embedded libevent-based HTTP server event2/rpc.h A framework for creating RPC servers and clients */ /** @file event2/event.h Core functions for waiting for and receiving events, and using event bases. */ #include #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include /* For int types. */ #include /** * Structure to hold information and state for a Libevent dispatch loop. * * The event_base lies at the center of Libevent; every application will * have one. It keeps track of all pending and active events, and * notifies your application of the active ones. * * This is an opaque structure; you can allocate one using * event_base_new() or event_base_new_with_config(). * * @see event_base_new(), event_base_free(), event_base_loop(), * event_base_new_with_config() */ struct event_base #ifdef EVENT_IN_DOXYGEN_ {/*Empty body so that doxygen will generate documentation here.*/} #endif ; /** * @struct event * * Structure to represent a single event. * * An event can have some underlying condition it represents: a socket * becoming readable or writeable (or both), or a signal becoming raised. * (An event that represents no underlying condition is still useful: you * can use one to implement a timer, or to communicate between threads.) * * Generally, you can create events with event_new(), then make them * pending with event_add(). As your event_base runs, it will run the * callbacks of an events whose conditions are triggered. When you * longer want the event, free it with event_free(). * * In more depth: * * An event may be "pending" (one whose condition we are watching), * "active" (one whose condition has triggered and whose callback is about * to run), neither, or both. Events come into existence via * event_assign() or event_new(), and are then neither active nor pending. * * To make an event pending, pass it to event_add(). When doing so, you * can also set a timeout for the event. * * Events become active during an event_base_loop() call when either their * condition has triggered, or when their timeout has elapsed. You can * also activate an event manually using event_active(). The even_base * loop will run the callbacks of active events; after it has done so, it * marks them as no longer active. * * You can make an event non-pending by passing it to event_del(). This * also makes the event non-active. * * Events can be "persistent" or "non-persistent". A non-persistent event * becomes non-pending as soon as it is triggered: thus, it only runs at * most once per call to event_add(). A persistent event remains pending * even when it becomes active: you'll need to event_del() it manually in * order to make it non-pending. When a persistent event with a timeout * becomes active, its timeout is reset: this means you can use persistent * events to implement periodic timeouts. * * This should be treated as an opaque structure; you should never read or * write any of its fields directly. For backward compatibility with old * code, it is defined in the event2/event_struct.h header; including this * header may make your code incompatible with other versions of Libevent. * * @see event_new(), event_free(), event_assign(), event_get_assignment(), * event_add(), event_del(), event_active(), event_pending(), * event_get_fd(), event_get_base(), event_get_events(), * event_get_callback(), event_get_callback_arg(), * event_priority_set() */ struct event #ifdef EVENT_IN_DOXYGEN_ {/*Empty body so that doxygen will generate documentation here.*/} #endif ; /** * Configuration for an event_base. * * There are many options that can be used to alter the behavior and * implementation of an event_base. To avoid having to pass them all in a * complex many-argument constructor, we provide an abstract data type * wrhere you set up configation information before passing it to * event_base_new_with_config(). * * @see event_config_new(), event_config_free(), event_base_new_with_config(), * event_config_avoid_method(), event_config_require_features(), * event_config_set_flag(), event_config_set_num_cpus_hint() */ struct event_config #ifdef EVENT_IN_DOXYGEN_ {/*Empty body so that doxygen will generate documentation here.*/} #endif ; /** * Enable some relatively expensive debugging checks in Libevent that * would normally be turned off. Generally, these checks cause code that * would otherwise crash mysteriously to fail earlier with an assertion * failure. Note that this method MUST be called before any events or * event_bases have been created. * * Debug mode can currently catch the following errors: * An event is re-assigned while it is added * Any function is called on a non-assigned event * * Note that debugging mode uses memory to track every event that has been * initialized (via event_assign, event_set, or event_new) but not yet * released (via event_free or event_debug_unassign). If you want to use * debug mode, and you find yourself running out of memory, you will need * to use event_debug_unassign to explicitly stop tracking events that * are no longer considered set-up. * * @see event_debug_unassign() */ EVENT2_EXPORT_SYMBOL void event_enable_debug_mode(void); /** * When debugging mode is enabled, informs Libevent that an event should no * longer be considered as assigned. When debugging mode is not enabled, does * nothing. * * This function must only be called on a non-added event. * * @see event_enable_debug_mode() */ EVENT2_EXPORT_SYMBOL void event_debug_unassign(struct event *); /** * Create and return a new event_base to use with the rest of Libevent. * * @return a new event_base on success, or NULL on failure. * * @see event_base_free(), event_base_new_with_config() */ EVENT2_EXPORT_SYMBOL struct event_base *event_base_new(void); /** Reinitialize the event base after a fork Some event mechanisms do not survive across fork. The event base needs to be reinitialized with the event_reinit() function. @param base the event base that needs to be re-initialized @return 0 if successful, or -1 if some events could not be re-added. @see event_base_new() */ EVENT2_EXPORT_SYMBOL int event_reinit(struct event_base *base); /** Event dispatching loop This loop will run the event base until either there are no more pending or active, or until something calls event_base_loopbreak() or event_base_loopexit(). @param base the event_base structure returned by event_base_new() or event_base_new_with_config() @return 0 if successful, -1 if an error occurred, or 1 if we exited because no events were pending or active. @see event_base_loop() */ EVENT2_EXPORT_SYMBOL int event_base_dispatch(struct event_base *); /** Get the kernel event notification mechanism used by Libevent. @param eb the event_base structure returned by event_base_new() @return a string identifying the kernel event mechanism (kqueue, epoll, etc.) */ EVENT2_EXPORT_SYMBOL const char *event_base_get_method(const struct event_base *); /** Gets all event notification mechanisms supported by Libevent. This functions returns the event mechanism in order preferred by Libevent. Note that this list will include all backends that Libevent has compiled-in support for, and will not necessarily check your OS to see whether it has the required resources. @return an array with pointers to the names of support methods. The end of the array is indicated by a NULL pointer. If an error is encountered NULL is returned. */ EVENT2_EXPORT_SYMBOL const char **event_get_supported_methods(void); /** Query the current monotonic time from a the timer for a struct * event_base. */ EVENT2_EXPORT_SYMBOL int event_gettime_monotonic(struct event_base *base, struct timeval *tp); /** @name event type flag Flags to pass to event_base_get_num_events() to specify the kinds of events we want to aggregate counts for */ /**@{*/ /** count the number of active events, which have been triggered.*/ #define EVENT_BASE_COUNT_ACTIVE 1U /** count the number of virtual events, which is used to represent an internal * condition, other than a pending event, that keeps the loop from exiting. */ #define EVENT_BASE_COUNT_VIRTUAL 2U /** count the number of events which have been added to event base, including * internal events. */ #define EVENT_BASE_COUNT_ADDED 4U /**@}*/ /** Gets the number of events in event_base, as specified in the flags. Since event base has some internal events added to make some of its functionalities work, EVENT_BASE_COUNT_ADDED may return more than the number of events you added using event_add(). If you pass EVENT_BASE_COUNT_ACTIVE and EVENT_BASE_COUNT_ADDED together, an active event will be counted twice. However, this might not be the case in future libevent versions. The return value is an indication of the work load, but the user shouldn't rely on the exact value as this may change in the future. @param eb the event_base structure returned by event_base_new() @param flags a bitwise combination of the kinds of events to aggregate counts for @return the number of events specified in the flags */ EVENT2_EXPORT_SYMBOL int event_base_get_num_events(struct event_base *, unsigned int); /** Get the maximum number of events in a given event_base as specified in the flags. @param eb the event_base structure returned by event_base_new() @param flags a bitwise combination of the kinds of events to aggregate counts for @param clear option used to reset the maximum count. @return the number of events specified in the flags */ EVENT2_EXPORT_SYMBOL int event_base_get_max_events(struct event_base *, unsigned int, int); /** Allocates a new event configuration object. The event configuration object can be used to change the behavior of an event base. @return an event_config object that can be used to store configuration, or NULL if an error is encountered. @see event_base_new_with_config(), event_config_free(), event_config */ EVENT2_EXPORT_SYMBOL struct event_config *event_config_new(void); /** Deallocates all memory associated with an event configuration object @param cfg the event configuration object to be freed. */ EVENT2_EXPORT_SYMBOL void event_config_free(struct event_config *cfg); /** Enters an event method that should be avoided into the configuration. This can be used to avoid event mechanisms that do not support certain file descriptor types, or for debugging to avoid certain event mechanisms. An application can make use of multiple event bases to accommodate incompatible file descriptor types. @param cfg the event configuration object @param method the name of the event method to avoid @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int event_config_avoid_method(struct event_config *cfg, const char *method); /** A flag used to describe which features an event_base (must) provide. Because of OS limitations, not every Libevent backend supports every possible feature. You can use this type with event_config_require_features() to tell Libevent to only proceed if your event_base implements a given feature, and you can receive this type from event_base_get_features() to see which features are available. */ enum event_method_feature { /** Require an event method that allows edge-triggered events with EV_ET. */ EV_FEATURE_ET = 0x01, /** Require an event method where having one event triggered among * many is [approximately] an O(1) operation. This excludes (for * example) select and poll, which are approximately O(N) for N * equal to the total number of possible events. */ EV_FEATURE_O1 = 0x02, /** Require an event method that allows file descriptors as well as * sockets. */ EV_FEATURE_FDS = 0x04, /** Require an event method that allows you to use EV_CLOSED to detect * connection close without the necessity of reading all the pending data. * * Methods that do support EV_CLOSED may not be able to provide support on * all kernel versions. **/ EV_FEATURE_EARLY_CLOSE = 0x08 }; /** A flag passed to event_config_set_flag(). These flags change the behavior of an allocated event_base. @see event_config_set_flag(), event_base_new_with_config(), event_method_feature */ enum event_base_config_flag { /** Do not allocate a lock for the event base, even if we have locking set up. Setting this option will make it unsafe and nonfunctional to call functions on the base concurrently from multiple threads. */ EVENT_BASE_FLAG_NOLOCK = 0x01, /** Do not check the EVENT_* environment variables when configuring an event_base */ EVENT_BASE_FLAG_IGNORE_ENV = 0x02, /** Windows only: enable the IOCP dispatcher at startup If this flag is set then bufferevent_socket_new() and evconn_listener_new() will use IOCP-backed implementations instead of the usual select-based one on Windows. */ EVENT_BASE_FLAG_STARTUP_IOCP = 0x04, /** Instead of checking the current time every time the event loop is ready to run timeout callbacks, check after each timeout callback. */ EVENT_BASE_FLAG_NO_CACHE_TIME = 0x08, /** If we are using the epoll backend, this flag says that it is safe to use Libevent's internal change-list code to batch up adds and deletes in order to try to do as few syscalls as possible. Setting this flag can make your code run faster, but it may trigger a Linux bug: it is not safe to use this flag if you have any fds cloned by dup() or its variants. Doing so will produce strange and hard-to-diagnose bugs. This flag can also be activated by setting the EVENT_EPOLL_USE_CHANGELIST environment variable. This flag has no effect if you wind up using a backend other than epoll. */ EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST = 0x10, /** Ordinarily, Libevent implements its time and timeout code using the fastest monotonic timer that we have. If this flag is set, however, we use less efficient more precise timer, assuming one is present. */ EVENT_BASE_FLAG_PRECISE_TIMER = 0x20 }; /** Return a bitmask of the features implemented by an event base. This will be a bitwise OR of one or more of the values of event_method_feature @see event_method_feature */ EVENT2_EXPORT_SYMBOL int event_base_get_features(const struct event_base *base); /** Enters a required event method feature that the application demands. Note that not every feature or combination of features is supported on every platform. Code that requests features should be prepared to handle the case where event_base_new_with_config() returns NULL, as in: 
     event_config_require_features(cfg, EV_FEATURE_ET);
     base = event_base_new_with_config(cfg);
     if (base == NULL) {
       // We can't get edge-triggered behavior here.
       event_config_require_features(cfg, 0);
       base = event_base_new_with_config(cfg);
     }
@param cfg the event configuration object @param feature a bitfield of one or more event_method_feature values. Replaces values from previous calls to this function. @return 0 on success, -1 on failure. @see event_method_feature, event_base_new_with_config() */ EVENT2_EXPORT_SYMBOL int event_config_require_features(struct event_config *cfg, int feature); /** * Sets one or more flags to configure what parts of the eventual event_base * will be initialized, and how they'll work. * * @see event_base_config_flags, event_base_new_with_config() **/ EVENT2_EXPORT_SYMBOL int event_config_set_flag(struct event_config *cfg, int flag); /** * Records a hint for the number of CPUs in the system. This is used for * tuning thread pools, etc, for optimal performance. In Libevent 2.0, * it is only on Windows, and only when IOCP is in use. * * @param cfg the event configuration object * @param cpus the number of cpus * @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int event_config_set_num_cpus_hint(struct event_config *cfg, int cpus); /** * Record an interval and/or a number of callbacks after which the event base * should check for new events. By default, the event base will run as many * events are as activated at the higest activated priority before checking * for new events. If you configure it by setting max_interval, it will check * the time after each callback, and not allow more than max_interval to * elapse before checking for new events. If you configure it by setting * max_callbacks to a value >= 0, it will run no more than max_callbacks * callbacks before checking for new events. * * This option can decrease the latency of high-priority events, and * avoid priority inversions where multiple low-priority events keep us from * polling for high-priority events, but at the expense of slightly decreasing * the throughput. Use it with caution! * * @param cfg The event_base configuration object. * @param max_interval An interval after which Libevent should stop running * callbacks and check for more events, or NULL if there should be * no such interval. * @param max_callbacks A number of callbacks after which Libevent should * stop running callbacks and check for more events, or -1 if there * should be no such limit. * @param min_priority A priority below which max_interval and max_callbacks * should not be enforced. If this is set to 0, they are enforced * for events of every priority; if it's set to 1, they're enforced * for events of priority 1 and above, and so on. * @return 0 on success, -1 on failure. **/ EVENT2_EXPORT_SYMBOL int event_config_set_max_dispatch_interval(struct event_config *cfg, const struct timeval *max_interval, int max_callbacks, int min_priority); /** Initialize the event API. Use event_base_new_with_config() to initialize a new event base, taking the specified configuration under consideration. The configuration object can currently be used to avoid certain event notification mechanisms. @param cfg the event configuration object @return an initialized event_base that can be used to registering events, or NULL if no event base can be created with the requested event_config. @see event_base_new(), event_base_free(), event_init(), event_assign() */ EVENT2_EXPORT_SYMBOL struct event_base *event_base_new_with_config(const struct event_config *); /** Deallocate all memory associated with an event_base, and free the base. Note that this function will not close any fds or free any memory passed to event_new as the argument to callback. If there are any pending finalizer callbacks, this function will invoke them. @param eb an event_base to be freed */ EVENT2_EXPORT_SYMBOL void event_base_free(struct event_base *); /** As event_free, but do not run finalizers. THIS IS AN EXPERIMENTAL API. IT MIGHT CHANGE BEFORE THE LIBEVENT 2.1 SERIES BECOMES STABLE. */ EVENT2_EXPORT_SYMBOL void event_base_free_nofinalize(struct event_base *); /** @name Log severities */ /**@{*/ #define EVENT_LOG_DEBUG 0 #define EVENT_LOG_MSG 1 #define EVENT_LOG_WARN 2 #define EVENT_LOG_ERR 3 /**@}*/ /* Obsolete names: these are deprecated, but older programs might use them. * They violate the reserved-identifier namespace. */ #define _EVENT_LOG_DEBUG EVENT_LOG_DEBUG #define _EVENT_LOG_MSG EVENT_LOG_MSG #define _EVENT_LOG_WARN EVENT_LOG_WARN #define _EVENT_LOG_ERR EVENT_LOG_ERR /** A callback function used to intercept Libevent's log messages. @see event_set_log_callback */ typedef void (*event_log_cb)(int severity, const char *msg); /** Redirect Libevent's log messages. @param cb a function taking two arguments: an integer severity between EVENT_LOG_DEBUG and EVENT_LOG_ERR, and a string. If cb is NULL, then the default log is used. NOTE: The function you provide *must not* call any other libevent functionality. Doing so can produce undefined behavior. */ EVENT2_EXPORT_SYMBOL void event_set_log_callback(event_log_cb cb); /** A function to be called if Libevent encounters a fatal internal error. @see event_set_fatal_callback */ typedef void (*event_fatal_cb)(int err); /** Override Libevent's behavior in the event of a fatal internal error. By default, Libevent will call exit(1) if a programming error makes it impossible to continue correct operation. This function allows you to supply another callback instead. Note that if the function is ever invoked, something is wrong with your program, or with Libevent: any subsequent calls to Libevent may result in undefined behavior. Libevent will (almost) always log an EVENT_LOG_ERR message before calling this function; look at the last log message to see why Libevent has died. */ EVENT2_EXPORT_SYMBOL void event_set_fatal_callback(event_fatal_cb cb); #define EVENT_DBG_ALL 0xffffffffu #define EVENT_DBG_NONE 0 /** Turn on debugging logs and have them sent to the default log handler. This is a global setting; if you are going to call it, you must call this before any calls that create an event-base. You must call it before any multithreaded use of Libevent. Debug logs are verbose. @param which Controls which debug messages are turned on. This option is unused for now; for forward compatibility, you must pass in the constant "EVENT_DBG_ALL" to turn debugging logs on, or "EVENT_DBG_NONE" to turn debugging logs off. */ EVENT2_EXPORT_SYMBOL void event_enable_debug_logging(ev_uint32_t which); /** Associate a different event base with an event. The event to be associated must not be currently active or pending. @param eb the event base @param ev the event @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int event_base_set(struct event_base *, struct event *); /** @name Loop flags These flags control the behavior of event_base_loop(). */ /**@{*/ /** Block until we have an active event, then exit once all active events * have had their callbacks run. */ #define EVLOOP_ONCE 0x01 /** Do not block: see which events are ready now, run the callbacks * of the highest-priority ones, then exit. */ #define EVLOOP_NONBLOCK 0x02 /** Do not exit the loop because we have no pending events. Instead, keep * running until event_base_loopexit() or event_base_loopbreak() makes us * stop. */ #define EVLOOP_NO_EXIT_ON_EMPTY 0x04 /**@}*/ /** Wait for events to become active, and run their callbacks. This is a more flexible version of event_base_dispatch(). By default, this loop will run the event base until either there are no more pending or active events, or until something calls event_base_loopbreak() or event_base_loopexit(). You can override this behavior with the 'flags' argument. @param eb the event_base structure returned by event_base_new() or event_base_new_with_config() @param flags any combination of EVLOOP_ONCE | EVLOOP_NONBLOCK @return 0 if successful, -1 if an error occurred, or 1 if we exited because no events were pending or active. @see event_base_loopexit(), event_base_dispatch(), EVLOOP_ONCE, EVLOOP_NONBLOCK */ EVENT2_EXPORT_SYMBOL int event_base_loop(struct event_base *, int); /** Exit the event loop after the specified time The next event_base_loop() iteration after the given timer expires will complete normally (handling all queued events) then exit without blocking for events again. Subsequent invocations of event_base_loop() will proceed normally. @param eb the event_base structure returned by event_init() @param tv the amount of time after which the loop should terminate, or NULL to exit after running all currently active events. @return 0 if successful, or -1 if an error occurred @see event_base_loopbreak() */ EVENT2_EXPORT_SYMBOL int event_base_loopexit(struct event_base *, const struct timeval *); /** Abort the active event_base_loop() immediately. event_base_loop() will abort the loop after the next event is completed; event_base_loopbreak() is typically invoked from this event's callback. This behavior is analogous to the "break;" statement. Subsequent invocations of event_base_loop() will proceed normally. @param eb the event_base structure returned by event_init() @return 0 if successful, or -1 if an error occurred @see event_base_loopexit() */ EVENT2_EXPORT_SYMBOL int event_base_loopbreak(struct event_base *); /** Tell the active event_base_loop() to scan for new events immediately. Calling this function makes the currently active event_base_loop() start the loop over again (scanning for new events) after the current event callback finishes. If the event loop is not running, this function has no effect. event_base_loopbreak() is typically invoked from this event's callback. This behavior is analogous to the "continue;" statement. Subsequent invocations of event loop will proceed normally. @param eb the event_base structure returned by event_init() @return 0 if successful, or -1 if an error occurred @see event_base_loopbreak() */ EVENT2_EXPORT_SYMBOL int event_base_loopcontinue(struct event_base *); /** Checks if the event loop was told to exit by event_base_loopexit(). This function will return true for an event_base at every point after event_loopexit() is called, until the event loop is next entered. @param eb the event_base structure returned by event_init() @return true if event_base_loopexit() was called on this event base, or 0 otherwise @see event_base_loopexit() @see event_base_got_break() */ EVENT2_EXPORT_SYMBOL int event_base_got_exit(struct event_base *); /** Checks if the event loop was told to abort immediately by event_base_loopbreak(). This function will return true for an event_base at every point after event_base_loopbreak() is called, until the event loop is next entered. @param eb the event_base structure returned by event_init() @return true if event_base_loopbreak() was called on this event base, or 0 otherwise @see event_base_loopbreak() @see event_base_got_exit() */ EVENT2_EXPORT_SYMBOL int event_base_got_break(struct event_base *); /** * @name event flags * * Flags to pass to event_new(), event_assign(), event_pending(), and * anything else with an argument of the form "short events" */ /**@{*/ /** Indicates that a timeout has occurred. It's not necessary to pass * this flag to event_for new()/event_assign() to get a timeout. */ #define EV_TIMEOUT 0x01 /** Wait for a socket or FD to become readable */ #define EV_READ 0x02 /** Wait for a socket or FD to become writeable */ #define EV_WRITE 0x04 /** Wait for a POSIX signal to be raised*/ #define EV_SIGNAL 0x08 /** * Persistent event: won't get removed automatically when activated. * * When a persistent event with a timeout becomes activated, its timeout * is reset to 0. */ #define EV_PERSIST 0x10 /** Select edge-triggered behavior, if supported by the backend. */ #define EV_ET 0x20 /** * If this option is provided, then event_del() will not block in one thread * while waiting for the event callback to complete in another thread. * * To use this option safely, you may need to use event_finalize() or * event_free_finalize() in order to safely tear down an event in a * multithreaded application. See those functions for more information. * * THIS IS AN EXPERIMENTAL API. IT MIGHT CHANGE BEFORE THE LIBEVENT 2.1 SERIES * BECOMES STABLE. **/ #define EV_FINALIZE 0x40 /** * Detects connection close events. You can use this to detect when a * connection has been closed, without having to read all the pending data * from a connection. * * Not all backends support EV_CLOSED. To detect or require it, use the * feature flag EV_FEATURE_EARLY_CLOSE. **/ #define EV_CLOSED 0x80 /**@}*/ /** @name evtimer_* macros Aliases for working with one-shot timer events */ /**@{*/ #define evtimer_assign(ev, b, cb, arg) \ event_assign((ev), (b), -1, 0, (cb), (arg)) #define evtimer_new(b, cb, arg) event_new((b), -1, 0, (cb), (arg)) #define evtimer_add(ev, tv) event_add((ev), (tv)) #define evtimer_del(ev) event_del(ev) #define evtimer_pending(ev, tv) event_pending((ev), EV_TIMEOUT, (tv)) #define evtimer_initialized(ev) event_initialized(ev) /**@}*/ /** @name evsignal_* macros Aliases for working with signal events */ /**@{*/ #define evsignal_add(ev, tv) event_add((ev), (tv)) #define evsignal_assign(ev, b, x, cb, arg) \ event_assign((ev), (b), (x), EV_SIGNAL|EV_PERSIST, cb, (arg)) #define evsignal_new(b, x, cb, arg) \ event_new((b), (x), EV_SIGNAL|EV_PERSIST, (cb), (arg)) #define evsignal_del(ev) event_del(ev) #define evsignal_pending(ev, tv) event_pending((ev), EV_SIGNAL, (tv)) #define evsignal_initialized(ev) event_initialized(ev) /**@}*/ /** A callback function for an event. It receives three arguments: @param fd An fd or signal @param events One or more EV_* flags @param arg A user-supplied argument. @see event_new() */ typedef void (*event_callback_fn)(evutil_socket_t, short, void *); /** Return a value used to specify that the event itself must be used as the callback argument. The function event_new() takes a callback argument which is passed to the event's callback function. To specify that the argument to be passed to the callback function is the event that event_new() returns, pass in the return value of event_self_cbarg() as the callback argument for event_new(). For example: 
      struct event *ev = event_new(base, sock, events, callback, %event_self_cbarg());
For consistency with event_new(), it is possible to pass the return value of this function as the callback argument for event_assign() – this achieves the same result as passing the event in directly. @return a value to be passed as the callback argument to event_new() or event_assign(). @see event_new(), event_assign() */ EVENT2_EXPORT_SYMBOL void *event_self_cbarg(void); /** Allocate and asssign a new event structure, ready to be added. The function event_new() returns a new event that can be used in future calls to event_add() and event_del(). The fd and events arguments determine which conditions will trigger the event; the callback and callback_arg arguments tell Libevent what to do when the event becomes active. If events contains one of EV_READ, EV_WRITE, or EV_READ|EV_WRITE, then fd is a file descriptor or socket that should get monitored for readiness to read, readiness to write, or readiness for either operation (respectively). If events contains EV_SIGNAL, then fd is a signal number to wait for. If events contains none of those flags, then the event can be triggered only by a timeout or by manual activation with event_active(): In this case, fd must be -1. The EV_PERSIST flag can also be passed in the events argument: it makes event_add() persistent until event_del() is called. The EV_ET flag is compatible with EV_READ and EV_WRITE, and supported only by certain backends. It tells Libevent to use edge-triggered events. The EV_TIMEOUT flag has no effect here. It is okay to have multiple events all listening on the same fds; but they must either all be edge-triggered, or all not be edge triggerd. When the event becomes active, the event loop will run the provided callbuck function, with three arguments. The first will be the provided fd value. The second will be a bitfield of the events that triggered: EV_READ, EV_WRITE, or EV_SIGNAL. Here the EV_TIMEOUT flag indicates that a timeout occurred, and EV_ET indicates that an edge-triggered event occurred. The third event will be the callback_arg pointer that you provide. @param base the event base to which the event should be attached. @param fd the file descriptor or signal to be monitored, or -1. @param events desired events to monitor: bitfield of EV_READ, EV_WRITE, EV_SIGNAL, EV_PERSIST, EV_ET. @param callback callback function to be invoked when the event occurs @param callback_arg an argument to be passed to the callback function @return a newly allocated struct event that must later be freed with event_free(). @see event_free(), event_add(), event_del(), event_assign() */ EVENT2_EXPORT_SYMBOL struct event *event_new(struct event_base *, evutil_socket_t, short, event_callback_fn, void *); /** Prepare a new, already-allocated event structure to be added. The function event_assign() prepares the event structure ev to be used in future calls to event_add() and event_del(). Unlike event_new(), it doesn't allocate memory itself: it requires that you have already allocated a struct event, probably on the heap. Doing this will typically make your code depend on the size of the event structure, and thereby create incompatibility with future versions of Libevent. The easiest way to avoid this problem is just to use event_new() and event_free() instead. A slightly harder way to future-proof your code is to use event_get_struct_event_size() to determine the required size of an event at runtime. Note that it is NOT safe to call this function on an event that is active or pending. Doing so WILL corrupt internal data structures in Libevent, and lead to strange, hard-to-diagnose bugs. You _can_ use event_assign to change an existing event, but only if it is not active or pending! The arguments for this function, and the behavior of the events that it makes, are as for event_new(). @param ev an event struct to be modified @param base the event base to which ev should be attached. @param fd the file descriptor to be monitored @param events desired events to monitor; can be EV_READ and/or EV_WRITE @param callback callback function to be invoked when the event occurs @param callback_arg an argument to be passed to the callback function @return 0 if success, or -1 on invalid arguments. @see event_new(), event_add(), event_del(), event_base_once(), event_get_struct_event_size() */ EVENT2_EXPORT_SYMBOL int event_assign(struct event *, struct event_base *, evutil_socket_t, short, event_callback_fn, void *); /** Deallocate a struct event * returned by event_new(). If the event is pending or active, first make it non-pending and non-active. */ EVENT2_EXPORT_SYMBOL void event_free(struct event *); /** * Callback type for event_finalize and event_free_finalize(). * * THIS IS AN EXPERIMENTAL API. IT MIGHT CHANGE BEFORE THE LIBEVENT 2.1 SERIES * BECOMES STABLE. * **/ typedef void (*event_finalize_callback_fn)(struct event *, void *); /** @name Finalization functions These functions are used to safely tear down an event in a multithreaded application. If you construct your events with EV_FINALIZE to avoid deadlocks, you will need a way to remove an event in the certainty that it will definitely not be running its callback when you deallocate it and its callback argument. To do this, call one of event_finalize() or event_free_finalize with 0 for its first argument, the event to tear down as its second argument, and a callback function as its third argument. The callback will be invoked as part of the event loop, with the event's priority. After you call a finalizer function, event_add() and event_active() will no longer work on the event, and event_del() will produce a no-op. You must not try to change the event's fields with event_assign() or event_set() while the finalize callback is in progress. Once the callback has been invoked, you should treat the event structure as containing uninitialized memory. The event_free_finalize() function frees the event after it's finalized; event_finalize() does not. A finalizer callback must not make events pending or active. It must not add events, activate events, or attempt to "resucitate" the event being finalized in any way. THIS IS AN EXPERIMENTAL API. IT MIGHT CHANGE BEFORE THE LIBEVENT 2.1 SERIES BECOMES STABLE. @return 0 on succes, -1 on failure. */ /**@{*/ EVENT2_EXPORT_SYMBOL int event_finalize(unsigned, struct event *, event_finalize_callback_fn); EVENT2_EXPORT_SYMBOL int event_free_finalize(unsigned, struct event *, event_finalize_callback_fn); /**@}*/ /** Schedule a one-time event The function event_base_once() is similar to event_new(). However, it schedules a callback to be called exactly once, and does not require the caller to prepare an event structure. Note that in Libevent 2.0 and earlier, if the event is never triggered, the internal memory used to hold it will never be freed. In Libevent 2.1, the internal memory will get freed by event_base_free() if the event is never triggered. The 'arg' value, however, will not get freed in either case--you'll need to free that on your own if you want it to go away. @param base an event_base @param fd a file descriptor to monitor, or -1 for no fd. @param events event(s) to monitor; can be any of EV_READ | EV_WRITE, or EV_TIMEOUT @param callback callback function to be invoked when the event occurs @param arg an argument to be passed to the callback function @param timeout the maximum amount of time to wait for the event. NULL makes an EV_READ/EV_WRITE event make forever; NULL makes an EV_TIMEOUT event succees immediately. @return 0 if successful, or -1 if an error occurred */ EVENT2_EXPORT_SYMBOL int event_base_once(struct event_base *, evutil_socket_t, short, event_callback_fn, void *, const struct timeval *); /** Add an event to the set of pending events. The function event_add() schedules the execution of the event 'ev' when the condition specified by event_assign() or event_new() occurs, or when the time specified in timeout has elapesed. If atimeout is NULL, no timeout occurs and the function will only be called if a matching event occurs. The event in the ev argument must be already initialized by event_assign() or event_new() and may not be used in calls to event_assign() until it is no longer pending. If the event in the ev argument already has a scheduled timeout, calling event_add() replaces the old timeout with the new one if tv is non-NULL. @param ev an event struct initialized via event_assign() or event_new() @param timeout the maximum amount of time to wait for the event, or NULL to wait forever @return 0 if successful, or -1 if an error occurred @see event_del(), event_assign(), event_new() */ EVENT2_EXPORT_SYMBOL int event_add(struct event *ev, const struct timeval *timeout); /** Remove a timer from a pending event without removing the event itself. If the event has a scheduled timeout, this function unschedules it but leaves the event otherwise pending. @param ev an event struct initialized via event_assign() or event_new() @return 0 on success, or -1 if an error occurrect. */ EVENT2_EXPORT_SYMBOL int event_remove_timer(struct event *ev); /** Remove an event from the set of monitored events. The function event_del() will cancel the event in the argument ev. If the event has already executed or has never been added the call will have no effect. @param ev an event struct to be removed from the working set @return 0 if successful, or -1 if an error occurred @see event_add() */ EVENT2_EXPORT_SYMBOL int event_del(struct event *); /** As event_del(), but never blocks while the event's callback is running in another thread, even if the event was constructed without the EV_FINALIZE flag. THIS IS AN EXPERIMENTAL API. IT MIGHT CHANGE BEFORE THE LIBEVENT 2.1 SERIES BECOMES STABLE. */ EVENT2_EXPORT_SYMBOL int event_del_noblock(struct event *ev); /** As event_del(), but always blocks while the event's callback is running in another thread, even if the event was constructed with the EV_FINALIZE flag. THIS IS AN EXPERIMENTAL API. IT MIGHT CHANGE BEFORE THE LIBEVENT 2.1 SERIES BECOMES STABLE. */ EVENT2_EXPORT_SYMBOL int event_del_block(struct event *ev); /** Make an event active. You can use this function on a pending or a non-pending event to make it active, so that its callback will be run by event_base_dispatch() or event_base_loop(). One common use in multithreaded programs is to wake the thread running event_base_loop() from another thread. @param ev an event to make active. @param res a set of flags to pass to the event's callback. @param ncalls an obsolete argument: this is ignored. **/ EVENT2_EXPORT_SYMBOL void event_active(struct event *ev, int res, short ncalls); /** Checks if a specific event is pending or scheduled. @param ev an event struct previously passed to event_add() @param events the requested event type; any of EV_TIMEOUT|EV_READ| EV_WRITE|EV_SIGNAL @param tv if this field is not NULL, and the event has a timeout, this field is set to hold the time at which the timeout will expire. @return true if the event is pending on any of the events in 'what', (that is to say, it has been added), or 0 if the event is not added. */ EVENT2_EXPORT_SYMBOL int event_pending(const struct event *ev, short events, struct timeval *tv); /** If called from within the callback for an event, returns that event. The behavior of this function is not defined when called from outside the callback function for an event. */ EVENT2_EXPORT_SYMBOL struct event *event_base_get_running_event(struct event_base *base); /** Test if an event structure might be initialized. The event_initialized() function can be used to check if an event has been initialized. Warning: This function is only useful for distinguishing a a zeroed-out piece of memory from an initialized event, it can easily be confused by uninitialized memory. Thus, it should ONLY be used to distinguish an initialized event from zero. @param ev an event structure to be tested @return 1 if the structure might be initialized, or 0 if it has not been initialized */ EVENT2_EXPORT_SYMBOL int event_initialized(const struct event *ev); /** Get the signal number assigned to a signal event */ #define event_get_signal(ev) ((int)event_get_fd(ev)) /** Get the socket or signal assigned to an event, or -1 if the event has no socket. */ EVENT2_EXPORT_SYMBOL evutil_socket_t event_get_fd(const struct event *ev); /** Get the event_base associated with an event. */ EVENT2_EXPORT_SYMBOL struct event_base *event_get_base(const struct event *ev); /** Return the events (EV_READ, EV_WRITE, etc) assigned to an event. */ EVENT2_EXPORT_SYMBOL short event_get_events(const struct event *ev); /** Return the callback assigned to an event. */ EVENT2_EXPORT_SYMBOL event_callback_fn event_get_callback(const struct event *ev); /** Return the callback argument assigned to an event. */ EVENT2_EXPORT_SYMBOL void *event_get_callback_arg(const struct event *ev); /** Return the priority of an event. @see event_priority_init(), event_get_priority() */ EVENT2_EXPORT_SYMBOL int event_get_priority(const struct event *ev); /** Extract _all_ of arguments given to construct a given event. The event_base is copied into *base_out, the fd is copied into *fd_out, and so on. If any of the "_out" arguments is NULL, it will be ignored. */ EVENT2_EXPORT_SYMBOL void event_get_assignment(const struct event *event, struct event_base **base_out, evutil_socket_t *fd_out, short *events_out, event_callback_fn *callback_out, void **arg_out); /** Return the size of struct event that the Libevent library was compiled with. This will be NO GREATER than sizeof(struct event) if you're running with the same version of Libevent that your application was built with, but otherwise might not. Note that it might be SMALLER than sizeof(struct event) if some future version of Libevent adds extra padding to the end of struct event. We might do this to help ensure ABI-compatibility between different versions of Libevent. */ EVENT2_EXPORT_SYMBOL size_t event_get_struct_event_size(void); /** Get the Libevent version. Note that this will give you the version of the library that you're currently linked against, not the version of the headers that you've compiled against. @return a string containing the version number of Libevent */ EVENT2_EXPORT_SYMBOL const char *event_get_version(void); /** Return a numeric representation of Libevent's version. Note that this will give you the version of the library that you're currently linked against, not the version of the headers you've used to compile. The format uses one byte each for the major, minor, and patchlevel parts of the version number. The low-order byte is unused. For example, version 2.0.1-alpha has a numeric representation of 0x02000100 */ EVENT2_EXPORT_SYMBOL ev_uint32_t event_get_version_number(void); /** As event_get_version, but gives the version of Libevent's headers. */ #define LIBEVENT_VERSION EVENT__VERSION /** As event_get_version_number, but gives the version number of Libevent's * headers. */ #define LIBEVENT_VERSION_NUMBER EVENT__NUMERIC_VERSION /** Largest number of priorities that Libevent can support. */ #define EVENT_MAX_PRIORITIES 256 /** Set the number of different event priorities By default Libevent schedules all active events with the same priority. However, some time it is desirable to process some events with a higher priority than others. For that reason, Libevent supports strict priority queues. Active events with a lower priority are always processed before events with a higher priority. The number of different priorities can be set initially with the event_base_priority_init() function. This function should be called before the first call to event_base_dispatch(). The event_priority_set() function can be used to assign a priority to an event. By default, Libevent assigns the middle priority to all events unless their priority is explicitly set. Note that urgent-priority events can starve less-urgent events: after running all urgent-priority callbacks, Libevent checks for more urgent events again, before running less-urgent events. Less-urgent events will not have their callbacks run until there are no events more urgent than them that want to be active. @param eb the event_base structure returned by event_base_new() @param npriorities the maximum number of priorities @return 0 if successful, or -1 if an error occurred @see event_priority_set() */ EVENT2_EXPORT_SYMBOL int event_base_priority_init(struct event_base *, int); /** Get the number of different event priorities. @param eb the event_base structure returned by event_base_new() @return Number of different event priorities @see event_base_priority_init() */ EVENT2_EXPORT_SYMBOL int event_base_get_npriorities(struct event_base *eb); /** Assign a priority to an event. @param ev an event struct @param priority the new priority to be assigned @return 0 if successful, or -1 if an error occurred @see event_priority_init(), event_get_priority() */ EVENT2_EXPORT_SYMBOL int event_priority_set(struct event *, int); /** Prepare an event_base to use a large number of timeouts with the same duration. Libevent's default scheduling algorithm is optimized for having a large number of timeouts with their durations more or less randomly distributed. But if you have a large number of timeouts that all have the same duration (for example, if you have a large number of connections that all have a 10-second timeout), then you can improve Libevent's performance by telling Libevent about it. To do this, call this function with the common duration. It will return a pointer to a different, opaque timeout value. (Don't depend on its actual contents!) When you use this timeout value in event_add(), Libevent will schedule the event more efficiently. (This optimization probably will not be worthwhile until you have thousands or tens of thousands of events with the same timeout.) */ EVENT2_EXPORT_SYMBOL const struct timeval *event_base_init_common_timeout(struct event_base *base, const struct timeval *duration); #if !defined(EVENT__DISABLE_MM_REPLACEMENT) || defined(EVENT_IN_DOXYGEN_) /** Override the functions that Libevent uses for memory management. Usually, Libevent uses the standard libc functions malloc, realloc, and free to allocate memory. Passing replacements for those functions to event_set_mem_functions() overrides this behavior. Note that all memory returned from Libevent will be allocated by the replacement functions rather than by malloc() and realloc(). Thus, if you have replaced those functions, it will not be appropriate to free() memory that you get from Libevent. Instead, you must use the free_fn replacement that you provided. Note also that if you are going to call this function, you should do so before any call to any Libevent function that does allocation. Otherwise, those funtions will allocate their memory using malloc(), but then later free it using your provided free_fn. @param malloc_fn A replacement for malloc. @param realloc_fn A replacement for realloc @param free_fn A replacement for free. **/ EVENT2_EXPORT_SYMBOL void event_set_mem_functions( void *(*malloc_fn)(size_t sz), void *(*realloc_fn)(void *ptr, size_t sz), void (*free_fn)(void *ptr)); /** This definition is present if Libevent was built with support for event_set_mem_functions() */ #define EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED #endif /** Writes a human-readable description of all inserted and/or active events to a provided stdio stream. This is intended for debugging; its format is not guaranteed to be the same between libevent versions. @param base An event_base on which to scan the events. @param output A stdio file to write on. */ EVENT2_EXPORT_SYMBOL void event_base_dump_events(struct event_base *, FILE *); /** Activates all pending events for the given fd and event mask. This function activates pending events only. Events which have not been added will not become active. @param base the event_base on which to activate the events. @param fd An fd to active events on. @param events One or more of EV_{READ,WRITE}. */ EVENT2_EXPORT_SYMBOL void event_base_active_by_fd(struct event_base *base, evutil_socket_t fd, short events); /** Activates all pending signals with a given signal number This function activates pending events only. Events which have not been added will not become active. @param base the event_base on which to activate the events. @param fd The signal to active events on. */ EVENT2_EXPORT_SYMBOL void event_base_active_by_signal(struct event_base *base, int sig); /** * Callback for iterating events in an event base via event_base_foreach_event */ typedef int (*event_base_foreach_event_cb)(const struct event_base *, const struct event *, void *); /** Iterate over all added or active events events in an event loop, and invoke a given callback on each one. The callback must not call any function that modifies the event base, that modifies any event in the event base, or that adds or removes any event to the event base. Doing so is unsupported and will lead to undefined behavior -- likely, to crashes. event_base_foreach_event() holds a lock on the event_base() for the whole time it's running: slow callbacks are not advisable. Note that Libevent adds some events of its own to make pieces of its functionality work. You must not assume that the only events you'll encounter will be the ones you added yourself. The callback function must return 0 to continue iteration, or some other integer to stop iterating. @param base An event_base on which to scan the events. @param fn A callback function to receive the events. @param arg An argument passed to the callback function. @return 0 if we iterated over every event, or the value returned by the callback function if the loop exited early. */ EVENT2_EXPORT_SYMBOL int event_base_foreach_event(struct event_base *base, event_base_foreach_event_cb fn, void *arg); /** Sets 'tv' to the current time (as returned by gettimeofday()), looking at the cached value in 'base' if possible, and calling gettimeofday() or clock_gettime() as appropriate if there is no cached time. Generally, this value will only be cached while actually processing event callbacks, and may be very inaccuate if your callbacks take a long time to execute. Returns 0 on success, negative on failure. */ EVENT2_EXPORT_SYMBOL int event_base_gettimeofday_cached(struct event_base *base, struct timeval *tv); /** Update cached_tv in the 'base' to the current time * * You can use this function is useful for selectively increasing * the accuracy of the cached time value in 'base' during callbacks * that take a long time to execute. * * This function has no effect if the base is currently not in its * event loop, or if timeval caching is disabled via * EVENT_BASE_FLAG_NO_CACHE_TIME. * * @return 0 on success, -1 on failure */ EVENT2_EXPORT_SYMBOL int event_base_update_cache_time(struct event_base *base); /** Release up all globally-allocated resources allocated by Libevent. This function does not free developer-controlled resources like event_bases, events, bufferevents, listeners, and so on. It only releases resources like global locks that there is no other way to free. It is not actually necessary to call this function before exit: every resource that it frees would be released anyway on exit. It mainly exists so that resource-leak debugging tools don't see Libevent as holding resources at exit. You should only call this function when no other Libevent functions will be invoked -- e.g., when cleanly exiting a program. */ EVENT2_EXPORT_SYMBOL void libevent_global_shutdown(void); #ifdef __cplusplus } #endif #endif /* EVENT2_EVENT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/event_compat.h000066400000000000000000000167271527052371200254760ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_EVENT_COMPAT_H_INCLUDED_ #define EVENT2_EVENT_COMPAT_H_INCLUDED_ /** @file event2/event_compat.h Potentially non-threadsafe versions of the functions in event.h: provided only for backwards compatibility. In the oldest versions of Libevent, event_base was not a first-class structure. Instead, there was a single event base that every function manipulated. Later, when separate event bases were added, the old functions that didn't take an event_base argument needed to work by manipulating the "current" event base. This could lead to thread-safety issues, and obscure, hard-to-diagnose bugs. @deprecated All functions in this file are by definition deprecated. */ #include #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include /** Initialize the event API. The event API needs to be initialized with event_init() before it can be used. Sets the global current base that gets used for events that have no base associated with them. @deprecated This function is deprecated because it replaces the "current" event_base, and is totally unsafe for multithreaded use. The replacement is event_base_new(). @see event_base_set(), event_base_new() */ EVENT2_EXPORT_SYMBOL struct event_base *event_init(void); /** Loop to process events. Like event_base_dispatch(), but uses the "current" base. @deprecated This function is deprecated because it is easily confused by multiple calls to event_init(), and because it is not safe for multithreaded use. The replacement is event_base_dispatch(). @see event_base_dispatch(), event_init() */ EVENT2_EXPORT_SYMBOL int event_dispatch(void); /** Handle events. This function behaves like event_base_loop(), but uses the "current" base @deprecated This function is deprecated because it uses the event base from the last call to event_init, and is therefore not safe for multithreaded use. The replacement is event_base_loop(). @see event_base_loop(), event_init() */ EVENT2_EXPORT_SYMBOL int event_loop(int); /** Exit the event loop after the specified time. This function behaves like event_base_loopexit(), except that it uses the "current" base. @deprecated This function is deprecated because it uses the event base from the last call to event_init, and is therefore not safe for multithreaded use. The replacement is event_base_loopexit(). @see event_init, event_base_loopexit() */ EVENT2_EXPORT_SYMBOL int event_loopexit(const struct timeval *); /** Abort the active event_loop() immediately. This function behaves like event_base_loopbreakt(), except that it uses the "current" base. @deprecated This function is deprecated because it uses the event base from the last call to event_init, and is therefore not safe for multithreaded use. The replacement is event_base_loopbreak(). @see event_base_loopbreak(), event_init() */ EVENT2_EXPORT_SYMBOL int event_loopbreak(void); /** Schedule a one-time event to occur. @deprecated This function is obsolete, and has been replaced by event_base_once(). Its use is deprecated because it relies on the "current" base configured by event_init(). @see event_base_once() */ EVENT2_EXPORT_SYMBOL int event_once(evutil_socket_t , short, void (*)(evutil_socket_t, short, void *), void *, const struct timeval *); /** Get the kernel event notification mechanism used by Libevent. @deprecated This function is obsolete, and has been replaced by event_base_get_method(). Its use is deprecated because it relies on the "current" base configured by event_init(). @see event_base_get_method() */ EVENT2_EXPORT_SYMBOL const char *event_get_method(void); /** Set the number of different event priorities. @deprecated This function is deprecated because it is easily confused by multiple calls to event_init(), and because it is not safe for multithreaded use. The replacement is event_base_priority_init(). @see event_base_priority_init() */ EVENT2_EXPORT_SYMBOL int event_priority_init(int); /** Prepare an event structure to be added. @deprecated event_set() is not recommended for new code, because it requires a subsequent call to event_base_set() to be safe under most circumstances. Use event_assign() or event_new() instead. */ EVENT2_EXPORT_SYMBOL void event_set(struct event *, evutil_socket_t, short, void (*)(evutil_socket_t, short, void *), void *); #define evtimer_set(ev, cb, arg) event_set((ev), -1, 0, (cb), (arg)) #define evsignal_set(ev, x, cb, arg) \ event_set((ev), (x), EV_SIGNAL|EV_PERSIST, (cb), (arg)) /** @name timeout_* macros @deprecated These macros are deprecated because their naming is inconsistent with the rest of Libevent. Use the evtimer_* macros instead. @{ */ #define timeout_add(ev, tv) event_add((ev), (tv)) #define timeout_set(ev, cb, arg) event_set((ev), -1, 0, (cb), (arg)) #define timeout_del(ev) event_del(ev) #define timeout_pending(ev, tv) event_pending((ev), EV_TIMEOUT, (tv)) #define timeout_initialized(ev) event_initialized(ev) /**@}*/ /** @name signal_* macros @deprecated These macros are deprecated because their naming is inconsistent with the rest of Libevent. Use the evsignal_* macros instead. @{ */ #define signal_add(ev, tv) event_add((ev), (tv)) #define signal_set(ev, x, cb, arg) \ event_set((ev), (x), EV_SIGNAL|EV_PERSIST, (cb), (arg)) #define signal_del(ev) event_del(ev) #define signal_pending(ev, tv) event_pending((ev), EV_SIGNAL, (tv)) #define signal_initialized(ev) event_initialized(ev) /**@}*/ #ifndef EVENT_FD /* These macros are obsolete; use event_get_fd and event_get_signal instead. */ #define EVENT_FD(ev) ((int)event_get_fd(ev)) #define EVENT_SIGNAL(ev) event_get_signal(ev) #endif #ifdef __cplusplus } #endif #endif /* EVENT2_EVENT_COMPAT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/event_struct.h000066400000000000000000000116401527052371200255240ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_EVENT_STRUCT_H_INCLUDED_ #define EVENT2_EVENT_STRUCT_H_INCLUDED_ /** @file event2/event_struct.h Structures used by event.h. Using these structures directly WILL harm forward compatibility: be careful. No field declared in this file should be used directly in user code. Except for historical reasons, these fields would not be exposed at all. */ #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include /* For evkeyvalq */ #include #define EVLIST_TIMEOUT 0x01 #define EVLIST_INSERTED 0x02 #define EVLIST_SIGNAL 0x04 #define EVLIST_ACTIVE 0x08 #define EVLIST_INTERNAL 0x10 #define EVLIST_ACTIVE_LATER 0x20 #define EVLIST_FINALIZING 0x40 #define EVLIST_INIT 0x80 #define EVLIST_ALL 0xff /* Fix so that people don't have to run with */ #ifndef TAILQ_ENTRY #define EVENT_DEFINED_TQENTRY_ #define TAILQ_ENTRY(type) \ struct { \ struct type *tqe_next; /* next element */ \ struct type **tqe_prev; /* address of previous next element */ \ } #endif /* !TAILQ_ENTRY */ #ifndef TAILQ_HEAD #define EVENT_DEFINED_TQHEAD_ #define TAILQ_HEAD(name, type) \ struct name { \ struct type *tqh_first; \ struct type **tqh_last; \ } #endif /* Fix so that people don't have to run with */ #ifndef LIST_ENTRY #define EVENT_DEFINED_LISTENTRY_ #define LIST_ENTRY(type) \ struct { \ struct type *le_next; /* next element */ \ struct type **le_prev; /* address of previous next element */ \ } #endif /* !LIST_ENTRY */ #ifndef LIST_HEAD #define EVENT_DEFINED_LISTHEAD_ #define LIST_HEAD(name, type) \ struct name { \ struct type *lh_first; /* first element */ \ } #endif /* !LIST_HEAD */ struct event; struct event_callback { TAILQ_ENTRY(event_callback) evcb_active_next; short evcb_flags; ev_uint8_t evcb_pri; /* smaller numbers are higher priority */ ev_uint8_t evcb_closure; /* allows us to adopt for different types of events */ union { void (*evcb_callback)(evutil_socket_t, short, void *); void (*evcb_selfcb)(struct event_callback *, void *); void (*evcb_evfinalize)(struct event *, void *); void (*evcb_cbfinalize)(struct event_callback *, void *); } evcb_cb_union; void *evcb_arg; }; struct event_base; struct event { struct event_callback ev_evcallback; /* for managing timeouts */ union { TAILQ_ENTRY(event) ev_next_with_common_timeout; int min_heap_idx; } ev_timeout_pos; evutil_socket_t ev_fd; struct event_base *ev_base; union { /* used for io events */ struct { LIST_ENTRY (event) ev_io_next; struct timeval ev_timeout; } ev_io; /* used by signal events */ struct { LIST_ENTRY (event) ev_signal_next; short ev_ncalls; /* Allows deletes in callback */ short *ev_pncalls; } ev_signal; } ev_; short ev_events; short ev_res; /* result passed to event callback */ struct timeval ev_timeout; }; TAILQ_HEAD (event_list, event); #ifdef EVENT_DEFINED_TQENTRY_ #undef TAILQ_ENTRY #endif #ifdef EVENT_DEFINED_TQHEAD_ #undef TAILQ_HEAD #endif LIST_HEAD (event_dlist, event); #ifdef EVENT_DEFINED_LISTENTRY_ #undef LIST_ENTRY #endif #ifdef EVENT_DEFINED_LISTHEAD_ #undef LIST_HEAD #endif #ifdef __cplusplus } #endif #endif /* EVENT2_EVENT_STRUCT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/http.h000066400000000000000000001234031527052371200237570ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_HTTP_H_INCLUDED_ #define EVENT2_HTTP_H_INCLUDED_ /* For int types. */ #include #include #ifdef __cplusplus extern "C" { #endif /* In case we haven't included the right headers yet. */ struct evbuffer; struct event_base; struct bufferevent; struct evhttp_connection; /** @file event2/http.h * * Basic support for HTTP serving. * * As Libevent is a library for dealing with event notification and most * interesting applications are networked today, I have often found the * need to write HTTP code. The following prototypes and definitions provide * an application with a minimal interface for making HTTP requests and for * creating a very simple HTTP server. */ /* Response codes */ #define HTTP_OK 200 /**< request completed ok */ #define HTTP_NOCONTENT 204 /**< request does not have content */ #define HTTP_MOVEPERM 301 /**< the uri moved permanently */ #define HTTP_MOVETEMP 302 /**< the uri moved temporarily */ #define HTTP_NOTMODIFIED 304 /**< page was not modified from last */ #define HTTP_BADREQUEST 400 /**< invalid http request was made */ #define HTTP_NOTFOUND 404 /**< could not find content for uri */ #define HTTP_BADMETHOD 405 /**< method not allowed for this uri */ #define HTTP_ENTITYTOOLARGE 413 /**< */ #define HTTP_EXPECTATIONFAILED 417 /**< we can't handle this expectation */ #define HTTP_INTERNAL 500 /**< internal error */ #define HTTP_NOTIMPLEMENTED 501 /**< not implemented */ #define HTTP_SERVUNAVAIL 503 /**< the server is not available */ struct evhttp; struct evhttp_request; struct evkeyvalq; struct evhttp_bound_socket; struct evconnlistener; struct evdns_base; /** * Create a new HTTP server. * * @param base (optional) the event base to receive the HTTP events * @return a pointer to a newly initialized evhttp server structure * @see evhttp_free() */ EVENT2_EXPORT_SYMBOL struct evhttp *evhttp_new(struct event_base *base); /** * Binds an HTTP server on the specified address and port. * * Can be called multiple times to bind the same http server * to multiple different ports. * * @param http a pointer to an evhttp object * @param address a string containing the IP address to listen(2) on * @param port the port number to listen on * @return 0 on success, -1 on failure. * @see evhttp_accept_socket() */ EVENT2_EXPORT_SYMBOL int evhttp_bind_socket(struct evhttp *http, const char *address, ev_uint16_t port); /** * Like evhttp_bind_socket(), but returns a handle for referencing the socket. * * The returned pointer is not valid after \a http is freed. * * @param http a pointer to an evhttp object * @param address a string containing the IP address to listen(2) on * @param port the port number to listen on * @return Handle for the socket on success, NULL on failure. * @see evhttp_bind_socket(), evhttp_del_accept_socket() */ EVENT2_EXPORT_SYMBOL struct evhttp_bound_socket *evhttp_bind_socket_with_handle(struct evhttp *http, const char *address, ev_uint16_t port); /** * Makes an HTTP server accept connections on the specified socket. * * This may be useful to create a socket and then fork multiple instances * of an http server, or when a socket has been communicated via file * descriptor passing in situations where an http servers does not have * permissions to bind to a low-numbered port. * * Can be called multiple times to have the http server listen to * multiple different sockets. * * @param http a pointer to an evhttp object * @param fd a socket fd that is ready for accepting connections * @return 0 on success, -1 on failure. * @see evhttp_bind_socket() */ EVENT2_EXPORT_SYMBOL int evhttp_accept_socket(struct evhttp *http, evutil_socket_t fd); /** * Like evhttp_accept_socket(), but returns a handle for referencing the socket. * * The returned pointer is not valid after \a http is freed. * * @param http a pointer to an evhttp object * @param fd a socket fd that is ready for accepting connections * @return Handle for the socket on success, NULL on failure. * @see evhttp_accept_socket(), evhttp_del_accept_socket() */ EVENT2_EXPORT_SYMBOL struct evhttp_bound_socket *evhttp_accept_socket_with_handle(struct evhttp *http, evutil_socket_t fd); /** * The most low-level evhttp_bind/accept method: takes an evconnlistener, and * returns an evhttp_bound_socket. The listener will be freed when the bound * socket is freed. */ EVENT2_EXPORT_SYMBOL struct evhttp_bound_socket *evhttp_bind_listener(struct evhttp *http, struct evconnlistener *listener); /** * Return the listener used to implement a bound socket. */ EVENT2_EXPORT_SYMBOL struct evconnlistener *evhttp_bound_socket_get_listener(struct evhttp_bound_socket *bound); typedef void evhttp_bound_socket_foreach_fn(struct evhttp_bound_socket *, void *); /** * Applies the function specified in the first argument to all * evhttp_bound_sockets associated with "http". The user must not * attempt to free or remove any connections, sockets or listeners * in the callback "function". * * @param http pointer to an evhttp object * @param function function to apply to every bound socket * @param argument pointer value passed to function for every socket iterated */ EVENT2_EXPORT_SYMBOL void evhttp_foreach_bound_socket(struct evhttp *http, evhttp_bound_socket_foreach_fn *function, void *argument); /** * Makes an HTTP server stop accepting connections on the specified socket * * This may be useful when a socket has been sent via file descriptor passing * and is no longer needed by the current process. * * If you created this bound socket with evhttp_bind_socket_with_handle or * evhttp_accept_socket_with_handle, this function closes the fd you provided. * If you created this bound socket with evhttp_bind_listener, this function * frees the listener you provided. * * \a bound_socket is an invalid pointer after this call returns. * * @param http a pointer to an evhttp object * @param bound_socket a handle returned by evhttp_{bind,accept}_socket_with_handle * @see evhttp_bind_socket_with_handle(), evhttp_accept_socket_with_handle() */ EVENT2_EXPORT_SYMBOL void evhttp_del_accept_socket(struct evhttp *http, struct evhttp_bound_socket *bound_socket); /** * Get the raw file descriptor referenced by an evhttp_bound_socket. * * @param bound_socket a handle returned by evhttp_{bind,accept}_socket_with_handle * @return the file descriptor used by the bound socket * @see evhttp_bind_socket_with_handle(), evhttp_accept_socket_with_handle() */ EVENT2_EXPORT_SYMBOL evutil_socket_t evhttp_bound_socket_get_fd(struct evhttp_bound_socket *bound_socket); /** * Free the previously created HTTP server. * * Works only if no requests are currently being served. * * @param http the evhttp server object to be freed * @see evhttp_start() */ EVENT2_EXPORT_SYMBOL void evhttp_free(struct evhttp* http); /** XXX Document. */ EVENT2_EXPORT_SYMBOL void evhttp_set_max_headers_size(struct evhttp* http, ev_ssize_t max_headers_size); /** XXX Document. */ EVENT2_EXPORT_SYMBOL void evhttp_set_max_body_size(struct evhttp* http, ev_ssize_t max_body_size); /** Set the value to use for the Content-Type header when none was provided. If the content type string is NULL, the Content-Type header will not be automatically added. @param http the http server on which to set the default content type @param content_type the value for the Content-Type header */ EVENT2_EXPORT_SYMBOL void evhttp_set_default_content_type(struct evhttp *http, const char *content_type); /** Sets the what HTTP methods are supported in requests accepted by this server, and passed to user callbacks. If not supported they will generate a "405 Method not allowed" response. By default this includes the following methods: GET, POST, HEAD, PUT, DELETE @param http the http server on which to set the methods @param methods bit mask constructed from evhttp_cmd_type values */ EVENT2_EXPORT_SYMBOL void evhttp_set_allowed_methods(struct evhttp* http, ev_uint16_t methods); /** Set a callback for a specified URI @param http the http sever on which to set the callback @param path the path for which to invoke the callback @param cb the callback function that gets invoked on requesting path @param cb_arg an additional context argument for the callback @return 0 on success, -1 if the callback existed already, -2 on failure */ EVENT2_EXPORT_SYMBOL int evhttp_set_cb(struct evhttp *http, const char *path, void (*cb)(struct evhttp_request *, void *), void *cb_arg); /** Removes the callback for a specified URI */ EVENT2_EXPORT_SYMBOL int evhttp_del_cb(struct evhttp *, const char *); /** Set a callback for all requests that are not caught by specific callbacks Invokes the specified callback for all requests that do not match any of the previously specified request paths. This is catchall for requests not specifically configured with evhttp_set_cb(). @param http the evhttp server object for which to set the callback @param cb the callback to invoke for any unmatched requests @param arg an context argument for the callback */ EVENT2_EXPORT_SYMBOL void evhttp_set_gencb(struct evhttp *http, void (*cb)(struct evhttp_request *, void *), void *arg); /** Set a callback used to create new bufferevents for connections to a given evhttp object. You can use this to override the default bufferevent type -- for example, to make this evhttp object use SSL bufferevents rather than unencrypted ones. New bufferevents must be allocated with no fd set on them. @param http the evhttp server object for which to set the callback @param cb the callback to invoke for incoming connections @param arg an context argument for the callback */ EVENT2_EXPORT_SYMBOL void evhttp_set_bevcb(struct evhttp *http, struct bufferevent *(*cb)(struct event_base *, void *), void *arg); /** Adds a virtual host to the http server. A virtual host is a newly initialized evhttp object that has request callbacks set on it via evhttp_set_cb() or evhttp_set_gencb(). It most not have any listing sockets associated with it. If the virtual host has not been removed by the time that evhttp_free() is called on the main http server, it will be automatically freed, too. It is possible to have hierarchical vhosts. For example: A vhost with the pattern *.example.com may have other vhosts with patterns foo.example.com and bar.example.com associated with it. @param http the evhttp object to which to add a virtual host @param pattern the glob pattern against which the hostname is matched. The match is case insensitive and follows otherwise regular shell matching. @param vhost the virtual host to add the regular http server. @return 0 on success, -1 on failure @see evhttp_remove_virtual_host() */ EVENT2_EXPORT_SYMBOL int evhttp_add_virtual_host(struct evhttp* http, const char *pattern, struct evhttp* vhost); /** Removes a virtual host from the http server. @param http the evhttp object from which to remove the virtual host @param vhost the virtual host to remove from the regular http server. @return 0 on success, -1 on failure @see evhttp_add_virtual_host() */ EVENT2_EXPORT_SYMBOL int evhttp_remove_virtual_host(struct evhttp* http, struct evhttp* vhost); /** Add a server alias to an http object. The http object can be a virtual host or the main server. @param http the evhttp object @param alias the alias to add @see evhttp_add_remove_alias() */ EVENT2_EXPORT_SYMBOL int evhttp_add_server_alias(struct evhttp *http, const char *alias); /** Remove a server alias from an http object. @param http the evhttp object @param alias the alias to remove @see evhttp_add_server_alias() */ EVENT2_EXPORT_SYMBOL int evhttp_remove_server_alias(struct evhttp *http, const char *alias); /** * Set the timeout for an HTTP request. * * @param http an evhttp object * @param timeout_in_secs the timeout, in seconds */ EVENT2_EXPORT_SYMBOL void evhttp_set_timeout(struct evhttp *http, int timeout_in_secs); /** * Set the timeout for an HTTP request. * * @param http an evhttp object * @param tv the timeout, or NULL */ EVENT2_EXPORT_SYMBOL void evhttp_set_timeout_tv(struct evhttp *http, const struct timeval* tv); /* Read all the clients body, and only after this respond with an error if the * clients body exceed max_body_size */ #define EVHTTP_SERVER_LINGERING_CLOSE 0x0001 /** * Set connection flags for HTTP server. * * @see EVHTTP_SERVER_* * @return 0 on success, otherwise non zero (for example if flag doesn't * supported). */ EVENT2_EXPORT_SYMBOL int evhttp_set_flags(struct evhttp *http, int flags); /* Request/Response functionality */ /** * Send an HTML error message to the client. * * @param req a request object * @param error the HTTP error code * @param reason a brief explanation of the error. If this is NULL, we'll * just use the standard meaning of the error code. */ EVENT2_EXPORT_SYMBOL void evhttp_send_error(struct evhttp_request *req, int error, const char *reason); /** * Send an HTML reply to the client. * * The body of the reply consists of the data in databuf. After calling * evhttp_send_reply() databuf will be empty, but the buffer is still * owned by the caller and needs to be deallocated by the caller if * necessary. * * @param req a request object * @param code the HTTP response code to send * @param reason a brief message to send with the response code * @param databuf the body of the response */ EVENT2_EXPORT_SYMBOL void evhttp_send_reply(struct evhttp_request *req, int code, const char *reason, struct evbuffer *databuf); /* Low-level response interface, for streaming/chunked replies */ /** Initiate a reply that uses Transfer-Encoding chunked. This allows the caller to stream the reply back to the client and is useful when either not all of the reply data is immediately available or when sending very large replies. The caller needs to supply data chunks with evhttp_send_reply_chunk() and complete the reply by calling evhttp_send_reply_end(). @param req a request object @param code the HTTP response code to send @param reason a brief message to send with the response code */ EVENT2_EXPORT_SYMBOL void evhttp_send_reply_start(struct evhttp_request *req, int code, const char *reason); /** Send another data chunk as part of an ongoing chunked reply. The reply chunk consists of the data in databuf. After calling evhttp_send_reply_chunk() databuf will be empty, but the buffer is still owned by the caller and needs to be deallocated by the caller if necessary. @param req a request object @param databuf the data chunk to send as part of the reply. */ EVENT2_EXPORT_SYMBOL void evhttp_send_reply_chunk(struct evhttp_request *req, struct evbuffer *databuf); /** Send another data chunk as part of an ongoing chunked reply. The reply chunk consists of the data in databuf. After calling evhttp_send_reply_chunk() databuf will be empty, but the buffer is still owned by the caller and needs to be deallocated by the caller if necessary. @param req a request object @param databuf the data chunk to send as part of the reply. @param cb callback funcion @param call back's argument. */ EVENT2_EXPORT_SYMBOL void evhttp_send_reply_chunk_with_cb(struct evhttp_request *, struct evbuffer *, void (*cb)(struct evhttp_connection *, void *), void *arg); /** Complete a chunked reply, freeing the request as appropriate. @param req a request object */ EVENT2_EXPORT_SYMBOL void evhttp_send_reply_end(struct evhttp_request *req); /* * Interfaces for making requests */ /** The different request types supported by evhttp. These are as specified * in RFC2616, except for PATCH which is specified by RFC5789. * * By default, only some of these methods are accepted and passed to user * callbacks; use evhttp_set_allowed_methods() to change which methods * are allowed. */ enum evhttp_cmd_type { EVHTTP_REQ_GET = 1 << 0, EVHTTP_REQ_POST = 1 << 1, EVHTTP_REQ_HEAD = 1 << 2, EVHTTP_REQ_PUT = 1 << 3, EVHTTP_REQ_DELETE = 1 << 4, EVHTTP_REQ_OPTIONS = 1 << 5, EVHTTP_REQ_TRACE = 1 << 6, EVHTTP_REQ_CONNECT = 1 << 7, EVHTTP_REQ_PATCH = 1 << 8 }; /** a request object can represent either a request or a reply */ enum evhttp_request_kind { EVHTTP_REQUEST, EVHTTP_RESPONSE }; /** * Create and return a connection object that can be used to for making HTTP * requests. The connection object tries to resolve address and establish the * connection when it is given an http request object. * * @param base the event_base to use for handling the connection * @param dnsbase the dns_base to use for resolving host names; if not * specified host name resolution will block. * @param bev a bufferevent to use for connecting to the server; if NULL, a * socket-based bufferevent will be created. This buffrevent will be freed * when the connection closes. It must have no fd set on it. * @param address the address to which to connect * @param port the port to connect to * @return an evhttp_connection object that can be used for making requests */ EVENT2_EXPORT_SYMBOL struct evhttp_connection *evhttp_connection_base_bufferevent_new( struct event_base *base, struct evdns_base *dnsbase, struct bufferevent* bev, const char *address, ev_uint16_t port); /** * Return the bufferevent that an evhttp_connection is using. */ EVENT2_EXPORT_SYMBOL struct bufferevent* evhttp_connection_get_bufferevent(struct evhttp_connection *evcon); /** * Return the HTTP server associated with this connection, or NULL. */ EVENT2_EXPORT_SYMBOL struct evhttp *evhttp_connection_get_server(struct evhttp_connection *evcon); /** * Creates a new request object that needs to be filled in with the request * parameters. The callback is executed when the request completed or an * error occurred. */ EVENT2_EXPORT_SYMBOL struct evhttp_request *evhttp_request_new( void (*cb)(struct evhttp_request *, void *), void *arg); /** * Enable delivery of chunks to requestor. * @param cb will be called after every read of data with the same argument * as the completion callback. Will never be called on an empty * response. May drain the input buffer; it will be drained * automatically on return. */ EVENT2_EXPORT_SYMBOL void evhttp_request_set_chunked_cb(struct evhttp_request *, void (*cb)(struct evhttp_request *, void *)); /** * Register callback for additional parsing of request headers. * @param cb will be called after receiving and parsing the full header. * It allows analyzing the header and possibly closing the connection * by returning a value < 0. */ EVENT2_EXPORT_SYMBOL void evhttp_request_set_header_cb(struct evhttp_request *, int (*cb)(struct evhttp_request *, void *)); /** * The different error types supported by evhttp * * @see evhttp_request_set_error_cb() */ enum evhttp_request_error { /** * Timeout reached, also @see evhttp_connection_set_timeout() */ EVREQ_HTTP_TIMEOUT, /** * EOF reached */ EVREQ_HTTP_EOF, /** * Error while reading header, or invalid header */ EVREQ_HTTP_INVALID_HEADER, /** * Error encountered while reading or writing */ EVREQ_HTTP_BUFFER_ERROR, /** * The evhttp_cancel_request() called on this request. */ EVREQ_HTTP_REQUEST_CANCEL, /** * Body is greater then evhttp_connection_set_max_body_size() */ EVREQ_HTTP_DATA_TOO_LONG }; /** * Set a callback for errors * @see evhttp_request_error for error types. * * On error, both the error callback and the regular callback will be called, * error callback is called before the regular callback. **/ EVENT2_EXPORT_SYMBOL void evhttp_request_set_error_cb(struct evhttp_request *, void (*)(enum evhttp_request_error, void *)); /** * Set a callback to be called on request completion of evhttp_send_* function. * * The callback function will be called on the completion of the request after * the output data has been written and before the evhttp_request object * is destroyed. This can be useful for tracking resources associated with a * request (ex: timing metrics). * * @param req a request object * @param cb callback function that will be called on request completion * @param cb_arg an additional context argument for the callback */ EVENT2_EXPORT_SYMBOL void evhttp_request_set_on_complete_cb(struct evhttp_request *req, void (*cb)(struct evhttp_request *, void *), void *cb_arg); /** Frees the request object and removes associated events. */ EVENT2_EXPORT_SYMBOL void evhttp_request_free(struct evhttp_request *req); /** * Create and return a connection object that can be used to for making HTTP * requests. The connection object tries to resolve address and establish the * connection when it is given an http request object. * * @param base the event_base to use for handling the connection * @param dnsbase the dns_base to use for resolving host names; if not * specified host name resolution will block. * @param address the address to which to connect * @param port the port to connect to * @return an evhttp_connection object that can be used for making requests */ EVENT2_EXPORT_SYMBOL struct evhttp_connection *evhttp_connection_base_new( struct event_base *base, struct evdns_base *dnsbase, const char *address, ev_uint16_t port); /** * Set family hint for DNS requests. */ EVENT2_EXPORT_SYMBOL void evhttp_connection_set_family(struct evhttp_connection *evcon, int family); /* reuse connection address on retry */ #define EVHTTP_CON_REUSE_CONNECTED_ADDR 0x0008 /* Try to read error, since server may already send and close * connection, but if at that time we have some data to send then we * can send get EPIPE and fail, while we can read that HTTP error. */ #define EVHTTP_CON_READ_ON_WRITE_ERROR 0x0010 /* @see EVHTTP_SERVER_LINGERING_CLOSE */ #define EVHTTP_CON_LINGERING_CLOSE 0x0020 /* Padding for public flags, @see EVHTTP_CON_* in http-internal.h */ #define EVHTTP_CON_PUBLIC_FLAGS_END 0x100000 /** * Set connection flags. * * @see EVHTTP_CON_* * @return 0 on success, otherwise non zero (for example if flag doesn't * supported). */ EVENT2_EXPORT_SYMBOL int evhttp_connection_set_flags(struct evhttp_connection *evcon, int flags); /** Takes ownership of the request object * * Can be used in a request callback to keep onto the request until * evhttp_request_free() is explicitly called by the user. */ EVENT2_EXPORT_SYMBOL void evhttp_request_own(struct evhttp_request *req); /** Returns 1 if the request is owned by the user */ EVENT2_EXPORT_SYMBOL int evhttp_request_is_owned(struct evhttp_request *req); /** * Returns the connection object associated with the request or NULL * * The user needs to either free the request explicitly or call * evhttp_send_reply_end(). */ EVENT2_EXPORT_SYMBOL struct evhttp_connection *evhttp_request_get_connection(struct evhttp_request *req); /** * Returns the underlying event_base for this connection */ EVENT2_EXPORT_SYMBOL struct event_base *evhttp_connection_get_base(struct evhttp_connection *req); EVENT2_EXPORT_SYMBOL void evhttp_connection_set_max_headers_size(struct evhttp_connection *evcon, ev_ssize_t new_max_headers_size); EVENT2_EXPORT_SYMBOL void evhttp_connection_set_max_body_size(struct evhttp_connection* evcon, ev_ssize_t new_max_body_size); /** Frees an http connection */ EVENT2_EXPORT_SYMBOL void evhttp_connection_free(struct evhttp_connection *evcon); /** Disowns a given connection object * * Can be used to tell libevent to free the connection object after * the last request has completed or failed. */ EVENT2_EXPORT_SYMBOL void evhttp_connection_free_on_completion(struct evhttp_connection *evcon); /** sets the ip address from which http connections are made */ EVENT2_EXPORT_SYMBOL void evhttp_connection_set_local_address(struct evhttp_connection *evcon, const char *address); /** sets the local port from which http connections are made */ EVENT2_EXPORT_SYMBOL void evhttp_connection_set_local_port(struct evhttp_connection *evcon, ev_uint16_t port); /** Sets the timeout in seconds for events related to this connection */ EVENT2_EXPORT_SYMBOL void evhttp_connection_set_timeout(struct evhttp_connection *evcon, int timeout_in_secs); /** Sets the timeout for events related to this connection. Takes a struct * timeval. */ EVENT2_EXPORT_SYMBOL void evhttp_connection_set_timeout_tv(struct evhttp_connection *evcon, const struct timeval *tv); /** Sets the delay before retrying requests on this connection. This is only * used if evhttp_connection_set_retries is used to make the number of retries * at least one. Each retry after the first is twice as long as the one before * it. */ EVENT2_EXPORT_SYMBOL void evhttp_connection_set_initial_retry_tv(struct evhttp_connection *evcon, const struct timeval *tv); /** Sets the retry limit for this connection - -1 repeats indefinitely */ EVENT2_EXPORT_SYMBOL void evhttp_connection_set_retries(struct evhttp_connection *evcon, int retry_max); /** Set a callback for connection close. */ EVENT2_EXPORT_SYMBOL void evhttp_connection_set_closecb(struct evhttp_connection *evcon, void (*)(struct evhttp_connection *, void *), void *); /** Get the remote address and port associated with this connection. */ EVENT2_EXPORT_SYMBOL void evhttp_connection_get_peer(struct evhttp_connection *evcon, char **address, ev_uint16_t *port); /** Get the remote address associated with this connection. * extracted from getpeername() OR from nameserver. * * @return NULL if getpeername() return non success, * or connection is not connected, * otherwise it return pointer to struct sockaddr_storage */ EVENT2_EXPORT_SYMBOL const struct sockaddr* evhttp_connection_get_addr(struct evhttp_connection *evcon); /** Make an HTTP request over the specified connection. The connection gets ownership of the request. On failure, the request object is no longer valid as it has been freed. @param evcon the evhttp_connection object over which to send the request @param req the previously created and configured request object @param type the request type EVHTTP_REQ_GET, EVHTTP_REQ_POST, etc. @param uri the URI associated with the request @return 0 on success, -1 on failure @see evhttp_cancel_request() */ EVENT2_EXPORT_SYMBOL int evhttp_make_request(struct evhttp_connection *evcon, struct evhttp_request *req, enum evhttp_cmd_type type, const char *uri); /** Cancels a pending HTTP request. Cancels an ongoing HTTP request. The callback associated with this request is not executed and the request object is freed. If the request is currently being processed, e.g. it is ongoing, the corresponding evhttp_connection object is going to get reset. A request cannot be canceled if its callback has executed already. A request may be canceled reentrantly from its chunked callback. @param req the evhttp_request to cancel; req becomes invalid after this call. */ EVENT2_EXPORT_SYMBOL void evhttp_cancel_request(struct evhttp_request *req); /** * A structure to hold a parsed URI or Relative-Ref conforming to RFC3986. */ struct evhttp_uri; /** Returns the request URI */ EVENT2_EXPORT_SYMBOL const char *evhttp_request_get_uri(const struct evhttp_request *req); /** Returns the request URI (parsed) */ EVENT2_EXPORT_SYMBOL const struct evhttp_uri *evhttp_request_get_evhttp_uri(const struct evhttp_request *req); /** Returns the request command */ EVENT2_EXPORT_SYMBOL enum evhttp_cmd_type evhttp_request_get_command(const struct evhttp_request *req); EVENT2_EXPORT_SYMBOL int evhttp_request_get_response_code(const struct evhttp_request *req); EVENT2_EXPORT_SYMBOL const char * evhttp_request_get_response_code_line(const struct evhttp_request *req); /** Returns the input headers */ EVENT2_EXPORT_SYMBOL struct evkeyvalq *evhttp_request_get_input_headers(struct evhttp_request *req); /** Returns the output headers */ EVENT2_EXPORT_SYMBOL struct evkeyvalq *evhttp_request_get_output_headers(struct evhttp_request *req); /** Returns the input buffer */ EVENT2_EXPORT_SYMBOL struct evbuffer *evhttp_request_get_input_buffer(struct evhttp_request *req); /** Returns the output buffer */ EVENT2_EXPORT_SYMBOL struct evbuffer *evhttp_request_get_output_buffer(struct evhttp_request *req); /** Returns the host associated with the request. If a client sends an absolute URI, the host part of that is preferred. Otherwise, the input headers are searched for a Host: header. NULL is returned if no absolute URI or Host: header is provided. */ EVENT2_EXPORT_SYMBOL const char *evhttp_request_get_host(struct evhttp_request *req); /* Interfaces for dealing with HTTP headers */ /** Finds the value belonging to a header. @param headers the evkeyvalq object in which to find the header @param key the name of the header to find @returns a pointer to the value for the header or NULL if the header could not be found. @see evhttp_add_header(), evhttp_remove_header() */ EVENT2_EXPORT_SYMBOL const char *evhttp_find_header(const struct evkeyvalq *headers, const char *key); /** Removes a header from a list of existing headers. @param headers the evkeyvalq object from which to remove a header @param key the name of the header to remove @returns 0 if the header was removed, -1 otherwise. @see evhttp_find_header(), evhttp_add_header() */ EVENT2_EXPORT_SYMBOL int evhttp_remove_header(struct evkeyvalq *headers, const char *key); /** Adds a header to a list of existing headers. @param headers the evkeyvalq object to which to add a header @param key the name of the header @param value the value belonging to the header @returns 0 on success, -1 otherwise. @see evhttp_find_header(), evhttp_clear_headers() */ EVENT2_EXPORT_SYMBOL int evhttp_add_header(struct evkeyvalq *headers, const char *key, const char *value); /** Removes all headers from the header list. @param headers the evkeyvalq object from which to remove all headers */ EVENT2_EXPORT_SYMBOL void evhttp_clear_headers(struct evkeyvalq *headers); /* Miscellaneous utility functions */ /** Helper function to encode a string for inclusion in a URI. All characters are replaced by their hex-escaped (%22) equivalents, except for characters explicitly unreserved by RFC3986 -- that is, ASCII alphanumeric characters, hyphen, dot, underscore, and tilde. The returned string must be freed by the caller. @param str an unencoded string @return a newly allocated URI-encoded string or NULL on failure */ EVENT2_EXPORT_SYMBOL char *evhttp_encode_uri(const char *str); /** As evhttp_encode_uri, but if 'size' is nonnegative, treat the string as being 'size' bytes long. This allows you to encode strings that may contain 0-valued bytes. The returned string must be freed by the caller. @param str an unencoded string @param size the length of the string to encode, or -1 if the string is NUL-terminated @param space_to_plus if true, space characters in 'str' are encoded as +, not %20. @return a newly allocate URI-encoded string, or NULL on failure. */ EVENT2_EXPORT_SYMBOL char *evhttp_uriencode(const char *str, ev_ssize_t size, int space_to_plus); /** Helper function to sort of decode a URI-encoded string. Unlike evhttp_get_decoded_uri, it decodes all plus characters that appear _after_ the first question mark character, but no plusses that occur before. This is not a good way to decode URIs in whole or in part. The returned string must be freed by the caller @deprecated This function is deprecated; you probably want to use evhttp_get_decoded_uri instead. @param uri an encoded URI @return a newly allocated unencoded URI or NULL on failure */ EVENT2_EXPORT_SYMBOL char *evhttp_decode_uri(const char *uri); /** Helper function to decode a URI-escaped string or HTTP parameter. If 'decode_plus' is 1, then we decode the string as an HTTP parameter value, and convert all plus ('+') characters to spaces. If 'decode_plus' is 0, we leave all plus characters unchanged. The returned string must be freed by the caller. @param uri a URI-encode encoded URI @param decode_plus determines whether we convert '+' to space. @param size_out if size_out is not NULL, *size_out is set to the size of the returned string @return a newly allocated unencoded URI or NULL on failure */ EVENT2_EXPORT_SYMBOL char *evhttp_uridecode(const char *uri, int decode_plus, size_t *size_out); /** Helper function to parse out arguments in a query. Parsing a URI like http://foo.com/?q=test&s=some+thing will result in two entries in the key value queue. The first entry is: key="q", value="test" The second entry is: key="s", value="some thing" @deprecated This function is deprecated as of Libevent 2.0.9. Use evhttp_uri_parse and evhttp_parse_query_str instead. @param uri the request URI @param headers the head of the evkeyval queue @return 0 on success, -1 on failure */ EVENT2_EXPORT_SYMBOL int evhttp_parse_query(const char *uri, struct evkeyvalq *headers); /** Helper function to parse out arguments from the query portion of an HTTP URI. Parsing a query string like q=test&s=some+thing will result in two entries in the key value queue. The first entry is: key="q", value="test" The second entry is: key="s", value="some thing" @param query_parse the query portion of the URI @param headers the head of the evkeyval queue @return 0 on success, -1 on failure */ EVENT2_EXPORT_SYMBOL int evhttp_parse_query_str(const char *uri, struct evkeyvalq *headers); /** * Escape HTML character entities in a string. * * Replaces <, >, ", ' and & with <, >, ", * ' and & correspondingly. * * The returned string needs to be freed by the caller. * * @param html an unescaped HTML string * @return an escaped HTML string or NULL on error */ EVENT2_EXPORT_SYMBOL char *evhttp_htmlescape(const char *html); /** * Return a new empty evhttp_uri with no fields set. */ EVENT2_EXPORT_SYMBOL struct evhttp_uri *evhttp_uri_new(void); /** * Changes the flags set on a given URI. See EVHTTP_URI_* for * a list of flags. **/ EVENT2_EXPORT_SYMBOL void evhttp_uri_set_flags(struct evhttp_uri *uri, unsigned flags); /** Return the scheme of an evhttp_uri, or NULL if there is no scheme has * been set and the evhttp_uri contains a Relative-Ref. */ EVENT2_EXPORT_SYMBOL const char *evhttp_uri_get_scheme(const struct evhttp_uri *uri); /** * Return the userinfo part of an evhttp_uri, or NULL if it has no userinfo * set. */ EVENT2_EXPORT_SYMBOL const char *evhttp_uri_get_userinfo(const struct evhttp_uri *uri); /** * Return the host part of an evhttp_uri, or NULL if it has no host set. * The host may either be a regular hostname (conforming to the RFC 3986 * "regname" production), or an IPv4 address, or the empty string, or a * bracketed IPv6 address, or a bracketed 'IP-Future' address. * * Note that having a NULL host means that the URI has no authority * section, but having an empty-string host means that the URI has an * authority section with no host part. For example, * "mailto:user@example.com" has a host of NULL, but "file:///etc/motd" * has a host of "". */ EVENT2_EXPORT_SYMBOL const char *evhttp_uri_get_host(const struct evhttp_uri *uri); /** Return the port part of an evhttp_uri, or -1 if there is no port set. */ EVENT2_EXPORT_SYMBOL int evhttp_uri_get_port(const struct evhttp_uri *uri); /** Return the path part of an evhttp_uri, or NULL if it has no path set */ EVENT2_EXPORT_SYMBOL const char *evhttp_uri_get_path(const struct evhttp_uri *uri); /** Return the query part of an evhttp_uri (excluding the leading "?"), or * NULL if it has no query set */ EVENT2_EXPORT_SYMBOL const char *evhttp_uri_get_query(const struct evhttp_uri *uri); /** Return the fragment part of an evhttp_uri (excluding the leading "#"), * or NULL if it has no fragment set */ EVENT2_EXPORT_SYMBOL const char *evhttp_uri_get_fragment(const struct evhttp_uri *uri); /** Set the scheme of an evhttp_uri, or clear the scheme if scheme==NULL. * Returns 0 on success, -1 if scheme is not well-formed. */ EVENT2_EXPORT_SYMBOL int evhttp_uri_set_scheme(struct evhttp_uri *uri, const char *scheme); /** Set the userinfo of an evhttp_uri, or clear the userinfo if userinfo==NULL. * Returns 0 on success, -1 if userinfo is not well-formed. */ EVENT2_EXPORT_SYMBOL int evhttp_uri_set_userinfo(struct evhttp_uri *uri, const char *userinfo); /** Set the host of an evhttp_uri, or clear the host if host==NULL. * Returns 0 on success, -1 if host is not well-formed. */ EVENT2_EXPORT_SYMBOL int evhttp_uri_set_host(struct evhttp_uri *uri, const char *host); /** Set the port of an evhttp_uri, or clear the port if port==-1. * Returns 0 on success, -1 if port is not well-formed. */ EVENT2_EXPORT_SYMBOL int evhttp_uri_set_port(struct evhttp_uri *uri, int port); /** Set the path of an evhttp_uri, or clear the path if path==NULL. * Returns 0 on success, -1 if path is not well-formed. */ EVENT2_EXPORT_SYMBOL int evhttp_uri_set_path(struct evhttp_uri *uri, const char *path); /** Set the query of an evhttp_uri, or clear the query if query==NULL. * The query should not include a leading "?". * Returns 0 on success, -1 if query is not well-formed. */ EVENT2_EXPORT_SYMBOL int evhttp_uri_set_query(struct evhttp_uri *uri, const char *query); /** Set the fragment of an evhttp_uri, or clear the fragment if fragment==NULL. * The fragment should not include a leading "#". * Returns 0 on success, -1 if fragment is not well-formed. */ EVENT2_EXPORT_SYMBOL int evhttp_uri_set_fragment(struct evhttp_uri *uri, const char *fragment); /** * Helper function to parse a URI-Reference as specified by RFC3986. * * This function matches the URI-Reference production from RFC3986, * which includes both URIs like * * scheme://[[userinfo]@]foo.com[:port]]/[path][?query][#fragment] * * and relative-refs like * * [path][?query][#fragment] * * Any optional elements portions not present in the original URI are * left set to NULL in the resulting evhttp_uri. If no port is * specified, the port is set to -1. * * Note that no decoding is performed on percent-escaped characters in * the string; if you want to parse them, use evhttp_uridecode or * evhttp_parse_query_str as appropriate. * * Note also that most URI schemes will have additional constraints that * this function does not know about, and cannot check. For example, * mailto://www.example.com/cgi-bin/fortune.pl is not a reasonable * mailto url, http://www.example.com:99999/ is not a reasonable HTTP * URL, and ftp:username@example.com is not a reasonable FTP URL. * Nevertheless, all of these URLs conform to RFC3986, and this function * accepts all of them as valid. * * @param source_uri the request URI * @param flags Zero or more EVHTTP_URI_* flags to affect the behavior * of the parser. * @return uri container to hold parsed data, or NULL if there is error * @see evhttp_uri_free() */ EVENT2_EXPORT_SYMBOL struct evhttp_uri *evhttp_uri_parse_with_flags(const char *source_uri, unsigned flags); /** Tolerate URIs that do not conform to RFC3986. * * Unfortunately, some HTTP clients generate URIs that, according to RFC3986, * are not conformant URIs. If you need to support these URIs, you can * do so by passing this flag to evhttp_uri_parse_with_flags. * * Currently, these changes are: *
    *
  • Nonconformant URIs are allowed to contain otherwise unreasonable * characters in their path, query, and fragment components. *
*/ #define EVHTTP_URI_NONCONFORMANT 0x01 /** Alias for evhttp_uri_parse_with_flags(source_uri, 0) */ EVENT2_EXPORT_SYMBOL struct evhttp_uri *evhttp_uri_parse(const char *source_uri); /** * Free all memory allocated for a parsed uri. Only use this for URIs * generated by evhttp_uri_parse. * * @param uri container with parsed data * @see evhttp_uri_parse() */ EVENT2_EXPORT_SYMBOL void evhttp_uri_free(struct evhttp_uri *uri); /** * Join together the uri parts from parsed data to form a URI-Reference. * * Note that no escaping of reserved characters is done on the members * of the evhttp_uri, so the generated string might not be a valid URI * unless the members of evhttp_uri are themselves valid. * * @param uri container with parsed data * @param buf destination buffer * @param limit destination buffer size * @return an joined uri as string or NULL on error * @see evhttp_uri_parse() */ EVENT2_EXPORT_SYMBOL char *evhttp_uri_join(struct evhttp_uri *uri, char *buf, size_t limit); #ifdef __cplusplus } #endif #endif /* EVENT2_HTTP_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/http_compat.h000066400000000000000000000061131527052371200253200ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_HTTP_COMPAT_H_INCLUDED_ #define EVENT2_HTTP_COMPAT_H_INCLUDED_ /** @file event2/http_compat.h Potentially non-threadsafe versions of the functions in http.h: provided only for backwards compatibility. */ #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include /** * Start an HTTP server on the specified address and port * * @deprecated It does not allow an event base to be specified * * @param address the address to which the HTTP server should be bound * @param port the port number on which the HTTP server should listen * @return an struct evhttp object */ struct evhttp *evhttp_start(const char *address, ev_uint16_t port); /** * A connection object that can be used to for making HTTP requests. The * connection object tries to establish the connection when it is given an * http request object. * * @deprecated It does not allow an event base to be specified */ struct evhttp_connection *evhttp_connection_new( const char *address, ev_uint16_t port); /** * Associates an event base with the connection - can only be called * on a freshly created connection object that has not been used yet. * * @deprecated XXXX Why? */ void evhttp_connection_set_base(struct evhttp_connection *evcon, struct event_base *base); /** Returns the request URI */ #define evhttp_request_uri evhttp_request_get_uri #ifdef __cplusplus } #endif #endif /* EVENT2_EVENT_COMPAT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/http_struct.h000066400000000000000000000113111527052371200253550ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_HTTP_STRUCT_H_INCLUDED_ #define EVENT2_HTTP_STRUCT_H_INCLUDED_ /** @file event2/http_struct.h Data structures for http. Using these structures may hurt forward compatibility with later versions of Libevent: be careful! */ #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include /** * the request structure that a server receives. * WARNING: expect this structure to change. I will try to provide * reasonable accessors. */ struct evhttp_request { #if defined(TAILQ_ENTRY) TAILQ_ENTRY(evhttp_request) next; #else struct { struct evhttp_request *tqe_next; struct evhttp_request **tqe_prev; } next; #endif /* the connection object that this request belongs to */ struct evhttp_connection *evcon; int flags; /** The request obj owns the evhttp connection and needs to free it */ #define EVHTTP_REQ_OWN_CONNECTION 0x0001 /** Request was made via a proxy */ #define EVHTTP_PROXY_REQUEST 0x0002 /** The request object is owned by the user; the user must free it */ #define EVHTTP_USER_OWNED 0x0004 /** The request will be used again upstack; freeing must be deferred */ #define EVHTTP_REQ_DEFER_FREE 0x0008 /** The request should be freed upstack */ #define EVHTTP_REQ_NEEDS_FREE 0x0010 struct evkeyvalq *input_headers; struct evkeyvalq *output_headers; /* address of the remote host and the port connection came from */ char *remote_host; ev_uint16_t remote_port; /* cache of the hostname for evhttp_request_get_host */ char *host_cache; enum evhttp_request_kind kind; enum evhttp_cmd_type type; size_t headers_size; size_t body_size; char *uri; /* uri after HTTP request was parsed */ struct evhttp_uri *uri_elems; /* uri elements */ char major; /* HTTP Major number */ char minor; /* HTTP Minor number */ int response_code; /* HTTP Response code */ char *response_code_line; /* Readable response */ struct evbuffer *input_buffer; /* read data */ ev_int64_t ntoread; unsigned chunked:1, /* a chunked request */ userdone:1; /* the user has sent all data */ struct evbuffer *output_buffer; /* outgoing post or data */ /* Callback */ void (*cb)(struct evhttp_request *, void *); void *cb_arg; /* * Chunked data callback - call for each completed chunk if * specified. If not specified, all the data is delivered via * the regular callback. */ void (*chunk_cb)(struct evhttp_request *, void *); /* * Callback added for forked-daapd so they can collect ICY * (shoutcast) metadata from the http header. If return * int is negative the connection will be closed. */ int (*header_cb)(struct evhttp_request *, void *); /* * Error callback - called when error is occured. * @see evhttp_request_error for error types. * * @see evhttp_request_set_error_cb() */ void (*error_cb)(enum evhttp_request_error, void *); /* * Send complete callback - called when the request is actually * sent and completed. */ void (*on_complete_cb)(struct evhttp_request *, void *); void *on_complete_cb_arg; }; #ifdef __cplusplus } #endif #endif /* EVENT2_HTTP_STRUCT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/keyvalq_struct.h000066400000000000000000000050531527052371200260600ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_KEYVALQ_STRUCT_H_INCLUDED_ #define EVENT2_KEYVALQ_STRUCT_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif /* Fix so that people don't have to run with */ /* XXXX This code is duplicated with event_struct.h */ #ifndef TAILQ_ENTRY #define EVENT_DEFINED_TQENTRY_ #define TAILQ_ENTRY(type) \ struct { \ struct type *tqe_next; /* next element */ \ struct type **tqe_prev; /* address of previous next element */ \ } #endif /* !TAILQ_ENTRY */ #ifndef TAILQ_HEAD #define EVENT_DEFINED_TQHEAD_ #define TAILQ_HEAD(name, type) \ struct name { \ struct type *tqh_first; \ struct type **tqh_last; \ } #endif /* * Key-Value pairs. Can be used for HTTP headers but also for * query argument parsing. */ struct evkeyval { TAILQ_ENTRY(evkeyval) next; char *key; char *value; }; TAILQ_HEAD (evkeyvalq, evkeyval); /* XXXX This code is duplicated with event_struct.h */ #ifdef EVENT_DEFINED_TQENTRY_ #undef TAILQ_ENTRY #endif #ifdef EVENT_DEFINED_TQHEAD_ #undef TAILQ_HEAD #endif #ifdef __cplusplus } #endif #endif kytensor-2.49.0.7/third_party/libevent/include/event2/listener.h000066400000000000000000000164251527052371200246320ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_LISTENER_H_INCLUDED_ #define EVENT2_LISTENER_H_INCLUDED_ #include #ifdef __cplusplus extern "C" { #endif #include struct sockaddr; struct evconnlistener; /** A callback that we invoke when a listener has a new connection. @param listener The evconnlistener @param fd The new file descriptor @param addr The source address of the connection @param socklen The length of addr @param user_arg the pointer passed to evconnlistener_new() */ typedef void (*evconnlistener_cb)(struct evconnlistener *, evutil_socket_t, struct sockaddr *, int socklen, void *); /** A callback that we invoke when a listener encounters a non-retriable error. @param listener The evconnlistener @param user_arg the pointer passed to evconnlistener_new() */ typedef void (*evconnlistener_errorcb)(struct evconnlistener *, void *); /** Flag: Indicates that we should not make incoming sockets nonblocking * before passing them to the callback. */ #define LEV_OPT_LEAVE_SOCKETS_BLOCKING (1u<<0) /** Flag: Indicates that freeing the listener should close the underlying * socket. */ #define LEV_OPT_CLOSE_ON_FREE (1u<<1) /** Flag: Indicates that we should set the close-on-exec flag, if possible */ #define LEV_OPT_CLOSE_ON_EXEC (1u<<2) /** Flag: Indicates that we should disable the timeout (if any) between when * this socket is closed and when we can listen again on the same port. */ #define LEV_OPT_REUSEABLE (1u<<3) /** Flag: Indicates that the listener should be locked so it's safe to use * from multiple threadcs at once. */ #define LEV_OPT_THREADSAFE (1u<<4) /** Flag: Indicates that the listener should be created in disabled * state. Use evconnlistener_enable() to enable it later. */ #define LEV_OPT_DISABLED (1u<<5) /** Flag: Indicates that the listener should defer accept() until data is * available, if possible. Ignored on platforms that do not support this. * * This option can help performance for protocols where the client transmits * immediately after connecting. Do not use this option if your protocol * _doesn't_ start out with the client transmitting data, since in that case * this option will sometimes cause the kernel to never tell you about the * connection. * * This option is only supported by evconnlistener_new_bind(): it can't * work with evconnlistener_new_fd(), since the listener needs to be told * to use the option before it is actually bound. */ #define LEV_OPT_DEFERRED_ACCEPT (1u<<6) /** Flag: Indicates that we ask to allow multiple servers (processes or * threads) to bind to the same port if they each set the option. * * SO_REUSEPORT is what most people would expect SO_REUSEADDR to be, however * SO_REUSEPORT does not imply SO_REUSEADDR. * * This is only available on Linux and kernel 3.9+ */ #define LEV_OPT_REUSEABLE_PORT (1u<<7) /** Allocate a new evconnlistener object to listen for incoming TCP connections on a given file descriptor. @param base The event base to associate the listener with. @param cb A callback to be invoked when a new connection arrives. If the callback is NULL, the listener will be treated as disabled until the callback is set. @param ptr A user-supplied pointer to give to the callback. @param flags Any number of LEV_OPT_* flags @param backlog Passed to the listen() call to determine the length of the acceptable connection backlog. Set to -1 for a reasonable default. Set to 0 if the socket is already listening. @param fd The file descriptor to listen on. It must be a nonblocking file descriptor, and it should already be bound to an appropriate port and address. */ EVENT2_EXPORT_SYMBOL struct evconnlistener *evconnlistener_new(struct event_base *base, evconnlistener_cb cb, void *ptr, unsigned flags, int backlog, evutil_socket_t fd); /** Allocate a new evconnlistener object to listen for incoming TCP connections on a given address. @param base The event base to associate the listener with. @param cb A callback to be invoked when a new connection arrives. If the callback is NULL, the listener will be treated as disabled until the callback is set. @param ptr A user-supplied pointer to give to the callback. @param flags Any number of LEV_OPT_* flags @param backlog Passed to the listen() call to determine the length of the acceptable connection backlog. Set to -1 for a reasonable default. @param addr The address to listen for connections on. @param socklen The length of the address. */ EVENT2_EXPORT_SYMBOL struct evconnlistener *evconnlistener_new_bind(struct event_base *base, evconnlistener_cb cb, void *ptr, unsigned flags, int backlog, const struct sockaddr *sa, int socklen); /** Disable and deallocate an evconnlistener. */ EVENT2_EXPORT_SYMBOL void evconnlistener_free(struct evconnlistener *lev); /** Re-enable an evconnlistener that has been disabled. */ EVENT2_EXPORT_SYMBOL int evconnlistener_enable(struct evconnlistener *lev); /** Stop listening for connections on an evconnlistener. */ EVENT2_EXPORT_SYMBOL int evconnlistener_disable(struct evconnlistener *lev); /** Return an evconnlistener's associated event_base. */ EVENT2_EXPORT_SYMBOL struct event_base *evconnlistener_get_base(struct evconnlistener *lev); /** Return the socket that an evconnlistner is listening on. */ EVENT2_EXPORT_SYMBOL evutil_socket_t evconnlistener_get_fd(struct evconnlistener *lev); /** Change the callback on the listener to cb and its user_data to arg. */ EVENT2_EXPORT_SYMBOL void evconnlistener_set_cb(struct evconnlistener *lev, evconnlistener_cb cb, void *arg); /** Set an evconnlistener's error callback. */ EVENT2_EXPORT_SYMBOL void evconnlistener_set_error_cb(struct evconnlistener *lev, evconnlistener_errorcb errorcb); #ifdef __cplusplus } #endif #endif kytensor-2.49.0.7/third_party/libevent/include/event2/rpc.h000066400000000000000000000510411527052371200235620ustar00rootroot00000000000000/* * Copyright (c) 2006-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_RPC_H_INCLUDED_ #define EVENT2_RPC_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif /** @file rpc.h * * This header files provides basic support for an RPC server and client. * * To support RPCs in a server, every supported RPC command needs to be * defined and registered. * * EVRPC_HEADER(SendCommand, Request, Reply); * * SendCommand is the name of the RPC command. * Request is the name of a structure generated by event_rpcgen.py. * It contains all parameters relating to the SendCommand RPC. The * server needs to fill in the Reply structure. * Reply is the name of a structure generated by event_rpcgen.py. It * contains the answer to the RPC. * * To register an RPC with an HTTP server, you need to first create an RPC * base with: * * struct evrpc_base *base = evrpc_init(http); * * A specific RPC can then be registered with * * EVRPC_REGISTER(base, SendCommand, Request, Reply, FunctionCB, arg); * * when the server receives an appropriately formatted RPC, the user callback * is invoked. The callback needs to fill in the reply structure. * * void FunctionCB(EVRPC_STRUCT(SendCommand)* rpc, void *arg); * * To send the reply, call EVRPC_REQUEST_DONE(rpc); * * See the regression test for an example. */ /** Determines if the member has been set in the message @param msg the message to inspect @param member the member variable to test for presences @return 1 if it's present or 0 otherwise. */ #define EVTAG_HAS(msg, member) \ ((msg)->member##_set == 1) #ifndef EVENT2_RPC_COMPAT_H_INCLUDED_ /** Assigns a value to the member in the message. @param msg the message to which to assign a value @param member the name of the member variable @param value the value to assign */ #define EVTAG_ASSIGN(msg, member, value) \ (*(msg)->base->member##_assign)((msg), (value)) /** Assigns a value to the member in the message. @param msg the message to which to assign a value @param member the name of the member variable @param value the value to assign @param len the length of the value */ #define EVTAG_ASSIGN_WITH_LEN(msg, member, value, len) \ (*(msg)->base->member##_assign)((msg), (value), (len)) /** Returns the value for a member. @param msg the message from which to get the value @param member the name of the member variable @param pvalue a pointer to the variable to hold the value @return 0 on success, -1 otherwise. */ #define EVTAG_GET(msg, member, pvalue) \ (*(msg)->base->member##_get)((msg), (pvalue)) /** Returns the value for a member. @param msg the message from which to get the value @param member the name of the member variable @param pvalue a pointer to the variable to hold the value @param plen a pointer to the length of the value @return 0 on success, -1 otherwise. */ #define EVTAG_GET_WITH_LEN(msg, member, pvalue, plen) \ (*(msg)->base->member##_get)((msg), (pvalue), (plen)) #endif /* EVENT2_RPC_COMPAT_H_INCLUDED_ */ /** Adds a value to an array. */ #define EVTAG_ARRAY_ADD_VALUE(msg, member, value) \ (*(msg)->base->member##_add)((msg), (value)) /** Allocates a new entry in the array and returns it. */ #define EVTAG_ARRAY_ADD(msg, member) \ (*(msg)->base->member##_add)(msg) /** Gets a variable at the specified offset from the array. */ #define EVTAG_ARRAY_GET(msg, member, offset, pvalue) \ (*(msg)->base->member##_get)((msg), (offset), (pvalue)) /** Returns the number of entries in the array. */ #define EVTAG_ARRAY_LEN(msg, member) ((msg)->member##_length) struct evbuffer; struct event_base; struct evrpc_req_generic; struct evrpc_request_wrapper; struct evrpc; /** The type of a specific RPC Message * * @param rpcname the name of the RPC message */ #define EVRPC_STRUCT(rpcname) struct evrpc_req__##rpcname struct evhttp_request; struct evrpc_status; struct evrpc_hook_meta; /** Creates the definitions and prototypes for an RPC * * You need to use EVRPC_HEADER to create structures and function prototypes * needed by the server and client implementation. The structures have to be * defined in an .rpc file and converted to source code via event_rpcgen.py * * @param rpcname the name of the RPC * @param reqstruct the name of the RPC request structure * @param replystruct the name of the RPC reply structure * @see EVRPC_GENERATE() */ #define EVRPC_HEADER(rpcname, reqstruct, rplystruct) \ EVRPC_STRUCT(rpcname) { \ struct evrpc_hook_meta *hook_meta; \ struct reqstruct* request; \ struct rplystruct* reply; \ struct evrpc* rpc; \ struct evhttp_request* http_req; \ struct evbuffer* rpc_data; \ }; \ int evrpc_send_request_##rpcname(struct evrpc_pool *, \ struct reqstruct *, struct rplystruct *, \ void (*)(struct evrpc_status *, \ struct reqstruct *, struct rplystruct *, void *cbarg), \ void *); struct evrpc_pool; /** use EVRPC_GENERATE instead */ struct evrpc_request_wrapper *evrpc_make_request_ctx( struct evrpc_pool *pool, void *request, void *reply, const char *rpcname, void (*req_marshal)(struct evbuffer*, void *), void (*rpl_clear)(void *), int (*rpl_unmarshal)(void *, struct evbuffer *), void (*cb)(struct evrpc_status *, void *, void *, void *), void *cbarg); /** Creates a context structure that contains rpc specific information. * * EVRPC_MAKE_CTX is used to populate a RPC specific context that * contains information about marshaling the RPC data types. * * @param rpcname the name of the RPC * @param reqstruct the name of the RPC request structure * @param replystruct the name of the RPC reply structure * @param pool the evrpc_pool over which to make the request * @param request a pointer to the RPC request structure object * @param reply a pointer to the RPC reply structure object * @param cb the callback function to call when the RPC has completed * @param cbarg the argument to supply to the callback */ #define EVRPC_MAKE_CTX(rpcname, reqstruct, rplystruct, \ pool, request, reply, cb, cbarg) \ evrpc_make_request_ctx(pool, request, reply, \ #rpcname, \ (void (*)(struct evbuffer *, void *))reqstruct##_marshal, \ (void (*)(void *))rplystruct##_clear, \ (int (*)(void *, struct evbuffer *))rplystruct##_unmarshal, \ (void (*)(struct evrpc_status *, void *, void *, void *))cb, \ cbarg) /** Generates the code for receiving and sending an RPC message * * EVRPC_GENERATE is used to create the code corresponding to sending * and receiving a particular RPC message * * @param rpcname the name of the RPC * @param reqstruct the name of the RPC request structure * @param replystruct the name of the RPC reply structure * @see EVRPC_HEADER() */ #define EVRPC_GENERATE(rpcname, reqstruct, rplystruct) \ int evrpc_send_request_##rpcname(struct evrpc_pool *pool, \ struct reqstruct *request, struct rplystruct *reply, \ void (*cb)(struct evrpc_status *, \ struct reqstruct *, struct rplystruct *, void *cbarg), \ void *cbarg) { \ return evrpc_send_request_generic(pool, request, reply, \ (void (*)(struct evrpc_status *, void *, void *, void *))cb, \ cbarg, \ #rpcname, \ (void (*)(struct evbuffer *, void *))reqstruct##_marshal, \ (void (*)(void *))rplystruct##_clear, \ (int (*)(void *, struct evbuffer *))rplystruct##_unmarshal); \ } /** Provides access to the HTTP request object underlying an RPC * * Access to the underlying http object; can be used to look at headers or * for getting the remote ip address * * @param rpc_req the rpc request structure provided to the server callback * @return an struct evhttp_request object that can be inspected for * HTTP headers or sender information. */ #define EVRPC_REQUEST_HTTP(rpc_req) (rpc_req)->http_req /** completes the server response to an rpc request */ void evrpc_request_done(struct evrpc_req_generic *req); /** accessors for request and reply */ void *evrpc_get_request(struct evrpc_req_generic *req); void *evrpc_get_reply(struct evrpc_req_generic *req); /** Creates the reply to an RPC request * * EVRPC_REQUEST_DONE is used to answer a request; the reply is expected * to have been filled in. The request and reply pointers become invalid * after this call has finished. * * @param rpc_req the rpc request structure provided to the server callback */ #define EVRPC_REQUEST_DONE(rpc_req) do { \ struct evrpc_req_generic *req_ = (struct evrpc_req_generic *)(rpc_req); \ evrpc_request_done(req_); \ } while (0) struct evrpc_base; struct evhttp; /* functions to start up the rpc system */ /** Creates a new rpc base from which RPC requests can be received * * @param server a pointer to an existing HTTP server * @return a newly allocated evrpc_base struct * @see evrpc_free() */ struct evrpc_base *evrpc_init(struct evhttp *server); /** * Frees the evrpc base * * For now, you are responsible for making sure that no rpcs are ongoing. * * @param base the evrpc_base object to be freed * @see evrpc_init */ void evrpc_free(struct evrpc_base *base); /** register RPCs with the HTTP Server * * registers a new RPC with the HTTP server, each RPC needs to have * a unique name under which it can be identified. * * @param base the evrpc_base structure in which the RPC should be * registered. * @param name the name of the RPC * @param request the name of the RPC request structure * @param reply the name of the RPC reply structure * @param callback the callback that should be invoked when the RPC * is received. The callback has the following prototype * void (*callback)(EVRPC_STRUCT(Message)* rpc, void *arg) * @param cbarg an additional parameter that can be passed to the callback. * The parameter can be used to carry around state. */ #define EVRPC_REGISTER(base, name, request, reply, callback, cbarg) \ evrpc_register_generic(base, #name, \ (void (*)(struct evrpc_req_generic *, void *))callback, cbarg, \ (void *(*)(void *))request##_new, NULL, \ (void (*)(void *))request##_free, \ (int (*)(void *, struct evbuffer *))request##_unmarshal, \ (void *(*)(void *))reply##_new, NULL, \ (void (*)(void *))reply##_free, \ (int (*)(void *))reply##_complete, \ (void (*)(struct evbuffer *, void *))reply##_marshal) /** Low level function for registering an RPC with a server. Use EVRPC_REGISTER() instead. @see EVRPC_REGISTER() */ int evrpc_register_rpc(struct evrpc_base *, struct evrpc *, void (*)(struct evrpc_req_generic*, void *), void *); /** * Unregisters an already registered RPC * * @param base the evrpc_base object from which to unregister an RPC * @param name the name of the rpc to unregister * @return -1 on error or 0 when successful. * @see EVRPC_REGISTER() */ #define EVRPC_UNREGISTER(base, name) evrpc_unregister_rpc((base), #name) int evrpc_unregister_rpc(struct evrpc_base *base, const char *name); /* * Client-side RPC support */ struct evhttp_connection; struct evrpc_status; /** launches an RPC and sends it to the server * * EVRPC_MAKE_REQUEST() is used by the client to send an RPC to the server. * * @param name the name of the RPC * @param pool the evrpc_pool that contains the connection objects over which * the request should be sent. * @param request a pointer to the RPC request structure - it contains the * data to be sent to the server. * @param reply a pointer to the RPC reply structure. It is going to be filled * if the request was answered successfully * @param cb the callback to invoke when the RPC request has been answered * @param cbarg an additional argument to be passed to the client * @return 0 on success, -1 on failure */ #define EVRPC_MAKE_REQUEST(name, pool, request, reply, cb, cbarg) \ evrpc_send_request_##name((pool), (request), (reply), (cb), (cbarg)) /** Makes an RPC request based on the provided context. This is a low-level function and should not be used directly unless a custom context object is provided. Use EVRPC_MAKE_REQUEST() instead. @param ctx a context from EVRPC_MAKE_CTX() @returns 0 on success, -1 otherwise. @see EVRPC_MAKE_REQUEST(), EVRPC_MAKE_CTX() */ int evrpc_make_request(struct evrpc_request_wrapper *ctx); /** creates an rpc connection pool * * a pool has a number of connections associated with it. * rpc requests are always made via a pool. * * @param base a pointer to an struct event_based object; can be left NULL * in singled-threaded applications * @return a newly allocated struct evrpc_pool object * @see evrpc_pool_free() */ struct evrpc_pool *evrpc_pool_new(struct event_base *base); /** frees an rpc connection pool * * @param pool a pointer to an evrpc_pool allocated via evrpc_pool_new() * @see evrpc_pool_new() */ void evrpc_pool_free(struct evrpc_pool *pool); /** * Adds a connection over which rpc can be dispatched to the pool. * * The connection object must have been newly created. * * @param pool the pool to which to add the connection * @param evcon the connection to add to the pool. */ void evrpc_pool_add_connection(struct evrpc_pool *pool, struct evhttp_connection *evcon); /** * Removes a connection from the pool. * * The connection object must have been newly created. * * @param pool the pool from which to remove the connection * @param evcon the connection to remove from the pool. */ void evrpc_pool_remove_connection(struct evrpc_pool *pool, struct evhttp_connection *evcon); /** * Sets the timeout in secs after which a request has to complete. The * RPC is completely aborted if it does not complete by then. Setting * the timeout to 0 means that it never timeouts and can be used to * implement callback type RPCs. * * Any connection already in the pool will be updated with the new * timeout. Connections added to the pool after set_timeout has be * called receive the pool timeout only if no timeout has been set * for the connection itself. * * @param pool a pointer to a struct evrpc_pool object * @param timeout_in_secs the number of seconds after which a request should * timeout and a failure be returned to the callback. */ void evrpc_pool_set_timeout(struct evrpc_pool *pool, int timeout_in_secs); /** * Hooks for changing the input and output of RPCs; this can be used to * implement compression, authentication, encryption, ... */ enum EVRPC_HOOK_TYPE { EVRPC_INPUT, /**< apply the function to an input hook */ EVRPC_OUTPUT /**< apply the function to an output hook */ }; #ifndef _WIN32 /** Deprecated alias for EVRPC_INPUT. Not available on windows, where it * conflicts with platform headers. */ #define INPUT EVRPC_INPUT /** Deprecated alias for EVRPC_OUTPUT. Not available on windows, where it * conflicts with platform headers. */ #define OUTPUT EVRPC_OUTPUT #endif /** * Return value from hook processing functions */ enum EVRPC_HOOK_RESULT { EVRPC_TERMINATE = -1, /**< indicates the rpc should be terminated */ EVRPC_CONTINUE = 0, /**< continue processing the rpc */ EVRPC_PAUSE = 1 /**< pause processing request until resumed */ }; /** adds a processing hook to either an rpc base or rpc pool * * If a hook returns TERMINATE, the processing is aborted. On CONTINUE, * the request is immediately processed after the hook returns. If the * hook returns PAUSE, request processing stops until evrpc_resume_request() * has been called. * * The add functions return handles that can be used for removing hooks. * * @param vbase a pointer to either struct evrpc_base or struct evrpc_pool * @param hook_type either INPUT or OUTPUT * @param cb the callback to call when the hook is activated * @param cb_arg an additional argument for the callback * @return a handle to the hook so it can be removed later * @see evrpc_remove_hook() */ void *evrpc_add_hook(void *vbase, enum EVRPC_HOOK_TYPE hook_type, int (*cb)(void *, struct evhttp_request *, struct evbuffer *, void *), void *cb_arg); /** removes a previously added hook * * @param vbase a pointer to either struct evrpc_base or struct evrpc_pool * @param hook_type either INPUT or OUTPUT * @param handle a handle returned by evrpc_add_hook() * @return 1 on success or 0 on failure * @see evrpc_add_hook() */ int evrpc_remove_hook(void *vbase, enum EVRPC_HOOK_TYPE hook_type, void *handle); /** resume a paused request * * @param vbase a pointer to either struct evrpc_base or struct evrpc_pool * @param ctx the context pointer provided to the original hook call */ int evrpc_resume_request(void *vbase, void *ctx, enum EVRPC_HOOK_RESULT res); /** adds meta data to request * * evrpc_hook_add_meta() allows hooks to add meta data to a request. for * a client request, the meta data can be inserted by an outgoing request hook * and retrieved by the incoming request hook. * * @param ctx the context provided to the hook call * @param key a NUL-terminated c-string * @param data the data to be associated with the key * @param data_size the size of the data */ void evrpc_hook_add_meta(void *ctx, const char *key, const void *data, size_t data_size); /** retrieves meta data previously associated * * evrpc_hook_find_meta() can be used to retrieve meta data associated to a * request by a previous hook. * @param ctx the context provided to the hook call * @param key a NUL-terminated c-string * @param data pointer to a data pointer that will contain the retrieved data * @param data_size pointer to the size of the data * @return 0 on success or -1 on failure */ int evrpc_hook_find_meta(void *ctx, const char *key, void **data, size_t *data_size); /** * returns the connection object associated with the request * * @param ctx the context provided to the hook call * @return a pointer to the evhttp_connection object */ struct evhttp_connection *evrpc_hook_get_connection(void *ctx); /** Function for sending a generic RPC request. Do not call this function directly, use EVRPC_MAKE_REQUEST() instead. @see EVRPC_MAKE_REQUEST() */ int evrpc_send_request_generic(struct evrpc_pool *pool, void *request, void *reply, void (*cb)(struct evrpc_status *, void *, void *, void *), void *cb_arg, const char *rpcname, void (*req_marshal)(struct evbuffer *, void *), void (*rpl_clear)(void *), int (*rpl_unmarshal)(void *, struct evbuffer *)); /** Function for registering a generic RPC with the RPC base. Do not call this function directly, use EVRPC_REGISTER() instead. @see EVRPC_REGISTER() */ int evrpc_register_generic(struct evrpc_base *base, const char *name, void (*callback)(struct evrpc_req_generic *, void *), void *cbarg, void *(*req_new)(void *), void *req_new_arg, void (*req_free)(void *), int (*req_unmarshal)(void *, struct evbuffer *), void *(*rpl_new)(void *), void *rpl_new_arg, void (*rpl_free)(void *), int (*rpl_complete)(void *), void (*rpl_marshal)(struct evbuffer *, void *)); /** accessors for obscure and undocumented functionality */ struct evrpc_pool* evrpc_request_get_pool(struct evrpc_request_wrapper *ctx); void evrpc_request_set_pool(struct evrpc_request_wrapper *ctx, struct evrpc_pool *pool); void evrpc_request_set_cb(struct evrpc_request_wrapper *ctx, void (*cb)(struct evrpc_status*, void *request, void *reply, void *arg), void *cb_arg); #ifdef __cplusplus } #endif #endif /* EVENT2_RPC_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/rpc_compat.h000066400000000000000000000044571527052371200251360ustar00rootroot00000000000000/* * Copyright (c) 2006-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_RPC_COMPAT_H_INCLUDED_ #define EVENT2_RPC_COMPAT_H_INCLUDED_ /** @file event2/rpc_compat.h Deprecated versions of the functions in rpc.h: provided only for backwards compatibility. */ #ifdef __cplusplus extern "C" { #endif /** backwards compatible accessors that work only with gcc */ #if defined(__GNUC__) && !defined(__STRICT_ANSI__) #undef EVTAG_ASSIGN #undef EVTAG_GET #undef EVTAG_ADD #define EVTAG_ASSIGN(msg, member, args...) \ (*(msg)->base->member##_assign)(msg, ## args) #define EVTAG_GET(msg, member, args...) \ (*(msg)->base->member##_get)(msg, ## args) #define EVTAG_ADD(msg, member, args...) \ (*(msg)->base->member##_add)(msg, ## args) #endif #define EVTAG_LEN(msg, member) ((msg)->member##_length) #ifdef __cplusplus } #endif #endif /* EVENT2_EVENT_COMPAT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/rpc_struct.h000066400000000000000000000062431527052371200251720ustar00rootroot00000000000000/* * Copyright (c) 2006-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_RPC_STRUCT_H_INCLUDED_ #define EVENT2_RPC_STRUCT_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif /** @file event2/rpc_struct.h Structures used by rpc.h. Using these structures directly may harm forward compatibility: be careful! */ /** * provides information about the completed RPC request. */ struct evrpc_status { #define EVRPC_STATUS_ERR_NONE 0 #define EVRPC_STATUS_ERR_TIMEOUT 1 #define EVRPC_STATUS_ERR_BADPAYLOAD 2 #define EVRPC_STATUS_ERR_UNSTARTED 3 #define EVRPC_STATUS_ERR_HOOKABORTED 4 int error; /* for looking at headers or other information */ struct evhttp_request *http_req; }; /* the structure below needs to be synchronized with evrpc_req_generic */ /* Encapsulates a request */ struct evrpc { TAILQ_ENTRY(evrpc) next; /* the URI at which the request handler lives */ const char* uri; /* creates a new request structure */ void *(*request_new)(void *); void *request_new_arg; /* frees the request structure */ void (*request_free)(void *); /* unmarshals the buffer into the proper request structure */ int (*request_unmarshal)(void *, struct evbuffer *); /* creates a new reply structure */ void *(*reply_new)(void *); void *reply_new_arg; /* frees the reply structure */ void (*reply_free)(void *); /* verifies that the reply is valid */ int (*reply_complete)(void *); /* marshals the reply into a buffer */ void (*reply_marshal)(struct evbuffer*, void *); /* the callback invoked for each received rpc */ void (*cb)(struct evrpc_req_generic *, void *); void *cb_arg; /* reference for further configuration */ struct evrpc_base *base; }; #ifdef __cplusplus } #endif #endif /* EVENT2_RPC_STRUCT_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/tag.h000066400000000000000000000114621527052371200235540ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_TAG_H_INCLUDED_ #define EVENT2_TAG_H_INCLUDED_ /** @file event2/tag.h Helper functions for reading and writing tagged data onto buffers. */ #include #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_SYS_TIME_H #include #endif /* For int types. */ #include struct evbuffer; /* * Marshaling tagged data - We assume that all tags are inserted in their * numeric order - so that unknown tags will always be higher than the * known ones - and we can just ignore the end of an event buffer. */ EVENT2_EXPORT_SYMBOL void evtag_init(void); /** Unmarshals the header and returns the length of the payload @param evbuf the buffer from which to unmarshal data @param ptag a pointer in which the tag id is being stored @returns -1 on failure or the number of bytes in the remaining payload. */ EVENT2_EXPORT_SYMBOL int evtag_unmarshal_header(struct evbuffer *evbuf, ev_uint32_t *ptag); EVENT2_EXPORT_SYMBOL void evtag_marshal(struct evbuffer *evbuf, ev_uint32_t tag, const void *data, ev_uint32_t len); EVENT2_EXPORT_SYMBOL void evtag_marshal_buffer(struct evbuffer *evbuf, ev_uint32_t tag, struct evbuffer *data); /** Encode an integer and store it in an evbuffer. We encode integers by nybbles; the first nibble contains the number of significant nibbles - 1; this allows us to encode up to 64-bit integers. This function is byte-order independent. @param evbuf evbuffer to store the encoded number @param number a 32-bit integer */ EVENT2_EXPORT_SYMBOL void evtag_encode_int(struct evbuffer *evbuf, ev_uint32_t number); EVENT2_EXPORT_SYMBOL void evtag_encode_int64(struct evbuffer *evbuf, ev_uint64_t number); EVENT2_EXPORT_SYMBOL void evtag_marshal_int(struct evbuffer *evbuf, ev_uint32_t tag, ev_uint32_t integer); EVENT2_EXPORT_SYMBOL void evtag_marshal_int64(struct evbuffer *evbuf, ev_uint32_t tag, ev_uint64_t integer); EVENT2_EXPORT_SYMBOL void evtag_marshal_string(struct evbuffer *buf, ev_uint32_t tag, const char *string); EVENT2_EXPORT_SYMBOL void evtag_marshal_timeval(struct evbuffer *evbuf, ev_uint32_t tag, struct timeval *tv); EVENT2_EXPORT_SYMBOL int evtag_unmarshal(struct evbuffer *src, ev_uint32_t *ptag, struct evbuffer *dst); EVENT2_EXPORT_SYMBOL int evtag_peek(struct evbuffer *evbuf, ev_uint32_t *ptag); EVENT2_EXPORT_SYMBOL int evtag_peek_length(struct evbuffer *evbuf, ev_uint32_t *plength); EVENT2_EXPORT_SYMBOL int evtag_payload_length(struct evbuffer *evbuf, ev_uint32_t *plength); EVENT2_EXPORT_SYMBOL int evtag_consume(struct evbuffer *evbuf); EVENT2_EXPORT_SYMBOL int evtag_unmarshal_int(struct evbuffer *evbuf, ev_uint32_t need_tag, ev_uint32_t *pinteger); EVENT2_EXPORT_SYMBOL int evtag_unmarshal_int64(struct evbuffer *evbuf, ev_uint32_t need_tag, ev_uint64_t *pinteger); EVENT2_EXPORT_SYMBOL int evtag_unmarshal_fixed(struct evbuffer *src, ev_uint32_t need_tag, void *data, size_t len); EVENT2_EXPORT_SYMBOL int evtag_unmarshal_string(struct evbuffer *evbuf, ev_uint32_t need_tag, char **pstring); EVENT2_EXPORT_SYMBOL int evtag_unmarshal_timeval(struct evbuffer *evbuf, ev_uint32_t need_tag, struct timeval *ptv); #ifdef __cplusplus } #endif #endif /* EVENT2_TAG_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/tag_compat.h000066400000000000000000000041351527052371200251160ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_TAG_COMPAT_H_INCLUDED_ #define EVENT2_TAG_COMPAT_H_INCLUDED_ /** @file event2/tag_compat.h Obsolete/deprecated functions from tag.h; provided only for backwards compatibility. */ /** @name Misnamed functions @deprecated These macros are deprecated because their names don't follow Libevent's naming conventions. Use evtag_encode_int and evtag_encode_int64 instead. @{ */ #define encode_int(evbuf, number) evtag_encode_int((evbuf), (number)) #define encode_int64(evbuf, number) evtag_encode_int64((evbuf), (number)) /**@}*/ #endif /* EVENT2_TAG_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/thread.h000066400000000000000000000233401527052371200242460ustar00rootroot00000000000000/* * Copyright (c) 2008-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_THREAD_H_INCLUDED_ #define EVENT2_THREAD_H_INCLUDED_ /** @file event2/thread.h Functions for multi-threaded applications using Libevent. When using a multi-threaded application in which multiple threads add and delete events from a single event base, Libevent needs to lock its data structures. Like the memory-management function hooks, all of the threading functions _must_ be set up before an event_base is created if you want the base to use them. Most programs will either be using Windows threads or Posix threads. You can configure Libevent to use one of these event_use_windows_threads() or event_use_pthreads() respectively. If you're using another threading library, you'll need to configure threading functions manually using evthread_set_lock_callbacks() and evthread_set_condition_callbacks(). */ #include #ifdef __cplusplus extern "C" { #endif #include /** @name Flags passed to lock functions @{ */ /** A flag passed to a locking callback when the lock was allocated as a * read-write lock, and we want to acquire or release the lock for writing. */ #define EVTHREAD_WRITE 0x04 /** A flag passed to a locking callback when the lock was allocated as a * read-write lock, and we want to acquire or release the lock for reading. */ #define EVTHREAD_READ 0x08 /** A flag passed to a locking callback when we don't want to block waiting * for the lock; if we can't get the lock immediately, we will instead * return nonzero from the locking callback. */ #define EVTHREAD_TRY 0x10 /**@}*/ #if !defined(EVENT__DISABLE_THREAD_SUPPORT) || defined(EVENT_IN_DOXYGEN_) #define EVTHREAD_LOCK_API_VERSION 1 /** @name Types of locks @{*/ /** A recursive lock is one that can be acquired multiple times at once by the * same thread. No other process can allocate the lock until the thread that * has been holding it has unlocked it as many times as it locked it. */ #define EVTHREAD_LOCKTYPE_RECURSIVE 1 /* A read-write lock is one that allows multiple simultaneous readers, but * where any one writer excludes all other writers and readers. */ #define EVTHREAD_LOCKTYPE_READWRITE 2 /**@}*/ /** This structure describes the interface a threading library uses for * locking. It's used to tell evthread_set_lock_callbacks() how to use * locking on this platform. */ struct evthread_lock_callbacks { /** The current version of the locking API. Set this to * EVTHREAD_LOCK_API_VERSION */ int lock_api_version; /** Which kinds of locks does this version of the locking API * support? A bitfield of EVTHREAD_LOCKTYPE_RECURSIVE and * EVTHREAD_LOCKTYPE_READWRITE. * * (Note that RECURSIVE locks are currently mandatory, and * READWRITE locks are not currently used.) **/ unsigned supported_locktypes; /** Function to allocate and initialize new lock of type 'locktype'. * Returns NULL on failure. */ void *(*alloc)(unsigned locktype); /** Funtion to release all storage held in 'lock', which was created * with type 'locktype'. */ void (*free)(void *lock, unsigned locktype); /** Acquire an already-allocated lock at 'lock' with mode 'mode'. * Returns 0 on success, and nonzero on failure. */ int (*lock)(unsigned mode, void *lock); /** Release a lock at 'lock' using mode 'mode'. Returns 0 on success, * and nonzero on failure. */ int (*unlock)(unsigned mode, void *lock); }; /** Sets a group of functions that Libevent should use for locking. * For full information on the required callback API, see the * documentation for the individual members of evthread_lock_callbacks. * * Note that if you're using Windows or the Pthreads threading library, you * probably shouldn't call this function; instead, use * evthread_use_windows_threads() or evthread_use_posix_threads() if you can. */ EVENT2_EXPORT_SYMBOL int evthread_set_lock_callbacks(const struct evthread_lock_callbacks *); #define EVTHREAD_CONDITION_API_VERSION 1 struct timeval; /** This structure describes the interface a threading library uses for * condition variables. It's used to tell evthread_set_condition_callbacks * how to use locking on this platform. */ struct evthread_condition_callbacks { /** The current version of the conditions API. Set this to * EVTHREAD_CONDITION_API_VERSION */ int condition_api_version; /** Function to allocate and initialize a new condition variable. * Returns the condition variable on success, and NULL on failure. * The 'condtype' argument will be 0 with this API version. */ void *(*alloc_condition)(unsigned condtype); /** Function to free a condition variable. */ void (*free_condition)(void *cond); /** Function to signal a condition variable. If 'broadcast' is 1, all * threads waiting on 'cond' should be woken; otherwise, only on one * thread is worken. Should return 0 on success, -1 on failure. * This function will only be called while holding the associated * lock for the condition. */ int (*signal_condition)(void *cond, int broadcast); /** Function to wait for a condition variable. The lock 'lock' * will be held when this function is called; should be released * while waiting for the condition to be come signalled, and * should be held again when this function returns. * If timeout is provided, it is interval of seconds to wait for * the event to become signalled; if it is NULL, the function * should wait indefinitely. * * The function should return -1 on error; 0 if the condition * was signalled, or 1 on a timeout. */ int (*wait_condition)(void *cond, void *lock, const struct timeval *timeout); }; /** Sets a group of functions that Libevent should use for condition variables. * For full information on the required callback API, see the * documentation for the individual members of evthread_condition_callbacks. * * Note that if you're using Windows or the Pthreads threading library, you * probably shouldn't call this function; instead, use * evthread_use_windows_threads() or evthread_use_pthreads() if you can. */ EVENT2_EXPORT_SYMBOL int evthread_set_condition_callbacks( const struct evthread_condition_callbacks *); /** Sets the function for determining the thread id. @param base the event base for which to set the id function @param id_fn the identify function Libevent should invoke to determine the identity of a thread. */ EVENT2_EXPORT_SYMBOL void evthread_set_id_callback( unsigned long (*id_fn)(void)); #if (defined(_WIN32) && !defined(EVENT__DISABLE_THREAD_SUPPORT)) || defined(EVENT_IN_DOXYGEN_) /** Sets up Libevent for use with Windows builtin locking and thread ID functions. Unavailable if Libevent is not built for Windows. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evthread_use_windows_threads(void); /** Defined if Libevent was built with support for evthread_use_windows_threads() */ #define EVTHREAD_USE_WINDOWS_THREADS_IMPLEMENTED 1 #endif #if defined(EVENT__HAVE_PTHREADS) || defined(EVENT_IN_DOXYGEN_) /** Sets up Libevent for use with Pthreads locking and thread ID functions. Unavailable if Libevent is not build for use with pthreads. Requires libraries to link against Libevent_pthreads as well as Libevent. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evthread_use_pthreads(void); /** Defined if Libevent was built with support for evthread_use_pthreads() */ #define EVTHREAD_USE_PTHREADS_IMPLEMENTED 1 #endif /** Enable debugging wrappers around the current lock callbacks. If Libevent * makes one of several common locking errors, exit with an assertion failure. * * If you're going to call this function, you must do so before any locks are * allocated. **/ EVENT2_EXPORT_SYMBOL void evthread_enable_lock_debugging(void); /* Old (misspelled) version: This is deprecated; use * evthread_enable_log_debugging instead. */ EVENT2_EXPORT_SYMBOL void evthread_enable_lock_debuging(void); #endif /* EVENT__DISABLE_THREAD_SUPPORT */ struct event_base; /** Make sure it's safe to tell an event base to wake up from another thread or a signal handler. You shouldn't need to call this by hand; configuring the base with thread support should be necessary and sufficient. @return 0 on success, -1 on failure. */ EVENT2_EXPORT_SYMBOL int evthread_make_base_notifiable(struct event_base *base); #ifdef __cplusplus } #endif #endif /* EVENT2_THREAD_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/util.h000066400000000000000000000673771527052371200237760ustar00rootroot00000000000000/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_UTIL_H_INCLUDED_ #define EVENT2_UTIL_H_INCLUDED_ /** @file event2/util.h Common convenience functions for cross-platform portability and related socket manipulations. */ #include #ifdef __cplusplus extern "C" { #endif #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #ifdef EVENT__HAVE_STDINT_H #include #elif defined(EVENT__HAVE_INTTYPES_H) #include #endif #ifdef EVENT__HAVE_SYS_TYPES_H #include #endif #ifdef EVENT__HAVE_STDDEF_H #include #endif #ifdef _MSC_VER #include #endif #include #ifdef EVENT__HAVE_NETDB_H #if !defined(_GNU_SOURCE) #define _GNU_SOURCE #endif #include #endif #ifdef _WIN32 #include #ifdef EVENT__HAVE_GETADDRINFO /* for EAI_* definitions. */ #include #endif #else #ifdef EVENT__HAVE_ERRNO_H #include #endif #include #endif #include /* Some openbsd autoconf versions get the name of this macro wrong. */ #if defined(EVENT__SIZEOF_VOID__) && !defined(EVENT__SIZEOF_VOID_P) #define EVENT__SIZEOF_VOID_P EVENT__SIZEOF_VOID__ #endif /** * @name Standard integer types. * * Integer type definitions for types that are supposed to be defined in the * C99-specified stdint.h. Shamefully, some platforms do not include * stdint.h, so we need to replace it. (If you are on a platform like this, * your C headers are now over 10 years out of date. You should bug them to * do something about this.) * * We define: * *
*
ev_uint64_t, ev_uint32_t, ev_uint16_t, ev_uint8_t
*
unsigned integer types of exactly 64, 32, 16, and 8 bits * respectively.
*
ev_int64_t, ev_int32_t, ev_int16_t, ev_int8_t
*
signed integer types of exactly 64, 32, 16, and 8 bits * respectively.
*
ev_uintptr_t, ev_intptr_t
*
unsigned/signed integers large enough * to hold a pointer without loss of bits.
*
ev_ssize_t
*
A signed type of the same size as size_t
*
ev_off_t
*
A signed type typically used to represent offsets within a * (potentially large) file
* * @{ */ #ifdef EVENT__HAVE_UINT64_T #define ev_uint64_t uint64_t #define ev_int64_t int64_t #elif defined(_WIN32) #define ev_uint64_t unsigned __int64 #define ev_int64_t signed __int64 #elif EVENT__SIZEOF_LONG_LONG == 8 #define ev_uint64_t unsigned long long #define ev_int64_t long long #elif EVENT__SIZEOF_LONG == 8 #define ev_uint64_t unsigned long #define ev_int64_t long #elif defined(EVENT_IN_DOXYGEN_) #define ev_uint64_t ... #define ev_int64_t ... #else #error "No way to define ev_uint64_t" #endif #ifdef EVENT__HAVE_UINT32_T #define ev_uint32_t uint32_t #define ev_int32_t int32_t #elif defined(_WIN32) #define ev_uint32_t unsigned int #define ev_int32_t signed int #elif EVENT__SIZEOF_LONG == 4 #define ev_uint32_t unsigned long #define ev_int32_t signed long #elif EVENT__SIZEOF_INT == 4 #define ev_uint32_t unsigned int #define ev_int32_t signed int #elif defined(EVENT_IN_DOXYGEN_) #define ev_uint32_t ... #define ev_int32_t ... #else #error "No way to define ev_uint32_t" #endif #ifdef EVENT__HAVE_UINT16_T #define ev_uint16_t uint16_t #define ev_int16_t int16_t #elif defined(_WIN32) #define ev_uint16_t unsigned short #define ev_int16_t signed short #elif EVENT__SIZEOF_INT == 2 #define ev_uint16_t unsigned int #define ev_int16_t signed int #elif EVENT__SIZEOF_SHORT == 2 #define ev_uint16_t unsigned short #define ev_int16_t signed short #elif defined(EVENT_IN_DOXYGEN_) #define ev_uint16_t ... #define ev_int16_t ... #else #error "No way to define ev_uint16_t" #endif #ifdef EVENT__HAVE_UINT8_T #define ev_uint8_t uint8_t #define ev_int8_t int8_t #elif defined(EVENT_IN_DOXYGEN_) #define ev_uint8_t ... #define ev_int8_t ... #else #define ev_uint8_t unsigned char #define ev_int8_t signed char #endif #ifdef EVENT__HAVE_UINTPTR_T #define ev_uintptr_t uintptr_t #define ev_intptr_t intptr_t #elif EVENT__SIZEOF_VOID_P <= 4 #define ev_uintptr_t ev_uint32_t #define ev_intptr_t ev_int32_t #elif EVENT__SIZEOF_VOID_P <= 8 #define ev_uintptr_t ev_uint64_t #define ev_intptr_t ev_int64_t #elif defined(EVENT_IN_DOXYGEN_) #define ev_uintptr_t ... #define ev_intptr_t ... #else #error "No way to define ev_uintptr_t" #endif #ifdef EVENT__ssize_t #define ev_ssize_t EVENT__ssize_t #else #define ev_ssize_t ssize_t #endif /* Note that we define ev_off_t based on the compile-time size of off_t that * we used to build Libevent, and not based on the current size of off_t. * (For example, we don't define ev_off_t to off_t.). We do this because * some systems let you build your software with different off_t sizes * at runtime, and so putting in any dependency on off_t would risk API * mismatch. */ #ifdef _WIN32 #define ev_off_t ev_int64_t #elif EVENT__SIZEOF_OFF_T == 8 #define ev_off_t ev_int64_t #elif EVENT__SIZEOF_OFF_T == 4 #define ev_off_t ev_int32_t #elif defined(EVENT_IN_DOXYGEN_) #define ev_off_t ... #else #define ev_off_t off_t #endif /**@}*/ /* Limits for integer types. We're making two assumptions here: - The compiler does constant folding properly. - The platform does signed arithmetic in two's complement. */ /** @name Limits for integer types These macros hold the largest or smallest values possible for the ev_[u]int*_t types. @{ */ #ifndef EVENT__HAVE_STDINT_H #define EV_UINT64_MAX ((((ev_uint64_t)0xffffffffUL) << 32) | 0xffffffffUL) #define EV_INT64_MAX ((((ev_int64_t) 0x7fffffffL) << 32) | 0xffffffffL) #define EV_INT64_MIN ((-EV_INT64_MAX) - 1) #define EV_UINT32_MAX ((ev_uint32_t)0xffffffffUL) #define EV_INT32_MAX ((ev_int32_t) 0x7fffffffL) #define EV_INT32_MIN ((-EV_INT32_MAX) - 1) #define EV_UINT16_MAX ((ev_uint16_t)0xffffUL) #define EV_INT16_MAX ((ev_int16_t) 0x7fffL) #define EV_INT16_MIN ((-EV_INT16_MAX) - 1) #define EV_UINT8_MAX 255 #define EV_INT8_MAX 127 #define EV_INT8_MIN ((-EV_INT8_MAX) - 1) #else #define EV_UINT64_MAX UINT64_MAX #define EV_INT64_MAX INT64_MAX #define EV_INT64_MIN INT64_MIN #define EV_UINT32_MAX UINT32_MAX #define EV_INT32_MAX INT32_MAX #define EV_INT32_MIN INT32_MIN #define EV_UINT16_MAX UINT16_MAX #define EV_INT16_MAX INT16_MAX #define EV_UINT8_MAX UINT8_MAX #define EV_INT8_MAX INT8_MAX #define EV_INT8_MIN INT8_MIN /** @} */ #endif /** @name Limits for SIZE_T and SSIZE_T @{ */ #if EVENT__SIZEOF_SIZE_T == 8 #define EV_SIZE_MAX EV_UINT64_MAX #define EV_SSIZE_MAX EV_INT64_MAX #elif EVENT__SIZEOF_SIZE_T == 4 #define EV_SIZE_MAX EV_UINT32_MAX #define EV_SSIZE_MAX EV_INT32_MAX #elif defined(EVENT_IN_DOXYGEN_) #define EV_SIZE_MAX ... #define EV_SSIZE_MAX ... #else #error "No way to define SIZE_MAX" #endif #define EV_SSIZE_MIN ((-EV_SSIZE_MAX) - 1) /**@}*/ #ifdef _WIN32 #define ev_socklen_t int #elif defined(EVENT__socklen_t) #define ev_socklen_t EVENT__socklen_t #else #define ev_socklen_t socklen_t #endif #ifdef EVENT__HAVE_STRUCT_SOCKADDR_STORAGE___SS_FAMILY #if !defined(EVENT__HAVE_STRUCT_SOCKADDR_STORAGE_SS_FAMILY) \ && !defined(ss_family) #define ss_family __ss_family #endif #endif /** * A type wide enough to hold the output of "socket()" or "accept()". On * Windows, this is an intptr_t; elsewhere, it is an int. */ #ifdef _WIN32 #define evutil_socket_t intptr_t #else #define evutil_socket_t int #endif /** * Structure to hold information about a monotonic timer * * Use this with evutil_configure_monotonic_time() and * evutil_gettime_monotonic(). * * This is an opaque structure; you can allocate one using * evutil_monotonic_timer_new(). * * @see evutil_monotonic_timer_new(), evutil_monotonic_timer_free(), * evutil_configure_monotonic_time(), evutil_gettime_monotonic() */ struct evutil_monotonic_timer #ifdef EVENT_IN_DOXYGEN_ {/*Empty body so that doxygen will generate documentation here.*/} #endif ; #define EV_MONOT_PRECISE 1 #define EV_MONOT_FALLBACK 2 /** Format a date string using RFC 1123 format (used in HTTP). * If `tm` is NULL, current system's time will be used. * The number of characters written will be returned. * One should check if the return value is smaller than `datelen` to check if * the result is truncated or not. */ EVENT2_EXPORT_SYMBOL int evutil_date_rfc1123(char *date, const size_t datelen, const struct tm *tm); /** Allocate a new struct evutil_monotonic_timer for use with the * evutil_configure_monotonic_time() and evutil_gettime_monotonic() * functions. You must configure the timer with * evutil_configure_monotonic_time() before using it. */ EVENT2_EXPORT_SYMBOL struct evutil_monotonic_timer * evutil_monotonic_timer_new(void); /** Free a struct evutil_monotonic_timer that was allocated using * evutil_monotonic_timer_new(). */ EVENT2_EXPORT_SYMBOL void evutil_monotonic_timer_free(struct evutil_monotonic_timer *timer); /** Set up a struct evutil_monotonic_timer; flags can include * EV_MONOT_PRECISE and EV_MONOT_FALLBACK. */ EVENT2_EXPORT_SYMBOL int evutil_configure_monotonic_time(struct evutil_monotonic_timer *timer, int flags); /** Query the current monotonic time from a struct evutil_monotonic_timer * previously configured with evutil_configure_monotonic_time(). Monotonic * time is guaranteed never to run in reverse, but is not necessarily epoch- * based, or relative to any other definite point. Use it to make reliable * measurements of elapsed time between events even when the system time * may be changed. * * It is not safe to use this funtion on the same timer from multiple * threads. */ EVENT2_EXPORT_SYMBOL int evutil_gettime_monotonic(struct evutil_monotonic_timer *timer, struct timeval *tp); /** Create two new sockets that are connected to each other. On Unix, this simply calls socketpair(). On Windows, it uses the loopback network interface on 127.0.0.1, and only AF_INET,SOCK_STREAM are supported. (This may fail on some Windows hosts where firewall software has cleverly decided to keep 127.0.0.1 from talking to itself.) Parameters and return values are as for socketpair() */ EVENT2_EXPORT_SYMBOL int evutil_socketpair(int d, int type, int protocol, evutil_socket_t sv[2]); /** Do platform-specific operations as needed to make a socket nonblocking. @param sock The socket to make nonblocking @return 0 on success, -1 on failure */ EVENT2_EXPORT_SYMBOL int evutil_make_socket_nonblocking(evutil_socket_t sock); /** Do platform-specific operations to make a listener socket reusable. Specifically, we want to make sure that another program will be able to bind this address right after we've closed the listener. This differs from Windows's interpretation of "reusable", which allows multiple listeners to bind the same address at the same time. @param sock The socket to make reusable @return 0 on success, -1 on failure */ EVENT2_EXPORT_SYMBOL int evutil_make_listen_socket_reuseable(evutil_socket_t sock); /** Do platform-specific operations to make a listener port reusable. Specifically, we want to make sure that multiple programs which also set the same socket option will be able to bind, listen at the same time. This is a feature available only to Linux 3.9+ @param sock The socket to make reusable @return 0 on success, -1 on failure */ EVENT2_EXPORT_SYMBOL int evutil_make_listen_socket_reuseable_port(evutil_socket_t sock); /** Do platform-specific operations as needed to close a socket upon a successful execution of one of the exec*() functions. @param sock The socket to be closed @return 0 on success, -1 on failure */ EVENT2_EXPORT_SYMBOL int evutil_make_socket_closeonexec(evutil_socket_t sock); /** Do the platform-specific call needed to close a socket returned from socket() or accept(). @param sock The socket to be closed @return 0 on success, -1 on failure */ EVENT2_EXPORT_SYMBOL int evutil_closesocket(evutil_socket_t sock); #define EVUTIL_CLOSESOCKET(s) evutil_closesocket(s) /** Do platform-specific operations, if possible, to make a tcp listener * socket defer accept()s until there is data to read. * * Not all platforms support this. You don't want to do this for every * listener socket: only the ones that implement a protocol where the * client transmits before the server needs to respond. * * @param sock The listening socket to to make deferred * @return 0 on success (whether the operation is supported or not), * -1 on failure */ EVENT2_EXPORT_SYMBOL int evutil_make_tcp_listen_socket_deferred(evutil_socket_t sock); #ifdef _WIN32 /** Return the most recent socket error. Not idempotent on all platforms. */ #define EVUTIL_SOCKET_ERROR() WSAGetLastError() /** Replace the most recent socket error with errcode */ #define EVUTIL_SET_SOCKET_ERROR(errcode) \ do { WSASetLastError(errcode); } while (0) /** Return the most recent socket error to occur on sock. */ EVENT2_EXPORT_SYMBOL int evutil_socket_geterror(evutil_socket_t sock); /** Convert a socket error to a string. */ EVENT2_EXPORT_SYMBOL const char *evutil_socket_error_to_string(int errcode); #elif defined(EVENT_IN_DOXYGEN_) /** @name Socket error functions These functions are needed for making programs compatible between Windows and Unix-like platforms. You see, Winsock handles socket errors differently from the rest of the world. Elsewhere, a socket error is like any other error and is stored in errno. But winsock functions require you to retrieve the error with a special function, and don't let you use strerror for the error codes. And handling EWOULDBLOCK is ... different. @{ */ /** Return the most recent socket error. Not idempotent on all platforms. */ #define EVUTIL_SOCKET_ERROR() ... /** Replace the most recent socket error with errcode */ #define EVUTIL_SET_SOCKET_ERROR(errcode) ... /** Return the most recent socket error to occur on sock. */ #define evutil_socket_geterror(sock) ... /** Convert a socket error to a string. */ #define evutil_socket_error_to_string(errcode) ... /**@}*/ #else #define EVUTIL_SOCKET_ERROR() (errno) #define EVUTIL_SET_SOCKET_ERROR(errcode) \ do { errno = (errcode); } while (0) #define evutil_socket_geterror(sock) (errno) #define evutil_socket_error_to_string(errcode) (strerror(errcode)) #endif /** * @name Manipulation macros for struct timeval. * * We define replacements * for timeradd, timersub, timerclear, timercmp, and timerisset. * * @{ */ #ifdef EVENT__HAVE_TIMERADD #define evutil_timeradd(tvp, uvp, vvp) timeradd((tvp), (uvp), (vvp)) #define evutil_timersub(tvp, uvp, vvp) timersub((tvp), (uvp), (vvp)) #else #define evutil_timeradd(tvp, uvp, vvp) \ do { \ (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \ (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec; \ if ((vvp)->tv_usec >= 1000000) { \ (vvp)->tv_sec++; \ (vvp)->tv_usec -= 1000000; \ } \ } while (0) #define evutil_timersub(tvp, uvp, vvp) \ do { \ (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ if ((vvp)->tv_usec < 0) { \ (vvp)->tv_sec--; \ (vvp)->tv_usec += 1000000; \ } \ } while (0) #endif /* !EVENT__HAVE_TIMERADD */ #ifdef EVENT__HAVE_TIMERCLEAR #define evutil_timerclear(tvp) timerclear(tvp) #else #define evutil_timerclear(tvp) (tvp)->tv_sec = (tvp)->tv_usec = 0 #endif /**@}*/ /** Return true iff the tvp is related to uvp according to the relational * operator cmp. Recognized values for cmp are ==, <=, <, >=, and >. */ #define evutil_timercmp(tvp, uvp, cmp) \ (((tvp)->tv_sec == (uvp)->tv_sec) ? \ ((tvp)->tv_usec cmp (uvp)->tv_usec) : \ ((tvp)->tv_sec cmp (uvp)->tv_sec)) #ifdef EVENT__HAVE_TIMERISSET #define evutil_timerisset(tvp) timerisset(tvp) #else #define evutil_timerisset(tvp) ((tvp)->tv_sec || (tvp)->tv_usec) #endif /** Replacement for offsetof on platforms that don't define it. */ #ifdef offsetof #define evutil_offsetof(type, field) offsetof(type, field) #else #define evutil_offsetof(type, field) ((off_t)(&((type *)0)->field)) #endif /* big-int related functions */ /** Parse a 64-bit value from a string. Arguments are as for strtol. */ EVENT2_EXPORT_SYMBOL ev_int64_t evutil_strtoll(const char *s, char **endptr, int base); /** Replacement for gettimeofday on platforms that lack it. */ #ifdef EVENT__HAVE_GETTIMEOFDAY #define evutil_gettimeofday(tv, tz) gettimeofday((tv), (tz)) #else struct timezone; EVENT2_EXPORT_SYMBOL int evutil_gettimeofday(struct timeval *tv, struct timezone *tz); #endif /** Replacement for snprintf to get consistent behavior on platforms for which the return value of snprintf does not conform to C99. */ EVENT2_EXPORT_SYMBOL int evutil_snprintf(char *buf, size_t buflen, const char *format, ...) #ifdef __GNUC__ __attribute__((format(printf, 3, 4))) #endif ; /** Replacement for vsnprintf to get consistent behavior on platforms for which the return value of snprintf does not conform to C99. */ EVENT2_EXPORT_SYMBOL int evutil_vsnprintf(char *buf, size_t buflen, const char *format, va_list ap) #ifdef __GNUC__ __attribute__((format(printf, 3, 0))) #endif ; /** Replacement for inet_ntop for platforms which lack it. */ EVENT2_EXPORT_SYMBOL const char *evutil_inet_ntop(int af, const void *src, char *dst, size_t len); /** Replacement for inet_pton for platforms which lack it. */ EVENT2_EXPORT_SYMBOL int evutil_inet_pton(int af, const char *src, void *dst); struct sockaddr; /** Parse an IPv4 or IPv6 address, with optional port, from a string. Recognized formats are: - [IPv6Address]:port - [IPv6Address] - IPv6Address - IPv4Address:port - IPv4Address If no port is specified, the port in the output is set to 0. @param str The string to parse. @param out A struct sockaddr to hold the result. This should probably be a struct sockaddr_storage. @param outlen A pointer to the number of bytes that that 'out' can safely hold. Set to the number of bytes used in 'out' on success. @return -1 if the address is not well-formed, if the port is out of range, or if out is not large enough to hold the result. Otherwise returns 0 on success. */ EVENT2_EXPORT_SYMBOL int evutil_parse_sockaddr_port(const char *str, struct sockaddr *out, int *outlen); /** Compare two sockaddrs; return 0 if they are equal, or less than 0 if sa1 * preceeds sa2, or greater than 0 if sa1 follows sa2. If include_port is * true, consider the port as well as the address. Only implemented for * AF_INET and AF_INET6 addresses. The ordering is not guaranteed to remain * the same between Libevent versions. */ EVENT2_EXPORT_SYMBOL int evutil_sockaddr_cmp(const struct sockaddr *sa1, const struct sockaddr *sa2, int include_port); /** As strcasecmp, but always compares the characters in locale-independent ASCII. That's useful if you're handling data in ASCII-based protocols. */ EVENT2_EXPORT_SYMBOL int evutil_ascii_strcasecmp(const char *str1, const char *str2); /** As strncasecmp, but always compares the characters in locale-independent ASCII. That's useful if you're handling data in ASCII-based protocols. */ EVENT2_EXPORT_SYMBOL int evutil_ascii_strncasecmp(const char *str1, const char *str2, size_t n); /* Here we define evutil_addrinfo to the native addrinfo type, or redefine it * if this system has no getaddrinfo(). */ #ifdef EVENT__HAVE_STRUCT_ADDRINFO #define evutil_addrinfo addrinfo #else /** A definition of struct addrinfo for systems that lack it. (This is just an alias for struct addrinfo if the system defines struct addrinfo.) */ struct evutil_addrinfo { int ai_flags; /* AI_PASSIVE, AI_CANONNAME, AI_NUMERICHOST */ int ai_family; /* PF_xxx */ int ai_socktype; /* SOCK_xxx */ int ai_protocol; /* 0 or IPPROTO_xxx for IPv4 and IPv6 */ size_t ai_addrlen; /* length of ai_addr */ char *ai_canonname; /* canonical name for nodename */ struct sockaddr *ai_addr; /* binary address */ struct evutil_addrinfo *ai_next; /* next structure in linked list */ }; #endif /** @name evutil_getaddrinfo() error codes These values are possible error codes for evutil_getaddrinfo() and related functions. @{ */ #if defined(EAI_ADDRFAMILY) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_ADDRFAMILY EAI_ADDRFAMILY #else #define EVUTIL_EAI_ADDRFAMILY -901 #endif #if defined(EAI_AGAIN) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_AGAIN EAI_AGAIN #else #define EVUTIL_EAI_AGAIN -902 #endif #if defined(EAI_BADFLAGS) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_BADFLAGS EAI_BADFLAGS #else #define EVUTIL_EAI_BADFLAGS -903 #endif #if defined(EAI_FAIL) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_FAIL EAI_FAIL #else #define EVUTIL_EAI_FAIL -904 #endif #if defined(EAI_FAMILY) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_FAMILY EAI_FAMILY #else #define EVUTIL_EAI_FAMILY -905 #endif #if defined(EAI_MEMORY) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_MEMORY EAI_MEMORY #else #define EVUTIL_EAI_MEMORY -906 #endif /* This test is a bit complicated, since some MS SDKs decide to * remove NODATA or redefine it to be the same as NONAME, in a * fun interpretation of RFC 2553 and RFC 3493. */ #if defined(EAI_NODATA) && defined(EVENT__HAVE_GETADDRINFO) && (!defined(EAI_NONAME) || EAI_NODATA != EAI_NONAME) #define EVUTIL_EAI_NODATA EAI_NODATA #else #define EVUTIL_EAI_NODATA -907 #endif #if defined(EAI_NONAME) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_NONAME EAI_NONAME #else #define EVUTIL_EAI_NONAME -908 #endif #if defined(EAI_SERVICE) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_SERVICE EAI_SERVICE #else #define EVUTIL_EAI_SERVICE -909 #endif #if defined(EAI_SOCKTYPE) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_SOCKTYPE EAI_SOCKTYPE #else #define EVUTIL_EAI_SOCKTYPE -910 #endif #if defined(EAI_SYSTEM) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_EAI_SYSTEM EAI_SYSTEM #else #define EVUTIL_EAI_SYSTEM -911 #endif #define EVUTIL_EAI_CANCEL -90001 #if defined(AI_PASSIVE) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_AI_PASSIVE AI_PASSIVE #else #define EVUTIL_AI_PASSIVE 0x1000 #endif #if defined(AI_CANONNAME) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_AI_CANONNAME AI_CANONNAME #else #define EVUTIL_AI_CANONNAME 0x2000 #endif #if defined(AI_NUMERICHOST) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_AI_NUMERICHOST AI_NUMERICHOST #else #define EVUTIL_AI_NUMERICHOST 0x4000 #endif #if defined(AI_NUMERICSERV) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_AI_NUMERICSERV AI_NUMERICSERV #else #define EVUTIL_AI_NUMERICSERV 0x8000 #endif #if defined(AI_V4MAPPED) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_AI_V4MAPPED AI_V4MAPPED #else #define EVUTIL_AI_V4MAPPED 0x10000 #endif #if defined(AI_ALL) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_AI_ALL AI_ALL #else #define EVUTIL_AI_ALL 0x20000 #endif #if defined(AI_ADDRCONFIG) && defined(EVENT__HAVE_GETADDRINFO) #define EVUTIL_AI_ADDRCONFIG AI_ADDRCONFIG #else #define EVUTIL_AI_ADDRCONFIG 0x40000 #endif /**@}*/ struct evutil_addrinfo; /** * This function clones getaddrinfo for systems that don't have it. For full * details, see RFC 3493, section 6.1. * * Limitations: * - When the system has no getaddrinfo, we fall back to gethostbyname_r or * gethostbyname, with their attendant issues. * - The AI_V4MAPPED and AI_ALL flags are not currently implemented. * * For a nonblocking variant, see evdns_getaddrinfo. */ EVENT2_EXPORT_SYMBOL int evutil_getaddrinfo(const char *nodename, const char *servname, const struct evutil_addrinfo *hints_in, struct evutil_addrinfo **res); /** Release storage allocated by evutil_getaddrinfo or evdns_getaddrinfo. */ EVENT2_EXPORT_SYMBOL void evutil_freeaddrinfo(struct evutil_addrinfo *ai); EVENT2_EXPORT_SYMBOL const char *evutil_gai_strerror(int err); /** Generate n bytes of secure pseudorandom data, and store them in buf. * * Current versions of Libevent use an ARC4-based random number generator, * seeded using the platform's entropy source (/dev/urandom on Unix-like * systems; CryptGenRandom on Windows). This is not actually as secure as it * should be: ARC4 is a pretty lousy cipher, and the current implementation * provides only rudimentary prediction- and backtracking-resistance. Don't * use this for serious cryptographic applications. */ EVENT2_EXPORT_SYMBOL void evutil_secure_rng_get_bytes(void *buf, size_t n); /** * Seed the secure random number generator if needed, and return 0 on * success or -1 on failure. * * It is okay to call this function more than once; it will still return * 0 if the RNG has been successfully seeded and -1 if it can't be * seeded. * * Ordinarily you don't need to call this function from your own code; * Libevent will seed the RNG itself the first time it needs good random * numbers. You only need to call it if (a) you want to double-check * that one of the seeding methods did succeed, or (b) you plan to drop * the capability to seed (by chrooting, or dropping capabilities, or * whatever), and you want to make sure that seeding happens before your * program loses the ability to do it. */ EVENT2_EXPORT_SYMBOL int evutil_secure_rng_init(void); /** * Set a filename to use in place of /dev/urandom for seeding the secure * PRNG. Return 0 on success, -1 on failure. * * Call this function BEFORE calling any other initialization or RNG * functions. * * (This string will _NOT_ be copied internally. Do not free it while any * user of the secure RNG might be running. Don't pass anything other than a * real /dev/...random device file here, or you might lose security.) * * This API is unstable, and might change in a future libevent version. */ EVENT2_EXPORT_SYMBOL int evutil_secure_rng_set_urandom_device_file(char *fname); /** Seed the random number generator with extra random bytes. You should almost never need to call this function; it should be sufficient to invoke evutil_secure_rng_init(), or let Libevent take care of calling evutil_secure_rng_init() on its own. If you call this function as a _replacement_ for the regular entropy sources, then you need to be sure that your input contains a fairly large amount of strong entropy. Doing so is notoriously hard: most people who try get it wrong. Watch out! @param dat a buffer full of a strong source of random numbers @param datlen the number of bytes to read from datlen */ EVENT2_EXPORT_SYMBOL void evutil_secure_rng_add_bytes(const char *dat, size_t datlen); #ifdef __cplusplus } #endif #endif /* EVENT1_EVUTIL_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/event2/visibility.h000066400000000000000000000044541527052371200251730ustar00rootroot00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ /* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT2_VISIBILITY_H_INCLUDED_ #define EVENT2_VISIBILITY_H_INCLUDED_ #include #if defined(event_EXPORTS) || defined(event_extra_EXPORTS) || defined(event_core_EXPORTS) # if defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550) # define EVENT2_EXPORT_SYMBOL __global # elif defined __GNUC__ # define EVENT2_EXPORT_SYMBOL __attribute__ ((visibility("default"))) # elif defined(_MSC_VER) # define EVENT2_EXPORT_SYMBOL extern __declspec(dllexport) # else # define EVENT2_EXPORT_SYMBOL /* unknown compiler */ # endif #else # if defined(EVENT__NEED_DLLIMPORT) && defined(_MSC_VER) && !defined(EVENT_BUILDING_REGRESS_TEST) # define EVENT2_EXPORT_SYMBOL extern __declspec(dllimport) # else # define EVENT2_EXPORT_SYMBOL # endif #endif #endif /* EVENT2_VISIBILITY_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/evhttp.h000066400000000000000000000037631527052371200231150ustar00rootroot00000000000000/* * Copyright 2000-2007 Niels Provos * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT1_EVHTTP_H_INCLUDED_ #define EVENT1_EVHTTP_H_INCLUDED_ /** @file evhttp.h An http implementation subsystem for Libevent. The header is deprecated in Libevent 2.0 and later; please use instead. Depending on what functionality you need, you may also want to include more of the other headers. */ #include #include #include #include #endif /* EVENT1_EVHTTP_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/evrpc.h000066400000000000000000000037371527052371200227230ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT1_EVRPC_H_INCLUDED_ #define EVENT1_EVRPC_H_INCLUDED_ /** @file evrpc.h An RPC system for Libevent. The header is deprecated in Libevent 2.0 and later; please use instead. Depending on what functionality you need, you may also want to include more of the other headers. */ #include #include #include #include #endif /* EVENT1_EVRPC_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/evutil.h000066400000000000000000000033661527052371200231120ustar00rootroot00000000000000/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef EVENT1_EVUTIL_H_INCLUDED_ #define EVENT1_EVUTIL_H_INCLUDED_ /** @file evutil.h Utility and compatibility functions for Libevent. The header is deprecated in Libevent 2.0 and later; please use instead. */ #include #endif /* EVENT1_EVUTIL_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/include/include.am000066400000000000000000000026351527052371200233710ustar00rootroot00000000000000# include/Makefile.am for libevent # Copyright 2000-2007 Niels Provos # Copyright 2007-2012 Niels Provos and Nick Mathewson # # See LICENSE for copying information. include_event2dir = $(includedir)/event2 EVENT2_EXPORT = \ include/event2/buffer.h \ include/event2/buffer_compat.h \ include/event2/bufferevent.h \ include/event2/bufferevent_compat.h \ include/event2/bufferevent_ssl.h \ include/event2/bufferevent_struct.h \ include/event2/dns.h \ include/event2/dns_compat.h \ include/event2/dns_struct.h \ include/event2/event.h \ include/event2/event_compat.h \ include/event2/event_struct.h \ include/event2/http.h \ include/event2/http_compat.h \ include/event2/http_struct.h \ include/event2/keyvalq_struct.h \ include/event2/listener.h \ include/event2/rpc.h \ include/event2/rpc_compat.h \ include/event2/rpc_struct.h \ include/event2/tag.h \ include/event2/tag_compat.h \ include/event2/thread.h \ include/event2/util.h \ include/event2/visibility.h ## Without the nobase_ prefixing, Automake would strip "include/event2/" from ## the source header filename to derive the installed header filename. ## With nobase_ the installed path is $(includedir)/include/event2/ev*.h. if INSTALL_LIBEVENT include_event2_HEADERS = $(EVENT2_EXPORT) nodist_include_event2_HEADERS = include/event2/event-config.h else noinst_HEADERS += $(EVENT2_EXPORT) nodist_noinst_HEADERS = include/event2/event-config.h endif kytensor-2.49.0.7/third_party/libevent/iocp-internal.h000066400000000000000000000170231527052371200227160ustar00rootroot00000000000000/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef IOCP_INTERNAL_H_INCLUDED_ #define IOCP_INTERNAL_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif struct event_overlapped; struct event_iocp_port; struct evbuffer; typedef void (*iocp_callback)(struct event_overlapped *, ev_uintptr_t, ev_ssize_t, int success); /* This whole file is actually win32 only. We wrap the structures in a win32 * ifdef so that we can test-compile code that uses these interfaces on * non-win32 platforms. */ #ifdef _WIN32 /** Internal use only. Wraps an OVERLAPPED that we're using for libevent functionality. Whenever an event_iocp_port gets an event for a given OVERLAPPED*, it upcasts the pointer to an event_overlapped, and calls the iocp_callback function with the event_overlapped, the iocp key, and the number of bytes transferred as arguments. */ struct event_overlapped { OVERLAPPED overlapped; iocp_callback cb; }; /* Mingw's headers don't define LPFN_ACCEPTEX. */ typedef BOOL (WINAPI *AcceptExPtr)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); typedef BOOL (WINAPI *ConnectExPtr)(SOCKET, const struct sockaddr *, int, PVOID, DWORD, LPDWORD, LPOVERLAPPED); typedef void (WINAPI *GetAcceptExSockaddrsPtr)(PVOID, DWORD, DWORD, DWORD, LPSOCKADDR *, LPINT, LPSOCKADDR *, LPINT); /** Internal use only. Holds pointers to functions that only some versions of Windows provide. */ struct win32_extension_fns { AcceptExPtr AcceptEx; ConnectExPtr ConnectEx; GetAcceptExSockaddrsPtr GetAcceptExSockaddrs; }; /** Internal use only. Stores a Windows IO Completion port, along with related data. */ struct event_iocp_port { /** The port itself */ HANDLE port; /* A lock to cover internal structures. */ CRITICAL_SECTION lock; /** Number of threads ever open on the port. */ short n_threads; /** True iff we're shutting down all the threads on this port */ short shutdown; /** How often the threads on this port check for shutdown and other * conditions */ long ms; /* The threads that are waiting for events. */ HANDLE *threads; /** Number of threads currently open on this port. */ short n_live_threads; /** A semaphore to signal when we are done shutting down. */ HANDLE *shutdownSemaphore; }; const struct win32_extension_fns *event_get_win32_extension_fns_(void); #else /* Dummy definition so we can test-compile more things on unix. */ struct event_overlapped { iocp_callback cb; }; #endif /** Initialize the fields in an event_overlapped. @param overlapped The struct event_overlapped to initialize @param cb The callback that should be invoked once the IO operation has finished. */ void event_overlapped_init_(struct event_overlapped *, iocp_callback cb); /** Allocate and return a new evbuffer that supports overlapped IO on a given socket. The socket must be associated with an IO completion port using event_iocp_port_associate_. */ struct evbuffer *evbuffer_overlapped_new_(evutil_socket_t fd); /** XXXX Document (nickm) */ evutil_socket_t evbuffer_overlapped_get_fd_(struct evbuffer *buf); void evbuffer_overlapped_set_fd_(struct evbuffer *buf, evutil_socket_t fd); /** Start reading data onto the end of an overlapped evbuffer. An evbuffer can only have one read pending at a time. While the read is in progress, no other data may be added to the end of the buffer. The buffer must be created with event_overlapped_init_(). evbuffer_commit_read_() must be called in the completion callback. @param buf The buffer to read onto @param n The number of bytes to try to read. @param ol Overlapped object with associated completion callback. @return 0 on success, -1 on error. */ int evbuffer_launch_read_(struct evbuffer *buf, size_t n, struct event_overlapped *ol); /** Start writing data from the start of an evbuffer. An evbuffer can only have one write pending at a time. While the write is in progress, no other data may be removed from the front of the buffer. The buffer must be created with event_overlapped_init_(). evbuffer_commit_write_() must be called in the completion callback. @param buf The buffer to read onto @param n The number of bytes to try to read. @param ol Overlapped object with associated completion callback. @return 0 on success, -1 on error. */ int evbuffer_launch_write_(struct evbuffer *buf, ev_ssize_t n, struct event_overlapped *ol); /** XXX document */ void evbuffer_commit_read_(struct evbuffer *, ev_ssize_t); void evbuffer_commit_write_(struct evbuffer *, ev_ssize_t); /** Create an IOCP, and launch its worker threads. Internal use only. This interface is unstable, and will change. */ struct event_iocp_port *event_iocp_port_launch_(int n_cpus); /** Associate a file descriptor with an iocp, such that overlapped IO on the fd will happen on one of the iocp's worker threads. */ int event_iocp_port_associate_(struct event_iocp_port *port, evutil_socket_t fd, ev_uintptr_t key); /** Tell all threads serving an iocp to stop. Wait for up to waitMsec for all the threads to finish whatever they're doing. If waitMsec is -1, wait as long as required. If all the threads are done, free the port and return 0. Otherwise, return -1. If you get a -1 return value, it is safe to call this function again. */ int event_iocp_shutdown_(struct event_iocp_port *port, long waitMsec); /* FIXME document. */ int event_iocp_activate_overlapped_(struct event_iocp_port *port, struct event_overlapped *o, ev_uintptr_t key, ev_uint32_t n_bytes); struct event_base; /* FIXME document. */ struct event_iocp_port *event_base_get_iocp_(struct event_base *base); /* FIXME document. */ int event_base_start_iocp_(struct event_base *base, int n_cpus); void event_base_stop_iocp_(struct event_base *base); /* FIXME document. */ struct bufferevent *bufferevent_async_new_(struct event_base *base, evutil_socket_t fd, int options); /* FIXME document. */ void bufferevent_async_set_connected_(struct bufferevent *bev); int bufferevent_async_can_connect_(struct bufferevent *bev); int bufferevent_async_connect_(struct bufferevent *bev, evutil_socket_t fd, const struct sockaddr *sa, int socklen); #ifdef __cplusplus } #endif #endif kytensor-2.49.0.7/third_party/libevent/ipv6-internal.h000066400000000000000000000047221527052371200226520ustar00rootroot00000000000000/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Internal use only: Fake IPv6 structures and values on platforms that * do not have them */ #ifndef IPV6_INTERNAL_H_INCLUDED_ #define IPV6_INTERNAL_H_INCLUDED_ #include "event2/event-config.h" #include "evconfig-private.h" #include #ifdef EVENT__HAVE_SYS_SOCKET_H #include #endif #include "event2/util.h" #ifdef __cplusplus extern "C" { #endif /** @file ipv6-internal.h * * Replacement types and functions for platforms that don't support ipv6 * properly. */ #ifndef EVENT__HAVE_STRUCT_IN6_ADDR struct in6_addr { ev_uint8_t s6_addr[16]; }; #endif #ifndef EVENT__HAVE_SA_FAMILY_T typedef int sa_family_t; #endif #ifndef EVENT__HAVE_STRUCT_SOCKADDR_IN6 struct sockaddr_in6 { /* This will fail if we find a struct sockaddr that doesn't have * sa_family as the first element. */ sa_family_t sin6_family; ev_uint16_t sin6_port; struct in6_addr sin6_addr; }; #endif #ifndef AF_INET6 #define AF_INET6 3333 #endif #ifndef PF_INET6 #define PF_INET6 AF_INET6 #endif #ifdef __cplusplus } #endif #endif kytensor-2.49.0.7/third_party/libevent/kqueue-internal.h000066400000000000000000000037061527052371200232660ustar00rootroot00000000000000/* * Copyright (c) 2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef KQUEUE_INTERNAL_H_INCLUDED_ #define KQUEUE_INTERNAL_H_INCLUDED_ /** Notification function, used to tell an event base to wake up from another * thread. Only works when event_kq_add_notify_event_() has previously been * called successfully on that base. */ int event_kq_notify_base_(struct event_base *base); /** Prepare a kqueue-using event base to receive notifications via an internal * EVFILT_USER event. Return 0 on sucess, -1 on failure. */ int event_kq_add_notify_event_(struct event_base *base); #endif kytensor-2.49.0.7/third_party/libevent/kqueue.c000066400000000000000000000345561527052371200214560ustar00rootroot00000000000000/* $OpenBSD: kqueue.c,v 1.5 2002/07/10 14:41:31 art Exp $ */ /* * Copyright 2000-2007 Niels Provos * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_KQUEUE #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #include #include #include #include #include #include #include #ifdef EVENT__HAVE_INTTYPES_H #include #endif /* Some platforms apparently define the udata field of struct kevent as * intptr_t, whereas others define it as void*. There doesn't seem to be an * easy way to tell them apart via autoconf, so we need to use OS macros. */ #if defined(EVENT__HAVE_INTTYPES_H) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__darwin__) && !defined(__APPLE__) && !defined(__CloudABI__) #define PTR_TO_UDATA(x) ((intptr_t)(x)) #define INT_TO_UDATA(x) ((intptr_t)(x)) #else #define PTR_TO_UDATA(x) (x) #define INT_TO_UDATA(x) ((void*)(x)) #endif #include "event-internal.h" #include "log-internal.h" #include "evmap-internal.h" #include "event2/thread.h" #include "evthread-internal.h" #include "changelist-internal.h" #include "kqueue-internal.h" #define NEVENT 64 struct kqop { struct kevent *changes; int changes_size; struct kevent *events; int events_size; int kq; int notify_event_added; pid_t pid; }; static void kqop_free(struct kqop *kqop); static void *kq_init(struct event_base *); static int kq_sig_add(struct event_base *, int, short, short, void *); static int kq_sig_del(struct event_base *, int, short, short, void *); static int kq_dispatch(struct event_base *, struct timeval *); static void kq_dealloc(struct event_base *); const struct eventop kqops = { "kqueue", kq_init, event_changelist_add_, event_changelist_del_, kq_dispatch, kq_dealloc, 1 /* need reinit */, EV_FEATURE_ET|EV_FEATURE_O1|EV_FEATURE_FDS, EVENT_CHANGELIST_FDINFO_SIZE }; static const struct eventop kqsigops = { "kqueue_signal", NULL, kq_sig_add, kq_sig_del, NULL, NULL, 1 /* need reinit */, 0, 0 }; static void * kq_init(struct event_base *base) { int kq = -1; struct kqop *kqueueop = NULL; if (!(kqueueop = mm_calloc(1, sizeof(struct kqop)))) return (NULL); /* Initialize the kernel queue */ if ((kq = kqueue()) == -1) { event_warn("kqueue"); goto err; } kqueueop->kq = kq; kqueueop->pid = getpid(); /* Initialize fields */ kqueueop->changes = mm_calloc(NEVENT, sizeof(struct kevent)); if (kqueueop->changes == NULL) goto err; kqueueop->events = mm_calloc(NEVENT, sizeof(struct kevent)); if (kqueueop->events == NULL) goto err; kqueueop->events_size = kqueueop->changes_size = NEVENT; /* Check for Mac OS X kqueue bug. */ memset(&kqueueop->changes[0], 0, sizeof kqueueop->changes[0]); kqueueop->changes[0].ident = -1; kqueueop->changes[0].filter = EVFILT_READ; kqueueop->changes[0].flags = EV_ADD; /* * If kqueue works, then kevent will succeed, and it will * stick an error in events[0]. If kqueue is broken, then * kevent will fail. */ if (kevent(kq, kqueueop->changes, 1, kqueueop->events, NEVENT, NULL) != 1 || (int)kqueueop->events[0].ident != -1 || !(kqueueop->events[0].flags & EV_ERROR)) { event_warn("%s: detected broken kqueue; not using.", __func__); goto err; } base->evsigsel = &kqsigops; return (kqueueop); err: if (kqueueop) kqop_free(kqueueop); return (NULL); } #define ADD_UDATA 0x30303 static void kq_setup_kevent(struct kevent *out, evutil_socket_t fd, int filter, short change) { memset(out, 0, sizeof(struct kevent)); out->ident = fd; out->filter = filter; if (change & EV_CHANGE_ADD) { out->flags = EV_ADD; /* We set a magic number here so that we can tell 'add' * errors from 'del' errors. */ out->udata = INT_TO_UDATA(ADD_UDATA); if (change & EV_ET) out->flags |= EV_CLEAR; #ifdef NOTE_EOF /* Make it behave like select() and poll() */ if (filter == EVFILT_READ) out->fflags = NOTE_EOF; #endif } else { EVUTIL_ASSERT(change & EV_CHANGE_DEL); out->flags = EV_DELETE; } } static int kq_build_changes_list(const struct event_changelist *changelist, struct kqop *kqop) { int i; int n_changes = 0; for (i = 0; i < changelist->n_changes; ++i) { struct event_change *in_ch = &changelist->changes[i]; struct kevent *out_ch; if (n_changes >= kqop->changes_size - 1) { int newsize = kqop->changes_size * 2; struct kevent *newchanges; newchanges = mm_realloc(kqop->changes, newsize * sizeof(struct kevent)); if (newchanges == NULL) { event_warn("%s: realloc", __func__); return (-1); } kqop->changes = newchanges; kqop->changes_size = newsize; } if (in_ch->read_change) { out_ch = &kqop->changes[n_changes++]; kq_setup_kevent(out_ch, in_ch->fd, EVFILT_READ, in_ch->read_change); } if (in_ch->write_change) { out_ch = &kqop->changes[n_changes++]; kq_setup_kevent(out_ch, in_ch->fd, EVFILT_WRITE, in_ch->write_change); } } return n_changes; } static int kq_grow_events(struct kqop *kqop, size_t new_size) { struct kevent *newresult; newresult = mm_realloc(kqop->events, new_size * sizeof(struct kevent)); if (newresult) { kqop->events = newresult; kqop->events_size = new_size; return 0; } else { return -1; } } static int kq_dispatch(struct event_base *base, struct timeval *tv) { struct kqop *kqop = base->evbase; struct kevent *events = kqop->events; struct kevent *changes; struct timespec ts, *ts_p = NULL; int i, n_changes, res; if (tv != NULL) { ts.tv_sec = tv->tv_sec; ts.tv_nsec = tv->tv_usec * 1000; ts_p = &ts; } /* Build "changes" from "base->changes" */ EVUTIL_ASSERT(kqop->changes); n_changes = kq_build_changes_list(&base->changelist, kqop); if (n_changes < 0) return -1; event_changelist_remove_all_(&base->changelist, base); /* steal the changes array in case some broken code tries to call * dispatch twice at once. */ changes = kqop->changes; kqop->changes = NULL; /* Make sure that 'events' is at least as long as the list of changes: * otherwise errors in the changes can get reported as a -1 return * value from kevent() rather than as EV_ERROR events in the events * array. * * (We could instead handle -1 return values from kevent() by * retrying with a smaller changes array or a larger events array, * but this approach seems less risky for now.) */ if (kqop->events_size < n_changes) { int new_size = kqop->events_size; do { new_size *= 2; } while (new_size < n_changes); kq_grow_events(kqop, new_size); events = kqop->events; } EVBASE_RELEASE_LOCK(base, th_base_lock); res = kevent(kqop->kq, changes, n_changes, events, kqop->events_size, ts_p); EVBASE_ACQUIRE_LOCK(base, th_base_lock); EVUTIL_ASSERT(kqop->changes == NULL); kqop->changes = changes; if (res == -1) { if (errno != EINTR) { event_warn("kevent"); return (-1); } return (0); } event_debug(("%s: kevent reports %d", __func__, res)); for (i = 0; i < res; i++) { int which = 0; if (events[i].flags & EV_ERROR) { switch (events[i].data) { /* Can occur on delete if we are not currently * watching any events on this fd. That can * happen when the fd was closed and another * file was opened with that fd. */ case ENOENT: /* Can occur for reasons not fully understood * on FreeBSD. */ case EINVAL: continue; #if defined(__FreeBSD__) /* * This currently occurs if an FD is closed * before the EV_DELETE makes it out via kevent(). * The FreeBSD capabilities code sees the blank * capability set and rejects the request to * modify an event. * * To be strictly correct - when an FD is closed, * all the registered events are also removed. * Queuing EV_DELETE to a closed FD is wrong. * The event(s) should just be deleted from * the pending changelist. */ case ENOTCAPABLE: continue; #endif /* Can occur on a delete if the fd is closed. */ case EBADF: /* XXXX On NetBSD, we can also get EBADF if we * try to add the write side of a pipe, but * the read side has already been closed. * Other BSDs call this situation 'EPIPE'. It * would be good if we had a way to report * this situation. */ continue; /* These two can occur on an add if the fd was one side * of a pipe, and the other side was closed. */ case EPERM: case EPIPE: /* Report read events, if we're listening for * them, so that the user can learn about any * add errors. (If the operation was a * delete, then udata should be cleared.) */ if (events[i].udata) { /* The operation was an add: * report the error as a read. */ which |= EV_READ; break; } else { /* The operation was a del: * report nothing. */ continue; } /* Other errors shouldn't occur. */ default: errno = events[i].data; return (-1); } } else if (events[i].filter == EVFILT_READ) { which |= EV_READ; } else if (events[i].filter == EVFILT_WRITE) { which |= EV_WRITE; } else if (events[i].filter == EVFILT_SIGNAL) { which |= EV_SIGNAL; #ifdef EVFILT_USER } else if (events[i].filter == EVFILT_USER) { base->is_notify_pending = 0; #endif } if (!which) continue; if (events[i].filter == EVFILT_SIGNAL) { evmap_signal_active_(base, events[i].ident, 1); } else { evmap_io_active_(base, events[i].ident, which | EV_ET); } } if (res == kqop->events_size) { /* We used all the events space that we have. Maybe we should make it bigger. */ kq_grow_events(kqop, kqop->events_size * 2); } return (0); } static void kqop_free(struct kqop *kqop) { if (kqop->changes) mm_free(kqop->changes); if (kqop->events) mm_free(kqop->events); if (kqop->kq >= 0 && kqop->pid == getpid()) close(kqop->kq); memset(kqop, 0, sizeof(struct kqop)); mm_free(kqop); } static void kq_dealloc(struct event_base *base) { struct kqop *kqop = base->evbase; evsig_dealloc_(base); kqop_free(kqop); } /* signal handling */ static int kq_sig_add(struct event_base *base, int nsignal, short old, short events, void *p) { struct kqop *kqop = base->evbase; struct kevent kev; struct timespec timeout = { 0, 0 }; (void)p; EVUTIL_ASSERT(nsignal >= 0 && nsignal < NSIG); memset(&kev, 0, sizeof(kev)); kev.ident = nsignal; kev.filter = EVFILT_SIGNAL; kev.flags = EV_ADD; /* Be ready for the signal if it is sent any * time between now and the next call to * kq_dispatch. */ if (kevent(kqop->kq, &kev, 1, NULL, 0, &timeout) == -1) return (-1); /* We can set the handler for most signals to SIG_IGN and * still have them reported to us in the queue. However, * if the handler for SIGCHLD is SIG_IGN, the system reaps * zombie processes for us, and we don't get any notification. * This appears to be the only signal with this quirk. */ if (evsig_set_handler_(base, nsignal, nsignal == SIGCHLD ? SIG_DFL : SIG_IGN) == -1) return (-1); return (0); } static int kq_sig_del(struct event_base *base, int nsignal, short old, short events, void *p) { struct kqop *kqop = base->evbase; struct kevent kev; struct timespec timeout = { 0, 0 }; (void)p; EVUTIL_ASSERT(nsignal >= 0 && nsignal < NSIG); memset(&kev, 0, sizeof(kev)); kev.ident = nsignal; kev.filter = EVFILT_SIGNAL; kev.flags = EV_DELETE; /* Because we insert signal events * immediately, we need to delete them * immediately, too */ if (kevent(kqop->kq, &kev, 1, NULL, 0, &timeout) == -1) return (-1); if (evsig_restore_handler_(base, nsignal) == -1) return (-1); return (0); } /* OSX 10.6 and FreeBSD 8.1 add support for EVFILT_USER, which we can use * to wake up the event loop from another thread. */ /* Magic number we use for our filter ID. */ #define NOTIFY_IDENT 42 int event_kq_add_notify_event_(struct event_base *base) { struct kqop *kqop = base->evbase; #if defined(EVFILT_USER) && defined(NOTE_TRIGGER) struct kevent kev; struct timespec timeout = { 0, 0 }; #endif if (kqop->notify_event_added) return 0; #if defined(EVFILT_USER) && defined(NOTE_TRIGGER) memset(&kev, 0, sizeof(kev)); kev.ident = NOTIFY_IDENT; kev.filter = EVFILT_USER; kev.flags = EV_ADD | EV_CLEAR; if (kevent(kqop->kq, &kev, 1, NULL, 0, &timeout) == -1) { event_warn("kevent: adding EVFILT_USER event"); return -1; } kqop->notify_event_added = 1; return 0; #else return -1; #endif } int event_kq_notify_base_(struct event_base *base) { struct kqop *kqop = base->evbase; #if defined(EVFILT_USER) && defined(NOTE_TRIGGER) struct kevent kev; struct timespec timeout = { 0, 0 }; #endif if (! kqop->notify_event_added) return -1; #if defined(EVFILT_USER) && defined(NOTE_TRIGGER) memset(&kev, 0, sizeof(kev)); kev.ident = NOTIFY_IDENT; kev.filter = EVFILT_USER; kev.fflags = NOTE_TRIGGER; if (kevent(kqop->kq, &kev, 1, NULL, 0, &timeout) == -1) { event_warn("kevent: triggering EVFILT_USER event"); return -1; } return 0; #else return -1; #endif } #endif /* EVENT__HAVE_KQUEUE */ kytensor-2.49.0.7/third_party/libevent/libevent.pc.in000066400000000000000000000004751527052371200225450ustar00rootroot00000000000000#libevent pkg-config source file prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ Name: libevent Description: libevent is an asynchronous notification event loop library Version: @VERSION@ Requires: Conflicts: Libs: -L${libdir} -levent Libs.private: @LIBS@ Cflags: -I${includedir} kytensor-2.49.0.7/third_party/libevent/libevent_core.pc.in000066400000000000000000000004311527052371200235450ustar00rootroot00000000000000#libevent pkg-config source file prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ Name: libevent_core Description: libevent_core Version: @VERSION@ Requires: Conflicts: Libs: -L${libdir} -levent_core Libs.private: @LIBS@ Cflags: -I${includedir} kytensor-2.49.0.7/third_party/libevent/libevent_extra.pc.in000066400000000000000000000004341527052371200237430ustar00rootroot00000000000000#libevent pkg-config source file prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ Name: libevent_extra Description: libevent_extra Version: @VERSION@ Requires: Conflicts: Libs: -L${libdir} -levent_extra Libs.private: @LIBS@ Cflags: -I${includedir} kytensor-2.49.0.7/third_party/libevent/libevent_openssl.pc.in000066400000000000000000000005641527052371200243070ustar00rootroot00000000000000#libevent pkg-config source file prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ Name: libevent_openssl Description: libevent_openssl adds openssl-based TLS support to libevent Version: @VERSION@ Requires: libevent Conflicts: Libs: -L${libdir} -levent_openssl Libs.private: @LIBS@ @OPENSSL_LIBS@ Cflags: -I${includedir} @OPENSSL_INCS@ kytensor-2.49.0.7/third_party/libevent/libevent_pthreads.pc.in000066400000000000000000000006001527052371200244250ustar00rootroot00000000000000#libevent pkg-config source file prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ Name: libevent_pthreads Description: libevent_pthreads adds pthreads-based threading support to libevent Version: @VERSION@ Requires: libevent Conflicts: Libs: -L${libdir} -levent_pthreads Libs.private: @LIBS@ @PTHREAD_LIBS@ Cflags: -I${includedir} @PTHREAD_CFLAGS@ kytensor-2.49.0.7/third_party/libevent/listener.c000066400000000000000000000515211527052371200217730ustar00rootroot00000000000000/* * Copyright (c) 2009-2012 Niels Provos, Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #include #ifdef _WIN32 #ifndef _WIN32_WINNT /* Minimum required for InitializeCriticalSectionAndSpinCount */ #define _WIN32_WINNT 0x0403 #endif #include #include #include #endif #include #ifdef EVENT__HAVE_SYS_SOCKET_H #include #endif #ifdef EVENT__HAVE_FCNTL_H #include #endif #ifdef EVENT__HAVE_UNISTD_H #include #endif #include "event2/listener.h" #include "event2/util.h" #include "event2/event.h" #include "event2/event_struct.h" #include "mm-internal.h" #include "util-internal.h" #include "log-internal.h" #include "evthread-internal.h" #ifdef _WIN32 #include "iocp-internal.h" #include "defer-internal.h" #include "event-internal.h" #endif struct evconnlistener_ops { int (*enable)(struct evconnlistener *); int (*disable)(struct evconnlistener *); void (*destroy)(struct evconnlistener *); void (*shutdown)(struct evconnlistener *); evutil_socket_t (*getfd)(struct evconnlistener *); struct event_base *(*getbase)(struct evconnlistener *); }; struct evconnlistener { const struct evconnlistener_ops *ops; void *lock; evconnlistener_cb cb; evconnlistener_errorcb errorcb; void *user_data; unsigned flags; short refcnt; int accept4_flags; unsigned enabled : 1; }; struct evconnlistener_event { struct evconnlistener base; struct event listener; }; #ifdef _WIN32 struct evconnlistener_iocp { struct evconnlistener base; evutil_socket_t fd; struct event_base *event_base; struct event_iocp_port *port; short n_accepting; unsigned shutting_down : 1; unsigned event_added : 1; struct accepting_socket **accepting; }; #endif #define LOCK(listener) EVLOCK_LOCK((listener)->lock, 0) #define UNLOCK(listener) EVLOCK_UNLOCK((listener)->lock, 0) struct evconnlistener * evconnlistener_new_async(struct event_base *base, evconnlistener_cb cb, void *ptr, unsigned flags, int backlog, evutil_socket_t fd); /* XXXX export this? */ static int event_listener_enable(struct evconnlistener *); static int event_listener_disable(struct evconnlistener *); static void event_listener_destroy(struct evconnlistener *); static evutil_socket_t event_listener_getfd(struct evconnlistener *); static struct event_base *event_listener_getbase(struct evconnlistener *); #if 0 static void listener_incref_and_lock(struct evconnlistener *listener) { LOCK(listener); ++listener->refcnt; } #endif static int listener_decref_and_unlock(struct evconnlistener *listener) { int refcnt = --listener->refcnt; if (refcnt == 0) { listener->ops->destroy(listener); UNLOCK(listener); EVTHREAD_FREE_LOCK(listener->lock, EVTHREAD_LOCKTYPE_RECURSIVE); mm_free(listener); return 1; } else { UNLOCK(listener); return 0; } } static const struct evconnlistener_ops evconnlistener_event_ops = { event_listener_enable, event_listener_disable, event_listener_destroy, NULL, /* shutdown */ event_listener_getfd, event_listener_getbase }; static void listener_read_cb(evutil_socket_t, short, void *); struct evconnlistener * evconnlistener_new(struct event_base *base, evconnlistener_cb cb, void *ptr, unsigned flags, int backlog, evutil_socket_t fd) { struct evconnlistener_event *lev; #ifdef _WIN32 if (base && event_base_get_iocp_(base)) { const struct win32_extension_fns *ext = event_get_win32_extension_fns_(); if (ext->AcceptEx && ext->GetAcceptExSockaddrs) return evconnlistener_new_async(base, cb, ptr, flags, backlog, fd); } #endif if (backlog > 0) { if (listen(fd, backlog) < 0) return NULL; } else if (backlog < 0) { if (listen(fd, 128) < 0) return NULL; } lev = mm_calloc(1, sizeof(struct evconnlistener_event)); if (!lev) return NULL; lev->base.ops = &evconnlistener_event_ops; lev->base.cb = cb; lev->base.user_data = ptr; lev->base.flags = flags; lev->base.refcnt = 1; lev->base.accept4_flags = 0; if (!(flags & LEV_OPT_LEAVE_SOCKETS_BLOCKING)) lev->base.accept4_flags |= EVUTIL_SOCK_NONBLOCK; if (flags & LEV_OPT_CLOSE_ON_EXEC) lev->base.accept4_flags |= EVUTIL_SOCK_CLOEXEC; if (flags & LEV_OPT_THREADSAFE) { EVTHREAD_ALLOC_LOCK(lev->base.lock, EVTHREAD_LOCKTYPE_RECURSIVE); } event_assign(&lev->listener, base, fd, EV_READ|EV_PERSIST, listener_read_cb, lev); if (!(flags & LEV_OPT_DISABLED)) evconnlistener_enable(&lev->base); return &lev->base; } struct evconnlistener * evconnlistener_new_bind(struct event_base *base, evconnlistener_cb cb, void *ptr, unsigned flags, int backlog, const struct sockaddr *sa, int socklen) { struct evconnlistener *listener; evutil_socket_t fd; int on = 1; int family = sa ? sa->sa_family : AF_UNSPEC; int socktype = SOCK_STREAM | EVUTIL_SOCK_NONBLOCK; if (backlog == 0) return NULL; if (flags & LEV_OPT_CLOSE_ON_EXEC) socktype |= EVUTIL_SOCK_CLOEXEC; fd = evutil_socket_(family, socktype, 0); if (fd == -1) return NULL; if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void*)&on, sizeof(on))<0) goto err; if (flags & LEV_OPT_REUSEABLE) { if (evutil_make_listen_socket_reuseable(fd) < 0) goto err; } if (flags & LEV_OPT_REUSEABLE_PORT) { if (evutil_make_listen_socket_reuseable_port(fd) < 0) goto err; } if (flags & LEV_OPT_DEFERRED_ACCEPT) { if (evutil_make_tcp_listen_socket_deferred(fd) < 0) goto err; } if (sa) { if (bind(fd, sa, socklen)<0) goto err; } listener = evconnlistener_new(base, cb, ptr, flags, backlog, fd); if (!listener) goto err; return listener; err: evutil_closesocket(fd); return NULL; } void evconnlistener_free(struct evconnlistener *lev) { LOCK(lev); lev->cb = NULL; lev->errorcb = NULL; if (lev->ops->shutdown) lev->ops->shutdown(lev); listener_decref_and_unlock(lev); } static void event_listener_destroy(struct evconnlistener *lev) { struct evconnlistener_event *lev_e = EVUTIL_UPCAST(lev, struct evconnlistener_event, base); event_del(&lev_e->listener); if (lev->flags & LEV_OPT_CLOSE_ON_FREE) evutil_closesocket(event_get_fd(&lev_e->listener)); event_debug_unassign(&lev_e->listener); } int evconnlistener_enable(struct evconnlistener *lev) { int r; LOCK(lev); lev->enabled = 1; if (lev->cb) r = lev->ops->enable(lev); else r = 0; UNLOCK(lev); return r; } int evconnlistener_disable(struct evconnlistener *lev) { int r; LOCK(lev); lev->enabled = 0; r = lev->ops->disable(lev); UNLOCK(lev); return r; } static int event_listener_enable(struct evconnlistener *lev) { struct evconnlistener_event *lev_e = EVUTIL_UPCAST(lev, struct evconnlistener_event, base); return event_add(&lev_e->listener, NULL); } static int event_listener_disable(struct evconnlistener *lev) { struct evconnlistener_event *lev_e = EVUTIL_UPCAST(lev, struct evconnlistener_event, base); return event_del(&lev_e->listener); } evutil_socket_t evconnlistener_get_fd(struct evconnlistener *lev) { evutil_socket_t fd; LOCK(lev); fd = lev->ops->getfd(lev); UNLOCK(lev); return fd; } static evutil_socket_t event_listener_getfd(struct evconnlistener *lev) { struct evconnlistener_event *lev_e = EVUTIL_UPCAST(lev, struct evconnlistener_event, base); return event_get_fd(&lev_e->listener); } struct event_base * evconnlistener_get_base(struct evconnlistener *lev) { struct event_base *base; LOCK(lev); base = lev->ops->getbase(lev); UNLOCK(lev); return base; } static struct event_base * event_listener_getbase(struct evconnlistener *lev) { struct evconnlistener_event *lev_e = EVUTIL_UPCAST(lev, struct evconnlistener_event, base); return event_get_base(&lev_e->listener); } void evconnlistener_set_cb(struct evconnlistener *lev, evconnlistener_cb cb, void *arg) { int enable = 0; LOCK(lev); if (lev->enabled && !lev->cb) enable = 1; lev->cb = cb; lev->user_data = arg; if (enable) evconnlistener_enable(lev); UNLOCK(lev); } void evconnlistener_set_error_cb(struct evconnlistener *lev, evconnlistener_errorcb errorcb) { LOCK(lev); lev->errorcb = errorcb; UNLOCK(lev); } static void listener_read_cb(evutil_socket_t fd, short what, void *p) { struct evconnlistener *lev = p; int err; evconnlistener_cb cb; evconnlistener_errorcb errorcb; void *user_data; LOCK(lev); while (1) { struct sockaddr_storage ss; ev_socklen_t socklen = sizeof(ss); evutil_socket_t new_fd = evutil_accept4_(fd, (struct sockaddr*)&ss, &socklen, lev->accept4_flags); if (new_fd < 0) break; if (socklen == 0) { /* This can happen with some older linux kernels in * response to nmap. */ evutil_closesocket(new_fd); continue; } if (lev->cb == NULL) { evutil_closesocket(new_fd); UNLOCK(lev); return; } ++lev->refcnt; cb = lev->cb; user_data = lev->user_data; UNLOCK(lev); cb(lev, new_fd, (struct sockaddr*)&ss, (int)socklen, user_data); LOCK(lev); if (lev->refcnt == 1) { int freed = listener_decref_and_unlock(lev); EVUTIL_ASSERT(freed); evutil_closesocket(new_fd); return; } --lev->refcnt; } err = evutil_socket_geterror(fd); if (EVUTIL_ERR_ACCEPT_RETRIABLE(err)) { UNLOCK(lev); return; } if (lev->errorcb != NULL) { ++lev->refcnt; errorcb = lev->errorcb; user_data = lev->user_data; UNLOCK(lev); errorcb(lev, user_data); LOCK(lev); listener_decref_and_unlock(lev); } else { event_sock_warn(fd, "Error from accept() call"); UNLOCK(lev); } } #ifdef _WIN32 struct accepting_socket { CRITICAL_SECTION lock; struct event_overlapped overlapped; SOCKET s; int error; struct event_callback deferred; struct evconnlistener_iocp *lev; ev_uint8_t buflen; ev_uint8_t family; unsigned free_on_cb:1; char addrbuf[1]; }; static void accepted_socket_cb(struct event_overlapped *o, ev_uintptr_t key, ev_ssize_t n, int ok); static void accepted_socket_invoke_user_cb(struct event_callback *cb, void *arg); static void iocp_listener_event_add(struct evconnlistener_iocp *lev) { if (lev->event_added) return; lev->event_added = 1; event_base_add_virtual_(lev->event_base); } static void iocp_listener_event_del(struct evconnlistener_iocp *lev) { if (!lev->event_added) return; lev->event_added = 0; event_base_del_virtual_(lev->event_base); } static struct accepting_socket * new_accepting_socket(struct evconnlistener_iocp *lev, int family) { struct accepting_socket *res; int addrlen; int buflen; if (family == AF_INET) addrlen = sizeof(struct sockaddr_in); else if (family == AF_INET6) addrlen = sizeof(struct sockaddr_in6); else return NULL; buflen = (addrlen+16)*2; res = mm_calloc(1,sizeof(struct accepting_socket)-1+buflen); if (!res) return NULL; event_overlapped_init_(&res->overlapped, accepted_socket_cb); res->s = INVALID_SOCKET; res->lev = lev; res->buflen = buflen; res->family = family; event_deferred_cb_init_(&res->deferred, event_base_get_npriorities(lev->event_base) / 2, accepted_socket_invoke_user_cb, res); InitializeCriticalSectionAndSpinCount(&res->lock, 1000); return res; } static void free_and_unlock_accepting_socket(struct accepting_socket *as) { /* requires lock. */ if (as->s != INVALID_SOCKET) closesocket(as->s); LeaveCriticalSection(&as->lock); DeleteCriticalSection(&as->lock); mm_free(as); } static int start_accepting(struct accepting_socket *as) { /* requires lock */ const struct win32_extension_fns *ext = event_get_win32_extension_fns_(); DWORD pending = 0; SOCKET s = socket(as->family, SOCK_STREAM, 0); int error = 0; if (!as->lev->base.enabled) return 0; if (s == INVALID_SOCKET) { error = WSAGetLastError(); goto report_err; } /* XXXX It turns out we need to do this again later. Does this call * have any effect? */ setsockopt(s, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, (char *)&as->lev->fd, sizeof(&as->lev->fd)); if (!(as->lev->base.flags & LEV_OPT_LEAVE_SOCKETS_BLOCKING)) evutil_make_socket_nonblocking(s); if (event_iocp_port_associate_(as->lev->port, s, 1) < 0) { closesocket(s); return -1; } as->s = s; if (ext->AcceptEx(as->lev->fd, s, as->addrbuf, 0, as->buflen/2, as->buflen/2, &pending, &as->overlapped.overlapped)) { /* Immediate success! */ accepted_socket_cb(&as->overlapped, 1, 0, 1); } else { error = WSAGetLastError(); if (error != ERROR_IO_PENDING) { goto report_err; } } return 0; report_err: as->error = error; event_deferred_cb_schedule_( as->lev->event_base, &as->deferred); return 0; } static void stop_accepting(struct accepting_socket *as) { /* requires lock. */ SOCKET s = as->s; as->s = INVALID_SOCKET; closesocket(s); } static void accepted_socket_invoke_user_cb(struct event_callback *dcb, void *arg) { struct accepting_socket *as = arg; struct sockaddr *sa_local=NULL, *sa_remote=NULL; int socklen_local=0, socklen_remote=0; const struct win32_extension_fns *ext = event_get_win32_extension_fns_(); struct evconnlistener *lev = &as->lev->base; evutil_socket_t sock=-1; void *data; evconnlistener_cb cb=NULL; evconnlistener_errorcb errorcb=NULL; int error; EVUTIL_ASSERT(ext->GetAcceptExSockaddrs); LOCK(lev); EnterCriticalSection(&as->lock); if (as->free_on_cb) { free_and_unlock_accepting_socket(as); listener_decref_and_unlock(lev); return; } ++lev->refcnt; error = as->error; if (error) { as->error = 0; errorcb = lev->errorcb; } else { ext->GetAcceptExSockaddrs( as->addrbuf, 0, as->buflen/2, as->buflen/2, &sa_local, &socklen_local, &sa_remote, &socklen_remote); sock = as->s; cb = lev->cb; as->s = INVALID_SOCKET; /* We need to call this so getsockname, getpeername, and * shutdown work correctly on the accepted socket. */ /* XXXX handle error? */ setsockopt(sock, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, (char *)&as->lev->fd, sizeof(&as->lev->fd)); } data = lev->user_data; LeaveCriticalSection(&as->lock); UNLOCK(lev); if (errorcb) { WSASetLastError(error); errorcb(lev, data); } else if (cb) { cb(lev, sock, sa_remote, socklen_remote, data); } LOCK(lev); if (listener_decref_and_unlock(lev)) return; EnterCriticalSection(&as->lock); start_accepting(as); LeaveCriticalSection(&as->lock); } static void accepted_socket_cb(struct event_overlapped *o, ev_uintptr_t key, ev_ssize_t n, int ok) { struct accepting_socket *as = EVUTIL_UPCAST(o, struct accepting_socket, overlapped); LOCK(&as->lev->base); EnterCriticalSection(&as->lock); if (ok) { /* XXXX Don't do this if some EV_MT flag is set. */ event_deferred_cb_schedule_( as->lev->event_base, &as->deferred); LeaveCriticalSection(&as->lock); } else if (as->free_on_cb) { struct evconnlistener *lev = &as->lev->base; free_and_unlock_accepting_socket(as); listener_decref_and_unlock(lev); return; } else if (as->s == INVALID_SOCKET) { /* This is okay; we were disabled by iocp_listener_disable. */ LeaveCriticalSection(&as->lock); } else { /* Some error on accept that we couldn't actually handle. */ BOOL ok; DWORD transfer = 0, flags=0; event_sock_warn(as->s, "Unexpected error on AcceptEx"); ok = WSAGetOverlappedResult(as->s, &o->overlapped, &transfer, FALSE, &flags); if (ok) { /* well, that was confusing! */ as->error = 1; } else { as->error = WSAGetLastError(); } event_deferred_cb_schedule_( as->lev->event_base, &as->deferred); LeaveCriticalSection(&as->lock); } UNLOCK(&as->lev->base); } static int iocp_listener_enable(struct evconnlistener *lev) { int i; struct evconnlistener_iocp *lev_iocp = EVUTIL_UPCAST(lev, struct evconnlistener_iocp, base); LOCK(lev); iocp_listener_event_add(lev_iocp); for (i = 0; i < lev_iocp->n_accepting; ++i) { struct accepting_socket *as = lev_iocp->accepting[i]; if (!as) continue; EnterCriticalSection(&as->lock); if (!as->free_on_cb && as->s == INVALID_SOCKET) start_accepting(as); LeaveCriticalSection(&as->lock); } UNLOCK(lev); return 0; } static int iocp_listener_disable_impl(struct evconnlistener *lev, int shutdown) { int i; struct evconnlistener_iocp *lev_iocp = EVUTIL_UPCAST(lev, struct evconnlistener_iocp, base); LOCK(lev); iocp_listener_event_del(lev_iocp); for (i = 0; i < lev_iocp->n_accepting; ++i) { struct accepting_socket *as = lev_iocp->accepting[i]; if (!as) continue; EnterCriticalSection(&as->lock); if (!as->free_on_cb && as->s != INVALID_SOCKET) { if (shutdown) as->free_on_cb = 1; stop_accepting(as); } LeaveCriticalSection(&as->lock); } if (shutdown && lev->flags & LEV_OPT_CLOSE_ON_FREE) evutil_closesocket(lev_iocp->fd); UNLOCK(lev); return 0; } static int iocp_listener_disable(struct evconnlistener *lev) { return iocp_listener_disable_impl(lev,0); } static void iocp_listener_destroy(struct evconnlistener *lev) { struct evconnlistener_iocp *lev_iocp = EVUTIL_UPCAST(lev, struct evconnlistener_iocp, base); if (! lev_iocp->shutting_down) { lev_iocp->shutting_down = 1; iocp_listener_disable_impl(lev,1); } } static evutil_socket_t iocp_listener_getfd(struct evconnlistener *lev) { struct evconnlistener_iocp *lev_iocp = EVUTIL_UPCAST(lev, struct evconnlistener_iocp, base); return lev_iocp->fd; } static struct event_base * iocp_listener_getbase(struct evconnlistener *lev) { struct evconnlistener_iocp *lev_iocp = EVUTIL_UPCAST(lev, struct evconnlistener_iocp, base); return lev_iocp->event_base; } static const struct evconnlistener_ops evconnlistener_iocp_ops = { iocp_listener_enable, iocp_listener_disable, iocp_listener_destroy, iocp_listener_destroy, /* shutdown */ iocp_listener_getfd, iocp_listener_getbase }; /* XXX define some way to override this. */ #define N_SOCKETS_PER_LISTENER 4 struct evconnlistener * evconnlistener_new_async(struct event_base *base, evconnlistener_cb cb, void *ptr, unsigned flags, int backlog, evutil_socket_t fd) { struct sockaddr_storage ss; int socklen = sizeof(ss); struct evconnlistener_iocp *lev; int i; flags |= LEV_OPT_THREADSAFE; if (!base || !event_base_get_iocp_(base)) goto err; /* XXXX duplicate code */ if (backlog > 0) { if (listen(fd, backlog) < 0) goto err; } else if (backlog < 0) { if (listen(fd, 128) < 0) goto err; } if (getsockname(fd, (struct sockaddr*)&ss, &socklen)) { event_sock_warn(fd, "getsockname"); goto err; } lev = mm_calloc(1, sizeof(struct evconnlistener_iocp)); if (!lev) { event_warn("calloc"); goto err; } lev->base.ops = &evconnlistener_iocp_ops; lev->base.cb = cb; lev->base.user_data = ptr; lev->base.flags = flags; lev->base.refcnt = 1; lev->base.enabled = 1; lev->port = event_base_get_iocp_(base); lev->fd = fd; lev->event_base = base; if (event_iocp_port_associate_(lev->port, fd, 1) < 0) goto err_free_lev; EVTHREAD_ALLOC_LOCK(lev->base.lock, EVTHREAD_LOCKTYPE_RECURSIVE); lev->n_accepting = N_SOCKETS_PER_LISTENER; lev->accepting = mm_calloc(lev->n_accepting, sizeof(struct accepting_socket *)); if (!lev->accepting) { event_warn("calloc"); goto err_delete_lock; } for (i = 0; i < lev->n_accepting; ++i) { lev->accepting[i] = new_accepting_socket(lev, ss.ss_family); if (!lev->accepting[i]) { event_warnx("Couldn't create accepting socket"); goto err_free_accepting; } if (cb && start_accepting(lev->accepting[i]) < 0) { event_warnx("Couldn't start accepting on socket"); EnterCriticalSection(&lev->accepting[i]->lock); free_and_unlock_accepting_socket(lev->accepting[i]); goto err_free_accepting; } ++lev->base.refcnt; } iocp_listener_event_add(lev); return &lev->base; err_free_accepting: mm_free(lev->accepting); /* XXXX free the other elements. */ err_delete_lock: EVTHREAD_FREE_LOCK(lev->base.lock, EVTHREAD_LOCKTYPE_RECURSIVE); err_free_lev: mm_free(lev); err: /* Don't close the fd, it is caller's responsibility. */ return NULL; } #endif kytensor-2.49.0.7/third_party/libevent/log-internal.h000066400000000000000000000062141527052371200225450ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef LOG_INTERNAL_H_INCLUDED_ #define LOG_INTERNAL_H_INCLUDED_ #include "event2/util.h" #ifdef __GNUC__ #define EV_CHECK_FMT(a,b) __attribute__((format(printf, a, b))) #define EV_NORETURN __attribute__((noreturn)) #else #define EV_CHECK_FMT(a,b) #define EV_NORETURN #endif #define EVENT_ERR_ABORT_ ((int)0xdeaddead) #define USE_GLOBAL_FOR_DEBUG_LOGGING #if !defined(EVENT__DISABLE_DEBUG_MODE) || defined(USE_DEBUG) #define EVENT_DEBUG_LOGGING_ENABLED #endif #ifdef EVENT_DEBUG_LOGGING_ENABLED #ifdef USE_GLOBAL_FOR_DEBUG_LOGGING extern ev_uint32_t event_debug_logging_mask_; #define event_debug_get_logging_mask_() (event_debug_logging_mask_) #else ev_uint32_t event_debug_get_logging_mask_(void); #endif #else #define event_debug_get_logging_mask_() (0) #endif void event_err(int eval, const char *fmt, ...) EV_CHECK_FMT(2,3) EV_NORETURN; void event_warn(const char *fmt, ...) EV_CHECK_FMT(1,2); void event_sock_err(int eval, evutil_socket_t sock, const char *fmt, ...) EV_CHECK_FMT(3,4) EV_NORETURN; void event_sock_warn(evutil_socket_t sock, const char *fmt, ...) EV_CHECK_FMT(2,3); void event_errx(int eval, const char *fmt, ...) EV_CHECK_FMT(2,3) EV_NORETURN; void event_warnx(const char *fmt, ...) EV_CHECK_FMT(1,2); void event_msgx(const char *fmt, ...) EV_CHECK_FMT(1,2); void event_debugx_(const char *fmt, ...) EV_CHECK_FMT(1,2); void event_logv_(int severity, const char *errstr, const char *fmt, va_list ap) EV_CHECK_FMT(3,0); #ifdef EVENT_DEBUG_LOGGING_ENABLED #define event_debug(x) do { \ if (event_debug_get_logging_mask_()) { \ event_debugx_ x; \ } \ } while (0) #else #define event_debug(x) ((void)0) #endif #undef EV_CHECK_FMT #endif kytensor-2.49.0.7/third_party/libevent/log.c000066400000000000000000000130111527052371200207170ustar00rootroot00000000000000/* $OpenBSD: err.c,v 1.2 2002/06/25 15:50:15 mickey Exp $ */ /* * log.c * * Based on err.c, which was adapted from OpenBSD libc *err* *warn* code. * * Copyright (c) 2005-2012 Niels Provos and Nick Mathewson * * Copyright (c) 2000 Dug Song * * Copyright (c) 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #include #define WIN32_LEAN_AND_MEAN #include #undef WIN32_LEAN_AND_MEAN #endif #include #include #include #include #include #include #include "event2/event.h" #include "event2/util.h" #include "log-internal.h" static void event_log(int severity, const char *msg); static void event_exit(int errcode) EV_NORETURN; static event_fatal_cb fatal_fn = NULL; #ifdef EVENT_DEBUG_LOGGING_ENABLED #ifdef USE_DEBUG #define DEFAULT_MASK EVENT_DBG_ALL #else #define DEFAULT_MASK 0 #endif #ifdef USE_GLOBAL_FOR_DEBUG_LOGGING ev_uint32_t event_debug_logging_mask_ = DEFAULT_MASK; #else static ev_uint32_t event_debug_logging_mask_ = DEFAULT_MASK; ev_uint32_t event_debug_get_logging_mask_(void) { return event_debug_logging_mask_; } #endif #endif /* EVENT_DEBUG_LOGGING_ENABLED */ void event_enable_debug_logging(ev_uint32_t which) { #ifdef EVENT_DEBUG_LOGGING_ENABLED event_debug_logging_mask_ = which; #endif } void event_set_fatal_callback(event_fatal_cb cb) { fatal_fn = cb; } static void event_exit(int errcode) { if (fatal_fn) { fatal_fn(errcode); exit(errcode); /* should never be reached */ } else if (errcode == EVENT_ERR_ABORT_) abort(); else exit(errcode); } void event_err(int eval, const char *fmt, ...) { va_list ap; va_start(ap, fmt); event_logv_(EVENT_LOG_ERR, strerror(errno), fmt, ap); va_end(ap); event_exit(eval); } void event_warn(const char *fmt, ...) { va_list ap; va_start(ap, fmt); event_logv_(EVENT_LOG_WARN, strerror(errno), fmt, ap); va_end(ap); } void event_sock_err(int eval, evutil_socket_t sock, const char *fmt, ...) { va_list ap; int err = evutil_socket_geterror(sock); va_start(ap, fmt); event_logv_(EVENT_LOG_ERR, evutil_socket_error_to_string(err), fmt, ap); va_end(ap); event_exit(eval); } void event_sock_warn(evutil_socket_t sock, const char *fmt, ...) { va_list ap; int err = evutil_socket_geterror(sock); va_start(ap, fmt); event_logv_(EVENT_LOG_WARN, evutil_socket_error_to_string(err), fmt, ap); va_end(ap); } void event_errx(int eval, const char *fmt, ...) { va_list ap; va_start(ap, fmt); event_logv_(EVENT_LOG_ERR, NULL, fmt, ap); va_end(ap); event_exit(eval); } void event_warnx(const char *fmt, ...) { va_list ap; va_start(ap, fmt); event_logv_(EVENT_LOG_WARN, NULL, fmt, ap); va_end(ap); } void event_msgx(const char *fmt, ...) { va_list ap; va_start(ap, fmt); event_logv_(EVENT_LOG_MSG, NULL, fmt, ap); va_end(ap); } void event_debugx_(const char *fmt, ...) { va_list ap; va_start(ap, fmt); event_logv_(EVENT_LOG_DEBUG, NULL, fmt, ap); va_end(ap); } void event_logv_(int severity, const char *errstr, const char *fmt, va_list ap) { char buf[1024]; size_t len; if (severity == EVENT_LOG_DEBUG && !event_debug_get_logging_mask_()) return; if (fmt != NULL) evutil_vsnprintf(buf, sizeof(buf), fmt, ap); else buf[0] = '\0'; if (errstr) { len = strlen(buf); if (len < sizeof(buf) - 3) { evutil_snprintf(buf + len, sizeof(buf) - len, ": %s", errstr); } } event_log(severity, buf); } static event_log_cb log_fn = NULL; void event_set_log_callback(event_log_cb cb) { log_fn = cb; } static void event_log(int severity, const char *msg) { if (log_fn) log_fn(severity, msg); else { const char *severity_str; switch (severity) { case EVENT_LOG_DEBUG: severity_str = "debug"; break; case EVENT_LOG_MSG: severity_str = "msg"; break; case EVENT_LOG_WARN: severity_str = "warn"; break; case EVENT_LOG_ERR: severity_str = "err"; break; default: severity_str = "???"; break; } (void)fprintf(stderr, "[%s] %s\n", severity_str, msg); } } kytensor-2.49.0.7/third_party/libevent/m4/000077500000000000000000000000001527052371200203165ustar00rootroot00000000000000kytensor-2.49.0.7/third_party/libevent/m4/ac_backport_259_ssizet.m4000066400000000000000000000001501527052371200250240ustar00rootroot00000000000000AN_IDENTIFIER([ssize_t], [AC_TYPE_SSIZE_T]) AC_DEFUN([AC_TYPE_SSIZE_T], [AC_CHECK_TYPE(ssize_t, int)]) kytensor-2.49.0.7/third_party/libevent/m4/acx_pthread.m4000066400000000000000000000252601527052371200230470ustar00rootroot00000000000000##### http://autoconf-archive.cryp.to/acx_pthread.html # # SYNOPSIS # # ACX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]]) # # DESCRIPTION # # This macro figures out how to build C programs using POSIX threads. # It sets the PTHREAD_LIBS output variable to the threads library and # linker flags, and the PTHREAD_CFLAGS output variable to any special # C compiler flags that are needed. (The user can also force certain # compiler flags/libs to be tested by setting these environment # variables.) # # Also sets PTHREAD_CC to any special C compiler that is needed for # multi-threaded programs (defaults to the value of CC otherwise). # (This is necessary on AIX to use the special cc_r compiler alias.) # # NOTE: You are assumed to not only compile your program with these # flags, but also link it with them as well. e.g. you should link # with $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS # $LIBS # # If you are only building threads programs, you may wish to use # these variables in your default LIBS, CFLAGS, and CC: # # LIBS="$PTHREAD_LIBS $LIBS" # CFLAGS="$CFLAGS $PTHREAD_CFLAGS" # CC="$PTHREAD_CC" # # In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute # constant has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to # that name (e.g. PTHREAD_CREATE_UNDETACHED on AIX). # # ACTION-IF-FOUND is a list of shell commands to run if a threads # library is found, and ACTION-IF-NOT-FOUND is a list of commands to # run it if it is not found. If ACTION-IF-FOUND is not specified, the # default action will define HAVE_PTHREAD. # # Please let the authors know if this macro fails on any platform, or # if you have any other suggestions or comments. This macro was based # on work by SGJ on autoconf scripts for FFTW (http://www.fftw.org/) # (with help from M. Frigo), as well as ac_pthread and hb_pthread # macros posted by Alejandro Forero Cuervo to the autoconf macro # repository. We are also grateful for the helpful feedback of # numerous users. # # LAST MODIFICATION # # 2007-07-29 # # COPYLEFT # # Copyright (c) 2007 Steven G. Johnson # # This program is free software: you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see # . # # As a special exception, the respective Autoconf Macro's copyright # owner gives unlimited permission to copy, distribute and modify the # configure scripts that are the output of Autoconf when processing # the Macro. You need not follow the terms of the GNU General Public # License when using or distributing such scripts, even though # portions of the text of the Macro appear in them. The GNU General # Public License (GPL) does govern all other use of the material that # constitutes the Autoconf Macro. # # This special exception to the GPL applies to versions of the # Autoconf Macro released by the Autoconf Macro Archive. When you # make and distribute a modified version of the Autoconf Macro, you # may extend this special exception to the GPL to apply to your # modified version as well. AC_DEFUN([ACX_PTHREAD], [ AC_REQUIRE([AC_CANONICAL_HOST]) AC_LANG_SAVE AC_LANG_C acx_pthread_ok=no # We used to check for pthread.h first, but this fails if pthread.h # requires special compiler flags (e.g. on True64 or Sequent). # It gets checked for in the link test anyway. # First of all, check if the user has set any of the PTHREAD_LIBS, # etcetera environment variables, and if threads linking works using # them: if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then save_CFLAGS="$CFLAGS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS" save_LIBS="$LIBS" LIBS="$PTHREAD_LIBS $LIBS" AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS]) AC_TRY_LINK_FUNC(pthread_join, acx_pthread_ok=yes) AC_MSG_RESULT($acx_pthread_ok) if test x"$acx_pthread_ok" = xno; then PTHREAD_LIBS="" PTHREAD_CFLAGS="" fi LIBS="$save_LIBS" CFLAGS="$save_CFLAGS" fi # We must check for the threads library under a number of different # names; the ordering is very important because some systems # (e.g. DEC) have both -lpthread and -lpthreads, where one of the # libraries is broken (non-POSIX). # Create a list of thread flags to try. Items starting with a "-" are # C compiler flags, and other items are library names, except for "none" # which indicates that we try without any flags at all, and "pthread-config" # which is a program returning the flags for the Pth emulation library. acx_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" # The ordering *is* (sometimes) important. Some notes on the # individual items follow: # pthreads: AIX (must check this before -lpthread) # none: in case threads are in libc; should be tried before -Kthread and # other compiler flags to prevent continual compiler warnings # -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h) # -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) # lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) # -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads) # -pthreads: Solaris/gcc # -mthreads: Mingw32/gcc, Lynx/gcc # -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it # doesn't hurt to check since this sometimes defines pthreads too; # also defines -D_REENTRANT) # ... -mt is also the pthreads flag for HP/aCC # pthread: Linux, etcetera # --thread-safe: KAI C++ # pthread-config: use pthread-config program (for GNU Pth library) case "${host_cpu}-${host_os}" in *solaris*) # On Solaris (at least, for some versions), libc contains stubbed # (non-functional) versions of the pthreads routines, so link-based # tests will erroneously succeed. (We need to link with -pthreads/-mt/ # -lpthread.) (The stubs are missing pthread_cleanup_push, or rather # a function called by this macro, so we could check for that, but # who knows whether they'll stub that too in a future libc.) So, # we'll just look for -pthreads and -lpthread first: acx_pthread_flags="-pthreads pthread -mt -pthread $acx_pthread_flags" ;; esac if test x"$acx_pthread_ok" = xno; then for flag in $acx_pthread_flags; do case $flag in none) AC_MSG_CHECKING([whether pthreads work without any flags]) ;; -*) AC_MSG_CHECKING([whether pthreads work with $flag]) PTHREAD_CFLAGS="$flag" ;; pthread-config) AC_CHECK_PROG(acx_pthread_config, pthread-config, yes, no) if test x"$acx_pthread_config" = xno; then continue; fi PTHREAD_CFLAGS="`pthread-config --cflags`" PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" ;; *) AC_MSG_CHECKING([for the pthreads library -l$flag]) PTHREAD_LIBS="-l$flag" ;; esac save_LIBS="$LIBS" save_CFLAGS="$CFLAGS" LIBS="$PTHREAD_LIBS $LIBS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS" # Check for various functions. We must include pthread.h, # since some functions may be macros. (On the Sequent, we # need a special flag -Kthread to make this header compile.) # We check for pthread_join because it is in -lpthread on IRIX # while pthread_create is in libc. We check for pthread_attr_init # due to DEC craziness with -lpthreads. We check for # pthread_cleanup_push because it is one of the few pthread # functions on Solaris that doesn't have a non-functional libc stub. # We try pthread_create on general principles. AC_TRY_LINK([#include ], [pthread_t th; pthread_join(th, 0); pthread_attr_init(0); pthread_cleanup_push(0, 0); pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], [acx_pthread_ok=yes]) LIBS="$save_LIBS" CFLAGS="$save_CFLAGS" AC_MSG_RESULT($acx_pthread_ok) if test "x$acx_pthread_ok" = xyes; then break; fi PTHREAD_LIBS="" PTHREAD_CFLAGS="" done fi # Various other checks: if test "x$acx_pthread_ok" = xyes; then save_LIBS="$LIBS" LIBS="$PTHREAD_LIBS $LIBS" save_CFLAGS="$CFLAGS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS" # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. AC_MSG_CHECKING([for joinable pthread attribute]) attr_name=unknown for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do AC_TRY_LINK([#include ], [int attr=$attr; return attr;], [attr_name=$attr; break]) done AC_MSG_RESULT($attr_name) if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name, [Define to necessary symbol if this constant uses a non-standard name on your system.]) fi AC_MSG_CHECKING([if more special flags are required for pthreads]) flag=no case "${host_cpu}-${host_os}" in *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";; *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";; esac AC_MSG_RESULT(${flag}) if test "x$flag" != xno; then PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS" fi LIBS="$save_LIBS" CFLAGS="$save_CFLAGS" # More AIX lossage: must compile with xlc_r or cc_r if test x"$GCC" != xyes; then AC_CHECK_PROGS(PTHREAD_CC, xlc_r cc_r, ${CC}) else PTHREAD_CC=$CC fi else PTHREAD_CC="$CC" fi AC_SUBST(PTHREAD_LIBS) AC_SUBST(PTHREAD_CFLAGS) AC_SUBST(PTHREAD_CC) # Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: if test x"$acx_pthread_ok" = xyes; then ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1]) : else acx_pthread_ok=no $2 fi AC_LANG_RESTORE ])dnl ACX_PTHREAD kytensor-2.49.0.7/third_party/libevent/m4/libevent_openssl.m4000066400000000000000000000024431527052371200241360ustar00rootroot00000000000000dnl ###################################################################### dnl OpenSSL support AC_DEFUN([LIBEVENT_OPENSSL], [ AC_REQUIRE([NTP_PKG_CONFIG])dnl case "$enable_openssl" in yes) have_openssl=no case "$PKG_CONFIG" in '') ;; *) OPENSSL_LIBS=`$PKG_CONFIG --libs openssl 2>/dev/null` case "$OPENSSL_LIBS" in '') ;; *) OPENSSL_LIBS="$OPENSSL_LIBS $EV_LIB_GDI $EV_LIB_WS32 $OPENSSL_LIBADD" have_openssl=yes ;; esac OPENSSL_INCS=`$PKG_CONFIG --cflags openssl 2>/dev/null` ;; esac case "$have_openssl" in yes) ;; *) save_LIBS="$LIBS" LIBS="" OPENSSL_LIBS="" for lib in crypto eay32; do # clear cache unset ac_cv_search_SSL_new AC_SEARCH_LIBS([SSL_new], [ssl ssl32], [have_openssl=yes OPENSSL_LIBS="$LIBS -l$lib $EV_LIB_GDI $EV_LIB_WS32 $OPENSSL_LIBADD"], [have_openssl=no], [-l$lib $EV_LIB_GDI $EV_LIB_WS32 $OPENSSL_LIBADD]) LIBS="$save_LIBS" test "$have_openssl" = "yes" && break done ;; esac AC_SUBST(OPENSSL_INCS) AC_SUBST(OPENSSL_LIBS) case "$have_openssl" in yes) AC_DEFINE(HAVE_OPENSSL, 1, [Define if the system has openssl]) ;; esac ;; esac # check if we have and should use openssl AM_CONDITIONAL(OPENSSL, [test "$enable_openssl" != "no" && test "$have_openssl" = "yes"]) ]) kytensor-2.49.0.7/third_party/libevent/m4/ntp_pkg_config.m4000066400000000000000000000012411527052371200235450ustar00rootroot00000000000000dnl NTP_PKG_CONFIG -*- Autoconf -*- dnl dnl Look for pkg-config, which must be at least dnl $ntp_pkgconfig_min_version. dnl AC_DEFUN([NTP_PKG_CONFIG], [ dnl lower the minimum version if you find an earlier one works ntp_pkgconfig_min_version='0.15.0' AC_PATH_TOOL([PKG_CONFIG], [pkg-config]) AS_UNSET([ac_cv_path_PKG_CONFIG]) AS_UNSET([ac_cv_path_ac_pt_PKG_CONFIG]) case "$PKG_CONFIG" in /*) AC_MSG_CHECKING([if pkg-config is at least version $ntp_pkgconfig_min_version]) if $PKG_CONFIG --atleast-pkgconfig-version $ntp_pkgconfig_min_version; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) PKG_CONFIG="" fi ;; esac ]) dnl NTP_PKG_CONFIG kytensor-2.49.0.7/third_party/libevent/make-event-config.sed000066400000000000000000000011141527052371200237670ustar00rootroot00000000000000# Sed script to postprocess config.h into event-config.h. 1i\ /* event2/event-config.h\ *\ * This file was generated by autoconf when libevent was built, and post-\ * processed by Libevent so that its macros would have a uniform prefix.\ *\ * DO NOT EDIT THIS FILE.\ *\ * Do not rely on macros in this file existing in later versions.\ */\ \ #ifndef EVENT2_EVENT_CONFIG_H_INCLUDED_\ #define EVENT2_EVENT_CONFIG_H_INCLUDED_\ $a\ \ #endif /* event2/event-config.h */ s/#\( *\)define /#\1define EVENT__/ s/#\( *\)undef /#\1undef EVENT__/ s/#\( *\)if\(n*\)def /#\1if\2def EVENT__/ kytensor-2.49.0.7/third_party/libevent/make_epoll_table.py000077500000000000000000000034171527052371200236370ustar00rootroot00000000000000#!/usr/bin/python2 def get(old,wc,rc,cc): if ('xxx' in (rc, wc, cc)): return "0",255 if ('add' in (rc, wc, cc)): events = [] if rc == 'add' or (rc != 'del' and 'r' in old): events.append("EPOLLIN") if wc == 'add' or (wc != 'del' and 'w' in old): events.append("EPOLLOUT") if cc == 'add' or (cc != 'del' and 'c' in old): events.append("EPOLLRDHUP") if old == "0": op = "EPOLL_CTL_ADD" else: op = "EPOLL_CTL_MOD" return "|".join(events), op if ('del' in (rc, wc, cc)): delevents = [] modevents = [] op = "EPOLL_CTL_DEL" if 'r' in old: modevents.append("EPOLLIN") if 'w' in old: modevents.append("EPOLLOUT") if 'c' in old: modevents.append("EPOLLRDHUP") for item, event in [(rc,"EPOLLIN"), (wc,"EPOLLOUT"), (cc,"EPOLLRDHUP")]: if item == 'del': delevents.append(event) if event in modevents: modevents.remove(event) if modevents: return "|".join(modevents), "EPOLL_CTL_MOD" else: return "|".join(delevents), "EPOLL_CTL_DEL" return 0, 0 def fmt(op, ev, old, wc, rc, cc): entry = "{ %s, %s },"%(op, ev) print "\t/* old=%3s, write:%3s, read:%3s, close:%3s */\n\t%s" % ( old, wc, rc, cc, entry) return len(entry) for old in ('0','r','w','rw','c','cr','cw','crw'): for wc in ('0', 'add', 'del', 'xxx'): for rc in ('0', 'add', 'del', 'xxx'): for cc in ('0', 'add', 'del', 'xxx'): op,ev = get(old,wc,rc,cc) fmt(op, ev, old, wc, rc, cc) kytensor-2.49.0.7/third_party/libevent/minheap-internal.h000066400000000000000000000154321527052371200234070ustar00rootroot00000000000000/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Copyright (c) 2006 Maxim Yegorushkin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef MINHEAP_INTERNAL_H_INCLUDED_ #define MINHEAP_INTERNAL_H_INCLUDED_ #include "event2/event-config.h" #include "evconfig-private.h" #include "event2/event.h" #include "event2/event_struct.h" #include "event2/util.h" #include "util-internal.h" #include "mm-internal.h" typedef struct min_heap { struct event** p; unsigned n, a; } min_heap_t; static inline void min_heap_ctor_(min_heap_t* s); static inline void min_heap_dtor_(min_heap_t* s); static inline void min_heap_elem_init_(struct event* e); static inline int min_heap_elt_is_top_(const struct event *e); static inline int min_heap_empty_(min_heap_t* s); static inline unsigned min_heap_size_(min_heap_t* s); static inline struct event* min_heap_top_(min_heap_t* s); static inline int min_heap_reserve_(min_heap_t* s, unsigned n); static inline int min_heap_push_(min_heap_t* s, struct event* e); static inline struct event* min_heap_pop_(min_heap_t* s); static inline int min_heap_adjust_(min_heap_t *s, struct event* e); static inline int min_heap_erase_(min_heap_t* s, struct event* e); static inline void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct event* e); static inline void min_heap_shift_up_unconditional_(min_heap_t* s, unsigned hole_index, struct event* e); static inline void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct event* e); #define min_heap_elem_greater(a, b) \ (evutil_timercmp(&(a)->ev_timeout, &(b)->ev_timeout, >)) void min_heap_ctor_(min_heap_t* s) { s->p = 0; s->n = 0; s->a = 0; } void min_heap_dtor_(min_heap_t* s) { if (s->p) mm_free(s->p); } void min_heap_elem_init_(struct event* e) { e->ev_timeout_pos.min_heap_idx = -1; } int min_heap_empty_(min_heap_t* s) { return 0u == s->n; } unsigned min_heap_size_(min_heap_t* s) { return s->n; } struct event* min_heap_top_(min_heap_t* s) { return s->n ? *s->p : 0; } int min_heap_push_(min_heap_t* s, struct event* e) { if (min_heap_reserve_(s, s->n + 1)) return -1; min_heap_shift_up_(s, s->n++, e); return 0; } struct event* min_heap_pop_(min_heap_t* s) { if (s->n) { struct event* e = *s->p; min_heap_shift_down_(s, 0u, s->p[--s->n]); e->ev_timeout_pos.min_heap_idx = -1; return e; } return 0; } int min_heap_elt_is_top_(const struct event *e) { return e->ev_timeout_pos.min_heap_idx == 0; } int min_heap_erase_(min_heap_t* s, struct event* e) { if (-1 != e->ev_timeout_pos.min_heap_idx) { struct event *last = s->p[--s->n]; unsigned parent = (e->ev_timeout_pos.min_heap_idx - 1) / 2; /* we replace e with the last element in the heap. We might need to shift it upward if it is less than its parent, or downward if it is greater than one or both its children. Since the children are known to be less than the parent, it can't need to shift both up and down. */ if (e->ev_timeout_pos.min_heap_idx > 0 && min_heap_elem_greater(s->p[parent], last)) min_heap_shift_up_unconditional_(s, e->ev_timeout_pos.min_heap_idx, last); else min_heap_shift_down_(s, e->ev_timeout_pos.min_heap_idx, last); e->ev_timeout_pos.min_heap_idx = -1; return 0; } return -1; } int min_heap_adjust_(min_heap_t *s, struct event *e) { if (-1 == e->ev_timeout_pos.min_heap_idx) { return min_heap_push_(s, e); } else { unsigned parent = (e->ev_timeout_pos.min_heap_idx - 1) / 2; /* The position of e has changed; we shift it up or down * as needed. We can't need to do both. */ if (e->ev_timeout_pos.min_heap_idx > 0 && min_heap_elem_greater(s->p[parent], e)) min_heap_shift_up_unconditional_(s, e->ev_timeout_pos.min_heap_idx, e); else min_heap_shift_down_(s, e->ev_timeout_pos.min_heap_idx, e); return 0; } } int min_heap_reserve_(min_heap_t* s, unsigned n) { if (s->a < n) { struct event** p; unsigned a = s->a ? s->a * 2 : 8; if (a < n) a = n; if (!(p = (struct event**)mm_realloc(s->p, a * sizeof *p))) return -1; s->p = p; s->a = a; } return 0; } void min_heap_shift_up_unconditional_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned parent = (hole_index - 1) / 2; do { (s->p[hole_index] = s->p[parent])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = parent; parent = (hole_index - 1) / 2; } while (hole_index && min_heap_elem_greater(s->p[parent], e)); (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned parent = (hole_index - 1) / 2; while (hole_index && min_heap_elem_greater(s->p[parent], e)) { (s->p[hole_index] = s->p[parent])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = parent; parent = (hole_index - 1) / 2; } (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned min_child = 2 * (hole_index + 1); while (min_child <= s->n) { min_child -= min_child == s->n || min_heap_elem_greater(s->p[min_child], s->p[min_child - 1]); if (!(min_heap_elem_greater(e, s->p[min_child]))) break; (s->p[hole_index] = s->p[min_child])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = min_child; min_child = 2 * (hole_index + 1); } (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } #endif /* MINHEAP_INTERNAL_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/mm-internal.h000066400000000000000000000065041527052371200223770ustar00rootroot00000000000000/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef MM_INTERNAL_H_INCLUDED_ #define MM_INTERNAL_H_INCLUDED_ #include #ifdef __cplusplus extern "C" { #endif #ifndef EVENT__DISABLE_MM_REPLACEMENT /* Internal use only: Memory allocation functions. We give them nice short * mm_names for our own use, but make sure that the symbols have longer names * so they don't conflict with other libraries (like, say, libmm). */ /** Allocate uninitialized memory. * * @return On success, return a pointer to sz newly allocated bytes. * On failure, set errno to ENOMEM and return NULL. * If the argument sz is 0, simply return NULL. */ void *event_mm_malloc_(size_t sz); /** Allocate memory initialized to zero. * * @return On success, return a pointer to (count * size) newly allocated * bytes, initialized to zero. * On failure, or if the product would result in an integer overflow, * set errno to ENOMEM and return NULL. * If either arguments are 0, simply return NULL. */ void *event_mm_calloc_(size_t count, size_t size); /** Duplicate a string. * * @return On success, return a pointer to a newly allocated duplicate * of a string. * Set errno to ENOMEM and return NULL if a memory allocation error * occurs (or would occur) in the process. * If the argument str is NULL, set errno to EINVAL and return NULL. */ char *event_mm_strdup_(const char *str); void *event_mm_realloc_(void *p, size_t sz); void event_mm_free_(void *p); #define mm_malloc(sz) event_mm_malloc_(sz) #define mm_calloc(count, size) event_mm_calloc_((count), (size)) #define mm_strdup(s) event_mm_strdup_(s) #define mm_realloc(p, sz) event_mm_realloc_((p), (sz)) #define mm_free(p) event_mm_free_(p) #else #define mm_malloc(sz) malloc(sz) #define mm_calloc(n, sz) calloc((n), (sz)) #define mm_strdup(s) strdup(s) #define mm_realloc(p, sz) realloc((p), (sz)) #define mm_free(p) free(p) #endif #ifdef __cplusplus } #endif #endif kytensor-2.49.0.7/third_party/libevent/openssl-compat.h000066400000000000000000000017161527052371200231200ustar00rootroot00000000000000#ifndef OPENSSL_COMPAT_H #define OPENSSL_COMPAT_H #if OPENSSL_VERSION_NUMBER < 0x10100000L static inline BIO_METHOD *BIO_meth_new(int type, const char *name) { BIO_METHOD *biom = calloc(1, sizeof(BIO_METHOD)); if (biom != NULL) { biom->type = type; biom->name = name; } return biom; } #define BIO_meth_set_write(b, f) (b)->bwrite = (f) #define BIO_meth_set_read(b, f) (b)->bread = (f) #define BIO_meth_set_puts(b, f) (b)->bputs = (f) #define BIO_meth_set_ctrl(b, f) (b)->ctrl = (f) #define BIO_meth_set_create(b, f) (b)->create = (f) #define BIO_meth_set_destroy(b, f) (b)->destroy = (f) #define BIO_set_init(b, val) (b)->init = (val) #define BIO_set_data(b, val) (b)->ptr = (val) #define BIO_set_shutdown(b, val) (b)->shutdown = (val) #define BIO_get_init(b) (b)->init #define BIO_get_data(b) (b)->ptr #define BIO_get_shutdown(b) (b)->shutdown #define TLS_method SSLv23_method #endif /* OPENSSL_VERSION_NUMBER < 0x10100000L */ #endif /* OPENSSL_COMPAT_H */ kytensor-2.49.0.7/third_party/libevent/poll.c000066400000000000000000000177611527052371200211240ustar00rootroot00000000000000/* $OpenBSD: poll.c,v 1.2 2002/06/25 15:50:15 mickey Exp $ */ /* * Copyright 2000-2007 Niels Provos * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_POLL #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #include #include #include #include #include #include #include #include #include "event-internal.h" #include "evsignal-internal.h" #include "log-internal.h" #include "evmap-internal.h" #include "event2/thread.h" #include "evthread-internal.h" #include "time-internal.h" struct pollidx { int idxplus1; }; struct pollop { int event_count; /* Highest number alloc */ int nfds; /* Highest number used */ int realloc_copy; /* True iff we must realloc * event_set_copy */ struct pollfd *event_set; struct pollfd *event_set_copy; }; static void *poll_init(struct event_base *); static int poll_add(struct event_base *, int, short old, short events, void *idx); static int poll_del(struct event_base *, int, short old, short events, void *idx); static int poll_dispatch(struct event_base *, struct timeval *); static void poll_dealloc(struct event_base *); const struct eventop pollops = { "poll", poll_init, poll_add, poll_del, poll_dispatch, poll_dealloc, 0, /* doesn't need_reinit */ EV_FEATURE_FDS, sizeof(struct pollidx), }; static void * poll_init(struct event_base *base) { struct pollop *pollop; if (!(pollop = mm_calloc(1, sizeof(struct pollop)))) return (NULL); evsig_init_(base); evutil_weakrand_seed_(&base->weakrand_seed, 0); return (pollop); } #ifdef CHECK_INVARIANTS static void poll_check_ok(struct pollop *pop) { int i, idx; struct event *ev; for (i = 0; i < pop->fd_count; ++i) { idx = pop->idxplus1_by_fd[i]-1; if (idx < 0) continue; EVUTIL_ASSERT(pop->event_set[idx].fd == i); } for (i = 0; i < pop->nfds; ++i) { struct pollfd *pfd = &pop->event_set[i]; EVUTIL_ASSERT(pop->idxplus1_by_fd[pfd->fd] == i+1); } } #else #define poll_check_ok(pop) #endif static int poll_dispatch(struct event_base *base, struct timeval *tv) { int res, i, j, nfds; long msec = -1; struct pollop *pop = base->evbase; struct pollfd *event_set; poll_check_ok(pop); nfds = pop->nfds; #ifndef EVENT__DISABLE_THREAD_SUPPORT if (base->th_base_lock) { /* If we're using this backend in a multithreaded setting, * then we need to work on a copy of event_set, so that we can * let other threads modify the main event_set while we're * polling. If we're not multithreaded, then we'll skip the * copy step here to save memory and time. */ if (pop->realloc_copy) { struct pollfd *tmp = mm_realloc(pop->event_set_copy, pop->event_count * sizeof(struct pollfd)); if (tmp == NULL) { event_warn("realloc"); return -1; } pop->event_set_copy = tmp; pop->realloc_copy = 0; } memcpy(pop->event_set_copy, pop->event_set, sizeof(struct pollfd)*nfds); event_set = pop->event_set_copy; } else { event_set = pop->event_set; } #else event_set = pop->event_set; #endif if (tv != NULL) { msec = evutil_tv_to_msec_(tv); if (msec < 0 || msec > INT_MAX) msec = INT_MAX; } EVBASE_RELEASE_LOCK(base, th_base_lock); res = poll(event_set, nfds, msec); EVBASE_ACQUIRE_LOCK(base, th_base_lock); if (res == -1) { if (errno != EINTR) { event_warn("poll"); return (-1); } return (0); } event_debug(("%s: poll reports %d", __func__, res)); if (res == 0 || nfds == 0) return (0); i = evutil_weakrand_range_(&base->weakrand_seed, nfds); for (j = 0; j < nfds; j++) { int what; if (++i == nfds) i = 0; what = event_set[i].revents; if (!what) continue; res = 0; /* If the file gets closed notify */ if (what & (POLLHUP|POLLERR|POLLNVAL)) what |= POLLIN|POLLOUT; if (what & POLLIN) res |= EV_READ; if (what & POLLOUT) res |= EV_WRITE; if (res == 0) continue; evmap_io_active_(base, event_set[i].fd, res); } return (0); } static int poll_add(struct event_base *base, int fd, short old, short events, void *idx_) { struct pollop *pop = base->evbase; struct pollfd *pfd = NULL; struct pollidx *idx = idx_; int i; EVUTIL_ASSERT((events & EV_SIGNAL) == 0); if (!(events & (EV_READ|EV_WRITE))) return (0); poll_check_ok(pop); if (pop->nfds + 1 >= pop->event_count) { struct pollfd *tmp_event_set; int tmp_event_count; if (pop->event_count < 32) tmp_event_count = 32; else tmp_event_count = pop->event_count * 2; /* We need more file descriptors */ tmp_event_set = mm_realloc(pop->event_set, tmp_event_count * sizeof(struct pollfd)); if (tmp_event_set == NULL) { event_warn("realloc"); return (-1); } pop->event_set = tmp_event_set; pop->event_count = tmp_event_count; pop->realloc_copy = 1; } i = idx->idxplus1 - 1; if (i >= 0) { pfd = &pop->event_set[i]; } else { i = pop->nfds++; pfd = &pop->event_set[i]; pfd->events = 0; pfd->fd = fd; idx->idxplus1 = i + 1; } pfd->revents = 0; if (events & EV_WRITE) pfd->events |= POLLOUT; if (events & EV_READ) pfd->events |= POLLIN; poll_check_ok(pop); return (0); } /* * Nothing to be done here. */ static int poll_del(struct event_base *base, int fd, short old, short events, void *idx_) { struct pollop *pop = base->evbase; struct pollfd *pfd = NULL; struct pollidx *idx = idx_; int i; EVUTIL_ASSERT((events & EV_SIGNAL) == 0); if (!(events & (EV_READ|EV_WRITE))) return (0); poll_check_ok(pop); i = idx->idxplus1 - 1; if (i < 0) return (-1); /* Do we still want to read or write? */ pfd = &pop->event_set[i]; if (events & EV_READ) pfd->events &= ~POLLIN; if (events & EV_WRITE) pfd->events &= ~POLLOUT; poll_check_ok(pop); if (pfd->events) /* Another event cares about that fd. */ return (0); /* Okay, so we aren't interested in that fd anymore. */ idx->idxplus1 = 0; --pop->nfds; if (i != pop->nfds) { /* * Shift the last pollfd down into the now-unoccupied * position. */ memcpy(&pop->event_set[i], &pop->event_set[pop->nfds], sizeof(struct pollfd)); idx = evmap_io_get_fdinfo_(&base->io, pop->event_set[i].fd); EVUTIL_ASSERT(idx); EVUTIL_ASSERT(idx->idxplus1 == pop->nfds + 1); idx->idxplus1 = i + 1; } poll_check_ok(pop); return (0); } static void poll_dealloc(struct event_base *base) { struct pollop *pop = base->evbase; evsig_dealloc_(base); if (pop->event_set) mm_free(pop->event_set); if (pop->event_set_copy) mm_free(pop->event_set_copy); memset(pop, 0, sizeof(struct pollop)); mm_free(pop); } #endif /* EVENT__HAVE_POLL */ kytensor-2.49.0.7/third_party/libevent/ratelim-internal.h000066400000000000000000000077741527052371200234350ustar00rootroot00000000000000/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef RATELIM_INTERNAL_H_INCLUDED_ #define RATELIM_INTERNAL_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif #include "event2/util.h" /** A token bucket is an internal structure that tracks how many bytes we are * currently willing to read or write on a given bufferevent or group of * bufferevents */ struct ev_token_bucket { /** How many bytes are we willing to read or write right now? These * values are signed so that we can do "defecit spending" */ ev_ssize_t read_limit, write_limit; /** When was this bucket last updated? Measured in abstract 'ticks' * relative to the token bucket configuration. */ ev_uint32_t last_updated; }; /** Configuration info for a token bucket or set of token buckets. */ struct ev_token_bucket_cfg { /** How many bytes are we willing to read on average per tick? */ size_t read_rate; /** How many bytes are we willing to read at most in any one tick? */ size_t read_maximum; /** How many bytes are we willing to write on average per tick? */ size_t write_rate; /** How many bytes are we willing to write at most in any one tick? */ size_t write_maximum; /* How long is a tick? Note that fractions of a millisecond are * ignored. */ struct timeval tick_timeout; /* How long is a tick, in milliseconds? Derived from tick_timeout. */ unsigned msec_per_tick; }; /** The current tick is 'current_tick': add bytes to 'bucket' as specified in * 'cfg'. */ int ev_token_bucket_update_(struct ev_token_bucket *bucket, const struct ev_token_bucket_cfg *cfg, ev_uint32_t current_tick); /** In which tick does 'tv' fall according to 'cfg'? Note that ticks can * overflow easily; your code needs to handle this. */ ev_uint32_t ev_token_bucket_get_tick_(const struct timeval *tv, const struct ev_token_bucket_cfg *cfg); /** Adjust 'bucket' to respect 'cfg', and note that it was last updated in * 'current_tick'. If 'reinitialize' is true, we are changing the * configuration of 'bucket'; otherwise, we are setting it up for the first * time. */ int ev_token_bucket_init_(struct ev_token_bucket *bucket, const struct ev_token_bucket_cfg *cfg, ev_uint32_t current_tick, int reinitialize); int bufferevent_remove_from_rate_limit_group_internal_(struct bufferevent *bev, int unsuspend); /** Decrease the read limit of 'b' by 'n' bytes */ #define ev_token_bucket_decrement_read(b,n) \ do { \ (b)->read_limit -= (n); \ } while (0) /** Decrease the write limit of 'b' by 'n' bytes */ #define ev_token_bucket_decrement_write(b,n) \ do { \ (b)->write_limit -= (n); \ } while (0) #ifdef __cplusplus } #endif #endif kytensor-2.49.0.7/third_party/libevent/sample/000077500000000000000000000000001527052371200212575ustar00rootroot00000000000000kytensor-2.49.0.7/third_party/libevent/sample/dns-example.c000066400000000000000000000147211527052371200236450ustar00rootroot00000000000000/* This example code shows how to use the high-level, low-level, and server-level interfaces of evdns. XXX It's pretty ugly and should probably be cleaned up. */ #include /* Compatibility for possible missing IPv6 declarations */ #include "../ipv6-internal.h" #include #ifdef EVENT__HAVE_UNISTD_H #include #endif #ifdef _WIN32 #include #include #include #else #include #include #include #endif #include #include #include #include #ifdef EVENT__HAVE_NETINET_IN6_H #include #endif #include #include #include #define u32 ev_uint32_t #define u8 ev_uint8_t static const char * debug_ntoa(u32 address) { static char buf[32]; u32 a = ntohl(address); evutil_snprintf(buf, sizeof(buf), "%d.%d.%d.%d", (int)(u8)((a>>24)&0xff), (int)(u8)((a>>16)&0xff), (int)(u8)((a>>8 )&0xff), (int)(u8)((a )&0xff)); return buf; } static void main_callback(int result, char type, int count, int ttl, void *addrs, void *orig) { char *n = (char*)orig; int i; for (i = 0; i < count; ++i) { if (type == DNS_IPv4_A) { printf("%s: %s\n", n, debug_ntoa(((u32*)addrs)[i])); } else if (type == DNS_PTR) { printf("%s: %s\n", n, ((char**)addrs)[i]); } } if (!count) { printf("%s: No answer (%d)\n", n, result); } fflush(stdout); } static void gai_callback(int err, struct evutil_addrinfo *ai, void *arg) { const char *name = arg; int i; if (err) { printf("%s: %s\n", name, evutil_gai_strerror(err)); } if (ai && ai->ai_canonname) printf(" %s ==> %s\n", name, ai->ai_canonname); for (i=0; ai; ai = ai->ai_next, ++i) { char buf[128]; if (ai->ai_family == PF_INET) { struct sockaddr_in *sin = (struct sockaddr_in*)ai->ai_addr; evutil_inet_ntop(AF_INET, &sin->sin_addr, buf, sizeof(buf)); printf("[%d] %s: %s\n",i,name,buf); } else { struct sockaddr_in6 *sin6 = (struct sockaddr_in6*)ai->ai_addr; evutil_inet_ntop(AF_INET6, &sin6->sin6_addr, buf, sizeof(buf)); printf("[%d] %s: %s\n",i,name,buf); } } } static void evdns_server_callback(struct evdns_server_request *req, void *data) { int i, r; (void)data; /* dummy; give 192.168.11.11 as an answer for all A questions, * give foo.bar.example.com as an answer for all PTR questions. */ for (i = 0; i < req->nquestions; ++i) { u32 ans = htonl(0xc0a80b0bUL); if (req->questions[i]->type == EVDNS_TYPE_A && req->questions[i]->dns_question_class == EVDNS_CLASS_INET) { printf(" -- replying for %s (A)\n", req->questions[i]->name); r = evdns_server_request_add_a_reply(req, req->questions[i]->name, 1, &ans, 10); if (r<0) printf("eeep, didn't work.\n"); } else if (req->questions[i]->type == EVDNS_TYPE_PTR && req->questions[i]->dns_question_class == EVDNS_CLASS_INET) { printf(" -- replying for %s (PTR)\n", req->questions[i]->name); r = evdns_server_request_add_ptr_reply(req, NULL, req->questions[i]->name, "foo.bar.example.com", 10); if (r<0) printf("ugh, no luck"); } else { printf(" -- skipping %s [%d %d]\n", req->questions[i]->name, req->questions[i]->type, req->questions[i]->dns_question_class); } } r = evdns_server_request_respond(req, 0); if (r<0) printf("eeek, couldn't send reply.\n"); } static int verbose = 0; static void logfn(int is_warn, const char *msg) { if (!is_warn && !verbose) return; fprintf(stderr, "%s: %s\n", is_warn?"WARN":"INFO", msg); } int main(int c, char **v) { struct options { int reverse; int use_getaddrinfo; int servertest; const char *resolv_conf; const char *ns; }; struct options o; char opt; struct event_base *event_base = NULL; struct evdns_base *evdns_base = NULL; memset(&o, 0, sizeof(o)); if (c < 2) { fprintf(stderr, "syntax: %s [-x] [-v] [-c resolv.conf] [-s ns] hostname\n", v[0]); fprintf(stderr, "syntax: %s [-T]\n", v[0]); return 1; } while ((opt = getopt(c, v, "xvc:Ts:")) != -1) { switch (opt) { case 'x': o.reverse = 1; break; case 'v': ++verbose; break; case 'g': o.use_getaddrinfo = 1; break; case 'T': o.servertest = 1; break; case 'c': o.resolv_conf = optarg; break; case 's': o.ns = optarg; break; default : fprintf(stderr, "Unknown option %c\n", opt); break; } } #ifdef _WIN32 { WSADATA WSAData; WSAStartup(0x101, &WSAData); } #endif event_base = event_base_new(); evdns_base = evdns_base_new(event_base, EVDNS_BASE_DISABLE_WHEN_INACTIVE); evdns_set_log_fn(logfn); if (o.servertest) { evutil_socket_t sock; struct sockaddr_in my_addr; sock = socket(PF_INET, SOCK_DGRAM, 0); if (sock == -1) { perror("socket"); exit(1); } evutil_make_socket_nonblocking(sock); my_addr.sin_family = AF_INET; my_addr.sin_port = htons(10053); my_addr.sin_addr.s_addr = INADDR_ANY; if (bind(sock, (struct sockaddr*)&my_addr, sizeof(my_addr))<0) { perror("bind"); exit(1); } evdns_add_server_port_with_base(event_base, sock, 0, evdns_server_callback, NULL); } if (optind < c) { int res; #ifdef _WIN32 if (o.resolv_conf == NULL && !o.ns) res = evdns_base_config_windows_nameservers(evdns_base); else #endif if (o.ns) res = evdns_base_nameserver_ip_add(evdns_base, o.ns); else res = evdns_base_resolv_conf_parse(evdns_base, DNS_OPTION_NAMESERVERS, o.resolv_conf); if (res < 0) { fprintf(stderr, "Couldn't configure nameservers"); return 1; } } printf("EVUTIL_AI_CANONNAME in example = %d\n", EVUTIL_AI_CANONNAME); for (; optind < c; ++optind) { if (o.reverse) { struct in_addr addr; if (evutil_inet_pton(AF_INET, v[optind], &addr)!=1) { fprintf(stderr, "Skipping non-IP %s\n", v[optind]); continue; } fprintf(stderr, "resolving %s...\n",v[optind]); evdns_base_resolve_reverse(evdns_base, &addr, 0, main_callback, v[optind]); } else if (o.use_getaddrinfo) { struct evutil_addrinfo hints; memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_protocol = IPPROTO_TCP; hints.ai_flags = EVUTIL_AI_CANONNAME; fprintf(stderr, "resolving (fwd) %s...\n",v[optind]); evdns_getaddrinfo(evdns_base, v[optind], NULL, &hints, gai_callback, v[optind]); } else { fprintf(stderr, "resolving (fwd) %s...\n",v[optind]); evdns_base_resolve_ipv4(evdns_base, v[optind], 0, main_callback, v[optind]); } } fflush(stdout); event_base_dispatch(event_base); return 0; } kytensor-2.49.0.7/third_party/libevent/sample/event-read-fifo.c000066400000000000000000000065331527052371200244050ustar00rootroot00000000000000/* * This sample code shows how to use Libevent to read from a named pipe. * XXX This code could make better use of the Libevent interfaces. * * XXX This does not work on Windows; ignore everything inside the _WIN32 block. * * On UNIX, compile with: * cc -I/usr/local/include -o event-read-fifo event-read-fifo.c \ * -L/usr/local/lib -levent */ #include #include #include #ifndef _WIN32 #include #include #include #include #else #include #include #endif #include #include #include #include #include #include static void fifo_read(evutil_socket_t fd, short event, void *arg) { char buf[255]; int len; struct event *ev = arg; #ifdef _WIN32 DWORD dwBytesRead; #endif fprintf(stderr, "fifo_read called with fd: %d, event: %d, arg: %p\n", (int)fd, event, arg); #ifdef _WIN32 len = ReadFile((HANDLE)fd, buf, sizeof(buf) - 1, &dwBytesRead, NULL); /* Check for end of file. */ if (len && dwBytesRead == 0) { fprintf(stderr, "End Of File"); event_del(ev); return; } buf[dwBytesRead] = '\0'; #else len = read(fd, buf, sizeof(buf) - 1); if (len <= 0) { if (len == -1) perror("read"); else if (len == 0) fprintf(stderr, "Connection closed\n"); event_del(ev); event_base_loopbreak(event_get_base(ev)); return; } buf[len] = '\0'; #endif fprintf(stdout, "Read: %s\n", buf); } /* On Unix, cleanup event.fifo if SIGINT is received. */ #ifndef _WIN32 static void signal_cb(evutil_socket_t fd, short event, void *arg) { struct event_base *base = arg; event_base_loopbreak(base); } #endif int main(int argc, char **argv) { struct event *evfifo; struct event_base* base; #ifdef _WIN32 HANDLE socket; /* Open a file. */ socket = CreateFileA("test.txt", /* open File */ GENERIC_READ, /* open for reading */ 0, /* do not share */ NULL, /* no security */ OPEN_EXISTING, /* existing file only */ FILE_ATTRIBUTE_NORMAL, /* normal file */ NULL); /* no attr. template */ if (socket == INVALID_HANDLE_VALUE) return 1; #else struct event *signal_int; struct stat st; const char *fifo = "event.fifo"; int socket; if (lstat(fifo, &st) == 0) { if ((st.st_mode & S_IFMT) == S_IFREG) { errno = EEXIST; perror("lstat"); exit(1); } } unlink(fifo); if (mkfifo(fifo, 0600) == -1) { perror("mkfifo"); exit(1); } socket = open(fifo, O_RDONLY | O_NONBLOCK, 0); if (socket == -1) { perror("open"); exit(1); } fprintf(stderr, "Write data to %s\n", fifo); #endif /* Initalize the event library */ base = event_base_new(); /* Initalize one event */ #ifdef _WIN32 evfifo = event_new(base, (evutil_socket_t)socket, EV_READ|EV_PERSIST, fifo_read, event_self_cbarg()); #else /* catch SIGINT so that event.fifo can be cleaned up */ signal_int = evsignal_new(base, SIGINT, signal_cb, base); event_add(signal_int, NULL); evfifo = event_new(base, socket, EV_READ|EV_PERSIST, fifo_read, event_self_cbarg()); #endif /* Add it to the active events, without a timeout */ event_add(evfifo, NULL); event_base_dispatch(base); event_base_free(base); #ifdef _WIN32 CloseHandle(socket); #else close(socket); unlink(fifo); #endif libevent_global_shutdown(); return (0); } kytensor-2.49.0.7/third_party/libevent/sample/hello-world.c000066400000000000000000000066501527052371200236620ustar00rootroot00000000000000/* This example program provides a trivial server program that listens for TCP connections on port 9995. When they arrive, it writes a short message to each client connection, and closes each connection once it is flushed. Where possible, it exits cleanly in response to a SIGINT (ctrl-c). */ #include #include #include #include #ifndef _WIN32 #include # ifdef _XOPEN_SOURCE_EXTENDED # include # endif #include #endif #include #include #include #include #include static const char MESSAGE[] = "Hello, World!\n"; static const int PORT = 9995; static void listener_cb(struct evconnlistener *, evutil_socket_t, struct sockaddr *, int socklen, void *); static void conn_writecb(struct bufferevent *, void *); static void conn_eventcb(struct bufferevent *, short, void *); static void signal_cb(evutil_socket_t, short, void *); int main(int argc, char **argv) { struct event_base *base; struct evconnlistener *listener; struct event *signal_event; struct sockaddr_in sin; #ifdef _WIN32 WSADATA wsa_data; WSAStartup(0x0201, &wsa_data); #endif base = event_base_new(); if (!base) { fprintf(stderr, "Could not initialize libevent!\n"); return 1; } memset(&sin, 0, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_port = htons(PORT); listener = evconnlistener_new_bind(base, listener_cb, (void *)base, LEV_OPT_REUSEABLE|LEV_OPT_CLOSE_ON_FREE, -1, (struct sockaddr*)&sin, sizeof(sin)); if (!listener) { fprintf(stderr, "Could not create a listener!\n"); return 1; } signal_event = evsignal_new(base, SIGINT, signal_cb, (void *)base); if (!signal_event || event_add(signal_event, NULL)<0) { fprintf(stderr, "Could not create/add a signal event!\n"); return 1; } event_base_dispatch(base); evconnlistener_free(listener); event_free(signal_event); event_base_free(base); printf("done\n"); return 0; } static void listener_cb(struct evconnlistener *listener, evutil_socket_t fd, struct sockaddr *sa, int socklen, void *user_data) { struct event_base *base = user_data; struct bufferevent *bev; bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE); if (!bev) { fprintf(stderr, "Error constructing bufferevent!"); event_base_loopbreak(base); return; } bufferevent_setcb(bev, NULL, conn_writecb, conn_eventcb, NULL); bufferevent_enable(bev, EV_WRITE); bufferevent_disable(bev, EV_READ); bufferevent_write(bev, MESSAGE, strlen(MESSAGE)); } static void conn_writecb(struct bufferevent *bev, void *user_data) { struct evbuffer *output = bufferevent_get_output(bev); if (evbuffer_get_length(output) == 0) { printf("flushed answer\n"); bufferevent_free(bev); } } static void conn_eventcb(struct bufferevent *bev, short events, void *user_data) { if (events & BEV_EVENT_EOF) { printf("Connection closed.\n"); } else if (events & BEV_EVENT_ERROR) { printf("Got an error on the connection: %s\n", strerror(errno));/*XXX win32*/ } /* None of the other events can happen here, since we haven't enabled * timeouts */ bufferevent_free(bev); } static void signal_cb(evutil_socket_t sig, short events, void *user_data) { struct event_base *base = user_data; struct timeval delay = { 2, 0 }; printf("Caught an interrupt signal; exiting cleanly in two seconds.\n"); event_base_loopexit(base, &delay); } kytensor-2.49.0.7/third_party/libevent/sample/hostcheck.c000066400000000000000000000147371527052371200234120ustar00rootroot00000000000000/*************************************************************************** * _ _ ____ _ * Project ___| | | | _ \| | * / __| | | | |_) | | * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms * are also available at http://curl.haxx.se/docs/copyright.html. * * You may opt to use, copy, modify, merge, publish, distribute and/or sell * copies of the Software, and permit persons to whom the Software is * furnished to do so, under the terms of the COPYING file. * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY * KIND, either express or implied. * ***************************************************************************/ /* This file is an amalgamation of hostcheck.c and most of rawstr.c from cURL. The contents of the COPYING file mentioned above are: COPYRIGHT AND PERMISSION NOTICE Copyright (c) 1996 - 2013, Daniel Stenberg, . All rights reserved. Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in this Software without prior written authorization of the copyright holder. */ #include "hostcheck.h" #include /* Portable, consistent toupper (remember EBCDIC). Do not use toupper() because its behavior is altered by the current locale. */ static char Curl_raw_toupper(char in) { switch (in) { case 'a': return 'A'; case 'b': return 'B'; case 'c': return 'C'; case 'd': return 'D'; case 'e': return 'E'; case 'f': return 'F'; case 'g': return 'G'; case 'h': return 'H'; case 'i': return 'I'; case 'j': return 'J'; case 'k': return 'K'; case 'l': return 'L'; case 'm': return 'M'; case 'n': return 'N'; case 'o': return 'O'; case 'p': return 'P'; case 'q': return 'Q'; case 'r': return 'R'; case 's': return 'S'; case 't': return 'T'; case 'u': return 'U'; case 'v': return 'V'; case 'w': return 'W'; case 'x': return 'X'; case 'y': return 'Y'; case 'z': return 'Z'; } return in; } /* * Curl_raw_equal() is for doing "raw" case insensitive strings. This is meant * to be locale independent and only compare strings we know are safe for * this. See http://daniel.haxx.se/blog/2008/10/15/strcasecmp-in-turkish/ for * some further explanation to why this function is necessary. * * The function is capable of comparing a-z case insensitively even for * non-ascii. */ static int Curl_raw_equal(const char *first, const char *second) { while(*first && *second) { if(Curl_raw_toupper(*first) != Curl_raw_toupper(*second)) /* get out of the loop as soon as they don't match */ break; first++; second++; } /* we do the comparison here (possibly again), just to make sure that if the loop above is skipped because one of the strings reached zero, we must not return this as a successful match */ return (Curl_raw_toupper(*first) == Curl_raw_toupper(*second)); } static int Curl_raw_nequal(const char *first, const char *second, size_t max) { while(*first && *second && max) { if(Curl_raw_toupper(*first) != Curl_raw_toupper(*second)) { break; } max--; first++; second++; } if(0 == max) return 1; /* they are equal this far */ return Curl_raw_toupper(*first) == Curl_raw_toupper(*second); } /* * Match a hostname against a wildcard pattern. * E.g. * "foo.host.com" matches "*.host.com". * * We use the matching rule described in RFC6125, section 6.4.3. * http://tools.ietf.org/html/rfc6125#section-6.4.3 */ static int hostmatch(const char *hostname, const char *pattern) { const char *pattern_label_end, *pattern_wildcard, *hostname_label_end; int wildcard_enabled; size_t prefixlen, suffixlen; pattern_wildcard = strchr(pattern, '*'); if(pattern_wildcard == NULL) return Curl_raw_equal(pattern, hostname) ? CURL_HOST_MATCH : CURL_HOST_NOMATCH; /* We require at least 2 dots in pattern to avoid too wide wildcard match. */ wildcard_enabled = 1; pattern_label_end = strchr(pattern, '.'); if(pattern_label_end == NULL || strchr(pattern_label_end+1, '.') == NULL || pattern_wildcard > pattern_label_end || Curl_raw_nequal(pattern, "xn--", 4)) { wildcard_enabled = 0; } if(!wildcard_enabled) return Curl_raw_equal(pattern, hostname) ? CURL_HOST_MATCH : CURL_HOST_NOMATCH; hostname_label_end = strchr(hostname, '.'); if(hostname_label_end == NULL || !Curl_raw_equal(pattern_label_end, hostname_label_end)) return CURL_HOST_NOMATCH; /* The wildcard must match at least one character, so the left-most label of the hostname is at least as large as the left-most label of the pattern. */ if(hostname_label_end - hostname < pattern_label_end - pattern) return CURL_HOST_NOMATCH; prefixlen = pattern_wildcard - pattern; suffixlen = pattern_label_end - (pattern_wildcard+1); return Curl_raw_nequal(pattern, hostname, prefixlen) && Curl_raw_nequal(pattern_wildcard+1, hostname_label_end - suffixlen, suffixlen) ? CURL_HOST_MATCH : CURL_HOST_NOMATCH; } int Curl_cert_hostcheck(const char *match_pattern, const char *hostname) { if(!match_pattern || !*match_pattern || !hostname || !*hostname) /* sanity check */ return 0; if(Curl_raw_equal(hostname, match_pattern)) /* trivial case */ return 1; if(hostmatch(hostname,match_pattern) == CURL_HOST_MATCH) return 1; return 0; } kytensor-2.49.0.7/third_party/libevent/sample/hostcheck.h000066400000000000000000000023531527052371200234060ustar00rootroot00000000000000#ifndef HEADER_CURL_HOSTCHECK_H #define HEADER_CURL_HOSTCHECK_H /*************************************************************************** * _ _ ____ _ * Project ___| | | | _ \| | * / __| | | | |_) | | * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms * are also available at http://curl.haxx.se/docs/copyright.html. * * You may opt to use, copy, modify, merge, publish, distribute and/or sell * copies of the Software, and permit persons to whom the Software is * furnished to do so, under the terms of the COPYING file. * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY * KIND, either express or implied. * ***************************************************************************/ #define CURL_HOST_NOMATCH 0 #define CURL_HOST_MATCH 1 int Curl_cert_hostcheck(const char *match_pattern, const char *hostname); #endif /* HEADER_CURL_HOSTCHECK_H */ kytensor-2.49.0.7/third_party/libevent/sample/http-connect.c000066400000000000000000000061101527052371200240270ustar00rootroot00000000000000#include "event2/event-config.h" #include #include #include #include #include #include #include #define VERIFY(cond) do { \ if (!(cond)) { \ fprintf(stderr, "[error] %s\n", #cond); \ } \ } while (0); \ #define URL_MAX 4096 struct connect_base { struct evhttp_connection *evcon; struct evhttp_uri *location; }; static void get_cb(struct evhttp_request *req, void *arg) { ev_ssize_t len; struct evbuffer *evbuf; VERIFY(req); evbuf = evhttp_request_get_input_buffer(req); len = evbuffer_get_length(evbuf); fwrite(evbuffer_pullup(evbuf, len), len, 1, stdout); evbuffer_drain(evbuf, len); } static void connect_cb(struct evhttp_request *proxy_req, void *arg) { char buffer[URL_MAX]; struct connect_base *base = arg; struct evhttp_connection *evcon = base->evcon; struct evhttp_uri *location = base->location; VERIFY(proxy_req); if (evcon) { struct evhttp_request *req = evhttp_request_new(get_cb, NULL); evhttp_add_header(req->output_headers, "Connection", "close"); VERIFY(!evhttp_make_request(evcon, req, EVHTTP_REQ_GET, evhttp_uri_join(location, buffer, URL_MAX))); } } int main(int argc, const char **argv) { char buffer[URL_MAX]; struct evhttp_uri *host_port; struct evhttp_uri *location; struct evhttp_uri *proxy; struct event_base *base; struct evhttp_connection *evcon; struct evhttp_request *req; struct connect_base connect_base; if (argc != 3) { printf("Usage: %s proxy url\n", argv[0]); return 1; } { proxy = evhttp_uri_parse(argv[1]); VERIFY(evhttp_uri_get_host(proxy)); VERIFY(evhttp_uri_get_port(proxy) > 0); } { host_port = evhttp_uri_parse(argv[2]); evhttp_uri_set_scheme(host_port, NULL); evhttp_uri_set_userinfo(host_port, NULL); evhttp_uri_set_path(host_port, NULL); evhttp_uri_set_query(host_port, NULL); evhttp_uri_set_fragment(host_port, NULL); VERIFY(evhttp_uri_get_host(host_port)); VERIFY(evhttp_uri_get_port(host_port) > 0); } { location = evhttp_uri_parse(argv[2]); evhttp_uri_set_scheme(location, NULL); evhttp_uri_set_userinfo(location, 0); evhttp_uri_set_host(location, NULL); evhttp_uri_set_port(location, -1); } VERIFY(base = event_base_new()); VERIFY(evcon = evhttp_connection_base_new(base, NULL, evhttp_uri_get_host(proxy), evhttp_uri_get_port(proxy))); connect_base.evcon = evcon; connect_base.location = location; VERIFY(req = evhttp_request_new(connect_cb, &connect_base)); evhttp_add_header(req->output_headers, "Connection", "keep-alive"); evhttp_add_header(req->output_headers, "Proxy-Connection", "keep-alive"); evutil_snprintf(buffer, URL_MAX, "%s:%d", evhttp_uri_get_host(host_port), evhttp_uri_get_port(host_port)); evhttp_make_request(evcon, req, EVHTTP_REQ_CONNECT, buffer); event_base_dispatch(base); evhttp_connection_free(evcon); event_base_free(base); evhttp_uri_free(proxy); evhttp_uri_free(host_port); evhttp_uri_free(location); return 0; } kytensor-2.49.0.7/third_party/libevent/sample/http-server.c000066400000000000000000000241111527052371200237050ustar00rootroot00000000000000/* A trivial static http webserver using Libevent's evhttp. This is not the best code in the world, and it does some fairly stupid stuff that you would never want to do in a production webserver. Caveat hackor! */ /* Compatibility for possible missing IPv6 declarations */ #include "../util-internal.h" #include #include #include #include #include #ifdef _WIN32 #include #include #include #include #include #ifndef S_ISDIR #define S_ISDIR(x) (((x) & S_IFMT) == S_IFDIR) #endif #else #include #include #include #include #include #include #endif #include #include #include #include #include #ifdef EVENT__HAVE_NETINET_IN_H #include # ifdef _XOPEN_SOURCE_EXTENDED # include # endif #endif #ifdef _WIN32 #ifndef stat #define stat _stat #endif #ifndef fstat #define fstat _fstat #endif #ifndef open #define open _open #endif #ifndef close #define close _close #endif #ifndef O_RDONLY #define O_RDONLY _O_RDONLY #endif #endif char uri_root[512]; static const struct table_entry { const char *extension; const char *content_type; } content_type_table[] = { { "txt", "text/plain" }, { "c", "text/plain" }, { "h", "text/plain" }, { "html", "text/html" }, { "htm", "text/htm" }, { "css", "text/css" }, { "gif", "image/gif" }, { "jpg", "image/jpeg" }, { "jpeg", "image/jpeg" }, { "png", "image/png" }, { "pdf", "application/pdf" }, { "ps", "application/postscript" }, { NULL, NULL }, }; /* Try to guess a good content-type for 'path' */ static const char * guess_content_type(const char *path) { const char *last_period, *extension; const struct table_entry *ent; last_period = strrchr(path, '.'); if (!last_period || strchr(last_period, '/')) goto not_found; /* no exension */ extension = last_period + 1; for (ent = &content_type_table[0]; ent->extension; ++ent) { if (!evutil_ascii_strcasecmp(ent->extension, extension)) return ent->content_type; } not_found: return "application/misc"; } /* Callback used for the /dump URI, and for every non-GET request: * dumps all information to stdout and gives back a trivial 200 ok */ static void dump_request_cb(struct evhttp_request *req, void *arg) { const char *cmdtype; struct evkeyvalq *headers; struct evkeyval *header; struct evbuffer *buf; switch (evhttp_request_get_command(req)) { case EVHTTP_REQ_GET: cmdtype = "GET"; break; case EVHTTP_REQ_POST: cmdtype = "POST"; break; case EVHTTP_REQ_HEAD: cmdtype = "HEAD"; break; case EVHTTP_REQ_PUT: cmdtype = "PUT"; break; case EVHTTP_REQ_DELETE: cmdtype = "DELETE"; break; case EVHTTP_REQ_OPTIONS: cmdtype = "OPTIONS"; break; case EVHTTP_REQ_TRACE: cmdtype = "TRACE"; break; case EVHTTP_REQ_CONNECT: cmdtype = "CONNECT"; break; case EVHTTP_REQ_PATCH: cmdtype = "PATCH"; break; default: cmdtype = "unknown"; break; } printf("Received a %s request for %s\nHeaders:\n", cmdtype, evhttp_request_get_uri(req)); headers = evhttp_request_get_input_headers(req); for (header = headers->tqh_first; header; header = header->next.tqe_next) { printf(" %s: %s\n", header->key, header->value); } buf = evhttp_request_get_input_buffer(req); puts("Input data: <<<"); while (evbuffer_get_length(buf)) { int n; char cbuf[128]; n = evbuffer_remove(buf, cbuf, sizeof(cbuf)); if (n > 0) (void) fwrite(cbuf, 1, n, stdout); } puts(">>>"); evhttp_send_reply(req, 200, "OK", NULL); } /* This callback gets invoked when we get any http request that doesn't match * any other callback. Like any evhttp server callback, it has a simple job: * it must eventually call evhttp_send_error() or evhttp_send_reply(). */ static void send_document_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb = NULL; const char *docroot = arg; const char *uri = evhttp_request_get_uri(req); struct evhttp_uri *decoded = NULL; const char *path; char *decoded_path; char *whole_path = NULL; size_t len; int fd = -1; struct stat st; if (evhttp_request_get_command(req) != EVHTTP_REQ_GET) { dump_request_cb(req, arg); return; } printf("Got a GET request for <%s>\n", uri); /* Decode the URI */ decoded = evhttp_uri_parse(uri); if (!decoded) { printf("It's not a good URI. Sending BADREQUEST\n"); evhttp_send_error(req, HTTP_BADREQUEST, 0); return; } /* Let's see what path the user asked for. */ path = evhttp_uri_get_path(decoded); if (!path) path = "/"; /* We need to decode it, to see what path the user really wanted. */ decoded_path = evhttp_uridecode(path, 0, NULL); if (decoded_path == NULL) goto err; /* Don't allow any ".."s in the path, to avoid exposing stuff outside * of the docroot. This test is both overzealous and underzealous: * it forbids aceptable paths like "/this/one..here", but it doesn't * do anything to prevent symlink following." */ if (strstr(decoded_path, "..")) goto err; len = strlen(decoded_path)+strlen(docroot)+2; if (!(whole_path = malloc(len))) { perror("malloc"); goto err; } evutil_snprintf(whole_path, len, "%s/%s", docroot, decoded_path); if (stat(whole_path, &st)<0) { goto err; } /* This holds the content we're sending. */ evb = evbuffer_new(); if (S_ISDIR(st.st_mode)) { /* If it's a directory, read the comments and make a little * index page */ #ifdef _WIN32 HANDLE d; WIN32_FIND_DATAA ent; char *pattern; size_t dirlen; #else DIR *d; struct dirent *ent; #endif const char *trailing_slash = ""; if (!strlen(path) || path[strlen(path)-1] != '/') trailing_slash = "/"; #ifdef _WIN32 dirlen = strlen(whole_path); pattern = malloc(dirlen+3); memcpy(pattern, whole_path, dirlen); pattern[dirlen] = '\\'; pattern[dirlen+1] = '*'; pattern[dirlen+2] = '\0'; d = FindFirstFileA(pattern, &ent); free(pattern); if (d == INVALID_HANDLE_VALUE) goto err; #else if (!(d = opendir(whole_path))) goto err; #endif evbuffer_add_printf(evb, "\n" "\n \n" " \n" " %s\n" " \n" " \n" " \n" "

%s

\n" "
    \n", decoded_path, /* XXX html-escape this. */ path, /* XXX html-escape this? */ trailing_slash, decoded_path /* XXX html-escape this */); #ifdef _WIN32 do { const char *name = ent.cFileName; #else while ((ent = readdir(d))) { const char *name = ent->d_name; #endif evbuffer_add_printf(evb, "
  • %s\n", name, name);/* XXX escape this */ #ifdef _WIN32 } while (FindNextFileA(d, &ent)); #else } #endif evbuffer_add_printf(evb, "
\n"); #ifdef _WIN32 FindClose(d); #else closedir(d); #endif evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "text/html"); } else { /* Otherwise it's a file; add it to the buffer to get * sent via sendfile */ const char *type = guess_content_type(decoded_path); if ((fd = open(whole_path, O_RDONLY)) < 0) { perror("open"); goto err; } if (fstat(fd, &st)<0) { /* Make sure the length still matches, now that we * opened the file :/ */ perror("fstat"); goto err; } evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", type); evbuffer_add_file(evb, fd, 0, st.st_size); } evhttp_send_reply(req, 200, "OK", evb); goto done; err: evhttp_send_error(req, 404, "Document was not found"); if (fd>=0) close(fd); done: if (decoded) evhttp_uri_free(decoded); if (decoded_path) free(decoded_path); if (whole_path) free(whole_path); if (evb) evbuffer_free(evb); } static void syntax(void) { fprintf(stdout, "Syntax: http-server \n"); } int main(int argc, char **argv) { struct event_base *base; struct evhttp *http; struct evhttp_bound_socket *handle; ev_uint16_t port = 0; #ifdef _WIN32 WSADATA WSAData; WSAStartup(0x101, &WSAData); #else if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) return (1); #endif if (argc < 2) { syntax(); return 1; } base = event_base_new(); if (!base) { fprintf(stderr, "Couldn't create an event_base: exiting\n"); return 1; } /* Create a new evhttp object to handle requests. */ http = evhttp_new(base); if (!http) { fprintf(stderr, "couldn't create evhttp. Exiting.\n"); return 1; } /* The /dump URI will dump all requests to stdout and say 200 ok. */ evhttp_set_cb(http, "/dump", dump_request_cb, NULL); /* We want to accept arbitrary requests, so we need to set a "generic" * cb. We can also add callbacks for specific paths. */ evhttp_set_gencb(http, send_document_cb, argv[1]); /* Now we tell the evhttp what port to listen on */ handle = evhttp_bind_socket_with_handle(http, "0.0.0.0", port); if (!handle) { fprintf(stderr, "couldn't bind to port %d. Exiting.\n", (int)port); return 1; } { /* Extract and display the address we're listening on. */ struct sockaddr_storage ss; evutil_socket_t fd; ev_socklen_t socklen = sizeof(ss); char addrbuf[128]; void *inaddr; const char *addr; int got_port = -1; fd = evhttp_bound_socket_get_fd(handle); memset(&ss, 0, sizeof(ss)); if (getsockname(fd, (struct sockaddr *)&ss, &socklen)) { perror("getsockname() failed"); return 1; } if (ss.ss_family == AF_INET) { got_port = ntohs(((struct sockaddr_in*)&ss)->sin_port); inaddr = &((struct sockaddr_in*)&ss)->sin_addr; } else if (ss.ss_family == AF_INET6) { got_port = ntohs(((struct sockaddr_in6*)&ss)->sin6_port); inaddr = &((struct sockaddr_in6*)&ss)->sin6_addr; } else { fprintf(stderr, "Weird address family %d\n", ss.ss_family); return 1; } addr = evutil_inet_ntop(ss.ss_family, inaddr, addrbuf, sizeof(addrbuf)); if (addr) { printf("Listening on %s:%d\n", addr, got_port); evutil_snprintf(uri_root, sizeof(uri_root), "http://%s:%d",addr,got_port); } else { fprintf(stderr, "evutil_inet_ntop failed\n"); return 1; } } event_base_dispatch(base); return 0; } kytensor-2.49.0.7/third_party/libevent/sample/https-client.c000066400000000000000000000302511527052371200240420ustar00rootroot00000000000000/* This is an example of how to hook up evhttp with bufferevent_ssl It just GETs an https URL given on the command-line and prints the response body to stdout. Actually, it also accepts plain http URLs to make it easy to compare http vs https code paths. Loosely based on le-proxy.c. */ // Get rid of OSX 10.7 and greater deprecation warnings. #if defined(__APPLE__) && defined(__clang__) #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif #include #include #include #include #include #ifdef _WIN32 #include #include #define snprintf _snprintf #define strcasecmp _stricmp #else #include #include #endif #include #include #include #include #include #include #include #include #include #include "openssl_hostname_validation.h" static struct event_base *base; static int ignore_cert = 0; static void http_request_done(struct evhttp_request *req, void *ctx) { char buffer[256]; int nread; if (req == NULL) { /* If req is NULL, it means an error occurred, but * sadly we are mostly left guessing what the error * might have been. We'll do our best... */ struct bufferevent *bev = (struct bufferevent *) ctx; unsigned long oslerr; int printed_err = 0; int errcode = EVUTIL_SOCKET_ERROR(); fprintf(stderr, "some request failed - no idea which one though!\n"); /* Print out the OpenSSL error queue that libevent * squirreled away for us, if any. */ while ((oslerr = bufferevent_get_openssl_error(bev))) { ERR_error_string_n(oslerr, buffer, sizeof(buffer)); fprintf(stderr, "%s\n", buffer); printed_err = 1; } /* If the OpenSSL error queue was empty, maybe it was a * socket error; let's try printing that. */ if (! printed_err) fprintf(stderr, "socket error = %s (%d)\n", evutil_socket_error_to_string(errcode), errcode); return; } fprintf(stderr, "Response line: %d %s\n", evhttp_request_get_response_code(req), evhttp_request_get_response_code_line(req)); while ((nread = evbuffer_remove(evhttp_request_get_input_buffer(req), buffer, sizeof(buffer))) > 0) { /* These are just arbitrary chunks of 256 bytes. * They are not lines, so we can't treat them as such. */ fwrite(buffer, nread, 1, stdout); } } static void syntax(void) { fputs("Syntax:\n", stderr); fputs(" https-client -url [-data data-file.bin] [-ignore-cert] [-retries num] [-timeout sec] [-crt crt]\n", stderr); fputs("Example:\n", stderr); fputs(" https-client -url https://ip.appspot.com/\n", stderr); } static void err(const char *msg) { fputs(msg, stderr); } static void err_openssl(const char *func) { fprintf (stderr, "%s failed:\n", func); /* This is the OpenSSL function that prints the contents of the * error stack to the specified file handle. */ ERR_print_errors_fp (stderr); exit(1); } #ifndef _WIN32 /* See http://archives.seul.org/libevent/users/Jan-2013/msg00039.html */ static int cert_verify_callback(X509_STORE_CTX *x509_ctx, void *arg) { char cert_str[256]; const char *host = (const char *) arg; const char *res_str = "X509_verify_cert failed"; HostnameValidationResult res = Error; /* This is the function that OpenSSL would call if we hadn't called * SSL_CTX_set_cert_verify_callback(). Therefore, we are "wrapping" * the default functionality, rather than replacing it. */ int ok_so_far = 0; X509 *server_cert = NULL; if (ignore_cert) { return 1; } ok_so_far = X509_verify_cert(x509_ctx); server_cert = X509_STORE_CTX_get_current_cert(x509_ctx); if (ok_so_far) { res = validate_hostname(host, server_cert); switch (res) { case MatchFound: res_str = "MatchFound"; break; case MatchNotFound: res_str = "MatchNotFound"; break; case NoSANPresent: res_str = "NoSANPresent"; break; case MalformedCertificate: res_str = "MalformedCertificate"; break; case Error: res_str = "Error"; break; default: res_str = "WTF!"; break; } } X509_NAME_oneline(X509_get_subject_name (server_cert), cert_str, sizeof (cert_str)); if (res == MatchFound) { printf("https server '%s' has this certificate, " "which looks good to me:\n%s\n", host, cert_str); return 1; } else { printf("Got '%s' for hostname '%s' and certificate:\n%s\n", res_str, host, cert_str); return 0; } } #endif int main(int argc, char **argv) { int r; struct evhttp_uri *http_uri = NULL; const char *url = NULL, *data_file = NULL; const char *crt = "/etc/ssl/certs/ca-certificates.crt"; const char *scheme, *host, *path, *query; char uri[256]; int port; int retries = 0; int timeout = -1; SSL_CTX *ssl_ctx = NULL; SSL *ssl = NULL; struct bufferevent *bev; struct evhttp_connection *evcon = NULL; struct evhttp_request *req; struct evkeyvalq *output_headers; struct evbuffer *output_buffer; int i; int ret = 0; enum { HTTP, HTTPS } type = HTTP; for (i = 1; i < argc; i++) { if (!strcmp("-url", argv[i])) { if (i < argc - 1) { url = argv[i + 1]; } else { syntax(); goto error; } } else if (!strcmp("-crt", argv[i])) { if (i < argc - 1) { crt = argv[i + 1]; } else { syntax(); goto error; } } else if (!strcmp("-ignore-cert", argv[i])) { ignore_cert = 1; } else if (!strcmp("-data", argv[i])) { if (i < argc - 1) { data_file = argv[i + 1]; } else { syntax(); goto error; } } else if (!strcmp("-retries", argv[i])) { if (i < argc - 1) { retries = atoi(argv[i + 1]); } else { syntax(); goto error; } } else if (!strcmp("-timeout", argv[i])) { if (i < argc - 1) { timeout = atoi(argv[i + 1]); } else { syntax(); goto error; } } else if (!strcmp("-help", argv[i])) { syntax(); goto error; } } if (!url) { syntax(); goto error; } #ifdef _WIN32 { WORD wVersionRequested; WSADATA wsaData; int err; wVersionRequested = MAKEWORD(2, 2); err = WSAStartup(wVersionRequested, &wsaData); if (err != 0) { printf("WSAStartup failed with error: %d\n", err); goto error; } } #endif // _WIN32 http_uri = evhttp_uri_parse(url); if (http_uri == NULL) { err("malformed url"); goto error; } scheme = evhttp_uri_get_scheme(http_uri); if (scheme == NULL || (strcasecmp(scheme, "https") != 0 && strcasecmp(scheme, "http") != 0)) { err("url must be http or https"); goto error; } host = evhttp_uri_get_host(http_uri); if (host == NULL) { err("url must have a host"); goto error; } port = evhttp_uri_get_port(http_uri); if (port == -1) { port = (strcasecmp(scheme, "http") == 0) ? 80 : 443; } path = evhttp_uri_get_path(http_uri); if (strlen(path) == 0) { path = "/"; } query = evhttp_uri_get_query(http_uri); if (query == NULL) { snprintf(uri, sizeof(uri) - 1, "%s", path); } else { snprintf(uri, sizeof(uri) - 1, "%s?%s", path, query); } uri[sizeof(uri) - 1] = '\0'; #if OPENSSL_VERSION_NUMBER < 0x10100000L // Initialize OpenSSL SSL_library_init(); ERR_load_crypto_strings(); SSL_load_error_strings(); OpenSSL_add_all_algorithms(); #endif /* This isn't strictly necessary... OpenSSL performs RAND_poll * automatically on first use of random number generator. */ r = RAND_poll(); if (r == 0) { err_openssl("RAND_poll"); goto error; } /* Create a new OpenSSL context */ ssl_ctx = SSL_CTX_new(SSLv23_method()); if (!ssl_ctx) { err_openssl("SSL_CTX_new"); goto error; } #ifndef _WIN32 /* TODO: Add certificate loading on Windows as well */ /* Attempt to use the system's trusted root certificates. * (This path is only valid for Debian-based systems.) */ if (1 != SSL_CTX_load_verify_locations(ssl_ctx, crt, NULL)) { err_openssl("SSL_CTX_load_verify_locations"); goto error; } /* Ask OpenSSL to verify the server certificate. Note that this * does NOT include verifying that the hostname is correct. * So, by itself, this means anyone with any legitimate * CA-issued certificate for any website, can impersonate any * other website in the world. This is not good. See "The * Most Dangerous Code in the World" article at * https://crypto.stanford.edu/~dabo/pubs/abstracts/ssl-client-bugs.html */ SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); /* This is how we solve the problem mentioned in the previous * comment. We "wrap" OpenSSL's validation routine in our * own routine, which also validates the hostname by calling * the code provided by iSECPartners. Note that even though * the "Everything You've Always Wanted to Know About * Certificate Validation With OpenSSL (But Were Afraid to * Ask)" paper from iSECPartners says very explicitly not to * call SSL_CTX_set_cert_verify_callback (at the bottom of * page 2), what we're doing here is safe because our * cert_verify_callback() calls X509_verify_cert(), which is * OpenSSL's built-in routine which would have been called if * we hadn't set the callback. Therefore, we're just * "wrapping" OpenSSL's routine, not replacing it. */ SSL_CTX_set_cert_verify_callback(ssl_ctx, cert_verify_callback, (void *) host); #else // _WIN32 (void)crt; #endif // _WIN32 // Create event base base = event_base_new(); if (!base) { perror("event_base_new()"); goto error; } // Create OpenSSL bufferevent and stack evhttp on top of it ssl = SSL_new(ssl_ctx); if (ssl == NULL) { err_openssl("SSL_new()"); goto error; } #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME // Set hostname for SNI extension SSL_set_tlsext_host_name(ssl, host); #endif if (strcasecmp(scheme, "http") == 0) { bev = bufferevent_socket_new(base, -1, BEV_OPT_CLOSE_ON_FREE); } else { type = HTTPS; bev = bufferevent_openssl_socket_new(base, -1, ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_CLOSE_ON_FREE|BEV_OPT_DEFER_CALLBACKS); } if (bev == NULL) { fprintf(stderr, "bufferevent_openssl_socket_new() failed\n"); goto error; } bufferevent_openssl_set_allow_dirty_shutdown(bev, 1); // For simplicity, we let DNS resolution block. Everything else should be // asynchronous though. evcon = evhttp_connection_base_bufferevent_new(base, NULL, bev, host, port); if (evcon == NULL) { fprintf(stderr, "evhttp_connection_base_bufferevent_new() failed\n"); goto error; } if (retries > 0) { evhttp_connection_set_retries(evcon, retries); } if (timeout >= 0) { evhttp_connection_set_timeout(evcon, timeout); } // Fire off the request req = evhttp_request_new(http_request_done, bev); if (req == NULL) { fprintf(stderr, "evhttp_request_new() failed\n"); goto error; } output_headers = evhttp_request_get_output_headers(req); evhttp_add_header(output_headers, "Host", host); evhttp_add_header(output_headers, "Connection", "close"); if (data_file) { /* NOTE: In production code, you'd probably want to use * evbuffer_add_file() or evbuffer_add_file_segment(), to * avoid needless copying. */ FILE * f = fopen(data_file, "rb"); char buf[1024]; size_t s; size_t bytes = 0; if (!f) { syntax(); goto error; } output_buffer = evhttp_request_get_output_buffer(req); while ((s = fread(buf, 1, sizeof(buf), f)) > 0) { evbuffer_add(output_buffer, buf, s); bytes += s; } evutil_snprintf(buf, sizeof(buf)-1, "%lu", (unsigned long)bytes); evhttp_add_header(output_headers, "Content-Length", buf); fclose(f); } r = evhttp_make_request(evcon, req, data_file ? EVHTTP_REQ_POST : EVHTTP_REQ_GET, uri); if (r != 0) { fprintf(stderr, "evhttp_make_request() failed\n"); goto error; } event_base_dispatch(base); goto cleanup; error: ret = 1; cleanup: if (evcon) evhttp_connection_free(evcon); if (http_uri) evhttp_uri_free(http_uri); event_base_free(base); if (ssl_ctx) SSL_CTX_free(ssl_ctx); if (type == HTTP && ssl) SSL_free(ssl); #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_cleanup(); ERR_free_strings(); #ifdef EVENT__HAVE_ERR_REMOVE_THREAD_STATE ERR_remove_thread_state(NULL); #else ERR_remove_state(0); #endif CRYPTO_cleanup_all_ex_data(); sk_SSL_COMP_free(SSL_COMP_get_compression_methods()); #endif /*OPENSSL_VERSION_NUMBER < 0x10100000L */ #ifdef _WIN32 WSACleanup(); #endif return ret; } kytensor-2.49.0.7/third_party/libevent/sample/include.am000066400000000000000000000034351527052371200232260ustar00rootroot00000000000000# sample/include.am for libevent # Copyright 2000-2007 Niels Provos # Copyright 2007-2012 Niels Provos and Nick Mathewson # # See LICENSE for copying information. SAMPLES = \ sample/dns-example \ sample/event-read-fifo \ sample/hello-world \ sample/http-server \ sample/http-connect \ sample/signal-test \ sample/time-test if OPENSSL SAMPLES += sample/le-proxy sample_le_proxy_SOURCES = sample/le-proxy.c sample_le_proxy_LDADD = libevent.la libevent_openssl.la ${OPENSSL_LIBS} ${OPENSSL_LIBADD} sample_le_proxy_INCLUDES = $(OPENSSL_INCS) SAMPLES += sample/https-client sample_https_client_SOURCES = \ sample/https-client.c \ sample/hostcheck.c \ sample/openssl_hostname_validation.c sample_https_client_LDADD = libevent.la libevent_openssl.la ${OPENSSL_LIBS} ${OPENSSL_LIBADD} sample_https_client_INCLUDES = $(OPENSSL_INCS) noinst_HEADERS += \ sample/hostcheck.h \ sample/openssl_hostname_validation.h endif if BUILD_SAMPLES noinst_PROGRAMS += $(SAMPLES) endif $(SAMPLES) : libevent.la sample_event_read_fifo_SOURCES = sample/event-read-fifo.c sample_event_read_fifo_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la sample_time_test_SOURCES = sample/time-test.c sample_time_test_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la sample_signal_test_SOURCES = sample/signal-test.c sample_signal_test_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la sample_dns_example_SOURCES = sample/dns-example.c sample_dns_example_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la sample_hello_world_SOURCES = sample/hello-world.c sample_hello_world_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la sample_http_server_SOURCES = sample/http-server.c sample_http_server_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la sample_http_connect_SOURCES = sample/http-connect.c sample_http_connect_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la kytensor-2.49.0.7/third_party/libevent/sample/le-proxy.c000066400000000000000000000160701527052371200232060ustar00rootroot00000000000000/* This example code shows how to write an (optionally encrypting) SSL proxy with Libevent's bufferevent layer. XXX It's a little ugly and should probably be cleaned up. */ // Get rid of OSX 10.7 and greater deprecation warnings. #if defined(__APPLE__) && defined(__clang__) #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif #include #include #include #include #include #ifdef _WIN32 #include #include #else #include #include #endif #include #include #include #include #include #include #include #include #include "openssl-compat.h" static struct event_base *base; static struct sockaddr_storage listen_on_addr; static struct sockaddr_storage connect_to_addr; static int connect_to_addrlen; static int use_wrapper = 1; static SSL_CTX *ssl_ctx = NULL; #define MAX_OUTPUT (512*1024) static void drained_writecb(struct bufferevent *bev, void *ctx); static void eventcb(struct bufferevent *bev, short what, void *ctx); static void readcb(struct bufferevent *bev, void *ctx) { struct bufferevent *partner = ctx; struct evbuffer *src, *dst; size_t len; src = bufferevent_get_input(bev); len = evbuffer_get_length(src); if (!partner) { evbuffer_drain(src, len); return; } dst = bufferevent_get_output(partner); evbuffer_add_buffer(dst, src); if (evbuffer_get_length(dst) >= MAX_OUTPUT) { /* We're giving the other side data faster than it can * pass it on. Stop reading here until we have drained the * other side to MAX_OUTPUT/2 bytes. */ bufferevent_setcb(partner, readcb, drained_writecb, eventcb, bev); bufferevent_setwatermark(partner, EV_WRITE, MAX_OUTPUT/2, MAX_OUTPUT); bufferevent_disable(bev, EV_READ); } } static void drained_writecb(struct bufferevent *bev, void *ctx) { struct bufferevent *partner = ctx; /* We were choking the other side until we drained our outbuf a bit. * Now it seems drained. */ bufferevent_setcb(bev, readcb, NULL, eventcb, partner); bufferevent_setwatermark(bev, EV_WRITE, 0, 0); if (partner) bufferevent_enable(partner, EV_READ); } static void close_on_finished_writecb(struct bufferevent *bev, void *ctx) { struct evbuffer *b = bufferevent_get_output(bev); if (evbuffer_get_length(b) == 0) { bufferevent_free(bev); } } static void eventcb(struct bufferevent *bev, short what, void *ctx) { struct bufferevent *partner = ctx; if (what & (BEV_EVENT_EOF|BEV_EVENT_ERROR)) { if (what & BEV_EVENT_ERROR) { unsigned long err; while ((err = (bufferevent_get_openssl_error(bev)))) { const char *msg = (const char*) ERR_reason_error_string(err); const char *lib = (const char*) ERR_lib_error_string(err); const char *func = (const char*) ERR_func_error_string(err); fprintf(stderr, "%s in %s %s\n", msg, lib, func); } if (errno) perror("connection error"); } if (partner) { /* Flush all pending data */ readcb(bev, ctx); if (evbuffer_get_length( bufferevent_get_output(partner))) { /* We still have to flush data from the other * side, but when that's done, close the other * side. */ bufferevent_setcb(partner, NULL, close_on_finished_writecb, eventcb, NULL); bufferevent_disable(partner, EV_READ); } else { /* We have nothing left to say to the other * side; close it. */ bufferevent_free(partner); } } bufferevent_free(bev); } } static void syntax(void) { fputs("Syntax:\n", stderr); fputs(" le-proxy [-s] [-W] \n", stderr); fputs("Example:\n", stderr); fputs(" le-proxy 127.0.0.1:8888 1.2.3.4:80\n", stderr); exit(1); } static void accept_cb(struct evconnlistener *listener, evutil_socket_t fd, struct sockaddr *a, int slen, void *p) { struct bufferevent *b_out, *b_in; /* Create two linked bufferevent objects: one to connect, one for the * new connection */ b_in = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE|BEV_OPT_DEFER_CALLBACKS); if (!ssl_ctx || use_wrapper) b_out = bufferevent_socket_new(base, -1, BEV_OPT_CLOSE_ON_FREE|BEV_OPT_DEFER_CALLBACKS); else { SSL *ssl = SSL_new(ssl_ctx); b_out = bufferevent_openssl_socket_new(base, -1, ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_CLOSE_ON_FREE|BEV_OPT_DEFER_CALLBACKS); } assert(b_in && b_out); if (bufferevent_socket_connect(b_out, (struct sockaddr*)&connect_to_addr, connect_to_addrlen)<0) { perror("bufferevent_socket_connect"); bufferevent_free(b_out); bufferevent_free(b_in); return; } if (ssl_ctx && use_wrapper) { struct bufferevent *b_ssl; SSL *ssl = SSL_new(ssl_ctx); b_ssl = bufferevent_openssl_filter_new(base, b_out, ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_CLOSE_ON_FREE|BEV_OPT_DEFER_CALLBACKS); if (!b_ssl) { perror("Bufferevent_openssl_new"); bufferevent_free(b_out); bufferevent_free(b_in); } b_out = b_ssl; } bufferevent_setcb(b_in, readcb, NULL, eventcb, b_out); bufferevent_setcb(b_out, readcb, NULL, eventcb, b_in); bufferevent_enable(b_in, EV_READ|EV_WRITE); bufferevent_enable(b_out, EV_READ|EV_WRITE); } int main(int argc, char **argv) { int i; int socklen; int use_ssl = 0; struct evconnlistener *listener; if (argc < 3) syntax(); for (i=1; i < argc; ++i) { if (!strcmp(argv[i], "-s")) { use_ssl = 1; } else if (!strcmp(argv[i], "-W")) { use_wrapper = 0; } else if (argv[i][0] == '-') { syntax(); } else break; } if (i+2 != argc) syntax(); memset(&listen_on_addr, 0, sizeof(listen_on_addr)); socklen = sizeof(listen_on_addr); if (evutil_parse_sockaddr_port(argv[i], (struct sockaddr*)&listen_on_addr, &socklen)<0) { int p = atoi(argv[i]); struct sockaddr_in *sin = (struct sockaddr_in*)&listen_on_addr; if (p < 1 || p > 65535) syntax(); sin->sin_port = htons(p); sin->sin_addr.s_addr = htonl(0x7f000001); sin->sin_family = AF_INET; socklen = sizeof(struct sockaddr_in); } memset(&connect_to_addr, 0, sizeof(connect_to_addr)); connect_to_addrlen = sizeof(connect_to_addr); if (evutil_parse_sockaddr_port(argv[i+1], (struct sockaddr*)&connect_to_addr, &connect_to_addrlen)<0) syntax(); base = event_base_new(); if (!base) { perror("event_base_new()"); return 1; } if (use_ssl) { int r; #if OPENSSL_VERSION_NUMBER < 0x10100000L SSL_library_init(); ERR_load_crypto_strings(); SSL_load_error_strings(); OpenSSL_add_all_algorithms(); #endif r = RAND_poll(); if (r == 0) { fprintf(stderr, "RAND_poll() failed.\n"); return 1; } ssl_ctx = SSL_CTX_new(TLS_method()); } listener = evconnlistener_new_bind(base, accept_cb, NULL, LEV_OPT_CLOSE_ON_FREE|LEV_OPT_CLOSE_ON_EXEC|LEV_OPT_REUSEABLE, -1, (struct sockaddr*)&listen_on_addr, socklen); if (! listener) { fprintf(stderr, "Couldn't open listener.\n"); event_base_free(base); return 1; } event_base_dispatch(base); evconnlistener_free(listener); event_base_free(base); return 0; } kytensor-2.49.0.7/third_party/libevent/sample/openssl_hostname_validation.c000066400000000000000000000155511527052371200272250ustar00rootroot00000000000000/* Obtained from: https://github.com/iSECPartners/ssl-conservatory */ /* Copyright (C) 2012, iSEC Partners. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* * Helper functions to perform basic hostname validation using OpenSSL. * * Please read "everything-you-wanted-to-know-about-openssl.pdf" before * attempting to use this code. This whitepaper describes how the code works, * how it should be used, and what its limitations are. * * Author: Alban Diquet * License: See LICENSE * */ // Get rid of OSX 10.7 and greater deprecation warnings. #if defined(__APPLE__) && defined(__clang__) #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif #include #include #include #include "openssl_hostname_validation.h" #include "hostcheck.h" #define HOSTNAME_MAX_SIZE 255 #if OPENSSL_VERSION_NUMBER < 0x10100000L #define ASN1_STRING_get0_data ASN1_STRING_data #endif /** * Tries to find a match for hostname in the certificate's Common Name field. * * Returns MatchFound if a match was found. * Returns MatchNotFound if no matches were found. * Returns MalformedCertificate if the Common Name had a NUL character embedded in it. * Returns Error if the Common Name could not be extracted. */ static HostnameValidationResult matches_common_name(const char *hostname, const X509 *server_cert) { int common_name_loc = -1; X509_NAME_ENTRY *common_name_entry = NULL; ASN1_STRING *common_name_asn1 = NULL; const char *common_name_str = NULL; // Find the position of the CN field in the Subject field of the certificate common_name_loc = X509_NAME_get_index_by_NID(X509_get_subject_name((X509 *) server_cert), NID_commonName, -1); if (common_name_loc < 0) { return Error; } // Extract the CN field common_name_entry = X509_NAME_get_entry(X509_get_subject_name((X509 *) server_cert), common_name_loc); if (common_name_entry == NULL) { return Error; } // Convert the CN field to a C string common_name_asn1 = X509_NAME_ENTRY_get_data(common_name_entry); if (common_name_asn1 == NULL) { return Error; } common_name_str = (char *) ASN1_STRING_get0_data(common_name_asn1); // Make sure there isn't an embedded NUL character in the CN if ((size_t)ASN1_STRING_length(common_name_asn1) != strlen(common_name_str)) { return MalformedCertificate; } // Compare expected hostname with the CN if (Curl_cert_hostcheck(common_name_str, hostname) == CURL_HOST_MATCH) { return MatchFound; } else { return MatchNotFound; } } /** * Tries to find a match for hostname in the certificate's Subject Alternative Name extension. * * Returns MatchFound if a match was found. * Returns MatchNotFound if no matches were found. * Returns MalformedCertificate if any of the hostnames had a NUL character embedded in it. * Returns NoSANPresent if the SAN extension was not present in the certificate. */ static HostnameValidationResult matches_subject_alternative_name(const char *hostname, const X509 *server_cert) { HostnameValidationResult result = MatchNotFound; int i; int san_names_nb = -1; STACK_OF(GENERAL_NAME) *san_names = NULL; // Try to extract the names within the SAN extension from the certificate san_names = X509_get_ext_d2i((X509 *) server_cert, NID_subject_alt_name, NULL, NULL); if (san_names == NULL) { return NoSANPresent; } san_names_nb = sk_GENERAL_NAME_num(san_names); // Check each name within the extension for (i=0; itype == GEN_DNS) { // Current name is a DNS name, let's check it const char *dns_name = (char *) ASN1_STRING_get0_data(current_name->d.dNSName); // Make sure there isn't an embedded NUL character in the DNS name if ((size_t)ASN1_STRING_length(current_name->d.dNSName) != strlen(dns_name)) { result = MalformedCertificate; break; } else { // Compare expected hostname with the DNS name if (Curl_cert_hostcheck(dns_name, hostname) == CURL_HOST_MATCH) { result = MatchFound; break; } } } } sk_GENERAL_NAME_pop_free(san_names, GENERAL_NAME_free); return result; } /** * Validates the server's identity by looking for the expected hostname in the * server's certificate. As described in RFC 6125, it first tries to find a match * in the Subject Alternative Name extension. If the extension is not present in * the certificate, it checks the Common Name instead. * * Returns MatchFound if a match was found. * Returns MatchNotFound if no matches were found. * Returns MalformedCertificate if any of the hostnames had a NUL character embedded in it. * Returns Error if there was an error. */ HostnameValidationResult validate_hostname(const char *hostname, const X509 *server_cert) { HostnameValidationResult result; if((hostname == NULL) || (server_cert == NULL)) return Error; // First try the Subject Alternative Names extension result = matches_subject_alternative_name(hostname, server_cert); if (result == NoSANPresent) { // Extension was not found: try the Common Name result = matches_common_name(hostname, server_cert); } return result; } kytensor-2.49.0.7/third_party/libevent/sample/openssl_hostname_validation.h000066400000000000000000000043131527052371200272240ustar00rootroot00000000000000/* Obtained from: https://github.com/iSECPartners/ssl-conservatory */ /* Copyright (C) 2012, iSEC Partners. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* * Helper functions to perform basic hostname validation using OpenSSL. * * Please read "everything-you-wanted-to-know-about-openssl.pdf" before * attempting to use this code. This whitepaper describes how the code works, * how it should be used, and what its limitations are. * * Author: Alban Diquet * License: See LICENSE * */ typedef enum { MatchFound, MatchNotFound, NoSANPresent, MalformedCertificate, Error } HostnameValidationResult; /** * Validates the server's identity by looking for the expected hostname in the * server's certificate. As described in RFC 6125, it first tries to find a match * in the Subject Alternative Name extension. If the extension is not present in * the certificate, it checks the Common Name instead. * * Returns MatchFound if a match was found. * Returns MatchNotFound if no matches were found. * Returns MalformedCertificate if any of the hostnames had a NUL character embedded in it. * Returns Error if there was an error. */ HostnameValidationResult validate_hostname(const char *hostname, const X509 *server_cert); kytensor-2.49.0.7/third_party/libevent/sample/signal-test.c000066400000000000000000000023251527052371200236570ustar00rootroot00000000000000/* * Compile with: * cc -I/usr/local/include -o signal-test \ * signal-test.c -L/usr/local/lib -levent */ #include #include #include #ifndef _WIN32 #include #include #include #else #include #include #endif #include #include #include #include #include #include #include int called = 0; static void signal_cb(evutil_socket_t fd, short event, void *arg) { struct event *signal = arg; printf("signal_cb: got signal %d\n", event_get_signal(signal)); if (called >= 2) event_del(signal); called++; } int main(int argc, char **argv) { struct event *signal_int; struct event_base* base; #ifdef _WIN32 WORD wVersionRequested; WSADATA wsaData; wVersionRequested = MAKEWORD(2, 2); (void) WSAStartup(wVersionRequested, &wsaData); #endif /* Initalize the event library */ base = event_base_new(); /* Initalize one event */ signal_int = evsignal_new(base, SIGINT, signal_cb, event_self_cbarg()); event_add(signal_int, NULL); event_base_dispatch(base); event_free(signal_int); event_base_free(base); return (0); } kytensor-2.49.0.7/third_party/libevent/sample/time-test.c000066400000000000000000000041701527052371200233400ustar00rootroot00000000000000/* * XXX This sample code was once meant to show how to use the basic Libevent * interfaces, but it never worked on non-Unix platforms, and some of the * interfaces have changed since it was first written. It should probably * be removed or replaced with something better. * * Compile with: * cc -I/usr/local/include -o time-test time-test.c -L/usr/local/lib -levent */ #include #include #include #ifndef _WIN32 #include #include #endif #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #include #include #include #include #include #include #include #ifdef _WIN32 #include #endif struct timeval lasttime; int event_is_persistent; static void timeout_cb(evutil_socket_t fd, short event, void *arg) { struct timeval newtime, difference; struct event *timeout = arg; double elapsed; evutil_gettimeofday(&newtime, NULL); evutil_timersub(&newtime, &lasttime, &difference); elapsed = difference.tv_sec + (difference.tv_usec / 1.0e6); printf("timeout_cb called at %d: %.3f seconds elapsed.\n", (int)newtime.tv_sec, elapsed); lasttime = newtime; if (! event_is_persistent) { struct timeval tv; evutil_timerclear(&tv); tv.tv_sec = 2; event_add(timeout, &tv); } } int main(int argc, char **argv) { struct event timeout; struct timeval tv; struct event_base *base; int flags; #ifdef _WIN32 WORD wVersionRequested; WSADATA wsaData; wVersionRequested = MAKEWORD(2, 2); (void)WSAStartup(wVersionRequested, &wsaData); #endif if (argc == 2 && !strcmp(argv[1], "-p")) { event_is_persistent = 1; flags = EV_PERSIST; } else { event_is_persistent = 0; flags = 0; } /* Initalize the event library */ base = event_base_new(); /* Initalize one event */ event_assign(&timeout, base, -1, flags, timeout_cb, (void*) &timeout); evutil_timerclear(&tv); tv.tv_sec = 2; event_add(&timeout, &tv); evutil_gettimeofday(&lasttime, NULL); event_base_dispatch(base); return (0); } kytensor-2.49.0.7/third_party/libevent/select.c000066400000000000000000000210031527052371200214150ustar00rootroot00000000000000/* $OpenBSD: select.c,v 1.2 2002/06/25 15:50:15 mickey Exp $ */ /* * Copyright 2000-2007 Niels Provos * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef EVENT__HAVE_SELECT #ifdef __APPLE__ /* Apple wants us to define this if we might ever pass more than * FD_SETSIZE bits to select(). */ #define _DARWIN_UNLIMITED_SELECT #endif #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #ifdef EVENT__HAVE_SYS_SELECT_H #include #endif #include #include #include #include #include #include #include #include "event-internal.h" #include "evsignal-internal.h" #include "event2/thread.h" #include "evthread-internal.h" #include "log-internal.h" #include "evmap-internal.h" #ifndef EVENT__HAVE_FD_MASK /* This type is mandatory, but Android doesn't define it. */ typedef unsigned long fd_mask; #endif #ifndef NFDBITS #define NFDBITS (sizeof(fd_mask)*8) #endif /* Divide positive x by y, rounding up. */ #define DIV_ROUNDUP(x, y) (((x)+((y)-1))/(y)) /* How many bytes to allocate for N fds? */ #define SELECT_ALLOC_SIZE(n) \ (DIV_ROUNDUP(n, NFDBITS) * sizeof(fd_mask)) struct selectop { int event_fds; /* Highest fd in fd set */ int event_fdsz; int resize_out_sets; fd_set *event_readset_in; fd_set *event_writeset_in; fd_set *event_readset_out; fd_set *event_writeset_out; }; static void *select_init(struct event_base *); static int select_add(struct event_base *, int, short old, short events, void*); static int select_del(struct event_base *, int, short old, short events, void*); static int select_dispatch(struct event_base *, struct timeval *); static void select_dealloc(struct event_base *); const struct eventop selectops = { "select", select_init, select_add, select_del, select_dispatch, select_dealloc, 0, /* doesn't need reinit. */ EV_FEATURE_FDS, 0, }; static int select_resize(struct selectop *sop, int fdsz); static void select_free_selectop(struct selectop *sop); static void * select_init(struct event_base *base) { struct selectop *sop; if (!(sop = mm_calloc(1, sizeof(struct selectop)))) return (NULL); if (select_resize(sop, SELECT_ALLOC_SIZE(32 + 1))) { select_free_selectop(sop); return (NULL); } evsig_init_(base); evutil_weakrand_seed_(&base->weakrand_seed, 0); return (sop); } #ifdef CHECK_INVARIANTS static void check_selectop(struct selectop *sop) { /* nothing to be done here */ } #else #define check_selectop(sop) do { (void) sop; } while (0) #endif static int select_dispatch(struct event_base *base, struct timeval *tv) { int res=0, i, j, nfds; struct selectop *sop = base->evbase; check_selectop(sop); if (sop->resize_out_sets) { fd_set *readset_out=NULL, *writeset_out=NULL; size_t sz = sop->event_fdsz; if (!(readset_out = mm_realloc(sop->event_readset_out, sz))) return (-1); sop->event_readset_out = readset_out; if (!(writeset_out = mm_realloc(sop->event_writeset_out, sz))) { /* We don't free readset_out here, since it was * already successfully reallocated. The next time * we call select_dispatch, the realloc will be a * no-op. */ return (-1); } sop->event_writeset_out = writeset_out; sop->resize_out_sets = 0; } memcpy(sop->event_readset_out, sop->event_readset_in, sop->event_fdsz); memcpy(sop->event_writeset_out, sop->event_writeset_in, sop->event_fdsz); nfds = sop->event_fds+1; EVBASE_RELEASE_LOCK(base, th_base_lock); res = select(nfds, sop->event_readset_out, sop->event_writeset_out, NULL, tv); EVBASE_ACQUIRE_LOCK(base, th_base_lock); check_selectop(sop); if (res == -1) { if (errno != EINTR) { event_warn("select"); return (-1); } return (0); } event_debug(("%s: select reports %d", __func__, res)); check_selectop(sop); i = evutil_weakrand_range_(&base->weakrand_seed, nfds); for (j = 0; j < nfds; ++j) { if (++i >= nfds) i = 0; res = 0; if (FD_ISSET(i, sop->event_readset_out)) res |= EV_READ; if (FD_ISSET(i, sop->event_writeset_out)) res |= EV_WRITE; if (res == 0) continue; evmap_io_active_(base, i, res); } check_selectop(sop); return (0); } static int select_resize(struct selectop *sop, int fdsz) { fd_set *readset_in = NULL; fd_set *writeset_in = NULL; if (sop->event_readset_in) check_selectop(sop); if ((readset_in = mm_realloc(sop->event_readset_in, fdsz)) == NULL) goto error; sop->event_readset_in = readset_in; if ((writeset_in = mm_realloc(sop->event_writeset_in, fdsz)) == NULL) { /* Note that this will leave event_readset_in expanded. * That's okay; we wouldn't want to free it, since that would * change the semantics of select_resize from "expand the * readset_in and writeset_in, or return -1" to "expand the * *set_in members, or trash them and return -1." */ goto error; } sop->event_writeset_in = writeset_in; sop->resize_out_sets = 1; memset((char *)sop->event_readset_in + sop->event_fdsz, 0, fdsz - sop->event_fdsz); memset((char *)sop->event_writeset_in + sop->event_fdsz, 0, fdsz - sop->event_fdsz); sop->event_fdsz = fdsz; check_selectop(sop); return (0); error: event_warn("malloc"); return (-1); } static int select_add(struct event_base *base, int fd, short old, short events, void *p) { struct selectop *sop = base->evbase; (void) p; EVUTIL_ASSERT((events & EV_SIGNAL) == 0); check_selectop(sop); /* * Keep track of the highest fd, so that we can calculate the size * of the fd_sets for select(2) */ if (sop->event_fds < fd) { int fdsz = sop->event_fdsz; if (fdsz < (int)sizeof(fd_mask)) fdsz = (int)sizeof(fd_mask); /* In theory we should worry about overflow here. In * reality, though, the highest fd on a unixy system will * not overflow here. XXXX */ while (fdsz < (int) SELECT_ALLOC_SIZE(fd + 1)) fdsz *= 2; if (fdsz != sop->event_fdsz) { if (select_resize(sop, fdsz)) { check_selectop(sop); return (-1); } } sop->event_fds = fd; } if (events & EV_READ) FD_SET(fd, sop->event_readset_in); if (events & EV_WRITE) FD_SET(fd, sop->event_writeset_in); check_selectop(sop); return (0); } /* * Nothing to be done here. */ static int select_del(struct event_base *base, int fd, short old, short events, void *p) { struct selectop *sop = base->evbase; (void)p; EVUTIL_ASSERT((events & EV_SIGNAL) == 0); check_selectop(sop); if (sop->event_fds < fd) { check_selectop(sop); return (0); } if (events & EV_READ) FD_CLR(fd, sop->event_readset_in); if (events & EV_WRITE) FD_CLR(fd, sop->event_writeset_in); check_selectop(sop); return (0); } static void select_free_selectop(struct selectop *sop) { if (sop->event_readset_in) mm_free(sop->event_readset_in); if (sop->event_writeset_in) mm_free(sop->event_writeset_in); if (sop->event_readset_out) mm_free(sop->event_readset_out); if (sop->event_writeset_out) mm_free(sop->event_writeset_out); memset(sop, 0, sizeof(struct selectop)); mm_free(sop); } static void select_dealloc(struct event_base *base) { evsig_dealloc_(base); select_free_selectop(base->evbase); } #endif /* EVENT__HAVE_SELECT */ kytensor-2.49.0.7/third_party/libevent/signal.c000066400000000000000000000303401527052371200214170ustar00rootroot00000000000000/* $OpenBSD: select.c,v 1.2 2002/06/25 15:50:15 mickey Exp $ */ /* * Copyright 2000-2007 Niels Provos * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include #include #undef WIN32_LEAN_AND_MEAN #endif #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #ifdef EVENT__HAVE_SYS_SOCKET_H #include #endif #include #include #include #include #ifdef EVENT__HAVE_UNISTD_H #include #endif #include #ifdef EVENT__HAVE_FCNTL_H #include #endif #include "event2/event.h" #include "event2/event_struct.h" #include "event-internal.h" #include "event2/util.h" #include "evsignal-internal.h" #include "log-internal.h" #include "evmap-internal.h" #include "evthread-internal.h" /* signal.c This is the signal-handling implementation we use for backends that don't have a better way to do signal handling. It uses sigaction() or signal() to set a signal handler, and a socket pair to tell the event base when Note that I said "the event base" : only one event base can be set up to use this at a time. For historical reasons and backward compatibility, if you add an event for a signal to event_base A, then add an event for a signal (any signal!) to event_base B, event_base B will get informed about the signal, but event_base A won't. It would be neat to change this behavior in some future version of Libevent. kqueue already does something far more sensible. We can make all backends on Linux do a reasonable thing using signalfd. */ #ifndef _WIN32 /* Windows wants us to call our signal handlers as __cdecl. Nobody else * expects you to do anything crazy like this. */ #define __cdecl #endif static int evsig_add(struct event_base *, evutil_socket_t, short, short, void *); static int evsig_del(struct event_base *, evutil_socket_t, short, short, void *); static const struct eventop evsigops = { "signal", NULL, evsig_add, evsig_del, NULL, NULL, 0, 0, 0 }; #ifndef EVENT__DISABLE_THREAD_SUPPORT /* Lock for evsig_base and evsig_base_n_signals_added fields. */ static void *evsig_base_lock = NULL; #endif /* The event base that's currently getting informed about signals. */ static struct event_base *evsig_base = NULL; /* A copy of evsig_base->sigev_n_signals_added. */ static int evsig_base_n_signals_added = 0; static evutil_socket_t evsig_base_fd = -1; static void __cdecl evsig_handler(int sig); #define EVSIGBASE_LOCK() EVLOCK_LOCK(evsig_base_lock, 0) #define EVSIGBASE_UNLOCK() EVLOCK_UNLOCK(evsig_base_lock, 0) void evsig_set_base_(struct event_base *base) { EVSIGBASE_LOCK(); evsig_base = base; evsig_base_n_signals_added = base->sig.ev_n_signals_added; evsig_base_fd = base->sig.ev_signal_pair[1]; EVSIGBASE_UNLOCK(); } /* Callback for when the signal handler write a byte to our signaling socket */ static void evsig_cb(evutil_socket_t fd, short what, void *arg) { static char signals[1024]; ev_ssize_t n; int i; int ncaught[NSIG]; struct event_base *base; base = arg; memset(&ncaught, 0, sizeof(ncaught)); while (1) { #ifdef _WIN32 n = recv(fd, signals, sizeof(signals), 0); #else n = read(fd, signals, sizeof(signals)); #endif if (n == -1) { int err = evutil_socket_geterror(fd); if (! EVUTIL_ERR_RW_RETRIABLE(err)) event_sock_err(1, fd, "%s: recv", __func__); break; } else if (n == 0) { /* XXX warn? */ break; } for (i = 0; i < n; ++i) { ev_uint8_t sig = signals[i]; if (sig < NSIG) ncaught[sig]++; } } EVBASE_ACQUIRE_LOCK(base, th_base_lock); for (i = 0; i < NSIG; ++i) { if (ncaught[i]) evmap_signal_active_(base, i, ncaught[i]); } EVBASE_RELEASE_LOCK(base, th_base_lock); } int evsig_init_(struct event_base *base) { /* * Our signal handler is going to write to one end of the socket * pair to wake up our event loop. The event loop then scans for * signals that got delivered. */ if (evutil_make_internal_pipe_(base->sig.ev_signal_pair) == -1) { #ifdef _WIN32 /* Make this nonfatal on win32, where sometimes people have localhost firewalled. */ event_sock_warn(-1, "%s: socketpair", __func__); #else event_sock_err(1, -1, "%s: socketpair", __func__); #endif return -1; } if (base->sig.sh_old) { mm_free(base->sig.sh_old); } base->sig.sh_old = NULL; base->sig.sh_old_max = 0; event_assign(&base->sig.ev_signal, base, base->sig.ev_signal_pair[0], EV_READ | EV_PERSIST, evsig_cb, base); base->sig.ev_signal.ev_flags |= EVLIST_INTERNAL; event_priority_set(&base->sig.ev_signal, 0); base->evsigsel = &evsigops; return 0; } /* Helper: set the signal handler for evsignal to handler in base, so that * we can restore the original handler when we clear the current one. */ int evsig_set_handler_(struct event_base *base, int evsignal, void (__cdecl *handler)(int)) { #ifdef EVENT__HAVE_SIGACTION struct sigaction sa; #else ev_sighandler_t sh; #endif struct evsig_info *sig = &base->sig; void *p; /* * resize saved signal handler array up to the highest signal number. * a dynamic array is used to keep footprint on the low side. */ if (evsignal >= sig->sh_old_max) { int new_max = evsignal + 1; event_debug(("%s: evsignal (%d) >= sh_old_max (%d), resizing", __func__, evsignal, sig->sh_old_max)); p = mm_realloc(sig->sh_old, new_max * sizeof(*sig->sh_old)); if (p == NULL) { event_warn("realloc"); return (-1); } memset((char *)p + sig->sh_old_max * sizeof(*sig->sh_old), 0, (new_max - sig->sh_old_max) * sizeof(*sig->sh_old)); sig->sh_old_max = new_max; sig->sh_old = p; } /* allocate space for previous handler out of dynamic array */ sig->sh_old[evsignal] = mm_malloc(sizeof *sig->sh_old[evsignal]); if (sig->sh_old[evsignal] == NULL) { event_warn("malloc"); return (-1); } /* save previous handler and setup new handler */ #ifdef EVENT__HAVE_SIGACTION memset(&sa, 0, sizeof(sa)); sa.sa_handler = handler; sa.sa_flags |= SA_RESTART; sigfillset(&sa.sa_mask); if (sigaction(evsignal, &sa, sig->sh_old[evsignal]) == -1) { event_warn("sigaction"); mm_free(sig->sh_old[evsignal]); sig->sh_old[evsignal] = NULL; return (-1); } #else if ((sh = signal(evsignal, handler)) == SIG_ERR) { event_warn("signal"); mm_free(sig->sh_old[evsignal]); sig->sh_old[evsignal] = NULL; return (-1); } *sig->sh_old[evsignal] = sh; #endif return (0); } static int evsig_add(struct event_base *base, evutil_socket_t evsignal, short old, short events, void *p) { struct evsig_info *sig = &base->sig; (void)p; EVUTIL_ASSERT(evsignal >= 0 && evsignal < NSIG); /* catch signals if they happen quickly */ EVSIGBASE_LOCK(); if (evsig_base != base && evsig_base_n_signals_added) { event_warnx("Added a signal to event base %p with signals " "already added to event_base %p. Only one can have " "signals at a time with the %s backend. The base with " "the most recently added signal or the most recent " "event_base_loop() call gets preference; do " "not rely on this behavior in future Libevent versions.", base, evsig_base, base->evsel->name); } evsig_base = base; evsig_base_n_signals_added = ++sig->ev_n_signals_added; evsig_base_fd = base->sig.ev_signal_pair[1]; EVSIGBASE_UNLOCK(); event_debug(("%s: %d: changing signal handler", __func__, (int)evsignal)); if (evsig_set_handler_(base, (int)evsignal, evsig_handler) == -1) { goto err; } if (!sig->ev_signal_added) { if (event_add_nolock_(&sig->ev_signal, NULL, 0)) goto err; sig->ev_signal_added = 1; } return (0); err: EVSIGBASE_LOCK(); --evsig_base_n_signals_added; --sig->ev_n_signals_added; EVSIGBASE_UNLOCK(); return (-1); } int evsig_restore_handler_(struct event_base *base, int evsignal) { int ret = 0; struct evsig_info *sig = &base->sig; #ifdef EVENT__HAVE_SIGACTION struct sigaction *sh; #else ev_sighandler_t *sh; #endif if (evsignal >= sig->sh_old_max) { /* Can't actually restore. */ /* XXXX.*/ return 0; } /* restore previous handler */ sh = sig->sh_old[evsignal]; sig->sh_old[evsignal] = NULL; #ifdef EVENT__HAVE_SIGACTION if (sigaction(evsignal, sh, NULL) == -1) { event_warn("sigaction"); ret = -1; } #else if (signal(evsignal, *sh) == SIG_ERR) { event_warn("signal"); ret = -1; } #endif mm_free(sh); return ret; } static int evsig_del(struct event_base *base, evutil_socket_t evsignal, short old, short events, void *p) { EVUTIL_ASSERT(evsignal >= 0 && evsignal < NSIG); event_debug(("%s: "EV_SOCK_FMT": restoring signal handler", __func__, EV_SOCK_ARG(evsignal))); EVSIGBASE_LOCK(); --evsig_base_n_signals_added; --base->sig.ev_n_signals_added; EVSIGBASE_UNLOCK(); return (evsig_restore_handler_(base, (int)evsignal)); } static void __cdecl evsig_handler(int sig) { int save_errno = errno; #ifdef _WIN32 int socket_errno = EVUTIL_SOCKET_ERROR(); #endif ev_uint8_t msg; if (evsig_base == NULL) { event_warnx( "%s: received signal %d, but have no base configured", __func__, sig); return; } #ifndef EVENT__HAVE_SIGACTION signal(sig, evsig_handler); #endif /* Wake up our notification mechanism */ msg = sig; #ifdef _WIN32 send(evsig_base_fd, (char*)&msg, 1, 0); #else { int r = write(evsig_base_fd, (char*)&msg, 1); (void)r; /* Suppress 'unused return value' and 'unused var' */ } #endif errno = save_errno; #ifdef _WIN32 EVUTIL_SET_SOCKET_ERROR(socket_errno); #endif } void evsig_dealloc_(struct event_base *base) { int i = 0; if (base->sig.ev_signal_added) { event_del(&base->sig.ev_signal); base->sig.ev_signal_added = 0; } /* debug event is created in evsig_init_/event_assign even when * ev_signal_added == 0, so unassign is required */ event_debug_unassign(&base->sig.ev_signal); for (i = 0; i < NSIG; ++i) { if (i < base->sig.sh_old_max && base->sig.sh_old[i] != NULL) evsig_restore_handler_(base, i); } EVSIGBASE_LOCK(); if (base == evsig_base) { evsig_base = NULL; evsig_base_n_signals_added = 0; evsig_base_fd = -1; } EVSIGBASE_UNLOCK(); if (base->sig.ev_signal_pair[0] != -1) { evutil_closesocket(base->sig.ev_signal_pair[0]); base->sig.ev_signal_pair[0] = -1; } if (base->sig.ev_signal_pair[1] != -1) { evutil_closesocket(base->sig.ev_signal_pair[1]); base->sig.ev_signal_pair[1] = -1; } base->sig.sh_old_max = 0; /* per index frees are handled in evsig_del() */ if (base->sig.sh_old) { mm_free(base->sig.sh_old); base->sig.sh_old = NULL; } } static void evsig_free_globals_locks(void) { #ifndef EVENT__DISABLE_THREAD_SUPPORT if (evsig_base_lock != NULL) { EVTHREAD_FREE_LOCK(evsig_base_lock, 0); evsig_base_lock = NULL; } #endif return; } void evsig_free_globals_(void) { evsig_free_globals_locks(); } #ifndef EVENT__DISABLE_THREAD_SUPPORT int evsig_global_setup_locks_(const int enable_locks) { EVTHREAD_SETUP_GLOBAL_LOCK(evsig_base_lock, 0); return 0; } #endif kytensor-2.49.0.7/third_party/libevent/strlcpy-internal.h000066400000000000000000000005561527052371200234670ustar00rootroot00000000000000#ifndef STRLCPY_INTERNAL_H_INCLUDED_ #define STRLCPY_INTERNAL_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif #include "event2/event-config.h" #include "evconfig-private.h" #ifndef EVENT__HAVE_STRLCPY #include size_t event_strlcpy_(char *dst, const char *src, size_t siz); #define strlcpy event_strlcpy_ #endif #ifdef __cplusplus } #endif #endif kytensor-2.49.0.7/third_party/libevent/strlcpy.c000066400000000000000000000050261527052371200216450ustar00rootroot00000000000000/* $OpenBSD: strlcpy.c,v 1.5 2001/05/13 15:40:16 deraadt Exp $ */ /* * Copyright (c) 1998 Todd C. Miller * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #if defined(LIBC_SCCS) && !defined(lint) static char *rcsid = "$OpenBSD: strlcpy.c,v 1.5 2001/05/13 15:40:16 deraadt Exp $"; #endif /* LIBC_SCCS and not lint */ #include "event2/event-config.h" #include "evconfig-private.h" #include #ifndef EVENT__HAVE_STRLCPY #include "strlcpy-internal.h" /* * Copy src to string dst of size siz. At most siz-1 characters * will be copied. Always NUL terminates (unless siz == 0). * Returns strlen(src); if retval >= siz, truncation occurred. */ size_t event_strlcpy_(dst, src, siz) char *dst; const char *src; size_t siz; { register char *d = dst; register const char *s = src; register size_t n = siz; /* Copy as many bytes as will fit */ if (n != 0 && --n != 0) { do { if ((*d++ = *s++) == 0) break; } while (--n != 0); } /* Not enough room in dst, add NUL and traverse rest of src */ if (n == 0) { if (siz != 0) *d = '\0'; /* NUL-terminate dst */ while (*s++) ; } return (s - src - 1); /* count does not include NUL */ } #endif kytensor-2.49.0.7/third_party/libevent/test/000077500000000000000000000000001527052371200207555ustar00rootroot00000000000000kytensor-2.49.0.7/third_party/libevent/test/Makefile.nmake000066400000000000000000000047411527052371200235150ustar00rootroot00000000000000# WATCH OUT! This makefile is a work in progress. -*- makefile -*- !IFDEF OPENSSL_DIR SSL_CFLAGS=/I$(OPENSSL_DIR)\include /DEVENT__HAVE_OPENSSL SSL_OBJS=regress_ssl.obj SSL_LIBS=..\libevent_openssl.lib $(OPENSSL_DIR)\lib\libeay32.lib $(OPENSSL_DIR)\lib\ssleay32.lib gdi32.lib User32.lib !ELSE SSL_CFLAGS= SSL_OBJS= SSL_LIBS= !ENDIF CFLAGS=/I.. /I../WIN32-Code /I../WIN32-Code/nmake /I../include /I../compat /DHAVE_CONFIG_H /DTINYTEST_LOCAL $(SSL_CFLAGS) CFLAGS=$(CFLAGS) /Ox /W3 /wd4996 /nologo REGRESS_OBJS=regress.obj regress_buffer.obj regress_http.obj regress_dns.obj \ regress_testutils.obj \ regress_rpc.obj regress.gen.obj \ regress_et.obj regress_bufferevent.obj \ regress_listener.obj regress_util.obj tinytest.obj \ regress_main.obj regress_minheap.obj regress_iocp.obj \ regress_thread.obj regress_finalize.obj $(SSL_OBJS) OTHER_OBJS=test-init.obj test-eof.obj test-closed.obj test-weof.obj test-time.obj \ bench.obj bench_cascade.obj bench_http.obj bench_httpclient.obj \ test-changelist.obj \ print-winsock-errors.obj PROGRAMS=regress.exe \ test-init.exe test-eof.exe test-closed.exe test-weof.exe test-time.exe \ test-changelist.exe \ print-winsock-errors.exe # Disabled for now: # bench.exe bench_cascade.exe bench_http.exe bench_httpclient.exe LIBS=..\libevent.lib ws2_32.lib shell32.lib advapi32.lib all: $(PROGRAMS) regress.exe: $(REGRESS_OBJS) $(CC) $(CFLAGS) $(LIBS) $(SSL_LIBS) $(REGRESS_OBJS) test-init.exe: test-init.obj $(CC) $(CFLAGS) $(LIBS) test-init.obj test-eof.exe: test-eof.obj $(CC) $(CFLAGS) $(LIBS) test-eof.obj test-closed.exe: test-closed.obj $(CC) $(CFLAGS) $(LIBS) test-closed.obj test-changelist.exe: test-changelist.obj $(CC) $(CFLAGS) $(LIBS) test-changelist.obj test-weof.exe: test-weof.obj $(CC) $(CFLAGS) $(LIBS) test-weof.obj test-time.exe: test-time.obj $(CC) $(CFLAGS) $(LIBS) test-time.obj print-winsock-errors.exe: print-winsock-errors.obj $(CC) $(CFLAGS) $(LIBS) print-winsock-errors.obj bench.exe: bench.obj $(CC) $(CFLAGS) $(LIBS) bench.obj bench_cascade.exe: bench_cascade.obj $(CC) $(CFLAGS) $(LIBS) bench_cascade.obj bench_http.exe: bench_http.obj $(CC) $(CFLAGS) $(LIBS) bench_http.obj bench_httpclient.exe: bench_httpclient.obj $(CC) $(CFLAGS) $(LIBS) bench_httpclient.obj regress.gen.c regress.gen.h: regress.rpc ../event_rpcgen.py echo // > regress.gen.c echo #define NO_PYTHON_EXISTS > regress.gen.h -python ..\event_rpcgen.py regress.rpc clean: -del $(REGRESS_OBJS) -del $(OTHER_OBJS) -del $(PROGRAMS) kytensor-2.49.0.7/third_party/libevent/test/bench.c000066400000000000000000000116341527052371200222050ustar00rootroot00000000000000/* * Copyright 2003-2007 Niels Provos * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * * Mon 03/10/2003 - Modified by Davide Libenzi * * Added chain event propagation to improve the sensitivity of * the measure respect to the event loop efficency. * * */ #include "event2/event-config.h" #include #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include #else #include #include #include #endif #include #include #include #include #ifdef EVENT__HAVE_UNISTD_H #include #endif #include #ifdef _WIN32 #include #endif #include #include static int count, writes, fired, failures; static evutil_socket_t *pipes; static int num_pipes, num_active, num_writes; static struct event *events; static void read_cb(evutil_socket_t fd, short which, void *arg) { ev_intptr_t idx = (ev_intptr_t) arg, widx = idx + 1; unsigned char ch; ev_ssize_t n; n = recv(fd, (char*)&ch, sizeof(ch), 0); if (n >= 0) count += n; else failures++; if (writes) { if (widx >= num_pipes) widx -= num_pipes; n = send(pipes[2 * widx + 1], "e", 1, 0); if (n != 1) failures++; writes--; fired++; } } static struct timeval * run_once(void) { evutil_socket_t *cp, space; long i; static struct timeval ts, te; for (cp = pipes, i = 0; i < num_pipes; i++, cp += 2) { if (event_initialized(&events[i])) event_del(&events[i]); event_set(&events[i], cp[0], EV_READ | EV_PERSIST, read_cb, (void *)(ev_intptr_t) i); event_add(&events[i], NULL); } event_loop(EVLOOP_ONCE | EVLOOP_NONBLOCK); fired = 0; space = num_pipes / num_active; space = space * 2; for (i = 0; i < num_active; i++, fired++) (void) send(pipes[i * space + 1], "e", 1, 0); count = 0; writes = num_writes; { int xcount = 0; evutil_gettimeofday(&ts, NULL); do { event_loop(EVLOOP_ONCE | EVLOOP_NONBLOCK); xcount++; } while (count != fired); evutil_gettimeofday(&te, NULL); if (xcount != count) fprintf(stderr, "Xcount: %d, Rcount: %d\n", xcount, count); } evutil_timersub(&te, &ts, &te); return (&te); } int main(int argc, char **argv) { #ifdef HAVE_SETRLIMIT struct rlimit rl; #endif int i, c; struct timeval *tv; evutil_socket_t *cp; #ifdef _WIN32 WSADATA WSAData; WSAStartup(0x101, &WSAData); #endif num_pipes = 100; num_active = 1; num_writes = num_pipes; while ((c = getopt(argc, argv, "n:a:w:")) != -1) { switch (c) { case 'n': num_pipes = atoi(optarg); break; case 'a': num_active = atoi(optarg); break; case 'w': num_writes = atoi(optarg); break; default: fprintf(stderr, "Illegal argument \"%c\"\n", c); exit(1); } } #ifdef HAVE_SETRLIMIT rl.rlim_cur = rl.rlim_max = num_pipes * 2 + 50; if (setrlimit(RLIMIT_NOFILE, &rl) == -1) { perror("setrlimit"); exit(1); } #endif events = calloc(num_pipes, sizeof(struct event)); pipes = calloc(num_pipes * 2, sizeof(evutil_socket_t)); if (events == NULL || pipes == NULL) { perror("malloc"); exit(1); } event_init(); for (cp = pipes, i = 0; i < num_pipes; i++, cp += 2) { #ifdef USE_PIPES if (pipe(cp) == -1) { #else if (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, cp) == -1) { #endif perror("pipe"); exit(1); } } for (i = 0; i < 25; i++) { tv = run_once(); if (tv == NULL) exit(1); fprintf(stdout, "%ld\n", tv->tv_sec * 1000000L + tv->tv_usec); } exit(0); } kytensor-2.49.0.7/third_party/libevent/test/bench_cascade.c000066400000000000000000000110741527052371200236460ustar00rootroot00000000000000/* * Copyright 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #include "event2/event-config.h" #include #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include #else #include #include #endif #include #include #include #include #include #ifdef EVENT__HAVE_UNISTD_H #include #endif #include #include #include #include /* * This benchmark tests how quickly we can propagate a write down a chain * of socket pairs. We start by writing to the first socket pair and all * events will fire subsequently until the last socket pair has been reached * and the benchmark terminates. */ static int fired; static evutil_socket_t *pipes; static struct event *events; static void read_cb(evutil_socket_t fd, short which, void *arg) { char ch; evutil_socket_t sock = (evutil_socket_t)(ev_intptr_t)arg; (void) recv(fd, &ch, sizeof(ch), 0); if (sock >= 0) { if (send(sock, "e", 1, 0) < 0) perror("send"); } fired++; } static struct timeval * run_once(int num_pipes) { int i; evutil_socket_t *cp; static struct timeval ts, te, tv_timeout; events = (struct event *)calloc(num_pipes, sizeof(struct event)); pipes = (evutil_socket_t *)calloc(num_pipes * 2, sizeof(evutil_socket_t)); if (events == NULL || pipes == NULL) { perror("malloc"); exit(1); } for (cp = pipes, i = 0; i < num_pipes; i++, cp += 2) { if (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, cp) == -1) { perror("socketpair"); exit(1); } } /* measurements includes event setup */ evutil_gettimeofday(&ts, NULL); /* provide a default timeout for events */ evutil_timerclear(&tv_timeout); tv_timeout.tv_sec = 60; for (cp = pipes, i = 0; i < num_pipes; i++, cp += 2) { evutil_socket_t fd = i < num_pipes - 1 ? cp[3] : -1; event_set(&events[i], cp[0], EV_READ, read_cb, (void *)(ev_intptr_t)fd); event_add(&events[i], &tv_timeout); } fired = 0; /* kick everything off with a single write */ if (send(pipes[1], "e", 1, 0) < 0) perror("send"); event_dispatch(); evutil_gettimeofday(&te, NULL); evutil_timersub(&te, &ts, &te); for (cp = pipes, i = 0; i < num_pipes; i++, cp += 2) { event_del(&events[i]); evutil_closesocket(cp[0]); evutil_closesocket(cp[1]); } free(pipes); free(events); return (&te); } int main(int argc, char **argv) { #ifdef HAVE_SETRLIMIT struct rlimit rl; #endif int i, c; struct timeval *tv; int num_pipes = 100; #ifdef _WIN32 WSADATA WSAData; WSAStartup(0x101, &WSAData); #endif while ((c = getopt(argc, argv, "n:")) != -1) { switch (c) { case 'n': num_pipes = atoi(optarg); break; default: fprintf(stderr, "Illegal argument \"%c\"\n", c); exit(1); } } #ifdef HAVE_SETRLIMIT rl.rlim_cur = rl.rlim_max = num_pipes * 2 + 50; if (setrlimit(RLIMIT_NOFILE, &rl) == -1) { perror("setrlimit"); exit(1); } #endif event_init(); for (i = 0; i < 25; i++) { tv = run_once(num_pipes); if (tv == NULL) exit(1); fprintf(stdout, "%ld\n", tv->tv_sec * 1000000L + tv->tv_usec); } #ifdef _WIN32 WSACleanup(); #endif exit(0); } kytensor-2.49.0.7/third_party/libevent/test/bench_http.c000066400000000000000000000116271527052371200232460ustar00rootroot00000000000000/* * Copyright 2008-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #include #include #ifdef _WIN32 #include #else #include #include #include #include #endif #include #include #include #include #include #include #include "event2/event.h" #include "event2/buffer.h" #include "event2/util.h" #include "event2/http.h" #include "event2/thread.h" static void http_basic_cb(struct evhttp_request *req, void *arg); static char *content; static size_t content_len = 0; static void http_basic_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb = evbuffer_new(); evbuffer_add(evb, content, content_len); /* allow sending of an empty reply */ evhttp_send_reply(req, HTTP_OK, "Everything is fine", evb); evbuffer_free(evb); } #if LIBEVENT_VERSION_NUMBER >= 0x02000200 static void http_ref_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb = evbuffer_new(); evbuffer_add_reference(evb, content, content_len, NULL, NULL); /* allow sending of an empty reply */ evhttp_send_reply(req, HTTP_OK, "Everything is fine", evb); evbuffer_free(evb); } #endif int main(int argc, char **argv) { struct event_config *cfg = event_config_new(); struct event_base *base; struct evhttp *http; int i; int c; int use_iocp = 0; ev_uint16_t port = 8080; char *endptr = NULL; #ifdef _WIN32 WSADATA WSAData; WSAStartup(0x101, &WSAData); #else if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) return (1); #endif for (i = 1; i < argc; ++i) { if (*argv[i] != '-') continue; c = argv[i][1]; if ((c == 'p' || c == 'l') && i + 1 >= argc) { fprintf(stderr, "-%c requires argument.\n", c); exit(1); } switch (c) { case 'p': if (i+1 >= argc || !argv[i+1]) { fprintf(stderr, "Missing port\n"); exit(1); } port = (int)strtol(argv[i+1], &endptr, 10); if (*endptr != '\0') { fprintf(stderr, "Bad port\n"); exit(1); } break; case 'l': if (i+1 >= argc || !argv[i+1]) { fprintf(stderr, "Missing content length\n"); exit(1); } content_len = (size_t)strtol(argv[i+1], &endptr, 10); if (*endptr != '\0' || content_len == 0) { fprintf(stderr, "Bad content length\n"); exit(1); } break; #ifdef _WIN32 case 'i': use_iocp = 1; #ifdef EVTHREAD_USE_WINDOWS_THREADS_IMPLEMENTED evthread_use_windows_threads(); #endif event_config_set_flag(cfg,EVENT_BASE_FLAG_STARTUP_IOCP); break; #endif default: fprintf(stderr, "Illegal argument \"%c\"\n", c); exit(1); } } base = event_base_new_with_config(cfg); if (!base) { fprintf(stderr, "creating event_base failed. Exiting.\n"); return 1; } http = evhttp_new(base); content = malloc(content_len); if (content == NULL) { fprintf(stderr, "Cannot allocate content\n"); exit(1); } else { int i = 0; for (i = 0; i < (int)content_len; ++i) content[i] = (i & 255); } evhttp_set_cb(http, "/ind", http_basic_cb, NULL); fprintf(stderr, "/ind - basic content (memory copy)\n"); evhttp_set_cb(http, "/ref", http_ref_cb, NULL); fprintf(stderr, "/ref - basic content (reference)\n"); fprintf(stderr, "Serving %d bytes on port %d using %s\n", (int)content_len, port, use_iocp? "IOCP" : event_base_get_method(base)); evhttp_bind_socket(http, "0.0.0.0", port); #ifdef _WIN32 if (use_iocp) { struct timeval tv={99999999,0}; event_base_loopexit(base, &tv); } #endif event_base_dispatch(base); #ifdef _WIN32 WSACleanup(); #endif /* NOTREACHED */ return (0); } kytensor-2.49.0.7/third_party/libevent/test/bench_httpclient.c000066400000000000000000000135761527052371200244520ustar00rootroot00000000000000/* * Copyright 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ /* for EVUTIL_ERR_CONNECT_RETRIABLE macro */ #include "util-internal.h" #include #ifdef _WIN32 #include #else #include #include # ifdef _XOPEN_SOURCE_EXTENDED # include # endif #endif #include #include #include #include "event2/event.h" #include "event2/bufferevent.h" #include "event2/buffer.h" #include "event2/util.h" const char *resource = NULL; struct event_base *base = NULL; int total_n_handled = 0; int total_n_errors = 0; int total_n_launched = 0; size_t total_n_bytes = 0; struct timeval total_time = {0,0}; int n_errors = 0; const int PARALLELISM = 200; const int N_REQUESTS = 20000; struct request_info { size_t n_read; struct timeval started; }; static int launch_request(void); static void readcb(struct bufferevent *b, void *arg); static void errorcb(struct bufferevent *b, short what, void *arg); static void readcb(struct bufferevent *b, void *arg) { struct request_info *ri = arg; struct evbuffer *input = bufferevent_get_input(b); size_t n = evbuffer_get_length(input); ri->n_read += n; evbuffer_drain(input, n); } static void errorcb(struct bufferevent *b, short what, void *arg) { struct request_info *ri = arg; struct timeval now, diff; if (what & BEV_EVENT_EOF) { ++total_n_handled; total_n_bytes += ri->n_read; evutil_gettimeofday(&now, NULL); evutil_timersub(&now, &ri->started, &diff); evutil_timeradd(&diff, &total_time, &total_time); if (total_n_handled && (total_n_handled%1000)==0) printf("%d requests done\n",total_n_handled); if (total_n_launched < N_REQUESTS) { if (launch_request() < 0) perror("Can't launch"); } } else { ++total_n_errors; perror("Unexpected error"); } bufferevent_setcb(b, NULL, NULL, NULL, NULL); free(ri); bufferevent_disable(b, EV_READ|EV_WRITE); bufferevent_free(b); } static void frob_socket(evutil_socket_t sock) { #ifdef HAVE_SO_LINGER struct linger l; #endif int one = 1; if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&one, sizeof(one))<0) perror("setsockopt(SO_REUSEADDR)"); #ifdef HAVE_SO_LINGER l.l_onoff = 1; l.l_linger = 0; if (setsockopt(sock, SOL_SOCKET, SO_LINGER, (void*)&l, sizeof(l))<0) perror("setsockopt(SO_LINGER)"); #endif } static int launch_request(void) { evutil_socket_t sock; struct sockaddr_in sin; struct bufferevent *b; struct request_info *ri; memset(&sin, 0, sizeof(sin)); ++total_n_launched; sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(0x7f000001); sin.sin_port = htons(8080); if ((sock = socket(AF_INET, SOCK_STREAM, 0)) < 0) return -1; if (evutil_make_socket_nonblocking(sock) < 0) { evutil_closesocket(sock); return -1; } frob_socket(sock); if (connect(sock, (struct sockaddr*)&sin, sizeof(sin)) < 0) { int e = evutil_socket_geterror(sock); if (! EVUTIL_ERR_CONNECT_RETRIABLE(e)) { evutil_closesocket(sock); return -1; } } ri = malloc(sizeof(*ri)); ri->n_read = 0; evutil_gettimeofday(&ri->started, NULL); b = bufferevent_socket_new(base, sock, BEV_OPT_CLOSE_ON_FREE); bufferevent_setcb(b, readcb, NULL, errorcb, ri); bufferevent_enable(b, EV_READ|EV_WRITE); evbuffer_add_printf(bufferevent_get_output(b), "GET %s HTTP/1.0\r\n\r\n", resource); return 0; } int main(int argc, char **argv) { int i; struct timeval start, end, total; long long usec; double throughput; #ifdef _WIN32 WSADATA WSAData; WSAStartup(0x101, &WSAData); #endif resource = "/ref"; setvbuf(stdout, NULL, _IONBF, 0); base = event_base_new(); for (i=0; i < PARALLELISM; ++i) { if (launch_request() < 0) perror("launch"); } evutil_gettimeofday(&start, NULL); event_base_dispatch(base); evutil_gettimeofday(&end, NULL); evutil_timersub(&end, &start, &total); usec = total_time.tv_sec * (long long)1000000 + total_time.tv_usec; if (!total_n_handled) { puts("Nothing worked. You probably did something dumb."); return 0; } throughput = total_n_handled / (total.tv_sec+ ((double)total.tv_usec)/1000000.0); #ifdef _WIN32 #define I64_FMT "%I64d" #define I64_TYP __int64 #else #define I64_FMT "%lld" #define I64_TYP long long int #endif printf("\n%d requests in %d.%06d sec. (%.2f throughput)\n" "Each took about %.02f msec latency\n" I64_FMT "bytes read. %d errors.\n", total_n_handled, (int)total.tv_sec, (int)total.tv_usec, throughput, (double)(usec/1000) / total_n_handled, (I64_TYP)total_n_bytes, n_errors); #ifdef _WIN32 WSACleanup(); #endif return 0; } kytensor-2.49.0.7/third_party/libevent/test/check-dumpevents.py000077500000000000000000000030201527052371200245720ustar00rootroot00000000000000#!/usr/bin/python2 # # Post-process the output of test-dumpevents and check it for correctness. # import math import re import sys text = sys.stdin.readlines() try: expect_inserted_pos = text.index("Inserted:\n") expect_active_pos = text.index("Active:\n") got_inserted_pos = text.index("Inserted events:\n") got_active_pos = text.index("Active events:\n") except ValueError: print >>sys.stderr, "Missing expected dividing line in dumpevents output" sys.exit(1) if not (expect_inserted_pos < expect_active_pos < got_inserted_pos < got_active_pos): print >>sys.stderr, "Sections out of order in dumpevents output" sys.exit(1) now,T= text[1].split() T = float(T) want_inserted = set(text[expect_inserted_pos+1:expect_active_pos]) want_active = set(text[expect_active_pos+1:got_inserted_pos-1]) got_inserted = set(text[got_inserted_pos+1:got_active_pos]) got_active = set(text[got_active_pos+1:]) pat = re.compile(r'Timeout=([0-9\.]+)') def replace_time(m): t = float(m.group(1)) if .9 < abs(t-T) < 1.1: return "Timeout=T+1" elif 2.4 < abs(t-T) < 2.6: return "Timeout=T+2.5" else: return m.group(0) cleaned_inserted = set( pat.sub(replace_time, s) for s in got_inserted if "Internal" not in s) if cleaned_inserted != want_inserted: print >>sys.stderr, "Inserted event lists were not as expected!" sys.exit(1) if set(got_active) != set(want_active): print >>sys.stderr, "Active event lists were not as expected!" sys.exit(1) kytensor-2.49.0.7/third_party/libevent/test/include.am000066400000000000000000000114201527052371200227150ustar00rootroot00000000000000# test/Makefile.am for libevent # Copyright 2000-2007 Niels Provos # Copyright 2007-2012 Niels Provos and Nick Mathewson # # See LICENSE for copying information. regress_CPPFLAGS = -DTINYTEST_LOCAL EXTRA_DIST+= \ test/check-dumpevents.py \ test/regress.gen.c \ test/regress.gen.h \ test/regress.rpc \ test/rpcgen_wrapper.sh \ test/test.sh TESTPROGRAMS = \ test/bench \ test/bench_cascade \ test/bench_http \ test/bench_httpclient \ test/test-changelist \ test/test-dumpevents \ test/test-eof \ test/test-closed \ test/test-fdleak \ test/test-init \ test/test-ratelim \ test/test-time \ test/test-weof \ test/regress if BUILD_REGRESS noinst_PROGRAMS += $(TESTPROGRAMS) EXTRA_PROGRAMS+= test/regress endif noinst_HEADERS+= \ test/regress.h \ test/regress_thread.h \ test/tinytest.h \ test/tinytest_local.h \ test/tinytest_macros.h TESTS = \ test_runner_epoll \ test_runner_select \ test_runner_kqueue \ test_runner_evport \ test_runner_devpoll \ test_runner_poll \ test_runner_win32 \ test_runner_timerfd \ test_runner_changelist \ test_runner_timerfd_changelist LOG_COMPILER = true TESTS_COMPILER = true test_runner_epoll: test/test.sh test/test.sh -b EPOLL test_runner_select: test/test.sh test/test.sh -b SELECT test_runner_kqueue: test/test.sh test/test.sh -b KQUEUE test_runner_evport: test/test.sh test/test.sh -b EVPORT test_runner_devpoll: test/test.sh test/test.sh -b DEVPOLL test_runner_poll: test/test.sh test/test.sh -b POLL test_runner_win32: test/test.sh test/test.sh -b WIN32 test_runner_timerfd: test/test.sh test/test.sh -b "" -t test_runner_changelist: test/test.sh test/test.sh -b "" -c test_runner_timerfd_changelist: test/test.sh test/test.sh -b "" -T DISTCLEANFILES += test/regress.gen.c test/regress.gen.h if BUILD_REGRESS BUILT_SOURCES += test/regress.gen.c test/regress.gen.h endif test_test_init_SOURCES = test/test-init.c test_test_init_LDADD = libevent_core.la test_test_dumpevents_SOURCES = test/test-dumpevents.c test_test_dumpevents_LDADD = libevent_core.la test_test_eof_SOURCES = test/test-eof.c test_test_eof_LDADD = libevent_core.la test_test_closed_SOURCES = test/test-closed.c test_test_closed_LDADD = libevent_core.la test_test_changelist_SOURCES = test/test-changelist.c test_test_changelist_LDADD = libevent_core.la test_test_weof_SOURCES = test/test-weof.c test_test_weof_LDADD = libevent_core.la test_test_time_SOURCES = test/test-time.c test_test_time_LDADD = libevent_core.la test_test_ratelim_SOURCES = test/test-ratelim.c test_test_ratelim_LDADD = libevent_core.la -lm test_test_fdleak_SOURCES = test/test-fdleak.c test_test_fdleak_LDADD = libevent_core.la test_regress_SOURCES = \ test/regress.c \ test/regress.gen.c \ test/regress.gen.h \ test/regress_buffer.c \ test/regress_bufferevent.c \ test/regress_dns.c \ test/regress_et.c \ test/regress_finalize.c \ test/regress_http.c \ test/regress_listener.c \ test/regress_main.c \ test/regress_minheap.c \ test/regress_rpc.c \ test/regress_testutils.c \ test/regress_testutils.h \ test/regress_util.c \ test/tinytest.c \ $(regress_thread_SOURCES) \ $(regress_zlib_SOURCES) if PTHREADS regress_thread_SOURCES = test/regress_thread.c PTHREAD_LIBS += libevent_pthreads.la endif if BUILD_WIN32 if THREADS regress_thread_SOURCES = test/regress_thread.c endif endif if ZLIB_REGRESS regress_zlib_SOURCES = test/regress_zlib.c endif if BUILD_WIN32 test_regress_SOURCES += test/regress_iocp.c endif test_regress_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la $(PTHREAD_LIBS) $(ZLIB_LIBS) test_regress_CPPFLAGS = $(AM_CPPFLAGS) $(PTHREAD_CFLAGS) $(ZLIB_CFLAGS) -Itest test_regress_LDFLAGS = $(PTHREAD_CFLAGS) if OPENSSL test_regress_SOURCES += test/regress_ssl.c test_regress_CPPFLAGS += $(OPENSSL_INCS) test_regress_LDADD += libevent_openssl.la $(OPENSSL_LIBS) ${OPENSSL_LIBADD} endif test_bench_SOURCES = test/bench.c test_bench_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la test_bench_cascade_SOURCES = test/bench_cascade.c test_bench_cascade_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la test_bench_http_SOURCES = test/bench_http.c test_bench_http_LDADD = $(LIBEVENT_GC_SECTIONS) libevent.la test_bench_httpclient_SOURCES = test/bench_httpclient.c test_bench_httpclient_LDADD = $(LIBEVENT_GC_SECTIONS) libevent_core.la test/regress.gen.c test/regress.gen.h: test/rpcgen-attempted test/rpcgen-attempted: test/regress.rpc event_rpcgen.py test/rpcgen_wrapper.sh $(AM_V_GEN)date -u > $@ $(AM_V_at)if $(srcdir)/test/rpcgen_wrapper.sh $(srcdir)/test; then \ true; \ else \ echo "No Python installed; stubbing out RPC test." >&2; \ echo " "> test/regress.gen.c; \ echo "#define NO_PYTHON_EXISTS" > test/regress.gen.h; \ fi CLEANFILES += test/rpcgen-attempted $(TESTPROGRAMS) : libevent.la kytensor-2.49.0.7/third_party/libevent/test/print-winsock-errors.c000066400000000000000000000035711527052371200252500ustar00rootroot00000000000000#include #include #include #include #include "event2/event.h" #include "event2/util.h" #include "event2/thread.h" #define E(x) printf (#x " -> \"%s\"\n", evutil_socket_error_to_string (x)); int main (int argc, char **argv) { int i, j; const char *s1, *s2; #ifdef EVTHREAD_USE_WINDOWS_THREADS_IMPLEMENTED evthread_use_windows_threads (); #endif s1 = evutil_socket_error_to_string (WSAEINTR); for (i = 0; i < 3; i++) { printf ("\niteration %d:\n\n", i); E(WSAEINTR); E(WSAEACCES); E(WSAEFAULT); E(WSAEINVAL); E(WSAEMFILE); E(WSAEWOULDBLOCK); E(WSAEINPROGRESS); E(WSAEALREADY); E(WSAENOTSOCK); E(WSAEDESTADDRREQ); E(WSAEMSGSIZE); E(WSAEPROTOTYPE); E(WSAENOPROTOOPT); E(WSAEPROTONOSUPPORT); E(WSAESOCKTNOSUPPORT); E(WSAEOPNOTSUPP); E(WSAEPFNOSUPPORT); E(WSAEAFNOSUPPORT); E(WSAEADDRINUSE); E(WSAEADDRNOTAVAIL); E(WSAENETDOWN); E(WSAENETUNREACH); E(WSAENETRESET); E(WSAECONNABORTED); E(WSAECONNRESET); E(WSAENOBUFS); E(WSAEISCONN); E(WSAENOTCONN); E(WSAESHUTDOWN); E(WSAETIMEDOUT); E(WSAECONNREFUSED); E(WSAEHOSTDOWN); E(WSAEHOSTUNREACH); E(WSAEPROCLIM); E(WSASYSNOTREADY); E(WSAVERNOTSUPPORTED); E(WSANOTINITIALISED); E(WSAEDISCON); E(WSATYPE_NOT_FOUND); E(WSAHOST_NOT_FOUND); E(WSATRY_AGAIN); E(WSANO_RECOVERY); E(WSANO_DATA); E(0xdeadbeef); /* test the case where no message is available */ /* fill up the hash table a bit to make sure it grows properly */ for (j = 0; j < 50; j++) { int err; evutil_secure_rng_get_bytes(&err, sizeof(err)); evutil_socket_error_to_string(err); } } s2 = evutil_socket_error_to_string (WSAEINTR); if (s1 != s2) printf ("caching failed!\n"); libevent_global_shutdown (); return EXIT_SUCCESS; } kytensor-2.49.0.7/third_party/libevent/test/regress.c000066400000000000000000002444561527052371200226120ustar00rootroot00000000000000/* * Copyright (c) 2003-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "util-internal.h" #ifdef _WIN32 #include #include #endif #ifdef EVENT__HAVE_PTHREADS #include #endif #include "event2/event-config.h" #include #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #ifndef _WIN32 #include #include #include #include #include #endif #include #include #include #include #include #include #include #include #include "event2/event.h" #include "event2/event_struct.h" #include "event2/event_compat.h" #include "event2/tag.h" #include "event2/buffer.h" #include "event2/buffer_compat.h" #include "event2/util.h" #include "event-internal.h" #include "evthread-internal.h" #include "log-internal.h" #include "time-internal.h" #include "regress.h" #ifndef _WIN32 #include "regress.gen.h" #endif evutil_socket_t pair[2]; int test_ok; int called; struct event_base *global_base; static char wbuf[4096]; static char rbuf[4096]; static int woff; static int roff; static int usepersist; static struct timeval tset; static struct timeval tcalled; #define TEST1 "this is a test" #ifdef _WIN32 #define write(fd,buf,len) send((fd),(buf),(int)(len),0) #define read(fd,buf,len) recv((fd),(buf),(int)(len),0) #endif struct basic_cb_args { struct event_base *eb; struct event *ev; unsigned int callcount; }; static void simple_read_cb(evutil_socket_t fd, short event, void *arg) { char buf[256]; int len; len = read(fd, buf, sizeof(buf)); if (len) { if (!called) { if (event_add(arg, NULL) == -1) exit(1); } } else if (called == 1) test_ok = 1; called++; } static void basic_read_cb(evutil_socket_t fd, short event, void *data) { char buf[256]; int len; struct basic_cb_args *arg = data; len = read(fd, buf, sizeof(buf)); if (len < 0) { tt_fail_perror("read (callback)"); } else { switch (arg->callcount++) { case 0: /* first call: expect to read data; cycle */ if (len > 0) return; tt_fail_msg("EOF before data read"); break; case 1: /* second call: expect EOF; stop */ if (len > 0) tt_fail_msg("not all data read on first cycle"); break; default: /* third call: should not happen */ tt_fail_msg("too many cycles"); } } event_del(arg->ev); event_base_loopexit(arg->eb, NULL); } static void dummy_read_cb(evutil_socket_t fd, short event, void *arg) { } static void simple_write_cb(evutil_socket_t fd, short event, void *arg) { int len; len = write(fd, TEST1, strlen(TEST1) + 1); if (len == -1) test_ok = 0; else test_ok = 1; } static void multiple_write_cb(evutil_socket_t fd, short event, void *arg) { struct event *ev = arg; int len; len = 128; if (woff + len >= (int)sizeof(wbuf)) len = sizeof(wbuf) - woff; len = write(fd, wbuf + woff, len); if (len == -1) { fprintf(stderr, "%s: write\n", __func__); if (usepersist) event_del(ev); return; } woff += len; if (woff >= (int)sizeof(wbuf)) { shutdown(fd, EVUTIL_SHUT_WR); if (usepersist) event_del(ev); return; } if (!usepersist) { if (event_add(ev, NULL) == -1) exit(1); } } static void multiple_read_cb(evutil_socket_t fd, short event, void *arg) { struct event *ev = arg; int len; len = read(fd, rbuf + roff, sizeof(rbuf) - roff); if (len == -1) fprintf(stderr, "%s: read\n", __func__); if (len <= 0) { if (usepersist) event_del(ev); return; } roff += len; if (!usepersist) { if (event_add(ev, NULL) == -1) exit(1); } } static void timeout_cb(evutil_socket_t fd, short event, void *arg) { evutil_gettimeofday(&tcalled, NULL); } struct both { struct event ev; int nread; }; static void combined_read_cb(evutil_socket_t fd, short event, void *arg) { struct both *both = arg; char buf[128]; int len; len = read(fd, buf, sizeof(buf)); if (len == -1) fprintf(stderr, "%s: read\n", __func__); if (len <= 0) return; both->nread += len; if (event_add(&both->ev, NULL) == -1) exit(1); } static void combined_write_cb(evutil_socket_t fd, short event, void *arg) { struct both *both = arg; char buf[128]; int len; len = sizeof(buf); if (len > both->nread) len = both->nread; memset(buf, 'q', len); len = write(fd, buf, len); if (len == -1) fprintf(stderr, "%s: write\n", __func__); if (len <= 0) { shutdown(fd, EVUTIL_SHUT_WR); return; } both->nread -= len; if (event_add(&both->ev, NULL) == -1) exit(1); } /* These macros used to replicate the work of the legacy test wrapper code */ #define setup_test(x) do { \ if (!in_legacy_test_wrapper) { \ TT_FAIL(("Legacy test %s not wrapped properly", x)); \ return; \ } \ } while (0) #define cleanup_test() setup_test("cleanup") static void test_simpleread(void) { struct event ev; /* Very simple read test */ setup_test("Simple read: "); if (write(pair[0], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); } shutdown(pair[0], EVUTIL_SHUT_WR); event_set(&ev, pair[1], EV_READ, simple_read_cb, &ev); if (event_add(&ev, NULL) == -1) exit(1); event_dispatch(); cleanup_test(); } static void test_simplewrite(void) { struct event ev; /* Very simple write test */ setup_test("Simple write: "); event_set(&ev, pair[0], EV_WRITE, simple_write_cb, &ev); if (event_add(&ev, NULL) == -1) exit(1); event_dispatch(); cleanup_test(); } static void simpleread_multiple_cb(evutil_socket_t fd, short event, void *arg) { if (++called == 2) test_ok = 1; } static void test_simpleread_multiple(void) { struct event one, two; /* Very simple read test */ setup_test("Simple read to multiple evens: "); if (write(pair[0], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); } shutdown(pair[0], EVUTIL_SHUT_WR); event_set(&one, pair[1], EV_READ, simpleread_multiple_cb, NULL); if (event_add(&one, NULL) == -1) exit(1); event_set(&two, pair[1], EV_READ, simpleread_multiple_cb, NULL); if (event_add(&two, NULL) == -1) exit(1); event_dispatch(); cleanup_test(); } static int have_closed = 0; static int premature_event = 0; static void simpleclose_close_fd_cb(evutil_socket_t s, short what, void *ptr) { evutil_socket_t **fds = ptr; TT_BLATHER(("Closing")); evutil_closesocket(*fds[0]); evutil_closesocket(*fds[1]); *fds[0] = -1; *fds[1] = -1; have_closed = 1; } static void record_event_cb(evutil_socket_t s, short what, void *ptr) { short *whatp = ptr; if (!have_closed) premature_event = 1; *whatp = what; TT_BLATHER(("Recorded %d on socket %d", (int)what, (int)s)); } static void test_simpleclose(void *ptr) { /* Test that a close of FD is detected as a read and as a write. */ struct event_base *base = event_base_new(); evutil_socket_t pair1[2]={-1,-1}, pair2[2] = {-1, -1}; evutil_socket_t *to_close[2]; struct event *rev=NULL, *wev=NULL, *closeev=NULL; struct timeval tv; short got_read_on_close = 0, got_write_on_close = 0; char buf[1024]; memset(buf, 99, sizeof(buf)); #ifdef _WIN32 #define LOCAL_SOCKETPAIR_AF AF_INET #else #define LOCAL_SOCKETPAIR_AF AF_UNIX #endif if (evutil_socketpair(LOCAL_SOCKETPAIR_AF, SOCK_STREAM, 0, pair1)<0) TT_DIE(("socketpair: %s", strerror(errno))); if (evutil_socketpair(LOCAL_SOCKETPAIR_AF, SOCK_STREAM, 0, pair2)<0) TT_DIE(("socketpair: %s", strerror(errno))); if (evutil_make_socket_nonblocking(pair1[1]) < 0) TT_DIE(("make_socket_nonblocking")); if (evutil_make_socket_nonblocking(pair2[1]) < 0) TT_DIE(("make_socket_nonblocking")); /** Stuff pair2[1] full of data, until write fails */ while (1) { int r = write(pair2[1], buf, sizeof(buf)); if (r<0) { int err = evutil_socket_geterror(pair2[1]); if (! EVUTIL_ERR_RW_RETRIABLE(err)) TT_DIE(("write failed strangely: %s", evutil_socket_error_to_string(err))); break; } } to_close[0] = &pair1[0]; to_close[1] = &pair2[0]; closeev = event_new(base, -1, EV_TIMEOUT, simpleclose_close_fd_cb, to_close); rev = event_new(base, pair1[1], EV_READ, record_event_cb, &got_read_on_close); TT_BLATHER(("Waiting for read on %d", (int)pair1[1])); wev = event_new(base, pair2[1], EV_WRITE, record_event_cb, &got_write_on_close); TT_BLATHER(("Waiting for write on %d", (int)pair2[1])); tv.tv_sec = 0; tv.tv_usec = 100*1000; /* Close pair1[0] after a little while, and make * sure we get a read event. */ event_add(closeev, &tv); event_add(rev, NULL); event_add(wev, NULL); /* Don't let the test go on too long. */ tv.tv_sec = 0; tv.tv_usec = 200*1000; event_base_loopexit(base, &tv); event_base_loop(base, 0); tt_int_op(got_read_on_close, ==, EV_READ); tt_int_op(got_write_on_close, ==, EV_WRITE); tt_int_op(premature_event, ==, 0); end: if (pair1[0] >= 0) evutil_closesocket(pair1[0]); if (pair1[1] >= 0) evutil_closesocket(pair1[1]); if (pair2[0] >= 0) evutil_closesocket(pair2[0]); if (pair2[1] >= 0) evutil_closesocket(pair2[1]); if (rev) event_free(rev); if (wev) event_free(wev); if (closeev) event_free(closeev); if (base) event_base_free(base); } static void test_multiple(void) { struct event ev, ev2; int i; /* Multiple read and write test */ setup_test("Multiple read/write: "); memset(rbuf, 0, sizeof(rbuf)); for (i = 0; i < (int)sizeof(wbuf); i++) wbuf[i] = i; roff = woff = 0; usepersist = 0; event_set(&ev, pair[0], EV_WRITE, multiple_write_cb, &ev); if (event_add(&ev, NULL) == -1) exit(1); event_set(&ev2, pair[1], EV_READ, multiple_read_cb, &ev2); if (event_add(&ev2, NULL) == -1) exit(1); event_dispatch(); if (roff == woff) test_ok = memcmp(rbuf, wbuf, sizeof(wbuf)) == 0; cleanup_test(); } static void test_persistent(void) { struct event ev, ev2; int i; /* Multiple read and write test with persist */ setup_test("Persist read/write: "); memset(rbuf, 0, sizeof(rbuf)); for (i = 0; i < (int)sizeof(wbuf); i++) wbuf[i] = i; roff = woff = 0; usepersist = 1; event_set(&ev, pair[0], EV_WRITE|EV_PERSIST, multiple_write_cb, &ev); if (event_add(&ev, NULL) == -1) exit(1); event_set(&ev2, pair[1], EV_READ|EV_PERSIST, multiple_read_cb, &ev2); if (event_add(&ev2, NULL) == -1) exit(1); event_dispatch(); if (roff == woff) test_ok = memcmp(rbuf, wbuf, sizeof(wbuf)) == 0; cleanup_test(); } static void test_combined(void) { struct both r1, r2, w1, w2; setup_test("Combined read/write: "); memset(&r1, 0, sizeof(r1)); memset(&r2, 0, sizeof(r2)); memset(&w1, 0, sizeof(w1)); memset(&w2, 0, sizeof(w2)); w1.nread = 4096; w2.nread = 8192; event_set(&r1.ev, pair[0], EV_READ, combined_read_cb, &r1); event_set(&w1.ev, pair[0], EV_WRITE, combined_write_cb, &w1); event_set(&r2.ev, pair[1], EV_READ, combined_read_cb, &r2); event_set(&w2.ev, pair[1], EV_WRITE, combined_write_cb, &w2); tt_assert(event_add(&r1.ev, NULL) != -1); tt_assert(!event_add(&w1.ev, NULL)); tt_assert(!event_add(&r2.ev, NULL)); tt_assert(!event_add(&w2.ev, NULL)); event_dispatch(); if (r1.nread == 8192 && r2.nread == 4096) test_ok = 1; end: cleanup_test(); } static void test_simpletimeout(void) { struct timeval tv; struct event ev; setup_test("Simple timeout: "); tv.tv_usec = 200*1000; tv.tv_sec = 0; evutil_timerclear(&tcalled); evtimer_set(&ev, timeout_cb, NULL); evtimer_add(&ev, &tv); evutil_gettimeofday(&tset, NULL); event_dispatch(); test_timeval_diff_eq(&tset, &tcalled, 200); test_ok = 1; end: cleanup_test(); } static void periodic_timeout_cb(evutil_socket_t fd, short event, void *arg) { int *count = arg; (*count)++; if (*count == 6) { /* call loopexit only once - on slow machines(?), it is * apparently possible for this to get called twice. */ test_ok = 1; event_base_loopexit(global_base, NULL); } } static void test_persistent_timeout(void) { struct timeval tv; struct event ev; int count = 0; evutil_timerclear(&tv); tv.tv_usec = 10000; event_assign(&ev, global_base, -1, EV_TIMEOUT|EV_PERSIST, periodic_timeout_cb, &count); event_add(&ev, &tv); event_dispatch(); event_del(&ev); } static void test_persistent_timeout_jump(void *ptr) { struct basic_test_data *data = ptr; struct event ev; int count = 0; struct timeval msec100 = { 0, 100 * 1000 }; struct timeval msec50 = { 0, 50 * 1000 }; struct timeval msec300 = { 0, 300 * 1000 }; event_assign(&ev, data->base, -1, EV_PERSIST, periodic_timeout_cb, &count); event_add(&ev, &msec100); /* Wait for a bit */ evutil_usleep_(&msec300); event_base_loopexit(data->base, &msec50); event_base_dispatch(data->base); tt_int_op(count, ==, 1); end: event_del(&ev); } struct persist_active_timeout_called { int n; short events[16]; struct timeval tvs[16]; }; static void activate_cb(evutil_socket_t fd, short event, void *arg) { struct event *ev = arg; event_active(ev, EV_READ, 1); } static void persist_active_timeout_cb(evutil_socket_t fd, short event, void *arg) { struct persist_active_timeout_called *c = arg; if (c->n < 15) { c->events[c->n] = event; evutil_gettimeofday(&c->tvs[c->n], NULL); ++c->n; } } static void test_persistent_active_timeout(void *ptr) { struct timeval tv, tv2, tv_exit, start; struct event ev; struct persist_active_timeout_called res; struct basic_test_data *data = ptr; struct event_base *base = data->base; memset(&res, 0, sizeof(res)); tv.tv_sec = 0; tv.tv_usec = 200 * 1000; event_assign(&ev, base, -1, EV_TIMEOUT|EV_PERSIST, persist_active_timeout_cb, &res); event_add(&ev, &tv); tv2.tv_sec = 0; tv2.tv_usec = 100 * 1000; event_base_once(base, -1, EV_TIMEOUT, activate_cb, &ev, &tv2); tv_exit.tv_sec = 0; tv_exit.tv_usec = 600 * 1000; event_base_loopexit(base, &tv_exit); event_base_assert_ok_(base); evutil_gettimeofday(&start, NULL); event_base_dispatch(base); event_base_assert_ok_(base); tt_int_op(res.n, ==, 3); tt_int_op(res.events[0], ==, EV_READ); tt_int_op(res.events[1], ==, EV_TIMEOUT); tt_int_op(res.events[2], ==, EV_TIMEOUT); test_timeval_diff_eq(&start, &res.tvs[0], 100); test_timeval_diff_eq(&start, &res.tvs[1], 300); test_timeval_diff_eq(&start, &res.tvs[2], 500); end: event_del(&ev); } struct common_timeout_info { struct event ev; struct timeval called_at; int which; int count; }; static void common_timeout_cb(evutil_socket_t fd, short event, void *arg) { struct common_timeout_info *ti = arg; ++ti->count; evutil_gettimeofday(&ti->called_at, NULL); if (ti->count >= 4) event_del(&ti->ev); } static void test_common_timeout(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = data->base; int i; struct common_timeout_info info[100]; struct timeval start; struct timeval tmp_100_ms = { 0, 100*1000 }; struct timeval tmp_200_ms = { 0, 200*1000 }; struct timeval tmp_5_sec = { 5, 0 }; struct timeval tmp_5M_usec = { 0, 5*1000*1000 }; const struct timeval *ms_100, *ms_200, *sec_5; ms_100 = event_base_init_common_timeout(base, &tmp_100_ms); ms_200 = event_base_init_common_timeout(base, &tmp_200_ms); sec_5 = event_base_init_common_timeout(base, &tmp_5_sec); tt_assert(ms_100); tt_assert(ms_200); tt_assert(sec_5); tt_ptr_op(event_base_init_common_timeout(base, &tmp_200_ms), ==, ms_200); tt_ptr_op(event_base_init_common_timeout(base, ms_200), ==, ms_200); tt_ptr_op(event_base_init_common_timeout(base, &tmp_5M_usec), ==, sec_5); tt_int_op(ms_100->tv_sec, ==, 0); tt_int_op(ms_200->tv_sec, ==, 0); tt_int_op(sec_5->tv_sec, ==, 5); tt_int_op(ms_100->tv_usec, ==, 100000|0x50000000); tt_int_op(ms_200->tv_usec, ==, 200000|0x50100000); tt_int_op(sec_5->tv_usec, ==, 0|0x50200000); memset(info, 0, sizeof(info)); for (i=0; i<100; ++i) { info[i].which = i; event_assign(&info[i].ev, base, -1, EV_TIMEOUT|EV_PERSIST, common_timeout_cb, &info[i]); if (i % 2) { if ((i%20)==1) { /* Glass-box test: Make sure we survive the * transition to non-common timeouts. It's * a little tricky. */ event_add(&info[i].ev, ms_200); event_add(&info[i].ev, &tmp_100_ms); } else if ((i%20)==3) { /* Check heap-to-common too. */ event_add(&info[i].ev, &tmp_200_ms); event_add(&info[i].ev, ms_100); } else if ((i%20)==5) { /* Also check common-to-common. */ event_add(&info[i].ev, ms_200); event_add(&info[i].ev, ms_100); } else { event_add(&info[i].ev, ms_100); } } else { event_add(&info[i].ev, ms_200); } } event_base_assert_ok_(base); evutil_gettimeofday(&start, NULL); event_base_dispatch(base); event_base_assert_ok_(base); for (i=0; i<10; ++i) { tt_int_op(info[i].count, ==, 4); if (i % 2) { test_timeval_diff_eq(&start, &info[i].called_at, 400); } else { test_timeval_diff_eq(&start, &info[i].called_at, 800); } } /* Make sure we can free the base with some events in. */ for (i=0; i<100; ++i) { if (i % 2) { event_add(&info[i].ev, ms_100); } else { event_add(&info[i].ev, ms_200); } } end: event_base_free(data->base); /* need to do this here before info is * out-of-scope */ data->base = NULL; } #ifndef _WIN32 #define current_base event_global_current_base_ extern struct event_base *current_base; static void fork_signal_cb(evutil_socket_t fd, short events, void *arg) { event_del(arg); } int child_pair[2] = { -1, -1 }; static void simple_child_read_cb(evutil_socket_t fd, short event, void *arg) { char buf[256]; int len; len = read(fd, buf, sizeof(buf)); if (write(child_pair[0], "", 1) < 0) tt_fail_perror("write"); if (len) { if (!called) { if (event_add(arg, NULL) == -1) exit(1); } } else if (called == 1) test_ok = 1; called++; } static void test_fork(void) { char c; int status; struct event ev, sig_ev, usr_ev, existing_ev; pid_t pid; int wait_flags = 0; #ifdef EVENT__HAVE_WAITPID_WITH_WNOWAIT wait_flags |= WNOWAIT; #endif setup_test("After fork: "); { if (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, child_pair) == -1) { fprintf(stderr, "%s: socketpair\n", __func__); exit(1); } if (evutil_make_socket_nonblocking(child_pair[0]) == -1) { fprintf(stderr, "fcntl(O_NONBLOCK)"); exit(1); } } tt_assert(current_base); evthread_make_base_notifiable(current_base); if (write(pair[0], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); } event_set(&ev, pair[1], EV_READ, simple_child_read_cb, &ev); if (event_add(&ev, NULL) == -1) exit(1); evsignal_set(&sig_ev, SIGCHLD, fork_signal_cb, &sig_ev); evsignal_add(&sig_ev, NULL); evsignal_set(&existing_ev, SIGUSR2, fork_signal_cb, &existing_ev); evsignal_add(&existing_ev, NULL); event_base_assert_ok_(current_base); TT_BLATHER(("Before fork")); if ((pid = regress_fork()) == 0) { /* in the child */ TT_BLATHER(("In child, before reinit")); event_base_assert_ok_(current_base); if (event_reinit(current_base) == -1) { fprintf(stdout, "FAILED (reinit)\n"); exit(1); } TT_BLATHER(("After reinit")); event_base_assert_ok_(current_base); TT_BLATHER(("After assert-ok")); evsignal_del(&sig_ev); evsignal_set(&usr_ev, SIGUSR1, fork_signal_cb, &usr_ev); evsignal_add(&usr_ev, NULL); raise(SIGUSR1); raise(SIGUSR2); called = 0; event_dispatch(); event_base_free(current_base); /* we do not send an EOF; simple_read_cb requires an EOF * to set test_ok. we just verify that the callback was * called. */ exit(test_ok != 0 || called != 2 ? -2 : 76); } /** wait until client read first message */ if (read(child_pair[1], &c, 1) < 0) { tt_fail_perror("read"); } if (write(pair[0], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); } TT_BLATHER(("Before waitpid")); if (waitpid(pid, &status, wait_flags) == -1) { perror("waitpid"); exit(1); } TT_BLATHER(("After waitpid")); if (WEXITSTATUS(status) != 76) { fprintf(stdout, "FAILED (exit): %d\n", WEXITSTATUS(status)); exit(1); } /* test that the current event loop still works */ if (write(pair[0], TEST1, strlen(TEST1)+1) < 0) { fprintf(stderr, "%s: write\n", __func__); } shutdown(pair[0], EVUTIL_SHUT_WR); evsignal_set(&usr_ev, SIGUSR1, fork_signal_cb, &usr_ev); evsignal_add(&usr_ev, NULL); raise(SIGUSR1); raise(SIGUSR2); event_dispatch(); evsignal_del(&sig_ev); tt_int_op(test_ok, ==, 1); end: cleanup_test(); if (child_pair[0] != -1) evutil_closesocket(child_pair[0]); if (child_pair[1] != -1) evutil_closesocket(child_pair[1]); } #ifdef EVENT__HAVE_PTHREADS static void* del_wait_thread(void *arg) { struct timeval tv_start, tv_end; evutil_gettimeofday(&tv_start, NULL); event_dispatch(); evutil_gettimeofday(&tv_end, NULL); test_timeval_diff_eq(&tv_start, &tv_end, 300); end: return NULL; } static void del_wait_cb(evutil_socket_t fd, short event, void *arg) { struct timeval delay = { 0, 300*1000 }; TT_BLATHER(("Sleeping")); evutil_usleep_(&delay); test_ok = 1; } static void test_del_wait(void) { struct event ev; pthread_t thread; setup_test("event_del will wait: "); event_set(&ev, pair[1], EV_READ, del_wait_cb, &ev); event_add(&ev, NULL); pthread_create(&thread, NULL, del_wait_thread, NULL); if (write(pair[0], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); } { struct timeval delay = { 0, 30*1000 }; evutil_usleep_(&delay); } { struct timeval tv_start, tv_end; evutil_gettimeofday(&tv_start, NULL); event_del(&ev); evutil_gettimeofday(&tv_end, NULL); test_timeval_diff_eq(&tv_start, &tv_end, 270); } pthread_join(thread, NULL); end: ; } #endif static void signal_cb_sa(int sig) { test_ok = 2; } static void signal_cb(evutil_socket_t fd, short event, void *arg) { struct event *ev = arg; evsignal_del(ev); test_ok = 1; } static void test_simplesignal_impl(int find_reorder) { struct event ev; struct itimerval itv; evsignal_set(&ev, SIGALRM, signal_cb, &ev); evsignal_add(&ev, NULL); /* find bugs in which operations are re-ordered */ if (find_reorder) { evsignal_del(&ev); evsignal_add(&ev, NULL); } memset(&itv, 0, sizeof(itv)); itv.it_value.tv_sec = 0; itv.it_value.tv_usec = 100000; if (setitimer(ITIMER_REAL, &itv, NULL) == -1) goto skip_simplesignal; event_dispatch(); skip_simplesignal: if (evsignal_del(&ev) == -1) test_ok = 0; cleanup_test(); } static void test_simplestsignal(void) { setup_test("Simplest one signal: "); test_simplesignal_impl(0); } static void test_simplesignal(void) { setup_test("Simple signal: "); test_simplesignal_impl(1); } static void test_multiplesignal(void) { struct event ev_one, ev_two; struct itimerval itv; setup_test("Multiple signal: "); evsignal_set(&ev_one, SIGALRM, signal_cb, &ev_one); evsignal_add(&ev_one, NULL); evsignal_set(&ev_two, SIGALRM, signal_cb, &ev_two); evsignal_add(&ev_two, NULL); memset(&itv, 0, sizeof(itv)); itv.it_value.tv_sec = 0; itv.it_value.tv_usec = 100000; if (setitimer(ITIMER_REAL, &itv, NULL) == -1) goto skip_simplesignal; event_dispatch(); skip_simplesignal: if (evsignal_del(&ev_one) == -1) test_ok = 0; if (evsignal_del(&ev_two) == -1) test_ok = 0; cleanup_test(); } static void test_immediatesignal(void) { struct event ev; test_ok = 0; evsignal_set(&ev, SIGUSR1, signal_cb, &ev); evsignal_add(&ev, NULL); raise(SIGUSR1); event_loop(EVLOOP_NONBLOCK); evsignal_del(&ev); cleanup_test(); } static void test_signal_dealloc(void) { /* make sure that evsignal_event is event_del'ed and pipe closed */ struct event ev; struct event_base *base = event_init(); evsignal_set(&ev, SIGUSR1, signal_cb, &ev); evsignal_add(&ev, NULL); evsignal_del(&ev); event_base_free(base); /* If we got here without asserting, we're fine. */ test_ok = 1; cleanup_test(); } static void test_signal_pipeloss(void) { /* make sure that the base1 pipe is closed correctly. */ struct event_base *base1, *base2; int pipe1; test_ok = 0; base1 = event_init(); pipe1 = base1->sig.ev_signal_pair[0]; base2 = event_init(); event_base_free(base2); event_base_free(base1); if (close(pipe1) != -1 || errno!=EBADF) { /* fd must be closed, so second close gives -1, EBADF */ printf("signal pipe not closed. "); test_ok = 0; } else { test_ok = 1; } cleanup_test(); } /* * make two bases to catch signals, use both of them. this only works * for event mechanisms that use our signal pipe trick. kqueue handles * signals internally, and all interested kqueues get all the signals. */ static void test_signal_switchbase(void) { struct event ev1, ev2; struct event_base *base1, *base2; int is_kqueue; test_ok = 0; base1 = event_init(); base2 = event_init(); is_kqueue = !strcmp(event_get_method(),"kqueue"); evsignal_set(&ev1, SIGUSR1, signal_cb, &ev1); evsignal_set(&ev2, SIGUSR1, signal_cb, &ev2); if (event_base_set(base1, &ev1) || event_base_set(base2, &ev2) || event_add(&ev1, NULL) || event_add(&ev2, NULL)) { fprintf(stderr, "%s: cannot set base, add\n", __func__); exit(1); } tt_ptr_op(event_get_base(&ev1), ==, base1); tt_ptr_op(event_get_base(&ev2), ==, base2); test_ok = 0; /* can handle signal before loop is called */ raise(SIGUSR1); event_base_loop(base2, EVLOOP_NONBLOCK); if (is_kqueue) { if (!test_ok) goto end; test_ok = 0; } event_base_loop(base1, EVLOOP_NONBLOCK); if (test_ok && !is_kqueue) { test_ok = 0; /* set base1 to handle signals */ event_base_loop(base1, EVLOOP_NONBLOCK); raise(SIGUSR1); event_base_loop(base1, EVLOOP_NONBLOCK); event_base_loop(base2, EVLOOP_NONBLOCK); } end: event_base_free(base1); event_base_free(base2); cleanup_test(); } /* * assert that a signal event removed from the event queue really is * removed - with no possibility of it's parent handler being fired. */ static void test_signal_assert(void) { struct event ev; struct event_base *base = event_init(); test_ok = 0; /* use SIGCONT so we don't kill ourselves when we signal to nowhere */ evsignal_set(&ev, SIGCONT, signal_cb, &ev); evsignal_add(&ev, NULL); /* * if evsignal_del() fails to reset the handler, it's current handler * will still point to evsig_handler(). */ evsignal_del(&ev); raise(SIGCONT); #if 0 /* only way to verify we were in evsig_handler() */ /* XXXX Now there's no longer a good way. */ if (base->sig.evsig_caught) test_ok = 0; else test_ok = 1; #else test_ok = 1; #endif event_base_free(base); cleanup_test(); return; } /* * assert that we restore our previous signal handler properly. */ static void test_signal_restore(void) { struct event ev; struct event_base *base = event_init(); #ifdef EVENT__HAVE_SIGACTION struct sigaction sa; #endif test_ok = 0; #ifdef EVENT__HAVE_SIGACTION sa.sa_handler = signal_cb_sa; sa.sa_flags = 0x0; sigemptyset(&sa.sa_mask); if (sigaction(SIGUSR1, &sa, NULL) == -1) goto out; #else if (signal(SIGUSR1, signal_cb_sa) == SIG_ERR) goto out; #endif evsignal_set(&ev, SIGUSR1, signal_cb, &ev); evsignal_add(&ev, NULL); evsignal_del(&ev); raise(SIGUSR1); /* 1 == signal_cb, 2 == signal_cb_sa, we want our previous handler */ if (test_ok != 2) test_ok = 0; out: event_base_free(base); cleanup_test(); return; } static void signal_cb_swp(int sig, short event, void *arg) { called++; if (called < 5) raise(sig); else event_loopexit(NULL); } static void timeout_cb_swp(evutil_socket_t fd, short event, void *arg) { if (called == -1) { struct timeval tv = {5, 0}; called = 0; evtimer_add((struct event *)arg, &tv); raise(SIGUSR1); return; } test_ok = 0; event_loopexit(NULL); } static void test_signal_while_processing(void) { struct event_base *base = event_init(); struct event ev, ev_timer; struct timeval tv = {0, 0}; setup_test("Receiving a signal while processing other signal: "); called = -1; test_ok = 1; signal_set(&ev, SIGUSR1, signal_cb_swp, NULL); signal_add(&ev, NULL); evtimer_set(&ev_timer, timeout_cb_swp, &ev_timer); evtimer_add(&ev_timer, &tv); event_dispatch(); event_base_free(base); cleanup_test(); return; } #endif static void test_free_active_base(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base1; struct event ev1; base1 = event_init(); if (base1) { event_assign(&ev1, base1, data->pair[1], EV_READ, dummy_read_cb, NULL); event_add(&ev1, NULL); event_base_free(base1); /* should not crash */ } else { tt_fail_msg("failed to create event_base for test"); } base1 = event_init(); tt_assert(base1); event_assign(&ev1, base1, 0, 0, dummy_read_cb, NULL); event_active(&ev1, EV_READ, 1); event_base_free(base1); end: ; } static void test_manipulate_active_events(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = data->base; struct event ev1; event_assign(&ev1, base, -1, EV_TIMEOUT, dummy_read_cb, NULL); /* Make sure an active event is pending. */ event_active(&ev1, EV_READ, 1); tt_int_op(event_pending(&ev1, EV_READ|EV_TIMEOUT|EV_WRITE, NULL), ==, EV_READ); /* Make sure that activating an event twice works. */ event_active(&ev1, EV_WRITE, 1); tt_int_op(event_pending(&ev1, EV_READ|EV_TIMEOUT|EV_WRITE, NULL), ==, EV_READ|EV_WRITE); end: event_del(&ev1); } static void event_selfarg_cb(evutil_socket_t fd, short event, void *arg) { struct event *ev = arg; struct event_base *base = event_get_base(ev); event_base_assert_ok_(base); event_base_loopexit(base, NULL); tt_want(ev == event_base_get_running_event(base)); } static void test_event_new_selfarg(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = data->base; struct event *ev = event_new(base, -1, EV_READ, event_selfarg_cb, event_self_cbarg()); event_active(ev, EV_READ, 1); event_base_dispatch(base); event_free(ev); } static void test_event_assign_selfarg(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = data->base; struct event ev; event_assign(&ev, base, -1, EV_READ, event_selfarg_cb, event_self_cbarg()); event_active(&ev, EV_READ, 1); event_base_dispatch(base); } static void test_event_base_get_num_events(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = data->base; struct event ev; int event_count_active; int event_count_virtual; int event_count_added; int event_count_active_virtual; int event_count_active_added; int event_count_virtual_added; int event_count_active_added_virtual; struct timeval qsec = {0, 100000}; event_assign(&ev, base, -1, EV_READ, event_selfarg_cb, event_self_cbarg()); event_add(&ev, &qsec); event_count_active = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE); event_count_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_VIRTUAL); event_count_added = event_base_get_num_events(base, EVENT_BASE_COUNT_ADDED); event_count_active_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE|EVENT_BASE_COUNT_VIRTUAL); event_count_active_added = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE|EVENT_BASE_COUNT_ADDED); event_count_virtual_added = event_base_get_num_events(base, EVENT_BASE_COUNT_VIRTUAL|EVENT_BASE_COUNT_ADDED); event_count_active_added_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE| EVENT_BASE_COUNT_ADDED| EVENT_BASE_COUNT_VIRTUAL); tt_int_op(event_count_active, ==, 0); tt_int_op(event_count_virtual, ==, 0); /* libevent itself adds a timeout event, so the event_count is 2 here */ tt_int_op(event_count_added, ==, 2); tt_int_op(event_count_active_virtual, ==, 0); tt_int_op(event_count_active_added, ==, 2); tt_int_op(event_count_virtual_added, ==, 2); tt_int_op(event_count_active_added_virtual, ==, 2); event_active(&ev, EV_READ, 1); event_count_active = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE); event_count_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_VIRTUAL); event_count_added = event_base_get_num_events(base, EVENT_BASE_COUNT_ADDED); event_count_active_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE|EVENT_BASE_COUNT_VIRTUAL); event_count_active_added = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE|EVENT_BASE_COUNT_ADDED); event_count_virtual_added = event_base_get_num_events(base, EVENT_BASE_COUNT_VIRTUAL|EVENT_BASE_COUNT_ADDED); event_count_active_added_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE| EVENT_BASE_COUNT_ADDED| EVENT_BASE_COUNT_VIRTUAL); tt_int_op(event_count_active, ==, 1); tt_int_op(event_count_virtual, ==, 0); tt_int_op(event_count_added, ==, 3); tt_int_op(event_count_active_virtual, ==, 1); tt_int_op(event_count_active_added, ==, 4); tt_int_op(event_count_virtual_added, ==, 3); tt_int_op(event_count_active_added_virtual, ==, 4); event_base_loop(base, 0); event_count_active = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE); event_count_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_VIRTUAL); event_count_added = event_base_get_num_events(base, EVENT_BASE_COUNT_ADDED); event_count_active_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE|EVENT_BASE_COUNT_VIRTUAL); event_count_active_added = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE|EVENT_BASE_COUNT_ADDED); event_count_virtual_added = event_base_get_num_events(base, EVENT_BASE_COUNT_VIRTUAL|EVENT_BASE_COUNT_ADDED); event_count_active_added_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE| EVENT_BASE_COUNT_ADDED| EVENT_BASE_COUNT_VIRTUAL); tt_int_op(event_count_active, ==, 0); tt_int_op(event_count_virtual, ==, 0); tt_int_op(event_count_added, ==, 0); tt_int_op(event_count_active_virtual, ==, 0); tt_int_op(event_count_active_added, ==, 0); tt_int_op(event_count_virtual_added, ==, 0); tt_int_op(event_count_active_added_virtual, ==, 0); event_base_add_virtual_(base); event_count_active = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE); event_count_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_VIRTUAL); event_count_added = event_base_get_num_events(base, EVENT_BASE_COUNT_ADDED); event_count_active_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE|EVENT_BASE_COUNT_VIRTUAL); event_count_active_added = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE|EVENT_BASE_COUNT_ADDED); event_count_virtual_added = event_base_get_num_events(base, EVENT_BASE_COUNT_VIRTUAL|EVENT_BASE_COUNT_ADDED); event_count_active_added_virtual = event_base_get_num_events(base, EVENT_BASE_COUNT_ACTIVE| EVENT_BASE_COUNT_ADDED| EVENT_BASE_COUNT_VIRTUAL); tt_int_op(event_count_active, ==, 0); tt_int_op(event_count_virtual, ==, 1); tt_int_op(event_count_added, ==, 0); tt_int_op(event_count_active_virtual, ==, 1); tt_int_op(event_count_active_added, ==, 0); tt_int_op(event_count_virtual_added, ==, 1); tt_int_op(event_count_active_added_virtual, ==, 1); end: ; } static void test_event_base_get_max_events(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = data->base; struct event ev; struct event ev2; int event_count_active; int event_count_virtual; int event_count_added; int event_count_active_virtual; int event_count_active_added; int event_count_virtual_added; int event_count_active_added_virtual; struct timeval qsec = {0, 100000}; event_assign(&ev, base, -1, EV_READ, event_selfarg_cb, event_self_cbarg()); event_assign(&ev2, base, -1, EV_READ, event_selfarg_cb, event_self_cbarg()); event_add(&ev, &qsec); event_add(&ev2, &qsec); event_del(&ev2); event_count_active = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE, 0); event_count_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL, 0); event_count_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ADDED, 0); event_count_active_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_VIRTUAL, 0); event_count_active_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_ADDED, 0); event_count_virtual_added = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL | EVENT_BASE_COUNT_ADDED, 0); event_count_active_added_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_ADDED | EVENT_BASE_COUNT_VIRTUAL, 0); tt_int_op(event_count_active, ==, 0); tt_int_op(event_count_virtual, ==, 0); /* libevent itself adds a timeout event, so the event_count is 4 here */ tt_int_op(event_count_added, ==, 4); tt_int_op(event_count_active_virtual, ==, 0); tt_int_op(event_count_active_added, ==, 4); tt_int_op(event_count_virtual_added, ==, 4); tt_int_op(event_count_active_added_virtual, ==, 4); event_active(&ev, EV_READ, 1); event_count_active = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE, 0); event_count_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL, 0); event_count_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ADDED, 0); event_count_active_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_VIRTUAL, 0); event_count_active_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_ADDED, 0); event_count_virtual_added = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL | EVENT_BASE_COUNT_ADDED, 0); event_count_active_added_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_ADDED | EVENT_BASE_COUNT_VIRTUAL, 0); tt_int_op(event_count_active, ==, 1); tt_int_op(event_count_virtual, ==, 0); tt_int_op(event_count_added, ==, 4); tt_int_op(event_count_active_virtual, ==, 1); tt_int_op(event_count_active_added, ==, 5); tt_int_op(event_count_virtual_added, ==, 4); tt_int_op(event_count_active_added_virtual, ==, 5); event_base_loop(base, 0); event_count_active = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE, 1); event_count_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL, 1); event_count_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ADDED, 1); event_count_active_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_VIRTUAL, 0); event_count_active_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_ADDED, 0); event_count_virtual_added = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL | EVENT_BASE_COUNT_ADDED, 0); event_count_active_added_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_ADDED | EVENT_BASE_COUNT_VIRTUAL, 1); tt_int_op(event_count_active, ==, 1); tt_int_op(event_count_virtual, ==, 0); tt_int_op(event_count_added, ==, 4); tt_int_op(event_count_active_virtual, ==, 0); tt_int_op(event_count_active_added, ==, 0); tt_int_op(event_count_virtual_added, ==, 0); tt_int_op(event_count_active_added_virtual, ==, 0); event_count_active = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE, 0); event_count_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL, 0); event_count_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ADDED, 0); tt_int_op(event_count_active, ==, 0); tt_int_op(event_count_virtual, ==, 0); tt_int_op(event_count_added, ==, 0); event_base_add_virtual_(base); event_count_active = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE, 0); event_count_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL, 0); event_count_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ADDED, 0); event_count_active_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_VIRTUAL, 0); event_count_active_added = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_ADDED, 0); event_count_virtual_added = event_base_get_max_events(base, EVENT_BASE_COUNT_VIRTUAL | EVENT_BASE_COUNT_ADDED, 0); event_count_active_added_virtual = event_base_get_max_events(base, EVENT_BASE_COUNT_ACTIVE | EVENT_BASE_COUNT_ADDED | EVENT_BASE_COUNT_VIRTUAL, 0); tt_int_op(event_count_active, ==, 0); tt_int_op(event_count_virtual, ==, 1); tt_int_op(event_count_added, ==, 0); tt_int_op(event_count_active_virtual, ==, 1); tt_int_op(event_count_active_added, ==, 0); tt_int_op(event_count_virtual_added, ==, 1); tt_int_op(event_count_active_added_virtual, ==, 1); end: ; } static void test_bad_assign(void *ptr) { struct event ev; int r; /* READ|SIGNAL is not allowed */ r = event_assign(&ev, NULL, -1, EV_SIGNAL|EV_READ, dummy_read_cb, NULL); tt_int_op(r,==,-1); end: ; } static int reentrant_cb_run = 0; static void bad_reentrant_run_loop_cb(evutil_socket_t fd, short what, void *ptr) { struct event_base *base = ptr; int r; reentrant_cb_run = 1; /* This reentrant call to event_base_loop should be detected and * should fail */ r = event_base_loop(base, 0); tt_int_op(r, ==, -1); end: ; } static void test_bad_reentrant(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = data->base; struct event ev; int r; event_assign(&ev, base, -1, 0, bad_reentrant_run_loop_cb, base); event_active(&ev, EV_WRITE, 1); r = event_base_loop(base, 0); tt_int_op(r, ==, 1); tt_int_op(reentrant_cb_run, ==, 1); end: ; } static int n_write_a_byte_cb=0; static int n_read_and_drain_cb=0; static int n_activate_other_event_cb=0; static void write_a_byte_cb(evutil_socket_t fd, short what, void *arg) { char buf[] = "x"; if (write(fd, buf, 1) == 1) ++n_write_a_byte_cb; } static void read_and_drain_cb(evutil_socket_t fd, short what, void *arg) { char buf[128]; int n; ++n_read_and_drain_cb; while ((n = read(fd, buf, sizeof(buf))) > 0) ; } static void activate_other_event_cb(evutil_socket_t fd, short what, void *other_) { struct event *ev_activate = other_; ++n_activate_other_event_cb; event_active_later_(ev_activate, EV_READ); } static void test_active_later(void *ptr) { struct basic_test_data *data = ptr; struct event *ev1 = NULL, *ev2 = NULL; struct event ev3, ev4; struct timeval qsec = {0, 100000}; ev1 = event_new(data->base, data->pair[0], EV_READ|EV_PERSIST, read_and_drain_cb, NULL); ev2 = event_new(data->base, data->pair[1], EV_WRITE|EV_PERSIST, write_a_byte_cb, NULL); event_assign(&ev3, data->base, -1, 0, activate_other_event_cb, &ev4); event_assign(&ev4, data->base, -1, 0, activate_other_event_cb, &ev3); event_add(ev1, NULL); event_add(ev2, NULL); event_active_later_(&ev3, EV_READ); event_base_loopexit(data->base, &qsec); event_base_loop(data->base, 0); TT_BLATHER(("%d write calls, %d read calls, %d activate-other calls.", n_write_a_byte_cb, n_read_and_drain_cb, n_activate_other_event_cb)); event_del(&ev3); event_del(&ev4); tt_int_op(n_write_a_byte_cb, ==, n_activate_other_event_cb); tt_int_op(n_write_a_byte_cb, >, 100); tt_int_op(n_read_and_drain_cb, >, 100); tt_int_op(n_activate_other_event_cb, >, 100); event_active_later_(&ev4, EV_READ); event_active(&ev4, EV_READ, 1); /* This should make the event active immediately. */ tt_assert((ev4.ev_flags & EVLIST_ACTIVE) != 0); tt_assert((ev4.ev_flags & EVLIST_ACTIVE_LATER) == 0); /* Now leave this one around, so that event_free sees it and removes * it. */ event_active_later_(&ev3, EV_READ); event_base_assert_ok_(data->base); end: if (ev1) event_free(ev1); if (ev2) event_free(ev2); event_base_free(data->base); data->base = NULL; } static void incr_arg_cb(evutil_socket_t fd, short what, void *arg) { int *intptr = arg; (void) fd; (void) what; ++*intptr; } static void remove_timers_cb(evutil_socket_t fd, short what, void *arg) { struct event **ep = arg; (void) fd; (void) what; event_remove_timer(ep[0]); event_remove_timer(ep[1]); } static void send_a_byte_cb(evutil_socket_t fd, short what, void *arg) { evutil_socket_t *sockp = arg; (void) fd; (void) what; (void) write(*sockp, "A", 1); } struct read_not_timeout_param { struct event **ev; int events; int count; }; static void read_not_timeout_cb(evutil_socket_t fd, short what, void *arg) { struct read_not_timeout_param *rntp = arg; char c; ev_ssize_t n; (void) fd; (void) what; n = read(fd, &c, 1); tt_int_op(n, ==, 1); rntp->events |= what; ++rntp->count; if(2 == rntp->count) event_del(rntp->ev[0]); end: ; } static void test_event_remove_timeout(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = data->base; struct event *ev[5]; int ev1_fired=0; struct timeval ms25 = { 0, 25*1000 }, ms40 = { 0, 40*1000 }, ms75 = { 0, 75*1000 }, ms125 = { 0, 125*1000 }; struct read_not_timeout_param rntp = { ev, 0, 0 }; event_base_assert_ok_(base); ev[0] = event_new(base, data->pair[0], EV_READ|EV_PERSIST, read_not_timeout_cb, &rntp); ev[1] = evtimer_new(base, incr_arg_cb, &ev1_fired); ev[2] = evtimer_new(base, remove_timers_cb, ev); ev[3] = evtimer_new(base, send_a_byte_cb, &data->pair[1]); ev[4] = evtimer_new(base, send_a_byte_cb, &data->pair[1]); tt_assert(base); event_add(ev[2], &ms25); /* remove timers */ event_add(ev[4], &ms40); /* write to test if timer re-activates */ event_add(ev[0], &ms75); /* read */ event_add(ev[1], &ms75); /* timer */ event_add(ev[3], &ms125); /* timeout. */ event_base_assert_ok_(base); event_base_dispatch(base); tt_int_op(ev1_fired, ==, 0); tt_int_op(rntp.events, ==, EV_READ); event_base_assert_ok_(base); end: event_free(ev[0]); event_free(ev[1]); event_free(ev[2]); event_free(ev[3]); event_free(ev[4]); } static void test_event_base_new(void *ptr) { struct basic_test_data *data = ptr; struct event_base *base = 0; struct event ev1; struct basic_cb_args args; int towrite = (int)strlen(TEST1)+1; int len = write(data->pair[0], TEST1, towrite); if (len < 0) tt_abort_perror("initial write"); else if (len != towrite) tt_abort_printf(("initial write fell short (%d of %d bytes)", len, towrite)); if (shutdown(data->pair[0], EVUTIL_SHUT_WR)) tt_abort_perror("initial write shutdown"); base = event_base_new(); if (!base) tt_abort_msg("failed to create event base"); args.eb = base; args.ev = &ev1; args.callcount = 0; event_assign(&ev1, base, data->pair[1], EV_READ|EV_PERSIST, basic_read_cb, &args); if (event_add(&ev1, NULL)) tt_abort_perror("initial event_add"); if (event_base_loop(base, 0)) tt_abort_msg("unsuccessful exit from event loop"); end: if (base) event_base_free(base); } static void test_loopexit(void) { struct timeval tv, tv_start, tv_end; struct event ev; setup_test("Loop exit: "); tv.tv_usec = 0; tv.tv_sec = 60*60*24; evtimer_set(&ev, timeout_cb, NULL); evtimer_add(&ev, &tv); tv.tv_usec = 300*1000; tv.tv_sec = 0; event_loopexit(&tv); evutil_gettimeofday(&tv_start, NULL); event_dispatch(); evutil_gettimeofday(&tv_end, NULL); evtimer_del(&ev); tt_assert(event_base_got_exit(global_base)); tt_assert(!event_base_got_break(global_base)); test_timeval_diff_eq(&tv_start, &tv_end, 300); test_ok = 1; end: cleanup_test(); } static void test_loopexit_multiple(void) { struct timeval tv, tv_start, tv_end; struct event_base *base; setup_test("Loop Multiple exit: "); base = event_base_new(); tv.tv_usec = 200*1000; tv.tv_sec = 0; event_base_loopexit(base, &tv); tv.tv_usec = 0; tv.tv_sec = 3; event_base_loopexit(base, &tv); evutil_gettimeofday(&tv_start, NULL); event_base_dispatch(base); evutil_gettimeofday(&tv_end, NULL); tt_assert(event_base_got_exit(base)); tt_assert(!event_base_got_break(base)); event_base_free(base); test_timeval_diff_eq(&tv_start, &tv_end, 200); test_ok = 1; end: cleanup_test(); } static void break_cb(evutil_socket_t fd, short events, void *arg) { test_ok = 1; event_loopbreak(); } static void fail_cb(evutil_socket_t fd, short events, void *arg) { test_ok = 0; } static void test_loopbreak(void) { struct event ev1, ev2; struct timeval tv; setup_test("Loop break: "); tv.tv_sec = 0; tv.tv_usec = 0; evtimer_set(&ev1, break_cb, NULL); evtimer_add(&ev1, &tv); evtimer_set(&ev2, fail_cb, NULL); evtimer_add(&ev2, &tv); event_dispatch(); tt_assert(!event_base_got_exit(global_base)); tt_assert(event_base_got_break(global_base)); evtimer_del(&ev1); evtimer_del(&ev2); end: cleanup_test(); } static struct event *readd_test_event_last_added = NULL; static void re_add_read_cb(evutil_socket_t fd, short event, void *arg) { char buf[256]; struct event *ev_other = arg; ev_ssize_t n_read; readd_test_event_last_added = ev_other; n_read = read(fd, buf, sizeof(buf)); if (n_read < 0) { tt_fail_perror("read"); event_base_loopbreak(event_get_base(ev_other)); return; } else { event_add(ev_other, NULL); ++test_ok; } } static void test_nonpersist_readd(void) { struct event ev1, ev2; setup_test("Re-add nonpersistent events: "); event_set(&ev1, pair[0], EV_READ, re_add_read_cb, &ev2); event_set(&ev2, pair[1], EV_READ, re_add_read_cb, &ev1); if (write(pair[0], "Hello", 5) < 0) { tt_fail_perror("write(pair[0])"); } if (write(pair[1], "Hello", 5) < 0) { tt_fail_perror("write(pair[1])\n"); } if (event_add(&ev1, NULL) == -1 || event_add(&ev2, NULL) == -1) { test_ok = 0; } if (test_ok != 0) exit(1); event_loop(EVLOOP_ONCE); if (test_ok != 2) exit(1); /* At this point, we executed both callbacks. Whichever one got * called first added the second, but the second then immediately got * deleted before its callback was called. At this point, though, it * re-added the first. */ if (!readd_test_event_last_added) { test_ok = 0; } else if (readd_test_event_last_added == &ev1) { if (!event_pending(&ev1, EV_READ, NULL) || event_pending(&ev2, EV_READ, NULL)) test_ok = 0; } else { if (event_pending(&ev1, EV_READ, NULL) || !event_pending(&ev2, EV_READ, NULL)) test_ok = 0; } event_del(&ev1); event_del(&ev2); cleanup_test(); } struct test_pri_event { struct event ev; int count; }; static void test_priorities_cb(evutil_socket_t fd, short what, void *arg) { struct test_pri_event *pri = arg; struct timeval tv; if (pri->count == 3) { event_loopexit(NULL); return; } pri->count++; evutil_timerclear(&tv); event_add(&pri->ev, &tv); } static void test_priorities_impl(int npriorities) { struct test_pri_event one, two; struct timeval tv; TT_BLATHER(("Testing Priorities %d: ", npriorities)); event_base_priority_init(global_base, npriorities); memset(&one, 0, sizeof(one)); memset(&two, 0, sizeof(two)); timeout_set(&one.ev, test_priorities_cb, &one); if (event_priority_set(&one.ev, 0) == -1) { fprintf(stderr, "%s: failed to set priority", __func__); exit(1); } timeout_set(&two.ev, test_priorities_cb, &two); if (event_priority_set(&two.ev, npriorities - 1) == -1) { fprintf(stderr, "%s: failed to set priority", __func__); exit(1); } evutil_timerclear(&tv); if (event_add(&one.ev, &tv) == -1) exit(1); if (event_add(&two.ev, &tv) == -1) exit(1); event_dispatch(); event_del(&one.ev); event_del(&two.ev); if (npriorities == 1) { if (one.count == 3 && two.count == 3) test_ok = 1; } else if (npriorities == 2) { /* Two is called once because event_loopexit is priority 1 */ if (one.count == 3 && two.count == 1) test_ok = 1; } else { if (one.count == 3 && two.count == 0) test_ok = 1; } } static void test_priorities(void) { test_priorities_impl(1); if (test_ok) test_priorities_impl(2); if (test_ok) test_priorities_impl(3); } /* priority-active-inversion: activate a higher-priority event, and make sure * it keeps us from running a lower-priority event first. */ static int n_pai_calls = 0; static struct event pai_events[3]; static void prio_active_inversion_cb(evutil_socket_t fd, short what, void *arg) { int *call_order = arg; *call_order = n_pai_calls++; if (n_pai_calls == 1) { /* This should activate later, even though it shares a priority with us. */ event_active(&pai_events[1], EV_READ, 1); /* This should activate next, since its priority is higher, even though we activated it second. */ event_active(&pai_events[2], EV_TIMEOUT, 1); } } static void test_priority_active_inversion(void *data_) { struct basic_test_data *data = data_; struct event_base *base = data->base; int call_order[3]; int i; tt_int_op(event_base_priority_init(base, 8), ==, 0); n_pai_calls = 0; memset(call_order, 0, sizeof(call_order)); for (i=0;i<3;++i) { event_assign(&pai_events[i], data->base, -1, 0, prio_active_inversion_cb, &call_order[i]); } event_priority_set(&pai_events[0], 4); event_priority_set(&pai_events[1], 4); event_priority_set(&pai_events[2], 0); event_active(&pai_events[0], EV_WRITE, 1); event_base_dispatch(base); tt_int_op(n_pai_calls, ==, 3); tt_int_op(call_order[0], ==, 0); tt_int_op(call_order[1], ==, 2); tt_int_op(call_order[2], ==, 1); end: ; } static void test_multiple_cb(evutil_socket_t fd, short event, void *arg) { if (event & EV_READ) test_ok |= 1; else if (event & EV_WRITE) test_ok |= 2; } static void test_multiple_events_for_same_fd(void) { struct event e1, e2; setup_test("Multiple events for same fd: "); event_set(&e1, pair[0], EV_READ, test_multiple_cb, NULL); event_add(&e1, NULL); event_set(&e2, pair[0], EV_WRITE, test_multiple_cb, NULL); event_add(&e2, NULL); event_loop(EVLOOP_ONCE); event_del(&e2); if (write(pair[1], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); } event_loop(EVLOOP_ONCE); event_del(&e1); if (test_ok != 3) test_ok = 0; cleanup_test(); } int evtag_decode_int(ev_uint32_t *pnumber, struct evbuffer *evbuf); int evtag_decode_int64(ev_uint64_t *pnumber, struct evbuffer *evbuf); int evtag_encode_tag(struct evbuffer *evbuf, ev_uint32_t number); int evtag_decode_tag(ev_uint32_t *pnumber, struct evbuffer *evbuf); static void read_once_cb(evutil_socket_t fd, short event, void *arg) { char buf[256]; int len; len = read(fd, buf, sizeof(buf)); if (called) { test_ok = 0; } else if (len) { /* Assumes global pair[0] can be used for writing */ if (write(pair[0], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); test_ok = 0; } else { test_ok = 1; } } called++; } static void test_want_only_once(void) { struct event ev; struct timeval tv; /* Very simple read test */ setup_test("Want read only once: "); if (write(pair[0], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); } /* Setup the loop termination */ evutil_timerclear(&tv); tv.tv_usec = 300*1000; event_loopexit(&tv); event_set(&ev, pair[1], EV_READ, read_once_cb, &ev); if (event_add(&ev, NULL) == -1) exit(1); event_dispatch(); cleanup_test(); } #define TEST_MAX_INT 6 static void evtag_int_test(void *ptr) { struct evbuffer *tmp = evbuffer_new(); ev_uint32_t integers[TEST_MAX_INT] = { 0xaf0, 0x1000, 0x1, 0xdeadbeef, 0x00, 0xbef000 }; ev_uint32_t integer; ev_uint64_t big_int; int i; evtag_init(); for (i = 0; i < TEST_MAX_INT; i++) { int oldlen, newlen; oldlen = (int)EVBUFFER_LENGTH(tmp); evtag_encode_int(tmp, integers[i]); newlen = (int)EVBUFFER_LENGTH(tmp); TT_BLATHER(("encoded 0x%08x with %d bytes", (unsigned)integers[i], newlen - oldlen)); big_int = integers[i]; big_int *= 1000000000; /* 1 billion */ evtag_encode_int64(tmp, big_int); } for (i = 0; i < TEST_MAX_INT; i++) { tt_int_op(evtag_decode_int(&integer, tmp), !=, -1); tt_uint_op(integer, ==, integers[i]); tt_int_op(evtag_decode_int64(&big_int, tmp), !=, -1); tt_assert((big_int / 1000000000) == integers[i]); } tt_uint_op(EVBUFFER_LENGTH(tmp), ==, 0); end: evbuffer_free(tmp); } static void evtag_fuzz(void *ptr) { unsigned char buffer[4096]; struct evbuffer *tmp = evbuffer_new(); struct timeval tv; int i, j; int not_failed = 0; evtag_init(); for (j = 0; j < 100; j++) { for (i = 0; i < (int)sizeof(buffer); i++) buffer[i] = test_weakrand(); evbuffer_drain(tmp, -1); evbuffer_add(tmp, buffer, sizeof(buffer)); if (evtag_unmarshal_timeval(tmp, 0, &tv) != -1) not_failed++; } /* The majority of decodes should fail */ tt_int_op(not_failed, <, 10); /* Now insert some corruption into the tag length field */ evbuffer_drain(tmp, -1); evutil_timerclear(&tv); tv.tv_sec = 1; evtag_marshal_timeval(tmp, 0, &tv); evbuffer_add(tmp, buffer, sizeof(buffer)); ((char *)EVBUFFER_DATA(tmp))[1] = '\xff'; if (evtag_unmarshal_timeval(tmp, 0, &tv) != -1) { tt_abort_msg("evtag_unmarshal_timeval should have failed"); } end: evbuffer_free(tmp); } static void evtag_tag_encoding(void *ptr) { struct evbuffer *tmp = evbuffer_new(); ev_uint32_t integers[TEST_MAX_INT] = { 0xaf0, 0x1000, 0x1, 0xdeadbeef, 0x00, 0xbef000 }; ev_uint32_t integer; int i; evtag_init(); for (i = 0; i < TEST_MAX_INT; i++) { int oldlen, newlen; oldlen = (int)EVBUFFER_LENGTH(tmp); evtag_encode_tag(tmp, integers[i]); newlen = (int)EVBUFFER_LENGTH(tmp); TT_BLATHER(("encoded 0x%08x with %d bytes", (unsigned)integers[i], newlen - oldlen)); } for (i = 0; i < TEST_MAX_INT; i++) { tt_int_op(evtag_decode_tag(&integer, tmp), !=, -1); tt_uint_op(integer, ==, integers[i]); } tt_uint_op(EVBUFFER_LENGTH(tmp), ==, 0); end: evbuffer_free(tmp); } static void evtag_test_peek(void *ptr) { struct evbuffer *tmp = evbuffer_new(); ev_uint32_t u32; evtag_marshal_int(tmp, 30, 0); evtag_marshal_string(tmp, 40, "Hello world"); tt_int_op(evtag_peek(tmp, &u32), ==, 1); tt_int_op(u32, ==, 30); tt_int_op(evtag_peek_length(tmp, &u32), ==, 0); tt_int_op(u32, ==, 1+1+1); tt_int_op(evtag_consume(tmp), ==, 0); tt_int_op(evtag_peek(tmp, &u32), ==, 1); tt_int_op(u32, ==, 40); tt_int_op(evtag_peek_length(tmp, &u32), ==, 0); tt_int_op(u32, ==, 1+1+11); tt_int_op(evtag_payload_length(tmp, &u32), ==, 0); tt_int_op(u32, ==, 11); end: evbuffer_free(tmp); } static void test_methods(void *ptr) { const char **methods = event_get_supported_methods(); struct event_config *cfg = NULL; struct event_base *base = NULL; const char *backend; int n_methods = 0; tt_assert(methods); backend = methods[0]; while (*methods != NULL) { TT_BLATHER(("Support method: %s", *methods)); ++methods; ++n_methods; } cfg = event_config_new(); assert(cfg != NULL); tt_int_op(event_config_avoid_method(cfg, backend), ==, 0); event_config_set_flag(cfg, EVENT_BASE_FLAG_IGNORE_ENV); base = event_base_new_with_config(cfg); if (n_methods > 1) { tt_assert(base); tt_str_op(backend, !=, event_base_get_method(base)); } else { tt_assert(base == NULL); } end: if (base) event_base_free(base); if (cfg) event_config_free(cfg); } static void test_version(void *arg) { const char *vstr; ev_uint32_t vint; int major, minor, patch, n; vstr = event_get_version(); vint = event_get_version_number(); tt_assert(vstr); tt_assert(vint); tt_str_op(vstr, ==, LIBEVENT_VERSION); tt_int_op(vint, ==, LIBEVENT_VERSION_NUMBER); n = sscanf(vstr, "%d.%d.%d", &major, &minor, &patch); tt_assert(3 == n); tt_int_op((vint&0xffffff00), ==, ((major<<24)|(minor<<16)|(patch<<8))); end: ; } static void test_base_features(void *arg) { struct event_base *base = NULL; struct event_config *cfg = NULL; cfg = event_config_new(); tt_assert(0 == event_config_require_features(cfg, EV_FEATURE_ET)); base = event_base_new_with_config(cfg); if (base) { tt_int_op(EV_FEATURE_ET, ==, event_base_get_features(base) & EV_FEATURE_ET); } else { base = event_base_new(); tt_int_op(0, ==, event_base_get_features(base) & EV_FEATURE_ET); } end: if (base) event_base_free(base); if (cfg) event_config_free(cfg); } #ifdef EVENT__HAVE_SETENV #define SETENV_OK #elif !defined(EVENT__HAVE_SETENV) && defined(EVENT__HAVE_PUTENV) static void setenv(const char *k, const char *v, int o_) { char b[256]; evutil_snprintf(b, sizeof(b), "%s=%s",k,v); putenv(b); } #define SETENV_OK #endif #ifdef EVENT__HAVE_UNSETENV #define UNSETENV_OK #elif !defined(EVENT__HAVE_UNSETENV) && defined(EVENT__HAVE_PUTENV) static void unsetenv(const char *k) { char b[256]; evutil_snprintf(b, sizeof(b), "%s=",k); putenv(b); } #define UNSETENV_OK #endif #if defined(SETENV_OK) && defined(UNSETENV_OK) static void methodname_to_envvar(const char *mname, char *buf, size_t buflen) { char *cp; evutil_snprintf(buf, buflen, "EVENT_NO%s", mname); for (cp = buf; *cp; ++cp) { *cp = EVUTIL_TOUPPER_(*cp); } } #endif static void test_base_environ(void *arg) { struct event_base *base = NULL; struct event_config *cfg = NULL; #if defined(SETENV_OK) && defined(UNSETENV_OK) const char **basenames; int i, n_methods=0; char varbuf[128]; const char *defaultname, *ignoreenvname; /* See if unsetenv works before we rely on it. */ setenv("EVENT_NOWAFFLES", "1", 1); unsetenv("EVENT_NOWAFFLES"); if (getenv("EVENT_NOWAFFLES") != NULL) { #ifndef EVENT__HAVE_UNSETENV TT_DECLARE("NOTE", ("Can't fake unsetenv; skipping test")); #else TT_DECLARE("NOTE", ("unsetenv doesn't work; skipping test")); #endif tt_skip(); } basenames = event_get_supported_methods(); for (i = 0; basenames[i]; ++i) { methodname_to_envvar(basenames[i], varbuf, sizeof(varbuf)); unsetenv(varbuf); ++n_methods; } base = event_base_new(); tt_assert(base); defaultname = event_base_get_method(base); TT_BLATHER(("default is <%s>", defaultname)); event_base_free(base); base = NULL; /* Can we disable the method with EVENT_NOfoo ? */ if (!strcmp(defaultname, "epoll (with changelist)")) { setenv("EVENT_NOEPOLL", "1", 1); ignoreenvname = "epoll"; } else { methodname_to_envvar(defaultname, varbuf, sizeof(varbuf)); setenv(varbuf, "1", 1); ignoreenvname = defaultname; } /* Use an empty cfg rather than NULL so a failure doesn't exit() */ cfg = event_config_new(); base = event_base_new_with_config(cfg); event_config_free(cfg); cfg = NULL; if (n_methods == 1) { tt_assert(!base); } else { tt_assert(base); tt_str_op(defaultname, !=, event_base_get_method(base)); event_base_free(base); base = NULL; } /* Can we disable looking at the environment with IGNORE_ENV ? */ cfg = event_config_new(); event_config_set_flag(cfg, EVENT_BASE_FLAG_IGNORE_ENV); base = event_base_new_with_config(cfg); tt_assert(base); tt_str_op(ignoreenvname, ==, event_base_get_method(base)); #else tt_skip(); #endif end: if (base) event_base_free(base); if (cfg) event_config_free(cfg); } static void read_called_once_cb(evutil_socket_t fd, short event, void *arg) { tt_int_op(event, ==, EV_READ); called += 1; end: ; } static void timeout_called_once_cb(evutil_socket_t fd, short event, void *arg) { tt_int_op(event, ==, EV_TIMEOUT); called += 100; end: ; } static void immediate_called_twice_cb(evutil_socket_t fd, short event, void *arg) { tt_int_op(event, ==, EV_TIMEOUT); called += 1000; end: ; } static void test_event_once(void *ptr) { struct basic_test_data *data = ptr; struct timeval tv; int r; tv.tv_sec = 0; tv.tv_usec = 50*1000; called = 0; r = event_base_once(data->base, data->pair[0], EV_READ, read_called_once_cb, NULL, NULL); tt_int_op(r, ==, 0); r = event_base_once(data->base, -1, EV_TIMEOUT, timeout_called_once_cb, NULL, &tv); tt_int_op(r, ==, 0); r = event_base_once(data->base, -1, 0, NULL, NULL, NULL); tt_int_op(r, <, 0); r = event_base_once(data->base, -1, EV_TIMEOUT, immediate_called_twice_cb, NULL, NULL); tt_int_op(r, ==, 0); tv.tv_sec = 0; tv.tv_usec = 0; r = event_base_once(data->base, -1, EV_TIMEOUT, immediate_called_twice_cb, NULL, &tv); tt_int_op(r, ==, 0); if (write(data->pair[1], TEST1, strlen(TEST1)+1) < 0) { tt_fail_perror("write"); } shutdown(data->pair[1], EVUTIL_SHUT_WR); event_base_dispatch(data->base); tt_int_op(called, ==, 2101); end: ; } static void test_event_once_never(void *ptr) { struct basic_test_data *data = ptr; struct timeval tv; /* Have one trigger in 10 seconds (don't worry, because) */ tv.tv_sec = 10; tv.tv_usec = 0; called = 0; event_base_once(data->base, -1, EV_TIMEOUT, timeout_called_once_cb, NULL, &tv); /* But shut down the base in 75 msec. */ tv.tv_sec = 0; tv.tv_usec = 75*1000; event_base_loopexit(data->base, &tv); event_base_dispatch(data->base); tt_int_op(called, ==, 0); end: ; } static void test_event_pending(void *ptr) { struct basic_test_data *data = ptr; struct event *r=NULL, *w=NULL, *t=NULL; struct timeval tv, now, tv2; tv.tv_sec = 0; tv.tv_usec = 500 * 1000; r = event_new(data->base, data->pair[0], EV_READ, simple_read_cb, NULL); w = event_new(data->base, data->pair[1], EV_WRITE, simple_write_cb, NULL); t = evtimer_new(data->base, timeout_cb, NULL); tt_assert(r); tt_assert(w); tt_assert(t); evutil_gettimeofday(&now, NULL); event_add(r, NULL); event_add(t, &tv); tt_assert( event_pending(r, EV_READ, NULL)); tt_assert(!event_pending(w, EV_WRITE, NULL)); tt_assert(!event_pending(r, EV_WRITE, NULL)); tt_assert( event_pending(r, EV_READ|EV_WRITE, NULL)); tt_assert(!event_pending(r, EV_TIMEOUT, NULL)); tt_assert( event_pending(t, EV_TIMEOUT, NULL)); tt_assert( event_pending(t, EV_TIMEOUT, &tv2)); tt_assert(evutil_timercmp(&tv2, &now, >)); test_timeval_diff_eq(&now, &tv2, 500); end: if (r) { event_del(r); event_free(r); } if (w) { event_del(w); event_free(w); } if (t) { event_del(t); event_free(t); } } static void dfd_cb(evutil_socket_t fd, short e, void *data) { *(int*)data = (int)e; } static void test_event_closed_fd_poll(void *arg) { struct timeval tv; struct event *e; struct basic_test_data *data = (struct basic_test_data *)arg; int i = 0; if (strcmp(event_base_get_method(data->base), "poll")) { tinytest_set_test_skipped_(); return; } e = event_new(data->base, data->pair[0], EV_READ, dfd_cb, &i); tt_assert(e); tv.tv_sec = 0; tv.tv_usec = 500 * 1000; event_add(e, &tv); tt_assert(event_pending(e, EV_READ, NULL)); close(data->pair[0]); data->pair[0] = -1; /** avoids double-close */ event_base_loop(data->base, EVLOOP_ONCE); tt_int_op(i, ==, EV_READ); end: if (e) { event_del(e); event_free(e); } } #ifndef _WIN32 /* You can't do this test on windows, since dup2 doesn't work on sockets */ /* Regression test for our workaround for a fun epoll/linux related bug * where fd2 = dup(fd1); add(fd2); close(fd2); dup2(fd1,fd2); add(fd2) * will get you an EEXIST */ static void test_dup_fd(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct event *ev1=NULL, *ev2=NULL; int fd, dfd=-1; int ev1_got, ev2_got; tt_int_op(write(data->pair[0], "Hello world", strlen("Hello world")), >, 0); fd = data->pair[1]; dfd = dup(fd); tt_int_op(dfd, >=, 0); ev1 = event_new(base, fd, EV_READ|EV_PERSIST, dfd_cb, &ev1_got); ev2 = event_new(base, dfd, EV_READ|EV_PERSIST, dfd_cb, &ev2_got); ev1_got = ev2_got = 0; event_add(ev1, NULL); event_add(ev2, NULL); event_base_loop(base, EVLOOP_ONCE); tt_int_op(ev1_got, ==, EV_READ); tt_int_op(ev2_got, ==, EV_READ); /* Now close and delete dfd then dispatch. We need to do the * dispatch here so that when we add it later, we think there * was an intermediate delete. */ close(dfd); event_del(ev2); ev1_got = ev2_got = 0; event_base_loop(base, EVLOOP_ONCE); tt_want_int_op(ev1_got, ==, EV_READ); tt_int_op(ev2_got, ==, 0); /* Re-duplicate the fd. We need to get the same duplicated * value that we closed to provoke the epoll quirk. Also, we * need to change the events to write, or else the old lingering * read event will make the test pass whether the change was * successful or not. */ tt_int_op(dup2(fd, dfd), ==, dfd); event_free(ev2); ev2 = event_new(base, dfd, EV_WRITE|EV_PERSIST, dfd_cb, &ev2_got); event_add(ev2, NULL); ev1_got = ev2_got = 0; event_base_loop(base, EVLOOP_ONCE); tt_want_int_op(ev1_got, ==, EV_READ); tt_int_op(ev2_got, ==, EV_WRITE); end: if (ev1) event_free(ev1); if (ev2) event_free(ev2); if (dfd >= 0) close(dfd); } #endif #ifdef EVENT__DISABLE_MM_REPLACEMENT static void test_mm_functions(void *arg) { tinytest_set_test_skipped_(); } #else static int check_dummy_mem_ok(void *mem_) { char *mem = mem_; mem -= 16; return !memcmp(mem, "{[]}", 16); } static void * dummy_malloc(size_t len) { char *mem = malloc(len+16); memcpy(mem, "{[]}", 16); return mem+16; } static void * dummy_realloc(void *mem_, size_t len) { char *mem = mem_; if (!mem) return dummy_malloc(len); tt_want(check_dummy_mem_ok(mem_)); mem -= 16; mem = realloc(mem, len+16); return mem+16; } static void dummy_free(void *mem_) { char *mem = mem_; tt_want(check_dummy_mem_ok(mem_)); mem -= 16; free(mem); } static void test_mm_functions(void *arg) { struct event_base *b = NULL; struct event_config *cfg = NULL; event_set_mem_functions(dummy_malloc, dummy_realloc, dummy_free); cfg = event_config_new(); event_config_avoid_method(cfg, "Nonesuch"); b = event_base_new_with_config(cfg); tt_assert(b); tt_assert(check_dummy_mem_ok(b)); end: if (cfg) event_config_free(cfg); if (b) event_base_free(b); } #endif static void many_event_cb(evutil_socket_t fd, short event, void *arg) { int *calledp = arg; *calledp += 1; } static void test_many_events(void *arg) { /* Try 70 events that should all be ready at once. This will * exercise the "resize" code on most of the backends, and will make * sure that we can get past the 64-handle limit of some windows * functions. */ #define MANY 70 struct basic_test_data *data = arg; struct event_base *base = data->base; int one_at_a_time = data->setup_data != NULL; evutil_socket_t sock[MANY]; struct event *ev[MANY]; int called[MANY]; int i; int loopflags = EVLOOP_NONBLOCK, evflags=0; if (one_at_a_time) { loopflags |= EVLOOP_ONCE; evflags = EV_PERSIST; } memset(sock, 0xff, sizeof(sock)); memset(ev, 0, sizeof(ev)); memset(called, 0, sizeof(called)); for (i = 0; i < MANY; ++i) { /* We need an event that will hit the backend, and that will * be ready immediately. "Send a datagram" is an easy * instance of that. */ sock[i] = socket(AF_INET, SOCK_DGRAM, 0); tt_assert(sock[i] >= 0); called[i] = 0; ev[i] = event_new(base, sock[i], EV_WRITE|evflags, many_event_cb, &called[i]); event_add(ev[i], NULL); if (one_at_a_time) event_base_loop(base, EVLOOP_NONBLOCK|EVLOOP_ONCE); } event_base_loop(base, loopflags); for (i = 0; i < MANY; ++i) { if (one_at_a_time) tt_int_op(called[i], ==, MANY - i + 1); else tt_int_op(called[i], ==, 1); } end: for (i = 0; i < MANY; ++i) { if (ev[i]) event_free(ev[i]); if (sock[i] >= 0) evutil_closesocket(sock[i]); } #undef MANY } static void test_struct_event_size(void *arg) { tt_int_op(event_get_struct_event_size(), <=, sizeof(struct event)); end: ; } static void test_get_assignment(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct event *ev1 = NULL; const char *str = "foo"; struct event_base *b; evutil_socket_t s; short what; event_callback_fn cb; void *cb_arg; ev1 = event_new(base, data->pair[1], EV_READ, dummy_read_cb, (void*)str); event_get_assignment(ev1, &b, &s, &what, &cb, &cb_arg); tt_ptr_op(b, ==, base); tt_int_op(s, ==, data->pair[1]); tt_int_op(what, ==, EV_READ); tt_ptr_op(cb, ==, dummy_read_cb); tt_ptr_op(cb_arg, ==, str); /* Now make sure this doesn't crash. */ event_get_assignment(ev1, NULL, NULL, NULL, NULL, NULL); end: if (ev1) event_free(ev1); } struct foreach_helper { int count; const struct event *ev; }; static int foreach_count_cb(const struct event_base *base, const struct event *ev, void *arg) { struct foreach_helper *h = event_get_callback_arg(ev); struct timeval *tv = arg; if (event_get_callback(ev) != timeout_cb) return 0; tt_ptr_op(event_get_base(ev), ==, base); tt_int_op(tv->tv_sec, ==, 10); h->ev = ev; h->count++; return 0; end: return -1; } static int foreach_find_cb(const struct event_base *base, const struct event *ev, void *arg) { const struct event **ev_out = arg; struct foreach_helper *h = event_get_callback_arg(ev); if (event_get_callback(ev) != timeout_cb) return 0; if (h->count == 99) { *ev_out = ev; return 101; } return 0; } static void test_event_foreach(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct event *ev[5]; struct foreach_helper visited[5]; int i; struct timeval ten_sec = {10,0}; const struct event *ev_found = NULL; for (i = 0; i < 5; ++i) { visited[i].count = 0; visited[i].ev = NULL; ev[i] = event_new(base, -1, 0, timeout_cb, &visited[i]); } tt_int_op(-1, ==, event_base_foreach_event(NULL, foreach_count_cb, NULL)); tt_int_op(-1, ==, event_base_foreach_event(base, NULL, NULL)); event_add(ev[0], &ten_sec); event_add(ev[1], &ten_sec); event_active(ev[1], EV_TIMEOUT, 1); event_active(ev[2], EV_TIMEOUT, 1); event_add(ev[3], &ten_sec); /* Don't touch ev[4]. */ tt_int_op(0, ==, event_base_foreach_event(base, foreach_count_cb, &ten_sec)); tt_int_op(1, ==, visited[0].count); tt_int_op(1, ==, visited[1].count); tt_int_op(1, ==, visited[2].count); tt_int_op(1, ==, visited[3].count); tt_ptr_op(ev[0], ==, visited[0].ev); tt_ptr_op(ev[1], ==, visited[1].ev); tt_ptr_op(ev[2], ==, visited[2].ev); tt_ptr_op(ev[3], ==, visited[3].ev); visited[2].count = 99; tt_int_op(101, ==, event_base_foreach_event(base, foreach_find_cb, &ev_found)); tt_ptr_op(ev_found, ==, ev[2]); end: for (i=0; i<5; ++i) { event_free(ev[i]); } } static struct event_base *cached_time_base = NULL; static int cached_time_reset = 0; static int cached_time_sleep = 0; static void cache_time_cb(evutil_socket_t fd, short what, void *arg) { struct timeval *tv = arg; tt_int_op(0, ==, event_base_gettimeofday_cached(cached_time_base, tv)); if (cached_time_sleep) { struct timeval delay = { 0, 30*1000 }; evutil_usleep_(&delay); } if (cached_time_reset) { event_base_update_cache_time(cached_time_base); } end: ; } static void test_gettimeofday_cached(void *arg) { struct basic_test_data *data = arg; struct event_config *cfg = NULL; struct event_base *base = NULL; struct timeval tv1, tv2, tv3, now; struct event *ev1=NULL, *ev2=NULL, *ev3=NULL; int cached_time_disable = strstr(data->setup_data, "disable") != NULL; cfg = event_config_new(); if (cached_time_disable) { event_config_set_flag(cfg, EVENT_BASE_FLAG_NO_CACHE_TIME); } cached_time_base = base = event_base_new_with_config(cfg); tt_assert(base); /* Try gettimeofday_cached outside of an event loop. */ evutil_gettimeofday(&now, NULL); tt_int_op(0, ==, event_base_gettimeofday_cached(NULL, &tv1)); tt_int_op(0, ==, event_base_gettimeofday_cached(base, &tv2)); tt_int_op(timeval_msec_diff(&tv1, &tv2), <, 10); tt_int_op(timeval_msec_diff(&tv1, &now), <, 10); cached_time_reset = strstr(data->setup_data, "reset") != NULL; cached_time_sleep = strstr(data->setup_data, "sleep") != NULL; ev1 = event_new(base, -1, 0, cache_time_cb, &tv1); ev2 = event_new(base, -1, 0, cache_time_cb, &tv2); ev3 = event_new(base, -1, 0, cache_time_cb, &tv3); event_active(ev1, EV_TIMEOUT, 1); event_active(ev2, EV_TIMEOUT, 1); event_active(ev3, EV_TIMEOUT, 1); event_base_dispatch(base); if (cached_time_reset && cached_time_sleep) { tt_int_op(labs(timeval_msec_diff(&tv1,&tv2)), >, 10); tt_int_op(labs(timeval_msec_diff(&tv2,&tv3)), >, 10); } else if (cached_time_disable && cached_time_sleep) { tt_int_op(labs(timeval_msec_diff(&tv1,&tv2)), >, 10); tt_int_op(labs(timeval_msec_diff(&tv2,&tv3)), >, 10); } else if (! cached_time_disable) { tt_assert(evutil_timercmp(&tv1, &tv2, ==)); tt_assert(evutil_timercmp(&tv2, &tv3, ==)); } end: if (ev1) event_free(ev1); if (ev2) event_free(ev2); if (ev3) event_free(ev3); if (base) event_base_free(base); if (cfg) event_config_free(cfg); } static void tabf_cb(evutil_socket_t fd, short what, void *arg) { int *ptr = arg; *ptr = what; *ptr += 0x10000; } static void test_active_by_fd(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct event *ev1 = NULL, *ev2 = NULL, *ev3 = NULL, *ev4 = NULL; int e1,e2,e3,e4; #ifndef _WIN32 struct event *evsig = NULL; int es; #endif struct timeval tenmin = { 600, 0 }; /* Ensure no crash on nonexistent FD. */ event_base_active_by_fd(base, 1000, EV_READ); /* Ensure no crash on bogus FD. */ event_base_active_by_fd(base, -1, EV_READ); /* Ensure no crash on nonexistent/bogus signal. */ event_base_active_by_signal(base, 1000); event_base_active_by_signal(base, -1); event_base_assert_ok_(base); e1 = e2 = e3 = e4 = 0; ev1 = event_new(base, data->pair[0], EV_READ, tabf_cb, &e1); ev2 = event_new(base, data->pair[0], EV_WRITE, tabf_cb, &e2); ev3 = event_new(base, data->pair[1], EV_READ, tabf_cb, &e3); ev4 = event_new(base, data->pair[1], EV_READ, tabf_cb, &e4); tt_assert(ev1); tt_assert(ev2); tt_assert(ev3); tt_assert(ev4); #ifndef _WIN32 evsig = event_new(base, SIGHUP, EV_SIGNAL, tabf_cb, &es); tt_assert(evsig); event_add(evsig, &tenmin); #endif event_add(ev1, &tenmin); event_add(ev2, NULL); event_add(ev3, NULL); event_add(ev4, &tenmin); event_base_assert_ok_(base); /* Trigger 2, 3, 4 */ event_base_active_by_fd(base, data->pair[0], EV_WRITE); event_base_active_by_fd(base, data->pair[1], EV_READ); #ifndef _WIN32 event_base_active_by_signal(base, SIGHUP); #endif event_base_assert_ok_(base); event_base_loop(base, EVLOOP_ONCE); tt_int_op(e1, ==, 0); tt_int_op(e2, ==, EV_WRITE | 0x10000); tt_int_op(e3, ==, EV_READ | 0x10000); /* Mask out EV_WRITE here, since it could be genuinely writeable. */ tt_int_op((e4 & ~EV_WRITE), ==, EV_READ | 0x10000); #ifndef _WIN32 tt_int_op(es, ==, EV_SIGNAL | 0x10000); #endif end: if (ev1) event_free(ev1); if (ev2) event_free(ev2); if (ev3) event_free(ev3); if (ev4) event_free(ev4); #ifndef _WIN32 if (evsig) event_free(evsig); #endif } struct testcase_t main_testcases[] = { /* Some converted-over tests */ { "methods", test_methods, TT_FORK, NULL, NULL }, { "version", test_version, 0, NULL, NULL }, BASIC(base_features, TT_FORK|TT_NO_LOGS), { "base_environ", test_base_environ, TT_FORK, NULL, NULL }, BASIC(event_base_new, TT_FORK|TT_NEED_SOCKETPAIR), BASIC(free_active_base, TT_FORK|TT_NEED_SOCKETPAIR), BASIC(manipulate_active_events, TT_FORK|TT_NEED_BASE), BASIC(event_new_selfarg, TT_FORK|TT_NEED_BASE), BASIC(event_assign_selfarg, TT_FORK|TT_NEED_BASE), BASIC(event_base_get_num_events, TT_FORK|TT_NEED_BASE), BASIC(event_base_get_max_events, TT_FORK|TT_NEED_BASE), BASIC(bad_assign, TT_FORK|TT_NEED_BASE|TT_NO_LOGS), BASIC(bad_reentrant, TT_FORK|TT_NEED_BASE|TT_NO_LOGS), BASIC(active_later, TT_FORK|TT_NEED_BASE|TT_NEED_SOCKETPAIR), BASIC(event_remove_timeout, TT_FORK|TT_NEED_BASE|TT_NEED_SOCKETPAIR), /* These are still using the old API */ LEGACY(persistent_timeout, TT_FORK|TT_NEED_BASE), { "persistent_timeout_jump", test_persistent_timeout_jump, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "persistent_active_timeout", test_persistent_active_timeout, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, LEGACY(priorities, TT_FORK|TT_NEED_BASE), BASIC(priority_active_inversion, TT_FORK|TT_NEED_BASE), { "common_timeout", test_common_timeout, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, /* These legacy tests may not all need all of these flags. */ LEGACY(simpleread, TT_ISOLATED), LEGACY(simpleread_multiple, TT_ISOLATED), LEGACY(simplewrite, TT_ISOLATED), { "simpleclose", test_simpleclose, TT_FORK, &basic_setup, NULL }, LEGACY(multiple, TT_ISOLATED), LEGACY(persistent, TT_ISOLATED), LEGACY(combined, TT_ISOLATED), LEGACY(simpletimeout, TT_ISOLATED), LEGACY(loopbreak, TT_ISOLATED), LEGACY(loopexit, TT_ISOLATED), LEGACY(loopexit_multiple, TT_ISOLATED), LEGACY(nonpersist_readd, TT_ISOLATED), LEGACY(multiple_events_for_same_fd, TT_ISOLATED), LEGACY(want_only_once, TT_ISOLATED), { "event_once", test_event_once, TT_ISOLATED, &basic_setup, NULL }, { "event_once_never", test_event_once_never, TT_ISOLATED, &basic_setup, NULL }, { "event_pending", test_event_pending, TT_ISOLATED, &basic_setup, NULL }, { "event_closed_fd_poll", test_event_closed_fd_poll, TT_ISOLATED, &basic_setup, NULL }, #ifndef _WIN32 { "dup_fd", test_dup_fd, TT_ISOLATED, &basic_setup, NULL }, #endif { "mm_functions", test_mm_functions, TT_FORK, NULL, NULL }, { "many_events", test_many_events, TT_ISOLATED, &basic_setup, NULL }, { "many_events_slow_add", test_many_events, TT_ISOLATED, &basic_setup, (void*)1 }, { "struct_event_size", test_struct_event_size, 0, NULL, NULL }, BASIC(get_assignment, TT_FORK|TT_NEED_BASE|TT_NEED_SOCKETPAIR), BASIC(event_foreach, TT_FORK|TT_NEED_BASE), { "gettimeofday_cached", test_gettimeofday_cached, TT_FORK, &basic_setup, (void*)"" }, { "gettimeofday_cached_sleep", test_gettimeofday_cached, TT_FORK, &basic_setup, (void*)"sleep" }, { "gettimeofday_cached_reset", test_gettimeofday_cached, TT_FORK, &basic_setup, (void*)"sleep reset" }, { "gettimeofday_cached_disabled", test_gettimeofday_cached, TT_FORK, &basic_setup, (void*)"sleep disable" }, { "gettimeofday_cached_disabled_nosleep", test_gettimeofday_cached, TT_FORK, &basic_setup, (void*)"disable" }, BASIC(active_by_fd, TT_FORK|TT_NEED_BASE|TT_NEED_SOCKETPAIR), #ifndef _WIN32 LEGACY(fork, TT_ISOLATED), #endif #ifdef EVENT__HAVE_PTHREADS /** TODO: support win32 */ LEGACY(del_wait, TT_ISOLATED|TT_NEED_THREADS), #endif END_OF_TESTCASES }; struct testcase_t evtag_testcases[] = { { "int", evtag_int_test, TT_FORK, NULL, NULL }, { "fuzz", evtag_fuzz, TT_FORK, NULL, NULL }, { "encoding", evtag_tag_encoding, TT_FORK, NULL, NULL }, { "peek", evtag_test_peek, 0, NULL, NULL }, END_OF_TESTCASES }; struct testcase_t signal_testcases[] = { #ifndef _WIN32 LEGACY(simplestsignal, TT_ISOLATED), LEGACY(simplesignal, TT_ISOLATED), LEGACY(multiplesignal, TT_ISOLATED), LEGACY(immediatesignal, TT_ISOLATED), LEGACY(signal_dealloc, TT_ISOLATED), LEGACY(signal_pipeloss, TT_ISOLATED), LEGACY(signal_switchbase, TT_ISOLATED|TT_NO_LOGS), LEGACY(signal_restore, TT_ISOLATED), LEGACY(signal_assert, TT_ISOLATED), LEGACY(signal_while_processing, TT_ISOLATED), #endif END_OF_TESTCASES }; kytensor-2.49.0.7/third_party/libevent/test/regress.gen.c000066400000000000000000000660511527052371200233530ustar00rootroot00000000000000/* * Automatically generated from regress.rpc * by event_rpcgen.py/0.1. DO NOT EDIT THIS FILE. */ #include #include #include #include #include #include #include #if defined(EVENT____func__) && !defined(__func__) #define __func__ EVENT____func__ #endif #include "regress.gen.h" void event_warn(const char *fmt, ...); void event_warnx(const char *fmt, ...); /* * Implementation of msg */ static struct msg_access_ msg_base__ = { msg_from_name_assign, msg_from_name_get, msg_to_name_assign, msg_to_name_get, msg_attack_assign, msg_attack_get, msg_run_assign, msg_run_get, msg_run_add, }; struct msg * msg_new(void) { return msg_new_with_arg(NULL); } struct msg * msg_new_with_arg(void *unused) { struct msg *tmp; if ((tmp = malloc(sizeof(struct msg))) == NULL) { event_warn("%s: malloc", __func__); return (NULL); } tmp->base = &msg_base__; tmp->from_name_data = NULL; tmp->from_name_set = 0; tmp->to_name_data = NULL; tmp->to_name_set = 0; tmp->attack_data = NULL; tmp->attack_set = 0; tmp->run_data = NULL; tmp->run_length = 0; tmp->run_num_allocated = 0; tmp->run_set = 0; return (tmp); } static int msg_run_expand_to_hold_more(struct msg *msg) { int tobe_allocated = msg->run_num_allocated; struct run** new_data = NULL; tobe_allocated = !tobe_allocated ? 1 : tobe_allocated << 1; new_data = (struct run**) realloc(msg->run_data, tobe_allocated * sizeof(struct run*)); if (new_data == NULL) return -1; msg->run_data = new_data; msg->run_num_allocated = tobe_allocated; return 0;} struct run* msg_run_add(struct msg *msg) { if (++msg->run_length >= msg->run_num_allocated) { if (msg_run_expand_to_hold_more(msg)<0) goto error; } msg->run_data[msg->run_length - 1] = run_new(); if (msg->run_data[msg->run_length - 1] == NULL) goto error; msg->run_set = 1; return (msg->run_data[msg->run_length - 1]); error: --msg->run_length; return (NULL); } int msg_from_name_assign(struct msg *msg, const char * value) { if (msg->from_name_data != NULL) free(msg->from_name_data); if ((msg->from_name_data = strdup(value)) == NULL) return (-1); msg->from_name_set = 1; return (0); } int msg_to_name_assign(struct msg *msg, const char * value) { if (msg->to_name_data != NULL) free(msg->to_name_data); if ((msg->to_name_data = strdup(value)) == NULL) return (-1); msg->to_name_set = 1; return (0); } int msg_attack_assign(struct msg *msg, const struct kill* value) { struct evbuffer *tmp = NULL; if (msg->attack_set) { kill_clear(msg->attack_data); msg->attack_set = 0; } else { msg->attack_data = kill_new(); if (msg->attack_data == NULL) { event_warn("%s: kill_new()", __func__); goto error; } } if ((tmp = evbuffer_new()) == NULL) { event_warn("%s: evbuffer_new()", __func__); goto error; } kill_marshal(tmp, value); if (kill_unmarshal(msg->attack_data, tmp) == -1) { event_warnx("%s: kill_unmarshal", __func__); goto error; } msg->attack_set = 1; evbuffer_free(tmp); return (0); error: if (tmp != NULL) evbuffer_free(tmp); if (msg->attack_data != NULL) { kill_free(msg->attack_data); msg->attack_data = NULL; } return (-1); } int msg_run_assign(struct msg *msg, int off, const struct run* value) { if (!msg->run_set || off < 0 || off >= msg->run_length) return (-1); { int had_error = 0; struct evbuffer *tmp = NULL; run_clear(msg->run_data[off]); if ((tmp = evbuffer_new()) == NULL) { event_warn("%s: evbuffer_new()", __func__); had_error = 1; goto done; } run_marshal(tmp, value); if (run_unmarshal(msg->run_data[off], tmp) == -1) { event_warnx("%s: run_unmarshal", __func__); had_error = 1; goto done; } done:if (tmp != NULL) evbuffer_free(tmp); if (had_error) { run_clear(msg->run_data[off]); return (-1); } } return (0); } int msg_from_name_get(struct msg *msg, char * *value) { if (msg->from_name_set != 1) return (-1); *value = msg->from_name_data; return (0); } int msg_to_name_get(struct msg *msg, char * *value) { if (msg->to_name_set != 1) return (-1); *value = msg->to_name_data; return (0); } int msg_attack_get(struct msg *msg, struct kill* *value) { if (msg->attack_set != 1) { msg->attack_data = kill_new(); if (msg->attack_data == NULL) return (-1); msg->attack_set = 1; } *value = msg->attack_data; return (0); } int msg_run_get(struct msg *msg, int offset, struct run* *value) { if (!msg->run_set || offset < 0 || offset >= msg->run_length) return (-1); *value = msg->run_data[offset]; return (0); } void msg_clear(struct msg *tmp) { if (tmp->from_name_set == 1) { free(tmp->from_name_data); tmp->from_name_data = NULL; tmp->from_name_set = 0; } if (tmp->to_name_set == 1) { free(tmp->to_name_data); tmp->to_name_data = NULL; tmp->to_name_set = 0; } if (tmp->attack_set == 1) { kill_free(tmp->attack_data); tmp->attack_data = NULL; tmp->attack_set = 0; } if (tmp->run_set == 1) { int i; for (i = 0; i < tmp->run_length; ++i) { run_free(tmp->run_data[i]); } free(tmp->run_data); tmp->run_data = NULL; tmp->run_set = 0; tmp->run_length = 0; tmp->run_num_allocated = 0; } } void msg_free(struct msg *tmp) { if (tmp->from_name_data != NULL) free (tmp->from_name_data); if (tmp->to_name_data != NULL) free (tmp->to_name_data); if (tmp->attack_data != NULL) kill_free(tmp->attack_data); if (tmp->run_set == 1) { int i; for (i = 0; i < tmp->run_length; ++i) { run_free(tmp->run_data[i]); } free(tmp->run_data); tmp->run_data = NULL; tmp->run_set = 0; tmp->run_length = 0; tmp->run_num_allocated = 0; } free(tmp->run_data); free(tmp); } void msg_marshal(struct evbuffer *evbuf, const struct msg *tmp){ evtag_marshal_string(evbuf, MSG_FROM_NAME, tmp->from_name_data); evtag_marshal_string(evbuf, MSG_TO_NAME, tmp->to_name_data); if (tmp->attack_set) { evtag_marshal_kill(evbuf, MSG_ATTACK, tmp->attack_data); } if (tmp->run_set) { { int i; for (i = 0; i < tmp->run_length; ++i) { evtag_marshal_run(evbuf, MSG_RUN, tmp->run_data[i]); } } } } int msg_unmarshal(struct msg *tmp, struct evbuffer *evbuf) { ev_uint32_t tag; while (evbuffer_get_length(evbuf) > 0) { if (evtag_peek(evbuf, &tag) == -1) return (-1); switch (tag) { case MSG_FROM_NAME: if (tmp->from_name_set) return (-1); if (evtag_unmarshal_string(evbuf, MSG_FROM_NAME, &tmp->from_name_data) == -1) { event_warnx("%s: failed to unmarshal from_name", __func__); return (-1); } tmp->from_name_set = 1; break; case MSG_TO_NAME: if (tmp->to_name_set) return (-1); if (evtag_unmarshal_string(evbuf, MSG_TO_NAME, &tmp->to_name_data) == -1) { event_warnx("%s: failed to unmarshal to_name", __func__); return (-1); } tmp->to_name_set = 1; break; case MSG_ATTACK: if (tmp->attack_set) return (-1); tmp->attack_data = kill_new(); if (tmp->attack_data == NULL) return (-1); if (evtag_unmarshal_kill(evbuf, MSG_ATTACK, tmp->attack_data) == -1) { event_warnx("%s: failed to unmarshal attack", __func__); return (-1); } tmp->attack_set = 1; break; case MSG_RUN: if (tmp->run_length >= tmp->run_num_allocated && msg_run_expand_to_hold_more(tmp) < 0) { puts("HEY NOW"); return (-1); } tmp->run_data[tmp->run_length] = run_new(); if (tmp->run_data[tmp->run_length] == NULL) return (-1); if (evtag_unmarshal_run(evbuf, MSG_RUN, tmp->run_data[tmp->run_length]) == -1) { event_warnx("%s: failed to unmarshal run", __func__); return (-1); } ++tmp->run_length; tmp->run_set = 1; break; default: return -1; } } if (msg_complete(tmp) == -1) return (-1); return (0); } int msg_complete(struct msg *msg) { if (!msg->from_name_set) return (-1); if (!msg->to_name_set) return (-1); if (msg->attack_set && kill_complete(msg->attack_data) == -1) return (-1); { int i; for (i = 0; i < msg->run_length; ++i) { if (msg->run_set && run_complete(msg->run_data[i]) == -1) return (-1); } } return (0); } int evtag_unmarshal_msg(struct evbuffer *evbuf, ev_uint32_t need_tag, struct msg *msg) { ev_uint32_t tag; int res = -1; struct evbuffer *tmp = evbuffer_new(); if (evtag_unmarshal(evbuf, &tag, tmp) == -1 || tag != need_tag) goto error; if (msg_unmarshal(msg, tmp) == -1) goto error; res = 0; error: evbuffer_free(tmp); return (res); } void evtag_marshal_msg(struct evbuffer *evbuf, ev_uint32_t tag, const struct msg *msg) { struct evbuffer *buf_ = evbuffer_new(); assert(buf_ != NULL); msg_marshal(buf_, msg); evtag_marshal_buffer(evbuf, tag, buf_); evbuffer_free(buf_); } /* * Implementation of kill */ static struct kill_access_ kill_base__ = { kill_weapon_assign, kill_weapon_get, kill_action_assign, kill_action_get, kill_how_often_assign, kill_how_often_get, kill_how_often_add, }; struct kill * kill_new(void) { return kill_new_with_arg(NULL); } struct kill * kill_new_with_arg(void *unused) { struct kill *tmp; if ((tmp = malloc(sizeof(struct kill))) == NULL) { event_warn("%s: malloc", __func__); return (NULL); } tmp->base = &kill_base__; tmp->weapon_data = NULL; tmp->weapon_set = 0; tmp->action_data = NULL; tmp->action_set = 0; tmp->how_often_data = NULL; tmp->how_often_length = 0; tmp->how_often_num_allocated = 0; tmp->how_often_set = 0; return (tmp); } static int kill_how_often_expand_to_hold_more(struct kill *msg) { int tobe_allocated = msg->how_often_num_allocated; ev_uint32_t* new_data = NULL; tobe_allocated = !tobe_allocated ? 1 : tobe_allocated << 1; new_data = (ev_uint32_t*) realloc(msg->how_often_data, tobe_allocated * sizeof(ev_uint32_t)); if (new_data == NULL) return -1; msg->how_often_data = new_data; msg->how_often_num_allocated = tobe_allocated; return 0;} ev_uint32_t * kill_how_often_add(struct kill *msg, const ev_uint32_t value) { if (++msg->how_often_length >= msg->how_often_num_allocated) { if (kill_how_often_expand_to_hold_more(msg)<0) goto error; } msg->how_often_data[msg->how_often_length - 1] = value; msg->how_often_set = 1; return &(msg->how_often_data[msg->how_often_length - 1]); error: --msg->how_often_length; return (NULL); } int kill_weapon_assign(struct kill *msg, const char * value) { if (msg->weapon_data != NULL) free(msg->weapon_data); if ((msg->weapon_data = strdup(value)) == NULL) return (-1); msg->weapon_set = 1; return (0); } int kill_action_assign(struct kill *msg, const char * value) { if (msg->action_data != NULL) free(msg->action_data); if ((msg->action_data = strdup(value)) == NULL) return (-1); msg->action_set = 1; return (0); } int kill_how_often_assign(struct kill *msg, int off, const ev_uint32_t value) { if (!msg->how_often_set || off < 0 || off >= msg->how_often_length) return (-1); { msg->how_often_data[off] = value; } return (0); } int kill_weapon_get(struct kill *msg, char * *value) { if (msg->weapon_set != 1) return (-1); *value = msg->weapon_data; return (0); } int kill_action_get(struct kill *msg, char * *value) { if (msg->action_set != 1) return (-1); *value = msg->action_data; return (0); } int kill_how_often_get(struct kill *msg, int offset, ev_uint32_t *value) { if (!msg->how_often_set || offset < 0 || offset >= msg->how_often_length) return (-1); *value = msg->how_often_data[offset]; return (0); } void kill_clear(struct kill *tmp) { if (tmp->weapon_set == 1) { free(tmp->weapon_data); tmp->weapon_data = NULL; tmp->weapon_set = 0; } if (tmp->action_set == 1) { free(tmp->action_data); tmp->action_data = NULL; tmp->action_set = 0; } if (tmp->how_often_set == 1) { free(tmp->how_often_data); tmp->how_often_data = NULL; tmp->how_often_set = 0; tmp->how_often_length = 0; tmp->how_often_num_allocated = 0; } } void kill_free(struct kill *tmp) { if (tmp->weapon_data != NULL) free (tmp->weapon_data); if (tmp->action_data != NULL) free (tmp->action_data); if (tmp->how_often_set == 1) { free(tmp->how_often_data); tmp->how_often_data = NULL; tmp->how_often_set = 0; tmp->how_often_length = 0; tmp->how_often_num_allocated = 0; } free(tmp->how_often_data); free(tmp); } void kill_marshal(struct evbuffer *evbuf, const struct kill *tmp){ evtag_marshal_string(evbuf, KILL_WEAPON, tmp->weapon_data); evtag_marshal_string(evbuf, KILL_ACTION, tmp->action_data); if (tmp->how_often_set) { { int i; for (i = 0; i < tmp->how_often_length; ++i) { evtag_marshal_int(evbuf, KILL_HOW_OFTEN, tmp->how_often_data[i]); } } } } int kill_unmarshal(struct kill *tmp, struct evbuffer *evbuf) { ev_uint32_t tag; while (evbuffer_get_length(evbuf) > 0) { if (evtag_peek(evbuf, &tag) == -1) return (-1); switch (tag) { case KILL_WEAPON: if (tmp->weapon_set) return (-1); if (evtag_unmarshal_string(evbuf, KILL_WEAPON, &tmp->weapon_data) == -1) { event_warnx("%s: failed to unmarshal weapon", __func__); return (-1); } tmp->weapon_set = 1; break; case KILL_ACTION: if (tmp->action_set) return (-1); if (evtag_unmarshal_string(evbuf, KILL_ACTION, &tmp->action_data) == -1) { event_warnx("%s: failed to unmarshal action", __func__); return (-1); } tmp->action_set = 1; break; case KILL_HOW_OFTEN: if (tmp->how_often_length >= tmp->how_often_num_allocated && kill_how_often_expand_to_hold_more(tmp) < 0) { puts("HEY NOW"); return (-1); } if (evtag_unmarshal_int(evbuf, KILL_HOW_OFTEN, &tmp->how_often_data[tmp->how_often_length]) == -1) { event_warnx("%s: failed to unmarshal how_often", __func__); return (-1); } ++tmp->how_often_length; tmp->how_often_set = 1; break; default: return -1; } } if (kill_complete(tmp) == -1) return (-1); return (0); } int kill_complete(struct kill *msg) { if (!msg->weapon_set) return (-1); if (!msg->action_set) return (-1); return (0); } int evtag_unmarshal_kill(struct evbuffer *evbuf, ev_uint32_t need_tag, struct kill *msg) { ev_uint32_t tag; int res = -1; struct evbuffer *tmp = evbuffer_new(); if (evtag_unmarshal(evbuf, &tag, tmp) == -1 || tag != need_tag) goto error; if (kill_unmarshal(msg, tmp) == -1) goto error; res = 0; error: evbuffer_free(tmp); return (res); } void evtag_marshal_kill(struct evbuffer *evbuf, ev_uint32_t tag, const struct kill *msg) { struct evbuffer *buf_ = evbuffer_new(); assert(buf_ != NULL); kill_marshal(buf_, msg); evtag_marshal_buffer(evbuf, tag, buf_); evbuffer_free(buf_); } /* * Implementation of run */ static struct run_access_ run_base__ = { run_how_assign, run_how_get, run_some_bytes_assign, run_some_bytes_get, run_fixed_bytes_assign, run_fixed_bytes_get, run_notes_assign, run_notes_get, run_notes_add, run_large_number_assign, run_large_number_get, run_other_numbers_assign, run_other_numbers_get, run_other_numbers_add, }; struct run * run_new(void) { return run_new_with_arg(NULL); } struct run * run_new_with_arg(void *unused) { struct run *tmp; if ((tmp = malloc(sizeof(struct run))) == NULL) { event_warn("%s: malloc", __func__); return (NULL); } tmp->base = &run_base__; tmp->how_data = NULL; tmp->how_set = 0; tmp->some_bytes_data = NULL; tmp->some_bytes_length = 0; tmp->some_bytes_set = 0; memset(tmp->fixed_bytes_data, 0, sizeof(tmp->fixed_bytes_data)); tmp->fixed_bytes_set = 0; tmp->notes_data = NULL; tmp->notes_length = 0; tmp->notes_num_allocated = 0; tmp->notes_set = 0; tmp->large_number_data = 0; tmp->large_number_set = 0; tmp->other_numbers_data = NULL; tmp->other_numbers_length = 0; tmp->other_numbers_num_allocated = 0; tmp->other_numbers_set = 0; return (tmp); } static int run_notes_expand_to_hold_more(struct run *msg) { int tobe_allocated = msg->notes_num_allocated; char ** new_data = NULL; tobe_allocated = !tobe_allocated ? 1 : tobe_allocated << 1; new_data = (char **) realloc(msg->notes_data, tobe_allocated * sizeof(char *)); if (new_data == NULL) return -1; msg->notes_data = new_data; msg->notes_num_allocated = tobe_allocated; return 0;} char * * run_notes_add(struct run *msg, const char * value) { if (++msg->notes_length >= msg->notes_num_allocated) { if (run_notes_expand_to_hold_more(msg)<0) goto error; } if (value != NULL) { msg->notes_data[msg->notes_length - 1] = strdup(value); if (msg->notes_data[msg->notes_length - 1] == NULL) { goto error; } } else { msg->notes_data[msg->notes_length - 1] = NULL; } msg->notes_set = 1; return &(msg->notes_data[msg->notes_length - 1]); error: --msg->notes_length; return (NULL); } static int run_other_numbers_expand_to_hold_more(struct run *msg) { int tobe_allocated = msg->other_numbers_num_allocated; ev_uint32_t* new_data = NULL; tobe_allocated = !tobe_allocated ? 1 : tobe_allocated << 1; new_data = (ev_uint32_t*) realloc(msg->other_numbers_data, tobe_allocated * sizeof(ev_uint32_t)); if (new_data == NULL) return -1; msg->other_numbers_data = new_data; msg->other_numbers_num_allocated = tobe_allocated; return 0;} ev_uint32_t * run_other_numbers_add(struct run *msg, const ev_uint32_t value) { if (++msg->other_numbers_length >= msg->other_numbers_num_allocated) { if (run_other_numbers_expand_to_hold_more(msg)<0) goto error; } msg->other_numbers_data[msg->other_numbers_length - 1] = value; msg->other_numbers_set = 1; return &(msg->other_numbers_data[msg->other_numbers_length - 1]); error: --msg->other_numbers_length; return (NULL); } int run_how_assign(struct run *msg, const char * value) { if (msg->how_data != NULL) free(msg->how_data); if ((msg->how_data = strdup(value)) == NULL) return (-1); msg->how_set = 1; return (0); } int run_some_bytes_assign(struct run *msg, const ev_uint8_t * value, ev_uint32_t len) { if (msg->some_bytes_data != NULL) free (msg->some_bytes_data); msg->some_bytes_data = malloc(len); if (msg->some_bytes_data == NULL) return (-1); msg->some_bytes_set = 1; msg->some_bytes_length = len; memcpy(msg->some_bytes_data, value, len); return (0); } int run_fixed_bytes_assign(struct run *msg, const ev_uint8_t *value) { msg->fixed_bytes_set = 1; memcpy(msg->fixed_bytes_data, value, 24); return (0); } int run_notes_assign(struct run *msg, int off, const char * value) { if (!msg->notes_set || off < 0 || off >= msg->notes_length) return (-1); { if (msg->notes_data[off] != NULL) free(msg->notes_data[off]); msg->notes_data[off] = strdup(value); if (msg->notes_data[off] == NULL) { event_warnx("%s: strdup", __func__); return (-1); } } return (0); } int run_large_number_assign(struct run *msg, const ev_uint64_t value) { msg->large_number_set = 1; msg->large_number_data = value; return (0); } int run_other_numbers_assign(struct run *msg, int off, const ev_uint32_t value) { if (!msg->other_numbers_set || off < 0 || off >= msg->other_numbers_length) return (-1); { msg->other_numbers_data[off] = value; } return (0); } int run_how_get(struct run *msg, char * *value) { if (msg->how_set != 1) return (-1); *value = msg->how_data; return (0); } int run_some_bytes_get(struct run *msg, ev_uint8_t * *value, ev_uint32_t *plen) { if (msg->some_bytes_set != 1) return (-1); *value = msg->some_bytes_data; *plen = msg->some_bytes_length; return (0); } int run_fixed_bytes_get(struct run *msg, ev_uint8_t **value) { if (msg->fixed_bytes_set != 1) return (-1); *value = msg->fixed_bytes_data; return (0); } int run_notes_get(struct run *msg, int offset, char * *value) { if (!msg->notes_set || offset < 0 || offset >= msg->notes_length) return (-1); *value = msg->notes_data[offset]; return (0); } int run_large_number_get(struct run *msg, ev_uint64_t *value) { if (msg->large_number_set != 1) return (-1); *value = msg->large_number_data; return (0); } int run_other_numbers_get(struct run *msg, int offset, ev_uint32_t *value) { if (!msg->other_numbers_set || offset < 0 || offset >= msg->other_numbers_length) return (-1); *value = msg->other_numbers_data[offset]; return (0); } void run_clear(struct run *tmp) { if (tmp->how_set == 1) { free(tmp->how_data); tmp->how_data = NULL; tmp->how_set = 0; } if (tmp->some_bytes_set == 1) { free (tmp->some_bytes_data); tmp->some_bytes_data = NULL; tmp->some_bytes_length = 0; tmp->some_bytes_set = 0; } tmp->fixed_bytes_set = 0; memset(tmp->fixed_bytes_data, 0, sizeof(tmp->fixed_bytes_data)); if (tmp->notes_set == 1) { int i; for (i = 0; i < tmp->notes_length; ++i) { if (tmp->notes_data[i] != NULL) free(tmp->notes_data[i]); } free(tmp->notes_data); tmp->notes_data = NULL; tmp->notes_set = 0; tmp->notes_length = 0; tmp->notes_num_allocated = 0; } tmp->large_number_set = 0; if (tmp->other_numbers_set == 1) { free(tmp->other_numbers_data); tmp->other_numbers_data = NULL; tmp->other_numbers_set = 0; tmp->other_numbers_length = 0; tmp->other_numbers_num_allocated = 0; } } void run_free(struct run *tmp) { if (tmp->how_data != NULL) free (tmp->how_data); if (tmp->some_bytes_data != NULL) free(tmp->some_bytes_data); if (tmp->notes_set == 1) { int i; for (i = 0; i < tmp->notes_length; ++i) { if (tmp->notes_data[i] != NULL) free(tmp->notes_data[i]); } free(tmp->notes_data); tmp->notes_data = NULL; tmp->notes_set = 0; tmp->notes_length = 0; tmp->notes_num_allocated = 0; } free(tmp->notes_data); if (tmp->other_numbers_set == 1) { free(tmp->other_numbers_data); tmp->other_numbers_data = NULL; tmp->other_numbers_set = 0; tmp->other_numbers_length = 0; tmp->other_numbers_num_allocated = 0; } free(tmp->other_numbers_data); free(tmp); } void run_marshal(struct evbuffer *evbuf, const struct run *tmp){ evtag_marshal_string(evbuf, RUN_HOW, tmp->how_data); if (tmp->some_bytes_set) { evtag_marshal(evbuf, RUN_SOME_BYTES, tmp->some_bytes_data, tmp->some_bytes_length); } evtag_marshal(evbuf, RUN_FIXED_BYTES, tmp->fixed_bytes_data, (24)); if (tmp->notes_set) { { int i; for (i = 0; i < tmp->notes_length; ++i) { evtag_marshal_string(evbuf, RUN_NOTES, tmp->notes_data[i]); } } } if (tmp->large_number_set) { evtag_marshal_int64(evbuf, RUN_LARGE_NUMBER, tmp->large_number_data); } if (tmp->other_numbers_set) { { int i; for (i = 0; i < tmp->other_numbers_length; ++i) { evtag_marshal_int(evbuf, RUN_OTHER_NUMBERS, tmp->other_numbers_data[i]); } } } } int run_unmarshal(struct run *tmp, struct evbuffer *evbuf) { ev_uint32_t tag; while (evbuffer_get_length(evbuf) > 0) { if (evtag_peek(evbuf, &tag) == -1) return (-1); switch (tag) { case RUN_HOW: if (tmp->how_set) return (-1); if (evtag_unmarshal_string(evbuf, RUN_HOW, &tmp->how_data) == -1) { event_warnx("%s: failed to unmarshal how", __func__); return (-1); } tmp->how_set = 1; break; case RUN_SOME_BYTES: if (tmp->some_bytes_set) return (-1); if (evtag_payload_length(evbuf, &tmp->some_bytes_length) == -1) return (-1); if (tmp->some_bytes_length > evbuffer_get_length(evbuf)) return (-1); if ((tmp->some_bytes_data = malloc(tmp->some_bytes_length)) == NULL) return (-1); if (evtag_unmarshal_fixed(evbuf, RUN_SOME_BYTES, tmp->some_bytes_data, tmp->some_bytes_length) == -1) { event_warnx("%s: failed to unmarshal some_bytes", __func__); return (-1); } tmp->some_bytes_set = 1; break; case RUN_FIXED_BYTES: if (tmp->fixed_bytes_set) return (-1); if (evtag_unmarshal_fixed(evbuf, RUN_FIXED_BYTES, tmp->fixed_bytes_data, (24)) == -1) { event_warnx("%s: failed to unmarshal fixed_bytes", __func__); return (-1); } tmp->fixed_bytes_set = 1; break; case RUN_NOTES: if (tmp->notes_length >= tmp->notes_num_allocated && run_notes_expand_to_hold_more(tmp) < 0) { puts("HEY NOW"); return (-1); } if (evtag_unmarshal_string(evbuf, RUN_NOTES, &tmp->notes_data[tmp->notes_length]) == -1) { event_warnx("%s: failed to unmarshal notes", __func__); return (-1); } ++tmp->notes_length; tmp->notes_set = 1; break; case RUN_LARGE_NUMBER: if (tmp->large_number_set) return (-1); if (evtag_unmarshal_int64(evbuf, RUN_LARGE_NUMBER, &tmp->large_number_data) == -1) { event_warnx("%s: failed to unmarshal large_number", __func__); return (-1); } tmp->large_number_set = 1; break; case RUN_OTHER_NUMBERS: if (tmp->other_numbers_length >= tmp->other_numbers_num_allocated && run_other_numbers_expand_to_hold_more(tmp) < 0) { puts("HEY NOW"); return (-1); } if (evtag_unmarshal_int(evbuf, RUN_OTHER_NUMBERS, &tmp->other_numbers_data[tmp->other_numbers_length]) == -1) { event_warnx("%s: failed to unmarshal other_numbers", __func__); return (-1); } ++tmp->other_numbers_length; tmp->other_numbers_set = 1; break; default: return -1; } } if (run_complete(tmp) == -1) return (-1); return (0); } int run_complete(struct run *msg) { if (!msg->how_set) return (-1); if (!msg->fixed_bytes_set) return (-1); return (0); } int evtag_unmarshal_run(struct evbuffer *evbuf, ev_uint32_t need_tag, struct run *msg) { ev_uint32_t tag; int res = -1; struct evbuffer *tmp = evbuffer_new(); if (evtag_unmarshal(evbuf, &tag, tmp) == -1 || tag != need_tag) goto error; if (run_unmarshal(msg, tmp) == -1) goto error; res = 0; error: evbuffer_free(tmp); return (res); } void evtag_marshal_run(struct evbuffer *evbuf, ev_uint32_t tag, const struct run *msg) { struct evbuffer *buf_ = evbuffer_new(); assert(buf_ != NULL); run_marshal(buf_, msg); evtag_marshal_buffer(evbuf, tag, buf_); evbuffer_free(buf_); } kytensor-2.49.0.7/third_party/libevent/test/regress.gen.h000066400000000000000000000150721527052371200233550ustar00rootroot00000000000000/* * Automatically generated from regress.rpc */ #ifndef EVENT_RPCOUT_REGRESS_RPC_ #define EVENT_RPCOUT_REGRESS_RPC_ #include /* for ev_uint*_t */ #include struct msg; struct kill; struct run; /* Tag definition for msg */ enum msg_ { MSG_FROM_NAME=1, MSG_TO_NAME=2, MSG_ATTACK=3, MSG_RUN=4, MSG_MAX_TAGS }; /* Structure declaration for msg */ struct msg_access_ { int (*from_name_assign)(struct msg *, const char *); int (*from_name_get)(struct msg *, char * *); int (*to_name_assign)(struct msg *, const char *); int (*to_name_get)(struct msg *, char * *); int (*attack_assign)(struct msg *, const struct kill*); int (*attack_get)(struct msg *, struct kill* *); int (*run_assign)(struct msg *, int, const struct run*); int (*run_get)(struct msg *, int, struct run* *); struct run* (*run_add)(struct msg *msg); }; struct msg { struct msg_access_ *base; char *from_name_data; char *to_name_data; struct kill* attack_data; struct run* *run_data; int run_length; int run_num_allocated; ev_uint8_t from_name_set; ev_uint8_t to_name_set; ev_uint8_t attack_set; ev_uint8_t run_set; }; struct msg *msg_new(void); struct msg *msg_new_with_arg(void *); void msg_free(struct msg *); void msg_clear(struct msg *); void msg_marshal(struct evbuffer *, const struct msg *); int msg_unmarshal(struct msg *, struct evbuffer *); int msg_complete(struct msg *); void evtag_marshal_msg(struct evbuffer *, ev_uint32_t, const struct msg *); int evtag_unmarshal_msg(struct evbuffer *, ev_uint32_t, struct msg *); int msg_from_name_assign(struct msg *, const char *); int msg_from_name_get(struct msg *, char * *); int msg_to_name_assign(struct msg *, const char *); int msg_to_name_get(struct msg *, char * *); int msg_attack_assign(struct msg *, const struct kill*); int msg_attack_get(struct msg *, struct kill* *); int msg_run_assign(struct msg *, int, const struct run*); int msg_run_get(struct msg *, int, struct run* *); struct run* msg_run_add(struct msg *msg); /* --- msg done --- */ /* Tag definition for kill */ enum kill_ { KILL_WEAPON=65825, KILL_ACTION=2, KILL_HOW_OFTEN=3, KILL_MAX_TAGS }; /* Structure declaration for kill */ struct kill_access_ { int (*weapon_assign)(struct kill *, const char *); int (*weapon_get)(struct kill *, char * *); int (*action_assign)(struct kill *, const char *); int (*action_get)(struct kill *, char * *); int (*how_often_assign)(struct kill *, int, const ev_uint32_t); int (*how_often_get)(struct kill *, int, ev_uint32_t *); ev_uint32_t * (*how_often_add)(struct kill *msg, const ev_uint32_t value); }; struct kill { struct kill_access_ *base; char *weapon_data; char *action_data; ev_uint32_t *how_often_data; int how_often_length; int how_often_num_allocated; ev_uint8_t weapon_set; ev_uint8_t action_set; ev_uint8_t how_often_set; }; struct kill *kill_new(void); struct kill *kill_new_with_arg(void *); void kill_free(struct kill *); void kill_clear(struct kill *); void kill_marshal(struct evbuffer *, const struct kill *); int kill_unmarshal(struct kill *, struct evbuffer *); int kill_complete(struct kill *); void evtag_marshal_kill(struct evbuffer *, ev_uint32_t, const struct kill *); int evtag_unmarshal_kill(struct evbuffer *, ev_uint32_t, struct kill *); int kill_weapon_assign(struct kill *, const char *); int kill_weapon_get(struct kill *, char * *); int kill_action_assign(struct kill *, const char *); int kill_action_get(struct kill *, char * *); int kill_how_often_assign(struct kill *, int, const ev_uint32_t); int kill_how_often_get(struct kill *, int, ev_uint32_t *); ev_uint32_t * kill_how_often_add(struct kill *msg, const ev_uint32_t value); /* --- kill done --- */ /* Tag definition for run */ enum run_ { RUN_HOW=1, RUN_SOME_BYTES=2, RUN_FIXED_BYTES=3, RUN_NOTES=4, RUN_LARGE_NUMBER=5, RUN_OTHER_NUMBERS=6, RUN_MAX_TAGS }; /* Structure declaration for run */ struct run_access_ { int (*how_assign)(struct run *, const char *); int (*how_get)(struct run *, char * *); int (*some_bytes_assign)(struct run *, const ev_uint8_t *, ev_uint32_t); int (*some_bytes_get)(struct run *, ev_uint8_t * *, ev_uint32_t *); int (*fixed_bytes_assign)(struct run *, const ev_uint8_t *); int (*fixed_bytes_get)(struct run *, ev_uint8_t **); int (*notes_assign)(struct run *, int, const char *); int (*notes_get)(struct run *, int, char * *); char * * (*notes_add)(struct run *msg, const char * value); int (*large_number_assign)(struct run *, const ev_uint64_t); int (*large_number_get)(struct run *, ev_uint64_t *); int (*other_numbers_assign)(struct run *, int, const ev_uint32_t); int (*other_numbers_get)(struct run *, int, ev_uint32_t *); ev_uint32_t * (*other_numbers_add)(struct run *msg, const ev_uint32_t value); }; struct run { struct run_access_ *base; char *how_data; ev_uint8_t *some_bytes_data; ev_uint32_t some_bytes_length; ev_uint8_t fixed_bytes_data[24]; char * *notes_data; int notes_length; int notes_num_allocated; ev_uint64_t large_number_data; ev_uint32_t *other_numbers_data; int other_numbers_length; int other_numbers_num_allocated; ev_uint8_t how_set; ev_uint8_t some_bytes_set; ev_uint8_t fixed_bytes_set; ev_uint8_t notes_set; ev_uint8_t large_number_set; ev_uint8_t other_numbers_set; }; struct run *run_new(void); struct run *run_new_with_arg(void *); void run_free(struct run *); void run_clear(struct run *); void run_marshal(struct evbuffer *, const struct run *); int run_unmarshal(struct run *, struct evbuffer *); int run_complete(struct run *); void evtag_marshal_run(struct evbuffer *, ev_uint32_t, const struct run *); int evtag_unmarshal_run(struct evbuffer *, ev_uint32_t, struct run *); int run_how_assign(struct run *, const char *); int run_how_get(struct run *, char * *); int run_some_bytes_assign(struct run *, const ev_uint8_t *, ev_uint32_t); int run_some_bytes_get(struct run *, ev_uint8_t * *, ev_uint32_t *); int run_fixed_bytes_assign(struct run *, const ev_uint8_t *); int run_fixed_bytes_get(struct run *, ev_uint8_t **); int run_notes_assign(struct run *, int, const char *); int run_notes_get(struct run *, int, char * *); char * * run_notes_add(struct run *msg, const char * value); int run_large_number_assign(struct run *, const ev_uint64_t); int run_large_number_get(struct run *, ev_uint64_t *); int run_other_numbers_assign(struct run *, int, const ev_uint32_t); int run_other_numbers_get(struct run *, int, ev_uint32_t *); ev_uint32_t * run_other_numbers_add(struct run *msg, const ev_uint32_t value); /* --- run done --- */ #endif /* EVENT_RPCOUT_REGRESS_RPC_ */ kytensor-2.49.0.7/third_party/libevent/test/regress.h000066400000000000000000000117171527052371200226070ustar00rootroot00000000000000/* * Copyright (c) 2000-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef REGRESS_H_INCLUDED_ #define REGRESS_H_INCLUDED_ #ifdef __cplusplus extern "C" { #endif #include "tinytest.h" #include "tinytest_macros.h" extern struct testcase_t main_testcases[]; extern struct testcase_t evtag_testcases[]; extern struct testcase_t evbuffer_testcases[]; extern struct testcase_t finalize_testcases[]; extern struct testcase_t bufferevent_testcases[]; extern struct testcase_t bufferevent_iocp_testcases[]; extern struct testcase_t util_testcases[]; extern struct testcase_t signal_testcases[]; extern struct testcase_t http_testcases[]; extern struct testcase_t dns_testcases[]; extern struct testcase_t rpc_testcases[]; extern struct testcase_t edgetriggered_testcases[]; extern struct testcase_t minheap_testcases[]; extern struct testcase_t iocp_testcases[]; extern struct testcase_t ssl_testcases[]; extern struct testcase_t listener_testcases[]; extern struct testcase_t listener_iocp_testcases[]; extern struct testcase_t thread_testcases[]; extern struct evutil_weakrand_state test_weakrand_state; #define test_weakrand() (evutil_weakrand_(&test_weakrand_state)) void regress_threads(void *); void test_bufferevent_zlib(void *); /* Helpers to wrap old testcases */ extern evutil_socket_t pair[2]; extern int test_ok; extern int called; extern struct event_base *global_base; extern int in_legacy_test_wrapper; int regress_make_tmpfile(const void *data, size_t datalen, char **filename_out); struct basic_test_data { struct event_base *base; evutil_socket_t pair[2]; void (*legacy_test_fn)(void); void *setup_data; }; extern const struct testcase_setup_t basic_setup; extern const struct testcase_setup_t legacy_setup; void run_legacy_test_fn(void *ptr); extern int libevent_tests_running_in_debug_mode; /* A couple of flags that basic/legacy_setup can support. */ #define TT_NEED_SOCKETPAIR TT_FIRST_USER_FLAG #define TT_NEED_BASE (TT_FIRST_USER_FLAG<<1) #define TT_NEED_DNS (TT_FIRST_USER_FLAG<<2) #define TT_LEGACY (TT_FIRST_USER_FLAG<<3) #define TT_NEED_THREADS (TT_FIRST_USER_FLAG<<4) #define TT_NO_LOGS (TT_FIRST_USER_FLAG<<5) #define TT_ENABLE_IOCP_FLAG (TT_FIRST_USER_FLAG<<6) #define TT_ENABLE_IOCP (TT_ENABLE_IOCP_FLAG|TT_NEED_THREADS) /* All the flags that a legacy test needs. */ #define TT_ISOLATED TT_FORK|TT_NEED_SOCKETPAIR|TT_NEED_BASE #define BASIC(name,flags) \ { #name, test_## name, flags, &basic_setup, NULL } #define LEGACY(name,flags) \ { #name, run_legacy_test_fn, flags|TT_LEGACY, &legacy_setup, \ test_## name } struct evutil_addrinfo; struct evutil_addrinfo *ai_find_by_family(struct evutil_addrinfo *ai, int f); struct evutil_addrinfo *ai_find_by_protocol(struct evutil_addrinfo *ai, int p); int test_ai_eq_(const struct evutil_addrinfo *ai, const char *sockaddr_port, int socktype, int protocol, int line); #define test_ai_eq(ai, str, s, p) do { \ if (test_ai_eq_((ai), (str), (s), (p), __LINE__)<0) \ goto end; \ } while (0) #define test_timeval_diff_leq(tv1, tv2, diff, tolerance) \ tt_int_op(labs(timeval_msec_diff((tv1), (tv2)) - diff), <=, tolerance) #define test_timeval_diff_eq(tv1, tv2, diff) \ test_timeval_diff_leq((tv1), (tv2), (diff), 50) long timeval_msec_diff(const struct timeval *start, const struct timeval *end); #ifndef _WIN32 pid_t regress_fork(void); #endif #ifdef EVENT__HAVE_OPENSSL #include EVP_PKEY *ssl_getkey(void); X509 *ssl_getcert(void); SSL_CTX *get_ssl_ctx(void); void init_ssl(void); #endif #ifdef __cplusplus } #endif #endif /* REGRESS_H_INCLUDED_ */ kytensor-2.49.0.7/third_party/libevent/test/regress.rpc000066400000000000000000000007211527052371200231350ustar00rootroot00000000000000/* tests data packing and unpacking */ struct msg { string /* sender */ from_name = 1; /* be verbose */ string to_name = 2; optional struct[kill] attack = 3; array struct[run] run = 4; } struct kill { string weapon = 0x10121; string action = 2; array int how_often = 3; } struct run { string how = 1; optional bytes some_bytes = 2; bytes fixed_bytes[24] = 3; array string notes = 4; optional int64 large_number = 5; array int other_numbers = 6; } kytensor-2.49.0.7/third_party/libevent/test/regress_buffer.c000066400000000000000000002174061527052371200241360ustar00rootroot00000000000000/* * Copyright (c) 2003-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "util-internal.h" #ifdef _WIN32 #include #include #endif #include "event2/event-config.h" #include #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #ifndef _WIN32 #include #include #include #include #include #endif #include #include #include #include #include #include "event2/event.h" #include "event2/buffer.h" #include "event2/buffer_compat.h" #include "event2/util.h" #include "defer-internal.h" #include "evbuffer-internal.h" #include "log-internal.h" #include "regress.h" /* Validates that an evbuffer is good. Returns false if it isn't, true if it * is*/ static int evbuffer_validate_(struct evbuffer *buf) { struct evbuffer_chain *chain; size_t sum = 0; int found_last_with_datap = 0; if (buf->first == NULL) { tt_assert(buf->last == NULL); tt_assert(buf->total_len == 0); } chain = buf->first; tt_assert(buf->last_with_datap); if (buf->last_with_datap == &buf->first) found_last_with_datap = 1; while (chain != NULL) { if (&chain->next == buf->last_with_datap) found_last_with_datap = 1; sum += chain->off; if (chain->next == NULL) { tt_assert(buf->last == chain); } tt_assert(chain->buffer_len >= chain->misalign + chain->off); chain = chain->next; } if (buf->first) tt_assert(*buf->last_with_datap); if (*buf->last_with_datap) { chain = *buf->last_with_datap; if (chain->off == 0 || buf->total_len == 0) { tt_assert(chain->off == 0) tt_assert(chain == buf->first); tt_assert(buf->total_len == 0); } chain = chain->next; while (chain != NULL) { tt_assert(chain->off == 0); chain = chain->next; } } else { tt_assert(buf->last_with_datap == &buf->first); } tt_assert(found_last_with_datap); tt_assert(sum == buf->total_len); return 1; end: return 0; } static void evbuffer_get_waste(struct evbuffer *buf, size_t *allocatedp, size_t *wastedp, size_t *usedp) { struct evbuffer_chain *chain; size_t a, w, u; int n = 0; u = a = w = 0; chain = buf->first; /* skip empty at start */ while (chain && chain->off==0) { ++n; a += chain->buffer_len; chain = chain->next; } /* first nonempty chain: stuff at the end only is wasted. */ if (chain) { ++n; a += chain->buffer_len; u += chain->off; if (chain->next && chain->next->off) w += (size_t)(chain->buffer_len - (chain->misalign + chain->off)); chain = chain->next; } /* subsequent nonempty chains */ while (chain && chain->off) { ++n; a += chain->buffer_len; w += (size_t)chain->misalign; u += chain->off; if (chain->next && chain->next->off) w += (size_t) (chain->buffer_len - (chain->misalign + chain->off)); chain = chain->next; } /* subsequent empty chains */ while (chain) { ++n; a += chain->buffer_len; } *allocatedp = a; *wastedp = w; *usedp = u; } #define evbuffer_validate(buf) \ TT_STMT_BEGIN if (!evbuffer_validate_(buf)) TT_DIE(("Buffer format invalid")); TT_STMT_END static void test_evbuffer(void *ptr) { static char buffer[512], *tmp; struct evbuffer *evb = evbuffer_new(); struct evbuffer *evb_two = evbuffer_new(); size_t sz_tmp; int i; evbuffer_validate(evb); evbuffer_add_printf(evb, "%s/%d", "hello", 1); evbuffer_validate(evb); tt_assert(evbuffer_get_length(evb) == 7); tt_assert(!memcmp((char*)EVBUFFER_DATA(evb), "hello/1", 1)); evbuffer_add_buffer(evb, evb_two); evbuffer_validate(evb); evbuffer_drain(evb, strlen("hello/")); evbuffer_validate(evb); tt_assert(evbuffer_get_length(evb) == 1); tt_assert(!memcmp((char*)EVBUFFER_DATA(evb), "1", 1)); evbuffer_add_printf(evb_two, "%s", "/hello"); evbuffer_validate(evb); evbuffer_add_buffer(evb, evb_two); evbuffer_validate(evb); tt_assert(evbuffer_get_length(evb_two) == 0); tt_assert(evbuffer_get_length(evb) == 7); tt_assert(!memcmp((char*)EVBUFFER_DATA(evb), "1/hello", 7)); memset(buffer, 0, sizeof(buffer)); evbuffer_add(evb, buffer, sizeof(buffer)); evbuffer_validate(evb); tt_assert(evbuffer_get_length(evb) == 7 + 512); tmp = (char *)evbuffer_pullup(evb, 7 + 512); tt_assert(tmp); tt_assert(!strncmp(tmp, "1/hello", 7)); tt_assert(!memcmp(tmp + 7, buffer, sizeof(buffer))); evbuffer_validate(evb); evbuffer_prepend(evb, "something", 9); evbuffer_validate(evb); evbuffer_prepend(evb, "else", 4); evbuffer_validate(evb); tmp = (char *)evbuffer_pullup(evb, 4 + 9 + 7); tt_assert(!strncmp(tmp, "elsesomething1/hello", 4 + 9 + 7)); evbuffer_validate(evb); evbuffer_drain(evb, -1); evbuffer_validate(evb); evbuffer_drain(evb_two, -1); evbuffer_validate(evb); for (i = 0; i < 3; ++i) { evbuffer_add(evb_two, buffer, sizeof(buffer)); evbuffer_validate(evb_two); evbuffer_add_buffer(evb, evb_two); evbuffer_validate(evb); evbuffer_validate(evb_two); } tt_assert(evbuffer_get_length(evb_two) == 0); tt_assert(evbuffer_get_length(evb) == i * sizeof(buffer)); /* test remove buffer */ sz_tmp = (size_t)(sizeof(buffer)*2.5); evbuffer_remove_buffer(evb, evb_two, sz_tmp); tt_assert(evbuffer_get_length(evb_two) == sz_tmp); tt_assert(evbuffer_get_length(evb) == sizeof(buffer) / 2); evbuffer_validate(evb); if (memcmp(evbuffer_pullup( evb, -1), buffer, sizeof(buffer) / 2) != 0 || memcmp(evbuffer_pullup( evb_two, -1), buffer, sizeof(buffer)) != 0) tt_abort_msg("Pullup did not preserve content"); evbuffer_validate(evb); /* testing one-vector reserve and commit */ { struct evbuffer_iovec v[1]; char *buf; int i, j, r; for (i = 0; i < 3; ++i) { r = evbuffer_reserve_space(evb, 10000, v, 1); tt_int_op(r, ==, 1); tt_assert(v[0].iov_len >= 10000); tt_assert(v[0].iov_base != NULL); evbuffer_validate(evb); buf = v[0].iov_base; for (j = 0; j < 10000; ++j) { buf[j] = j; } evbuffer_validate(evb); tt_int_op(evbuffer_commit_space(evb, v, 1), ==, 0); evbuffer_validate(evb); tt_assert(evbuffer_get_length(evb) >= 10000); evbuffer_drain(evb, j * 5000); evbuffer_validate(evb); } } end: evbuffer_free(evb); evbuffer_free(evb_two); } static void no_cleanup(const void *data, size_t datalen, void *extra) { } static void test_evbuffer_remove_buffer_with_empty(void *ptr) { struct evbuffer *src = evbuffer_new(); struct evbuffer *dst = evbuffer_new(); char buf[2]; evbuffer_validate(src); evbuffer_validate(dst); /* setup the buffers */ /* we need more data in src than we will move later */ evbuffer_add_reference(src, buf, sizeof(buf), no_cleanup, NULL); evbuffer_add_reference(src, buf, sizeof(buf), no_cleanup, NULL); /* we need one buffer in dst and one empty buffer at the end */ evbuffer_add(dst, buf, sizeof(buf)); evbuffer_add_reference(dst, buf, 0, no_cleanup, NULL); evbuffer_validate(src); evbuffer_validate(dst); /* move three bytes over */ evbuffer_remove_buffer(src, dst, 3); evbuffer_validate(src); evbuffer_validate(dst); end: evbuffer_free(src); evbuffer_free(dst); } static void test_evbuffer_remove_buffer_with_empty2(void *ptr) { struct evbuffer *src = evbuffer_new(); struct evbuffer *dst = evbuffer_new(); struct evbuffer *buf = evbuffer_new(); evbuffer_add(buf, "foo", 3); evbuffer_add_reference(buf, "foo", 3, NULL, NULL); evbuffer_add_reference(src, "foo", 3, NULL, NULL); evbuffer_add_reference(src, NULL, 0, NULL, NULL); evbuffer_add_buffer(src, buf); evbuffer_add(buf, "foo", 3); evbuffer_add_reference(buf, "foo", 3, NULL, NULL); evbuffer_add_reference(dst, "foo", 3, NULL, NULL); evbuffer_add_reference(dst, NULL, 0, NULL, NULL); evbuffer_add_buffer(dst, buf); tt_int_op(evbuffer_get_length(src), ==, 9); tt_int_op(evbuffer_get_length(dst), ==, 9); evbuffer_validate(src); evbuffer_validate(dst); evbuffer_remove_buffer(src, dst, 8); evbuffer_validate(src); evbuffer_validate(dst); tt_int_op(evbuffer_get_length(src), ==, 1); tt_int_op(evbuffer_get_length(dst), ==, 17); end: evbuffer_free(src); evbuffer_free(dst); evbuffer_free(buf); } static void test_evbuffer_remove_buffer_with_empty3(void *ptr) { struct evbuffer *src = evbuffer_new(); struct evbuffer *dst = evbuffer_new(); struct evbuffer *buf = evbuffer_new(); evbuffer_add(buf, "foo", 3); evbuffer_add_reference(buf, NULL, 0, NULL, NULL); evbuffer_add_reference(src, "foo", 3, NULL, NULL); evbuffer_add_reference(src, NULL, 0, NULL, NULL); evbuffer_prepend_buffer(src, buf); evbuffer_add(buf, "foo", 3); evbuffer_add_reference(buf, NULL, 0, NULL, NULL); evbuffer_add_reference(dst, "foo", 3, NULL, NULL); evbuffer_add_reference(dst, NULL, 0, NULL, NULL); evbuffer_prepend_buffer(dst, buf); tt_int_op(evbuffer_get_length(src), ==, 6); tt_int_op(evbuffer_get_length(dst), ==, 6); evbuffer_validate(src); evbuffer_validate(dst); evbuffer_remove_buffer(src, dst, 5); evbuffer_validate(src); evbuffer_validate(dst); tt_int_op(evbuffer_get_length(src), ==, 1); tt_int_op(evbuffer_get_length(dst), ==, 11); end: evbuffer_free(src); evbuffer_free(dst); evbuffer_free(buf); } static void test_evbuffer_add_buffer_with_empty(void *ptr) { struct evbuffer *src = evbuffer_new(); struct evbuffer *dst = evbuffer_new(); struct evbuffer *buf = evbuffer_new(); evbuffer_add(buf, "foo", 3); evbuffer_add_reference(src, "foo", 3, NULL, NULL); evbuffer_add_reference(src, NULL, 0, NULL, NULL); evbuffer_add_buffer(src, buf); evbuffer_add(buf, "foo", 3); evbuffer_add_reference(dst, "foo", 3, NULL, NULL); evbuffer_add_reference(dst, NULL, 0, NULL, NULL); evbuffer_add_buffer(dst, buf); tt_int_op(evbuffer_get_length(src), ==, 6); tt_int_op(evbuffer_get_length(dst), ==, 6); evbuffer_validate(src); evbuffer_validate(dst); end: evbuffer_free(src); evbuffer_free(dst); evbuffer_free(buf); } static void test_evbuffer_add_buffer_with_empty2(void *ptr) { struct evbuffer *src = evbuffer_new(); struct evbuffer *dst = evbuffer_new(); struct evbuffer *buf = evbuffer_new(); evbuffer_add(buf, "foo", 3); evbuffer_add_reference(src, NULL, 0, NULL, NULL); evbuffer_add_buffer(src, buf); evbuffer_add(buf, "foo", 3); evbuffer_add_reference(dst, NULL, 0, NULL, NULL); evbuffer_add_buffer(dst, buf); tt_int_op(evbuffer_get_length(src), ==, 3); tt_int_op(evbuffer_get_length(dst), ==, 3); evbuffer_validate(src); evbuffer_validate(dst); end: evbuffer_free(src); evbuffer_free(dst); evbuffer_free(buf); } static void test_evbuffer_reserve2(void *ptr) { /* Test the two-vector cases of reserve/commit. */ struct evbuffer *buf = evbuffer_new(); int n, i; struct evbuffer_iovec v[2]; size_t remaining; char *cp, *cp2; /* First chunk will necessarily be one chunk. Use 512 bytes of it.*/ n = evbuffer_reserve_space(buf, 1024, v, 2); tt_int_op(n, ==, 1); tt_int_op(evbuffer_get_length(buf), ==, 0); tt_assert(v[0].iov_base != NULL); tt_int_op(v[0].iov_len, >=, 1024); memset(v[0].iov_base, 'X', 512); cp = v[0].iov_base; remaining = v[0].iov_len - 512; v[0].iov_len = 512; evbuffer_validate(buf); tt_int_op(0, ==, evbuffer_commit_space(buf, v, 1)); tt_int_op(evbuffer_get_length(buf), ==, 512); evbuffer_validate(buf); /* Ask for another same-chunk request, in an existing chunk. Use 8 * bytes of it. */ n = evbuffer_reserve_space(buf, 32, v, 2); tt_int_op(n, ==, 1); tt_assert(cp + 512 == v[0].iov_base); tt_int_op(remaining, ==, v[0].iov_len); memset(v[0].iov_base, 'Y', 8); v[0].iov_len = 8; tt_int_op(0, ==, evbuffer_commit_space(buf, v, 1)); tt_int_op(evbuffer_get_length(buf), ==, 520); remaining -= 8; evbuffer_validate(buf); /* Now ask for a request that will be split. Use only one byte of it, though. */ n = evbuffer_reserve_space(buf, remaining+64, v, 2); tt_int_op(n, ==, 2); tt_assert(cp + 520 == v[0].iov_base); tt_int_op(remaining, ==, v[0].iov_len); tt_assert(v[1].iov_base); tt_assert(v[1].iov_len >= 64); cp2 = v[1].iov_base; memset(v[0].iov_base, 'Z', 1); v[0].iov_len = 1; tt_int_op(0, ==, evbuffer_commit_space(buf, v, 1)); tt_int_op(evbuffer_get_length(buf), ==, 521); remaining -= 1; evbuffer_validate(buf); /* Now ask for a request that will be split. Use some of the first * part and some of the second. */ n = evbuffer_reserve_space(buf, remaining+64, v, 2); evbuffer_validate(buf); tt_int_op(n, ==, 2); tt_assert(cp + 521 == v[0].iov_base); tt_int_op(remaining, ==, v[0].iov_len); tt_assert(v[1].iov_base == cp2); tt_assert(v[1].iov_len >= 64); memset(v[0].iov_base, 'W', 400); v[0].iov_len = 400; memset(v[1].iov_base, 'x', 60); v[1].iov_len = 60; tt_int_op(0, ==, evbuffer_commit_space(buf, v, 2)); tt_int_op(evbuffer_get_length(buf), ==, 981); evbuffer_validate(buf); /* Now peek to make sure stuff got made how we like. */ memset(v,0,sizeof(v)); n = evbuffer_peek(buf, -1, NULL, v, 2); tt_int_op(n, ==, 2); tt_int_op(v[0].iov_len, ==, 921); tt_int_op(v[1].iov_len, ==, 60); cp = v[0].iov_base; for (i=0; i<512; ++i) tt_int_op(cp[i], ==, 'X'); for (i=512; i<520; ++i) tt_int_op(cp[i], ==, 'Y'); for (i=520; i<521; ++i) tt_int_op(cp[i], ==, 'Z'); for (i=521; i<921; ++i) tt_int_op(cp[i], ==, 'W'); cp = v[1].iov_base; for (i=0; i<60; ++i) tt_int_op(cp[i], ==, 'x'); end: evbuffer_free(buf); } static void test_evbuffer_reserve_many(void *ptr) { /* This is a glass-box test to handle expanding a buffer with more * chunks and reallocating chunks as needed */ struct evbuffer *buf = evbuffer_new(); struct evbuffer_iovec v[8]; int n; size_t sz; int add_data = ptr && !strcmp(ptr, "add"); int fill_first = ptr && !strcmp(ptr, "fill"); char *cp1, *cp2; /* When reserving the the first chunk, we just allocate it */ n = evbuffer_reserve_space(buf, 128, v, 2); evbuffer_validate(buf); tt_int_op(n, ==, 1); tt_assert(v[0].iov_len >= 128); sz = v[0].iov_len; cp1 = v[0].iov_base; if (add_data) { *(char*)v[0].iov_base = 'X'; v[0].iov_len = 1; n = evbuffer_commit_space(buf, v, 1); tt_int_op(n, ==, 0); } else if (fill_first) { memset(v[0].iov_base, 'X', v[0].iov_len); n = evbuffer_commit_space(buf, v, 1); tt_int_op(n, ==, 0); n = evbuffer_reserve_space(buf, 128, v, 2); tt_int_op(n, ==, 1); sz = v[0].iov_len; tt_assert(v[0].iov_base != cp1); cp1 = v[0].iov_base; } /* Make another chunk get added. */ n = evbuffer_reserve_space(buf, sz+128, v, 2); evbuffer_validate(buf); tt_int_op(n, ==, 2); sz = v[0].iov_len + v[1].iov_len; tt_int_op(sz, >=, v[0].iov_len+128); if (add_data) { tt_assert(v[0].iov_base == cp1 + 1); } else { tt_assert(v[0].iov_base == cp1); } cp1 = v[0].iov_base; cp2 = v[1].iov_base; /* And a third chunk. */ n = evbuffer_reserve_space(buf, sz+128, v, 3); evbuffer_validate(buf); tt_int_op(n, ==, 3); tt_assert(cp1 == v[0].iov_base); tt_assert(cp2 == v[1].iov_base); sz = v[0].iov_len + v[1].iov_len + v[2].iov_len; /* Now force a reallocation by asking for more space in only 2 * buffers. */ n = evbuffer_reserve_space(buf, sz+128, v, 2); evbuffer_validate(buf); if (add_data) { tt_int_op(n, ==, 2); tt_assert(cp1 == v[0].iov_base); } else { tt_int_op(n, ==, 1); } end: evbuffer_free(buf); } static void test_evbuffer_expand(void *ptr) { char data[4096]; struct evbuffer *buf; size_t a,w,u; void *buffer; memset(data, 'X', sizeof(data)); /* Make sure that expand() works on an empty buffer */ buf = evbuffer_new(); tt_int_op(evbuffer_expand(buf, 20000), ==, 0); evbuffer_validate(buf); a=w=u=0; evbuffer_get_waste(buf, &a,&w,&u); tt_assert(w == 0); tt_assert(u == 0); tt_assert(a >= 20000); tt_assert(buf->first); tt_assert(buf->first == buf->last); tt_assert(buf->first->off == 0); tt_assert(buf->first->buffer_len >= 20000); /* Make sure that expand() works as a no-op when there's enough * contiguous space already. */ buffer = buf->first->buffer; evbuffer_add(buf, data, 1024); tt_int_op(evbuffer_expand(buf, 1024), ==, 0); tt_assert(buf->first->buffer == buffer); evbuffer_validate(buf); evbuffer_free(buf); /* Make sure that expand() can work by moving misaligned data * when it makes sense to do so. */ buf = evbuffer_new(); evbuffer_add(buf, data, 400); { int n = (int)(buf->first->buffer_len - buf->first->off - 1); tt_assert(n < (int)sizeof(data)); evbuffer_add(buf, data, n); } tt_assert(buf->first == buf->last); tt_assert(buf->first->off == buf->first->buffer_len - 1); evbuffer_drain(buf, buf->first->off - 1); tt_assert(1 == evbuffer_get_length(buf)); tt_assert(buf->first->misalign > 0); tt_assert(buf->first->off == 1); buffer = buf->first->buffer; tt_assert(evbuffer_expand(buf, 40) == 0); tt_assert(buf->first == buf->last); tt_assert(buf->first->off == 1); tt_assert(buf->first->buffer == buffer); tt_assert(buf->first->misalign == 0); evbuffer_validate(buf); evbuffer_free(buf); /* add, expand, pull-up: This used to crash libevent. */ buf = evbuffer_new(); evbuffer_add(buf, data, sizeof(data)); evbuffer_add(buf, data, sizeof(data)); evbuffer_add(buf, data, sizeof(data)); evbuffer_validate(buf); evbuffer_expand(buf, 1024); evbuffer_validate(buf); evbuffer_pullup(buf, -1); evbuffer_validate(buf); end: evbuffer_free(buf); } static void test_evbuffer_expand_overflow(void *ptr) { struct evbuffer *buf; buf = evbuffer_new(); evbuffer_add(buf, "1", 1); evbuffer_expand(buf, EVBUFFER_CHAIN_MAX); evbuffer_validate(buf); evbuffer_expand(buf, EV_SIZE_MAX); evbuffer_validate(buf); end: evbuffer_free(buf); } static void test_evbuffer_add1(void *ptr) { struct evbuffer *buf; char *str; buf = evbuffer_new(); evbuffer_add(buf, "1", 1); evbuffer_validate(buf); evbuffer_expand(buf, 2048); evbuffer_validate(buf); evbuffer_add(buf, "2", 1); evbuffer_validate(buf); evbuffer_add_printf(buf, "3"); evbuffer_validate(buf); tt_assert(evbuffer_get_length(buf) == 3); str = (char *)evbuffer_pullup(buf, -1); tt_assert(str[0] == '1'); tt_assert(str[1] == '2'); tt_assert(str[2] == '3'); end: evbuffer_free(buf); } static void test_evbuffer_add2(void *ptr) { struct evbuffer *buf; static char data[4096]; int data_len = MIN_BUFFER_SIZE-EVBUFFER_CHAIN_SIZE-10; char *str; int len; memset(data, 'P', sizeof(data)); buf = evbuffer_new(); evbuffer_add(buf, data, data_len); evbuffer_validate(buf); evbuffer_expand(buf, 100); evbuffer_validate(buf); evbuffer_add(buf, "2", 1); evbuffer_validate(buf); evbuffer_add_printf(buf, "3"); evbuffer_validate(buf); len = evbuffer_get_length(buf); tt_assert(len == data_len+2); str = (char *)evbuffer_pullup(buf, -1); tt_assert(str[len-3] == 'P'); tt_assert(str[len-2] == '2'); tt_assert(str[len-1] == '3'); end: evbuffer_free(buf); } static int reference_cb_called; static void reference_cb(const void *data, size_t len, void *extra) { tt_str_op(data, ==, "this is what we add as read-only memory."); tt_int_op(len, ==, strlen(data)); tt_want(extra == (void *)0xdeadaffe); ++reference_cb_called; end: ; } static void test_evbuffer_reference(void *ptr) { struct evbuffer *src = evbuffer_new(); struct evbuffer *dst = evbuffer_new(); struct evbuffer_iovec v[1]; const char *data = "this is what we add as read-only memory."; reference_cb_called = 0; tt_assert(evbuffer_add_reference(src, data, strlen(data), reference_cb, (void *)0xdeadaffe) != -1); evbuffer_reserve_space(dst, strlen(data), v, 1); tt_assert(evbuffer_remove(src, v[0].iov_base, 10) != -1); evbuffer_validate(src); evbuffer_validate(dst); /* make sure that we don't write data at the beginning */ evbuffer_prepend(src, "aaaaa", 5); evbuffer_validate(src); evbuffer_drain(src, 5); tt_assert(evbuffer_remove(src, ((char*)(v[0].iov_base)) + 10, strlen(data) - 10) != -1); v[0].iov_len = strlen(data); evbuffer_commit_space(dst, v, 1); evbuffer_validate(src); evbuffer_validate(dst); tt_int_op(reference_cb_called, ==, 1); tt_assert(!memcmp(evbuffer_pullup(dst, strlen(data)), data, strlen(data))); evbuffer_validate(dst); end: evbuffer_free(dst); evbuffer_free(src); } static void test_evbuffer_reference2(void *ptr) { struct evbuffer *buf; static char data[4096]; int data_len = MIN_BUFFER_SIZE-EVBUFFER_CHAIN_SIZE-10; char *str; int len; memset(data, 'P', sizeof(data)); buf = evbuffer_new(); evbuffer_add(buf, data, data_len); evbuffer_validate(buf); evbuffer_expand(buf, 100); evbuffer_validate(buf); evbuffer_add_reference(buf, "2", 1, no_cleanup, NULL); evbuffer_validate(buf); evbuffer_add_printf(buf, "3"); evbuffer_validate(buf); len = evbuffer_get_length(buf); tt_assert(len == data_len+2); str = (char *)evbuffer_pullup(buf, -1); tt_assert(str[len-3] == 'P'); tt_assert(str[len-2] == '2'); tt_assert(str[len-1] == '3'); end: evbuffer_free(buf); } static struct event_base *addfile_test_event_base; static int addfile_test_done_writing; static int addfile_test_total_written; static int addfile_test_total_read; static void addfile_test_writecb(evutil_socket_t fd, short what, void *arg) { struct evbuffer *b = arg; int r; evbuffer_validate(b); while (evbuffer_get_length(b)) { r = evbuffer_write(b, fd); if (r > 0) { addfile_test_total_written += r; TT_BLATHER(("Wrote %d/%d bytes", r, addfile_test_total_written)); } else { int e = evutil_socket_geterror(fd); if (EVUTIL_ERR_RW_RETRIABLE(e)) return; tt_fail_perror("write"); event_base_loopexit(addfile_test_event_base,NULL); } evbuffer_validate(b); } addfile_test_done_writing = 1; return; end: event_base_loopexit(addfile_test_event_base,NULL); } static void addfile_test_readcb(evutil_socket_t fd, short what, void *arg) { struct evbuffer *b = arg; int e, r = 0; do { r = evbuffer_read(b, fd, 1024); if (r > 0) { addfile_test_total_read += r; TT_BLATHER(("Read %d/%d bytes", r, addfile_test_total_read)); } } while (r > 0); if (r < 0) { e = evutil_socket_geterror(fd); if (! EVUTIL_ERR_RW_RETRIABLE(e)) { tt_fail_perror("read"); event_base_loopexit(addfile_test_event_base,NULL); } } if (addfile_test_done_writing && addfile_test_total_read >= addfile_test_total_written) { event_base_loopexit(addfile_test_event_base,NULL); } } static void test_evbuffer_add_file(void *ptr) { struct basic_test_data *testdata = ptr; const char *impl = testdata->setup_data; struct evbuffer *src = evbuffer_new(), *dest = evbuffer_new(); char *tmpfilename = NULL; char *data = NULL; const char *expect_data; size_t datalen, expect_len; const char *compare; int fd = -1; int want_ismapping = -1, want_cansendfile = -1; unsigned flags = 0; int use_segment = 1, use_bigfile = 0, map_from_offset = 0, view_from_offset = 0; struct evbuffer_file_segment *seg = NULL; ev_off_t starting_offset = 0, mapping_len = -1; ev_off_t segment_offset = 0, segment_len = -1; struct event *rev=NULL, *wev=NULL; struct event_base *base = testdata->base; evutil_socket_t pair[2] = {-1, -1}; struct evutil_weakrand_state seed = { 123456789U }; /* This test is highly parameterized based on substrings of its * argument. The strings are: */ tt_assert(impl); if (strstr(impl, "nosegment")) { /* If nosegment is set, use the older evbuffer_add_file * interface */ use_segment = 0; } if (strstr(impl, "bigfile")) { /* If bigfile is set, use a 512K file. Else use a smaller * one. */ use_bigfile = 1; } if (strstr(impl, "map_offset")) { /* If map_offset is set, we build the file segment starting * from a point other than byte 0 and ending somewhere other * than the last byte. Otherwise we map the whole thing */ map_from_offset = 1; } if (strstr(impl, "offset_in_segment")) { /* If offset_in_segment is set, we add a subsection of the * file semgment starting from a point other than byte 0 of * the segment. */ view_from_offset = 1; } if (strstr(impl, "sendfile")) { /* If sendfile is set, we try to use a sendfile/splice style * backend. */ flags = EVBUF_FS_DISABLE_MMAP; want_cansendfile = 1; want_ismapping = 0; } else if (strstr(impl, "mmap")) { /* If sendfile is set, we try to use a mmap/CreateFileMapping * style backend. */ flags = EVBUF_FS_DISABLE_SENDFILE; want_ismapping = 1; want_cansendfile = 0; } else if (strstr(impl, "linear")) { /* If linear is set, we try to use a read-the-whole-thing * backend. */ flags = EVBUF_FS_DISABLE_SENDFILE|EVBUF_FS_DISABLE_MMAP; want_ismapping = 0; want_cansendfile = 0; } else if (strstr(impl, "default")) { /* The caller doesn't care which backend we use. */ ; } else { /* The caller must choose a backend. */ TT_DIE(("Didn't recognize the implementation")); } if (use_bigfile) { unsigned int i; datalen = 1024*512; data = malloc(1024*512); tt_assert(data); for (i = 0; i < datalen; ++i) data[i] = (char)evutil_weakrand_(&seed); } else { data = strdup("here is a relatively small string."); tt_assert(data); datalen = strlen(data); } fd = regress_make_tmpfile(data, datalen, &tmpfilename); if (map_from_offset) { starting_offset = datalen/4 + 1; mapping_len = datalen / 2 - 1; expect_data = data + starting_offset; expect_len = mapping_len; } else { expect_data = data; expect_len = datalen; } if (view_from_offset) { tt_assert(use_segment); /* Can't do this with add_file*/ segment_offset = expect_len / 3; segment_len = expect_len / 2; expect_data = expect_data + segment_offset; expect_len = segment_len; } if (use_segment) { seg = evbuffer_file_segment_new(fd, starting_offset, mapping_len, flags); tt_assert(seg); if (want_ismapping >= 0) { if (seg->is_mapping != (unsigned)want_ismapping) tt_skip(); } if (want_cansendfile >= 0) { if (seg->can_sendfile != (unsigned)want_cansendfile) tt_skip(); } } /* Say that it drains to a fd so that we can use sendfile. */ evbuffer_set_flags(src, EVBUFFER_FLAG_DRAINS_TO_FD); #if defined(EVENT__HAVE_SENDFILE) && defined(__sun__) && defined(__svr4__) /* We need to use a pair of AF_INET sockets, since Solaris doesn't support sendfile() over AF_UNIX. */ if (evutil_ersatz_socketpair_(AF_INET, SOCK_STREAM, 0, pair) == -1) tt_abort_msg("ersatz_socketpair failed"); #else if (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, pair) == -1) tt_abort_msg("socketpair failed"); #endif evutil_make_socket_nonblocking(pair[0]); evutil_make_socket_nonblocking(pair[1]); tt_assert(fd != -1); if (use_segment) { tt_assert(evbuffer_add_file_segment(src, seg, segment_offset, segment_len)!=-1); } else { tt_assert(evbuffer_add_file(src, fd, starting_offset, mapping_len) != -1); } evbuffer_validate(src); addfile_test_event_base = base; addfile_test_done_writing = 0; addfile_test_total_written = 0; addfile_test_total_read = 0; wev = event_new(base, pair[0], EV_WRITE|EV_PERSIST, addfile_test_writecb, src); rev = event_new(base, pair[1], EV_READ|EV_PERSIST, addfile_test_readcb, dest); event_add(wev, NULL); event_add(rev, NULL); event_base_dispatch(base); evbuffer_validate(src); evbuffer_validate(dest); tt_assert(addfile_test_done_writing); tt_int_op(addfile_test_total_written, ==, expect_len); tt_int_op(addfile_test_total_read, ==, expect_len); compare = (char *)evbuffer_pullup(dest, expect_len); tt_assert(compare != NULL); if (memcmp(compare, expect_data, expect_len)) { tt_abort_msg("Data from add_file differs."); } evbuffer_validate(dest); end: if (data) free(data); if (seg) evbuffer_file_segment_free(seg); if (src) evbuffer_free(src); if (dest) evbuffer_free(dest); if (pair[0] >= 0) evutil_closesocket(pair[0]); if (pair[1] >= 0) evutil_closesocket(pair[1]); if (wev) event_free(wev); if (rev) event_free(rev); if (tmpfilename) { unlink(tmpfilename); free(tmpfilename); } } static int file_segment_cleanup_cb_called_count = 0; static struct evbuffer_file_segment const* file_segment_cleanup_cb_called_with = NULL; static int file_segment_cleanup_cb_called_with_flags = 0; static void* file_segment_cleanup_cb_called_with_arg = NULL; static void file_segment_cleanup_cp(struct evbuffer_file_segment const* seg, int flags, void* arg) { ++file_segment_cleanup_cb_called_count; file_segment_cleanup_cb_called_with = seg; file_segment_cleanup_cb_called_with_flags = flags; file_segment_cleanup_cb_called_with_arg = arg; } static void test_evbuffer_file_segment_add_cleanup_cb(void* ptr) { char *tmpfilename = NULL; int fd = -1; struct evbuffer *evb = NULL; struct evbuffer_file_segment *seg = NULL, *segptr; char const* arg = "token"; fd = regress_make_tmpfile("file_segment_test_file", 22, &tmpfilename); tt_int_op(fd, >=, 0); evb = evbuffer_new(); tt_assert(evb); segptr = seg = evbuffer_file_segment_new(fd, 0, -1, 0); tt_assert(seg); evbuffer_file_segment_add_cleanup_cb( seg, &file_segment_cleanup_cp, (void*)arg); tt_assert(fd != -1); tt_assert(evbuffer_add_file_segment(evb, seg, 0, -1)!=-1); evbuffer_validate(evb); tt_int_op(file_segment_cleanup_cb_called_count, ==, 0); evbuffer_file_segment_free(seg); seg = NULL; /* Prevent double-free. */ tt_int_op(file_segment_cleanup_cb_called_count, ==, 0); evbuffer_free(evb); evb = NULL; /* pevent double-free */ tt_int_op(file_segment_cleanup_cb_called_count, ==, 1); tt_assert(file_segment_cleanup_cb_called_with == segptr); tt_assert(file_segment_cleanup_cb_called_with_flags == 0); tt_assert(file_segment_cleanup_cb_called_with_arg == (void*)arg); end: if (evb) evbuffer_free(evb); if (seg) evbuffer_file_segment_free(seg); if (tmpfilename) { unlink(tmpfilename); free(tmpfilename); } } #ifndef EVENT__DISABLE_MM_REPLACEMENT static void * failing_malloc(size_t how_much) { errno = ENOMEM; return NULL; } #endif static void test_evbuffer_readln(void *ptr) { struct evbuffer *evb = evbuffer_new(); struct evbuffer *evb_tmp = evbuffer_new(); const char *s; char *cp = NULL; size_t sz; #define tt_line_eq(content) \ TT_STMT_BEGIN \ if (!cp || sz != strlen(content) || strcmp(cp, content)) { \ TT_DIE(("Wanted %s; got %s [%d]", content, cp, (int)sz)); \ } \ TT_STMT_END /* Test EOL_ANY. */ s = "complex silly newline\r\n\n\r\n\n\rmore\0\n"; evbuffer_add(evb, s, strlen(s)+2); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_ANY); tt_line_eq("complex silly newline"); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_ANY); if (!cp || sz != 5 || memcmp(cp, "more\0\0", 6)) tt_abort_msg("Not as expected"); tt_uint_op(evbuffer_get_length(evb), ==, 0); evbuffer_validate(evb); s = "\nno newline"; evbuffer_add(evb, s, strlen(s)); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_ANY); tt_line_eq(""); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_ANY); tt_assert(!cp); evbuffer_validate(evb); evbuffer_drain(evb, evbuffer_get_length(evb)); tt_assert(evbuffer_get_length(evb) == 0); evbuffer_validate(evb); /* Test EOL_CRLF */ s = "Line with\rin the middle\nLine with good crlf\r\n\nfinal\n"; evbuffer_add(evb, s, strlen(s)); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF); tt_line_eq("Line with\rin the middle"); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF); tt_line_eq("Line with good crlf"); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF); tt_line_eq(""); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF); tt_line_eq("final"); s = "x"; evbuffer_validate(evb); evbuffer_add(evb, s, 1); evbuffer_validate(evb); free(cp); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF); tt_assert(!cp); evbuffer_validate(evb); /* Test CRLF_STRICT */ s = " and a bad crlf\nand a good one\r\n\r\nMore\r"; evbuffer_add(evb, s, strlen(s)); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_line_eq("x and a bad crlf\nand a good one"); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_line_eq(""); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_assert(!cp); evbuffer_validate(evb); evbuffer_add(evb, "\n", 1); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_line_eq("More"); free(cp); tt_assert(evbuffer_get_length(evb) == 0); evbuffer_validate(evb); s = "An internal CR\r is not an eol\r\nNor is a lack of one"; evbuffer_add(evb, s, strlen(s)); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_line_eq("An internal CR\r is not an eol"); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_assert(!cp); evbuffer_validate(evb); evbuffer_add(evb, "\r\n", 2); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_line_eq("Nor is a lack of one"); free(cp); tt_assert(evbuffer_get_length(evb) == 0); evbuffer_validate(evb); /* Test LF */ s = "An\rand a nl\n\nText"; evbuffer_add(evb, s, strlen(s)); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_LF); tt_line_eq("An\rand a nl"); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_LF); tt_line_eq(""); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_LF); tt_assert(!cp); free(cp); evbuffer_add(evb, "\n", 1); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_LF); tt_line_eq("Text"); free(cp); evbuffer_validate(evb); /* Test NUL */ tt_int_op(evbuffer_get_length(evb), ==, 0); { char x[] = "NUL\n\0\0" "The all-zeros character which may serve\0" "to accomplish time fill\0and media fill"; /* Add all but the final NUL of x. */ evbuffer_add(evb, x, sizeof(x)-1); } cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_NUL); tt_line_eq("NUL\n"); free(cp); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_NUL); tt_line_eq(""); free(cp); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_NUL); tt_line_eq("The all-zeros character which may serve"); free(cp); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_NUL); tt_line_eq("to accomplish time fill"); free(cp); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_NUL); tt_ptr_op(cp, ==, NULL); evbuffer_drain(evb, -1); /* Test CRLF_STRICT - across boundaries*/ s = " and a bad crlf\nand a good one\r"; evbuffer_add(evb_tmp, s, strlen(s)); evbuffer_validate(evb); evbuffer_add_buffer(evb, evb_tmp); evbuffer_validate(evb); s = "\n\r"; evbuffer_add(evb_tmp, s, strlen(s)); evbuffer_validate(evb); evbuffer_add_buffer(evb, evb_tmp); evbuffer_validate(evb); s = "\nMore\r"; evbuffer_add(evb_tmp, s, strlen(s)); evbuffer_validate(evb); evbuffer_add_buffer(evb, evb_tmp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_line_eq(" and a bad crlf\nand a good one"); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_line_eq(""); free(cp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_assert(!cp); free(cp); evbuffer_validate(evb); evbuffer_add(evb, "\n", 1); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_CRLF_STRICT); tt_line_eq("More"); free(cp); cp = NULL; evbuffer_validate(evb); tt_assert(evbuffer_get_length(evb) == 0); /* Test memory problem*/ s = "one line\ntwo line\nblue line"; evbuffer_add(evb_tmp, s, strlen(s)); evbuffer_validate(evb); evbuffer_add_buffer(evb, evb_tmp); evbuffer_validate(evb); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_LF); tt_line_eq("one line"); free(cp); cp = NULL; evbuffer_validate(evb); /* the next call to readline should fail */ #ifndef EVENT__DISABLE_MM_REPLACEMENT event_set_mem_functions(failing_malloc, realloc, free); cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_LF); tt_assert(cp == NULL); evbuffer_validate(evb); /* now we should get the next line back */ event_set_mem_functions(malloc, realloc, free); #endif cp = evbuffer_readln(evb, &sz, EVBUFFER_EOL_LF); tt_line_eq("two line"); free(cp); cp = NULL; evbuffer_validate(evb); end: evbuffer_free(evb); evbuffer_free(evb_tmp); if (cp) free(cp); } static void test_evbuffer_search_eol(void *ptr) { struct evbuffer *buf = evbuffer_new(); struct evbuffer_ptr ptr1, ptr2; const char *s; size_t eol_len; s = "string! \r\n\r\nx\n"; evbuffer_add(buf, s, strlen(s)); eol_len = -1; ptr1 = evbuffer_search_eol(buf, NULL, &eol_len, EVBUFFER_EOL_CRLF); tt_int_op(ptr1.pos, ==, 8); tt_int_op(eol_len, ==, 2); eol_len = -1; ptr2 = evbuffer_search_eol(buf, &ptr1, &eol_len, EVBUFFER_EOL_CRLF); tt_int_op(ptr2.pos, ==, 8); tt_int_op(eol_len, ==, 2); evbuffer_ptr_set(buf, &ptr1, 1, EVBUFFER_PTR_ADD); eol_len = -1; ptr2 = evbuffer_search_eol(buf, &ptr1, &eol_len, EVBUFFER_EOL_CRLF); tt_int_op(ptr2.pos, ==, 9); tt_int_op(eol_len, ==, 1); eol_len = -1; ptr2 = evbuffer_search_eol(buf, &ptr1, &eol_len, EVBUFFER_EOL_CRLF_STRICT); tt_int_op(ptr2.pos, ==, 10); tt_int_op(eol_len, ==, 2); eol_len = -1; ptr1 = evbuffer_search_eol(buf, NULL, &eol_len, EVBUFFER_EOL_LF); tt_int_op(ptr1.pos, ==, 9); tt_int_op(eol_len, ==, 1); eol_len = -1; ptr2 = evbuffer_search_eol(buf, &ptr1, &eol_len, EVBUFFER_EOL_LF); tt_int_op(ptr2.pos, ==, 9); tt_int_op(eol_len, ==, 1); evbuffer_ptr_set(buf, &ptr1, 1, EVBUFFER_PTR_ADD); eol_len = -1; ptr2 = evbuffer_search_eol(buf, &ptr1, &eol_len, EVBUFFER_EOL_LF); tt_int_op(ptr2.pos, ==, 11); tt_int_op(eol_len, ==, 1); tt_assert(evbuffer_ptr_set(buf, &ptr1, evbuffer_get_length(buf), EVBUFFER_PTR_SET) == 0); eol_len = -1; ptr2 = evbuffer_search_eol(buf, &ptr1, &eol_len, EVBUFFER_EOL_LF); tt_int_op(ptr2.pos, ==, -1); tt_int_op(eol_len, ==, 0); end: evbuffer_free(buf); } static void test_evbuffer_iterative(void *ptr) { struct evbuffer *buf = evbuffer_new(); const char *abc = "abcdefghijklmnopqrstvuwxyzabcdefghijklmnopqrstvuwxyzabcdefghijklmnopqrstvuwxyzabcdefghijklmnopqrstvuwxyz"; unsigned i, j, sum, n; sum = 0; n = 0; for (i = 0; i < 1000; ++i) { for (j = 1; j < strlen(abc); ++j) { char format[32]; evutil_snprintf(format, sizeof(format), "%%%u.%us", j, j); evbuffer_add_printf(buf, format, abc); /* Only check for rep violations every so often. Walking over the whole list of chains can get pretty expensive as it gets long. */ if ((n % 337) == 0) evbuffer_validate(buf); sum += j; n++; } } evbuffer_validate(buf); tt_uint_op(sum, ==, evbuffer_get_length(buf)); { size_t a,w,u; a=w=u=0; evbuffer_get_waste(buf, &a, &w, &u); if (0) printf("Allocated: %u.\nWasted: %u.\nUsed: %u.", (unsigned)a, (unsigned)w, (unsigned)u); tt_assert( ((double)w)/a < .125); } end: evbuffer_free(buf); } static void test_evbuffer_find(void *ptr) { unsigned char* p; const char* test1 = "1234567890\r\n"; const char* test2 = "1234567890\r"; #define EVBUFFER_INITIAL_LENGTH 256 char test3[EVBUFFER_INITIAL_LENGTH]; unsigned int i; struct evbuffer * buf = evbuffer_new(); tt_assert(buf); /* make sure evbuffer_find doesn't match past the end of the buffer */ evbuffer_add(buf, (unsigned char*)test1, strlen(test1)); evbuffer_validate(buf); evbuffer_drain(buf, strlen(test1)); evbuffer_validate(buf); evbuffer_add(buf, (unsigned char*)test2, strlen(test2)); evbuffer_validate(buf); p = evbuffer_find(buf, (unsigned char*)"\r\n", 2); tt_want(p == NULL); /* * drain the buffer and do another find; in r309 this would * read past the allocated buffer causing a valgrind error. */ evbuffer_drain(buf, strlen(test2)); evbuffer_validate(buf); for (i = 0; i < EVBUFFER_INITIAL_LENGTH; ++i) test3[i] = 'a'; test3[EVBUFFER_INITIAL_LENGTH - 1] = 'x'; evbuffer_add(buf, (unsigned char *)test3, EVBUFFER_INITIAL_LENGTH); evbuffer_validate(buf); p = evbuffer_find(buf, (unsigned char *)"xy", 2); tt_want(p == NULL); /* simple test for match at end of allocated buffer */ p = evbuffer_find(buf, (unsigned char *)"ax", 2); tt_assert(p != NULL); tt_want(strncmp((char*)p, "ax", 2) == 0); end: if (buf) evbuffer_free(buf); } static void test_evbuffer_ptr_set(void *ptr) { struct evbuffer *buf = evbuffer_new(); struct evbuffer_ptr pos; struct evbuffer_iovec v[1]; tt_assert(buf); tt_int_op(evbuffer_get_length(buf), ==, 0); tt_assert(evbuffer_ptr_set(buf, &pos, 0, EVBUFFER_PTR_SET) == 0); tt_assert(pos.pos == 0); tt_assert(evbuffer_ptr_set(buf, &pos, 1, EVBUFFER_PTR_ADD) == -1); tt_assert(pos.pos == -1); tt_assert(evbuffer_ptr_set(buf, &pos, 1, EVBUFFER_PTR_SET) == -1); tt_assert(pos.pos == -1); /* create some chains */ evbuffer_reserve_space(buf, 5000, v, 1); v[0].iov_len = 5000; memset(v[0].iov_base, 1, v[0].iov_len); evbuffer_commit_space(buf, v, 1); evbuffer_validate(buf); evbuffer_reserve_space(buf, 4000, v, 1); v[0].iov_len = 4000; memset(v[0].iov_base, 2, v[0].iov_len); evbuffer_commit_space(buf, v, 1); evbuffer_reserve_space(buf, 3000, v, 1); v[0].iov_len = 3000; memset(v[0].iov_base, 3, v[0].iov_len); evbuffer_commit_space(buf, v, 1); evbuffer_validate(buf); tt_int_op(evbuffer_get_length(buf), ==, 12000); tt_assert(evbuffer_ptr_set(buf, &pos, 13000, EVBUFFER_PTR_SET) == -1); tt_assert(pos.pos == -1); tt_assert(evbuffer_ptr_set(buf, &pos, 0, EVBUFFER_PTR_SET) == 0); tt_assert(pos.pos == 0); tt_assert(evbuffer_ptr_set(buf, &pos, 13000, EVBUFFER_PTR_ADD) == -1); tt_assert(evbuffer_ptr_set(buf, &pos, 0, EVBUFFER_PTR_SET) == 0); tt_assert(pos.pos == 0); tt_assert(evbuffer_ptr_set(buf, &pos, 10000, EVBUFFER_PTR_ADD) == 0); tt_assert(pos.pos == 10000); tt_assert(evbuffer_ptr_set(buf, &pos, 1000, EVBUFFER_PTR_ADD) == 0); tt_assert(pos.pos == 11000); tt_assert(evbuffer_ptr_set(buf, &pos, 1000, EVBUFFER_PTR_ADD) == 0); tt_assert(pos.pos == 12000); tt_assert(evbuffer_ptr_set(buf, &pos, 1000, EVBUFFER_PTR_ADD) == -1); tt_assert(pos.pos == -1); end: if (buf) evbuffer_free(buf); } static void test_evbuffer_search(void *ptr) { struct evbuffer *buf = evbuffer_new(); struct evbuffer *tmp = evbuffer_new(); struct evbuffer_ptr pos, end; tt_assert(buf); tt_assert(tmp); pos = evbuffer_search(buf, "x", 1, NULL); tt_int_op(pos.pos, ==, -1); tt_assert(evbuffer_ptr_set(buf, &pos, 0, EVBUFFER_PTR_SET) == 0); pos = evbuffer_search(buf, "x", 1, &pos); tt_int_op(pos.pos, ==, -1); tt_assert(evbuffer_ptr_set(buf, &pos, 0, EVBUFFER_PTR_SET) == 0); pos = evbuffer_search_range(buf, "x", 1, &pos, &pos); tt_int_op(pos.pos, ==, -1); tt_assert(evbuffer_ptr_set(buf, &pos, 0, EVBUFFER_PTR_SET) == 0); pos = evbuffer_search_range(buf, "x", 1, &pos, NULL); tt_int_op(pos.pos, ==, -1); /* set up our chains */ evbuffer_add_printf(tmp, "hello"); /* 5 chars */ evbuffer_add_buffer(buf, tmp); evbuffer_add_printf(tmp, "foo"); /* 3 chars */ evbuffer_add_buffer(buf, tmp); evbuffer_add_printf(tmp, "cat"); /* 3 chars */ evbuffer_add_buffer(buf, tmp); evbuffer_add_printf(tmp, "attack"); evbuffer_add_buffer(buf, tmp); pos = evbuffer_search(buf, "attack", 6, NULL); tt_int_op(pos.pos, ==, 11); pos = evbuffer_search(buf, "attacker", 8, NULL); tt_int_op(pos.pos, ==, -1); /* test continuing search */ pos = evbuffer_search(buf, "oc", 2, NULL); tt_int_op(pos.pos, ==, 7); pos = evbuffer_search(buf, "cat", 3, &pos); tt_int_op(pos.pos, ==, 8); pos = evbuffer_search(buf, "tacking", 7, &pos); tt_int_op(pos.pos, ==, -1); evbuffer_ptr_set(buf, &pos, 5, EVBUFFER_PTR_SET); pos = evbuffer_search(buf, "foo", 3, &pos); tt_int_op(pos.pos, ==, 5); evbuffer_ptr_set(buf, &pos, 2, EVBUFFER_PTR_ADD); pos = evbuffer_search(buf, "tat", 3, &pos); tt_int_op(pos.pos, ==, 10); /* test bounded search. */ /* Set "end" to the first t in "attack". */ evbuffer_ptr_set(buf, &end, 12, EVBUFFER_PTR_SET); pos = evbuffer_search_range(buf, "foo", 3, NULL, &end); tt_int_op(pos.pos, ==, 5); pos = evbuffer_search_range(buf, "foocata", 7, NULL, &end); tt_int_op(pos.pos, ==, 5); pos = evbuffer_search_range(buf, "foocatat", 8, NULL, &end); tt_int_op(pos.pos, ==, -1); pos = evbuffer_search_range(buf, "ack", 3, NULL, &end); tt_int_op(pos.pos, ==, -1); /* Set "end" after the last byte in the buffer. */ tt_assert(evbuffer_ptr_set(buf, &end, 17, EVBUFFER_PTR_SET) == 0); pos = evbuffer_search_range(buf, "attack", 6, NULL, &end); tt_int_op(pos.pos, ==, 11); tt_assert(evbuffer_ptr_set(buf, &pos, 11, EVBUFFER_PTR_SET) == 0); pos = evbuffer_search_range(buf, "attack", 6, &pos, &end); tt_int_op(pos.pos, ==, 11); tt_assert(evbuffer_ptr_set(buf, &pos, 17, EVBUFFER_PTR_SET) == 0); pos = evbuffer_search_range(buf, "attack", 6, &pos, &end); tt_int_op(pos.pos, ==, -1); tt_assert(evbuffer_ptr_set(buf, &pos, 17, EVBUFFER_PTR_SET) == 0); pos = evbuffer_search_range(buf, "attack", 6, &pos, NULL); tt_int_op(pos.pos, ==, -1); end: if (buf) evbuffer_free(buf); if (tmp) evbuffer_free(tmp); } static void log_change_callback(struct evbuffer *buffer, const struct evbuffer_cb_info *cbinfo, void *arg) { size_t old_len = cbinfo->orig_size; size_t new_len = old_len + cbinfo->n_added - cbinfo->n_deleted; struct evbuffer *out = arg; evbuffer_add_printf(out, "%lu->%lu; ", (unsigned long)old_len, (unsigned long)new_len); } static void self_draining_callback(struct evbuffer *evbuffer, size_t old_len, size_t new_len, void *arg) { if (new_len > old_len) evbuffer_drain(evbuffer, new_len); } static void test_evbuffer_callbacks(void *ptr) { struct evbuffer *buf = evbuffer_new(); struct evbuffer *buf_out1 = evbuffer_new(); struct evbuffer *buf_out2 = evbuffer_new(); struct evbuffer_cb_entry *cb1, *cb2; tt_assert(buf); tt_assert(buf_out1); tt_assert(buf_out2); cb1 = evbuffer_add_cb(buf, log_change_callback, buf_out1); cb2 = evbuffer_add_cb(buf, log_change_callback, buf_out2); /* Let's run through adding and deleting some stuff from the buffer * and turning the callbacks on and off and removing them. The callback * adds a summary of length changes to buf_out1/buf_out2 when called. */ /* size: 0-> 36. */ evbuffer_add_printf(buf, "The %d magic words are spotty pudding", 2); evbuffer_validate(buf); evbuffer_cb_clear_flags(buf, cb2, EVBUFFER_CB_ENABLED); evbuffer_drain(buf, 10); /*36->26*/ evbuffer_validate(buf); evbuffer_prepend(buf, "Hello", 5);/*26->31*/ evbuffer_cb_set_flags(buf, cb2, EVBUFFER_CB_ENABLED); evbuffer_add_reference(buf, "Goodbye", 7, NULL, NULL); /*31->38*/ evbuffer_remove_cb_entry(buf, cb1); evbuffer_validate(buf); evbuffer_drain(buf, evbuffer_get_length(buf)); /*38->0*/; tt_assert(-1 == evbuffer_remove_cb(buf, log_change_callback, NULL)); evbuffer_add(buf, "X", 1); /* 0->1 */ tt_assert(!evbuffer_remove_cb(buf, log_change_callback, buf_out2)); evbuffer_validate(buf); tt_str_op((const char *) evbuffer_pullup(buf_out1, -1), ==, "0->36; 36->26; 26->31; 31->38; "); tt_str_op((const char *) evbuffer_pullup(buf_out2, -1), ==, "0->36; 31->38; 38->0; 0->1; "); evbuffer_drain(buf_out1, evbuffer_get_length(buf_out1)); evbuffer_drain(buf_out2, evbuffer_get_length(buf_out2)); /* Let's test the obsolete buffer_setcb function too. */ cb1 = evbuffer_add_cb(buf, log_change_callback, buf_out1); tt_assert(cb1 != NULL); cb2 = evbuffer_add_cb(buf, log_change_callback, buf_out2); tt_assert(cb2 != NULL); evbuffer_setcb(buf, self_draining_callback, NULL); evbuffer_add_printf(buf, "This should get drained right away."); tt_uint_op(evbuffer_get_length(buf), ==, 0); tt_uint_op(evbuffer_get_length(buf_out1), ==, 0); tt_uint_op(evbuffer_get_length(buf_out2), ==, 0); evbuffer_setcb(buf, NULL, NULL); evbuffer_add_printf(buf, "This will not."); tt_str_op((const char *) evbuffer_pullup(buf, -1), ==, "This will not."); evbuffer_validate(buf); evbuffer_drain(buf, evbuffer_get_length(buf)); evbuffer_validate(buf); #if 0 /* Now let's try a suspended callback. */ cb1 = evbuffer_add_cb(buf, log_change_callback, buf_out1); cb2 = evbuffer_add_cb(buf, log_change_callback, buf_out2); evbuffer_cb_suspend(buf,cb2); evbuffer_prepend(buf,"Hello world",11); /*0->11*/ evbuffer_validate(buf); evbuffer_cb_suspend(buf,cb1); evbuffer_add(buf,"more",4); /* 11->15 */ evbuffer_cb_unsuspend(buf,cb2); evbuffer_drain(buf, 4); /* 15->11 */ evbuffer_cb_unsuspend(buf,cb1); evbuffer_drain(buf, evbuffer_get_length(buf)); /* 11->0 */ tt_str_op(evbuffer_pullup(buf_out1, -1), ==, "0->11; 11->11; 11->0; "); tt_str_op(evbuffer_pullup(buf_out2, -1), ==, "0->15; 15->11; 11->0; "); #endif end: if (buf) evbuffer_free(buf); if (buf_out1) evbuffer_free(buf_out1); if (buf_out2) evbuffer_free(buf_out2); } static int ref_done_cb_called_count = 0; static void *ref_done_cb_called_with = NULL; static const void *ref_done_cb_called_with_data = NULL; static size_t ref_done_cb_called_with_len = 0; static void ref_done_cb(const void *data, size_t len, void *info) { ++ref_done_cb_called_count; ref_done_cb_called_with = info; ref_done_cb_called_with_data = data; ref_done_cb_called_with_len = len; } static void test_evbuffer_add_reference(void *ptr) { const char chunk1[] = "If you have found the answer to such a problem"; const char chunk2[] = "you ought to write it up for publication"; /* -- Knuth's "Notes on the Exercises" from TAOCP */ char tmp[16]; size_t len1 = strlen(chunk1), len2=strlen(chunk2); struct evbuffer *buf1 = NULL, *buf2 = NULL; buf1 = evbuffer_new(); tt_assert(buf1); evbuffer_add_reference(buf1, chunk1, len1, ref_done_cb, (void*)111); evbuffer_add(buf1, ", ", 2); evbuffer_add_reference(buf1, chunk2, len2, ref_done_cb, (void*)222); tt_int_op(evbuffer_get_length(buf1), ==, len1+len2+2); /* Make sure we can drain a little from a reference. */ tt_int_op(evbuffer_remove(buf1, tmp, 6), ==, 6); tt_int_op(memcmp(tmp, "If you", 6), ==, 0); tt_int_op(evbuffer_remove(buf1, tmp, 5), ==, 5); tt_int_op(memcmp(tmp, " have", 5), ==, 0); /* Make sure that prepending does not meddle with immutable data */ tt_int_op(evbuffer_prepend(buf1, "I have ", 7), ==, 0); tt_int_op(memcmp(chunk1, "If you", 6), ==, 0); evbuffer_validate(buf1); /* Make sure that when the chunk is over, the callback is invoked. */ evbuffer_drain(buf1, 7); /* Remove prepended stuff. */ evbuffer_drain(buf1, len1-11-1); /* remove all but one byte of chunk1 */ tt_int_op(ref_done_cb_called_count, ==, 0); evbuffer_remove(buf1, tmp, 1); tt_int_op(tmp[0], ==, 'm'); tt_assert(ref_done_cb_called_with == (void*)111); tt_assert(ref_done_cb_called_with_data == chunk1); tt_assert(ref_done_cb_called_with_len == len1); tt_int_op(ref_done_cb_called_count, ==, 1); evbuffer_validate(buf1); /* Drain some of the remaining chunk, then add it to another buffer */ evbuffer_drain(buf1, 6); /* Remove the ", you ". */ buf2 = evbuffer_new(); tt_assert(buf2); tt_int_op(ref_done_cb_called_count, ==, 1); evbuffer_add(buf2, "I ", 2); evbuffer_add_buffer(buf2, buf1); tt_int_op(ref_done_cb_called_count, ==, 1); evbuffer_remove(buf2, tmp, 16); tt_int_op(memcmp("I ought to write", tmp, 16), ==, 0); evbuffer_drain(buf2, evbuffer_get_length(buf2)); tt_int_op(ref_done_cb_called_count, ==, 2); tt_assert(ref_done_cb_called_with == (void*)222); evbuffer_validate(buf2); /* Now add more stuff to buf1 and make sure that it gets removed on * free. */ evbuffer_add(buf1, "You shake and shake the ", 24); evbuffer_add_reference(buf1, "ketchup bottle", 14, ref_done_cb, (void*)3333); evbuffer_add(buf1, ". Nothing comes and then a lot'll.", 35); evbuffer_free(buf1); buf1 = NULL; tt_int_op(ref_done_cb_called_count, ==, 3); tt_assert(ref_done_cb_called_with == (void*)3333); end: if (buf1) evbuffer_free(buf1); if (buf2) evbuffer_free(buf2); } static void test_evbuffer_multicast(void *ptr) { const char chunk1[] = "If you have found the answer to such a problem"; const char chunk2[] = "you ought to write it up for publication"; /* -- Knuth's "Notes on the Exercises" from TAOCP */ char tmp[16]; size_t len1 = strlen(chunk1), len2=strlen(chunk2); struct evbuffer *buf1 = NULL, *buf2 = NULL; buf1 = evbuffer_new(); tt_assert(buf1); evbuffer_add(buf1, chunk1, len1); evbuffer_add(buf1, ", ", 2); evbuffer_add(buf1, chunk2, len2); tt_int_op(evbuffer_get_length(buf1), ==, len1+len2+2); buf2 = evbuffer_new(); tt_assert(buf2); tt_int_op(evbuffer_add_buffer_reference(buf2, buf1), ==, 0); /* nested references are not allowed */ tt_int_op(evbuffer_add_buffer_reference(buf2, buf2), ==, -1); tt_int_op(evbuffer_add_buffer_reference(buf1, buf2), ==, -1); /* both buffers contain the same amount of data */ tt_int_op(evbuffer_get_length(buf1), ==, evbuffer_get_length(buf1)); /* Make sure we can drain a little from the first buffer. */ tt_int_op(evbuffer_remove(buf1, tmp, 6), ==, 6); tt_int_op(memcmp(tmp, "If you", 6), ==, 0); tt_int_op(evbuffer_remove(buf1, tmp, 5), ==, 5); tt_int_op(memcmp(tmp, " have", 5), ==, 0); /* Make sure that prepending does not meddle with immutable data */ tt_int_op(evbuffer_prepend(buf1, "I have ", 7), ==, 0); tt_int_op(memcmp(chunk1, "If you", 6), ==, 0); evbuffer_validate(buf1); /* Make sure we can drain a little from the second buffer. */ tt_int_op(evbuffer_remove(buf2, tmp, 6), ==, 6); tt_int_op(memcmp(tmp, "If you", 6), ==, 0); tt_int_op(evbuffer_remove(buf2, tmp, 5), ==, 5); tt_int_op(memcmp(tmp, " have", 5), ==, 0); /* Make sure that prepending does not meddle with immutable data */ tt_int_op(evbuffer_prepend(buf2, "I have ", 7), ==, 0); tt_int_op(memcmp(chunk1, "If you", 6), ==, 0); evbuffer_validate(buf2); /* Make sure the data can be read from the second buffer when the first is freed */ evbuffer_free(buf1); buf1 = NULL; tt_int_op(evbuffer_remove(buf2, tmp, 6), ==, 6); tt_int_op(memcmp(tmp, "I have", 6), ==, 0); tt_int_op(evbuffer_remove(buf2, tmp, 6), ==, 6); tt_int_op(memcmp(tmp, " foun", 6), ==, 0); end: if (buf1) evbuffer_free(buf1); if (buf2) evbuffer_free(buf2); } static void test_evbuffer_multicast_drain(void *ptr) { const char chunk1[] = "If you have found the answer to such a problem"; const char chunk2[] = "you ought to write it up for publication"; /* -- Knuth's "Notes on the Exercises" from TAOCP */ size_t len1 = strlen(chunk1), len2=strlen(chunk2); struct evbuffer *buf1 = NULL, *buf2 = NULL; buf1 = evbuffer_new(); tt_assert(buf1); evbuffer_add(buf1, chunk1, len1); evbuffer_add(buf1, ", ", 2); evbuffer_add(buf1, chunk2, len2); tt_int_op(evbuffer_get_length(buf1), ==, len1+len2+2); buf2 = evbuffer_new(); tt_assert(buf2); tt_int_op(evbuffer_add_buffer_reference(buf2, buf1), ==, 0); tt_int_op(evbuffer_get_length(buf2), ==, len1+len2+2); tt_int_op(evbuffer_drain(buf1, evbuffer_get_length(buf1)), ==, 0); tt_int_op(evbuffer_get_length(buf2), ==, len1+len2+2); tt_int_op(evbuffer_drain(buf2, evbuffer_get_length(buf2)), ==, 0); evbuffer_validate(buf1); evbuffer_validate(buf2); end: if (buf1) evbuffer_free(buf1); if (buf2) evbuffer_free(buf2); } static void check_prepend(struct evbuffer *buffer, const struct evbuffer_cb_info *cbinfo, void *arg) { tt_int_op(cbinfo->orig_size, ==, 3); tt_int_op(cbinfo->n_added, ==, 8096); tt_int_op(cbinfo->n_deleted, ==, 0); end: ; } /* Some cases that we didn't get in test_evbuffer() above, for more coverage. */ static void test_evbuffer_prepend(void *ptr) { struct evbuffer *buf1 = NULL, *buf2 = NULL; char tmp[128], *buffer = malloc(8096); int n; buf1 = evbuffer_new(); tt_assert(buf1); /* Case 0: The evbuffer is entirely empty. */ evbuffer_prepend(buf1, "This string has 29 characters", 29); evbuffer_validate(buf1); /* Case 1: Prepend goes entirely in new chunk. */ evbuffer_prepend(buf1, "Short.", 6); evbuffer_validate(buf1); /* Case 2: prepend goes entirely in first chunk. */ evbuffer_drain(buf1, 6+11); evbuffer_prepend(buf1, "it", 2); evbuffer_validate(buf1); tt_assert(!memcmp(buf1->first->buffer+buf1->first->misalign, "it has", 6)); /* Case 3: prepend is split over multiple chunks. */ evbuffer_prepend(buf1, "It is no longer true to say ", 28); evbuffer_validate(buf1); n = evbuffer_remove(buf1, tmp, sizeof(tmp)-1); tt_int_op(n, >=, 0); tmp[n]='\0'; tt_str_op(tmp,==,"It is no longer true to say it has 29 characters"); buf2 = evbuffer_new(); tt_assert(buf2); /* Case 4: prepend a buffer to an empty buffer. */ n = 999; evbuffer_add_printf(buf1, "Here is string %d. ", n++); evbuffer_prepend_buffer(buf2, buf1); evbuffer_validate(buf2); /* Case 5: prepend a buffer to a nonempty buffer. */ evbuffer_add_printf(buf1, "Here is string %d. ", n++); evbuffer_prepend_buffer(buf2, buf1); evbuffer_validate(buf2); evbuffer_validate(buf1); n = evbuffer_remove(buf2, tmp, sizeof(tmp)-1); tt_int_op(n, >=, 0); tmp[n]='\0'; tt_str_op(tmp,==,"Here is string 1000. Here is string 999. "); /* Case 5: evbuffer_prepend() will need a new buffer, with callbacks */ memset(buffer, 'A', 8096); evbuffer_free(buf2); buf2 = evbuffer_new(); tt_assert(buf2); evbuffer_prepend(buf2, "foo", 3); evbuffer_add_cb(buf2, check_prepend, NULL); evbuffer_prepend(buf2, buffer, 8096); evbuffer_remove_cb(buf2, check_prepend, NULL); evbuffer_validate(buf2); tt_nstr_op(8096,(char *)evbuffer_pullup(buf2, 8096),==,buffer); evbuffer_drain(buf2, 8096); tt_nstr_op(3,(char *)evbuffer_pullup(buf2, 3),==,"foo"); evbuffer_drain(buf2, 3); end: free(buffer); if (buf1) evbuffer_free(buf1); if (buf2) evbuffer_free(buf2); } static void test_evbuffer_peek_first_gt(void *info) { struct evbuffer *buf = NULL, *tmp_buf = NULL; struct evbuffer_ptr ptr; struct evbuffer_iovec v[2]; buf = evbuffer_new(); tmp_buf = evbuffer_new(); evbuffer_add_printf(tmp_buf, "Contents of chunk 100\n"); evbuffer_add_buffer(buf, tmp_buf); evbuffer_add_printf(tmp_buf, "Contents of chunk 1\n"); evbuffer_add_buffer(buf, tmp_buf); evbuffer_ptr_set(buf, &ptr, 0, EVBUFFER_PTR_SET); /** The only case that matters*/ tt_int_op(evbuffer_peek(buf, -1, &ptr, NULL, 0), ==, 2); /** Just in case */ tt_int_op(evbuffer_peek(buf, -1, &ptr, v, 2), ==, 2); evbuffer_ptr_set(buf, &ptr, 20, EVBUFFER_PTR_ADD); tt_int_op(evbuffer_peek(buf, -1, &ptr, NULL, 0), ==, 2); tt_int_op(evbuffer_peek(buf, -1, &ptr, v, 2), ==, 2); tt_int_op(evbuffer_peek(buf, 2, &ptr, NULL, 0), ==, 1); tt_int_op(evbuffer_peek(buf, 2, &ptr, v, 2), ==, 1); tt_int_op(evbuffer_peek(buf, 3, &ptr, NULL, 0), ==, 2); tt_int_op(evbuffer_peek(buf, 3, &ptr, v, 2), ==, 2); end: if (buf) evbuffer_free(buf); if (tmp_buf) evbuffer_free(tmp_buf); } static void test_evbuffer_peek(void *info) { struct evbuffer *buf = NULL, *tmp_buf = NULL; int i; struct evbuffer_iovec v[20]; struct evbuffer_ptr ptr; #define tt_iov_eq(v, s) \ tt_int_op((v)->iov_len, ==, strlen(s)); \ tt_assert(!memcmp((v)->iov_base, (s), strlen(s))) /* Let's make a very fragmented buffer. */ buf = evbuffer_new(); tmp_buf = evbuffer_new(); for (i = 0; i < 16; ++i) { evbuffer_add_printf(tmp_buf, "Contents of chunk [%d]\n", i); evbuffer_add_buffer(buf, tmp_buf); } /* How many chunks do we need for everything? */ i = evbuffer_peek(buf, -1, NULL, NULL, 0); tt_int_op(i, ==, 16); /* Simple peek: get everything. */ i = evbuffer_peek(buf, -1, NULL, v, 20); tt_int_op(i, ==, 16); /* we used only 16 chunks. */ tt_iov_eq(&v[0], "Contents of chunk [0]\n"); tt_iov_eq(&v[3], "Contents of chunk [3]\n"); tt_iov_eq(&v[12], "Contents of chunk [12]\n"); tt_iov_eq(&v[15], "Contents of chunk [15]\n"); /* Just get one chunk worth. */ memset(v, 0, sizeof(v)); i = evbuffer_peek(buf, -1, NULL, v, 1); tt_int_op(i, ==, 1); tt_iov_eq(&v[0], "Contents of chunk [0]\n"); tt_assert(v[1].iov_base == NULL); /* Suppose we want at least the first 40 bytes. */ memset(v, 0, sizeof(v)); i = evbuffer_peek(buf, 40, NULL, v, 16); tt_int_op(i, ==, 2); tt_iov_eq(&v[0], "Contents of chunk [0]\n"); tt_iov_eq(&v[1], "Contents of chunk [1]\n"); tt_assert(v[2].iov_base == NULL); /* How many chunks do we need for 100 bytes? */ memset(v, 0, sizeof(v)); i = evbuffer_peek(buf, 100, NULL, NULL, 0); tt_int_op(i, ==, 5); tt_assert(v[0].iov_base == NULL); /* Now we ask for more bytes than we provide chunks for */ memset(v, 0, sizeof(v)); i = evbuffer_peek(buf, 60, NULL, v, 1); tt_int_op(i, ==, 3); tt_iov_eq(&v[0], "Contents of chunk [0]\n"); tt_assert(v[1].iov_base == NULL); /* Now we ask for more bytes than the buffer has. */ memset(v, 0, sizeof(v)); i = evbuffer_peek(buf, 65536, NULL, v, 20); tt_int_op(i, ==, 16); /* we used only 16 chunks. */ tt_iov_eq(&v[0], "Contents of chunk [0]\n"); tt_iov_eq(&v[3], "Contents of chunk [3]\n"); tt_iov_eq(&v[12], "Contents of chunk [12]\n"); tt_iov_eq(&v[15], "Contents of chunk [15]\n"); tt_assert(v[16].iov_base == NULL); /* What happens if we try an empty buffer? */ memset(v, 0, sizeof(v)); i = evbuffer_peek(tmp_buf, -1, NULL, v, 20); tt_int_op(i, ==, 0); tt_assert(v[0].iov_base == NULL); memset(v, 0, sizeof(v)); i = evbuffer_peek(tmp_buf, 50, NULL, v, 20); tt_int_op(i, ==, 0); tt_assert(v[0].iov_base == NULL); /* Okay, now time to have fun with pointers. */ memset(v, 0, sizeof(v)); evbuffer_ptr_set(buf, &ptr, 30, EVBUFFER_PTR_SET); i = evbuffer_peek(buf, 50, &ptr, v, 20); tt_int_op(i, ==, 3); tt_iov_eq(&v[0], " of chunk [1]\n"); tt_iov_eq(&v[1], "Contents of chunk [2]\n"); tt_iov_eq(&v[2], "Contents of chunk [3]\n"); /*more than we asked for*/ /* advance to the start of another chain. */ memset(v, 0, sizeof(v)); evbuffer_ptr_set(buf, &ptr, 14, EVBUFFER_PTR_ADD); i = evbuffer_peek(buf, 44, &ptr, v, 20); tt_int_op(i, ==, 2); tt_iov_eq(&v[0], "Contents of chunk [2]\n"); tt_iov_eq(&v[1], "Contents of chunk [3]\n"); /*more than we asked for*/ /* peek at the end of the buffer */ memset(v, 0, sizeof(v)); tt_assert(evbuffer_ptr_set(buf, &ptr, evbuffer_get_length(buf), EVBUFFER_PTR_SET) == 0); i = evbuffer_peek(buf, 44, &ptr, v, 20); tt_int_op(i, ==, 0); tt_assert(v[0].iov_base == NULL); end: if (buf) evbuffer_free(buf); if (tmp_buf) evbuffer_free(tmp_buf); } /* Check whether evbuffer freezing works right. This is called twice, once with the argument "start" and once with the argument "end". When we test "start", we freeze the start of an evbuffer and make sure that modifying the start of the buffer doesn't work. When we test "end", we freeze the end of an evbuffer and make sure that modifying the end of the buffer doesn't work. */ static void test_evbuffer_freeze(void *ptr) { struct evbuffer *buf = NULL, *tmp_buf=NULL; const char string[] = /* Year's End, Richard Wilbur */ "I've known the wind by water banks to shake\n" "The late leaves down, which frozen where they fell\n" "And held in ice as dancers in a spell\n" "Fluttered all winter long into a lake..."; const int start = !strcmp(ptr, "start"); char *cp; char charbuf[128]; int r; size_t orig_length; struct evbuffer_iovec v[1]; if (!start) tt_str_op(ptr, ==, "end"); buf = evbuffer_new(); tmp_buf = evbuffer_new(); tt_assert(tmp_buf); evbuffer_add(buf, string, strlen(string)); evbuffer_freeze(buf, start); /* Freeze the start or the end.*/ #define FREEZE_EQ(a, startcase, endcase) \ do { \ if (start) { \ tt_int_op((a), ==, (startcase)); \ } else { \ tt_int_op((a), ==, (endcase)); \ } \ } while (0) orig_length = evbuffer_get_length(buf); /* These functions all manipulate the end of buf. */ r = evbuffer_add(buf, "abc", 0); FREEZE_EQ(r, 0, -1); r = evbuffer_reserve_space(buf, 10, v, 1); FREEZE_EQ(r, 1, -1); if (r == 1) { memset(v[0].iov_base, 'X', 10); v[0].iov_len = 10; } r = evbuffer_commit_space(buf, v, 1); FREEZE_EQ(r, 0, -1); r = evbuffer_add_reference(buf, string, 5, NULL, NULL); FREEZE_EQ(r, 0, -1); r = evbuffer_add_printf(buf, "Hello %s", "world"); FREEZE_EQ(r, 11, -1); /* TODO: test add_buffer, add_file, read */ if (!start) tt_int_op(orig_length, ==, evbuffer_get_length(buf)); orig_length = evbuffer_get_length(buf); /* These functions all manipulate the start of buf. */ r = evbuffer_remove(buf, charbuf, 1); FREEZE_EQ(r, -1, 1); r = evbuffer_drain(buf, 3); FREEZE_EQ(r, -1, 0); r = evbuffer_prepend(buf, "dummy", 5); FREEZE_EQ(r, -1, 0); cp = evbuffer_readln(buf, NULL, EVBUFFER_EOL_LF); FREEZE_EQ(cp==NULL, 1, 0); if (cp) free(cp); /* TODO: Test remove_buffer, add_buffer, write, prepend_buffer */ if (start) tt_int_op(orig_length, ==, evbuffer_get_length(buf)); end: if (buf) evbuffer_free(buf); if (tmp_buf) evbuffer_free(tmp_buf); } static void test_evbuffer_add_iovec(void * ptr) { struct evbuffer * buf = NULL; struct evbuffer_iovec vec[4]; const char * data[] = { "Guilt resembles a sword with two edges.", "On the one hand, it cuts for Justice, imposing practical morality upon those who fear it.", "Conscience does not always adhere to rational judgment.", "Guilt is always a self-imposed burden, but it is not always rightly imposed." /* -- R.A. Salvatore, _Sojurn_ */ }; size_t expected_length = 0; size_t returned_length = 0; int i; buf = evbuffer_new(); tt_assert(buf); for (i = 0; i < 4; i++) { vec[i].iov_len = strlen(data[i]); vec[i].iov_base = (char*) data[i]; expected_length += vec[i].iov_len; } returned_length = evbuffer_add_iovec(buf, vec, 4); tt_int_op(returned_length, ==, evbuffer_get_length(buf)); tt_int_op(evbuffer_get_length(buf), ==, expected_length); for (i = 0; i < 4; i++) { char charbuf[1024]; memset(charbuf, 0, 1024); evbuffer_remove(buf, charbuf, strlen(data[i])); tt_assert(strcmp(charbuf, data[i]) == 0); } tt_assert(evbuffer_get_length(buf) == 0); end: if (buf) { evbuffer_free(buf); } } static void test_evbuffer_copyout(void *dummy) { const char string[] = "Still they skirmish to and fro, men my messmates on the snow " "When we headed off the aurochs turn for turn; " "When the rich Allobrogenses never kept amanuenses, " "And our only plots were piled in lakes at Berne."; /* -- Kipling, "In The Neolithic Age" */ char tmp[1024]; struct evbuffer_ptr ptr; struct evbuffer *buf; (void)dummy; buf = evbuffer_new(); tt_assert(buf); tt_int_op(strlen(string), ==, 206); /* Ensure separate chains */ evbuffer_add_reference(buf, string, 80, no_cleanup, NULL); evbuffer_add_reference(buf, string+80, 80, no_cleanup, NULL); evbuffer_add(buf, string+160, strlen(string)-160); tt_int_op(206, ==, evbuffer_get_length(buf)); /* First, let's test plain old copyout. */ /* Copy a little from the beginning. */ tt_int_op(10, ==, evbuffer_copyout(buf, tmp, 10)); tt_int_op(0, ==, memcmp(tmp, "Still they", 10)); /* Now copy more than a little from the beginning */ memset(tmp, 0, sizeof(tmp)); tt_int_op(100, ==, evbuffer_copyout(buf, tmp, 100)); tt_int_op(0, ==, memcmp(tmp, string, 100)); /* Copy too much; ensure truncation. */ memset(tmp, 0, sizeof(tmp)); tt_int_op(206, ==, evbuffer_copyout(buf, tmp, 230)); tt_int_op(0, ==, memcmp(tmp, string, 206)); /* That was supposed to be nondestructive, btw */ tt_int_op(206, ==, evbuffer_get_length(buf)); /* Now it's time to test copyout_from! First, let's start in the * first chain. */ evbuffer_ptr_set(buf, &ptr, 15, EVBUFFER_PTR_SET); memset(tmp, 0, sizeof(tmp)); tt_int_op(10, ==, evbuffer_copyout_from(buf, &ptr, tmp, 10)); tt_int_op(0, ==, memcmp(tmp, "mish to an", 10)); /* Right up to the end of the first chain */ memset(tmp, 0, sizeof(tmp)); tt_int_op(65, ==, evbuffer_copyout_from(buf, &ptr, tmp, 65)); tt_int_op(0, ==, memcmp(tmp, string+15, 65)); /* Span into the second chain */ memset(tmp, 0, sizeof(tmp)); tt_int_op(90, ==, evbuffer_copyout_from(buf, &ptr, tmp, 90)); tt_int_op(0, ==, memcmp(tmp, string+15, 90)); /* Span into the third chain */ memset(tmp, 0, sizeof(tmp)); tt_int_op(160, ==, evbuffer_copyout_from(buf, &ptr, tmp, 160)); tt_int_op(0, ==, memcmp(tmp, string+15, 160)); /* Overrun */ memset(tmp, 0, sizeof(tmp)); tt_int_op(206-15, ==, evbuffer_copyout_from(buf, &ptr, tmp, 999)); tt_int_op(0, ==, memcmp(tmp, string+15, 206-15)); /* That was supposed to be nondestructive, too */ tt_int_op(206, ==, evbuffer_get_length(buf)); end: if (buf) evbuffer_free(buf); } static void * setup_passthrough(const struct testcase_t *testcase) { return testcase->setup_data; } static int cleanup_passthrough(const struct testcase_t *testcase, void *ptr) { (void) ptr; return 1; } static const struct testcase_setup_t nil_setup = { setup_passthrough, cleanup_passthrough }; struct testcase_t evbuffer_testcases[] = { { "evbuffer", test_evbuffer, 0, NULL, NULL }, { "remove_buffer_with_empty", test_evbuffer_remove_buffer_with_empty, 0, NULL, NULL }, { "remove_buffer_with_empty2", test_evbuffer_remove_buffer_with_empty2, 0, NULL, NULL }, { "remove_buffer_with_empty3", test_evbuffer_remove_buffer_with_empty3, 0, NULL, NULL }, { "add_buffer_with_empty", test_evbuffer_add_buffer_with_empty, 0, NULL, NULL }, { "add_buffer_with_empty2", test_evbuffer_add_buffer_with_empty2, 0, NULL, NULL }, { "reserve2", test_evbuffer_reserve2, 0, NULL, NULL }, { "reserve_many", test_evbuffer_reserve_many, 0, NULL, NULL }, { "reserve_many2", test_evbuffer_reserve_many, 0, &nil_setup, (void*)"add" }, { "reserve_many3", test_evbuffer_reserve_many, 0, &nil_setup, (void*)"fill" }, { "expand", test_evbuffer_expand, 0, NULL, NULL }, { "expand_overflow", test_evbuffer_expand_overflow, 0, NULL, NULL }, { "add1", test_evbuffer_add1, 0, NULL, NULL }, { "add2", test_evbuffer_add2, 0, NULL, NULL }, { "reference", test_evbuffer_reference, 0, NULL, NULL }, { "reference2", test_evbuffer_reference2, 0, NULL, NULL }, { "iterative", test_evbuffer_iterative, 0, NULL, NULL }, { "readln", test_evbuffer_readln, TT_NO_LOGS, &basic_setup, NULL }, { "search_eol", test_evbuffer_search_eol, 0, NULL, NULL }, { "find", test_evbuffer_find, 0, NULL, NULL }, { "ptr_set", test_evbuffer_ptr_set, 0, NULL, NULL }, { "search", test_evbuffer_search, 0, NULL, NULL }, { "callbacks", test_evbuffer_callbacks, 0, NULL, NULL }, { "add_reference", test_evbuffer_add_reference, 0, NULL, NULL }, { "multicast", test_evbuffer_multicast, 0, NULL, NULL }, { "multicast_drain", test_evbuffer_multicast_drain, 0, NULL, NULL }, { "prepend", test_evbuffer_prepend, TT_FORK, NULL, NULL }, { "peek", test_evbuffer_peek, 0, NULL, NULL }, { "peek_first_gt", test_evbuffer_peek_first_gt, 0, NULL, NULL }, { "freeze_start", test_evbuffer_freeze, 0, &nil_setup, (void*)"start" }, { "freeze_end", test_evbuffer_freeze, 0, &nil_setup, (void*)"end" }, { "add_iovec", test_evbuffer_add_iovec, 0, NULL, NULL}, { "copyout", test_evbuffer_copyout, 0, NULL, NULL}, { "file_segment_add_cleanup_cb", test_evbuffer_file_segment_add_cleanup_cb, 0, NULL, NULL }, #define ADDFILE_TEST(name, parameters) \ { name, test_evbuffer_add_file, TT_FORK|TT_NEED_BASE, \ &basic_setup, (void*)(parameters) } #define ADDFILE_TEST_GROUP(name, parameters) \ ADDFILE_TEST(name "_sendfile", "sendfile " parameters), \ ADDFILE_TEST(name "_mmap", "mmap " parameters), \ ADDFILE_TEST(name "_linear", "linear " parameters) ADDFILE_TEST_GROUP("add_file", ""), ADDFILE_TEST("add_file_nosegment", "default nosegment"), ADDFILE_TEST_GROUP("add_big_file", "bigfile"), ADDFILE_TEST("add_big_file_nosegment", "default nosegment bigfile"), ADDFILE_TEST_GROUP("add_file_offset", "bigfile map_offset"), ADDFILE_TEST("add_file_offset_nosegment", "default nosegment bigfile map_offset"), ADDFILE_TEST_GROUP("add_file_offset2", "bigfile offset_in_segment"), ADDFILE_TEST_GROUP("add_file_offset3", "bigfile offset_in_segment map_offset"), END_OF_TESTCASES }; kytensor-2.49.0.7/third_party/libevent/test/regress_bufferevent.c000066400000000000000000001157361527052371200252030ustar00rootroot00000000000000/* * Copyright (c) 2003-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "util-internal.h" /* The old tests here need assertions to work. */ #undef NDEBUG #ifdef _WIN32 #include #include #endif #include "event2/event-config.h" #include #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #ifndef _WIN32 #include #include #include #include #include #include #endif #include #include #include #include #include #include #include #ifdef EVENT__HAVE_ARPA_INET_H #include #endif #include "event2/event-config.h" #include "event2/event.h" #include "event2/event_struct.h" #include "event2/event_compat.h" #include "event2/tag.h" #include "event2/buffer.h" #include "event2/bufferevent.h" #include "event2/bufferevent_compat.h" #include "event2/bufferevent_struct.h" #include "event2/listener.h" #include "event2/util.h" #include "bufferevent-internal.h" #include "evthread-internal.h" #include "util-internal.h" #ifdef _WIN32 #include "iocp-internal.h" #endif #include "regress.h" #include "regress_testutils.h" /* * simple bufferevent test */ static void readcb(struct bufferevent *bev, void *arg) { if (evbuffer_get_length(bev->input) == 8333) { struct evbuffer *evbuf = evbuffer_new(); assert(evbuf != NULL); /* gratuitous test of bufferevent_read_buffer */ bufferevent_read_buffer(bev, evbuf); bufferevent_disable(bev, EV_READ); if (evbuffer_get_length(evbuf) == 8333) { test_ok++; } evbuffer_free(evbuf); } } static void writecb(struct bufferevent *bev, void *arg) { if (evbuffer_get_length(bev->output) == 0) { test_ok++; } } static void errorcb(struct bufferevent *bev, short what, void *arg) { test_ok = -2; } static void test_bufferevent_impl(int use_pair, int flush) { struct bufferevent *bev1 = NULL, *bev2 = NULL; char buffer[8333]; int i; int expected = 2; if (use_pair) { struct bufferevent *pair[2]; tt_assert(0 == bufferevent_pair_new(NULL, 0, pair)); bev1 = pair[0]; bev2 = pair[1]; bufferevent_setcb(bev1, readcb, writecb, errorcb, bev1); bufferevent_setcb(bev2, readcb, writecb, errorcb, NULL); tt_int_op(bufferevent_getfd(bev1), ==, -1); tt_ptr_op(bufferevent_get_underlying(bev1), ==, NULL); tt_ptr_op(bufferevent_pair_get_partner(bev1), ==, bev2); tt_ptr_op(bufferevent_pair_get_partner(bev2), ==, bev1); } else { bev1 = bufferevent_new(pair[0], readcb, writecb, errorcb, NULL); bev2 = bufferevent_new(pair[1], readcb, writecb, errorcb, NULL); tt_int_op(bufferevent_getfd(bev1), ==, pair[0]); tt_ptr_op(bufferevent_get_underlying(bev1), ==, NULL); tt_ptr_op(bufferevent_pair_get_partner(bev1), ==, NULL); tt_ptr_op(bufferevent_pair_get_partner(bev2), ==, NULL); } { /* Test getcb. */ bufferevent_data_cb r, w; bufferevent_event_cb e; void *a; bufferevent_getcb(bev1, &r, &w, &e, &a); tt_ptr_op(r, ==, readcb); tt_ptr_op(w, ==, writecb); tt_ptr_op(e, ==, errorcb); tt_ptr_op(a, ==, use_pair ? bev1 : NULL); } bufferevent_disable(bev1, EV_READ); bufferevent_enable(bev2, EV_READ); tt_int_op(bufferevent_get_enabled(bev1), ==, EV_WRITE); tt_int_op(bufferevent_get_enabled(bev2), ==, EV_WRITE|EV_READ); for (i = 0; i < (int)sizeof(buffer); i++) buffer[i] = i; bufferevent_write(bev1, buffer, sizeof(buffer)); if (flush >= 0) { tt_int_op(bufferevent_flush(bev1, EV_WRITE, flush), >=, 0); } event_dispatch(); bufferevent_free(bev2); tt_ptr_op(bufferevent_pair_get_partner(bev1), ==, NULL); bufferevent_free(bev1); /** Only pair call errorcb for BEV_FINISHED */ if (use_pair && flush == BEV_FINISHED) { expected = -1; } if (test_ok != expected) test_ok = 0; end: ; } static void test_bufferevent(void) { test_bufferevent_impl(0, -1); } static void test_bufferevent_pair(void) { test_bufferevent_impl(1, -1); } static void test_bufferevent_flush_normal(void) { test_bufferevent_impl(0, BEV_NORMAL); } static void test_bufferevent_flush_flush(void) { test_bufferevent_impl(0, BEV_FLUSH); } static void test_bufferevent_flush_finished(void) { test_bufferevent_impl(0, BEV_FINISHED); } static void test_bufferevent_pair_flush_normal(void) { test_bufferevent_impl(1, BEV_NORMAL); } static void test_bufferevent_pair_flush_flush(void) { test_bufferevent_impl(1, BEV_FLUSH); } static void test_bufferevent_pair_flush_finished(void) { test_bufferevent_impl(1, BEV_FINISHED); } #if defined(EVTHREAD_USE_PTHREADS_IMPLEMENTED) /** * Trace lock/unlock/alloc/free for locks. * (More heavier then evthread_debug*) */ typedef struct { void *lock; enum { ALLOC, FREE, } status; size_t locked /** allow recursive locking */; } lock_wrapper; struct lock_unlock_base { /* Original callbacks */ struct evthread_lock_callbacks cbs; /* Map of locks */ lock_wrapper *locks; size_t nr_locks; } lu_base = { .locks = NULL, }; static lock_wrapper *lu_find(void *lock_) { size_t i; for (i = 0; i < lu_base.nr_locks; ++i) { lock_wrapper *lock = &lu_base.locks[i]; if (lock->lock == lock_) return lock; } return NULL; } static void *trace_lock_alloc(unsigned locktype) { void *lock; ++lu_base.nr_locks; lu_base.locks = realloc(lu_base.locks, sizeof(lock_wrapper) * lu_base.nr_locks); lock = lu_base.cbs.alloc(locktype); lu_base.locks[lu_base.nr_locks - 1] = (lock_wrapper){ lock, ALLOC, 0 }; return lock; } static void trace_lock_free(void *lock_, unsigned locktype) { lock_wrapper *lock = lu_find(lock_); if (!lock || lock->status == FREE || lock->locked) { TT_FAIL(("lock: free error")); } else { lock->status = FREE; lu_base.cbs.free(lock_, locktype); } } static int trace_lock_lock(unsigned mode, void *lock_) { lock_wrapper *lock = lu_find(lock_); if (!lock || lock->status == FREE) { TT_FAIL(("lock: lock error")); return -1; } else { ++lock->locked; return lu_base.cbs.lock(mode, lock_); } } static int trace_lock_unlock(unsigned mode, void *lock_) { lock_wrapper *lock = lu_find(lock_); if (!lock || lock->status == FREE || !lock->locked) { TT_FAIL(("lock: unlock error")); return -1; } else { --lock->locked; return lu_base.cbs.unlock(mode, lock_); } } static void lock_unlock_free_thread_cbs(void) { event_base_free(NULL); if (libevent_tests_running_in_debug_mode) libevent_global_shutdown(); /** drop immutable flag */ evthread_set_lock_callbacks(NULL); /** avoid calling of event_global_setup_locks_() for new cbs */ libevent_global_shutdown(); /** drop immutable flag for non-debug ops (since called after shutdown) */ evthread_set_lock_callbacks(NULL); } static int use_lock_unlock_profiler(void) { struct evthread_lock_callbacks cbs = { EVTHREAD_LOCK_API_VERSION, EVTHREAD_LOCKTYPE_RECURSIVE, trace_lock_alloc, trace_lock_free, trace_lock_lock, trace_lock_unlock, }; memcpy(&lu_base.cbs, evthread_get_lock_callbacks(), sizeof(lu_base.cbs)); { lock_unlock_free_thread_cbs(); evthread_set_lock_callbacks(&cbs); /** re-create debug locks correctly */ evthread_enable_lock_debugging(); event_init(); } return 0; } static void free_lock_unlock_profiler(struct basic_test_data *data) { /** fix "held_by" for kqueue */ evthread_set_lock_callbacks(NULL); lock_unlock_free_thread_cbs(); free(lu_base.locks); data->base = NULL; } static void test_bufferevent_pair_release_lock(void *arg) { struct basic_test_data *data = arg; use_lock_unlock_profiler(); { struct bufferevent *pair[2]; if (!bufferevent_pair_new(NULL, BEV_OPT_THREADSAFE, pair)) { bufferevent_free(pair[0]); bufferevent_free(pair[1]); } else tt_abort_perror("bufferevent_pair_new"); } free_lock_unlock_profiler(data); end: ; } #endif /* * test watermarks and bufferevent */ static void wm_readcb(struct bufferevent *bev, void *arg) { struct evbuffer *evbuf = evbuffer_new(); int len = (int)evbuffer_get_length(bev->input); static int nread; assert(len >= 10 && len <= 20); assert(evbuf != NULL); /* gratuitous test of bufferevent_read_buffer */ bufferevent_read_buffer(bev, evbuf); nread += len; if (nread == 65000) { bufferevent_disable(bev, EV_READ); test_ok++; } evbuffer_free(evbuf); } static void wm_writecb(struct bufferevent *bev, void *arg) { assert(evbuffer_get_length(bev->output) <= 100); if (evbuffer_get_length(bev->output) == 0) { evbuffer_drain(bev->output, evbuffer_get_length(bev->output)); test_ok++; } } static void wm_errorcb(struct bufferevent *bev, short what, void *arg) { test_ok = -2; } static void test_bufferevent_watermarks_impl(int use_pair) { struct bufferevent *bev1 = NULL, *bev2 = NULL; char buffer[65000]; size_t low, high; int i; test_ok = 0; if (use_pair) { struct bufferevent *pair[2]; tt_assert(0 == bufferevent_pair_new(NULL, 0, pair)); bev1 = pair[0]; bev2 = pair[1]; bufferevent_setcb(bev1, NULL, wm_writecb, errorcb, NULL); bufferevent_setcb(bev2, wm_readcb, NULL, errorcb, NULL); } else { bev1 = bufferevent_new(pair[0], NULL, wm_writecb, wm_errorcb, NULL); bev2 = bufferevent_new(pair[1], wm_readcb, NULL, wm_errorcb, NULL); } tt_assert(bev1); tt_assert(bev2); bufferevent_disable(bev1, EV_READ); bufferevent_enable(bev2, EV_READ); /* By default, low watermarks are set to 0 */ bufferevent_getwatermark(bev1, EV_READ, &low, NULL); tt_int_op(low, ==, 0); bufferevent_getwatermark(bev2, EV_WRITE, &low, NULL); tt_int_op(low, ==, 0); for (i = 0; i < (int)sizeof(buffer); i++) buffer[i] = (char)i; /* limit the reading on the receiving bufferevent */ bufferevent_setwatermark(bev2, EV_READ, 10, 20); bufferevent_getwatermark(bev2, EV_READ, &low, &high); tt_int_op(low, ==, 10); tt_int_op(high, ==, 20); /* Tell the sending bufferevent not to notify us till it's down to 100 bytes. */ bufferevent_setwatermark(bev1, EV_WRITE, 100, 2000); bufferevent_getwatermark(bev1, EV_WRITE, &low, &high); tt_int_op(low, ==, 100); tt_int_op(high, ==, 2000); { int r = bufferevent_getwatermark(bev1, EV_WRITE | EV_READ, &low, &high); tt_int_op(r, !=, 0); } bufferevent_write(bev1, buffer, sizeof(buffer)); event_dispatch(); tt_int_op(test_ok, ==, 2); /* The write callback drained all the data from outbuf, so we * should have removed the write event... */ tt_assert(!event_pending(&bev2->ev_write, EV_WRITE, NULL)); end: if (bev1) bufferevent_free(bev1); if (bev2) bufferevent_free(bev2); } static void test_bufferevent_watermarks(void) { test_bufferevent_watermarks_impl(0); } static void test_bufferevent_pair_watermarks(void) { test_bufferevent_watermarks_impl(1); } /* * Test bufferevent filters */ /* strip an 'x' from each byte */ static enum bufferevent_filter_result bufferevent_input_filter(struct evbuffer *src, struct evbuffer *dst, ev_ssize_t lim, enum bufferevent_flush_mode state, void *ctx) { const unsigned char *buffer; unsigned i; buffer = evbuffer_pullup(src, evbuffer_get_length(src)); for (i = 0; i < evbuffer_get_length(src); i += 2) { if (buffer[i] == '-') continue; assert(buffer[i] == 'x'); evbuffer_add(dst, buffer + i + 1, 1); } evbuffer_drain(src, i); return (BEV_OK); } /* add an 'x' before each byte */ static enum bufferevent_filter_result bufferevent_output_filter(struct evbuffer *src, struct evbuffer *dst, ev_ssize_t lim, enum bufferevent_flush_mode state, void *ctx) { const unsigned char *buffer; unsigned i; struct bufferevent **bevp = ctx; ++test_ok; if (test_ok == 1) { buffer = evbuffer_pullup(src, evbuffer_get_length(src)); for (i = 0; i < evbuffer_get_length(src); ++i) { evbuffer_add(dst, "x", 1); evbuffer_add(dst, buffer + i, 1); } evbuffer_drain(src, evbuffer_get_length(src)); } else { return BEV_ERROR; } if (bevp && test_ok == 1) { int prev = ++test_ok; bufferevent_write(*bevp, "-", 1); /* check that during this bufferevent_write() * bufferevent_output_filter() will not be called again */ assert(test_ok == prev); --test_ok; } return (BEV_OK); } static void test_bufferevent_filters_impl(int use_pair, int disable) { struct bufferevent *bev1 = NULL, *bev2 = NULL; struct bufferevent *bev1_base = NULL, *bev2_base = NULL; char buffer[8333]; int i; test_ok = 0; if (use_pair) { struct bufferevent *pair[2]; tt_assert(0 == bufferevent_pair_new(NULL, 0, pair)); bev1 = pair[0]; bev2 = pair[1]; } else { bev1 = bufferevent_socket_new(NULL, pair[0], 0); bev2 = bufferevent_socket_new(NULL, pair[1], 0); } bev1_base = bev1; bev2_base = bev2; for (i = 0; i < (int)sizeof(buffer); i++) buffer[i] = i; bev1 = bufferevent_filter_new(bev1, NULL, bufferevent_output_filter, BEV_OPT_CLOSE_ON_FREE, NULL, disable ? &bev1 : NULL); bev2 = bufferevent_filter_new(bev2, bufferevent_input_filter, NULL, BEV_OPT_CLOSE_ON_FREE, NULL, NULL); bufferevent_setcb(bev1, NULL, writecb, errorcb, NULL); bufferevent_setcb(bev2, readcb, NULL, errorcb, NULL); tt_ptr_op(bufferevent_get_underlying(bev1), ==, bev1_base); tt_ptr_op(bufferevent_get_underlying(bev2), ==, bev2_base); tt_int_op(bufferevent_getfd(bev1), ==, -1); tt_int_op(bufferevent_getfd(bev2), ==, -1); bufferevent_disable(bev1, EV_READ); bufferevent_enable(bev2, EV_READ); /* insert some filters */ bufferevent_write(bev1, buffer, sizeof(buffer)); event_dispatch(); if (test_ok != 3 + !!disable) test_ok = 0; end: if (bev1) bufferevent_free(bev1); if (bev2) bufferevent_free(bev2); } static void test_bufferevent_filters(void) { test_bufferevent_filters_impl(0, 0); } static void test_bufferevent_pair_filters(void) { test_bufferevent_filters_impl(1, 0); } static void test_bufferevent_filters_disable(void) { test_bufferevent_filters_impl(0, 1); } static void test_bufferevent_pair_filters_disable(void) { test_bufferevent_filters_impl(1, 1); } static void sender_writecb(struct bufferevent *bev, void *ctx) { if (evbuffer_get_length(bufferevent_get_output(bev)) == 0) { bufferevent_disable(bev,EV_READ|EV_WRITE); TT_BLATHER(("Flushed %d: freeing it.", (int)bufferevent_getfd(bev))); bufferevent_free(bev); } } static void sender_errorcb(struct bufferevent *bev, short what, void *ctx) { TT_FAIL(("Got sender error %d",(int)what)); } static int bufferevent_connect_test_flags = 0; static int bufferevent_trigger_test_flags = 0; static int n_strings_read = 0; static int n_reads_invoked = 0; static int n_events_invoked = 0; #define TEST_STR "Now is the time for all good events to signal for " \ "the good of their protocol" static void listen_cb(struct evconnlistener *listener, evutil_socket_t fd, struct sockaddr *sa, int socklen, void *arg) { struct event_base *base = arg; struct bufferevent *bev; const char s[] = TEST_STR; TT_BLATHER(("Got a request on socket %d", (int)fd )); bev = bufferevent_socket_new(base, fd, bufferevent_connect_test_flags); tt_assert(bev); bufferevent_setcb(bev, NULL, sender_writecb, sender_errorcb, NULL); bufferevent_write(bev, s, sizeof(s)); end: ; } static int fake_listener_create(struct sockaddr_in *localhost) { struct sockaddr *sa = (struct sockaddr *)localhost; evutil_socket_t fd = -1; ev_socklen_t slen = sizeof(*localhost); memset(localhost, 0, sizeof(*localhost)); localhost->sin_port = 0; /* have the kernel pick a port */ localhost->sin_addr.s_addr = htonl(0x7f000001L); localhost->sin_family = AF_INET; /* bind, but don't listen or accept. should trigger "Connection refused" reliably on most platforms. */ fd = socket(localhost->sin_family, SOCK_STREAM, 0); tt_assert(fd >= 0); tt_assert(bind(fd, sa, slen) == 0); tt_assert(getsockname(fd, sa, &slen) == 0); return fd; end: return -1; } static void reader_eventcb(struct bufferevent *bev, short what, void *ctx) { struct event_base *base = ctx; if (what & BEV_EVENT_ERROR) { perror("foobar"); TT_FAIL(("got connector error %d", (int)what)); return; } if (what & BEV_EVENT_CONNECTED) { TT_BLATHER(("connected on %d", (int)bufferevent_getfd(bev))); bufferevent_enable(bev, EV_READ); } if (what & BEV_EVENT_EOF) { char buf[512]; size_t n; n = bufferevent_read(bev, buf, sizeof(buf)-1); tt_int_op(n, >=, 0); buf[n] = '\0'; tt_str_op(buf, ==, TEST_STR); if (++n_strings_read == 2) event_base_loopexit(base, NULL); TT_BLATHER(("EOF on %d: %d strings read.", (int)bufferevent_getfd(bev), n_strings_read)); } end: ; } static void reader_eventcb_simple(struct bufferevent *bev, short what, void *ctx) { TT_BLATHER(("Read eventcb simple invoked on %d.", (int)bufferevent_getfd(bev))); n_events_invoked++; } static void reader_readcb(struct bufferevent *bev, void *ctx) { TT_BLATHER(("Read invoked on %d.", (int)bufferevent_getfd(bev))); n_reads_invoked++; } static void test_bufferevent_connect(void *arg) { struct basic_test_data *data = arg; struct evconnlistener *lev=NULL; struct bufferevent *bev1=NULL, *bev2=NULL; struct sockaddr_in localhost; struct sockaddr_storage ss; struct sockaddr *sa; ev_socklen_t slen; int be_flags=BEV_OPT_CLOSE_ON_FREE; if (strstr((char*)data->setup_data, "defer")) { be_flags |= BEV_OPT_DEFER_CALLBACKS; } if (strstr((char*)data->setup_data, "unlocked")) { be_flags |= BEV_OPT_UNLOCK_CALLBACKS; } if (strstr((char*)data->setup_data, "lock")) { be_flags |= BEV_OPT_THREADSAFE; } bufferevent_connect_test_flags = be_flags; #ifdef _WIN32 if (!strcmp((char*)data->setup_data, "unset_connectex")) { struct win32_extension_fns *ext = (struct win32_extension_fns *) event_get_win32_extension_fns_(); ext->ConnectEx = NULL; } #endif memset(&localhost, 0, sizeof(localhost)); localhost.sin_port = 0; /* pick-a-port */ localhost.sin_addr.s_addr = htonl(0x7f000001L); localhost.sin_family = AF_INET; sa = (struct sockaddr *)&localhost; lev = evconnlistener_new_bind(data->base, listen_cb, data->base, LEV_OPT_CLOSE_ON_FREE|LEV_OPT_REUSEABLE, 16, sa, sizeof(localhost)); tt_assert(lev); sa = (struct sockaddr *)&ss; slen = sizeof(ss); if (regress_get_listener_addr(lev, sa, &slen) < 0) { tt_abort_perror("getsockname"); } tt_assert(!evconnlistener_enable(lev)); bev1 = bufferevent_socket_new(data->base, -1, be_flags); bev2 = bufferevent_socket_new(data->base, -1, be_flags); tt_assert(bev1); tt_assert(bev2); bufferevent_setcb(bev1, reader_readcb,NULL, reader_eventcb, data->base); bufferevent_setcb(bev2, reader_readcb,NULL, reader_eventcb, data->base); bufferevent_enable(bev1, EV_READ); bufferevent_enable(bev2, EV_READ); tt_want(!bufferevent_socket_connect(bev1, sa, sizeof(localhost))); tt_want(!bufferevent_socket_connect(bev2, sa, sizeof(localhost))); event_base_dispatch(data->base); tt_int_op(n_strings_read, ==, 2); tt_int_op(n_reads_invoked, >=, 2); end: if (lev) evconnlistener_free(lev); if (bev1) bufferevent_free(bev1); if (bev2) bufferevent_free(bev2); } static void test_bufferevent_connect_fail_eventcb(void *arg) { struct basic_test_data *data = arg; int flags = BEV_OPT_CLOSE_ON_FREE | (long)data->setup_data; struct bufferevent *bev = NULL; struct evconnlistener *lev = NULL; struct sockaddr_in localhost; ev_socklen_t slen = sizeof(localhost); evutil_socket_t fake_listener = -1; fake_listener = fake_listener_create(&localhost); tt_int_op(n_events_invoked, ==, 0); bev = bufferevent_socket_new(data->base, -1, flags); tt_assert(bev); bufferevent_setcb(bev, reader_readcb, reader_readcb, reader_eventcb_simple, data->base); bufferevent_enable(bev, EV_READ|EV_WRITE); tt_int_op(n_events_invoked, ==, 0); tt_int_op(n_reads_invoked, ==, 0); /** @see also test_bufferevent_connect_fail() */ bufferevent_socket_connect(bev, (struct sockaddr *)&localhost, slen); tt_int_op(n_events_invoked, ==, 0); tt_int_op(n_reads_invoked, ==, 0); event_base_dispatch(data->base); tt_int_op(n_events_invoked, ==, 1); tt_int_op(n_reads_invoked, ==, 0); end: if (lev) evconnlistener_free(lev); if (bev) bufferevent_free(bev); if (fake_listener >= 0) evutil_closesocket(fake_listener); } static void want_fail_eventcb(struct bufferevent *bev, short what, void *ctx) { struct event_base *base = ctx; const char *err; evutil_socket_t s; if (what & BEV_EVENT_ERROR) { s = bufferevent_getfd(bev); err = evutil_socket_error_to_string(evutil_socket_geterror(s)); TT_BLATHER(("connection failure on "EV_SOCK_FMT": %s", EV_SOCK_ARG(s), err)); test_ok = 1; } else { TT_FAIL(("didn't fail? what %hd", what)); } event_base_loopexit(base, NULL); } static void close_socket_cb(evutil_socket_t fd, short what, void *arg) { evutil_socket_t *fdp = arg; if (*fdp >= 0) { evutil_closesocket(*fdp); *fdp = -1; } } static void test_bufferevent_connect_fail(void *arg) { struct basic_test_data *data = (struct basic_test_data *)arg; struct bufferevent *bev=NULL; struct event close_listener_event; int close_listener_event_added = 0; struct timeval one_second = { 1, 0 }; struct sockaddr_in localhost; ev_socklen_t slen = sizeof(localhost); evutil_socket_t fake_listener = -1; int r; test_ok = 0; fake_listener = fake_listener_create(&localhost); bev = bufferevent_socket_new(data->base, -1, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS); tt_assert(bev); bufferevent_setcb(bev, NULL, NULL, want_fail_eventcb, data->base); r = bufferevent_socket_connect(bev, (struct sockaddr *)&localhost, slen); /* XXXX we'd like to test the '0' case everywhere, but FreeBSD tells * detects the error immediately, which is not really wrong of it. */ tt_want(r == 0 || r == -1); /* Close the listener socket after a second. This should trigger "connection refused" on some other platforms, including OSX. */ evtimer_assign(&close_listener_event, data->base, close_socket_cb, &fake_listener); event_add(&close_listener_event, &one_second); close_listener_event_added = 1; event_base_dispatch(data->base); tt_int_op(test_ok, ==, 1); end: if (fake_listener >= 0) evutil_closesocket(fake_listener); if (bev) bufferevent_free(bev); if (close_listener_event_added) event_del(&close_listener_event); } struct timeout_cb_result { struct timeval read_timeout_at; struct timeval write_timeout_at; struct timeval last_wrote_at; int n_read_timeouts; int n_write_timeouts; int total_calls; }; static void bev_timeout_write_cb(struct bufferevent *bev, void *arg) { struct timeout_cb_result *res = arg; evutil_gettimeofday(&res->last_wrote_at, NULL); } static void bev_timeout_event_cb(struct bufferevent *bev, short what, void *arg) { struct timeout_cb_result *res = arg; ++res->total_calls; if ((what & (BEV_EVENT_READING|BEV_EVENT_TIMEOUT)) == (BEV_EVENT_READING|BEV_EVENT_TIMEOUT)) { evutil_gettimeofday(&res->read_timeout_at, NULL); ++res->n_read_timeouts; } if ((what & (BEV_EVENT_WRITING|BEV_EVENT_TIMEOUT)) == (BEV_EVENT_WRITING|BEV_EVENT_TIMEOUT)) { evutil_gettimeofday(&res->write_timeout_at, NULL); ++res->n_write_timeouts; } } static void test_bufferevent_timeouts(void *arg) { /* "arg" is a string containing "pair" and/or "filter". */ struct bufferevent *bev1 = NULL, *bev2 = NULL; struct basic_test_data *data = arg; int use_pair = 0, use_filter = 0; struct timeval tv_w, tv_r, started_at; struct timeout_cb_result res1, res2; char buf[1024]; memset(&res1, 0, sizeof(res1)); memset(&res2, 0, sizeof(res2)); if (strstr((char*)data->setup_data, "pair")) use_pair = 1; if (strstr((char*)data->setup_data, "filter")) use_filter = 1; if (use_pair) { struct bufferevent *p[2]; tt_int_op(0, ==, bufferevent_pair_new(data->base, 0, p)); bev1 = p[0]; bev2 = p[1]; } else { bev1 = bufferevent_socket_new(data->base, data->pair[0], 0); bev2 = bufferevent_socket_new(data->base, data->pair[1], 0); } tt_assert(bev1); tt_assert(bev2); if (use_filter) { struct bufferevent *bevf1, *bevf2; bevf1 = bufferevent_filter_new(bev1, NULL, NULL, BEV_OPT_CLOSE_ON_FREE, NULL, NULL); bevf2 = bufferevent_filter_new(bev2, NULL, NULL, BEV_OPT_CLOSE_ON_FREE, NULL, NULL); tt_assert(bevf1); tt_assert(bevf2); bev1 = bevf1; bev2 = bevf2; } /* Do this nice and early. */ bufferevent_disable(bev2, EV_READ); /* bev1 will try to write and read. Both will time out. */ evutil_gettimeofday(&started_at, NULL); tv_w.tv_sec = tv_r.tv_sec = 0; tv_w.tv_usec = 100*1000; tv_r.tv_usec = 150*1000; bufferevent_setcb(bev1, NULL, bev_timeout_write_cb, bev_timeout_event_cb, &res1); bufferevent_setwatermark(bev1, EV_WRITE, 1024*1024+10, 0); bufferevent_set_timeouts(bev1, &tv_r, &tv_w); if (use_pair) { /* For a pair, the fact that the other side isn't reading * makes the writer stall */ bufferevent_write(bev1, "ABCDEFG", 7); } else { /* For a real socket, the kernel's TCP buffers can eat a * fair number of bytes; make sure that at some point we * have some bytes that will stall. */ struct evbuffer *output = bufferevent_get_output(bev1); int i; memset(buf, 0xbb, sizeof(buf)); for (i=0;i<1024;++i) { evbuffer_add_reference(output, buf, sizeof(buf), NULL, NULL); } } bufferevent_enable(bev1, EV_READ|EV_WRITE); /* bev2 has nothing to say, and isn't listening. */ bufferevent_setcb(bev2, NULL, bev_timeout_write_cb, bev_timeout_event_cb, &res2); tv_w.tv_sec = tv_r.tv_sec = 0; tv_w.tv_usec = 200*1000; tv_r.tv_usec = 100*1000; bufferevent_set_timeouts(bev2, &tv_r, &tv_w); bufferevent_enable(bev2, EV_WRITE); tv_r.tv_sec = 0; tv_r.tv_usec = 350000; event_base_loopexit(data->base, &tv_r); event_base_dispatch(data->base); /* XXXX Test that actually reading or writing a little resets the * timeouts. */ /* Each buf1 timeout happens, and happens only once. */ tt_want(res1.n_read_timeouts); tt_want(res1.n_write_timeouts); tt_want(res1.n_read_timeouts == 1); tt_want(res1.n_write_timeouts == 1); test_timeval_diff_eq(&started_at, &res1.read_timeout_at, 150); test_timeval_diff_eq(&started_at, &res1.write_timeout_at, 100); end: if (bev1) bufferevent_free(bev1); if (bev2) bufferevent_free(bev2); } static void trigger_failure_cb(evutil_socket_t fd, short what, void *ctx) { TT_FAIL(("The triggered callback did not fire or the machine is really slow (try increasing timeout).")); } static void trigger_eventcb(struct bufferevent *bev, short what, void *ctx) { struct event_base *base = ctx; if (what == ~0) { TT_BLATHER(("Event successfully triggered.")); event_base_loopexit(base, NULL); return; } reader_eventcb(bev, what, ctx); } static void trigger_readcb_triggered(struct bufferevent *bev, void *ctx) { TT_BLATHER(("Read successfully triggered.")); n_reads_invoked++; bufferevent_trigger_event(bev, ~0, bufferevent_trigger_test_flags); } static void trigger_readcb(struct bufferevent *bev, void *ctx) { struct timeval timeout = { 30, 0 }; struct event_base *base = ctx; size_t low, high, len; int expected_reads; TT_BLATHER(("Read invoked on %d.", (int)bufferevent_getfd(bev))); expected_reads = ++n_reads_invoked; bufferevent_setcb(bev, trigger_readcb_triggered, NULL, trigger_eventcb, ctx); bufferevent_getwatermark(bev, EV_READ, &low, &high); len = evbuffer_get_length(bufferevent_get_input(bev)); bufferevent_setwatermark(bev, EV_READ, len + 1, 0); bufferevent_trigger(bev, EV_READ, bufferevent_trigger_test_flags); /* no callback expected */ tt_int_op(n_reads_invoked, ==, expected_reads); if ((bufferevent_trigger_test_flags & BEV_TRIG_DEFER_CALLBACKS) || (bufferevent_connect_test_flags & BEV_OPT_DEFER_CALLBACKS)) { /* will be deferred */ } else { expected_reads++; } event_base_once(base, -1, EV_TIMEOUT, trigger_failure_cb, NULL, &timeout); bufferevent_trigger(bev, EV_READ, bufferevent_trigger_test_flags | BEV_TRIG_IGNORE_WATERMARKS); tt_int_op(n_reads_invoked, ==, expected_reads); bufferevent_setwatermark(bev, EV_READ, low, high); end: ; } static void test_bufferevent_trigger(void *arg) { struct basic_test_data *data = arg; struct evconnlistener *lev=NULL; struct bufferevent *bev=NULL; struct sockaddr_in localhost; struct sockaddr_storage ss; struct sockaddr *sa; ev_socklen_t slen; int be_flags=BEV_OPT_CLOSE_ON_FREE; int trig_flags=0; if (strstr((char*)data->setup_data, "defer")) { be_flags |= BEV_OPT_DEFER_CALLBACKS; } bufferevent_connect_test_flags = be_flags; if (strstr((char*)data->setup_data, "postpone")) { trig_flags |= BEV_TRIG_DEFER_CALLBACKS; } bufferevent_trigger_test_flags = trig_flags; memset(&localhost, 0, sizeof(localhost)); localhost.sin_port = 0; /* pick-a-port */ localhost.sin_addr.s_addr = htonl(0x7f000001L); localhost.sin_family = AF_INET; sa = (struct sockaddr *)&localhost; lev = evconnlistener_new_bind(data->base, listen_cb, data->base, LEV_OPT_CLOSE_ON_FREE|LEV_OPT_REUSEABLE, 16, sa, sizeof(localhost)); tt_assert(lev); sa = (struct sockaddr *)&ss; slen = sizeof(ss); if (regress_get_listener_addr(lev, sa, &slen) < 0) { tt_abort_perror("getsockname"); } tt_assert(!evconnlistener_enable(lev)); bev = bufferevent_socket_new(data->base, -1, be_flags); tt_assert(bev); bufferevent_setcb(bev, trigger_readcb, NULL, trigger_eventcb, data->base); bufferevent_enable(bev, EV_READ); tt_want(!bufferevent_socket_connect(bev, sa, sizeof(localhost))); event_base_dispatch(data->base); tt_int_op(n_reads_invoked, ==, 2); end: if (lev) evconnlistener_free(lev); if (bev) bufferevent_free(bev); } static void test_bufferevent_socket_filter_inactive(void *arg) { struct basic_test_data *data = arg; struct bufferevent *bev = NULL, *bevf = NULL; bev = bufferevent_socket_new(data->base, -1, 0); tt_assert(bev); bevf = bufferevent_filter_new(bev, NULL, NULL, 0, NULL, NULL); tt_assert(bevf); end: if (bevf) bufferevent_free(bevf); if (bev) bufferevent_free(bev); } static void pair_flush_eventcb(struct bufferevent *bev, short what, void *ctx) { int *callback_what = ctx; *callback_what = what; } static void test_bufferevent_pair_flush(void *arg) { struct basic_test_data *data = arg; struct bufferevent *pair[2]; struct bufferevent *bev1 = NULL; struct bufferevent *bev2 = NULL; int callback_what = 0; tt_assert(0 == bufferevent_pair_new(data->base, 0, pair)); bev1 = pair[0]; bev2 = pair[1]; tt_assert(0 == bufferevent_enable(bev1, EV_WRITE)); tt_assert(0 == bufferevent_enable(bev2, EV_READ)); bufferevent_setcb(bev2, NULL, NULL, pair_flush_eventcb, &callback_what); bufferevent_flush(bev1, EV_WRITE, BEV_FINISHED); event_base_loop(data->base, EVLOOP_ONCE); tt_assert(callback_what == (BEV_EVENT_READING | BEV_EVENT_EOF)); end: if (bev1) bufferevent_free(bev1); if (bev2) bufferevent_free(bev2); } struct bufferevent_filter_data_stuck { size_t header_size; size_t total_read; }; static void bufferevent_filter_data_stuck_readcb(struct bufferevent *bev, void *arg) { struct bufferevent_filter_data_stuck *filter_data = arg; struct evbuffer *input = bufferevent_get_input(bev); size_t read_size = evbuffer_get_length(input); evbuffer_drain(input, read_size); filter_data->total_read += read_size; } /** * This filter prepends header once before forwarding data. */ static enum bufferevent_filter_result bufferevent_filter_data_stuck_inputcb( struct evbuffer *src, struct evbuffer *dst, ev_ssize_t dst_limit, enum bufferevent_flush_mode mode, void *ctx) { struct bufferevent_filter_data_stuck *filter_data = ctx; static int header_inserted = 0; size_t payload_size; size_t header_size = 0; if (!header_inserted) { char *header = calloc(filter_data->header_size, 1); evbuffer_add(dst, header, filter_data->header_size); free(header); header_size = filter_data->header_size; header_inserted = 1; } payload_size = evbuffer_get_length(src); if (payload_size > dst_limit - header_size) { payload_size = dst_limit - header_size; } tt_int_op(payload_size, ==, evbuffer_remove_buffer(src, dst, payload_size)); end: return BEV_OK; } static void test_bufferevent_filter_data_stuck(void *arg) { const size_t read_high_wm = 4096; struct bufferevent_filter_data_stuck filter_data; struct basic_test_data *data = arg; struct bufferevent *pair[2]; struct bufferevent *filter = NULL; int options = BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS; char payload[4096]; int payload_size = sizeof(payload); memset(&filter_data, 0, sizeof(filter_data)); filter_data.header_size = 20; tt_assert(bufferevent_pair_new(data->base, options, pair) == 0); bufferevent_setwatermark(pair[0], EV_READ, 0, read_high_wm); bufferevent_setwatermark(pair[1], EV_READ, 0, read_high_wm); tt_assert( filter = bufferevent_filter_new(pair[1], bufferevent_filter_data_stuck_inputcb, NULL, options, NULL, &filter_data)); bufferevent_setcb(filter, bufferevent_filter_data_stuck_readcb, NULL, NULL, &filter_data); tt_assert(bufferevent_enable(filter, EV_READ|EV_WRITE) == 0); bufferevent_setwatermark(filter, EV_READ, 0, read_high_wm); tt_assert(bufferevent_write(pair[0], payload, sizeof(payload)) == 0); event_base_dispatch(data->base); tt_int_op(filter_data.total_read, ==, payload_size + filter_data.header_size); end: if (pair[0]) bufferevent_free(pair[0]); if (filter) bufferevent_free(filter); } struct testcase_t bufferevent_testcases[] = { LEGACY(bufferevent, TT_ISOLATED), LEGACY(bufferevent_pair, TT_ISOLATED), LEGACY(bufferevent_flush_normal, TT_ISOLATED), LEGACY(bufferevent_flush_flush, TT_ISOLATED), LEGACY(bufferevent_flush_finished, TT_ISOLATED), LEGACY(bufferevent_pair_flush_normal, TT_ISOLATED), LEGACY(bufferevent_pair_flush_flush, TT_ISOLATED), LEGACY(bufferevent_pair_flush_finished, TT_ISOLATED), #if defined(EVTHREAD_USE_PTHREADS_IMPLEMENTED) { "bufferevent_pair_release_lock", test_bufferevent_pair_release_lock, TT_FORK|TT_ISOLATED|TT_NEED_THREADS|TT_NEED_BASE|TT_LEGACY, &basic_setup, NULL }, #endif LEGACY(bufferevent_watermarks, TT_ISOLATED), LEGACY(bufferevent_pair_watermarks, TT_ISOLATED), LEGACY(bufferevent_filters, TT_ISOLATED), LEGACY(bufferevent_pair_filters, TT_ISOLATED), LEGACY(bufferevent_filters_disable, TT_ISOLATED), LEGACY(bufferevent_pair_filters_disable, TT_ISOLATED), { "bufferevent_connect", test_bufferevent_connect, TT_FORK|TT_NEED_BASE, &basic_setup, (void*)"" }, { "bufferevent_connect_defer", test_bufferevent_connect, TT_FORK|TT_NEED_BASE, &basic_setup, (void*)"defer" }, { "bufferevent_connect_lock", test_bufferevent_connect, TT_FORK|TT_NEED_BASE|TT_NEED_THREADS, &basic_setup, (void*)"lock" }, { "bufferevent_connect_lock_defer", test_bufferevent_connect, TT_FORK|TT_NEED_BASE|TT_NEED_THREADS, &basic_setup, (void*)"defer lock" }, { "bufferevent_connect_unlocked_cbs", test_bufferevent_connect, TT_FORK|TT_NEED_BASE|TT_NEED_THREADS, &basic_setup, (void*)"lock defer unlocked" }, { "bufferevent_connect_fail", test_bufferevent_connect_fail, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "bufferevent_timeout", test_bufferevent_timeouts, TT_FORK|TT_NEED_BASE|TT_NEED_SOCKETPAIR, &basic_setup, (void*)"" }, { "bufferevent_timeout_pair", test_bufferevent_timeouts, TT_FORK|TT_NEED_BASE, &basic_setup, (void*)"pair" }, { "bufferevent_timeout_filter", test_bufferevent_timeouts, TT_FORK|TT_NEED_BASE, &basic_setup, (void*)"filter" }, { "bufferevent_timeout_filter_pair", test_bufferevent_timeouts, TT_FORK|TT_NEED_BASE, &basic_setup, (void*)"filter pair" }, { "bufferevent_trigger", test_bufferevent_trigger, TT_FORK|TT_NEED_BASE, &basic_setup, (void*)"" }, { "bufferevent_trigger_defer", test_bufferevent_trigger, TT_FORK|TT_NEED_BASE, &basic_setup, (void*)"defer" }, { "bufferevent_trigger_postpone", test_bufferevent_trigger, TT_FORK|TT_NEED_BASE|TT_NEED_THREADS, &basic_setup, (void*)"postpone" }, { "bufferevent_trigger_defer_postpone", test_bufferevent_trigger, TT_FORK|TT_NEED_BASE|TT_NEED_THREADS, &basic_setup, (void*)"defer postpone" }, #ifdef EVENT__HAVE_LIBZ LEGACY(bufferevent_zlib, TT_ISOLATED), #else { "bufferevent_zlib", NULL, TT_SKIP, NULL, NULL }, #endif { "bufferevent_connect_fail_eventcb_defer", test_bufferevent_connect_fail_eventcb, TT_FORK|TT_NEED_BASE, &basic_setup, (void*)BEV_OPT_DEFER_CALLBACKS }, { "bufferevent_connect_fail_eventcb", test_bufferevent_connect_fail_eventcb, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "bufferevent_socket_filter_inactive", test_bufferevent_socket_filter_inactive, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "bufferevent_pair_flush", test_bufferevent_pair_flush, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "bufferevent_filter_data_stuck", test_bufferevent_filter_data_stuck, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, END_OF_TESTCASES, }; struct testcase_t bufferevent_iocp_testcases[] = { LEGACY(bufferevent, TT_ISOLATED|TT_ENABLE_IOCP), LEGACY(bufferevent_flush_normal, TT_ISOLATED), LEGACY(bufferevent_flush_flush, TT_ISOLATED), LEGACY(bufferevent_flush_finished, TT_ISOLATED), LEGACY(bufferevent_watermarks, TT_ISOLATED|TT_ENABLE_IOCP), LEGACY(bufferevent_filters, TT_ISOLATED|TT_ENABLE_IOCP), LEGACY(bufferevent_filters_disable, TT_ISOLATED|TT_ENABLE_IOCP), { "bufferevent_connect", test_bufferevent_connect, TT_FORK|TT_NEED_BASE|TT_ENABLE_IOCP, &basic_setup, (void*)"" }, { "bufferevent_connect_defer", test_bufferevent_connect, TT_FORK|TT_NEED_BASE|TT_ENABLE_IOCP, &basic_setup, (void*)"defer" }, { "bufferevent_connect_lock", test_bufferevent_connect, TT_FORK|TT_NEED_BASE|TT_NEED_THREADS|TT_ENABLE_IOCP, &basic_setup, (void*)"lock" }, { "bufferevent_connect_lock_defer", test_bufferevent_connect, TT_FORK|TT_NEED_BASE|TT_NEED_THREADS|TT_ENABLE_IOCP, &basic_setup, (void*)"defer lock" }, { "bufferevent_connect_fail", test_bufferevent_connect_fail, TT_FORK|TT_NEED_BASE|TT_ENABLE_IOCP, &basic_setup, NULL }, { "bufferevent_connect_nonblocking", test_bufferevent_connect, TT_FORK|TT_NEED_BASE|TT_ENABLE_IOCP, &basic_setup, (void*)"unset_connectex" }, { "bufferevent_connect_fail_eventcb_defer", test_bufferevent_connect_fail_eventcb, TT_FORK|TT_NEED_BASE|TT_ENABLE_IOCP, &basic_setup, (void*)BEV_OPT_DEFER_CALLBACKS }, { "bufferevent_connect_fail_eventcb", test_bufferevent_connect_fail_eventcb, TT_FORK|TT_NEED_BASE|TT_ENABLE_IOCP, &basic_setup, NULL }, END_OF_TESTCASES, }; kytensor-2.49.0.7/third_party/libevent/test/regress_dns.c000066400000000000000000001720751527052371200234530ustar00rootroot00000000000000/* * Copyright (c) 2003-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "../util-internal.h" #ifdef _WIN32 #include #include #include #endif #include "event2/event-config.h" #include #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #ifndef _WIN32 #include #include #include #include #include #endif #ifdef EVENT__HAVE_NETINET_IN6_H #include #endif #ifdef HAVE_NETDB_H #include #endif #include #include #include #include #include #include "event2/dns.h" #include "event2/dns_compat.h" #include "event2/dns_struct.h" #include "event2/event.h" #include "event2/event_compat.h" #include "event2/event_struct.h" #include "event2/util.h" #include "event2/listener.h" #include "event2/bufferevent.h" #include "log-internal.h" #include "regress.h" #include "regress_testutils.h" #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) static int dns_ok = 0; static int dns_got_cancel = 0; static int dns_err = 0; static void dns_gethostbyname_cb(int result, char type, int count, int ttl, void *addresses, void *arg) { dns_ok = dns_err = 0; if (result == DNS_ERR_TIMEOUT) { printf("[Timed out] "); dns_err = result; goto out; } if (result != DNS_ERR_NONE) { printf("[Error code %d] ", result); goto out; } TT_BLATHER(("type: %d, count: %d, ttl: %d: ", type, count, ttl)); switch (type) { case DNS_IPv6_AAAA: { #if defined(EVENT__HAVE_STRUCT_IN6_ADDR) && defined(EVENT__HAVE_INET_NTOP) && defined(INET6_ADDRSTRLEN) struct in6_addr *in6_addrs = addresses; char buf[INET6_ADDRSTRLEN+1]; int i; /* a resolution that's not valid does not help */ if (ttl < 0) goto out; for (i = 0; i < count; ++i) { const char *b = evutil_inet_ntop(AF_INET6, &in6_addrs[i], buf,sizeof(buf)); if (b) TT_BLATHER(("%s ", b)); else TT_BLATHER(("%s ", strerror(errno))); } #endif break; } case DNS_IPv4_A: { struct in_addr *in_addrs = addresses; int i; /* a resolution that's not valid does not help */ if (ttl < 0) goto out; for (i = 0; i < count; ++i) TT_BLATHER(("%s ", inet_ntoa(in_addrs[i]))); break; } case DNS_PTR: /* may get at most one PTR */ if (count != 1) goto out; TT_BLATHER(("%s ", *(char **)addresses)); break; default: goto out; } dns_ok = type; out: if (arg == NULL) event_loopexit(NULL); else event_base_loopexit((struct event_base *)arg, NULL); } static void dns_gethostbyname(void) { dns_ok = 0; evdns_resolve_ipv4("www.monkey.org", 0, dns_gethostbyname_cb, NULL); event_dispatch(); tt_int_op(dns_ok, ==, DNS_IPv4_A); test_ok = dns_ok; end: ; } static void dns_gethostbyname6(void) { dns_ok = 0; evdns_resolve_ipv6("www.ietf.org", 0, dns_gethostbyname_cb, NULL); event_dispatch(); if (!dns_ok && dns_err == DNS_ERR_TIMEOUT) { tt_skip(); } tt_int_op(dns_ok, ==, DNS_IPv6_AAAA); test_ok = 1; end: ; } static void dns_gethostbyaddr(void) { struct in_addr in; in.s_addr = htonl(0x7f000001ul); /* 127.0.0.1 */ dns_ok = 0; evdns_resolve_reverse(&in, 0, dns_gethostbyname_cb, NULL); event_dispatch(); tt_int_op(dns_ok, ==, DNS_PTR); test_ok = dns_ok; end: ; } static void dns_resolve_reverse(void *ptr) { struct in_addr in; struct event_base *base = event_base_new(); struct evdns_base *dns = evdns_base_new(base, 1/* init name servers */); struct evdns_request *req = NULL; tt_assert(base); tt_assert(dns); in.s_addr = htonl(0x7f000001ul); /* 127.0.0.1 */ dns_ok = 0; req = evdns_base_resolve_reverse( dns, &in, 0, dns_gethostbyname_cb, base); tt_assert(req); event_base_dispatch(base); tt_int_op(dns_ok, ==, DNS_PTR); end: if (dns) evdns_base_free(dns, 0); if (base) event_base_free(base); } static int n_server_responses = 0; static void dns_server_request_cb(struct evdns_server_request *req, void *data) { int i, r; const char TEST_ARPA[] = "11.11.168.192.in-addr.arpa"; const char TEST_IN6[] = "f.e.f.e." "0.0.0.0." "0.0.0.0." "1.1.1.1." "a.a.a.a." "0.0.0.0." "0.0.0.0." "0.f.f.f.ip6.arpa"; for (i = 0; i < req->nquestions; ++i) { const int qtype = req->questions[i]->type; const int qclass = req->questions[i]->dns_question_class; const char *qname = req->questions[i]->name; struct in_addr ans; ans.s_addr = htonl(0xc0a80b0bUL); /* 192.168.11.11 */ if (qtype == EVDNS_TYPE_A && qclass == EVDNS_CLASS_INET && !evutil_ascii_strcasecmp(qname, "zz.example.com")) { r = evdns_server_request_add_a_reply(req, qname, 1, &ans.s_addr, 12345); if (r<0) dns_ok = 0; } else if (qtype == EVDNS_TYPE_AAAA && qclass == EVDNS_CLASS_INET && !evutil_ascii_strcasecmp(qname, "zz.example.com")) { char addr6[17] = "abcdefghijklmnop"; r = evdns_server_request_add_aaaa_reply(req, qname, 1, addr6, 123); if (r<0) dns_ok = 0; } else if (qtype == EVDNS_TYPE_PTR && qclass == EVDNS_CLASS_INET && !evutil_ascii_strcasecmp(qname, TEST_ARPA)) { r = evdns_server_request_add_ptr_reply(req, NULL, qname, "ZZ.EXAMPLE.COM", 54321); if (r<0) dns_ok = 0; } else if (qtype == EVDNS_TYPE_PTR && qclass == EVDNS_CLASS_INET && !evutil_ascii_strcasecmp(qname, TEST_IN6)){ r = evdns_server_request_add_ptr_reply(req, NULL, qname, "ZZ-INET6.EXAMPLE.COM", 54322); if (r<0) dns_ok = 0; } else if (qtype == EVDNS_TYPE_A && qclass == EVDNS_CLASS_INET && !evutil_ascii_strcasecmp(qname, "drop.example.com")) { if (evdns_server_request_drop(req)<0) dns_ok = 0; return; } else { printf("Unexpected question %d %d \"%s\" ", qtype, qclass, qname); dns_ok = 0; } } r = evdns_server_request_respond(req, 0); if (r<0) { printf("Couldn't send reply. "); dns_ok = 0; } } static void dns_server_gethostbyname_cb(int result, char type, int count, int ttl, void *addresses, void *arg) { if (result == DNS_ERR_CANCEL) { if (arg != (void*)(char*)90909) { printf("Unexpected cancelation"); dns_ok = 0; } dns_got_cancel = 1; goto out; } if (result != DNS_ERR_NONE) { printf("Unexpected result %d. ", result); dns_ok = 0; goto out; } if (count != 1) { printf("Unexpected answer count %d. ", count); dns_ok = 0; goto out; } switch (type) { case DNS_IPv4_A: { struct in_addr *in_addrs = addresses; if (in_addrs[0].s_addr != htonl(0xc0a80b0bUL) || ttl != 12345) { printf("Bad IPv4 response \"%s\" %d. ", inet_ntoa(in_addrs[0]), ttl); dns_ok = 0; goto out; } break; } case DNS_IPv6_AAAA: { #if defined (EVENT__HAVE_STRUCT_IN6_ADDR) && defined(EVENT__HAVE_INET_NTOP) && defined(INET6_ADDRSTRLEN) struct in6_addr *in6_addrs = addresses; char buf[INET6_ADDRSTRLEN+1]; if (memcmp(&in6_addrs[0].s6_addr, "abcdefghijklmnop", 16) || ttl != 123) { const char *b = evutil_inet_ntop(AF_INET6, &in6_addrs[0],buf,sizeof(buf)); printf("Bad IPv6 response \"%s\" %d. ", b, ttl); dns_ok = 0; goto out; } #endif break; } case DNS_PTR: { char **addrs = addresses; if (arg != (void*)6) { if (strcmp(addrs[0], "ZZ.EXAMPLE.COM") || ttl != 54321) { printf("Bad PTR response \"%s\" %d. ", addrs[0], ttl); dns_ok = 0; goto out; } } else { if (strcmp(addrs[0], "ZZ-INET6.EXAMPLE.COM") || ttl != 54322) { printf("Bad ipv6 PTR response \"%s\" %d. ", addrs[0], ttl); dns_ok = 0; goto out; } } break; } default: printf("Bad response type %d. ", type); dns_ok = 0; } out: if (++n_server_responses == 3) { event_loopexit(NULL); } } static void dns_server(void) { evutil_socket_t sock=-1; struct sockaddr_in my_addr; struct sockaddr_storage ss; ev_socklen_t slen; struct evdns_server_port *port=NULL; struct in_addr resolve_addr; struct in6_addr resolve_addr6; struct evdns_base *base=NULL; struct evdns_request *req=NULL; dns_ok = 1; base = evdns_base_new(NULL, 0); /* Now configure a nameserver port. */ sock = socket(AF_INET, SOCK_DGRAM, 0); if (sock<0) { tt_abort_perror("socket"); } evutil_make_socket_nonblocking(sock); memset(&my_addr, 0, sizeof(my_addr)); my_addr.sin_family = AF_INET; my_addr.sin_port = 0; /* kernel picks */ my_addr.sin_addr.s_addr = htonl(0x7f000001UL); if (bind(sock, (struct sockaddr*)&my_addr, sizeof(my_addr)) < 0) { tt_abort_perror("bind"); } slen = sizeof(ss); if (getsockname(sock, (struct sockaddr*)&ss, &slen) < 0) { tt_abort_perror("getsockname"); } port = evdns_add_server_port(sock, 0, dns_server_request_cb, NULL); /* Add ourself as the only nameserver, and make sure we really are * the only nameserver. */ evdns_base_nameserver_sockaddr_add(base, (struct sockaddr*)&ss, slen, 0); tt_int_op(evdns_base_count_nameservers(base), ==, 1); { struct sockaddr_storage ss2; int slen2; memset(&ss2, 0, sizeof(ss2)); slen2 = evdns_base_get_nameserver_addr(base, 0, (struct sockaddr *)&ss2, 3); tt_int_op(slen2, ==, slen); tt_int_op(ss2.ss_family, ==, 0); slen2 = evdns_base_get_nameserver_addr(base, 0, (struct sockaddr *)&ss2, sizeof(ss2)); tt_int_op(slen2, ==, slen); tt_mem_op(&ss2, ==, &ss, slen); slen2 = evdns_base_get_nameserver_addr(base, 1, (struct sockaddr *)&ss2, sizeof(ss2)); tt_int_op(-1, ==, slen2); } /* Send some queries. */ evdns_base_resolve_ipv4(base, "zz.example.com", DNS_QUERY_NO_SEARCH, dns_server_gethostbyname_cb, NULL); evdns_base_resolve_ipv6(base, "zz.example.com", DNS_QUERY_NO_SEARCH, dns_server_gethostbyname_cb, NULL); resolve_addr.s_addr = htonl(0xc0a80b0bUL); /* 192.168.11.11 */ evdns_base_resolve_reverse(base, &resolve_addr, 0, dns_server_gethostbyname_cb, NULL); memcpy(resolve_addr6.s6_addr, "\xff\xf0\x00\x00\x00\x00\xaa\xaa" "\x11\x11\x00\x00\x00\x00\xef\xef", 16); evdns_base_resolve_reverse_ipv6(base, &resolve_addr6, 0, dns_server_gethostbyname_cb, (void*)6); req = evdns_base_resolve_ipv4(base, "drop.example.com", DNS_QUERY_NO_SEARCH, dns_server_gethostbyname_cb, (void*)(char*)90909); evdns_cancel_request(base, req); event_dispatch(); tt_assert(dns_got_cancel); test_ok = dns_ok; end: if (port) evdns_close_server_port(port); if (sock >= 0) evutil_closesocket(sock); if (base) evdns_base_free(base, 0); } static int n_replies_left; static struct event_base *exit_base; static struct evdns_server_port *exit_port; struct generic_dns_callback_result { int result; char type; int count; int ttl; size_t addrs_len; void *addrs; char addrs_buf[256]; }; static void generic_dns_callback(int result, char type, int count, int ttl, void *addresses, void *arg) { size_t len; struct generic_dns_callback_result *res = arg; res->result = result; res->type = type; res->count = count; res->ttl = ttl; if (type == DNS_IPv4_A) len = count * 4; else if (type == DNS_IPv6_AAAA) len = count * 16; else if (type == DNS_PTR) len = strlen(addresses)+1; else { res->addrs_len = len = 0; res->addrs = NULL; } if (len) { res->addrs_len = len; if (len > 256) len = 256; memcpy(res->addrs_buf, addresses, len); res->addrs = res->addrs_buf; } --n_replies_left; if (n_replies_left == 0) { if (exit_port) { evdns_close_server_port(exit_port); exit_port = NULL; } else event_base_loopexit(exit_base, NULL); } } static struct regress_dns_server_table search_table[] = { { "host.a.example.com", "err", "3", 0, 0 }, { "host.b.example.com", "err", "3", 0, 0 }, { "host.c.example.com", "A", "11.22.33.44", 0, 0 }, { "host2.a.example.com", "err", "3", 0, 0 }, { "host2.b.example.com", "A", "200.100.0.100", 0, 0 }, { "host2.c.example.com", "err", "3", 0, 0 }, { "hostn.a.example.com", "errsoa", "0", 0, 0 }, { "hostn.b.example.com", "errsoa", "3", 0, 0 }, { "hostn.c.example.com", "err", "0", 0, 0 }, { "host", "err", "3", 0, 0 }, { "host2", "err", "3", 0, 0 }, { "*", "err", "3", 0, 0 }, { NULL, NULL, NULL, 0, 0 } }; static void dns_search_test_impl(void *arg, int lower) { struct regress_dns_server_table table[ARRAY_SIZE(search_table)]; struct basic_test_data *data = arg; struct event_base *base = data->base; struct evdns_base *dns = NULL; ev_uint16_t portnum = 0; char buf[64]; struct generic_dns_callback_result r[8]; size_t i; for (i = 0; i < ARRAY_SIZE(table); ++i) { table[i] = search_table[i]; table[i].lower = lower; } tt_assert(regress_dnsserver(base, &portnum, table)); evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", (int)portnum); dns = evdns_base_new(base, 0); tt_assert(!evdns_base_nameserver_ip_add(dns, buf)); evdns_base_search_add(dns, "a.example.com"); evdns_base_search_add(dns, "b.example.com"); evdns_base_search_add(dns, "c.example.com"); n_replies_left = ARRAY_SIZE(r); exit_base = base; evdns_base_resolve_ipv4(dns, "host", 0, generic_dns_callback, &r[0]); evdns_base_resolve_ipv4(dns, "host2", 0, generic_dns_callback, &r[1]); evdns_base_resolve_ipv4(dns, "host", DNS_NO_SEARCH, generic_dns_callback, &r[2]); evdns_base_resolve_ipv4(dns, "host2", DNS_NO_SEARCH, generic_dns_callback, &r[3]); evdns_base_resolve_ipv4(dns, "host3", 0, generic_dns_callback, &r[4]); evdns_base_resolve_ipv4(dns, "hostn.a.example.com", DNS_NO_SEARCH, generic_dns_callback, &r[5]); evdns_base_resolve_ipv4(dns, "hostn.b.example.com", DNS_NO_SEARCH, generic_dns_callback, &r[6]); evdns_base_resolve_ipv4(dns, "hostn.c.example.com", DNS_NO_SEARCH, generic_dns_callback, &r[7]); event_base_dispatch(base); tt_int_op(r[0].type, ==, DNS_IPv4_A); tt_int_op(r[0].count, ==, 1); tt_int_op(((ev_uint32_t*)r[0].addrs)[0], ==, htonl(0x0b16212c)); tt_int_op(r[1].type, ==, DNS_IPv4_A); tt_int_op(r[1].count, ==, 1); tt_int_op(((ev_uint32_t*)r[1].addrs)[0], ==, htonl(0xc8640064)); tt_int_op(r[2].result, ==, DNS_ERR_NOTEXIST); tt_int_op(r[3].result, ==, DNS_ERR_NOTEXIST); tt_int_op(r[4].result, ==, DNS_ERR_NOTEXIST); tt_int_op(r[5].result, ==, DNS_ERR_NODATA); tt_int_op(r[5].ttl, ==, 42); tt_int_op(r[6].result, ==, DNS_ERR_NOTEXIST); tt_int_op(r[6].ttl, ==, 42); tt_int_op(r[7].result, ==, DNS_ERR_NODATA); tt_int_op(r[7].ttl, ==, 0); end: if (dns) evdns_base_free(dns, 0); regress_clean_dnsserver(); } static void dns_search_empty_test(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct evdns_base *dns = NULL; dns = evdns_base_new(base, 0); evdns_base_search_add(dns, "whatever.example.com"); n_replies_left = 1; exit_base = base; tt_ptr_op(evdns_base_resolve_ipv4(dns, "", 0, generic_dns_callback, NULL), ==, NULL); end: if (dns) evdns_base_free(dns, 0); } static void dns_search_test(void *arg) { return dns_search_test_impl(arg, 0); } static void dns_search_lower_test(void *arg) { return dns_search_test_impl(arg, 1); } static int request_count = 0; static struct evdns_request *current_req = NULL; static void search_cancel_server_cb(struct evdns_server_request *req, void *data) { const char *question; if (req->nquestions != 1) TT_DIE(("Only handling one question at a time; got %d", req->nquestions)); question = req->questions[0]->name; TT_BLATHER(("got question, %s", question)); tt_assert(request_count > 0); tt_assert(!evdns_server_request_respond(req, 3)); if (!--request_count) evdns_cancel_request(NULL, current_req); end: ; } static void dns_search_cancel_test(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct evdns_base *dns = NULL; struct evdns_server_port *port = NULL; ev_uint16_t portnum = 0; struct generic_dns_callback_result r1; char buf[64]; port = regress_get_dnsserver(base, &portnum, NULL, search_cancel_server_cb, NULL); tt_assert(port); evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", (int)portnum); dns = evdns_base_new(base, 0); tt_assert(!evdns_base_nameserver_ip_add(dns, buf)); evdns_base_search_add(dns, "a.example.com"); evdns_base_search_add(dns, "b.example.com"); evdns_base_search_add(dns, "c.example.com"); evdns_base_search_add(dns, "d.example.com"); exit_base = base; request_count = 3; n_replies_left = 1; current_req = evdns_base_resolve_ipv4(dns, "host", 0, generic_dns_callback, &r1); event_base_dispatch(base); tt_int_op(r1.result, ==, DNS_ERR_CANCEL); end: if (port) evdns_close_server_port(port); if (dns) evdns_base_free(dns, 0); } static void fail_server_cb(struct evdns_server_request *req, void *data) { const char *question; int *count = data; struct in_addr in; /* Drop the first N requests that we get. */ if (*count > 0) { --*count; tt_want(! evdns_server_request_drop(req)); return; } if (req->nquestions != 1) TT_DIE(("Only handling one question at a time; got %d", req->nquestions)); question = req->questions[0]->name; if (!evutil_ascii_strcasecmp(question, "google.com")) { /* Detect a probe, and get out of the loop. */ event_base_loopexit(exit_base, NULL); } tt_assert(evutil_inet_pton(AF_INET, "16.32.64.128", &in)); evdns_server_request_add_a_reply(req, question, 1, &in.s_addr, 100); tt_assert(! evdns_server_request_respond(req, 0)) return; end: tt_want(! evdns_server_request_drop(req)); } static void dns_retry_test_impl(void *arg, int flags) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct evdns_server_port *port = NULL; struct evdns_base *dns = NULL; int drop_count = 2; ev_uint16_t portnum = 0; char buf[64]; struct generic_dns_callback_result r1; port = regress_get_dnsserver(base, &portnum, NULL, fail_server_cb, &drop_count); tt_assert(port); evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", (int)portnum); dns = evdns_base_new(base, flags); tt_assert(!evdns_base_nameserver_ip_add(dns, buf)); tt_assert(! evdns_base_set_option(dns, "timeout", "0.2")); tt_assert(! evdns_base_set_option(dns, "max-timeouts:", "10")); tt_assert(! evdns_base_set_option(dns, "initial-probe-timeout", "0.1")); evdns_base_resolve_ipv4(dns, "host.example.com", 0, generic_dns_callback, &r1); n_replies_left = 1; exit_base = base; event_base_dispatch(base); tt_int_op(drop_count, ==, 0); tt_int_op(r1.type, ==, DNS_IPv4_A); tt_int_op(r1.count, ==, 1); tt_int_op(((ev_uint32_t*)r1.addrs)[0], ==, htonl(0x10204080)); /* Now try again, but this time have the server get treated as * failed, so we can send it a test probe. */ drop_count = 4; tt_assert(! evdns_base_set_option(dns, "max-timeouts:", "2")); tt_assert(! evdns_base_set_option(dns, "attempts:", "3")); memset(&r1, 0, sizeof(r1)); evdns_base_resolve_ipv4(dns, "host.example.com", 0, generic_dns_callback, &r1); n_replies_left = 2; /* This will run until it answers the "google.com" probe request. */ event_base_dispatch(base); /* We'll treat the server as failed here. */ tt_int_op(r1.result, ==, DNS_ERR_TIMEOUT); /* It should work this time. */ tt_int_op(drop_count, ==, 0); evdns_base_resolve_ipv4(dns, "host.example.com", 0, generic_dns_callback, &r1); event_base_dispatch(base); tt_int_op(r1.result, ==, DNS_ERR_NONE); tt_int_op(r1.type, ==, DNS_IPv4_A); tt_int_op(r1.count, ==, 1); tt_int_op(((ev_uint32_t*)r1.addrs)[0], ==, htonl(0x10204080)); end: if (dns) evdns_base_free(dns, 0); if (port) evdns_close_server_port(port); } static void dns_retry_test(void *arg) { dns_retry_test_impl(arg, 0); } static void dns_retry_disable_when_inactive_test(void *arg) { dns_retry_test_impl(arg, EVDNS_BASE_DISABLE_WHEN_INACTIVE); } static struct regress_dns_server_table internal_error_table[] = { /* Error 4 (NOTIMPL) makes us reissue the request to another server if we can. XXXX we should reissue under a much wider set of circumstances! */ { "foof.example.com", "err", "4", 0, 0 }, { NULL, NULL, NULL, 0, 0 } }; static struct regress_dns_server_table reissue_table[] = { { "foof.example.com", "A", "240.15.240.15", 0, 0 }, { NULL, NULL, NULL, 0, 0 } }; static void dns_reissue_test_impl(void *arg, int flags) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct evdns_server_port *port1 = NULL, *port2 = NULL; struct evdns_base *dns = NULL; struct generic_dns_callback_result r1; ev_uint16_t portnum1 = 0, portnum2=0; char buf1[64], buf2[64]; port1 = regress_get_dnsserver(base, &portnum1, NULL, regress_dns_server_cb, internal_error_table); tt_assert(port1); port2 = regress_get_dnsserver(base, &portnum2, NULL, regress_dns_server_cb, reissue_table); tt_assert(port2); evutil_snprintf(buf1, sizeof(buf1), "127.0.0.1:%d", (int)portnum1); evutil_snprintf(buf2, sizeof(buf2), "127.0.0.1:%d", (int)portnum2); dns = evdns_base_new(base, flags); tt_assert(!evdns_base_nameserver_ip_add(dns, buf1)); tt_assert(! evdns_base_set_option(dns, "timeout:", "0.3")); tt_assert(! evdns_base_set_option(dns, "max-timeouts:", "2")); tt_assert(! evdns_base_set_option(dns, "attempts:", "5")); memset(&r1, 0, sizeof(r1)); evdns_base_resolve_ipv4(dns, "foof.example.com", 0, generic_dns_callback, &r1); /* Add this after, so that we are sure to get a reissue. */ tt_assert(!evdns_base_nameserver_ip_add(dns, buf2)); n_replies_left = 1; exit_base = base; event_base_dispatch(base); tt_int_op(r1.result, ==, DNS_ERR_NONE); tt_int_op(r1.type, ==, DNS_IPv4_A); tt_int_op(r1.count, ==, 1); tt_int_op(((ev_uint32_t*)r1.addrs)[0], ==, htonl(0xf00ff00f)); /* Make sure we dropped at least once. */ tt_int_op(internal_error_table[0].seen, >, 0); end: if (dns) evdns_base_free(dns, 0); if (port1) evdns_close_server_port(port1); if (port2) evdns_close_server_port(port2); } static void dns_reissue_test(void *arg) { dns_reissue_test_impl(arg, 0); } static void dns_reissue_disable_when_inactive_test(void *arg) { dns_reissue_test_impl(arg, EVDNS_BASE_DISABLE_WHEN_INACTIVE); } #if 0 static void dumb_bytes_fn(char *p, size_t n) { unsigned i; /* This gets us 6 bits of entropy per transaction ID, which means we * will have probably have collisions and need to pick again. */ for (i=0;ibase; struct evdns_base *dns = NULL; struct evdns_server_port *dns_port = NULL; ev_uint16_t portnum = 0; char buf[64]; int disable_when_inactive = flags & EVDNS_BASE_DISABLE_WHEN_INACTIVE; struct generic_dns_callback_result r[20]; int i; dns_port = regress_get_dnsserver(base, &portnum, NULL, regress_dns_server_cb, reissue_table); tt_assert(dns_port); if (disable_when_inactive) { exit_port = dns_port; } evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", (int)portnum); dns = evdns_base_new(base, flags); tt_assert(!evdns_base_nameserver_ip_add(dns, buf)); tt_assert(! evdns_base_set_option(dns, "max-inflight:", "3")); tt_assert(! evdns_base_set_option(dns, "randomize-case:", "0")); for (i=0;i<20;++i) evdns_base_resolve_ipv4(dns, "foof.example.com", 0, generic_dns_callback, &r[i]); n_replies_left = 20; exit_base = base; event_base_dispatch(base); for (i=0;i<20;++i) { tt_int_op(r[i].type, ==, DNS_IPv4_A); tt_int_op(r[i].count, ==, 1); tt_int_op(((ev_uint32_t*)r[i].addrs)[0], ==, htonl(0xf00ff00f)); } end: if (dns) evdns_base_free(dns, 0); if (exit_port) { evdns_close_server_port(exit_port); exit_port = NULL; } else if (! disable_when_inactive) { evdns_close_server_port(dns_port); } } static void dns_inflight_test(void *arg) { dns_inflight_test_impl(arg, 0); } static void dns_disable_when_inactive_test(void *arg) { dns_inflight_test_impl(arg, EVDNS_BASE_DISABLE_WHEN_INACTIVE); } static void dns_disable_when_inactive_no_ns_test(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base, *inactive_base; struct evdns_base *dns = NULL; ev_uint16_t portnum = 0; char buf[64]; struct generic_dns_callback_result r; inactive_base = event_base_new(); tt_assert(inactive_base); /** Create dns server with inactive base, to avoid replying to clients */ tt_assert(regress_dnsserver(inactive_base, &portnum, search_table)); evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", (int)portnum); dns = evdns_base_new(base, EVDNS_BASE_DISABLE_WHEN_INACTIVE); tt_assert(!evdns_base_nameserver_ip_add(dns, buf)); tt_assert(! evdns_base_set_option(dns, "timeout:", "0.1")); evdns_base_resolve_ipv4(dns, "foof.example.com", 0, generic_dns_callback, &r); n_replies_left = 1; exit_base = base; event_base_dispatch(base); tt_int_op(n_replies_left, ==, 0); tt_int_op(r.result, ==, DNS_ERR_TIMEOUT); tt_int_op(r.count, ==, 0); tt_ptr_op(r.addrs, ==, NULL); end: if (dns) evdns_base_free(dns, 0); regress_clean_dnsserver(); if (inactive_base) event_base_free(inactive_base); } /* === Test for bufferevent_socket_connect_hostname */ static int total_connected_or_failed = 0; static int total_n_accepted = 0; static struct event_base *be_connect_hostname_base = NULL; /* Implements a DNS server for the connect_hostname test and the * getaddrinfo_async test */ static void be_getaddrinfo_server_cb(struct evdns_server_request *req, void *data) { int i; int *n_got_p=data; int added_any=0; ++*n_got_p; for (i=0;inquestions;++i) { const int qtype = req->questions[i]->type; const int qclass = req->questions[i]->dns_question_class; const char *qname = req->questions[i]->name; struct in_addr ans; struct in6_addr ans6; memset(&ans6, 0, sizeof(ans6)); TT_BLATHER(("Got question about %s, type=%d", qname, qtype)); if (qtype == EVDNS_TYPE_A && qclass == EVDNS_CLASS_INET && !evutil_ascii_strcasecmp(qname, "nobodaddy.example.com")) { ans.s_addr = htonl(0x7f000001); evdns_server_request_add_a_reply(req, qname, 1, &ans.s_addr, 2000); added_any = 1; } else if (!evutil_ascii_strcasecmp(qname, "nosuchplace.example.com")) { /* ok, just say notfound. */ } else if (!evutil_ascii_strcasecmp(qname, "both.example.com")) { if (qtype == EVDNS_TYPE_A) { ans.s_addr = htonl(0x50502020); evdns_server_request_add_a_reply(req, qname, 1, &ans.s_addr, 2000); added_any = 1; } else if (qtype == EVDNS_TYPE_AAAA) { ans6.s6_addr[0] = 0x80; ans6.s6_addr[1] = 0xff; ans6.s6_addr[14] = 0xbb; ans6.s6_addr[15] = 0xbb; evdns_server_request_add_aaaa_reply(req, qname, 1, &ans6.s6_addr, 2000); added_any = 1; } evdns_server_request_add_cname_reply(req, qname, "both-canonical.example.com", 1000); } else if (!evutil_ascii_strcasecmp(qname, "v4only.example.com") || !evutil_ascii_strcasecmp(qname, "v4assert.example.com")) { if (qtype == EVDNS_TYPE_A) { ans.s_addr = htonl(0x12345678); evdns_server_request_add_a_reply(req, qname, 1, &ans.s_addr, 2000); added_any = 1; } else if (!evutil_ascii_strcasecmp(qname, "v4assert.example.com")) { TT_FAIL(("Got an AAAA request for v4assert")); } } else if (!evutil_ascii_strcasecmp(qname, "v6only.example.com") || !evutil_ascii_strcasecmp(qname, "v6assert.example.com")) { if (qtype == EVDNS_TYPE_AAAA) { ans6.s6_addr[0] = 0x0b; ans6.s6_addr[1] = 0x0b; ans6.s6_addr[14] = 0xf0; ans6.s6_addr[15] = 0x0d; evdns_server_request_add_aaaa_reply(req, qname, 1, &ans6.s6_addr, 2000); added_any = 1; } else if (!evutil_ascii_strcasecmp(qname, "v6assert.example.com")) { TT_FAIL(("Got a A request for v6assert")); } } else if (!evutil_ascii_strcasecmp(qname, "v6timeout.example.com")) { if (qtype == EVDNS_TYPE_A) { ans.s_addr = htonl(0xabcdef01); evdns_server_request_add_a_reply(req, qname, 1, &ans.s_addr, 2000); added_any = 1; } else if (qtype == EVDNS_TYPE_AAAA) { /* Let the v6 request time out.*/ evdns_server_request_drop(req); return; } } else if (!evutil_ascii_strcasecmp(qname, "v4timeout.example.com")) { if (qtype == EVDNS_TYPE_AAAA) { ans6.s6_addr[0] = 0x0a; ans6.s6_addr[1] = 0x0a; ans6.s6_addr[14] = 0xff; ans6.s6_addr[15] = 0x01; evdns_server_request_add_aaaa_reply(req, qname, 1, &ans6.s6_addr, 2000); added_any = 1; } else if (qtype == EVDNS_TYPE_A) { /* Let the v4 request time out.*/ evdns_server_request_drop(req); return; } } else if (!evutil_ascii_strcasecmp(qname, "v6timeout-nonexist.example.com")) { if (qtype == EVDNS_TYPE_A) { /* Fall through, give an nexist. */ } else if (qtype == EVDNS_TYPE_AAAA) { /* Let the v6 request time out.*/ evdns_server_request_drop(req); return; } } else if (!evutil_ascii_strcasecmp(qname, "all-timeout.example.com")) { /* drop all requests */ evdns_server_request_drop(req); return; } else { TT_GRIPE(("Got weird request for %s",qname)); } } if (added_any) { TT_BLATHER(("answering")); evdns_server_request_respond(req, 0); } else { TT_BLATHER(("saying nexist.")); evdns_server_request_respond(req, 3); } } /* Implements a listener for connect_hostname test. */ static void nil_accept_cb(struct evconnlistener *l, evutil_socket_t fd, struct sockaddr *s, int socklen, void *arg) { int *p = arg; (*p)++; ++total_n_accepted; /* don't do anything with the socket; let it close when we exit() */ if (total_n_accepted >= 3 && total_connected_or_failed >= 5) event_base_loopexit(be_connect_hostname_base, NULL); } struct be_conn_hostname_result { int dnserr; int what; }; /* Bufferevent event callback for the connect_hostname test: remembers what * event we got. */ static void be_connect_hostname_event_cb(struct bufferevent *bev, short what, void *ctx) { struct be_conn_hostname_result *got = ctx; if (!got->what) { TT_BLATHER(("Got a bufferevent event %d", what)); got->what = what; if ((what & BEV_EVENT_CONNECTED) || (what & BEV_EVENT_ERROR)) { int r; if ((r = bufferevent_socket_get_dns_error(bev))) { got->dnserr = r; TT_BLATHER(("DNS error %d: %s", r, evutil_gai_strerror(r))); } ++total_connected_or_failed; TT_BLATHER(("Got %d connections or errors.", total_connected_or_failed)); if (total_n_accepted >= 3 && total_connected_or_failed >= 5) event_base_loopexit(be_connect_hostname_base, NULL); } } else { TT_FAIL(("Two events on one bufferevent. %d,%d", got->what, (int)what)); } } static void test_bufferevent_connect_hostname(void *arg) { struct basic_test_data *data = arg; struct evconnlistener *listener = NULL; struct bufferevent *be1=NULL, *be2=NULL, *be3=NULL, *be4=NULL, *be5=NULL; struct be_conn_hostname_result be1_outcome={0,0}, be2_outcome={0,0}, be3_outcome={0,0}, be4_outcome={0,0}, be5_outcome={0,0}; int expect_err5; struct evdns_base *dns=NULL; struct evdns_server_port *port=NULL; struct sockaddr_in sin; int listener_port=-1; ev_uint16_t dns_port=0; int n_accept=0, n_dns=0; char buf[128]; be_connect_hostname_base = data->base; /* Bind an address and figure out what port it's on. */ memset(&sin, 0, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(0x7f000001); /* 127.0.0.1 */ sin.sin_port = 0; listener = evconnlistener_new_bind(data->base, nil_accept_cb, &n_accept, LEV_OPT_REUSEABLE|LEV_OPT_CLOSE_ON_EXEC, -1, (struct sockaddr *)&sin, sizeof(sin)); tt_assert(listener); listener_port = regress_get_socket_port( evconnlistener_get_fd(listener)); port = regress_get_dnsserver(data->base, &dns_port, NULL, be_getaddrinfo_server_cb, &n_dns); tt_assert(port); tt_int_op(dns_port, >=, 0); /* Start an evdns_base that uses the server as its resolver. */ dns = evdns_base_new(data->base, 0); evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", (int)dns_port); evdns_base_nameserver_ip_add(dns, buf); /* Now, finally, at long last, launch the bufferevents. One should do * a failing lookup IP, one should do a successful lookup by IP, * and one should do a successful lookup by hostname. */ be1 = bufferevent_socket_new(data->base, -1, BEV_OPT_CLOSE_ON_FREE); be2 = bufferevent_socket_new(data->base, -1, BEV_OPT_CLOSE_ON_FREE); be3 = bufferevent_socket_new(data->base, -1, BEV_OPT_CLOSE_ON_FREE); be4 = bufferevent_socket_new(data->base, -1, BEV_OPT_CLOSE_ON_FREE); be5 = bufferevent_socket_new(data->base, -1, BEV_OPT_CLOSE_ON_FREE); bufferevent_setcb(be1, NULL, NULL, be_connect_hostname_event_cb, &be1_outcome); bufferevent_setcb(be2, NULL, NULL, be_connect_hostname_event_cb, &be2_outcome); bufferevent_setcb(be3, NULL, NULL, be_connect_hostname_event_cb, &be3_outcome); bufferevent_setcb(be4, NULL, NULL, be_connect_hostname_event_cb, &be4_outcome); bufferevent_setcb(be5, NULL, NULL, be_connect_hostname_event_cb, &be5_outcome); /* Use the blocking resolver. This one will fail if your resolver * can't resolve localhost to 127.0.0.1 */ tt_assert(!bufferevent_socket_connect_hostname(be4, NULL, AF_INET, "localhost", listener_port)); /* Use the blocking resolver with a nonexistent hostname. */ tt_assert(!bufferevent_socket_connect_hostname(be5, NULL, AF_INET, "nonesuch.nowhere.example.com", 80)); { /* The blocking resolver will use the system nameserver, which * might tell us anything. (Yes, some twits even pretend that * example.com is real.) Let's see what answer to expect. */ struct evutil_addrinfo hints, *ai = NULL; memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = IPPROTO_TCP; expect_err5 = evutil_getaddrinfo( "nonesuch.nowhere.example.com", "80", &hints, &ai); } /* Launch an async resolve that will fail. */ tt_assert(!bufferevent_socket_connect_hostname(be1, dns, AF_INET, "nosuchplace.example.com", listener_port)); /* Connect to the IP without resolving. */ tt_assert(!bufferevent_socket_connect_hostname(be2, dns, AF_INET, "127.0.0.1", listener_port)); /* Launch an async resolve that will succeed. */ tt_assert(!bufferevent_socket_connect_hostname(be3, dns, AF_INET, "nobodaddy.example.com", listener_port)); event_base_dispatch(data->base); tt_int_op(be1_outcome.what, ==, BEV_EVENT_ERROR); tt_int_op(be1_outcome.dnserr, ==, EVUTIL_EAI_NONAME); tt_int_op(be2_outcome.what, ==, BEV_EVENT_CONNECTED); tt_int_op(be2_outcome.dnserr, ==, 0); tt_int_op(be3_outcome.what, ==, BEV_EVENT_CONNECTED); tt_int_op(be3_outcome.dnserr, ==, 0); tt_int_op(be4_outcome.what, ==, BEV_EVENT_CONNECTED); tt_int_op(be4_outcome.dnserr, ==, 0); if (expect_err5) { tt_int_op(be5_outcome.what, ==, BEV_EVENT_ERROR); tt_int_op(be5_outcome.dnserr, ==, expect_err5); } tt_int_op(n_accept, ==, 3); tt_int_op(n_dns, ==, 2); end: if (listener) evconnlistener_free(listener); if (port) evdns_close_server_port(port); if (dns) evdns_base_free(dns, 0); if (be1) bufferevent_free(be1); if (be2) bufferevent_free(be2); if (be3) bufferevent_free(be3); if (be4) bufferevent_free(be4); if (be5) bufferevent_free(be5); } struct gai_outcome { int err; struct evutil_addrinfo *ai; }; static int n_gai_results_pending = 0; static struct event_base *exit_base_on_no_pending_results = NULL; static void gai_cb(int err, struct evutil_addrinfo *res, void *ptr) { struct gai_outcome *go = ptr; go->err = err; go->ai = res; if (--n_gai_results_pending <= 0 && exit_base_on_no_pending_results) event_base_loopexit(exit_base_on_no_pending_results, NULL); if (n_gai_results_pending < 900) TT_BLATHER(("Got an answer; expecting %d more.", n_gai_results_pending)); } static void cancel_gai_cb(evutil_socket_t fd, short what, void *ptr) { struct evdns_getaddrinfo_request *r = ptr; evdns_getaddrinfo_cancel(r); } static void test_getaddrinfo_async(void *arg) { struct basic_test_data *data = arg; struct evutil_addrinfo hints, *a; struct gai_outcome local_outcome; struct gai_outcome a_out[12]; int i; struct evdns_getaddrinfo_request *r; char buf[128]; struct evdns_server_port *port = NULL; ev_uint16_t dns_port = 0; int n_dns_questions = 0; struct evdns_base *dns_base; memset(a_out, 0, sizeof(a_out)); memset(&local_outcome, 0, sizeof(local_outcome)); dns_base = evdns_base_new(data->base, 0); tt_assert(dns_base); /* for localhost */ evdns_base_load_hosts(dns_base, NULL); tt_assert(! evdns_base_set_option(dns_base, "timeout", "0.3")); tt_assert(! evdns_base_set_option(dns_base, "getaddrinfo-allow-skew", "0.2")); n_gai_results_pending = 10000; /* don't think about exiting yet. */ /* 1. Try some cases that will never hit the asynchronous resolver. */ /* 1a. Simple case with a symbolic service name */ memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; memset(&local_outcome, 0, sizeof(local_outcome)); r = evdns_getaddrinfo(dns_base, "1.2.3.4", "http", &hints, gai_cb, &local_outcome); tt_assert(! r); if (!local_outcome.err) { tt_ptr_op(local_outcome.ai,!=,NULL); test_ai_eq(local_outcome.ai, "1.2.3.4:80", SOCK_STREAM, IPPROTO_TCP); evutil_freeaddrinfo(local_outcome.ai); local_outcome.ai = NULL; } else { TT_BLATHER(("Apparently we have no getservbyname.")); } /* 1b. EVUTIL_AI_NUMERICHOST is set */ memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_flags = EVUTIL_AI_NUMERICHOST; memset(&local_outcome, 0, sizeof(local_outcome)); r = evdns_getaddrinfo(dns_base, "www.google.com", "80", &hints, gai_cb, &local_outcome); tt_ptr_op(r,==,NULL); tt_int_op(local_outcome.err,==,EVUTIL_EAI_NONAME); tt_ptr_op(local_outcome.ai,==,NULL); /* 1c. We give a numeric address (ipv6) */ memset(&hints, 0, sizeof(hints)); memset(&local_outcome, 0, sizeof(local_outcome)); hints.ai_family = PF_UNSPEC; hints.ai_protocol = IPPROTO_TCP; r = evdns_getaddrinfo(dns_base, "f::f", "8008", &hints, gai_cb, &local_outcome); tt_assert(!r); tt_int_op(local_outcome.err,==,0); tt_assert(local_outcome.ai); tt_ptr_op(local_outcome.ai->ai_next,==,NULL); test_ai_eq(local_outcome.ai, "[f::f]:8008", SOCK_STREAM, IPPROTO_TCP); evutil_freeaddrinfo(local_outcome.ai); local_outcome.ai = NULL; /* 1d. We give a numeric address (ipv4) */ memset(&hints, 0, sizeof(hints)); memset(&local_outcome, 0, sizeof(local_outcome)); hints.ai_family = PF_UNSPEC; r = evdns_getaddrinfo(dns_base, "5.6.7.8", NULL, &hints, gai_cb, &local_outcome); tt_assert(!r); tt_int_op(local_outcome.err,==,0); tt_assert(local_outcome.ai); a = ai_find_by_protocol(local_outcome.ai, IPPROTO_TCP); tt_assert(a); test_ai_eq(a, "5.6.7.8", SOCK_STREAM, IPPROTO_TCP); a = ai_find_by_protocol(local_outcome.ai, IPPROTO_UDP); tt_assert(a); test_ai_eq(a, "5.6.7.8", SOCK_DGRAM, IPPROTO_UDP); evutil_freeaddrinfo(local_outcome.ai); local_outcome.ai = NULL; /* 1e. nodename is NULL (bind) */ memset(&hints, 0, sizeof(hints)); memset(&local_outcome, 0, sizeof(local_outcome)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_DGRAM; hints.ai_flags = EVUTIL_AI_PASSIVE; r = evdns_getaddrinfo(dns_base, NULL, "9090", &hints, gai_cb, &local_outcome); tt_assert(!r); tt_int_op(local_outcome.err,==,0); tt_assert(local_outcome.ai); /* we should get a v4 address of 0.0.0.0... */ a = ai_find_by_family(local_outcome.ai, PF_INET); tt_assert(a); test_ai_eq(a, "0.0.0.0:9090", SOCK_DGRAM, IPPROTO_UDP); /* ... and a v6 address of ::0 */ a = ai_find_by_family(local_outcome.ai, PF_INET6); tt_assert(a); test_ai_eq(a, "[::]:9090", SOCK_DGRAM, IPPROTO_UDP); evutil_freeaddrinfo(local_outcome.ai); local_outcome.ai = NULL; /* 1f. nodename is NULL (connect) */ memset(&hints, 0, sizeof(hints)); memset(&local_outcome, 0, sizeof(local_outcome)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; r = evdns_getaddrinfo(dns_base, NULL, "2", &hints, gai_cb, &local_outcome); tt_assert(!r); tt_int_op(local_outcome.err,==,0); tt_assert(local_outcome.ai); /* we should get a v4 address of 127.0.0.1 .... */ a = ai_find_by_family(local_outcome.ai, PF_INET); tt_assert(a); test_ai_eq(a, "127.0.0.1:2", SOCK_STREAM, IPPROTO_TCP); /* ... and a v6 address of ::1 */ a = ai_find_by_family(local_outcome.ai, PF_INET6); tt_assert(a); test_ai_eq(a, "[::1]:2", SOCK_STREAM, IPPROTO_TCP); evutil_freeaddrinfo(local_outcome.ai); local_outcome.ai = NULL; /* 1g. We find localhost immediately. (pf_unspec) */ memset(&hints, 0, sizeof(hints)); memset(&local_outcome, 0, sizeof(local_outcome)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; r = evdns_getaddrinfo(dns_base, "LOCALHOST", "80", &hints, gai_cb, &local_outcome); tt_assert(!r); tt_int_op(local_outcome.err,==,0); tt_assert(local_outcome.ai); /* we should get a v4 address of 127.0.0.1 .... */ a = ai_find_by_family(local_outcome.ai, PF_INET); tt_assert(a); test_ai_eq(a, "127.0.0.1:80", SOCK_STREAM, IPPROTO_TCP); /* ... and a v6 address of ::1 */ a = ai_find_by_family(local_outcome.ai, PF_INET6); tt_assert(a); test_ai_eq(a, "[::1]:80", SOCK_STREAM, IPPROTO_TCP); evutil_freeaddrinfo(local_outcome.ai); local_outcome.ai = NULL; /* 1g. We find localhost immediately. (pf_inet6) */ memset(&hints, 0, sizeof(hints)); memset(&local_outcome, 0, sizeof(local_outcome)); hints.ai_family = PF_INET6; hints.ai_socktype = SOCK_STREAM; r = evdns_getaddrinfo(dns_base, "LOCALHOST", "9999", &hints, gai_cb, &local_outcome); tt_assert(! r); tt_int_op(local_outcome.err,==,0); tt_assert(local_outcome.ai); a = local_outcome.ai; test_ai_eq(a, "[::1]:9999", SOCK_STREAM, IPPROTO_TCP); tt_ptr_op(a->ai_next, ==, NULL); evutil_freeaddrinfo(local_outcome.ai); local_outcome.ai = NULL; /* 2. Okay, now we can actually test the asynchronous resolver. */ /* Start a dummy local dns server... */ port = regress_get_dnsserver(data->base, &dns_port, NULL, be_getaddrinfo_server_cb, &n_dns_questions); tt_assert(port); tt_int_op(dns_port, >=, 0); /* ... and tell the evdns_base about it. */ evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", dns_port); evdns_base_nameserver_ip_add(dns_base, buf); memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = EVUTIL_AI_CANONNAME; /* 0: Request for both.example.com should return both addresses. */ r = evdns_getaddrinfo(dns_base, "both.example.com", "8000", &hints, gai_cb, &a_out[0]); tt_assert(r); /* 1: Request for v4only.example.com should return one address. */ r = evdns_getaddrinfo(dns_base, "v4only.example.com", "8001", &hints, gai_cb, &a_out[1]); tt_assert(r); /* 2: Request for v6only.example.com should return one address. */ hints.ai_flags = 0; r = evdns_getaddrinfo(dns_base, "v6only.example.com", "8002", &hints, gai_cb, &a_out[2]); tt_assert(r); /* 3: PF_INET request for v4assert.example.com should not generate a * v6 request. The server will fail the test if it does. */ hints.ai_family = PF_INET; r = evdns_getaddrinfo(dns_base, "v4assert.example.com", "8003", &hints, gai_cb, &a_out[3]); tt_assert(r); /* 4: PF_INET6 request for v6assert.example.com should not generate a * v4 request. The server will fail the test if it does. */ hints.ai_family = PF_INET6; r = evdns_getaddrinfo(dns_base, "v6assert.example.com", "8004", &hints, gai_cb, &a_out[4]); tt_assert(r); /* 5: PF_INET request for nosuchplace.example.com should give NEXIST. */ hints.ai_family = PF_INET; r = evdns_getaddrinfo(dns_base, "nosuchplace.example.com", "8005", &hints, gai_cb, &a_out[5]); tt_assert(r); /* 6: PF_UNSPEC request for nosuchplace.example.com should give NEXIST. */ hints.ai_family = PF_UNSPEC; r = evdns_getaddrinfo(dns_base, "nosuchplace.example.com", "8006", &hints, gai_cb, &a_out[6]); tt_assert(r); /* 7: PF_UNSPEC request for v6timeout.example.com should give an ipv4 * address only. */ hints.ai_family = PF_UNSPEC; r = evdns_getaddrinfo(dns_base, "v6timeout.example.com", "8007", &hints, gai_cb, &a_out[7]); tt_assert(r); /* 8: PF_UNSPEC request for v6timeout-nonexist.example.com should give * a NEXIST */ hints.ai_family = PF_UNSPEC; r = evdns_getaddrinfo(dns_base, "v6timeout-nonexist.example.com", "8008", &hints, gai_cb, &a_out[8]); tt_assert(r); /* 9: AI_ADDRCONFIG should at least not crash. Can't test it more * without knowing what kind of internet we have. */ hints.ai_flags |= EVUTIL_AI_ADDRCONFIG; r = evdns_getaddrinfo(dns_base, "both.example.com", "8009", &hints, gai_cb, &a_out[9]); tt_assert(r); /* 10: PF_UNSPEC for v4timeout.example.com should give an ipv6 address * only. */ hints.ai_family = PF_UNSPEC; hints.ai_flags = 0; r = evdns_getaddrinfo(dns_base, "v4timeout.example.com", "8010", &hints, gai_cb, &a_out[10]); tt_assert(r); /* 11: timeout.example.com: cancel it after 100 msec. */ r = evdns_getaddrinfo(dns_base, "all-timeout.example.com", "8011", &hints, gai_cb, &a_out[11]); tt_assert(r); { struct timeval tv; tv.tv_sec = 0; tv.tv_usec = 100*1000; /* 100 msec */ event_base_once(data->base, -1, EV_TIMEOUT, cancel_gai_cb, r, &tv); } /* XXXXX There are more tests we could do, including: - A test to elicit NODATA. */ n_gai_results_pending = 12; exit_base_on_no_pending_results = data->base; event_base_dispatch(data->base); /* 0: both.example.com */ tt_int_op(a_out[0].err, ==, 0); tt_assert(a_out[0].ai); tt_assert(a_out[0].ai->ai_next); tt_assert(!a_out[0].ai->ai_next->ai_next); a = ai_find_by_family(a_out[0].ai, PF_INET); tt_assert(a); test_ai_eq(a, "80.80.32.32:8000", SOCK_STREAM, IPPROTO_TCP); a = ai_find_by_family(a_out[0].ai, PF_INET6); tt_assert(a); test_ai_eq(a, "[80ff::bbbb]:8000", SOCK_STREAM, IPPROTO_TCP); tt_assert(a_out[0].ai->ai_canonname); tt_str_op(a_out[0].ai->ai_canonname, ==, "both-canonical.example.com"); /* 1: v4only.example.com */ tt_int_op(a_out[1].err, ==, 0); tt_assert(a_out[1].ai); tt_assert(! a_out[1].ai->ai_next); test_ai_eq(a_out[1].ai, "18.52.86.120:8001", SOCK_STREAM, IPPROTO_TCP); tt_assert(a_out[1].ai->ai_canonname == NULL); /* 2: v6only.example.com */ tt_int_op(a_out[2].err, ==, 0); tt_assert(a_out[2].ai); tt_assert(! a_out[2].ai->ai_next); test_ai_eq(a_out[2].ai, "[b0b::f00d]:8002", SOCK_STREAM, IPPROTO_TCP); /* 3: v4assert.example.com */ tt_int_op(a_out[3].err, ==, 0); tt_assert(a_out[3].ai); tt_assert(! a_out[3].ai->ai_next); test_ai_eq(a_out[3].ai, "18.52.86.120:8003", SOCK_STREAM, IPPROTO_TCP); /* 4: v6assert.example.com */ tt_int_op(a_out[4].err, ==, 0); tt_assert(a_out[4].ai); tt_assert(! a_out[4].ai->ai_next); test_ai_eq(a_out[4].ai, "[b0b::f00d]:8004", SOCK_STREAM, IPPROTO_TCP); /* 5: nosuchplace.example.com (inet) */ tt_int_op(a_out[5].err, ==, EVUTIL_EAI_NONAME); tt_assert(! a_out[5].ai); /* 6: nosuchplace.example.com (unspec) */ tt_int_op(a_out[6].err, ==, EVUTIL_EAI_NONAME); tt_assert(! a_out[6].ai); /* 7: v6timeout.example.com */ tt_int_op(a_out[7].err, ==, 0); tt_assert(a_out[7].ai); tt_assert(! a_out[7].ai->ai_next); test_ai_eq(a_out[7].ai, "171.205.239.1:8007", SOCK_STREAM, IPPROTO_TCP); /* 8: v6timeout-nonexist.example.com */ tt_int_op(a_out[8].err, ==, EVUTIL_EAI_NONAME); tt_assert(! a_out[8].ai); /* 9: both (ADDRCONFIG) */ tt_int_op(a_out[9].err, ==, 0); tt_assert(a_out[9].ai); a = ai_find_by_family(a_out[9].ai, PF_INET); if (a) test_ai_eq(a, "80.80.32.32:8009", SOCK_STREAM, IPPROTO_TCP); else tt_assert(ai_find_by_family(a_out[9].ai, PF_INET6)); a = ai_find_by_family(a_out[9].ai, PF_INET6); if (a) test_ai_eq(a, "[80ff::bbbb]:8009", SOCK_STREAM, IPPROTO_TCP); else tt_assert(ai_find_by_family(a_out[9].ai, PF_INET)); /* 10: v4timeout.example.com */ tt_int_op(a_out[10].err, ==, 0); tt_assert(a_out[10].ai); tt_assert(! a_out[10].ai->ai_next); test_ai_eq(a_out[10].ai, "[a0a::ff01]:8010", SOCK_STREAM, IPPROTO_TCP); /* 11: cancelled request. */ tt_int_op(a_out[11].err, ==, EVUTIL_EAI_CANCEL); tt_assert(a_out[11].ai == NULL); end: if (local_outcome.ai) evutil_freeaddrinfo(local_outcome.ai); for (i=0;i<(int)ARRAY_SIZE(a_out);++i) { if (a_out[i].ai) evutil_freeaddrinfo(a_out[i].ai); } if (port) evdns_close_server_port(port); if (dns_base) evdns_base_free(dns_base, 0); } struct gaic_request_status { int magic; struct event_base *base; struct evdns_base *dns_base; struct evdns_getaddrinfo_request *request; struct event cancel_event; int canceled; }; #define GAIC_MAGIC 0x1234abcd static int pending = 0; static void gaic_cancel_request_cb(evutil_socket_t fd, short what, void *arg) { struct gaic_request_status *status = arg; tt_assert(status->magic == GAIC_MAGIC); status->canceled = 1; evdns_getaddrinfo_cancel(status->request); return; end: event_base_loopexit(status->base, NULL); } static void gaic_server_cb(struct evdns_server_request *req, void *arg) { ev_uint32_t answer = 0x7f000001; tt_assert(req->nquestions); evdns_server_request_add_a_reply(req, req->questions[0]->name, 1, &answer, 100); evdns_server_request_respond(req, 0); return; end: evdns_server_request_respond(req, DNS_ERR_REFUSED); } static void gaic_getaddrinfo_cb(int result, struct evutil_addrinfo *res, void *arg) { struct gaic_request_status *status = arg; struct event_base *base = status->base; tt_assert(status->magic == GAIC_MAGIC); if (result == EVUTIL_EAI_CANCEL) { tt_assert(status->canceled); } event_del(&status->cancel_event); memset(status, 0xf0, sizeof(*status)); free(status); end: if (--pending <= 0) event_base_loopexit(base, NULL); } static void gaic_launch(struct event_base *base, struct evdns_base *dns_base) { struct gaic_request_status *status = calloc(1,sizeof(*status)); struct timeval tv = { 0, 10000 }; status->magic = GAIC_MAGIC; status->base = base; status->dns_base = dns_base; event_assign(&status->cancel_event, base, -1, 0, gaic_cancel_request_cb, status); status->request = evdns_getaddrinfo(dns_base, "foobar.bazquux.example.com", "80", NULL, gaic_getaddrinfo_cb, status); event_add(&status->cancel_event, &tv); ++pending; } #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED /* FIXME: We should move this to regress_main.c if anything else needs it.*/ /* Trivial replacements for malloc/free/realloc to check for memory leaks. * Not threadsafe. */ static int allocated_chunks = 0; static void * cnt_malloc(size_t sz) { allocated_chunks += 1; return malloc(sz); } static void * cnt_realloc(void *old, size_t sz) { if (!old) allocated_chunks += 1; if (!sz) allocated_chunks -= 1; return realloc(old, sz); } static void cnt_free(void *ptr) { allocated_chunks -= 1; free(ptr); } struct testleak_env_t { struct event_base *base; struct evdns_base *dns_base; struct evdns_request *req; struct generic_dns_callback_result r; }; static void * testleak_setup(const struct testcase_t *testcase) { struct testleak_env_t *env; allocated_chunks = 0; /* Reset allocation counter, to start allocations from the very beginning. * (this will avoid false-positive negative numbers for allocated_chunks) */ libevent_global_shutdown(); event_set_mem_functions(cnt_malloc, cnt_realloc, cnt_free); event_enable_debug_mode(); /* not mm_calloc: we don't want to mess with the count. */ env = calloc(1, sizeof(struct testleak_env_t)); env->base = event_base_new(); env->dns_base = evdns_base_new(env->base, 0); env->req = evdns_base_resolve_ipv4( env->dns_base, "example.com", DNS_QUERY_NO_SEARCH, generic_dns_callback, &env->r); return env; } static int testleak_cleanup(const struct testcase_t *testcase, void *env_) { int ok = 0; struct testleak_env_t *env = env_; tt_assert(env); #ifdef EVENT__DISABLE_DEBUG_MODE tt_int_op(allocated_chunks, ==, 0); #else libevent_global_shutdown(); tt_int_op(allocated_chunks, ==, 0); #endif ok = 1; end: if (env) { if (env->dns_base) evdns_base_free(env->dns_base, 0); if (env->base) event_base_free(env->base); free(env); } return ok; } static struct testcase_setup_t testleak_funcs = { testleak_setup, testleak_cleanup }; static void test_dbg_leak_cancel(void *env_) { /* cancel, loop, free/dns, free/base */ struct testleak_env_t *env = env_; int send_err_shutdown = 1; evdns_cancel_request(env->dns_base, env->req); env->req = 0; /* `req` is freed in callback, that's why one loop is required. */ event_base_loop(env->base, EVLOOP_NONBLOCK); /* send_err_shutdown means nothing as soon as our request is * already canceled */ evdns_base_free(env->dns_base, send_err_shutdown); env->dns_base = 0; event_base_free(env->base); env->base = 0; } static void dbg_leak_resume(void *env_, int cancel, int send_err_shutdown) { /* cancel, loop, free/dns, free/base */ struct testleak_env_t *env = env_; if (cancel) { evdns_cancel_request(env->dns_base, env->req); tt_assert(!evdns_base_resume(env->dns_base)); } else { /* TODO: No nameservers, request can't be processed, must be errored */ tt_assert(!evdns_base_resume(env->dns_base)); } event_base_loop(env->base, EVLOOP_NONBLOCK); /** * Because we don't cancel request, and want our callback to recieve * DNS_ERR_SHUTDOWN, we use deferred callback, and there was: * - one extra malloc(), * @see reply_schedule_callback() * - and one missing free * @see request_finished() (req->handle->pending_cb = 1) * than we don't need to count in testleak_cleanup(), but we can clean them * if we will run loop once again, but *after* evdns base freed. */ evdns_base_free(env->dns_base, send_err_shutdown); env->dns_base = 0; event_base_loop(env->base, EVLOOP_NONBLOCK); end: event_base_free(env->base); env->base = 0; } #define IMPL_DBG_LEAK_RESUME(name, cancel, send_err_shutdown) \ static void \ test_dbg_leak_##name##_(void *env_) \ { \ dbg_leak_resume(env_, cancel, send_err_shutdown); \ } IMPL_DBG_LEAK_RESUME(resume, 0, 0) IMPL_DBG_LEAK_RESUME(cancel_and_resume, 1, 0) IMPL_DBG_LEAK_RESUME(resume_send_err, 0, 1) IMPL_DBG_LEAK_RESUME(cancel_and_resume_send_err, 1, 1) static void test_dbg_leak_shutdown(void *env_) { /* free/dns, loop, free/base */ struct testleak_env_t *env = env_; int send_err_shutdown = 1; /* `req` is freed both with `send_err_shutdown` and without it, * the only difference is `evdns_callback` call */ env->req = 0; evdns_base_free(env->dns_base, send_err_shutdown); env->dns_base = 0; /* `req` is freed in callback, that's why one loop is required */ event_base_loop(env->base, EVLOOP_NONBLOCK); event_base_free(env->base); env->base = 0; } #endif static void test_getaddrinfo_async_cancel_stress(void *ptr) { struct event_base *base; struct evdns_base *dns_base = NULL; struct evdns_server_port *server = NULL; evutil_socket_t fd = -1; struct sockaddr_in sin; struct sockaddr_storage ss; ev_socklen_t slen; int i; base = event_base_new(); dns_base = evdns_base_new(base, 0); memset(&sin, 0, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_port = 0; sin.sin_addr.s_addr = htonl(0x7f000001); if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { tt_abort_perror("socket"); } evutil_make_socket_nonblocking(fd); if (bind(fd, (struct sockaddr*)&sin, sizeof(sin))<0) { tt_abort_perror("bind"); } server = evdns_add_server_port_with_base(base, fd, 0, gaic_server_cb, base); memset(&ss, 0, sizeof(ss)); slen = sizeof(ss); if (getsockname(fd, (struct sockaddr*)&ss, &slen)<0) { tt_abort_perror("getsockname"); } evdns_base_nameserver_sockaddr_add(dns_base, (struct sockaddr*)&ss, slen, 0); for (i = 0; i < 1000; ++i) { gaic_launch(base, dns_base); } event_base_dispatch(base); end: if (dns_base) evdns_base_free(dns_base, 1); if (server) evdns_close_server_port(server); if (base) event_base_free(base); if (fd >= 0) evutil_closesocket(fd); } static void dns_client_fail_requests_test(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct evdns_base *dns = NULL; struct evdns_server_port *dns_port = NULL; ev_uint16_t portnum = 0; char buf[64]; struct generic_dns_callback_result r[20]; int i; dns_port = regress_get_dnsserver(base, &portnum, NULL, regress_dns_server_cb, reissue_table); tt_assert(dns_port); evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", (int)portnum); dns = evdns_base_new(base, EVDNS_BASE_DISABLE_WHEN_INACTIVE); tt_assert(!evdns_base_nameserver_ip_add(dns, buf)); for (i = 0; i < 20; ++i) evdns_base_resolve_ipv4(dns, "foof.example.com", 0, generic_dns_callback, &r[i]); n_replies_left = 20; exit_base = base; evdns_base_free(dns, 1 /** fail requests */); /** run defered callbacks, to trigger UAF */ event_base_dispatch(base); tt_int_op(n_replies_left, ==, 0); for (i = 0; i < 20; ++i) tt_int_op(r[i].result, ==, DNS_ERR_SHUTDOWN); end: evdns_close_server_port(dns_port); } static void getaddrinfo_cb(int err, struct evutil_addrinfo *res, void *ptr) { generic_dns_callback(err, 0, 0, 0, NULL, ptr); } static void dns_client_fail_requests_getaddrinfo_test(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct evdns_base *dns = NULL; struct evdns_server_port *dns_port = NULL; ev_uint16_t portnum = 0; char buf[64]; struct generic_dns_callback_result r[20]; int i; dns_port = regress_get_dnsserver(base, &portnum, NULL, regress_dns_server_cb, reissue_table); tt_assert(dns_port); evutil_snprintf(buf, sizeof(buf), "127.0.0.1:%d", (int)portnum); dns = evdns_base_new(base, EVDNS_BASE_DISABLE_WHEN_INACTIVE); tt_assert(!evdns_base_nameserver_ip_add(dns, buf)); for (i = 0; i < 20; ++i) tt_assert(evdns_getaddrinfo(dns, "foof.example.com", "ssh", NULL, getaddrinfo_cb, &r[i])); n_replies_left = 20; exit_base = base; evdns_base_free(dns, 1 /** fail requests */); /** run defered callbacks, to trigger UAF */ event_base_dispatch(base); tt_int_op(n_replies_left, ==, 0); for (i = 0; i < 20; ++i) tt_int_op(r[i].result, ==, EVUTIL_EAI_FAIL); end: evdns_close_server_port(dns_port); } #define DNS_LEGACY(name, flags) \ { #name, run_legacy_test_fn, flags|TT_LEGACY, &legacy_setup, \ dns_##name } struct testcase_t dns_testcases[] = { DNS_LEGACY(server, TT_FORK|TT_NEED_BASE), DNS_LEGACY(gethostbyname, TT_FORK|TT_NEED_BASE|TT_NEED_DNS|TT_OFF_BY_DEFAULT), DNS_LEGACY(gethostbyname6, TT_FORK|TT_NEED_BASE|TT_NEED_DNS|TT_OFF_BY_DEFAULT), DNS_LEGACY(gethostbyaddr, TT_FORK|TT_NEED_BASE|TT_NEED_DNS|TT_OFF_BY_DEFAULT), { "resolve_reverse", dns_resolve_reverse, TT_FORK|TT_OFF_BY_DEFAULT, NULL, NULL }, { "search_empty", dns_search_empty_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "search", dns_search_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "search_lower", dns_search_lower_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "search_cancel", dns_search_cancel_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "retry", dns_retry_test, TT_FORK|TT_NEED_BASE|TT_NO_LOGS, &basic_setup, NULL }, { "retry_disable_when_inactive", dns_retry_disable_when_inactive_test, TT_FORK|TT_NEED_BASE|TT_NO_LOGS, &basic_setup, NULL }, { "reissue", dns_reissue_test, TT_FORK|TT_NEED_BASE|TT_NO_LOGS, &basic_setup, NULL }, { "reissue_disable_when_inactive", dns_reissue_disable_when_inactive_test, TT_FORK|TT_NEED_BASE|TT_NO_LOGS, &basic_setup, NULL }, { "inflight", dns_inflight_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "bufferevent_connect_hostname", test_bufferevent_connect_hostname, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "disable_when_inactive", dns_disable_when_inactive_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "disable_when_inactive_no_ns", dns_disable_when_inactive_no_ns_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "getaddrinfo_async", test_getaddrinfo_async, TT_FORK|TT_NEED_BASE, &basic_setup, (char*)"" }, { "getaddrinfo_cancel_stress", test_getaddrinfo_async_cancel_stress, TT_FORK, NULL, NULL }, #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED { "leak_shutdown", test_dbg_leak_shutdown, TT_FORK, &testleak_funcs, NULL }, { "leak_cancel", test_dbg_leak_cancel, TT_FORK, &testleak_funcs, NULL }, { "leak_resume", test_dbg_leak_resume_, TT_FORK, &testleak_funcs, NULL }, { "leak_cancel_and_resume", test_dbg_leak_cancel_and_resume_, TT_FORK, &testleak_funcs, NULL }, { "leak_resume_send_err", test_dbg_leak_resume_send_err_, TT_FORK, &testleak_funcs, NULL }, { "leak_cancel_and_resume_send_err", test_dbg_leak_cancel_and_resume_send_err_, TT_FORK, &testleak_funcs, NULL }, #endif { "client_fail_requests", dns_client_fail_requests_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, { "client_fail_requests_getaddrinfo", dns_client_fail_requests_getaddrinfo_test, TT_FORK|TT_NEED_BASE, &basic_setup, NULL }, END_OF_TESTCASES }; kytensor-2.49.0.7/third_party/libevent/test/regress_et.c000066400000000000000000000136351527052371200232730ustar00rootroot00000000000000/* * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "../util-internal.h" #include "event2/event-config.h" #ifdef _WIN32 #include #endif #include #include #ifdef EVENT__HAVE_SYS_SOCKET_H #include #endif #include #include #include #include #ifndef _WIN32 #include #include #endif #include #include "event2/event.h" #include "event2/util.h" #include "regress.h" static int was_et = 0; static void read_cb(evutil_socket_t fd, short event, void *arg) { char buf; int len; len = recv(fd, &buf, sizeof(buf), 0); called++; if (event & EV_ET) was_et = 1; if (!len) event_del(arg); } #ifdef _WIN32 #define LOCAL_SOCKETPAIR_AF AF_INET #else #define LOCAL_SOCKETPAIR_AF AF_UNIX #endif static void test_edgetriggered(void *et) { struct event *ev = NULL; struct event_base *base = NULL; const char *test = "test string"; evutil_socket_t pair[2] = {-1,-1}; int supports_et; /* On Linux 3.2.1 (at least, as patched by Fedora and tested by Nick), * doing a "recv" on an AF_UNIX socket resets the readability of the * socket, even though there is no state change, so we don't actually * get edge-triggered behavior. Yuck! Linux 3.1.9 didn't have this * problem. */ #ifdef __linux__ if (evutil_ersatz_socketpair_(AF_INET, SOCK_STREAM, 0, pair) == -1) { tt_abort_perror("socketpair"); } #else if (evutil_socketpair(LOCAL_SOCKETPAIR_AF, SOCK_STREAM, 0, pair) == -1) { tt_abort_perror("socketpair"); } #endif called = was_et = 0; tt_int_op(send(pair[0], test, (int)strlen(test)+1, 0), >, 0); shutdown(pair[0], EVUTIL_SHUT_WR); /* Initalize the event library */ base = event_base_new(); if (!strcmp(event_base_get_method(base), "epoll") || !strcmp(event_base_get_method(base), "epoll (with changelist)") || !strcmp(event_base_get_method(base), "kqueue")) supports_et = 1; else supports_et = 0; TT_BLATHER(("Checking for edge-triggered events with %s, which should %s" "support edge-triggering", event_base_get_method(base), supports_et?"":"not ")); /* Initalize one event */ ev = event_new(base, pair[1], EV_READ|EV_ET|EV_PERSIST, read_cb, &ev); event_add(ev, NULL); /* We're going to call the dispatch function twice. The first invocation * will read a single byte from pair[1] in either case. If we're edge * triggered, we'll only see the event once (since we only see transitions * from no data to data), so the second invocation of event_base_loop will * do nothing. If we're level triggered, the second invocation of * event_base_loop will also activate the event (because there's still * data to read). */ event_base_loop(base,EVLOOP_NONBLOCK|EVLOOP_ONCE); event_base_loop(base,EVLOOP_NONBLOCK|EVLOOP_ONCE); if (supports_et) { tt_int_op(called, ==, 1); tt_assert(was_et); } else { tt_int_op(called, ==, 2); tt_assert(!was_et); } end: if (ev) { event_del(ev); event_free(ev); } if (base) event_base_free(base); evutil_closesocket(pair[0]); evutil_closesocket(pair[1]); } static void test_edgetriggered_mix_error(void *data_) { struct basic_test_data *data = data_; struct event_base *base = NULL; struct event *ev_et=NULL, *ev_lt=NULL; #ifdef EVENT__DISABLE_DEBUG_MODE if (1) tt_skip(); #endif if (!libevent_tests_running_in_debug_mode) event_enable_debug_mode(); base = event_base_new(); /* try mixing edge-triggered and level-triggered to make sure it fails*/ ev_et = event_new(base, data->pair[0], EV_READ|EV_ET, read_cb, ev_et); tt_assert(ev_et); ev_lt = event_new(base, data->pair[0], EV_READ, read_cb, ev_lt); tt_assert(ev_lt); /* Add edge-triggered, then level-triggered. Get an error. */ tt_int_op(0, ==, event_add(ev_et, NULL)); tt_int_op(-1, ==, event_add(ev_lt, NULL)); tt_int_op(EV_READ, ==, event_pending(ev_et, EV_READ, NULL)); tt_int_op(0, ==, event_pending(ev_lt, EV_READ, NULL)); tt_int_op(0, ==, event_del(ev_et)); /* Add level-triggered, then edge-triggered. Get an error. */ tt_int_op(0, ==, event_add(ev_lt, NULL)); tt_int_op(-1, ==, event_add(ev_et, NULL)); tt_int_op(EV_READ, ==, event_pending(ev_lt, EV_READ, NULL)); tt_int_op(0, ==, event_pending(ev_et, EV_READ, NULL)); end: if (ev_et) event_free(ev_et); if (ev_lt) event_free(ev_lt); if (base) event_base_free(base); } struct testcase_t edgetriggered_testcases[] = { { "et", test_edgetriggered, TT_FORK, NULL, NULL }, { "et_mix_error", test_edgetriggered_mix_error, TT_FORK|TT_NEED_SOCKETPAIR|TT_NO_LOGS, &basic_setup, NULL }, END_OF_TESTCASES }; kytensor-2.49.0.7/third_party/libevent/test/regress_finalize.c000066400000000000000000000212231527052371200244540ustar00rootroot00000000000000/* * Copyright (c) 2013 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "event2/event-config.h" #include "evconfig-private.h" #include "tinytest.h" #include "tinytest_macros.h" #include #include "event2/event.h" #include "event2/util.h" #include "event-internal.h" #include "defer-internal.h" #include "regress.h" #include "regress_thread.h" static void timer_callback(evutil_socket_t fd, short what, void *arg) { int *int_arg = arg; *int_arg += 1; (void)fd; (void)what; } static void simple_callback(struct event_callback *evcb, void *arg) { int *int_arg = arg; *int_arg += 1; (void)evcb; } static void event_finalize_callback_1(struct event *ev, void *arg) { int *int_arg = arg; *int_arg += 100; (void)ev; } static void callback_finalize_callback_1(struct event_callback *evcb, void *arg) { int *int_arg = arg; *int_arg += 100; (void)evcb; } static void test_fin_cb_invoked(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct event *ev; struct event ev2; struct event_callback evcb; int cb_called = 0; int ev_called = 0; const struct timeval ten_sec = {10,0}; event_deferred_cb_init_(&evcb, 0, simple_callback, &cb_called); ev = evtimer_new(base, timer_callback, &ev_called); /* Just finalize them; don't bother adding. */ event_free_finalize(0, ev, event_finalize_callback_1); event_callback_finalize_(base, 0, &evcb, callback_finalize_callback_1); event_base_dispatch(base); tt_int_op(cb_called, ==, 100); tt_int_op(ev_called, ==, 100); ev_called = cb_called = 0; event_base_assert_ok_(base); /* Now try it when they're active. (actually, don't finalize: make * sure activation can happen! */ ev = evtimer_new(base, timer_callback, &ev_called); event_deferred_cb_init_(&evcb, 0, simple_callback, &cb_called); event_active(ev, EV_TIMEOUT, 1); event_callback_activate_(base, &evcb); event_base_dispatch(base); tt_int_op(cb_called, ==, 1); tt_int_op(ev_called, ==, 1); ev_called = cb_called = 0; event_base_assert_ok_(base); /* Great, it worked. Now activate and finalize and make sure only * finalizing happens. */ event_active(ev, EV_TIMEOUT, 1); event_callback_activate_(base, &evcb); event_free_finalize(0, ev, event_finalize_callback_1); event_callback_finalize_(base, 0, &evcb, callback_finalize_callback_1); event_base_dispatch(base); tt_int_op(cb_called, ==, 100); tt_int_op(ev_called, ==, 100); ev_called = 0; event_base_assert_ok_(base); /* Okay, now add but don't have it become active, and make sure *that* * works. */ ev = evtimer_new(base, timer_callback, &ev_called); event_add(ev, &ten_sec); event_free_finalize(0, ev, event_finalize_callback_1); event_base_dispatch(base); tt_int_op(ev_called, ==, 100); ev_called = 0; event_base_assert_ok_(base); /* Now try adding and deleting after finalizing. */ ev = evtimer_new(base, timer_callback, &ev_called); evtimer_assign(&ev2, base, timer_callback, &ev_called); event_add(ev, &ten_sec); event_free_finalize(0, ev, event_finalize_callback_1); event_finalize(0, &ev2, event_finalize_callback_1); event_add(&ev2, &ten_sec); event_del(ev); event_active(&ev2, EV_TIMEOUT, 1); event_base_dispatch(base); tt_int_op(ev_called, ==, 200); event_base_assert_ok_(base); end: ; } #ifndef EVENT__DISABLE_MM_REPLACEMENT static void * tfff_malloc(size_t n) { return malloc(n); } static void *tfff_p1=NULL, *tfff_p2=NULL; static int tfff_p1_freed=0, tfff_p2_freed=0; static void tfff_free(void *p) { if (! p) return; if (p == tfff_p1) ++tfff_p1_freed; if (p == tfff_p2) ++tfff_p2_freed; free(p); } static void * tfff_realloc(void *p, size_t sz) { return realloc(p,sz); } #endif static void test_fin_free_finalize(void *arg) { #ifdef EVENT__DISABLE_MM_REPLACEMENT tinytest_set_test_skipped_(); #else struct event_base *base = NULL; struct event *ev, *ev2; int ev_called = 0; int ev2_called = 0; (void)arg; event_set_mem_functions(tfff_malloc, tfff_realloc, tfff_free); base = event_base_new(); tt_assert(base); ev = evtimer_new(base, timer_callback, &ev_called); ev2 = evtimer_new(base, timer_callback, &ev2_called); tfff_p1 = ev; tfff_p2 = ev2; event_free_finalize(0, ev, event_finalize_callback_1); event_finalize(0, ev2, event_finalize_callback_1); event_base_dispatch(base); tt_int_op(ev_called, ==, 100); tt_int_op(ev2_called, ==, 100); event_base_assert_ok_(base); tt_int_op(tfff_p1_freed, ==, 1); tt_int_op(tfff_p2_freed, ==, 0); event_free(ev2); end: if (base) event_base_free(base); #endif } /* For test_fin_within_cb */ struct event_and_count { struct event *ev; struct event *ev2; int count; }; static void event_finalize_callback_2(struct event *ev, void *arg) { struct event_and_count *evc = arg; evc->count += 100; event_free(ev); } static void timer_callback_2(evutil_socket_t fd, short what, void *arg) { struct event_and_count *evc = arg; event_finalize(0, evc->ev, event_finalize_callback_2); event_finalize(0, evc->ev2, event_finalize_callback_2); ++ evc->count; (void)fd; (void)what; } static void test_fin_within_cb(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct event_and_count evc1, evc2; evc1.count = evc2.count = 0; evc2.ev2 = evc1.ev = evtimer_new(base, timer_callback_2, &evc1); evc1.ev2 = evc2.ev = evtimer_new(base, timer_callback_2, &evc2); /* Activate both. The first one will have its callback run, which * will finalize both of them, preventing the second one's callback * from running. */ event_active(evc1.ev, EV_TIMEOUT, 1); event_active(evc2.ev, EV_TIMEOUT, 1); event_base_dispatch(base); tt_int_op(evc1.count, ==, 101); tt_int_op(evc2.count, ==, 100); event_base_assert_ok_(base); /* Now try with EV_PERSIST events. */ evc1.count = evc2.count = 0; evc2.ev2 = evc1.ev = event_new(base, -1, EV_PERSIST, timer_callback_2, &evc1); evc1.ev2 = evc2.ev = event_new(base, -1, EV_PERSIST, timer_callback_2, &evc2); event_active(evc1.ev, EV_TIMEOUT, 1); event_active(evc2.ev, EV_TIMEOUT, 1); event_base_dispatch(base); tt_int_op(evc1.count, ==, 101); tt_int_op(evc2.count, ==, 100); event_base_assert_ok_(base); end: ; } #if 0 static void timer_callback_3(evutil_socket_t *fd, short what, void *arg) { (void)fd; (void)what; } static void test_fin_many(void *arg) { struct basic_test_data *data = arg; struct event_base *base = data->base; struct event *ev1, *ev2; struct event_callback evcb1, evcb2; int ev1_count = 0, ev2_count = 0; int evcb1_count = 0, evcb2_count = 0; struct event_callback *array[4]; int n; /* First attempt: call finalize_many with no events running */ ev1 = evtimer_new(base, timer_callback, &ev1_count); ev1 = evtimer_new(base, timer_callback, &ev2_count); event_deferred_cb_init_(&evcb1, 0, simple_callback, &evcb1_called); event_deferred_cb_init_(&evcb2, 0, simple_callback, &evcb2_called); array[0] = &ev1->ev_evcallback; array[1] = &ev2->ev_evcallback; array[2] = &evcb1; array[3] = &evcb2; n = event_callback_finalize_many(base, 4, array, callback_finalize_callback_1); } #endif #define TEST(name, flags) \ { #name, test_fin_##name, (flags), &basic_setup, NULL } struct testcase_t finalize_testcases[] = { TEST(cb_invoked, TT_FORK|TT_NEED_BASE), TEST(free_finalize, TT_FORK), TEST(within_cb, TT_FORK|TT_NEED_BASE), // TEST(many, TT_FORK|TT_NEED_BASE), END_OF_TESTCASES }; kytensor-2.49.0.7/third_party/libevent/test/regress_http.c000066400000000000000000003702261527052371200236440ustar00rootroot00000000000000/* * Copyright (c) 2003-2007 Niels Provos * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "util-internal.h" #ifdef _WIN32 #include #include #include #endif #include "event2/event-config.h" #include #include #ifdef EVENT__HAVE_SYS_TIME_H #include #endif #include #ifndef _WIN32 #include #include #include #include #endif #include #include #include #include #include #include "event2/dns.h" #include "event2/event.h" #include "event2/http.h" #include "event2/buffer.h" #include "event2/bufferevent.h" #include "event2/bufferevent_ssl.h" #include "event2/util.h" #include "event2/listener.h" #include "log-internal.h" #include "http-internal.h" #include "regress.h" #include "regress_testutils.h" /* set if a test needs to call loopexit on a base */ static struct event_base *exit_base; static char const BASIC_REQUEST_BODY[] = "This is funny"; static void http_basic_cb(struct evhttp_request *req, void *arg); static void http_large_cb(struct evhttp_request *req, void *arg); static void http_chunked_cb(struct evhttp_request *req, void *arg); static void http_post_cb(struct evhttp_request *req, void *arg); static void http_put_cb(struct evhttp_request *req, void *arg); static void http_delete_cb(struct evhttp_request *req, void *arg); static void http_delay_cb(struct evhttp_request *req, void *arg); static void http_large_delay_cb(struct evhttp_request *req, void *arg); static void http_badreq_cb(struct evhttp_request *req, void *arg); static void http_dispatcher_cb(struct evhttp_request *req, void *arg); static void http_on_complete_cb(struct evhttp_request *req, void *arg); #define HTTP_BIND_IPV6 1 #define HTTP_BIND_SSL 2 #define HTTP_SSL_FILTER 4 static int http_bind(struct evhttp *myhttp, ev_uint16_t *pport, int mask) { int port; struct evhttp_bound_socket *sock; int ipv6 = mask & HTTP_BIND_IPV6; if (ipv6) sock = evhttp_bind_socket_with_handle(myhttp, "::1", *pport); else sock = evhttp_bind_socket_with_handle(myhttp, "127.0.0.1", *pport); if (sock == NULL) { if (ipv6) return -1; else event_errx(1, "Could not start web server"); } port = regress_get_socket_port(evhttp_bound_socket_get_fd(sock)); if (port < 0) return -1; *pport = (ev_uint16_t) port; return 0; } #ifdef EVENT__HAVE_OPENSSL static struct bufferevent * https_bev(struct event_base *base, void *arg) { SSL *ssl = SSL_new(get_ssl_ctx()); SSL_use_certificate(ssl, ssl_getcert()); SSL_use_PrivateKey(ssl, ssl_getkey()); return bufferevent_openssl_socket_new( base, -1, ssl, BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_CLOSE_ON_FREE); } #endif static struct evhttp * http_setup(ev_uint16_t *pport, struct event_base *base, int mask) { struct evhttp *myhttp; /* Try a few different ports */ myhttp = evhttp_new(base); if (http_bind(myhttp, pport, mask) < 0) return NULL; #ifdef EVENT__HAVE_OPENSSL if (mask & HTTP_BIND_SSL) { init_ssl(); evhttp_set_bevcb(myhttp, https_bev, NULL); } #endif /* Register a callback for certain types of requests */ evhttp_set_cb(myhttp, "/test", http_basic_cb, myhttp); evhttp_set_cb(myhttp, "/large", http_large_cb, base); evhttp_set_cb(myhttp, "/chunked", http_chunked_cb, base); evhttp_set_cb(myhttp, "/streamed", http_chunked_cb, base); evhttp_set_cb(myhttp, "/postit", http_post_cb, base); evhttp_set_cb(myhttp, "/putit", http_put_cb, base); evhttp_set_cb(myhttp, "/deleteit", http_delete_cb, base); evhttp_set_cb(myhttp, "/delay", http_delay_cb, base); evhttp_set_cb(myhttp, "/largedelay", http_large_delay_cb, base); evhttp_set_cb(myhttp, "/badrequest", http_badreq_cb, base); evhttp_set_cb(myhttp, "/oncomplete", http_on_complete_cb, base); evhttp_set_cb(myhttp, "/", http_dispatcher_cb, base); return (myhttp); } #ifndef NI_MAXSERV #define NI_MAXSERV 1024 #endif static evutil_socket_t http_connect(const char *address, ev_uint16_t port) { /* Stupid code for connecting */ struct evutil_addrinfo ai, *aitop; char strport[NI_MAXSERV]; struct sockaddr *sa; int slen; evutil_socket_t fd; memset(&ai, 0, sizeof(ai)); ai.ai_family = AF_INET; ai.ai_socktype = SOCK_STREAM; evutil_snprintf(strport, sizeof(strport), "%d", port); if (evutil_getaddrinfo(address, strport, &ai, &aitop) != 0) { event_warn("getaddrinfo"); return (-1); } sa = aitop->ai_addr; slen = aitop->ai_addrlen; fd = socket(AF_INET, SOCK_STREAM, 0); if (fd == -1) event_err(1, "socket failed"); evutil_make_socket_nonblocking(fd); if (connect(fd, sa, slen) == -1) { #ifdef _WIN32 int tmp_err = WSAGetLastError(); if (tmp_err != WSAEINPROGRESS && tmp_err != WSAEINVAL && tmp_err != WSAEWOULDBLOCK) event_err(1, "connect failed"); #else if (errno != EINPROGRESS) event_err(1, "connect failed"); #endif } evutil_freeaddrinfo(aitop); return (fd); } /* Helper: do a strcmp on the contents of buf and the string s. */ static int evbuffer_datacmp(struct evbuffer *buf, const char *s) { size_t b_sz = evbuffer_get_length(buf); size_t s_sz = strlen(s); unsigned char *d; int r; if (b_sz < s_sz) return -1; d = evbuffer_pullup(buf, s_sz); if ((r = memcmp(d, s, s_sz))) return r; if (b_sz > s_sz) return 1; else return 0; } /* Helper: Return true iff buf contains s */ static int evbuffer_contains(struct evbuffer *buf, const char *s) { struct evbuffer_ptr ptr; ptr = evbuffer_search(buf, s, strlen(s), NULL); return ptr.pos != -1; } static void http_readcb(struct bufferevent *bev, void *arg) { const char *what = BASIC_REQUEST_BODY; struct event_base *my_base = arg; if (evbuffer_contains(bufferevent_get_input(bev), what)) { struct evhttp_request *req = evhttp_request_new(NULL, NULL); enum message_read_status done; /* req->kind = EVHTTP_RESPONSE; */ done = evhttp_parse_firstline_(req, bufferevent_get_input(bev)); if (done != ALL_DATA_READ) goto out; done = evhttp_parse_headers_(req, bufferevent_get_input(bev)); if (done != ALL_DATA_READ) goto out; if (done == 1 && evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Type") != NULL) test_ok++; out: evhttp_request_free(req); bufferevent_disable(bev, EV_READ); if (exit_base) event_base_loopexit(exit_base, NULL); else if (my_base) event_base_loopexit(my_base, NULL); else { fprintf(stderr, "No way to exit loop!\n"); exit(1); } } } static void http_writecb(struct bufferevent *bev, void *arg) { if (evbuffer_get_length(bufferevent_get_output(bev)) == 0) { /* enable reading of the reply */ bufferevent_enable(bev, EV_READ); test_ok++; } } static void http_errorcb(struct bufferevent *bev, short what, void *arg) { /** For ssl */ if (what & BEV_EVENT_CONNECTED) return; test_ok = -2; event_base_loopexit(arg, NULL); } static int found_multi = 0; static int found_multi2 = 0; static void http_basic_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb = evbuffer_new(); struct evhttp_connection *evcon; int empty = evhttp_find_header(evhttp_request_get_input_headers(req), "Empty") != NULL; event_debug(("%s: called\n", __func__)); evbuffer_add_printf(evb, BASIC_REQUEST_BODY); evcon = evhttp_request_get_connection(req); tt_assert(evhttp_connection_get_server(evcon) == arg); /* For multi-line headers test */ { const char *multi = evhttp_find_header(evhttp_request_get_input_headers(req),"X-Multi"); if (multi) { found_multi = !strcmp(multi,"aaaaaaaa a END"); if (strcmp("END", multi + strlen(multi) - 3) == 0) test_ok++; if (evhttp_find_header(evhttp_request_get_input_headers(req), "X-Last")) test_ok++; } } { const char *multi2 = evhttp_find_header(evhttp_request_get_input_headers(req),"X-Multi-Extra-WS"); if (multi2) { found_multi2 = !strcmp(multi2,"libevent 2.1"); } } /* injecting a bad content-length */ if (evhttp_find_header(evhttp_request_get_input_headers(req), "X-Negative")) evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Length", "-100"); /* allow sending of an empty reply */ evhttp_send_reply(req, HTTP_OK, "Everything is fine", !empty ? evb : NULL); end: evbuffer_free(evb); } static void http_large_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb = evbuffer_new(); int i; for (i = 0; i < 1<<20; ++i) { evbuffer_add_printf(evb, BASIC_REQUEST_BODY); } evhttp_send_reply(req, HTTP_OK, "Everything is fine", evb); evbuffer_free(evb); } static char const* const CHUNKS[] = { "This is funny", "but not hilarious.", "bwv 1052" }; struct chunk_req_state { struct event_base *base; struct evhttp_request *req; int i; }; static void http_chunked_trickle_cb(evutil_socket_t fd, short events, void *arg) { struct evbuffer *evb = evbuffer_new(); struct chunk_req_state *state = arg; struct timeval when = { 0, 0 }; evbuffer_add_printf(evb, "%s", CHUNKS[state->i]); evhttp_send_reply_chunk(state->req, evb); evbuffer_free(evb); if (++state->i < (int) (sizeof(CHUNKS)/sizeof(CHUNKS[0]))) { event_base_once(state->base, -1, EV_TIMEOUT, http_chunked_trickle_cb, state, &when); } else { evhttp_send_reply_end(state->req); free(state); } } static void http_chunked_cb(struct evhttp_request *req, void *arg) { struct timeval when = { 0, 0 }; struct chunk_req_state *state = malloc(sizeof(struct chunk_req_state)); event_debug(("%s: called\n", __func__)); memset(state, 0, sizeof(struct chunk_req_state)); state->req = req; state->base = arg; if (strcmp(evhttp_request_get_uri(req), "/streamed") == 0) { evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Length", "39"); } /* generate a chunked/streamed reply */ evhttp_send_reply_start(req, HTTP_OK, "Everything is fine"); /* but trickle it across several iterations to ensure we're not * assuming it comes all at once */ event_base_once(arg, -1, EV_TIMEOUT, http_chunked_trickle_cb, state, &when); } static void http_complete_write(evutil_socket_t fd, short what, void *arg) { struct bufferevent *bev = arg; const char *http_request = "host\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); } static struct bufferevent * create_bev(struct event_base *base, int fd, int ssl_mask) { int flags = BEV_OPT_DEFER_CALLBACKS; struct bufferevent *bev = NULL; if (!ssl_mask) { bev = bufferevent_socket_new(base, fd, flags); } else { #ifdef EVENT__HAVE_OPENSSL SSL *ssl = SSL_new(get_ssl_ctx()); if (ssl_mask & HTTP_SSL_FILTER) { struct bufferevent *underlying = bufferevent_socket_new(base, fd, flags); bev = bufferevent_openssl_filter_new( base, underlying, ssl, BUFFEREVENT_SSL_CONNECTING, flags); } else { bev = bufferevent_openssl_socket_new( base, fd, ssl, BUFFEREVENT_SSL_CONNECTING, flags); } bufferevent_openssl_set_allow_dirty_shutdown(bev, 1); #endif } return bev; } static void http_basic_test_impl(void *arg, int ssl) { struct basic_test_data *data = arg; struct timeval tv; struct bufferevent *bev = NULL; evutil_socket_t fd; const char *http_request; ev_uint16_t port = 0, port2 = 0; int server_flags = ssl ? HTTP_BIND_SSL : 0; struct evhttp *http = http_setup(&port, data->base, server_flags); exit_base = data->base; test_ok = 0; /* bind to a second socket */ if (http_bind(http, &port2, server_flags) == -1) { fprintf(stdout, "FAILED (bind)\n"); exit(1); } fd = http_connect("127.0.0.1", port); /* Stupid thing to send a request */ bev = create_bev(data->base, fd, ssl); bufferevent_setcb(bev, http_readcb, http_writecb, http_errorcb, data->base); /* first half of the http request */ http_request = "GET /test HTTP/1.1\r\n" "Host: some"; bufferevent_write(bev, http_request, strlen(http_request)); evutil_timerclear(&tv); tv.tv_usec = 100000; event_base_once(data->base, -1, EV_TIMEOUT, http_complete_write, bev, &tv); event_base_dispatch(data->base); tt_assert(test_ok == 3); /* connect to the second port */ bufferevent_free(bev); evutil_closesocket(fd); fd = http_connect("127.0.0.1", port2); /* Stupid thing to send a request */ bev = create_bev(data->base, fd, ssl); bufferevent_setcb(bev, http_readcb, http_writecb, http_errorcb, data->base); http_request = "GET /test HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); event_base_dispatch(data->base); tt_assert(test_ok == 5); /* Connect to the second port again. This time, send an absolute uri. */ bufferevent_free(bev); evutil_closesocket(fd); fd = http_connect("127.0.0.1", port2); /* Stupid thing to send a request */ bev = create_bev(data->base, fd, ssl); bufferevent_setcb(bev, http_readcb, http_writecb, http_errorcb, data->base); http_request = "GET http://somehost.net/test HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); event_base_dispatch(data->base); tt_assert(test_ok == 7); evhttp_free(http); end: if (bev) bufferevent_free(bev); } static void http_basic_test(void *arg) { return http_basic_test_impl(arg, 0); } static void http_delay_reply(evutil_socket_t fd, short what, void *arg) { struct evhttp_request *req = arg; evhttp_send_reply(req, HTTP_OK, "Everything is fine", NULL); ++test_ok; } static void http_delay_cb(struct evhttp_request *req, void *arg) { struct timeval tv; evutil_timerclear(&tv); tv.tv_sec = 0; tv.tv_usec = 200 * 1000; event_base_once(arg, -1, EV_TIMEOUT, http_delay_reply, req, &tv); } static void http_badreq_cb(struct evhttp_request *req, void *arg) { struct evbuffer *buf = evbuffer_new(); evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "text/xml; charset=UTF-8"); evbuffer_add_printf(buf, "Hello, %s!", "127.0.0.1"); evhttp_send_reply(req, HTTP_OK, "OK", buf); evbuffer_free(buf); } static void http_badreq_errorcb(struct bufferevent *bev, short what, void *arg) { event_debug(("%s: called (what=%04x, arg=%p)", __func__, what, arg)); /* ignore */ } static void http_badreq_readcb(struct bufferevent *bev, void *arg) { const char *what = "Hello, 127.0.0.1"; const char *bad_request = "400 Bad Request"; if (evbuffer_contains(bufferevent_get_input(bev), bad_request)) { TT_FAIL(("%s:bad request detected", __func__)); bufferevent_disable(bev, EV_READ); event_base_loopexit(arg, NULL); return; } if (evbuffer_contains(bufferevent_get_input(bev), what)) { struct evhttp_request *req = evhttp_request_new(NULL, NULL); enum message_read_status done; /* req->kind = EVHTTP_RESPONSE; */ done = evhttp_parse_firstline_(req, bufferevent_get_input(bev)); if (done != ALL_DATA_READ) goto out; done = evhttp_parse_headers_(req, bufferevent_get_input(bev)); if (done != ALL_DATA_READ) goto out; if (done == 1 && evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Type") != NULL) test_ok++; out: evhttp_request_free(req); evbuffer_drain(bufferevent_get_input(bev), evbuffer_get_length(bufferevent_get_input(bev))); } shutdown(bufferevent_getfd(bev), EVUTIL_SHUT_WR); } static void http_badreq_successcb(evutil_socket_t fd, short what, void *arg) { event_debug(("%s: called (what=%04x, arg=%p)", __func__, what, arg)); event_base_loopexit(exit_base, NULL); } static void http_bad_request_test(void *arg) { struct basic_test_data *data = arg; struct timeval tv; struct bufferevent *bev = NULL; evutil_socket_t fd = -1; const char *http_request; ev_uint16_t port=0, port2=0; struct evhttp *http = http_setup(&port, data->base, 0); test_ok = 0; exit_base = data->base; /* bind to a second socket */ if (http_bind(http, &port2, 0) == -1) TT_DIE(("Bind socket failed")); /* NULL request test */ fd = http_connect("127.0.0.1", port); tt_int_op(fd, >=, 0); /* Stupid thing to send a request */ bev = bufferevent_socket_new(data->base, fd, 0); bufferevent_setcb(bev, http_badreq_readcb, http_writecb, http_badreq_errorcb, data->base); bufferevent_enable(bev, EV_READ); /* real NULL request */ http_request = ""; bufferevent_write(bev, http_request, strlen(http_request)); shutdown(fd, EVUTIL_SHUT_WR); timerclear(&tv); tv.tv_usec = 10000; event_base_once(data->base, -1, EV_TIMEOUT, http_badreq_successcb, bev, &tv); event_base_dispatch(data->base); bufferevent_free(bev); evutil_closesocket(fd); if (test_ok != 0) { fprintf(stdout, "FAILED\n"); exit(1); } /* Second answer (BAD REQUEST) on connection close */ /* connect to the second port */ fd = http_connect("127.0.0.1", port2); /* Stupid thing to send a request */ bev = bufferevent_socket_new(data->base, fd, 0); bufferevent_setcb(bev, http_badreq_readcb, http_writecb, http_badreq_errorcb, data->base); bufferevent_enable(bev, EV_READ); /* first half of the http request */ http_request = "GET /badrequest HTTP/1.0\r\n" \ "Connection: Keep-Alive\r\n" \ "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); timerclear(&tv); tv.tv_usec = 10000; event_base_once(data->base, -1, EV_TIMEOUT, http_badreq_successcb, bev, &tv); event_base_dispatch(data->base); tt_int_op(test_ok, ==, 2); end: evhttp_free(http); if (bev) bufferevent_free(bev); if (fd >= 0) evutil_closesocket(fd); } static struct evhttp_connection *delayed_client; static void http_large_delay_cb(struct evhttp_request *req, void *arg) { struct timeval tv; evutil_timerclear(&tv); tv.tv_usec = 500000; event_base_once(arg, -1, EV_TIMEOUT, http_delay_reply, req, &tv); evhttp_connection_fail_(delayed_client, EVREQ_HTTP_EOF); } /* * HTTP DELETE test, just piggyback on the basic test */ static void http_delete_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb = evbuffer_new(); int empty = evhttp_find_header(evhttp_request_get_input_headers(req), "Empty") != NULL; /* Expecting a DELETE request */ if (evhttp_request_get_command(req) != EVHTTP_REQ_DELETE) { fprintf(stdout, "FAILED (delete type)\n"); exit(1); } event_debug(("%s: called\n", __func__)); evbuffer_add_printf(evb, BASIC_REQUEST_BODY); /* allow sending of an empty reply */ evhttp_send_reply(req, HTTP_OK, "Everything is fine", !empty ? evb : NULL); evbuffer_free(evb); } static void http_delete_test(void *arg) { struct basic_test_data *data = arg; struct bufferevent *bev; evutil_socket_t fd = -1; const char *http_request; ev_uint16_t port = 0; struct evhttp *http = http_setup(&port, data->base, 0); exit_base = data->base; test_ok = 0; tt_assert(http); fd = http_connect("127.0.0.1", port); tt_int_op(fd, >=, 0); /* Stupid thing to send a request */ bev = bufferevent_socket_new(data->base, fd, 0); bufferevent_setcb(bev, http_readcb, http_writecb, http_errorcb, data->base); http_request = "DELETE /deleteit HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); event_base_dispatch(data->base); bufferevent_free(bev); evutil_closesocket(fd); fd = -1; evhttp_free(http); tt_int_op(test_ok, ==, 2); end: if (fd >= 0) evutil_closesocket(fd); } static void http_sent_cb(struct evhttp_request *req, void *arg) { ev_uintptr_t val = (ev_uintptr_t)arg; struct evbuffer *b; if (val != 0xDEADBEEF) { fprintf(stdout, "FAILED on_complete_cb argument\n"); exit(1); } b = evhttp_request_get_output_buffer(req); if (evbuffer_get_length(b) != 0) { fprintf(stdout, "FAILED on_complete_cb output buffer not written\n"); exit(1); } event_debug(("%s: called\n", __func__)); ++test_ok; } static void http_on_complete_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb = evbuffer_new(); evhttp_request_set_on_complete_cb(req, http_sent_cb, (void *)0xDEADBEEF); event_debug(("%s: called\n", __func__)); evbuffer_add_printf(evb, BASIC_REQUEST_BODY); /* allow sending of an empty reply */ evhttp_send_reply(req, HTTP_OK, "Everything is fine", evb); evbuffer_free(evb); ++test_ok; } static void http_on_complete_test(void *arg) { struct basic_test_data *data = arg; struct bufferevent *bev; evutil_socket_t fd = -1; const char *http_request; ev_uint16_t port = 0; struct evhttp *http = http_setup(&port, data->base, 0); exit_base = data->base; test_ok = 0; fd = http_connect("127.0.0.1", port); tt_int_op(fd, >=, 0); /* Stupid thing to send a request */ bev = bufferevent_socket_new(data->base, fd, 0); bufferevent_setcb(bev, http_readcb, http_writecb, http_errorcb, data->base); http_request = "GET /oncomplete HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); event_base_dispatch(data->base); bufferevent_free(bev); evhttp_free(http); tt_int_op(test_ok, ==, 4); end: if (fd >= 0) evutil_closesocket(fd); } static void http_allowed_methods_eventcb(struct bufferevent *bev, short what, void *arg) { char **output = arg; if ((what & (BEV_EVENT_ERROR|BEV_EVENT_EOF))) { char buf[4096]; int n; n = evbuffer_remove(bufferevent_get_input(bev), buf, sizeof(buf)-1); if (n >= 0) { buf[n]='\0'; if (*output) free(*output); *output = strdup(buf); } event_base_loopexit(exit_base, NULL); } } static void http_allowed_methods_test(void *arg) { struct basic_test_data *data = arg; struct bufferevent *bev1, *bev2, *bev3; evutil_socket_t fd1=-1, fd2=-1, fd3=-1; const char *http_request; char *result1=NULL, *result2=NULL, *result3=NULL; ev_uint16_t port = 0; struct evhttp *http = http_setup(&port, data->base, 0); exit_base = data->base; test_ok = 0; fd1 = http_connect("127.0.0.1", port); tt_int_op(fd1, >=, 0); /* GET is out; PATCH is in. */ evhttp_set_allowed_methods(http, EVHTTP_REQ_PATCH); /* Stupid thing to send a request */ bev1 = bufferevent_socket_new(data->base, fd1, 0); bufferevent_enable(bev1, EV_READ|EV_WRITE); bufferevent_setcb(bev1, NULL, NULL, http_allowed_methods_eventcb, &result1); http_request = "GET /index.html HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev1, http_request, strlen(http_request)); event_base_dispatch(data->base); fd2 = http_connect("127.0.0.1", port); tt_int_op(fd2, >=, 0); bev2 = bufferevent_socket_new(data->base, fd2, 0); bufferevent_enable(bev2, EV_READ|EV_WRITE); bufferevent_setcb(bev2, NULL, NULL, http_allowed_methods_eventcb, &result2); http_request = "PATCH /test HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev2, http_request, strlen(http_request)); event_base_dispatch(data->base); fd3 = http_connect("127.0.0.1", port); tt_int_op(fd3, >=, 0); bev3 = bufferevent_socket_new(data->base, fd3, 0); bufferevent_enable(bev3, EV_READ|EV_WRITE); bufferevent_setcb(bev3, NULL, NULL, http_allowed_methods_eventcb, &result3); http_request = "FLOOP /test HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev3, http_request, strlen(http_request)); event_base_dispatch(data->base); bufferevent_free(bev1); bufferevent_free(bev2); bufferevent_free(bev3); evhttp_free(http); /* Method known but disallowed */ tt_assert(result1); tt_assert(!strncmp(result1, "HTTP/1.1 501 ", strlen("HTTP/1.1 501 "))); /* Method known and allowed */ tt_assert(result2); tt_assert(!strncmp(result2, "HTTP/1.1 200 ", strlen("HTTP/1.1 200 "))); /* Method unknown */ tt_assert(result3); tt_assert(!strncmp(result3, "HTTP/1.1 501 ", strlen("HTTP/1.1 501 "))); end: if (result1) free(result1); if (result2) free(result2); if (result3) free(result3); if (fd1 >= 0) evutil_closesocket(fd1); if (fd2 >= 0) evutil_closesocket(fd2); if (fd3 >= 0) evutil_closesocket(fd3); } static void http_request_no_action_done(struct evhttp_request *, void *); static void http_request_done(struct evhttp_request *, void *); static void http_request_empty_done(struct evhttp_request *, void *); static void http_connection_test_(struct basic_test_data *data, int persistent, const char *address, struct evdns_base *dnsbase, int ipv6, int family, int ssl) { ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct evhttp *http; int mask = 0; if (ipv6) mask |= HTTP_BIND_IPV6; if (ssl) mask |= HTTP_BIND_SSL; http = http_setup(&port, data->base, mask); test_ok = 0; if (!http && ipv6) { tt_skip(); } tt_assert(http); if (ssl) { #ifdef EVENT__HAVE_OPENSSL SSL *ssl = SSL_new(get_ssl_ctx()); struct bufferevent *bev = bufferevent_openssl_socket_new( data->base, -1, ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(bev, 1); evcon = evhttp_connection_base_bufferevent_new(data->base, dnsbase, bev, address, port); #else tt_skip(); #endif } else { evcon = evhttp_connection_base_new(data->base, dnsbase, address, port); } tt_assert(evcon); evhttp_connection_set_family(evcon, family); tt_assert(evhttp_connection_get_base(evcon) == data->base); exit_base = data->base; tt_assert(evhttp_connection_get_server(evcon) == NULL); /* * At this point, we want to schedule a request to the HTTP * server using our make request method. */ req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test") == -1) { fprintf(stdout, "FAILED\n"); exit(1); } event_base_dispatch(data->base); tt_assert(test_ok); /* try to make another request over the same connection */ test_ok = 0; req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* * if our connections are not supposed to be persistent; request * a close from the server. */ if (!persistent) evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "close"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test") == -1) { tt_abort_msg("couldn't make request"); } event_base_dispatch(data->base); /* make another request: request empty reply */ test_ok = 0; req = evhttp_request_new(http_request_empty_done, data->base); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Empty", "itis"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } static void http_connection_test(void *arg) { http_connection_test_(arg, 0, "127.0.0.1", NULL, 0, AF_UNSPEC, 0); } static void http_persist_connection_test(void *arg) { http_connection_test_(arg, 1, "127.0.0.1", NULL, 0, AF_UNSPEC, 0); } static struct regress_dns_server_table search_table[] = { { "localhost", "A", "127.0.0.1", 0, 0 }, { NULL, NULL, NULL, 0, 0 } }; static void http_connection_async_test(void *arg) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct evdns_base *dns_base = NULL; ev_uint16_t portnum = 0; char address[64]; struct evhttp *http = http_setup(&port, data->base, 0); exit_base = data->base; tt_assert(regress_dnsserver(data->base, &portnum, search_table)); dns_base = evdns_base_new(data->base, 0/* init name servers */); tt_assert(dns_base); /* Add ourself as the only nameserver, and make sure we really are * the only nameserver. */ evutil_snprintf(address, sizeof(address), "127.0.0.1:%d", portnum); evdns_base_nameserver_ip_add(dns_base, address); test_ok = 0; evcon = evhttp_connection_base_new(data->base, dns_base, "127.0.0.1", port); tt_assert(evcon); /* * At this point, we want to schedule a request to the HTTP * server using our make request method. */ req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test") == -1) { fprintf(stdout, "FAILED\n"); exit(1); } event_base_dispatch(data->base); tt_assert(test_ok); /* try to make another request over the same connection */ test_ok = 0; req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* * if our connections are not supposed to be persistent; request * a close from the server. */ evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "close"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test") == -1) { tt_abort_msg("couldn't make request"); } event_base_dispatch(data->base); /* make another request: request empty reply */ test_ok = 0; req = evhttp_request_new(http_request_empty_done, data->base); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Empty", "itis"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); if (dns_base) evdns_base_free(dns_base, 0); regress_clean_dnsserver(); } static void http_autofree_connection_test(void *arg) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req[2] = { NULL }; struct evhttp *http = http_setup(&port, data->base, 0); test_ok = 0; evcon = evhttp_connection_base_new(data->base, NULL, "127.0.0.1", port); tt_assert(evcon); /* * At this point, we want to schedule two request to the HTTP * server using our make request method. */ req[0] = evhttp_request_new(http_request_empty_done, data->base); req[1] = evhttp_request_new(http_request_empty_done, data->base); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req[0]), "Host", "somehost"); evhttp_add_header(evhttp_request_get_output_headers(req[0]), "Connection", "close"); evhttp_add_header(evhttp_request_get_output_headers(req[0]), "Empty", "itis"); evhttp_add_header(evhttp_request_get_output_headers(req[1]), "Host", "somehost"); evhttp_add_header(evhttp_request_get_output_headers(req[1]), "Connection", "close"); evhttp_add_header(evhttp_request_get_output_headers(req[1]), "Empty", "itis"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req[0], EVHTTP_REQ_GET, "/test") == -1) { tt_abort_msg("couldn't make request"); } if (evhttp_make_request(evcon, req[1], EVHTTP_REQ_GET, "/test") == -1) { tt_abort_msg("couldn't make request"); } /* * Tell libevent to free the connection when the request completes * We then set the evcon pointer to NULL since we don't want to free it * when this function ends. */ evhttp_connection_free_on_completion(evcon); evcon = NULL; event_base_dispatch(data->base); /* at this point, the http server should have no connection */ tt_assert(TAILQ_FIRST(&http->connections) == NULL); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } static void http_request_never_call(struct evhttp_request *req, void *arg) { fprintf(stdout, "FAILED\n"); exit(1); } static void http_failed_request_done(struct evhttp_request *req, void *arg) { tt_assert(!req); end: event_base_loopexit(arg, NULL); } #ifndef _WIN32 static void http_timed_out_request_done(struct evhttp_request *req, void *arg) { tt_assert(req); tt_int_op(evhttp_request_get_response_code(req), !=, HTTP_OK); end: event_base_loopexit(arg, NULL); } #endif static void http_request_error_cb_with_cancel(enum evhttp_request_error error, void *arg) { if (error != EVREQ_HTTP_REQUEST_CANCEL) { fprintf(stderr, "FAILED\n"); exit(1); } test_ok = 1; { struct timeval tv; evutil_timerclear(&tv); tv.tv_sec = 0; tv.tv_usec = 500 * 1000; event_base_loopexit(exit_base, &tv); } } static void http_do_cancel(evutil_socket_t fd, short what, void *arg) { struct evhttp_request *req = arg; evhttp_cancel_request(req); ++test_ok; } static void http_no_write(struct evbuffer *buffer, const struct evbuffer_cb_info *info, void *arg) { fprintf(stdout, "FAILED\n"); exit(1); } static void http_free_evcons(struct evhttp_connection **evcons) { struct evhttp_connection *evcon, **orig = evcons; if (!evcons) return; while ((evcon = *evcons++)) { evhttp_connection_free(evcon); } free(orig); } /** fill the backlog to force server drop packages for timeouts */ static struct evhttp_connection ** http_fill_backlog(struct event_base *base, int port) { #define BACKLOG_SIZE 256 struct evhttp_connection **evcon = malloc(sizeof(*evcon) * (BACKLOG_SIZE + 1)); int i; for (i = 0; i < BACKLOG_SIZE; ++i) { struct evhttp_request *req; evcon[i] = evhttp_connection_base_new(base, NULL, "127.0.0.1", port); tt_assert(evcon[i]); evhttp_connection_set_timeout(evcon[i], 5); req = evhttp_request_new(http_request_never_call, NULL); tt_assert(req); tt_int_op(evhttp_make_request(evcon[i], req, EVHTTP_REQ_GET, "/delay"), !=, -1); } evcon[i] = NULL; return evcon; end: fprintf(stderr, "Couldn't fill the backlog"); return NULL; } enum http_cancel_test_type { BASIC = 1, BY_HOST = 2, NO_NS = 4, INACTIVE_SERVER = 8, SERVER_TIMEOUT = 16, NS_TIMEOUT = 32, }; static struct evhttp_request * http_cancel_test_bad_request_new(enum http_cancel_test_type type, struct event_base *base) { #ifndef _WIN32 if (!(type & NO_NS) && (type & SERVER_TIMEOUT)) return evhttp_request_new(http_timed_out_request_done, base); else #endif if ((type & INACTIVE_SERVER) || (type & NO_NS)) return evhttp_request_new(http_failed_request_done, base); else return NULL; } static void http_cancel_test(void *arg) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct bufferevent *bufev = NULL; struct timeval tv; struct evdns_base *dns_base = NULL; ev_uint16_t portnum = 0; char address[64]; struct evhttp *inactive_http = NULL; struct event_base *inactive_base = NULL; struct evhttp_connection **evcons = NULL; struct event_base *base_to_fill = data->base; enum http_cancel_test_type type = (enum http_cancel_test_type)data->setup_data; struct evhttp *http = http_setup(&port, data->base, 0); if (type & BY_HOST) { const char *timeout = (type & NS_TIMEOUT) ? "6" : "3"; tt_assert(regress_dnsserver(data->base, &portnum, search_table)); dns_base = evdns_base_new(data->base, 0/* init name servers */); tt_assert(dns_base); /** XXX: Hack the port to make timeout after resolving */ if (type & NO_NS) ++portnum; evutil_snprintf(address, sizeof(address), "127.0.0.1:%d", portnum); evdns_base_nameserver_ip_add(dns_base, address); evdns_base_set_option(dns_base, "timeout:", timeout); evdns_base_set_option(dns_base, "initial-probe-timeout:", timeout); evdns_base_set_option(dns_base, "attempts:", "1"); } exit_base = data->base; test_ok = 0; if (type & INACTIVE_SERVER) { port = 0; inactive_base = event_base_new(); inactive_http = http_setup(&port, inactive_base, 0); base_to_fill = inactive_base; } if (type & SERVER_TIMEOUT) evcons = http_fill_backlog(base_to_fill, port); evcon = evhttp_connection_base_new( data->base, dns_base, type & BY_HOST ? "localhost" : "127.0.0.1", port); if (type & INACTIVE_SERVER) evhttp_connection_set_timeout(evcon, 5); tt_assert(evcon); bufev = evhttp_connection_get_bufferevent(evcon); /* Guarantee that we stack in connect() not after waiting EV_READ after * write() */ if (type & SERVER_TIMEOUT) evbuffer_add_cb(bufferevent_get_output(bufev), http_no_write, NULL); /* * At this point, we want to schedule a request to the HTTP * server using our make request method. */ req = evhttp_request_new(http_request_never_call, NULL); evhttp_request_set_error_cb(req, http_request_error_cb_with_cancel); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ tt_int_op(evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/delay"), !=, -1); evutil_timerclear(&tv); tv.tv_sec = 0; tv.tv_usec = 100 * 1000; event_base_once(data->base, -1, EV_TIMEOUT, http_do_cancel, req, &tv); event_base_dispatch(data->base); if (type & NO_NS || type & INACTIVE_SERVER) tt_int_op(test_ok, ==, 2); /** no servers responses */ else tt_int_op(test_ok, ==, 3); /* try to make another request over the same connection */ test_ok = 0; http_free_evcons(evcons); if (type & SERVER_TIMEOUT) evcons = http_fill_backlog(base_to_fill, port); req = http_cancel_test_bad_request_new(type, data->base); if (!req) req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ tt_int_op(evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test"), !=, -1); event_base_dispatch(data->base); /* make another request: request empty reply */ test_ok = 0; http_free_evcons(evcons); if (type & SERVER_TIMEOUT) evcons = http_fill_backlog(base_to_fill, port); req = http_cancel_test_bad_request_new(type, data->base); if (!req) req = evhttp_request_new(http_request_empty_done, data->base); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Empty", "itis"); /* We give ownership of the request to the connection */ tt_int_op(evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test"), !=, -1); event_base_dispatch(data->base); end: http_free_evcons(evcons); if (bufev) evbuffer_remove_cb(bufferevent_get_output(bufev), http_no_write, NULL); if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); if (dns_base) evdns_base_free(dns_base, 0); regress_clean_dnsserver(); if (inactive_http) evhttp_free(inactive_http); if (inactive_base) event_base_free(inactive_base); } static void http_request_no_action_done(struct evhttp_request *req, void *arg) { EVUTIL_ASSERT(exit_base); event_base_loopexit(exit_base, NULL); } static void http_request_done(struct evhttp_request *req, void *arg) { const char *what = arg; if (!req) { fprintf(stderr, "FAILED\n"); exit(1); } if (evhttp_request_get_response_code(req) != HTTP_OK) { fprintf(stderr, "FAILED\n"); exit(1); } if (evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Type") == NULL) { fprintf(stderr, "FAILED\n"); exit(1); } if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != strlen(what)) { fprintf(stderr, "FAILED\n"); exit(1); } if (evbuffer_datacmp(evhttp_request_get_input_buffer(req), what) != 0) { fprintf(stderr, "FAILED\n"); exit(1); } test_ok = 1; EVUTIL_ASSERT(exit_base); event_base_loopexit(exit_base, NULL); } static void http_request_expect_error(struct evhttp_request *req, void *arg) { if (evhttp_request_get_response_code(req) == HTTP_OK) { fprintf(stderr, "FAILED\n"); exit(1); } test_ok = 1; EVUTIL_ASSERT(arg); event_base_loopexit(arg, NULL); } /* test virtual hosts */ static void http_virtual_host_test(void *arg) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct evhttp *second = NULL, *third = NULL; evutil_socket_t fd; struct bufferevent *bev; const char *http_request; struct evhttp *http = http_setup(&port, data->base, 0); exit_base = data->base; /* virtual host */ second = evhttp_new(NULL); evhttp_set_cb(second, "/funnybunny", http_basic_cb, http); third = evhttp_new(NULL); evhttp_set_cb(third, "/blackcoffee", http_basic_cb, http); if (evhttp_add_virtual_host(http, "foo.com", second) == -1) { tt_abort_msg("Couldn't add vhost"); } if (evhttp_add_virtual_host(http, "bar.*.foo.com", third) == -1) { tt_abort_msg("Couldn't add wildcarded vhost"); } /* add some aliases to the vhosts */ tt_assert(evhttp_add_server_alias(second, "manolito.info") == 0); tt_assert(evhttp_add_server_alias(third, "bonkers.org") == 0); evcon = evhttp_connection_base_new(data->base, NULL, "127.0.0.1", port); tt_assert(evcon); /* make a request with a different host and expect an error */ req = evhttp_request_new(http_request_expect_error, data->base); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/funnybunny") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_assert(test_ok == 1); test_ok = 0; /* make a request with the right host and expect a response */ req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "foo.com"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/funnybunny") == -1) { fprintf(stdout, "FAILED\n"); exit(1); } event_base_dispatch(data->base); tt_assert(test_ok == 1); test_ok = 0; /* make a request with the right host and expect a response */ req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "bar.magic.foo.com"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/blackcoffee") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_assert(test_ok == 1) test_ok = 0; /* make a request with the right host and expect a response */ req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "manolito.info"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/funnybunny") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_assert(test_ok == 1) test_ok = 0; /* make a request with the right host and expect a response */ req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); /* Add the Host header. This time with the optional port. */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "bonkers.org:8000"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/blackcoffee") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_assert(test_ok == 1) test_ok = 0; /* Now make a raw request with an absolute URI. */ fd = http_connect("127.0.0.1", port); /* Stupid thing to send a request */ bev = bufferevent_socket_new(data->base, fd, 0); bufferevent_setcb(bev, http_readcb, http_writecb, http_errorcb, NULL); /* The host in the URI should override the Host: header */ http_request = "GET http://manolito.info/funnybunny HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); event_base_dispatch(data->base); tt_int_op(test_ok, ==, 2); bufferevent_free(bev); evutil_closesocket(fd); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } /* test date header and content length */ static void http_request_empty_done(struct evhttp_request *req, void *arg) { if (!req) { fprintf(stderr, "FAILED\n"); exit(1); } if (evhttp_request_get_response_code(req) != HTTP_OK) { fprintf(stderr, "FAILED\n"); exit(1); } if (evhttp_find_header(evhttp_request_get_input_headers(req), "Date") == NULL) { fprintf(stderr, "FAILED\n"); exit(1); } if (evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Length") == NULL) { fprintf(stderr, "FAILED\n"); exit(1); } if (strcmp(evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Length"), "0")) { fprintf(stderr, "FAILED\n"); exit(1); } if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != 0) { fprintf(stderr, "FAILED\n"); exit(1); } test_ok = 1; EVUTIL_ASSERT(arg); event_base_loopexit(arg, NULL); } /* * HTTP DISPATCHER test */ void http_dispatcher_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb = evbuffer_new(); event_debug(("%s: called\n", __func__)); evbuffer_add_printf(evb, "DISPATCHER_TEST"); evhttp_send_reply(req, HTTP_OK, "Everything is fine", evb); evbuffer_free(evb); } static void http_dispatcher_test_done(struct evhttp_request *req, void *arg) { struct event_base *base = arg; const char *what = "DISPATCHER_TEST"; if (!req) { fprintf(stderr, "FAILED\n"); exit(1); } if (evhttp_request_get_response_code(req) != HTTP_OK) { fprintf(stderr, "FAILED\n"); exit(1); } if (evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Type") == NULL) { fprintf(stderr, "FAILED (content type)\n"); exit(1); } if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != strlen(what)) { fprintf(stderr, "FAILED (length %lu vs %lu)\n", (unsigned long)evbuffer_get_length(evhttp_request_get_input_buffer(req)), (unsigned long)strlen(what)); exit(1); } if (evbuffer_datacmp(evhttp_request_get_input_buffer(req), what) != 0) { fprintf(stderr, "FAILED (data)\n"); exit(1); } test_ok = 1; event_base_loopexit(base, NULL); } static void http_dispatcher_test(void *arg) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct evhttp *http = http_setup(&port, data->base, 0); test_ok = 0; evcon = evhttp_connection_base_new(data->base, NULL, "127.0.0.1", port); tt_assert(evcon); /* also bind to local host */ evhttp_connection_set_local_address(evcon, "127.0.0.1"); /* * At this point, we want to schedule an HTTP GET request * server using our make request method. */ req = evhttp_request_new(http_dispatcher_test_done, data->base); tt_assert(req); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/?arg=val") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } /* * HTTP POST test. */ void http_postrequest_done(struct evhttp_request *, void *); #define POST_DATA "Okay. Not really printf" static void http_post_test(void *arg) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct evhttp *http = http_setup(&port, data->base, 0); test_ok = 0; evcon = evhttp_connection_base_new(data->base, NULL, "127.0.0.1", port); tt_assert(evcon); /* * At this point, we want to schedule an HTTP POST request * server using our make request method. */ req = evhttp_request_new(http_postrequest_done, data->base); tt_assert(req); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); evbuffer_add_printf(evhttp_request_get_output_buffer(req), POST_DATA); if (evhttp_make_request(evcon, req, EVHTTP_REQ_POST, "/postit") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_int_op(test_ok, ==, 1); test_ok = 0; req = evhttp_request_new(http_postrequest_done, data->base); tt_assert(req); /* Now try with 100-continue. */ /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); evhttp_add_header(evhttp_request_get_output_headers(req), "Expect", "100-continue"); evbuffer_add_printf(evhttp_request_get_output_buffer(req), POST_DATA); if (evhttp_make_request(evcon, req, EVHTTP_REQ_POST, "/postit") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_int_op(test_ok, ==, 1); evhttp_connection_free(evcon); evhttp_free(http); end: ; } void http_post_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb; event_debug(("%s: called\n", __func__)); /* Yes, we are expecting a post request */ if (evhttp_request_get_command(req) != EVHTTP_REQ_POST) { fprintf(stdout, "FAILED (post type)\n"); exit(1); } if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != strlen(POST_DATA)) { fprintf(stdout, "FAILED (length: %lu vs %lu)\n", (unsigned long) evbuffer_get_length(evhttp_request_get_input_buffer(req)), (unsigned long) strlen(POST_DATA)); exit(1); } if (evbuffer_datacmp(evhttp_request_get_input_buffer(req), POST_DATA) != 0) { fprintf(stdout, "FAILED (data)\n"); fprintf(stdout, "Got :%s\n", evbuffer_pullup(evhttp_request_get_input_buffer(req),-1)); fprintf(stdout, "Want:%s\n", POST_DATA); exit(1); } evb = evbuffer_new(); evbuffer_add_printf(evb, BASIC_REQUEST_BODY); evhttp_send_reply(req, HTTP_OK, "Everything is fine", evb); evbuffer_free(evb); } void http_postrequest_done(struct evhttp_request *req, void *arg) { const char *what = BASIC_REQUEST_BODY; struct event_base *base = arg; if (req == NULL) { fprintf(stderr, "FAILED (timeout)\n"); exit(1); } if (evhttp_request_get_response_code(req) != HTTP_OK) { fprintf(stderr, "FAILED (response code)\n"); exit(1); } if (evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Type") == NULL) { fprintf(stderr, "FAILED (content type)\n"); exit(1); } if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != strlen(what)) { fprintf(stderr, "FAILED (length %lu vs %lu)\n", (unsigned long)evbuffer_get_length(evhttp_request_get_input_buffer(req)), (unsigned long)strlen(what)); exit(1); } if (evbuffer_datacmp(evhttp_request_get_input_buffer(req), what) != 0) { fprintf(stderr, "FAILED (data)\n"); exit(1); } test_ok = 1; event_base_loopexit(base, NULL); } /* * HTTP PUT test, basically just like POST, but ... */ void http_putrequest_done(struct evhttp_request *, void *); #define PUT_DATA "Hi, I'm some PUT data" static void http_put_test(void *arg) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct evhttp *http = http_setup(&port, data->base, 0); test_ok = 0; evcon = evhttp_connection_base_new(data->base, NULL, "127.0.0.1", port); tt_assert(evcon); /* * Schedule the HTTP PUT request */ req = evhttp_request_new(http_putrequest_done, data->base); tt_assert(req); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "someotherhost"); evbuffer_add_printf(evhttp_request_get_output_buffer(req), PUT_DATA); if (evhttp_make_request(evcon, req, EVHTTP_REQ_PUT, "/putit") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); evhttp_connection_free(evcon); evhttp_free(http); tt_int_op(test_ok, ==, 1); end: ; } void http_put_cb(struct evhttp_request *req, void *arg) { struct evbuffer *evb; event_debug(("%s: called\n", __func__)); /* Expecting a PUT request */ if (evhttp_request_get_command(req) != EVHTTP_REQ_PUT) { fprintf(stdout, "FAILED (put type)\n"); exit(1); } if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != strlen(PUT_DATA)) { fprintf(stdout, "FAILED (length: %lu vs %lu)\n", (unsigned long)evbuffer_get_length(evhttp_request_get_input_buffer(req)), (unsigned long)strlen(PUT_DATA)); exit(1); } if (evbuffer_datacmp(evhttp_request_get_input_buffer(req), PUT_DATA) != 0) { fprintf(stdout, "FAILED (data)\n"); fprintf(stdout, "Got :%s\n", evbuffer_pullup(evhttp_request_get_input_buffer(req),-1)); fprintf(stdout, "Want:%s\n", PUT_DATA); exit(1); } evb = evbuffer_new(); evbuffer_add_printf(evb, "That ain't funny"); evhttp_send_reply(req, HTTP_OK, "Everything is great", evb); evbuffer_free(evb); } void http_putrequest_done(struct evhttp_request *req, void *arg) { struct event_base *base = arg; const char *what = "That ain't funny"; if (req == NULL) { fprintf(stderr, "FAILED (timeout)\n"); exit(1); } if (evhttp_request_get_response_code(req) != HTTP_OK) { fprintf(stderr, "FAILED (response code)\n"); exit(1); } if (evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Type") == NULL) { fprintf(stderr, "FAILED (content type)\n"); exit(1); } if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != strlen(what)) { fprintf(stderr, "FAILED (length %lu vs %lu)\n", (unsigned long)evbuffer_get_length(evhttp_request_get_input_buffer(req)), (unsigned long)strlen(what)); exit(1); } if (evbuffer_datacmp(evhttp_request_get_input_buffer(req), what) != 0) { fprintf(stderr, "FAILED (data)\n"); exit(1); } test_ok = 1; event_base_loopexit(base, NULL); } static void http_failure_readcb(struct bufferevent *bev, void *arg) { const char *what = "400 Bad Request"; if (evbuffer_contains(bufferevent_get_input(bev), what)) { test_ok = 2; bufferevent_disable(bev, EV_READ); event_base_loopexit(arg, NULL); } } /* * Testing that the HTTP server can deal with a malformed request. */ static void http_failure_test(void *arg) { struct basic_test_data *data = arg; struct bufferevent *bev; evutil_socket_t fd = -1; const char *http_request; ev_uint16_t port = 0; struct evhttp *http = http_setup(&port, data->base, 0); test_ok = 0; fd = http_connect("127.0.0.1", port); tt_int_op(fd, >=, 0); /* Stupid thing to send a request */ bev = bufferevent_socket_new(data->base, fd, 0); bufferevent_setcb(bev, http_failure_readcb, http_writecb, http_errorcb, data->base); http_request = "illegal request\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); event_base_dispatch(data->base); bufferevent_free(bev); evhttp_free(http); tt_int_op(test_ok, ==, 2); end: if (fd >= 0) evutil_closesocket(fd); } static void close_detect_done(struct evhttp_request *req, void *arg) { struct timeval tv; tt_assert(req); tt_assert(evhttp_request_get_response_code(req) == HTTP_OK); test_ok = 1; end: evutil_timerclear(&tv); tv.tv_usec = 150000; event_base_loopexit(arg, &tv); } static void close_detect_launch(evutil_socket_t fd, short what, void *arg) { struct evhttp_connection *evcon = arg; struct event_base *base = evhttp_connection_get_base(evcon); struct evhttp_request *req; req = evhttp_request_new(close_detect_done, base); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test") == -1) { tt_fail_msg("Couldn't make request"); } } static void close_detect_cb(struct evhttp_request *req, void *arg) { struct evhttp_connection *evcon = arg; struct event_base *base = evhttp_connection_get_base(evcon); struct timeval tv; if (req != NULL && evhttp_request_get_response_code(req) != HTTP_OK) { tt_abort_msg("Failed"); } evutil_timerclear(&tv); tv.tv_sec = 0; /* longer than the http time out */ tv.tv_usec = 600000; /* longer than the http time out */ /* launch a new request on the persistent connection in .3 seconds */ event_base_once(base, -1, EV_TIMEOUT, close_detect_launch, evcon, &tv); end: ; } static void http_close_detection_(struct basic_test_data *data, int with_delay) { ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; const struct timeval sec_tenth = { 0, 100000 }; struct evhttp *http = http_setup(&port, data->base, 0); test_ok = 0; /* .1 second timeout */ evhttp_set_timeout_tv(http, &sec_tenth); evcon = evhttp_connection_base_new(data->base, NULL, "127.0.0.1", port); tt_assert(evcon); evhttp_connection_set_timeout_tv(evcon, &sec_tenth); tt_assert(evcon); delayed_client = evcon; /* * At this point, we want to schedule a request to the HTTP * server using our make request method. */ req = evhttp_request_new(close_detect_cb, evcon); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, with_delay ? "/largedelay" : "/test") == -1) { tt_abort_msg("couldn't make request"); } event_base_dispatch(data->base); /* at this point, the http server should have no connection */ tt_assert(TAILQ_FIRST(&http->connections) == NULL); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } static void http_close_detection_test(void *arg) { http_close_detection_(arg, 0); } static void http_close_detection_delay_test(void *arg) { http_close_detection_(arg, 1); } static void http_highport_test(void *arg) { struct basic_test_data *data = arg; int i = -1; struct evhttp *myhttp = NULL; /* Try a few different ports */ for (i = 0; i < 50; ++i) { myhttp = evhttp_new(data->base); if (evhttp_bind_socket(myhttp, "127.0.0.1", 65535 - i) == 0) { test_ok = 1; evhttp_free(myhttp); return; } evhttp_free(myhttp); } tt_fail_msg("Couldn't get a high port"); } static void http_bad_header_test(void *ptr) { struct evkeyvalq headers; TAILQ_INIT(&headers); tt_want(evhttp_add_header(&headers, "One", "Two") == 0); tt_want(evhttp_add_header(&headers, "One", "Two\r\n Three") == 0); tt_want(evhttp_add_header(&headers, "One\r", "Two") == -1); tt_want(evhttp_add_header(&headers, "One\n", "Two") == -1); tt_want(evhttp_add_header(&headers, "One", "Two\r") == -1); tt_want(evhttp_add_header(&headers, "One", "Two\n") == -1); evhttp_clear_headers(&headers); } static int validate_header( const struct evkeyvalq* headers, const char *key, const char *value) { const char *real_val = evhttp_find_header(headers, key); tt_assert(real_val != NULL); tt_want(strcmp(real_val, value) == 0); end: return (0); } static void http_parse_query_test(void *ptr) { struct evkeyvalq headers; int r; TAILQ_INIT(&headers); r = evhttp_parse_query("http://www.test.com/?q=test", &headers); tt_want(validate_header(&headers, "q", "test") == 0); tt_int_op(r, ==, 0); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=test&foo=bar", &headers); tt_want(validate_header(&headers, "q", "test") == 0); tt_want(validate_header(&headers, "foo", "bar") == 0); tt_int_op(r, ==, 0); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=test+foo", &headers); tt_want(validate_header(&headers, "q", "test foo") == 0); tt_int_op(r, ==, 0); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=test%0Afoo", &headers); tt_want(validate_header(&headers, "q", "test\nfoo") == 0); tt_int_op(r, ==, 0); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=test%0Dfoo", &headers); tt_want(validate_header(&headers, "q", "test\rfoo") == 0); tt_int_op(r, ==, 0); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=test&&q2", &headers); tt_int_op(r, ==, -1); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=test+this", &headers); tt_want(validate_header(&headers, "q", "test this") == 0); tt_int_op(r, ==, 0); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=test&q2=foo", &headers); tt_int_op(r, ==, 0); tt_want(validate_header(&headers, "q", "test") == 0); tt_want(validate_header(&headers, "q2", "foo") == 0); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q&q2=foo", &headers); tt_int_op(r, ==, -1); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=foo&q2", &headers); tt_int_op(r, ==, -1); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=foo&q2&q3=x", &headers); tt_int_op(r, ==, -1); evhttp_clear_headers(&headers); r = evhttp_parse_query("http://www.test.com/?q=&q2=&q3=", &headers); tt_int_op(r, ==, 0); tt_want(validate_header(&headers, "q", "") == 0); tt_want(validate_header(&headers, "q2", "") == 0); tt_want(validate_header(&headers, "q3", "") == 0); evhttp_clear_headers(&headers); end: evhttp_clear_headers(&headers); } static void http_parse_uri_test(void *ptr) { const int nonconform = (ptr != NULL); const unsigned parse_flags = nonconform ? EVHTTP_URI_NONCONFORMANT : 0; struct evhttp_uri *uri = NULL; char url_tmp[4096]; #define URI_PARSE(uri) \ evhttp_uri_parse_with_flags((uri), parse_flags) #define TT_URI(want) do { \ char *ret = evhttp_uri_join(uri, url_tmp, sizeof(url_tmp)); \ tt_want(ret != NULL); \ tt_want(ret == url_tmp); \ if (strcmp(ret,want) != 0) \ TT_FAIL(("\"%s\" != \"%s\"",ret,want)); \ } while(0) tt_want(evhttp_uri_join(NULL, 0, 0) == NULL); tt_want(evhttp_uri_join(NULL, url_tmp, 0) == NULL); tt_want(evhttp_uri_join(NULL, url_tmp, sizeof(url_tmp)) == NULL); /* bad URIs: parsing */ #define BAD(s) do { \ if (URI_PARSE(s) != NULL) \ TT_FAIL(("Expected error parsing \"%s\"",s)); \ } while(0) /* Nonconformant URIs we can parse: parsing */ #define NCF(s) do { \ uri = URI_PARSE(s); \ if (uri != NULL && !nonconform) { \ TT_FAIL(("Expected error parsing \"%s\"",s)); \ } else if (uri == NULL && nonconform) { \ TT_FAIL(("Couldn't parse nonconformant URI \"%s\"", \ s)); \ } \ if (uri) { \ tt_want(evhttp_uri_join(uri, url_tmp, \ sizeof(url_tmp))); \ evhttp_uri_free(uri); \ } \ } while(0) NCF("http://www.test.com/ why hello"); NCF("http://www.test.com/why-hello\x01"); NCF("http://www.test.com/why-hello?\x01"); NCF("http://www.test.com/why-hello#\x01"); BAD("http://www.\x01.test.com/why-hello"); BAD("http://www.%7test.com/why-hello"); NCF("http://www.test.com/why-hell%7o"); BAD("h%3ttp://www.test.com/why-hello"); NCF("http://www.test.com/why-hello%7"); NCF("http://www.test.com/why-hell%7o"); NCF("http://www.test.com/foo?ba%r"); NCF("http://www.test.com/foo#ba%r"); BAD("99:99/foo"); BAD("http://www.test.com:999x/"); BAD("http://www.test.com:x/"); BAD("http://[hello-there]/"); BAD("http://[::1]]/"); BAD("http://[::1/"); BAD("http://[foob/"); BAD("http://[/"); BAD("http://[ffff:ffff:ffff:ffff:Ffff:ffff:ffff:" "ffff:ffff:ffff:ffff:ffff:ffff:ffff]/"); BAD("http://[vX.foo]/"); BAD("http://[vX.foo]/"); BAD("http://[v.foo]/"); BAD("http://[v5.fo%o]/"); BAD("http://[v5X]/"); BAD("http://[v5]/"); BAD("http://[]/"); BAD("http://f\x01red@www.example.com/"); BAD("http://f%0red@www.example.com/"); BAD("http://www.example.com:9999999999999999999999999999999999999/"); BAD("http://www.example.com:hihi/"); BAD("://www.example.com/"); /* bad URIs: joining */ uri = evhttp_uri_new(); tt_want(0==evhttp_uri_set_host(uri, "www.example.com")); tt_want(evhttp_uri_join(uri, url_tmp, sizeof(url_tmp)) != NULL); /* not enough space: */ tt_want(evhttp_uri_join(uri, url_tmp, 3) == NULL); /* host is set, but path doesn't start with "/": */ tt_want(0==evhttp_uri_set_path(uri, "hi_mom")); tt_want(evhttp_uri_join(uri, url_tmp, sizeof(url_tmp)) == NULL); tt_want(evhttp_uri_join(uri, NULL, sizeof(url_tmp))==NULL); tt_want(evhttp_uri_join(uri, url_tmp, 0)==NULL); evhttp_uri_free(uri); uri = URI_PARSE("mailto:foo@bar"); tt_want(uri != NULL); tt_want(evhttp_uri_get_host(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(!strcmp(evhttp_uri_get_scheme(uri), "mailto")); tt_want(!strcmp(evhttp_uri_get_path(uri), "foo@bar")); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("mailto:foo@bar"); evhttp_uri_free(uri); uri = evhttp_uri_new(); /* Bad URI usage: setting invalid values */ tt_want(-1 == evhttp_uri_set_scheme(uri,"")); tt_want(-1 == evhttp_uri_set_scheme(uri,"33")); tt_want(-1 == evhttp_uri_set_scheme(uri,"hi!")); tt_want(-1 == evhttp_uri_set_userinfo(uri,"hello@")); tt_want(-1 == evhttp_uri_set_host(uri,"[1.2.3.4]")); tt_want(-1 == evhttp_uri_set_host(uri,"[")); tt_want(-1 == evhttp_uri_set_host(uri,"www.[foo].com")); tt_want(-1 == evhttp_uri_set_port(uri,-3)); tt_want(-1 == evhttp_uri_set_path(uri,"hello?world")); tt_want(-1 == evhttp_uri_set_query(uri,"hello#world")); tt_want(-1 == evhttp_uri_set_fragment(uri,"hello#world")); /* Valid URI usage: setting valid values */ tt_want(0 == evhttp_uri_set_scheme(uri,"http")); tt_want(0 == evhttp_uri_set_scheme(uri,NULL)); tt_want(0 == evhttp_uri_set_userinfo(uri,"username:pass")); tt_want(0 == evhttp_uri_set_userinfo(uri,NULL)); tt_want(0 == evhttp_uri_set_host(uri,"www.example.com")); tt_want(0 == evhttp_uri_set_host(uri,"1.2.3.4")); tt_want(0 == evhttp_uri_set_host(uri,"[1:2:3:4::]")); tt_want(0 == evhttp_uri_set_host(uri,"[v7.wobblewobble]")); tt_want(0 == evhttp_uri_set_host(uri,NULL)); tt_want(0 == evhttp_uri_set_host(uri,"")); tt_want(0 == evhttp_uri_set_port(uri, -1)); tt_want(0 == evhttp_uri_set_port(uri, 80)); tt_want(0 == evhttp_uri_set_port(uri, 65535)); tt_want(0 == evhttp_uri_set_path(uri, "")); tt_want(0 == evhttp_uri_set_path(uri, "/documents/public/index.html")); tt_want(0 == evhttp_uri_set_path(uri, NULL)); tt_want(0 == evhttp_uri_set_query(uri, "key=val&key2=val2")); tt_want(0 == evhttp_uri_set_query(uri, "keyvalblarg")); tt_want(0 == evhttp_uri_set_query(uri, "")); tt_want(0 == evhttp_uri_set_query(uri, NULL)); tt_want(0 == evhttp_uri_set_fragment(uri, "")); tt_want(0 == evhttp_uri_set_fragment(uri, "here?i?am")); tt_want(0 == evhttp_uri_set_fragment(uri, NULL)); evhttp_uri_free(uri); /* Valid parsing */ uri = URI_PARSE("http://www.test.com/?q=t%33est"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "http") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "www.test.com") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=t%33est") == 0); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("http://www.test.com/?q=t%33est"); evhttp_uri_free(uri); uri = URI_PARSE("http://%77ww.test.com"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "http") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "%77ww.test.com") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("http://%77ww.test.com"); evhttp_uri_free(uri); uri = URI_PARSE("http://www.test.com?q=test"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "http") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "www.test.com") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=test") == 0); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("http://www.test.com?q=test"); evhttp_uri_free(uri); uri = URI_PARSE("http://www.test.com#fragment"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "http") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "www.test.com") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want_str_op(evhttp_uri_get_fragment(uri), ==, "fragment"); TT_URI("http://www.test.com#fragment"); evhttp_uri_free(uri); uri = URI_PARSE("http://8000/"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "http") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "8000") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("http://8000/"); evhttp_uri_free(uri); uri = URI_PARSE("http://:8000/"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "http") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == 8000); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("http://:8000/"); evhttp_uri_free(uri); uri = URI_PARSE("http://www.test.com:/"); /* empty port */ tt_want(strcmp(evhttp_uri_get_scheme(uri), "http") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "www.test.com") == 0); tt_want_str_op(evhttp_uri_get_path(uri), ==, "/"); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("http://www.test.com/"); evhttp_uri_free(uri); uri = URI_PARSE("http://www.test.com:"); /* empty port 2 */ tt_want(strcmp(evhttp_uri_get_scheme(uri), "http") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "www.test.com") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("http://www.test.com"); evhttp_uri_free(uri); uri = URI_PARSE("ftp://www.test.com/?q=test"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "ftp") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "www.test.com") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=test") == 0); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("ftp://www.test.com/?q=test"); evhttp_uri_free(uri); uri = URI_PARSE("ftp://[::1]:999/?q=test"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "ftp") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "[::1]") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=test") == 0); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == 999); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("ftp://[::1]:999/?q=test"); evhttp_uri_free(uri); uri = URI_PARSE("ftp://[ff00::127.0.0.1]/?q=test"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "ftp") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "[ff00::127.0.0.1]") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=test") == 0); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("ftp://[ff00::127.0.0.1]/?q=test"); evhttp_uri_free(uri); uri = URI_PARSE("ftp://[v99.not_(any:time)_soon]/?q=test"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "ftp") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "[v99.not_(any:time)_soon]") == 0); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=test") == 0); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("ftp://[v99.not_(any:time)_soon]/?q=test"); evhttp_uri_free(uri); uri = URI_PARSE("scheme://user:pass@foo.com:42/?q=test&s=some+thing#fragment"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "scheme") == 0); tt_want(strcmp(evhttp_uri_get_userinfo(uri), "user:pass") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "foo.com") == 0); tt_want(evhttp_uri_get_port(uri) == 42); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=test&s=some+thing") == 0); tt_want(strcmp(evhttp_uri_get_fragment(uri), "fragment") == 0); TT_URI("scheme://user:pass@foo.com:42/?q=test&s=some+thing#fragment"); evhttp_uri_free(uri); uri = URI_PARSE("scheme://user@foo.com/#fragment"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "scheme") == 0); tt_want(strcmp(evhttp_uri_get_userinfo(uri), "user") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "foo.com") == 0); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(strcmp(evhttp_uri_get_fragment(uri), "fragment") == 0); TT_URI("scheme://user@foo.com/#fragment"); evhttp_uri_free(uri); uri = URI_PARSE("scheme://%75ser@foo.com/#frag@ment"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "scheme") == 0); tt_want(strcmp(evhttp_uri_get_userinfo(uri), "%75ser") == 0); tt_want(strcmp(evhttp_uri_get_host(uri), "foo.com") == 0); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(strcmp(evhttp_uri_get_path(uri), "/") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(strcmp(evhttp_uri_get_fragment(uri), "frag@ment") == 0); TT_URI("scheme://%75ser@foo.com/#frag@ment"); evhttp_uri_free(uri); uri = URI_PARSE("file:///some/path/to/the/file"); tt_want(strcmp(evhttp_uri_get_scheme(uri), "file") == 0); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(strcmp(evhttp_uri_get_host(uri), "") == 0); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(strcmp(evhttp_uri_get_path(uri), "/some/path/to/the/file") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("file:///some/path/to/the/file"); evhttp_uri_free(uri); uri = URI_PARSE("///some/path/to/the-file"); tt_want(uri != NULL); tt_want(evhttp_uri_get_scheme(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(strcmp(evhttp_uri_get_host(uri), "") == 0); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(strcmp(evhttp_uri_get_path(uri), "/some/path/to/the-file") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("///some/path/to/the-file"); evhttp_uri_free(uri); uri = URI_PARSE("/s:ome/path/to/the-file?q=99#fred"); tt_want(uri != NULL); tt_want(evhttp_uri_get_scheme(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_host(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(strcmp(evhttp_uri_get_path(uri), "/s:ome/path/to/the-file") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=99") == 0); tt_want(strcmp(evhttp_uri_get_fragment(uri), "fred") == 0); TT_URI("/s:ome/path/to/the-file?q=99#fred"); evhttp_uri_free(uri); uri = URI_PARSE("relative/path/with/co:lon"); tt_want(uri != NULL); tt_want(evhttp_uri_get_scheme(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_host(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(strcmp(evhttp_uri_get_path(uri), "relative/path/with/co:lon") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(evhttp_uri_get_fragment(uri) == NULL); TT_URI("relative/path/with/co:lon"); evhttp_uri_free(uri); uri = URI_PARSE("bob?q=99&q2=q?33#fr?ed"); tt_want(uri != NULL); tt_want(evhttp_uri_get_scheme(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_host(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(strcmp(evhttp_uri_get_path(uri), "bob") == 0); tt_want(strcmp(evhttp_uri_get_query(uri), "q=99&q2=q?33") == 0); tt_want(strcmp(evhttp_uri_get_fragment(uri), "fr?ed") == 0); TT_URI("bob?q=99&q2=q?33#fr?ed"); evhttp_uri_free(uri); uri = URI_PARSE("#fr?ed"); tt_want(uri != NULL); tt_want(evhttp_uri_get_scheme(uri) == NULL); tt_want(evhttp_uri_get_userinfo(uri) == NULL); tt_want(evhttp_uri_get_host(uri) == NULL); tt_want(evhttp_uri_get_port(uri) == -1); tt_want(strcmp(evhttp_uri_get_path(uri), "") == 0); tt_want(evhttp_uri_get_query(uri) == NULL); tt_want(strcmp(evhttp_uri_get_fragment(uri), "fr?ed") == 0); TT_URI("#fr?ed"); evhttp_uri_free(uri); #undef URI_PARSE #undef TT_URI #undef BAD } static void http_uriencode_test(void *ptr) { char *s=NULL, *s2=NULL; size_t sz; int bytes_decoded; #define ENC(from,want,plus) do { \ s = evhttp_uriencode((from), -1, (plus)); \ tt_assert(s); \ tt_str_op(s,==,(want)); \ sz = -1; \ s2 = evhttp_uridecode((s), (plus), &sz); \ tt_assert(s2); \ tt_str_op(s2,==,(from)); \ tt_int_op(sz,==,strlen(from)); \ free(s); \ free(s2); \ s = s2 = NULL; \ } while (0) #define DEC(from,want,dp) do { \ s = evhttp_uridecode((from),(dp),&sz); \ tt_assert(s); \ tt_str_op(s,==,(want)); \ tt_int_op(sz,==,strlen(want)); \ free(s); \ s = NULL; \ } while (0) #define OLD_DEC(from,want) do { \ s = evhttp_decode_uri((from)); \ tt_assert(s); \ tt_str_op(s,==,(want)); \ free(s); \ s = NULL; \ } while (0) ENC("Hello", "Hello",0); ENC("99", "99",0); ENC("", "",0); ENC( "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789-.~_", "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789-.~_",0); ENC(" ", "%20",0); ENC(" ", "+",1); ENC("\xff\xf0\xe0", "%FF%F0%E0",0); ENC("\x01\x19", "%01%19",1); ENC("http://www.ietf.org/rfc/rfc3986.txt", "http%3A%2F%2Fwww.ietf.org%2Frfc%2Frfc3986.txt",1); ENC("1+2=3", "1%2B2%3D3",1); ENC("1+2=3", "1%2B2%3D3",0); /* Now try encoding with internal NULs. */ s = evhttp_uriencode("hello\0world", 11, 0); tt_assert(s); tt_str_op(s,==,"hello%00world"); free(s); s = NULL; /* Now try decoding just part of string. */ s = malloc(6 + 1 /* NUL byte */); bytes_decoded = evhttp_decode_uri_internal("hello%20%20", 6, s, 0); tt_assert(s); tt_int_op(bytes_decoded,==,6); tt_str_op(s,==,"hello%"); free(s); s = NULL; /* Now try out some decoding cases that we don't generate with * encode_uri: Make sure that malformed stuff doesn't crash... */ DEC("%%xhello th+ere \xff", "%%xhello th+ere \xff", 0); /* Make sure plus decoding works */ DEC("plus+should%20work+", "plus should work ",1); /* Try some lowercase hex */ DEC("%f0%a0%b0", "\xf0\xa0\xb0",1); /* Try an internal NUL. */ sz = 0; s = evhttp_uridecode("%00%00x%00%00", 1, &sz); tt_int_op(sz,==,5); tt_assert(!memcmp(s, "\0\0x\0\0", 5)); free(s); s = NULL; /* Try with size == NULL */ sz = 0; s = evhttp_uridecode("%00%00x%00%00", 1, NULL); tt_assert(!memcmp(s, "\0\0x\0\0", 5)); free(s); s = NULL; /* Test out the crazy old behavior of the deprecated * evhttp_decode_uri */ OLD_DEC("http://example.com/normal+path/?key=val+with+spaces", "http://example.com/normal+path/?key=val with spaces"); end: if (s) free(s); if (s2) free(s2); #undef ENC #undef DEC #undef OLD_DEC } static void http_base_test(void *ptr) { struct event_base *base = NULL; struct bufferevent *bev; evutil_socket_t fd; const char *http_request; ev_uint16_t port = 0; struct evhttp *http; test_ok = 0; base = event_base_new(); tt_assert(base); http = http_setup(&port, base, 0); fd = http_connect("127.0.0.1", port); tt_int_op(fd, >=, 0); /* Stupid thing to send a request */ bev = bufferevent_socket_new(base, fd, 0); bufferevent_setcb(bev, http_readcb, http_writecb, http_errorcb, base); bufferevent_base_set(base, bev); http_request = "GET /test HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); event_base_dispatch(base); bufferevent_free(bev); evutil_closesocket(fd); evhttp_free(http); tt_int_op(test_ok, ==, 2); end: if (base) event_base_free(base); } /* * the server is just going to close the connection if it times out during * reading the headers. */ static void http_incomplete_readcb(struct bufferevent *bev, void *arg) { test_ok = -1; event_base_loopexit(exit_base,NULL); } static void http_incomplete_errorcb(struct bufferevent *bev, short what, void *arg) { /** For ssl */ if (what & BEV_EVENT_CONNECTED) return; if (what == (BEV_EVENT_READING|BEV_EVENT_EOF)) test_ok++; else test_ok = -2; event_base_loopexit(exit_base,NULL); } static void http_incomplete_writecb(struct bufferevent *bev, void *arg) { if (arg != NULL) { evutil_socket_t fd = *(evutil_socket_t *)arg; /* terminate the write side to simulate EOF */ shutdown(fd, EVUTIL_SHUT_WR); } if (evbuffer_get_length(bufferevent_get_output(bev)) == 0) { /* enable reading of the reply */ bufferevent_enable(bev, EV_READ); test_ok++; } } static void http_incomplete_test_(struct basic_test_data *data, int use_timeout, int ssl) { struct bufferevent *bev; evutil_socket_t fd; const char *http_request; ev_uint16_t port = 0; struct timeval tv_start, tv_end; struct evhttp *http = http_setup(&port, data->base, ssl ? HTTP_BIND_SSL : 0); exit_base = data->base; test_ok = 0; evhttp_set_timeout(http, 1); fd = http_connect("127.0.0.1", port); tt_int_op(fd, >=, 0); /* Stupid thing to send a request */ bev = create_bev(data->base, fd, ssl); bufferevent_setcb(bev, http_incomplete_readcb, http_incomplete_writecb, http_incomplete_errorcb, use_timeout ? NULL : &fd); http_request = "GET /test HTTP/1.1\r\n" "Host: somehost\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); evutil_gettimeofday(&tv_start, NULL); event_base_dispatch(data->base); evutil_gettimeofday(&tv_end, NULL); evutil_timersub(&tv_end, &tv_start, &tv_end); bufferevent_free(bev); if (use_timeout) { evutil_closesocket(fd); fd = -1; } evhttp_free(http); if (use_timeout && tv_end.tv_sec >= 3) { tt_abort_msg("time"); } else if (!use_timeout && tv_end.tv_sec >= 1) { /* we should be done immediately */ tt_abort_msg("time"); } tt_int_op(test_ok, ==, 2); end: if (fd >= 0) evutil_closesocket(fd); } static void http_incomplete_test(void *arg) { http_incomplete_test_(arg, 0, 0); } static void http_incomplete_timeout_test(void *arg) { http_incomplete_test_(arg, 1, 0); } /* * the server is going to reply with chunked data. */ static void http_chunked_readcb(struct bufferevent *bev, void *arg) { /* nothing here */ } static void http_chunked_errorcb(struct bufferevent *bev, short what, void *arg) { struct evhttp_request *req = NULL; /** SSL */ if (what & BEV_EVENT_CONNECTED) return; if (!test_ok) goto out; test_ok = -1; if ((what & BEV_EVENT_EOF) != 0) { const char *header; enum message_read_status done; req = evhttp_request_new(NULL, NULL); /* req->kind = EVHTTP_RESPONSE; */ done = evhttp_parse_firstline_(req, bufferevent_get_input(bev)); if (done != ALL_DATA_READ) goto out; done = evhttp_parse_headers_(req, bufferevent_get_input(bev)); if (done != ALL_DATA_READ) goto out; header = evhttp_find_header(evhttp_request_get_input_headers(req), "Transfer-Encoding"); if (header == NULL || strcmp(header, "chunked")) goto out; header = evhttp_find_header(evhttp_request_get_input_headers(req), "Connection"); if (header == NULL || strcmp(header, "close")) goto out; header = evbuffer_readln(bufferevent_get_input(bev), NULL, EVBUFFER_EOL_CRLF); if (header == NULL) goto out; /* 13 chars */ if (strcmp(header, "d")) { free((void*)header); goto out; } free((void*)header); if (strncmp((char *)evbuffer_pullup(bufferevent_get_input(bev), 13), "This is funny", 13)) goto out; evbuffer_drain(bufferevent_get_input(bev), 13 + 2); header = evbuffer_readln(bufferevent_get_input(bev), NULL, EVBUFFER_EOL_CRLF); if (header == NULL) goto out; /* 18 chars */ if (strcmp(header, "12")) goto out; free((char *)header); if (strncmp((char *)evbuffer_pullup(bufferevent_get_input(bev), 18), "but not hilarious.", 18)) goto out; evbuffer_drain(bufferevent_get_input(bev), 18 + 2); header = evbuffer_readln(bufferevent_get_input(bev), NULL, EVBUFFER_EOL_CRLF); if (header == NULL) goto out; /* 8 chars */ if (strcmp(header, "8")) { free((void*)header); goto out; } free((char *)header); if (strncmp((char *)evbuffer_pullup(bufferevent_get_input(bev), 8), "bwv 1052.", 8)) goto out; evbuffer_drain(bufferevent_get_input(bev), 8 + 2); header = evbuffer_readln(bufferevent_get_input(bev), NULL, EVBUFFER_EOL_CRLF); if (header == NULL) goto out; /* 0 chars */ if (strcmp(header, "0")) { free((void*)header); goto out; } free((void *)header); test_ok = 2; } out: if (req) evhttp_request_free(req); event_base_loopexit(arg, NULL); } static void http_chunked_writecb(struct bufferevent *bev, void *arg) { if (evbuffer_get_length(bufferevent_get_output(bev)) == 0) { /* enable reading of the reply */ bufferevent_enable(bev, EV_READ); test_ok++; } } static void http_chunked_request_done(struct evhttp_request *req, void *arg) { if (evhttp_request_get_response_code(req) != HTTP_OK) { fprintf(stderr, "FAILED\n"); exit(1); } if (evhttp_find_header(evhttp_request_get_input_headers(req), "Transfer-Encoding") == NULL) { fprintf(stderr, "FAILED\n"); exit(1); } if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != 13 + 18 + 8) { fprintf(stderr, "FAILED\n"); exit(1); } if (strncmp((char *)evbuffer_pullup(evhttp_request_get_input_buffer(req), 13 + 18 + 8), "This is funnybut not hilarious.bwv 1052", 13 + 18 + 8)) { fprintf(stderr, "FAILED\n"); exit(1); } test_ok = 1; event_base_loopexit(arg, NULL); } static void http_chunk_out_test_impl(void *arg, int ssl) { struct basic_test_data *data = arg; struct bufferevent *bev; evutil_socket_t fd; const char *http_request; ev_uint16_t port = 0; struct timeval tv_start, tv_end; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; int i; struct evhttp *http = http_setup(&port, data->base, ssl ? HTTP_BIND_SSL : 0); exit_base = data->base; test_ok = 0; fd = http_connect("127.0.0.1", port); /* Stupid thing to send a request */ bev = create_bev(data->base, fd, ssl); bufferevent_setcb(bev, http_chunked_readcb, http_chunked_writecb, http_chunked_errorcb, data->base); http_request = "GET /chunked HTTP/1.1\r\n" "Host: somehost\r\n" "Connection: close\r\n" "\r\n"; bufferevent_write(bev, http_request, strlen(http_request)); evutil_gettimeofday(&tv_start, NULL); event_base_dispatch(data->base); bufferevent_free(bev); evutil_gettimeofday(&tv_end, NULL); evutil_timersub(&tv_end, &tv_start, &tv_end); tt_int_op(tv_end.tv_sec, <, 1); tt_int_op(test_ok, ==, 2); /* now try again with the regular connection object */ bev = create_bev(data->base, -1, ssl); evcon = evhttp_connection_base_bufferevent_new( data->base, NULL, bev, "127.0.0.1", port); tt_assert(evcon); /* make two requests to check the keepalive behavior */ for (i = 0; i < 2; i++) { test_ok = 0; req = evhttp_request_new(http_chunked_request_done,data->base); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/chunked") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_assert(test_ok == 1); } end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } static void http_chunk_out_test(void *arg) { return http_chunk_out_test_impl(arg, 0); } static void http_stream_out_test_impl(void *arg, int ssl) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct bufferevent *bev; struct evhttp *http = http_setup(&port, data->base, ssl ? HTTP_BIND_SSL : 0); test_ok = 0; exit_base = data->base; bev = create_bev(data->base, -1, ssl); evcon = evhttp_connection_base_bufferevent_new( data->base, NULL, bev, "127.0.0.1", port); tt_assert(evcon); /* * At this point, we want to schedule a request to the HTTP * server using our make request method. */ req = evhttp_request_new(http_request_done, (void *)"This is funnybut not hilarious.bwv 1052"); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/streamed") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } static void http_stream_out_test(void *arg) { return http_stream_out_test_impl(arg, 0); } static void http_stream_in_chunk(struct evhttp_request *req, void *arg) { struct evbuffer *reply = arg; if (evhttp_request_get_response_code(req) != HTTP_OK) { fprintf(stderr, "FAILED\n"); exit(1); } evbuffer_add_buffer(reply, evhttp_request_get_input_buffer(req)); } static void http_stream_in_done(struct evhttp_request *req, void *arg) { if (evbuffer_get_length(evhttp_request_get_input_buffer(req)) != 0) { fprintf(stderr, "FAILED\n"); exit(1); } event_base_loopexit(exit_base, NULL); } /** * Makes a request and reads the response in chunks. */ static void http_stream_in_test_(struct basic_test_data *data, char const *url, size_t expected_len, char const *expected) { struct evhttp_connection *evcon; struct evbuffer *reply = evbuffer_new(); struct evhttp_request *req = NULL; ev_uint16_t port = 0; struct evhttp *http = http_setup(&port, data->base, 0); exit_base = data->base; evcon = evhttp_connection_base_new(data->base, NULL,"127.0.0.1", port); tt_assert(evcon); req = evhttp_request_new(http_stream_in_done, reply); evhttp_request_set_chunked_cb(req, http_stream_in_chunk); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, url) == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); if (evbuffer_get_length(reply) != expected_len) { TT_DIE(("reply length %lu; expected %lu; FAILED (%s)\n", (unsigned long)evbuffer_get_length(reply), (unsigned long)expected_len, (char*)evbuffer_pullup(reply, -1))); } if (memcmp(evbuffer_pullup(reply, -1), expected, expected_len) != 0) { tt_abort_msg("Memory mismatch"); } test_ok = 1; end: if (reply) evbuffer_free(reply); if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } static void http_stream_in_test(void *arg) { http_stream_in_test_(arg, "/chunked", 13 + 18 + 8, "This is funnybut not hilarious.bwv 1052"); http_stream_in_test_(arg, "/test", strlen(BASIC_REQUEST_BODY), BASIC_REQUEST_BODY); } static void http_stream_in_cancel_chunk(struct evhttp_request *req, void *arg) { tt_int_op(evhttp_request_get_response_code(req), ==, HTTP_OK); end: evhttp_cancel_request(req); event_base_loopexit(arg, NULL); } static void http_stream_in_cancel_done(struct evhttp_request *req, void *arg) { /* should never be called */ tt_fail_msg("In cancel done"); } static void http_stream_in_cancel_test(void *arg) { struct basic_test_data *data = arg; struct evhttp_connection *evcon; struct evhttp_request *req = NULL; ev_uint16_t port = 0; struct evhttp *http = http_setup(&port, data->base, 0); evcon = evhttp_connection_base_new(data->base, NULL, "127.0.0.1", port); tt_assert(evcon); req = evhttp_request_new(http_stream_in_cancel_done, data->base); evhttp_request_set_chunked_cb(req, http_stream_in_cancel_chunk); /* We give ownership of the request to the connection */ if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/chunked") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); test_ok = 1; end: evhttp_connection_free(evcon); evhttp_free(http); } static void http_connection_fail_done(struct evhttp_request *req, void *arg) { struct evhttp_connection *evcon = arg; struct event_base *base = evhttp_connection_get_base(evcon); /* An ENETUNREACH error results in an unrecoverable * evhttp_connection error (see evhttp_connection_fail_()). The * connection will be reset, and the user will be notified with a NULL * req parameter. */ tt_assert(!req); evhttp_connection_free(evcon); test_ok = 1; end: event_base_loopexit(base, NULL); } /* Test unrecoverable evhttp_connection errors by generating an ENETUNREACH * error on connection. */ static void http_connection_fail_test_impl(void *arg, int ssl) { struct basic_test_data *data = arg; ev_uint16_t port = 0; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct bufferevent *bev; struct evhttp *http = http_setup(&port, data->base, ssl ? HTTP_BIND_SSL : 0); exit_base = data->base; test_ok = 0; /* auto detect a port */ evhttp_free(http); bev = create_bev(data->base, -1, ssl); /* Pick an unroutable address. This administratively scoped multicast * address should do when working with TCP. */ evcon = evhttp_connection_base_bufferevent_new( data->base, NULL, bev, "239.10.20.30", 80); tt_assert(evcon); /* * At this point, we want to schedule an HTTP GET request * server using our make request method. */ req = evhttp_request_new(http_connection_fail_done, evcon); tt_assert(req); if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_int_op(test_ok, ==, 1); end: ; } static void http_connection_fail_test(void *arg) { return http_connection_fail_test_impl(arg, 0); } static void http_connection_retry_done(struct evhttp_request *req, void *arg) { tt_assert(req); tt_int_op(evhttp_request_get_response_code(req), !=, HTTP_OK); if (evhttp_find_header(evhttp_request_get_input_headers(req), "Content-Type") != NULL) { tt_abort_msg("(content type)\n"); } tt_uint_op(evbuffer_get_length(evhttp_request_get_input_buffer(req)), ==, 0); test_ok = 1; end: event_base_loopexit(arg,NULL); } struct http_server { ev_uint16_t port; int ssl; struct evhttp *http; }; static struct event_base *http_make_web_server_base=NULL; static void http_make_web_server(evutil_socket_t fd, short what, void *arg) { struct http_server *hs = (struct http_server *)arg; hs->http = http_setup(&hs->port, http_make_web_server_base, hs->ssl ? HTTP_BIND_SSL : 0); } static void http_simple_test_impl(void *arg, int ssl, int dirty) { struct basic_test_data *data = arg; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct bufferevent *bev; struct http_server hs = { .port = 0, .ssl = ssl, }; struct evhttp *http = http_setup(&hs.port, data->base, ssl ? HTTP_BIND_SSL : 0); exit_base = data->base; test_ok = 0; bev = create_bev(data->base, -1, ssl); #ifdef EVENT__HAVE_OPENSSL bufferevent_openssl_set_allow_dirty_shutdown(bev, dirty); #endif evcon = evhttp_connection_base_bufferevent_new( data->base, NULL, bev, "127.0.0.1", hs.port); tt_assert(evcon); evhttp_connection_set_local_address(evcon, "127.0.0.1"); req = evhttp_request_new(http_request_done, (void*) BASIC_REQUEST_BODY); tt_assert(req); if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/test") == -1) { tt_abort_msg("Couldn't make request"); } event_base_dispatch(data->base); tt_int_op(test_ok, ==, 1); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(http); } static void http_simple_test(void *arg) { return http_simple_test_impl(arg, 0, 0); } static void http_connection_retry_test_basic(void *arg, const char *addr, struct evdns_base *dns_base, int ssl) { struct basic_test_data *data = arg; struct evhttp_connection *evcon = NULL; struct evhttp_request *req = NULL; struct timeval tv, tv_start, tv_end; struct bufferevent *bev; struct http_server hs = { .port = 0, .ssl = ssl, }; struct evhttp *http = http_setup(&hs.port, data->base, ssl ? HTTP_BIND_SSL : 0); exit_base = data->base; test_ok = 0; /* auto detect a port */ evhttp_free(http); bev = create_bev(data->base, -1, ssl); evcon = evhttp_connection_base_bufferevent_new(data->base, dns_base, bev, addr, hs.port); tt_assert(evcon); if (dns_base) tt_assert(!evhttp_connection_set_flags(evcon, EVHTTP_CON_REUSE_CONNECTED_ADDR)); evhttp_connection_set_timeout(evcon, 1); /* also bind to local host */ evhttp_connection_set_local_address(evcon, "127.0.0.1"); /* * At this point, we want to schedule an HTTP GET request * server using our make request method. */ req = evhttp_request_new(http_connection_retry_done, data->base); tt_assert(req); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/?arg=val") == -1) { tt_abort_msg("Couldn't make request"); } evutil_gettimeofday(&tv_start, NULL); event_base_dispatch(data->base); evutil_gettimeofday(&tv_end, NULL); evutil_timersub(&tv_end, &tv_start, &tv_end); tt_int_op(tv_end.tv_sec, <, 1); tt_int_op(test_ok, ==, 1); /* * now test the same but with retries */ test_ok = 0; /** Shutdown dns server, to test conn_address reusing */ if (dns_base) regress_clean_dnsserver(); { const struct timeval tv_timeout = { 0, 500000 }; const struct timeval tv_retry = { 0, 500000 }; evhttp_connection_set_timeout_tv(evcon, &tv_timeout); evhttp_connection_set_initial_retry_tv(evcon, &tv_retry); } evhttp_connection_set_retries(evcon, 1); req = evhttp_request_new(http_connection_retry_done, data->base); tt_assert(req); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/?arg=val") == -1) { tt_abort_msg("Couldn't make request"); } evutil_gettimeofday(&tv_start, NULL); event_base_dispatch(data->base); evutil_gettimeofday(&tv_end, NULL); /* fails fast, .5 sec to wait to retry, fails fast again. */ test_timeval_diff_leq(&tv_start, &tv_end, 500, 200); tt_assert(test_ok == 1); /* * now test the same but with retries and give it a web server * at the end */ test_ok = 0; evhttp_connection_set_timeout(evcon, 1); evhttp_connection_set_retries(evcon, 3); req = evhttp_request_new(http_dispatcher_test_done, data->base); tt_assert(req); /* Add the information that we care about */ evhttp_add_header(evhttp_request_get_output_headers(req), "Host", "somehost"); if (evhttp_make_request(evcon, req, EVHTTP_REQ_GET, "/?arg=val") == -1) { tt_abort_msg("Couldn't make request"); } /* start up a web server .2 seconds after the connection tried * to send a request */ evutil_timerclear(&tv); tv.tv_usec = 200000; http_make_web_server_base = data->base; event_base_once(data->base, -1, EV_TIMEOUT, http_make_web_server, &hs, &tv); evutil_gettimeofday(&tv_start, NULL); event_base_dispatch(data->base); evutil_gettimeofday(&tv_end, NULL); /* We'll wait twice as long as we did last time. */ test_timeval_diff_leq(&tv_start, &tv_end, 1000, 400); tt_int_op(test_ok, ==, 1); end: if (evcon) evhttp_connection_free(evcon); if (http) evhttp_free(hs.http); } static void http_connection_retry_conn_address_test_impl(void *arg, int ssl) { struct basic_test_data *data = arg; ev_uint16_t portnum = 0; struct evdns_base *dns_base = NULL; char address[64]; tt_assert(regress_dnsserver(data->base, &portnum, search_table)); dns_base = evdns_base_new(data->base, 0/* init name servers */); tt_assert(dns_base); /* Add ourself as the only nameserver, and make sure we really are * the only nameserver. */ evutil_snprintf(address, sizeof(address), "127.0.0.1:%d", portnum); evdns_base_nameserver_ip_add(dns_base, address); http_connection_retry_test_basic(arg, "localhost", dns_base, ssl); end: if (dns_base) evdns_base_free(dns_base, 0); /** dnsserver will be cleaned in http_connection_retry_test_basic() */ } static void http_connection_retry_conn_address_test(void *arg) { return http_connection_retry_conn_address_test_impl(arg, 0); } static void http_connection_retry_test_impl(void *arg, int ssl) { return http_connection_retry_test_basic(arg, "127.0.0.1", NULL, ssl); } static void http_connection_retry_test(void *arg) { return http_connection_retry_test_impl(arg, 0); } static void http_primitives(void *ptr) { char *escaped = NULL; struct evhttp *http = NULL; escaped = evhttp_htmlescape("